diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..444bc4d24783eb43e3dc5474bc54433c6d4f4a8b 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+audiocraft/dataset/example/clip/sample_1/no_vocal.wav filter=lfs diff=lfs merge=lfs -text
+audiocraft/dataset/example/clip/sample_2/no_vocal.wav filter=lfs diff=lfs merge=lfs -text
+preproc/1_beats-crop/1_mm.wav filter=lfs diff=lfs merge=lfs -text
+preproc/1_beats-crop/1_nn.wav filter=lfs diff=lfs merge=lfs -text
diff --git a/5_genre_songs_list.json b/5_genre_songs_list.json
new file mode 100644
index 0000000000000000000000000000000000000000..1fc1c02ce281db922f76259a664f3ed2bfc1d2de
--- /dev/null
+++ b/5_genre_songs_list.json
@@ -0,0 +1 @@
+{"Funk": ["8RM9X2vCbQw", "5zadWs6531g", "Au-GI2_jLcQ", "EAOFM9PyFNc", "HMc0P0MTSnY", "NEetSdEnMnk", "jhMsk9dlz_E", "2FINRkmWVCE", "7ULII2-asSs", "7iUxrQvwg40", "DakMQVNlWTQ", "JmhFmR3u3FM", "Jy3vHaAa7LA", "L3lq5tvUSMQ", "QnP5Fh7fnN0", "UCRrMLfgMJs", "gKyvhnhMUts", "nU80bEG-A4o", "3j16qv7ZpDQ", "6997aNKeWeA", "8zZo_2_dbro", "BwpQoOejIHg", "GYO4dhNAaN8", "JO-ehtZJ31w", "JsntV4fxsOI", "LYsxnRY4WKI", "VjifkLcNc9Y", "cCGasWfGBf8", "nacVOM8R9mE", "oOPSLQf5sp4", "p4XEIdeh5Zw", "pcaJcXFzbv8", "rZr-BYmeQdY", "w1yMkudP5cg", "wW_ZPvUsqcw", "B4_f0pxKOJY", "K0U2bu25H-o", "ghVwifzniwg", "uETDfIP1m4s", "-fc07rk8FNg", "-hjfbkJP6rI", "-ooH4zWv6KI", "0lxsKB3YYws", "0m6EdNrE8KM", "0zmpmTJXse4", "12NkQgvrvO0", "2TYOrDUjMGk", "399ivnDZHMA", "3HNWPtt43Ac", "3ep7CGp8cdQ", "40o2Y4W0qh8", "42p-cPN4RWY", "49n9KLA-ZOI", "50_QY7_uXeQ", "5q5JGuzfuJc", "68-IG-0FbNQ", "6QxPP9v7gAI", "6Y7-kh2Rff8", "6nvpt7I9uSw", "7Bswf0v37OM", "7UIvddNn250", "9GV2nuyJsoU", "9Snm-IKreJw", "9WHRQADbUFU", "9o6MyvedMUY", "9t_d64Vgmkg", "9vA516qEXoY", "A6HtAp1HbZs", "AM7EOhr_QQw", "ARF_pRXUacs", "AWd5DQMxfeg", "AZ4wN8xjo8Q", "An5OXwRg91I", "AqzuBws7EUI", "DpgE901_jeQ", "E873o8bGCs8", "Ec-rOgFIY5A", "F0PfVlUFNcw", "FQQPCuQDW_A", "Fe_dbxjZktk", "FlGOazzVpRo", "FtonGWaAQhA", "G4eQgnXwalc", "G4hx5VWm-9k", "GLmqdOD_urA", "GiRLJMEUBGk", "GvbLi3sh67I", "Gzvw4DiDx1Y", "HS8zIvoqN_U", "Hu_6qmVhpnM", "IDXvggLIxPo", "IO8PTIjQWr4", "IbIQk2RWIN0", "Iu4sFMdoxp8", "JCcIRmsUS2U", "JOce_oZ8LeE", "JlxJeXrbNSU", "Jov1U4cRXX0", "JtHRhrBgrVU", "JxXOGCLn1uU", "K9h5Y8rco10", "KiV4ZAgTId4", "KlhElaycFHk", "M0lAOIBwxQQ", "MIOL--VoDkU", "MP0GLAgF4Kg", "MV9VIhY6UIc", "Mh94IYWRQoM", "MjltWlMqia8", "MuZtOKwMHxk", "Mx7ycYy1A78", "Mz3gaqBoECQ", "NE4xofX7ZEQ", "NVQGGxr-Gl0", "NakoZeRwzl8", "NejQTizNg5A", "Nlixj44vkEs", "OPV1SSDhCBE", "OcX5pz0PnTU", "P1HXWIeIDCA", "PKX8IS-Fc90", "PQrNEUtBnK0", "PRLRCN8QIu0", "PmiHOAdchnI", "Q0kg1VBw4aI", "QINWCyC-sx8", "QXe4AKEYQfw", "Qs5cu-n1nkI", "Rpj5wC3vvws", "RuQzl-BSSuY", "SWbr_xi6Ce4", "Sbd6z5cyP7k", "SgXcRKQ_bKk", "SueddLdz2G8", "T7bMqzoIhP4", "TC2u5IECKUQ", "UI2-Nnio_Ac", "UVqk0I3FmQs", "UVsotK_lllY", "VDXEgifMioA", "VjNoJzXDdgk", "WIRIdTm5N1Q", "WNzqFiw5cgw", "WePOx8KKJNE", "Wf8yqDbX1_A", "X--XRaajt6Q", "Y9m-wri8Lz8", "YjnAa0jLWVw", "YsGq6G67rN8", "Z-OdtZdT-wQ", "Zgm2QacfD0I", "ZkTtniN1H90", "_GgGc1KuNzA", "_OsR4djZHMc", "a2vUUC2wQcA", "ak1-cDhABvM", "apq-ybN8RU4", "b3yyc30zPDo", "b4ilGu6qRtg", "bHm5QWeb9KY", "bQuI4o5nF20", "cRlYcsJuz4s", "cYuTczBXgN8", "ctrpCBuAslo", "dB4Itlw1yqc", "dFfy0GAKS4E", "d_WNgvbcLP0", "daeM6nsukKo", "ejQmF6WpxHg", "eqqJPr3kUYM", "fcvp64p1AhI", "fl3dZeCnTnE", "h9cym0eYRPc", "hJWLIK5FfFg", "hNEbK6L60GM", "h_1wanJelfQ", "igJLTailYTM", "jF5513JYIqs", "jFcdymVEk18", "jL3Kl7rUfz0", "jUc8rUAAshg", "jeGaV9OCqc4", "jzY3lK6hFNk", "kCrQuhrhQI4", "kmyyWonvaBk", "knaJNcjce38", "lBtBH3FHvNs", "lMfwAzXdq00", "lPy16sOByPc", "lnG5vDCwYjw", "m11Ez1Zl6Hg", "m1VitXaY3Ok", "mQ4iZ6cssaY", "nVlOkZHtXt4", "nbrZR4QbEKg", "npt67f6nHIw", "o6ph2fUBbT8", "oNVVrZiet6w", "ob_P2M5KYN0", "p6safB1ThmM", "pI14vO5R_Us", "pIKqew9J7QQ", "pfgcYEP3HOA", "piswxKD1pY8", "qI2FFsrC7TA", "qOm7uNFzs8c", "qR4-WcefBPk", "qUQw1Sgl5qg", "qV6M9qbcIhI", "qpkCQTSQXa8", "r_qY_zi9u5c", "sozTtqunU3o", "sz3utK8d3Dk", "tAtQLQpcmnk", "tdpcv9lkZ-o", "tzjMazqKPn0", "u3hx1oWsVq0", "u4y5E0nf5A0", "uIotPcxeDT4", "ucUyR1-uJaY", "udOVwpwbTsM", "vHzHeO_z3cU", "vbKLrcdx80U", "viPm9Sr77m0", "w9zMQTMd3io", "wswILBI-fok", "xEJLbzg6gHg", "xHZLHpgacSM", "xnQqLJ6juxg", "yTcZG1dCegs", "yVI51US44KE", "ysOyVLwgUyw", "z0EDd7Jaomw", "zfiR6dNqO-E", "-Lg5fq4TH0E", "8vYt1lhdn2w", "X9dHiHSDq40", "aMNEstIAl3g", "cg_AycnTkVc", "qUN5iugeqDY", "qiTwASQrJCo", "z7arQnP_CW8", "4dWsoJDWPYI", "4tekXIif42Y", "5E0mxgsFqKM", "7ifOuxpNLKU", "9s_PkotxHRg", "ASQbebYIImY", "G3YCUM7begQ", "HXpJHMPFfTY", "KIMb-DiGUOk", "KdcRB3NKAXg", "KiH-VI0xNnI", "NU7f5Kn06dw", "U9bz7T-1MDY", "XJq2DkHsQ4w", "_Xtb5aCJtK0", "aTgX9Mv66WY", "cFfQkNlViqA", "fUV6DSIES1o", "gpssMQJnoaw", "hFqfo6etYEc", "j9W48wMny6w", "jdAoDiimOl0", "k1aER0GGcs0", "kDGBUCYCjkQ", "kFAxICIGHHc", "nhsjrf5j9_w", "zo3aAA8uWpY", "Au8IryB8isE", "B9jK-PvR-2E", "JAOZ29yEqP4", "TmNRIgC7PG4", "yBeIlp6iBws", "06vq7D03ago", "3-gyCHIc8O8", "JgqZ8wRzhsM", "Sly9izOjFSA", "UjL8wi-BsSc", "iLICZxfPzkw", "jibDA0OHCpg", "m0Qp6Mk0FwA", "9fcgDzPAp_E", "cSnpfjoQXUc", "xVxjskBA6ZM", "-gYAoi-T52k", "1Xl3Dsk5dFE", "1aW-egSK2tg", "1uijfEoKsWc", "2YTSSUMkSKU", "2pOgPD1S21E", "2uoYsTATu9g", "3N8Ngnqy3-M", "3Yt1hTH427E", "3qNkgnDBZ70", "4a3R32mrgAQ", "4jW1wSBuU5o", "4tO3mO_sQnc", "4yA7gy9T664", "5adKe-KxLO8", "5pH0H_FPTk8", "6j-mng8LubE", "7YFmRxRsEVw", "9aO7B1EIsjs", "AarB4cT7RIU", "B606sAglPfg", "BBe6E42BTB0", "Bk_JBiu3Z7U", "C1_d1NhFBag", "CdDGrhmZVhY", "Cy-LQdnU1x8", "D2stKt8ise4", "DV9FkqhnsMo", "DtrEtTXLMb4", "EQnQDiUW19w", "G7cNDDejEQo", "G_Fa8rAZGz4", "GlNyFlV1BFU", "Gq7LDTlYDkg", "H3Sn7pG0Y2M", "HIMs4HdER6Y", "Hd_Dd3arung", "I2JWnxhrwes", "Imun9eV8wT0", "JJWl_PtM1lU", "JRvSIpt_m-U", "JWHPNXSF1cE", "Jw686OLQE90", "K_vSqKxfKpw", "LFyw-QhWEIo", "LxKEJX6dI38", "MSvQVe5LH4k", "MzA-yGht_nQ", "NB5r2Vra1ZE", "Nej2r_ZwBDE", "OOvwqY8m5CA", "OTquK2CpiYQ", "OWYo4ipdojI", "OwJdOO9DnE0", "PMDAEHX9A7w", "QMAKRQ3nxiw", "Qn9pLkk8Fm0", "RPuWomTqC_I", "SnkiGAyTZbE", "TKbNUqF_-X8", "TsKaXOd5Buw", "UNhr-zp4-iM", "UzSMaqsVEmc", "VeoWkKqLNbQ", "WFfqQIkL0Og", "Xr3YEY0NSTM", "YKtywjx3Aas", "Yqtk8Nyd-n4", "YyrgzsD9NRo", "Z7HHvWkkQIE", "Z8c6c-hBES0", "ZXd8oWZ9xSA", "aIKtTpc7V34", "aWr7pOfNWBk", "abwpNInRlwc", "bxxeW3XOF_0", "cUT9XK1gkfQ", "eSsnL4l7IwI", "fBiVQIxKDMM", "fWJ8z2DpH5g", "fxG9j91Mslg", "fxH3LUsf7jw", "ga1jFfbEtMM", "hCsNFJQpNYU", "hK1Hd9qWYt4", "i78rHPmrigM", "iKnIETgosyY", "iRS9kKGjy60", "ihAiNbvLWRU", "k1zxYAeZ7CU", "kTA83e1DD-Q", "kUe7RTAzhCc", "kv0fKC6dY3s", "lGXpZ9wVvT0", "lgRlder2bc4", "mDnItz4i2NE", "m_m_5hL27nw", "mk9YtVcUt3U", "mmKpXsZK87U", "nPY_YSJckQw", "ncpeW-PoC-Q", "niQv84TXbuQ", "p1mxRXbGjnA", "pAZvVdEfqQI", "pGWUTmBbIyY", "pbGOaclf_hE", "pwRtaUkqVVk", "q8FQUMt4e48", "qPwiDpYQxMY", "r1OkER_-o0c", "st50ICUr_So", "uIKUtMjD2Wc", "uiIKXUnI5aA", "unQieYcJbjY", "vpXKOoMezWU", "w3m9vGyJ9kY", "wPR4ybUJCPA", "xZVPfDOp1JQ", "yspohI5ElyA", "4TKHzvY83aE", "PlKWPha1OrA", "j8h1wyIzeb4", "VOcr4znACh0", "ymdOWiVmul4", "10fgVhvdPpA", "CtaMuPKFE1o", "JqihiGIcDf0", "bZdmdnWXCtU", "-f24vuufUxw", "-fuA3unVBsI", "0G-5O9yFtTs", "0qdZQcKMQP4", "1gpJcjb7tV4", "2E4oOON0jeI", "2EUOYON3eZg", "3RiVkgVk88A", "4MPYRltRx3w", "5J2PYeJqF5o", "5lXEoVyYR4U", "5wW0s9AleO4", "6FmNShyPcMM", "CN1AgfIJ7e0", "EEXNNpWyBWQ", "GdHixLV6jEM", "JMmaNxaHaD8", "JQGTyqIKEzc", "KNr9daii9cg", "L80JpiEDaUo", "LUh3VQhZTQY", "MpI5x1tb_Gk", "PWH2eEk90mk", "PvOtXvUwE_0", "Q2KxhJ0zvhs", "S5QtWdKRQGs", "SkpRjNVnk4s", "TMjSF7xnYVg", "TkL2w-O4Tdo", "V7DsaCnji30", "VIpKuoO8Zc4", "VjZobH3XqQI", "WnE97LpVPsU", "XKZaITdAHgs", "XtpFflC0QzA", "Xv7uB6x5CCk", "YNAVGbu38CQ", "aN5QkTuAzdA", "bbss7StwSsA", "c6W1a3ygqOM", "ckX_j1iJnr4", "cqT4aERwFpk", "dTKaTtD0ky4", "fAeApgO61iQ", "fLk1Ml3dcA0", "gcbFcPxEFjE", "gtUbDC-CsMs", "hybfK9kPWew", "iKOze7YtHkY", "iaxZMovHAnk", "j6jDPDgSy7Q", "jC_jOs7urYo", "nGO_KqJw9b0", "nsVSR6SbL-M", "pEW28PjDFRc", "pHWGKLniHCU", "r__M0IguzuU", "tzBZMUsbg_g", "uUUgzwK3AIs", "uhcZo9S9hvE", "upVQV_7fRZE", "wI8vfpqR1h4", "xqhvKtJnhX0", "yDlydW2p8xs", "z4-EcJ6W0JU", "ScjqpIZFflc", "nQWvxsmxYHo", "-hSKuWKu4GU", "-r0PTM0KbO4", "3Q7Av7EmdNM", "3ZufeSUpUT0", "6NCbCrBdXN0", "6OqV2fIsp2M", "8-Bk0z8HWjc", "89fdm1VRayM", "99kj06OYPOo", "9jLsVlIG5O8", "A59v1qUqTmo", "AJh6U76A1dE", "BKa6NtHmwfI", "BO-MSRZbU7A", "D7j-WMr2JRY", "EvVDOUpfYRk", "IYJwJRKEDTk", "IalQaZW_Yj4", "JHB-5h4yn_A", "JLL1iy7EST4", "LeZq19M4L8s", "LprnbyokSyo", "N3dFS6ddpFg", "OHy4WAY687o", "QGRE2i7FFgE", "RQeapR2iDiQ", "Tv6OcQH7hXA", "UeF5VuF_X5s", "VELN1-kKhug", "XYa-0JKZl-U", "Xb69_lwsBdo", "XwawDHRZXWg", "Y3l7I9vgga0", "YZTLv-g3G7k", "ZmPDDn3qO_g", "_vgVw8yuS3M", "bOwuzfBzSGA", "d-qqoqhpx5Y", "dWIiQsE757s", "enlAMVUHEDA", "f58pYdIiYJM", "get3I9IC2D0", "hiL-QjBHx48", "mabj7RG9Bs8", "oc0yGA8j_a4", "pOze-Iic4wQ", "psQa2zY3QxE", "q0-SbkdxKO8", "rNvvgOsgrsQ", "ryvC5vQ7F4o", "tkL-4FlA5Gk", "tluKawtTvo0", "xPASN86buGg", "xPwa71KtrL4", "yQVpk8QdnaI", "yWD297bspEw", "ybWlypfXoPE", "GpngM-hWr6o", "KYW7K1DlXns", "QxopKhrUuQg", "YXAVXD2Wh9Y", "lcLOWixDMbQ", "mvRijJxVjgg", "rvxRYX-rZYA", "uBasOqmDm2U", "3Ak4Qxikf4Q", "CFi_LVGvQsg", "b5CNtQyY5Oo", "ePOqMlijt0Y", "xebhFQlmRvQ", "yCouR59nwpI", "qknFalbI-J8", "BHomZfGTPxg", "D-Toz2rmaPU", "ETbW28vj5SE", "FRuMsSmsBDg", "Hbo5s5GNUto", "N5L0LGVz-l4", "PQldBk_jn10", "Vp_gjvL00CA", "afwsLO6O9oQ", "ls0fBYmRxzU", "qb7NxO3us10", "OHT05LsYjBc", "ODkk7UfrQas", "RYz6H5BkZlA", "ZkYbX8eG22A", "4X2DNc1MdGo", "ABnbACZlctI", "PdyJRb6Adg4", "o_5y_W9S8Yk", "7qq8q0CrXRU", "A2l3E2-v114", "Qmj50Wj70u4", "azDXugI28Pw", "bivlaOOLJfU", "inXNpWLtYyQ", "4-vrNZFnG3I", "5ppOFlwOIZ4", "AjsxVjzjc9U", "G9Pxx7DonbA", "HYu5U1HRSGM", "HpyDbEzgT0g", "JBzdTeqzQ80", "JkQK2BzafqM", "LIPBVGMJeuo", "LKOtIW9vBoQ", "Oe9BKBVAIeo", "Onp9JYHfGvw", "Y7z5VfcfggU", "ZA5RlQpfNEw", "Zn4nPg6rfVQ", "_TxY6uz1vOQ", "dzQXmBc3DSI", "f0XFIrYULDY", "fp7ctf3_SJQ", "gZCmBNoqVk4", "h3571jraioE", "hzvQmeqPsNU", "jbd2Bm0i4Nw", "krgRswPFNc8", "lhZW2e-AmrQ", "lhn5XDTjleA", "mG7rvWSW3b4", "oLoqs1-ejqs", "q2hpFDvaP-0", "r4zFkKu0hRc", "ritPwwk_zbc", "sq34eyrjuUA", "ur6N8ZhjDa4", "w_i9205xliY", "xM6Eyuc4B4o", "yLhEymWN69w", "TcJq8myir9w", "dzdWSfh9M-0", "0tdm4s8366o", "1Gu4IAegBew", "1GvSob_1TW8", "1jJg2lFYASM", "2pXBWUu-KaE", "3rt10YFZifA", "3zflAXCniwU", "4gkFnqCT-Ec", "7KA_AEWATuo", "87QlFxMf2sU", "C07KFNKBkys", "DJRzVX0J_3E", "EEp-up3r4do", "EmH--rrnuQ4", "KDBq8GkgE_U", "M2qB6bWwDlo", "RvQGrDUDJes", "VWIq5bZdcnA", "WeUX7LozhdY", "XnVeRUGcabs", "Y8ykiAJ1PKI", "Z87W-JrjM0c", "aVlo4fco-h4", "j_Eix7Kgg_0", "jmurF2TUizc", "kO7-zqKxMiw", "kqBwGMHnbJY", "oavCE9eU8bs", "olAfqJ1jnRY", "uWy5etsMAos", "wnjva_rvTW4", "xcNVMH4w-5o", "-Xer9vhJopM", "70xL0UzlTVw", "836wh4LjnAE", "D0f_LNskfcE", "IKiXTWeG5k0", "N3syK68lQB0", "cvfr5BSGrFM", "twNbUqfNbp0", "HY8Y5I_jt2U", "J02ZfrA0yms", "ekpXpXAZQ88", "n4vWfHKFynk", "xfOPtEGdkhU", "Z8LGfAU6U-U", "0QKyQ3cp0nA", "17FXmetEPR4", "1VM_yg1yeDI", "1kje-kd6mKQ", "2WPTd0dZG9o", "2XfoXgRoPu8", "46YOpbJjVEk", "4QV-9-A_pyQ", "5EUHprKKWi8", "5QyBpwrQShs", "69YR4pVvoRg", "6xqRxY5d0rw", "7-QfXbK7GqY", "704B_DZzqYI", "7hWkTF7-Rt0", "8_FF-Vt-Vok", "8dxFTrpV3fw", "90JXXSyOv1k", "9Lyn0w9hAKs", "9o7RZesJ1DA", "AlykIVuwA9Q", "AptiX6Bxvao", "AvDSJa8dWsQ", "Awx8cQiYdqA", "B7m4FhYKJjs", "BrT1NOHVDP4", "CNdfoUsQPbQ", "CmttnbXLk3U", "DxMQR6YCX8g", "EG7arqDf37k", "G5RyIR417gY", "GE4emTLS9Hg", "GRk8uPUOTMs", "Glc5XPc3vv0", "Ha8uptWnCNk", "HfHhu2a2BTE", "HrkS-aPNpYg", "IAC9JSQJugQ", "JHkF4R2zXTE", "JUnb5M69p1o", "JyCyQspQXC4", "KAUedltnZEY", "LYbko93MZJE", "MBpxppbu3aQ", "MwZlRhYSJVk", "NJEwJtLOONs", "NgthkNQXwXA", "NqyNDbIyJsg", "OPocidA24sY", "QSnJKzldMbg", "QVW-nWAZJ4E", "QaUOrfivvgU", "Qm_LtI61NQ4", "R928S6qNqhk", "RkMDj2HBKjA", "RrDZw8Vm6oA", "SPqnnOMu22Y", "SR1I38l5gl4", "Sxh0evRYq5U", "TDWEUrAZvWE", "TtflXPOyj2s", "U7QrMXyvEEk", "UXuWTSknPXU", "UzrjETjkFdw", "XpTmcpVlkJQ", "YD0JT4jc6Og", "Z8LLeyA4bx4", "ZlqZjiH2N9Q", "_1w7Ml7jrII", "_MTbL3_v3Lc", "_dwJnhLJDpA", "aBIT-Hrqh7Y", "auETo69ebCk", "b114v-dYWXs", "bZzFTm2ryTU", "dy-uN4OYq1o", "fja12b23BHw", "flOtLyn-LfA", "gFVJcuriv98", "gHYusP2IsKk", "gW6klXyxXNc", "gZoHK9r9eOQ", "gh3Pip7QCAQ", "h9zyFAYE5s8", "iCpUO34CfdM", "iXoTlZ2jtv0", "j6MqhhO1BdY", "k1WHrPTwW6Y", "k6X10O_HPYE", "kXTbyjuaqns", "mEZGynapOWo", "oLTK0Vbf_IU", "ohDGI6RloPE", "pjoiOO96NNI", "poRTm6JGN-8", "pqujD94IwEY", "qJLnPgmqxJM", "rJPCAelvKVQ", "rPEf6HfDpkM", "s2K6pzVTnOM", "sx2OrfYCCMI", "tG_ZXN41NLM", "tP4GehPkxeQ", "tu0hKLWRPp8", "vSHoTRZ3euM", "vkRrOs_dC18", "wGRVdxRtBl4", "wbkgXHmNIBw", "xcecGEdTvCA", "xdYrjJtr1ao", "yBk1q2ozx6M", "CZ-bbo-o5oA", "GJAMmpuzilk", "e7FLhntIZUI", "jNiLF8lzIVU", "XF_Zq1h9K2A", "-FBSDmbby7g", "-slOlo7uWxs", "080e-stTcgU", "0VhHE1TCwq4", "4_oVOsepaDA", "66_KGLZhvCA", "6KYckRkIuAM", "6syS_EaqJ5M", "8_9BmeC__g4", "97FIHU0fJIA", "Aeg3opW8gXQ", "CNRlaiqfe9A", "DQQ71eEiW7I", "F0kqnqlwiKw", "FD6soGdlNYc", "GE4urfdpPU8", "GPADlR7dnl4", "HHz0kkIro2Q", "IqMwO0NVF6E", "Jq8e5o8oP3g", "KBd-hmTT41o", "N2SOoPS3bk8", "NmJZMnwYR0g", "R6WZeK_iqh0", "SwAtcejo8hI", "T5PlFpcX5hA", "WYZSjxe9_HY", "XKZeqvgdvNI", "_34phkEeVl4", "_JzPlcpiD8w", "a_Q4BPHFmP8", "cLF8HUNeyoY", "gNaOhiP2S0k", "hAmwm2xukZw", "hQEI7gz8eKs", "lxWTiZDwbis", "nFo07pIQbuk", "nnn5DGrNHaQ", "oyUwKpfNsCA", "q-mIVRlvUBk", "s2m86_dtXsc", "x5l6kmO-Mu8", "x7qa9nEvC9I", "-bStEPjmns0", "jcSYEJgFSDg", "7p83YnKf10o", "AfzoxcEMw0o", "J-9a3IySk3U", "Nz_C1dZjd8E", "SzlMTmd0nqM", "_Aoon_vDM9s", "xY_kjbKaJYU", "lZp8Oyg0Png", "1tgP-RA3Yio", "3TzZcDPhatU", "3VEsAyQQQHE", "3fmlK1gp6fY", "4ZuRsXeVJqY", "4yIC3U7E2gI", "6sm2JK9T2Co", "87LxgGLOYIU", "Bo-ug4LFxZQ", "DPhEhFJL8OI", "G4RccBP9l0o", "I2qaEmx7L04", "Jmy-iHg8Ihc", "JuzbUpCJi7w", "LX9KpUWOsVQ", "M5AfZc0vnyE", "MKaUhJDFbpc", "MLImU880CyQ", "MwdOgOwS9m8", "N6978oznn-Y", "OEY4rGuSNyw", "Pr2RFTjAA2A", "PyP4hYXPu_M", "T9r9qt260K8", "TqUouXqvuYc", "VMzxLJGiSPs", "WM5RoCRhwAo", "Zi0qa401pjI", "ZsJeqyXqdR0", "_sU8R7_D7TI", "_wLMqiAFy_k", "bH5Lt55q2_k", "d7jRYHAW_Jo", "dJMzLfWc9-M", "dWx6djaFDXM", "em4IgxHm4Vw", "hQhTnwDMHzU", "iBFLB5Uwepo", "iR9wr0OSLk0", "itJtIZHYMK4", "jEJfsRD9qNg", "kGXtR79ppX8", "mMioI745Jc0", "oOYfJRNk_J4", "oTFSJfPBH8c", "oyMhr63ivSA", "pkQVm1kqHOA", "r3H-6xginmA", "tbBBlRlPNhg", "uPV3IZ_dDNc", "uQCH3C8RK5g", "vcxp6A93TfQ", "xvIkKdPjw_A", "yINymT-YDjM", "zWmwNjOnqGc", "ztJ4kZHyKMg", "-kPPRuB5JYU", "01E0XE26744", "0KDVNIQDQF8", "0jg9trU3azg", "0r8a53pIq7o", "4EDRX5jaoRA", "5IodkuBv1XM", "5ezJUyizQfE", "5s2Oy0ylIJA", "6JBJowNVYcE", "6gPDgRV0s_Y", "7FXOD8LSvvM", "7Q-EiwczyVM", "7gcWEKzZJd8", "7oiL3nu7mRI", "9pOWHyjXDXU", "9q13b1rkVNM", "AbiHVEN95nA", "DVeKb7ZkJPE", "DmBpBGPjRw8", "ER2gucAdtWQ", "FbagutLfFo0", "G4osBDCpb3s", "GAGsu6O4-34", "HTZcfQBrYqY", "HcQqZKbmvi0", "Hl7YnGGHj1g", "ICsVDDaF2_Q", "IEC3jvXbhWY", "IspyvI1dPjA", "JXGRHetgoR8", "KjPfgF0pn-8", "KlhgkPdYNCM", "L1L0eiC_3nI", "N9Zfjwvlhfs", "O-BnVVB7Zw0", "Oa7-U5Z1PFI", "P8hzVg8y7dM", "QXGs5BXu1kI", "QcdbM9CKm08", "QvEPs8f0ESc", "RLTnnZ39TlM", "TNbMZWJfrJ8", "TvyDJVh0dXg", "U-QUccbCnXU", "UAYodXw3cAg", "UM8QIJz918U", "V3r-7LV0qRU", "WB3cdE1ftsU", "WN5td82hoWY", "XrqrZ2hWx0Q", "YdpnWCZ1Bec", "YnKP-EC9bVk", "ZV7dk7W1l2g", "ZlSRjtmH4bw", "ZxG66dKW9cc", "_O_QblSmRJ0", "_brCF6018lo", "a2hcLzZO_1g", "bANPZwKGvW8", "cK_6Z-MdIZg", "cl3x6ub4Bkk", "d4NwZzfky94", "daoWtESpehU", "eiK9DZO1PGA", "gD17w_NO_Vo", "gN4OrvjEXxA", "iY4USyO_0Vg", "ifC9zIJv3Sg", "j96cp8a6D8Q", "jGAZOtEkDfE", "k4zCQI-0GVI", "mAtXIXGBUzs", "nt1KJte9Qqg", "qtwvdgw4tyk", "r5xft-8wn6I", "sj7ksc5yyzo", "tta-3uOyGr8", "ty0kgWdWyP0", "uTVqXDL23cA", "vdO_jzTIff8", "wNzHDRIp6CU", "wQj3UBy1qNE", "wZPAV_gejZ4", "w_zGV91dl7E", "yhsZkHdFyAw", "-FY1L7Duffw", "-gsJNpKtv48", "-wZSqyedBTE", "0doYi9zK6n4", "0u1fWW3MaMw", "0zqRJFEi948", "2MqI356687c", "2rfGtFqH9-o", "3i7U59rRTUE", "3nJBoKbtWqA", "4DQ8FezhCK8", "5RWxl6Ec4CA", "5hSHR8oUbKU", "5jTVfQK72Ac", "64ph6l_ngaE", "68aAwWiuJ4A", "6o1hADaB2dQ", "6pSa775QpTM", "6qMa0UDG_HE", "75XKONUhqp8", "7hktABPtnCU", "8LyEv8zSgfE", "8TNO_JxaCEg", "8esA2ZidVAw", "99lP9sDvZvc", "9XWDPDCcUtA", "9po4C6usxog", "A0nAU5g20IA", "A90US72vj7c", "AhY0Muobba8", "Axhf6Gga2jk", "BEhX3FhP9bE", "B_xAs9CYVdw", "Bsl7jGkeKss", "CATCMZOoqpM", "CF79JwGYbVo", "Cp93SIt5mkw", "DHhyg4OBCAo", "ECdXK4PHDVY", "EOdD2j36TNU", "EnHYREyxi4o", "ErpXTpBLvgU", "Fdf1C4lFViE", "FsnsURNl6AY", "GHigRAv-rew", "GRUiohjy7jc", "GvbqZTUyjDw", "I2Ytt60kaRY", "ITMfo1uWFL0", "JcKSqoaPp2s", "KDedOcejTeA", "LBrfFb_3X1E", "M5jXDUVgVKs", "MSMeDOfzFVY", "O-t9aplkptE", "PClsrOyO9lo", "PpSB30OCn44", "Q0Ru7057CQs", "Q7YVMkam5Ec", "QRfmO3IjPgU", "Qm0-EONWIyk", "QpM8rxVMg2M", "QpOX-89nAZA", "QyKWs3UtLT8", "RNz0PHn9yeI", "RadsgTM4LO8", "STyzqp6aYC8", "SwxrewMAUDI", "TIlAYUNzOfY", "TZPxy5k-8tU", "U6iPLjAuHwM", "UNFmIgvlFG8", "UXaw81yAeZk", "UcsenYzAmg0", "V4TZUty7fLk", "VGQtyto8IiA", "VVdWZFwp--c", "VxGLsSGnZNI", "W0flCxt1VDY", "W0kqKenZRhc", "WM0BnlfNnDo", "WPjVc1T168E", "XK-BNwuEP7Q", "XqD06-w2hVw", "XsJyPQx6IBk", "YC5JJc8YeLI", "YSb9sbGDmFA", "ZPqAWFzLkr4", "ZVMMfKSTpGo", "_rrD_LgpxTE", "aMD3pJsBAW0", "agDAVE7lqio", "as8lsPummU8", "bLBUUTXnbu0", "bOknxL4_T5w", "c0mEd1D7Gqc", "ck59YzHSnms", "cttnZ2cyyiY", "dGwJkckDgJA", "eUygWyaAbgk", "ecK2rQjhUcQ", "es2uvwbsrp8", "esrvLY1BDsM", "fpDegy5oeW4", "gCeg9soRENY", "gENLJbpQA1g", "gi3r2CcFxII", "gn-JK-bR9TU", "hlHVfzkPkeE", "kXBrhpt57SM", "k_3pFywnFf4", "kkRBKTGTo08", "n-7kmGifzOs", "nPj5T3hy4Tw", "n_xB4g3gonk", "nmYQ-2mlwDY", "nza36P68yC0", "oBgxrx5OGj4", "pFi3PBtLpi0", "pazSrd874OY", "qhL8JXqDpHo", "r5-LuiIzkks", "rfbi2zBGq9U", "rhJLpLOCqwg", "sAGInrirPPk", "tRHR4VQ2h4A", "u2H2YXJkEAg", "uSxWQXK6J1o", "vOG-LF_Cv0Q", "wo5o3PBkAc0", "xUdDp3bhLIY", "xa8IhXyC_Q8", "y5NTWBQCeRk", "y77H9XxkDJU", "yV1FT2LKIh8", "ymdLRagVpqM", "ystV-Uj_E8Y", "zBtDOmAGz1I", "-Dz_wNbGw3o", "0-xu36o06Yw", "0rqvtaKTgmU", "1qIcq7j92UA", "2xs0CqhPJ9E", "3RpisSWWKsI", "3_YeNoe1eCk", "3iX_XyRv3pw", "3o7J6C1mq2A", "3q2efgDueS4", "3uznXt9qyMg", "4NBkwE-Cduc", "4QVAgx4112s", "4cc-_IBqYEk", "4h3qzBKH1zs", "5P9YzaTVT0U", "6HIlRmpv5vE", "6VrEX2-9rzY", "6ZDL3gQt4Wc", "76DOwdvY3cg", "7hSAchcUAnQ", "8ratLhWGLh0", "8yTaz3w-cj4", "9F_cnAgDni4", "Bb8_ePoZ5xk", "CpJcIv_1tUs", "DrCtywkh4ns", "FqPuVhRb80w", "FzSuy9zh38k", "GKnlJ6b9Gv4", "H1MbV-gyfAk", "HLF8GJR4SuU", "I-6EP934wUY", "In1go79ZM1Y", "IsMs-0IGzik", "KAJNRjwzI-0", "N49alDDnCJQ", "NINYRnjWcyw", "Nbc1bvbefvs", "OB3xjqnyOsk", "PepKpNcIE0M", "Q3lcel_FUdU", "Q7wwAD04kac", "QL6WWrBojf8", "RWm4Nbw52j0", "S95GrI9km_I", "S_PscrIevDk", "Sc4HxPcA4qU", "ShVv_P3Kplo", "TehPoJi_dPc", "TsXtBwubeNg", "TzNHoVMjgsE", "UOIsvH8Y8EA", "V4K5S0coEBw", "VGqzsbDsvPA", "VsjSpakd4ZQ", "WDoPpEu_dCg", "WaMM1Cb-2b0", "WruCqHTEmNE", "X0kNyn-EosU", "XrnD3UfqZpI", "Xu2sOOMcp-g", "Y8KLDZOOZeo", "YBPOPLgf_xE", "YtPNFuhc4xw", "Yxoh1OwWrWU", "_UWtAEB0gGI", "aMCOugPrK7o", "aZv95MlXrcM", "azO8rGt0Pxg", "bBnkVQYDutw", "b_TpEcJVuLw", "crFAyTrNjAc", "dC8iNdL2oIo", "dCi1q6aumpI", "dHfk8qXkh78", "f0AZIjg1xWI", "fvZ_pyZWUN4", "gYOvf-ArgpA", "hKlEPfslo6o", "ick-HrvCNTA", "igbSeLOJJLo", "kH6EroIfr8A", "kvPOmMKbvSk", "l49jdILDskM", "l9lGCnmZxJs", "lB661dn8URU", "lHab9TEZSd4", "lRW1blLVO3o", "mUU4uQKnIxQ", "mV-ww80kzQg", "mrcH8XR5mFE", "nUD1vyuVyEI", "npEnbyItI1o", "o8ALDcWNQKE", "of1Opj-D_BU", "p-Ir4DlcW0c", "p1Ra3FF9n9E", "pL-B84YDrsM", "pLfj-aPoPdQ", "pSSrnyTVZLA", "pZ0DfJiuiJg", "qI9QpWhbcw8", "qpB4xni_C6s", "tmIlapR96fo", "uajI8Yp7nxw", "vJE0Z3KoxKo", "wcxFTkZvKA4", "wjBUoqnGI6w", "x8pvwt1DwbU", "y8wm1LJwJlE", "yVV2VVJIlHA", "ykALVWo7fDs", "zFNCqZ8CMl0", "zI4uvoEDsZQ", "zesMJZ2JXZc", "zfJSGaL4e3I", "znCEvXsAM-Q", "zvv23PjirCw", "-a51kWFjlWo", "3ElAkOvzQe0", "8_083HjF-SQ", "CcSgwH44-IQ", "I5hWZhPeO9M", "Rh9sgxWm5gA", "Y3arS_4-CJU", "kxVhpQ-CT_4", "o7aDznBoezA", "190thNfhfFo", "LFVqUTOCx34", "R1I9getwUyw", "RqBml4cxKrM", "FcGF5WeZwd8", "m5S1EdVOQ4Q", "4FTZxYJVGOA", "MO4uosD06Y4", "QZvNs_iYXqA", "V45EVicw-A4", "Xl93eoquZtM", "m_IjoZbRhhQ", "p349cqLcxDk", "sB90Pqpig40", "4GioaMiccWQ", "RY_d3RbV1Es", "V9oWZt7eHNM", "kFMn2PFCl2c", "nijZ_cCzieU", "24fRg1P64Dw", "2gBNhZIT0Dg", "3-7lbHO9yl4", "5cNkf74s8LY", "84bsrb6j5pI", "9m7ig7zWMUs", "ACo_qZ54Cnw", "AJc-94zXL0w", "AnUo1bJ9uXU", "FH9mGeNICos", "HlM0w5dHAeo", "INYW3jIyu0k", "JIoMinC5FvE", "LGFq-tHwecY", "LhaYuIfNg0s", "RBzdAz_B9uQ", "RMXHbO2l9Do", "UDpaLF24sGo", "UsV_dNOnXdk", "YvKgoHS04Us", "ZAZwkF2QyoI", "b-jxPF9kdZc", "bqQVbMdFn-0", "ckdgl-eq-sk", "d9fkeEq-lk0", "eKpJENZdetk", "hP0nfKd3SSU", "i9zCyeRBHuo", "ikmBF-GKvZc", "mUHqIQhC6-E", "mrY35TbWF0A", "nrxaDMQ-yoQ", "qr-CxH5QPDQ", "r7o7E09kHwY", "rQp9MHsBcZk", "rqkJzGNQr7k", "tVOCifGtNk4", "toTqsyjtp0Y", "ucAvsOe1R9g", "w0HPM5Obspc", "wVd1jyYVegw", "xCeOga83ksk", "xGc2ac-gEEM", "2QdICmuK5-o", "4HLXnUrh7vQ", "4PzBPI5eXAQ", "5JyGZD9GSpE", "7TBTCINxdPE", "AW_t4irqzws", "CA0q1EpLeoQ", "FpWRw9OpPXM", "G9Zx5aKPAbw", "GIljJ-N7IV4", "HQqDMe2C8yA", "IlEccrriALc", "KGBEKU_1r94", "L2UcjgX2rbY", "MrxJ3CUSXqw", "NNIoRORo20E", "O-tMva_8AEo", "PFALeTpozAY", "Qy0PdI4beL0", "RW5T_OgjJ3k", "RbQ2YwnK0-s", "SSkvmgU1r0g", "UYzYbcUWLCI", "Wu2G8OX2Fas", "XQk5FcPTM8Y", "Z7m0ixE_O6o", "ZQxsi-gElDY", "ZSH0ATfPzNc", "aBbgzz-uAsM", "c8111HJYt38", "cKUAjZqRlRA", "dNjDxi-sfw4", "ehIUq7QwAF0", "fU_4GkfmFJw", "gb2Kqlfwb-8", "kU-R3T2xvyY", "kuR0ccmHZuU", "lAk7D8johZg", "mwAlJRvjHXU", "o0L1TMjE4Nc", "o76yQTapmJg", "puhgvnojeWA", "qLusdgYxvfc", "qXfnV_ggn1U", "tNaFltwt2C8", "uJF_uzgUD-M", "v9Nav6Rz4e8", "vKEGmN6vpRc", "vrQiD6Mpi78", "w4N1IWR6IQE", "wR0Z1eTT8Tk", "xRApaBUzqCs", "yN5xb9eAJH0", "z-0loRSaJ1A", "-_n3dMJ4OTI", "-a76Bug3LS4", "040JETP4lE4", "2EstAhBtAOU", "3MEMM1sqEbo", "4fH8PDQpFdg", "5lfryq6K7cc", "75YIX0eJSwM", "9NpZvCT6zv8", "9ObmBPyIvRA", "BEGW7pzNWNM", "BLVDV3J2C50", "Bh9Psu5TGxM", "CRfHyNZjD-Y", "DdLu9-44UC4", "DtEhbUIU3Wc", "H7yffIbWxwA", "HhULB5578rE", "IeEP5ROuC2c", "KxRiwE_QU3s", "NRv07xre2zg", "OOzWAr96xTA", "P6AKO0hGhrs", "Q0aTCGpN5SQ", "TdyRi33Lol8", "VYwPyx-d0o0", "V_J_keEIdSE", "Vra6ScepbPI", "Vs4dHuXLiPo", "X4D5jvdeAyM", "X7UfOVP7TxY", "X7fquNdtRU0", "Y-zx9G9eNLc", "_XjAxcHA8SQ", "bK1hMou5Ne8", "c7Ye0l2ptTk", "cpF5AoRtF0I", "d63FS2gLLlc", "dYDDs5JEhhM", "e6QwMyx0IN8", "ezitJzFJeeU", "hHWRodwB-p4", "icyTL6m-3us", "iy6396Myy2E", "jM6eKeS0mmE", "m2YRTryZ_LA", "n2zpB25OdFg", "q0jPfWjLkSs", "q4LYzl3uXvs", "qHAnVi4dOLY", "qjlyW7nIQxs", "rQKUm_Mgle0", "r_J9upN7DN4", "slSjLBk9HH4", "tPCya96Dl_Q", "toxVpIXRqk8", "vjElxfaGmBU", "-Mi-PPrxsXE", "4-q1dKlNHrA", "ixwwI_cHepc", "GyUuoWP4g3g", "HiQOiuFe46k", "HjafFZIkw_k", "QvOzUOGeifo", "ey0_K67UX_U", "jOOd7yKYcgs", "kBzyLM5DLI8", "nCbfbRagmgc", "rcYvouYO9dQ", "wE0jRuZvLFw", "wR63jXXfs00", "02bpwUjD7RM", "7-w4oJdU7Ac", "PxAEZAoSzNg", "XfHlQhCjw8Q", "f31V5tiHHeE", "mlynGoZMRVU", "-hjcNG_pXH8", "1KVb2j0SLDA", "74ekfnGO1Zg", "7ori8VgXmbg", "A1Jb_WIALWI", "AOJhbbstjYo", "DVXXQ1CvyTg", "E50hU6JLUHI", "PVxURGVHZQ8", "QFXXAwLUkW4", "RSA3wdrqpYo", "YOuNOubk1aw", "Zv4IWIx5aic", "_7PMQeBqDbo", "aDK7RsyKmBo", "be15tfY41tc", "eSZmh286pVA", "fPZ4qbFdYVY", "mm5mU2ddxsg", "x6TGH47fYQQ", "449HRFneP_Q", "JbF_e6lKv98", "V8PeauGTPO4", "Ws1QHcypZDo", "Ys52o7_8jNY", "ZZpnzPpFKWY", "bqB38SQdwOM", "lrNlA-3ujbw", "tPJvXi05MwU", "w2vO-LgEJY8", "0H0DvTalKyk", "6B8Q5P15jxk", "7LYeB8LRQRk", "7zhpkC4Y8e0", "872FAfEwqyI", "DM2ZvNzpJ2w", "EBhz_7iRIy4", "Fu6g-hr1Qac", "HYc0d8rReCs", "JhifkAswEVw", "RmXMPNcQLX4", "TyW2rXp-Gc4", "UB0qPnZ9yM8", "VbacKac0iNU", "ZJWQXV2fsgQ", "_9IY3Q79D6g", "eqOTjlSPJz0", "fULSBmYkhbw", "hZ0Zm1WUBg4", "hgL_-d1_rlQ", "iF-P0TQfg-0", "iajGtLYeQuM", "kgWNAK6_gZc", "lyhYqxet9iY", "oduMoUnGF4I", "qe00lfRl5ck", "tF2qUAj_Dss", "vNtUF9_LP0c", "vagG9XE9-H8", "vxvgxmjQ1rY", "yX04d-LYzYo", "8l2V3ZjeO7g", "AJ8KQAkAspo", "HQmYI73UFxI", "LhLR4GUg45I", "Qa0H2DvDz5w", "X-fbnBVGSuk", "XQKedxlGkck", "_Xnms7oRu3w", "cgMsJ6gbwkE", "cu94OcZbXQQ", "iEHlpPl7iEU", "iLLRCfBv35U", "lVHZZnW7hDk", "luRvkIrubkE", "t733P8lrbkw", "xS05XJCVS-I", "yPy2mLXpplM", "0W3QYPHgpQY", "1FuL4tKqLPc", "2vi0eXrOMZ4", "3r-CVyFyiFM", "4bf3zGJ1EJE", "5KO5O71dLIA", "703QiwKV47k", "8iSJ2c18kwk", "9IlhqXW6wR4", "A0vyKrU4qu4", "A6aa599aC20", "ARU-HFE7_-U", "As-XjdGn6-k", "BQuzoYqjxy0", "BpsWwwRjGVk", "E2ddOrqg6uM", "ECTy1XHs1yc", "FwCphh8DBQI", "Ga6eR1lC0P0", "HVYw1fGFfts", "HdeKvhAltcU", "HhPq1J93uMI", "IA1rMxZbqv8", "IP1xvhsteQ0", "IYrGOjHPH0s", "IiA6h0uGDVk", "Iz-S19Pt1uA", "Je_pvwr80fM", "Jnxp0exZKAM", "KNkPGz2ZGQk", "LTuRbodTe9Q", "LehMzv6V3b0", "MMsbXiiGdFA", "NOc54zK_8IY", "NR6goFyZ0_c", "Nh14HRufqWM", "PTajyYcxv4I", "PndDl58tg7k", "QN17bzJkehQ", "R5zee4ZBYhM", "RlP_3isrqBY", "Rz2xki3GV5E", "SMIXM_UZErw", "TzWirI2Ozew", "VB_d1UJa3lQ", "VwpWJ8kvxPs", "WKpo5F3exCY", "WNagYrCuLeA", "WSI44ZghK1s", "XzxvE0A7J6U", "YLmmdvYg-6I", "YpZ4VgZrUDs", "Z6yRy2h2tRo", "ZSjMBqgasU8", "ZTdy_6G2GkU", "Zbr8bn-gCL8", "_cIkkn8dQdQ", "_rMI23M3XW0", "anuj3BYL5CE", "bIlGcvDXdwY", "bT-qoWveiwg", "bYSdS52jWks", "buvWLVZrv6c", "byWzdxmigFo", "c2cPwqsQz8k", "cE6dCZQppUQ", "cW6VGICXUNw", "chhVrYJA64U", "cqoLYgSKDGk", "d3f-MY_-dbk", "dInY1l5ONx4", "dq6uRaK954Q", "e5jt6GLrczs", "ePNd_6Ry7h4", "gIW3aGfwyDM", "hBsqarLUl1o", "hPaP65OvERE", "j7OdYvgJpeo", "jCCy2IerBEY", "jFhMA_tX4HY", "k45WCRiZaYI", "kMoee16Owu8", "kYPmxUPnsVY", "kvCRb_IZy-g", "lud9colMdaE", "mVRipK5hHdc", "mmewQjRt8Ss", "o5VsrZFUXtc", "oP1hoQAqx-w", "q7utj_l_Wng", "qfjm4m6xs68", "rQRB3MBmJCg", "s6jOA7fM9LA", "sGygybcX_xU", "sLWl_keBpUY", "sMMQlBpJ4O4", "tWOsXCKAG60", "uZUBjdAjTSw", "um6Klg-PiPY", "ww-T8WBO38I", "xbsgCSz18rc", "xkBunCIHSGI", "xubRiK92ACc", "y7OGlefObyo", "yTmECzXG220", "y_4m5e7E4BA", "-TOWhOCY69s", "22U2vkHpCBQ", "3TFpb_DxnrU", "5k98l-b4OB0", "9K2_e7PK7rQ", "C-OUQAWRmsg", "HWdE7HE8Lig", "L-nOOGTnIjo", "NXG35YWfP0o", "OgxcVT4EoDg", "QXIh36fNRyM", "QnYncF6BoyE", "WWASP4z8UwU", "hgCYxKf8ooY", "hzAYzsD7mR0", "nZRqE16uXlY", "o8gGcCzTjuA", "rYon0b9h2Dc", "snOTCT74WtQ", "xc4tmd-wBtQ", "IlyprupIqqg", "xAjVbaAvXCc", "zSx2qUX7RLE", "gG0uO6tHd40", "25ytJfYOYeg", "4Pov9NKL_OM", "7oTQVIx4PaU", "wdS8_FoghlI", "-IFUoN-Jy-U", "-t-1PHXdAEU", "0DVVW8Kj4Kk", "0RC1bLCQxes", "0wdyjU3X1Y8", "1IBvJNaFai0", "1Q_ETnYRbuk", "1cf0ctEfvK4", "1i3DKjiRevE", "1t4t2ykitN8", "34JdM1U-ajo", "3GkzQC-I2IE", "3P48DfIrT30", "3XyGKH8PrCk", "3Yp8uGy4HVc", "3q33fZGkcgM", "3sN6BEgtE9c", "4TDsXuFVHbI", "4eiC6bAz7P4", "4skOU1cwbO0", "5PC07-255M4", "6qbrVzubFA8", "78rLGGvEaNE", "8K2T1OB7lh4", "8KvrdfznKrE", "9ty10Obmv68", "CTEDxsGsZmY", "Cy1K0MuapUk", "D6CIDyj2L5U", "DAwavXV3zlE", "F--Ujfga1ZY", "GBY_aWd42KY", "GILKiGgfj7g", "H3Ina2qLgiI", "HSPzV-bx6NA", "Ha3-9xd7aCU", "I-E8lnnKAyc", "ISsZGnrjPrU", "Ij5ErNco_do", "IsguYLed29I", "J5x0qVPhgFw", "JXiHQQx12qA", "K6JTADPJgfE", "KFwxAiDxIec", "KOszKAQcFWs", "KjcTR3GxVfs", "LZSq83mrJ68", "M9m81yFKhgc", "N4Z46C2d1yc", "Nr9XwsYNek4", "ODEkd0bTCrY", "OIhr0O0X1Z0", "ONt709UhYd8", "OlHQ770VZC8", "OphEcdAsmeM", "OwPXyqaJ26s", "P8wqo1MT0f0", "Q3Ik6ya0Dmk", "QJ5nRQtbEFQ", "R-Y93Gchz2U", "R7ObW0GoTD4", "RKc1rI2LZzU", "Rfi3y_UbOZ8", "SDwiw9JSfVs", "SHJ26av_QkU", "TY65tGT2F44", "Th814uHpNcU", "TobMrFv2nkQ", "USeg1peaNQI", "VCX4X-9Iv7M", "VPmc8FMXlc8", "Vg62AMHTus8", "W47i4jqirwE", "WOo1Uu4uoMo", "XNCt8WrwIAc", "XYfdYDaIX20", "Y42w0WGEpTk", "Y8acmwoD-ZI", "Y_9Kd-3pfPI", "Z5OLqFD4N2Y", "ZOIwU1_pFEI", "ZkPHSAg9Enw", "ZpnuWqK1pvk", "ZzIuubkT-oc", "_T5GuAkrk_o", "_XAibb0SlMk", "a3gL5I4oegQ", "algXHFwjslk", "apiScpnmNw8", "bk-12-k-xH8", "bvuP6DvMlrM", "bythdEjtiTc", "cAAsg7jaIQg", "cPLxHavFkdc", "csEgDtOwjEw", "cv15qX5yT5s", "dzQzsiu_30w", "e1GgeHthsFM", "eGPx91a_i5g", "eTBYhHTcOEo", "ecc48HRUBkc", "ezeQFVtxuic", "fwL9MfMgCfU", "g5OIlK8-VqA", "hFYyMCYpfZc", "h_hNhqeAM_g", "hezqPdWdRd8", "hfwNbzvKh5A", "i81vamfzxho", "j83XUWBBErU", "jE9DTpZzLwg", "k4ETvsUcns0", "kH6LGvHVENc", "lJdrwxm4UEY", "m28UaTODSdg", "mDUQmdKNp5A", "mMol8tSmetY", "mTEtPFmfPYQ", "nYbq1q1NOUw", "nnwM7jnGpwQ", "oWp-CasK_-Q", "qM0jHanwwM0", "rarxbRTF0Sk", "sHGB-1yZoUk", "tEbl5Rzp3a0", "uDv8KCM_djs", "uWUckac2MIY", "um1bfM_JdfU", "vEfO58r2geI", "vZPQG9TF2n8", "vo2hpMk53o4", "wQs42T3E3UQ", "xX4LGzjlYSg", "yolIATcdwvk", "2G583INfGVc", "2h7zPAO-FeQ", "8EUrFc2jb00", "LCWkJ7PocTE", "M76cjbE_4GE", "OVD1a_g6I0s", "Ph1k5Tx586s", "R4jEaNVJAZA", "Rs5vek_-8No", "TeV4vEio6f8", "biLfMMTMvic", "iePQ_tLcp1s", "jU70O5nsCUc", "jUaSojRYWHA", "kJCOA37ETpQ"], "Rock": ["9Sj62YGZXv4", "KaazaRZJIIA", "NifYKW7yj-8", "S0ohddtxdlU", "TfLd6vx6UWY", "XwzSqEaJS8s", "YoOJmvr5puI", "Zpwn2BIEBNg", "_cm4roAdGxs", "dlFPu0e2irg", "e-ivj-02hSg", "uJssDGPwwvs", "x9fT8-NIJEY", "-KPluXupX3E", "2l8XtrSFL6k", "4lD-0mBJIRw", "4z3Udp5dFp4", "6wlqA_9zftk", "8eHAznLrmRo", "BRGeTHVyeIE", "Dsk9Mb2HhYw", "Jv7eAYHAQ6s", "LSjUIzRSNu4", "Ukpv_vb0HGI", "_4Puht79WbA", "_zUQcVT0P6Y", "bmF330a4HmA", "duRlnhXShq0", "e1wtStJ4Pp8", "f-Ba9J9WNSw", "fMOKqHWCb1o", "fvhjKuvQEZk", "gKJTW_-TlxE", "g_BrH4cAbLE", "hMf9mVMy_dI", "pYtFjf1mE-k", "tljHbATdkKY", "w84ZuHBfiOM", "zuMunZPoK7M", "-_vLZeaVD-c", "-j1ZL_BwQWU", "00Y6QKchAlE", "2DZC4bpGcMU", "2ZL1IAHRkNA", "3DwjBhpEdYc", "7QLZkHFdWKU", "7t6fY_g7Y30", "8eKDUPRyCqY", "BQK8VPgsJTU", "IwXGYDJ-R1w", "SBByo89EVeA", "V1ICdLLFw-E", "Y2b_js9kKTs", "YfbAnFJsRIU", "ZkMBMlRX9SA", "c4n6XvuP3p8", "g1Zu7fF_g4Y", "hr4Jadl-CnY", "pg0JUdvWeYU", "rd5TYtfw05A", "H-oMDkaWb_o", "IbayzUGUjnE", "NpQHXX83XNo", "1D5DV3KxUOg", "IWlHY0Qy0q4", "K6AckIqKbhM", "KNJdD7kZCmM", "LmHB5CU-sRk", "MvU3MvXbGxA", "P4sNkC-_MBY", "XK4THymdR9M", "Z1CUE7289WY", "uCcw8RJNWtM", "-OoMHYVxPL4", "-PJf3j-Inf0", "-RGP4r3EiLw", "-klSPhUg5Kw", "-ma8XCSU7Fk", "-sBE94rBbc8", "-syxE67aUtQ", "-uOOkmD7aiM", "0B4gAKs4tsA", "0DAHboBHbPw", "0MDHcxL2hWQ", "0T734n2B5OI", "0UGL6UVH7ww", "0UdanbPXSvY", "0WytfK_xMf8", "0aPNI935-08", "0ckJjvy3wws", "12puT0KcE-M", "17TuzmK2QRk", "17p1aZZpiDQ", "1K1SQzL9Tgg", "1NSvYphBvW4", "1UBxEkHdpow", "1codnyvsaVY", "1gHF2FH1T1s", "1otI77_ukIo", "1sGysPTWiGI", "1t5QKXB7Gro", "24TM9acV6oU", "2FHblxPpzU8", "2HtuSQZINSA", "2OoWfT30oNM", "2QErtJwSEQE", "2UIJwLv7sVI", "2eHOH39ZyBw", "34icdOapNXA", "3M-DJMY73Wc", "3MpCOS0TevI", "3NfszKX0a9Q", "3ReNzzi4MeE", "3nJZagQ6cFI", "3ntYmWc1Ytw", "3tpHYcgkNrI", "4OlIdQOrdW0", "4Zstk1uQiBc", "4ZukKt8Vrhk", "4aOMSOul6tQ", "4iTlLWnYc0M", "4kJNZ9jhfNw", "4str1aTCMls", "53rGLh6yhzg", "5HlEiG81GDw", "5IBUGFEedfA", "5NXOb9cl5j4", "5PR9eEkTE1s", "5kMyDvFn96o", "5n0xYquTEig", "5yF1sag15ho", "62yqa5LgdyI", "68QShRJ0ioc", "6SjHJFkU9Zo", "6n69hm8Usrg", "6r_K4N4p_4E", "6spbIv-fW0I", "7DFYyoP_h_I", "7KVJlbm4Z5w", "7M9dSLP73hI", "7NGbioYoLjI", "7tnMdehauqY", "7yLLOd6KeSQ", "7zqOp6JQhdU", "8-vB2Fh2xsw", "86J90kw1xXY", "8CPsvvXTEXI", "8VvtwZ4NOy4", "8W0gcRhrf8I", "8_TP1LhL9hY", "8bHGYRwKPNY", "8bmrJbS7WLE", "8l2kKDabD8g", "91iAL4XgZ5k", "9FF3QcTZ47s", "9JIguakOPJk", "9Qgy1rR_WXA", "9boS7sGnocA", "9fUv5MoezVw", "9fzpX1gdKhg", "9nHqTx5diHU", "A4qIufS5iRU", "ADLQfDd8AOQ", "ANsJFHGCDY0", "A_UakP9_sqw", "AfMUAB1ZyXM", "Ain9BnuBymc", "ApLw_z8k258", "B0cY9KbjV4U", "BToN9NNVMXg", "BXSXOwU9-rY", "BjU89rVFWUc", "BtQh-wjN3Kw", "Byzd2WiPgow", "C0aRgHZGh5w", "C1CevWqgOvw", "C4a4P9wj9Ls", "C5sVJrbPlFw", "CS8SyR4U6U0", "CVxhhdutQzU", "CXz0KqSK_wM", "Ck7tmXMZ1OI", "CtRcFXi-giI", "Cw_8tsBFvWI", "Cx7OdafoZno", "DDziDQanFCk", "DHMlx0YxuP8", "DK9N5oTuh9g", "DP0pWJy_4O0", "DQHgH8kNbjo", "DaMH3vxeaJQ", "DbI9CIo4eSo", "Dg3nSKgNRKM", "DrDg4Y5v5Sg", "E4rED2p_AKE", "EEbfZ4Zj7b8", "EHnuopwhgB0", "FBvTXzbzJvs", "FJKMkoRtSoo", "FeQn-5TplhU", "Fob_Kps4zmY", "FrlRqupCFJI", "G0rJvCPGNE8", "G2jLD4Ta4fo", "G6j0Knke76w", "G9cDSs-gQh4", "GH1ACfxCmEc", "GbO0wphBBJw", "Gd2pPpjjZmI", "Gr7soLmkGys", "HZc2l7g5lJg", "HnEgb7gGu5o", "HrTy9jTiEQw", "IEpktUghD9M", "IIblpRHns2w", "IKFUbqnzrsY", "ILy9y8Xr8dI", "IbU-Hntutmw", "IngJcv2J-Cg", "Iw3SVYl2VMI", "IxrZqKDuo8s", "J9JZVG5raUM", "JAkvWEnBjR8", "JpeIOxUWmxE", "JrHoqvErHMI", "K1q91LA_l2c", "K7SqrQvUQMo", "KNhJ02Jv2ns", "KaVilkqrBCs", "KeXXy1q0szY", "Kf1ufxqnXs4", "KgCrOvgID0o", "KyXKtFrVsSw", "L0K0of1yZaI", "L2kU3N_3mdw", "L8a0ee4MxcY", "LAIsBIaMTu0", "LB5VJzl65Xs", "LGV6gHs8ROg", "LQNtdfH8zck", "LhM3K7zAJ2k", "LjjeS8khhxo", "LnbzO99Nz-Q", "Lq0LOmPIB48", "LsCeB05nhVA", "MQHU7_OGeAg", "MnCrO71kkmQ", "MvX8s3EKQbA", "N1gPA8_Vkgs", "N6zmQC0vazY", "NBeWvsvRBbU", "NYv4hHDxhbk", "N_5TFv8X5-A", "NdUUdoGm3v8", "NeHsa9A0NWQ", "NgA4lH_bgp0", "NoBU6boVxeA", "Nrd0RoswmBs", "Nryxk5gEiB0", "OMItV-YY0VQ", "OQOdtG5cF3A", "Ofz0CbVPUpw", "OjohB2FIlbw", "OkfxQNI6wtQ", "OntYEsIpheU", "OtgSIthzpTc", "OwNJwlSf1kQ", "Oxw4FT5skH4", "PJc6tE4xDW0", "PJtIh5wjxJg", "PTlWK-68AhI", "PVY-RjsLiBY", "Pq-0Fh2zzRE", "PqXlB99tK6M", "PqnNa1KU_VQ", "PwyfFgsD1PI", "Pzm3QIg7i0k", "Q2umVtT4Ug0", "QGOoey9AyF0", "Qd24McU32rI", "QgAW4zBl1Hw", "Quq4cd9ow0E", "R-fKXESs1Ow", "R2PsjUH6_Zg", "R2ZnVv4lQ6I", "R51j0hRI6eo", "RJv7c2aQKMU", "RO4ImK2-tsA", "RSxAmKKAPdc", "RU5McYhqz1g", "RUXSR5XxsbM", "RVC3puPm9p0", "Rh4hq39NTkE", "RlUAQ_JVraY", "Ru14XlWKEo4", "RvZ4CppmAnM", "RxWIBoJwvc0", "S9zxtvYhdvs", "SjHhwbn2cDU", "Sm81eM777k8", "SmoxSBxAyyU", "SnQv3n-1dtY", "SvZBBTV1o5w", "SweanK4e4kE", "T77mhDoNfeY", "TBBMWu94CCs", "TPgHmTwPfns", "TQrkmvWtDMA", "TUwtxkZwc-Q", "TVMQfBHXk10", "TZqhPEnDxg8", "TaivAB4nqVM", "TeZypjF_H7M", "TfROXtJDk7I", "TgE2AGDSck4", "Tk4ACFi-HS8", "TrabIfgKYAM", "TzUy2TXgRCY", "UELR5K9DjDc", "UGWx4VV7jV8", "UMlSiPzVvB4", "UQ9giREOyK0", "URdqI_c0E4M", "UfMG6vUS0Ow", "UiQ0VVilHjw", "UvcWLMcOaOs", "UwWE7tVJxCQ", "V5ybeZdu9oM", "V68fzTR3TR8", "VKriSQaR3eY", "VPWOevftIpU", "VXkMaMcL5Wo", "Vf1BscAzhgM", "Vj5Uh3Y8pUU", "VjC3kMNRZlw", "VnMyPdWMNzA", "VrRp4C8icU4", "VzWtPTZ5ZHI", "W6JHEaoCOfI", "W7QwNPSmy4M", "WMenXBWkyks", "WrhKSsTD5g0", "XAqhUI6W-XE", "XSmCTe3L1QM", "Xsqat0FSHqY", "XuWqzG7_gRA", "YAiAGLpRrXA", "YPZFQieBaFg", "YcVWaVPIqdM", "Yg1rwaSyxeU", "Ylm4JC3wtUE", "Ypjui2glUJE", "Z42BGGFoIgs", "ZBQRXV8Gb8s", "ZDuHmNC2V6U", "ZTRhimfxCzQ", "ZTmRe8vIV2w", "ZXsfT5zNCHE", "Zc8B0dTiuBQ", "ZdGxkLoeNfU", "Zu7vN27mve4", "ZwHvURg47ZA", "_4MP74zaESQ", "_4mH2HwVF-0", "_EGx6CSJtC4", "_EfpKSPC_Y8", "_O23wmMDnUc", "_eSyghrHarg", "_p3fpa5KBTE", "_u4uE0V1IsA", "a0ShltLFrH4", "a0bmxsnI8Hs", "a7qi435xPMo", "aDp1V17Hrv8", "aG91_wCxOq0", "aPO7V6-A1IQ", "aWjYDexnJJg", "aYFqyCXsHUs", "ah8VwXmIUb0", "ahT7lELUg8Y", "ahoOGIDQ8KQ", "aoAvoaL_JpE", "aobs9oa1qw8", "arFzmVLp7pE", "asD3MT7xUdg", "aw_NJ06AAqA", "b-PBocb7aEs", "b1K4E3VN8pM", "b5pIA9Jg1OA", "bEZXNojLbg0", "bOnG6RChe_4", "bWt1cZ5a8-g", "bagUaRzIVpk", "bkUrerpKPJc", "blhN8AyBL68", "bs9YmJ0jW-U", "bz1r33CEBMY", "bzf7xFO21-0", "c7ZUT66s0uQ", "cBifHw-4I8k", "cnMgZoOWOX8", "cz2GQiXuO2Y", "d6J5K1akBKg", "dC6ipJyW_i8", "dIxlWDC-OXY", "dLxZzpPTTJo", "dN13_HICwO4", "dTM4n977u1w", "dXt6NmIYiz8", "dulQo1ojkto", "e4UG54oonk4", "eO9__pAt7x8", "eRAOygElT6w", "e_sQsafGCTg", "eaE2-iqd7eo", "ejjA9KRfJUI", "ejjLb0TaoLo", "esQphRz_4i8", "f1QAEdMXLqU", "f2k-NbpkpSY", "f7o4xj2G3aM", "fLgjyo_YWf0", "fPoKagXzObs", "ffC2JR5lzBw", "fglK3B5yQjw", "fkro5qQWDCc", "fkyoM4-kJEI", "fqufaA53Bcw", "g08urOFR1zI", "g9CgLnazq20", "gXiQ1SQIdYk", "gbwyzw2kWPk", "gmvUNNB41bM", "hBE1NatzVpg", "hMOfZpXhaS8", "hOfl5GOQ2oM", "hVzmvV866sA", "hf2kGGezmqY", "hoVi1d2HgCw", "hvlU7oumWDw", "i5Dr-yl40BM", "i9MaCbplE-8", "iAK5xG5In74", "iCK-X2bae0A", "iF2lriO4xG4", "iH4vm47lNOY", "iHGJAMkfWLM", "iSAI0fgPTc8", "iYzeCcrHkns", "ifrG8wzuFfM", "ikRsurQvzg8", "irFZJmFsyFo", "itzKmbXOmP4", "iyZs6Gywg6s", "j84ID7EfFBc", "jExlyKPi2So", "jQB59QxpoiY", "jTeyfqTl27w", "jV-9St66v8w", "jcCwSbIzd3Q", "jenxmIP5qiM", "jv9jbn6Uams", "k8BiRFsfvtA", "k8k2M2pEDNM", "kHdZFcV_75s", "kJ6VdJO_cxc", "kNBmCXFcehg", "k_0bEqGY19A", "kx4zt855OsI", "l3v8j9fHwpA", "lAPr-S2Spk4", "lPZlFKciPdA", "lZrfHLjY4GM", "lca0Z_Efl9w", "m9_GYIDI5O8", "mCfd4EMgmNw", "mHHXWH5iFkU", "mKa5ekt3f7c", "mRFmu5XYy2k", "mWb9MQJdj9w", "mgV0EgLYNso", "mgiu60fXgA0", "mhSC9Uj4dOc", "mwVddJsnznc", "myEEYR05sNo", "n02QR-bGN8Y", "n9PxA4fl3a8", "n9o-jsESkiw", "nCRHCb0ULfQ", "nQAmCuzXYjo", "nXrg5K4aScg", "nZNv_lTyRuw", "niNGJ8zqdp4", "nrWgs_m4HVs", "nvNoYt5b2c8", "nwenIiJYoPY", "nwpTAs6dmc8", "oEDgrx2vvWQ", "oFCfWAUmggg", "oGSU37TJnkQ", "oGzKYR1mfg4", "oNtYCuIVIto", "ob1ySBRwc8o", "oeu-AYpcdfo", "pNTrJAfk1Xg", "pc8KD0-DHCY", "pedP3JvjzPE", "pkiIEcNCQQU", "pqBbi7hw7uU", "pq_IZZ6c75I", "q15Mwu5kF9A", "qN7_qtRXlTA", "qTP9HVxKsZw", "qX0isJLefXc", "qeEDddqhoIQ", "qjIQaVRx5Z8", "qjNaG6rmchc", "qm9RjxQFV9U", "qwDjroewunQ", "qwrcDOASqe4", "qxWD348SNv8", "rGyUAHXnZv0", "rPN9zwiFI2U", "r_P5KlvL6dg", "rciCowK_GfI", "rl0iZkCKJN4", "rmOPXwFy6fE", "rqfEtbj1ey0", "s-bTMy_KMRk", "sa6yfbZv7iE", "tEgb1o68fwI", "tGDO-VOuybE", "tIS5pSviZWs", "tISjL-rkGAU", "tIYCfaxoLj8", "tNx4jYNoRNM", "tO1Ka33pRw8", "tUpsYPluECc", "twZwO2Bf17s", "u56Mz1Gz_cE", "uBfcuR4Vp2Q", "uiCG4Dcd3Fw", "upFn_YZsnt0", "usWJvZz42bc", "uu1kt75nzvw", "uyI069xXHM8", "uystU0G4EBY", "uzXaC19tK_4", "v0tkJvOnpz4", "v4XQpRKrDCE", "v4kuG86a3TA", "v7_pbASexvQ", "vfbxoWl7rFU", "viytHtiIwSo", "vjYyWWrgDs8", "vkEOnzJ0FEo", "vyaa-rAKySk", "w3LqXYq4ybg", "wO2Dwse157U", "wX8EHCEAhEM", "wdSSsK6gXmM", "wrALykuYQPs", "wtcnIn8Gyjg", "x0WGPTbXZso", "x6XDZYCzYy0", "xL5PQelU-No", "xXVj46hs3pM", "x_4gsZQ3aGs", "xsfuvL4xDXI", "y41pmFAD6SE", "y4n5xK0zLjU", "yJS51F2RByQ", "yJmRNWDHaOE", "yKmM8Yw1UNU", "yNR0e6mTjUs", "yPBAuzX6hDg", "yee2oC37_hE", "yr9HgQYwhsM", "yukno8YIvD0", "ywvE339VUtc", "z-cC2hHV058", "zCQ3RLjsrhY", "zTkLoqTmaVU", "zuF337qMUqQ", "zxj3NgdKmnQ", "-HxQ0ZLm1L0", "01D-VMoiITw", "08-ZTPqsh1s", "18aCZR_as8c", "1Er3Py1x8_o", "1peW3gvmgko", "36gSmwfbUfE", "37nxSaNOvxc", "48y_gDvUOlU", "66VOERJdZmU", "6DxaUPn3hkE", "6mSGC5dokrY", "6zbcqlCR68w", "9GM8-aeuU9w", "9LJ8oOr4cRU", "9ZH6UxnJenc", "A8fvLGrdiNI", "AUMNfri1jVI", "CIROR-tSDps", "CVwfcvjvGtY", "D7_WM10BTEs", "Dqozh3fvzvo", "DyRi1gkoxfA", "EO6qiMn8tD8", "EeV8YsjjIAk", "FKVDvxH7F2o", "G4U1eEI7Rc8", "IItqmx9PH34", "Il7pIo-rl-U", "JxqM_eQJHVY", "KZJIJAHtJ9k", "KajBratywJs", "Ks4G5Yw4iCw", "N-serNDmlEo", "O9gnuiI2M1Q", "PWnTWVMiA7c", "PiPbBWQgExc", "QGJBJwsw03s", "QNsM0h9Q_YE", "Rz9vyGqfq74", "TWKziyz0HeM", "Uiv0bsNsCLw", "V2fKtDiC44U", "W6-I493001U", "WZyQ0-8d-ko", "Wtrf3k8KwHk", "Yc9NHBOlk-4", "Z2Dn_kh145A", "ZLEEJ1b6_i4", "_GIeWDO-bR8", "_V4b5NZ5lCk", "_t17aHEXaqQ", "a3DWDnzrhvA", "abZQYCsU-Bc", "cJWL7K4U31A", "czJJJWeUyTM", "dK7ltydzmbE", "eD8cNWa7nLU", "ezphwX2mB2Y", "f1fHB3IP2Wg", "f8ljWhBEUng", "ffVXE9-_Hz0", "flZrNDcGG5U", "g7372N99gKk", "gCy-LiHuqaQ", "hRlMrSxPHus", "hRpdXWiFEP0", "ih5RZ56l270", "ikmEXs2MFvo", "jFClHuqYIP4", "jKMpznXDfPE", "jWS7Zh38c64", "joQgRMqrat0", "lqwvyAtZzZ8", "lspsexfg9oM", "m_QK3E0G_gk", "mcQin46ZLIk", "nvUmn3n6WkI", "olvk5xbXElo", "p8H7noHVUGA", "qVUwE3kwqEY", "rzaPH1L-LJQ", "vcCEBYLLtmw", "w2R5m4x3_CU", "x6hqf3Ty3rc", "xoiL_FYnLVE", "yBeWmg8qTqo", "zCqahMjkPAs", "-ENJUz0KyGI", "-LwMQq390Eo", "0aLtMymD-9M", "0ubAd4-0S5A", "5Do1_Py69WE", "7BRT39xswcQ", "7RiqKXc_b8g", "7XAQUFWKc0o", "7pbmNlWsX-o", "8FKZajbAMyY", "8ZfUkMLfCJw", "8hDy1DGHTwk", "9IIVQXXWL9o", "A0wKWZqkNGI", "AAwOAxj1KZQ", "Akb0kCl0kCA", "CRZmdRgYFa0", "DhiqqqfOXfU", "E1Y7aPCGNls", "EKL9VcyuBqs", "EQfF8h5bVeM", "F1v-NW5U964", "FIq1ALSxVk8", "GmDExAlUQHs", "JHdGQPYAJxQ", "Ko3MQ1DdOE8", "KpPHmumM01k", "KwY9aynMlqo", "Lu1HA_hzyTs", "OHzFwEdrWcc", "Olu918fuPRs", "RDL0Kfq9rOQ", "Sh5cVrENGvw", "UvsxySAexho", "VoWfPuq5LhU", "XQS24u0YnII", "Xu0MmwIWc4E", "ZGx2ONSX0Ig", "ZtlMYCNqdKQ", "_7VaGdKrtfk", "_d2o0-m6Ihg", "b1JmFqNdULw", "bTT7aX0m4q4", "bWap9sz81Nc", "bpdPW898Se8", "byhGVFT0anE", "fRsIk-Hy5VQ", "g4bLIwkQ5_Y", "hhoMCy1UjoI", "iS9nrdp7XeU", "if6ZO8NKRFw", "mYLly7I_99s", "mi8riHvRp2E", "mxSFGrFbt0A", "nQ1Sn6QidHI", "o95ahhg_lUg", "ocvBOm-CL3o", "ofJEtGXAfjs", "qBBOiw8DxtY", "qvAMVCOFJRA", "rD6L6tFElIU", "t2lfINURhss", "tmUWw8V61XE", "u1sXzDncwpI", "u7dbXoJS9wI", "v99WDWHcBfk", "vA7zqbnnPfk", "vlp2ODhBGDY", "wwCm1nN6y-Y", "xC2GvSDjWI8", "xqoEa0iX8J8", "zYYtKwjNnq0", "28knXXi0uq0", "37OY8-dYgqs", "4TFPO7zDyIw", "50ZWc3eId34", "Iu7lI4ylNlg", "OShOn_eUoiI", "RTS802V0gVY", "T28f7uQmmfo", "UzSaay1GlWw", "VhAXvh3mFII", "Vr0z2gg7kSg", "aOJ7chNBF5Y", "aQfjkHHad0I", "cnZnibk3oro", "gd0qZIj8KFY", "grILe6Xetm0", "iiUutn--uU0", "ldR_dvkB0NE", "pogcj_Be1E4", "rP-7v2tR5Q4", "xC2FjlwzSPE", "53CrnpcVtrY", "6yynKnW-fl0", "Bbt7LA-1_a8", "DYQlWdWeHFQ", "HRy2EYWwSMk", "MP7wKPsyRYY", "cGJSnhJEbpY", "eLxwXeRpJXE", "jpDlH2rXFgQ", "qGQ8fjUqWKY", "qgzc_D0AdjA", "sJcHsq8dNjo", "sguOll9iMeM", "w27qmvfEf-M", "z9-jgJJvArI", "--e9liMIBOk", "-6Qi0vLKXzg", "17HC5HtQfkw", "4Gpwjev-5Qc", "4H-Wgk-XPrI", "4XRlEbPmT04", "4g5vX8e0emY", "4ovBi12lQcA", "57n1RIVd4lM", "5T5dadQvI1I", "5mdWHdSAdxk", "6GjHhEG02Ic", "7CajRnuQeEw", "94MtiP_lKSs", "9vV_lNL3HoU", "BE8GlZrxG2g", "BW2vrSIZykI", "DZWGt53ch5k", "DaYYWL43kII", "E7hQdJL9NIg", "EI8Lxib3eiY", "GJwOmVAYrXk", "IMT6HE8YltQ", "JBjrF4t8-mE", "K57BlvHmOqc", "KEg1FBkWQ2E", "KWUpgllLWtE", "L7b7SXVRx0w", "LcXTVK-gMas", "NBu_1Vnv3ck", "NvDkP_HHDjU", "PxxE-lvCC7E", "Q4oz4zV1azw", "TV2U09I_uzg", "U3DhwHKagy8", "UErF6CFL3LI", "UErFZsupOWw", "UNdTT9-Vp5A", "UrhkHZzWpFM", "VZCLMvRglgs", "XXd5T98xhQ8", "XwkRCOAENUk", "YrucVFRPZlY", "_YlVnsCJPnk", "cPSthH_W0kk", "d0cUiKZx3sc", "dVd8iaKm3BQ", "ds4C29zTxOs", "dxvT5f073QE", "f8-eDdiS9kM", "fj7LbfiXIa4", "g8XKUIHJByA", "iWxFyWqo0-Q", "jl6LR65Pavs", "kucXDE6EW6g", "lzSUh6tQGSQ", "nxqqUnksxtQ", "pM8hjz-LPyQ", "q5N50nJgKj0", "qOE_V8jc2mI", "quzW5sQAlTI", "r8l1xzQTmHY", "rSlU5Qm1nC8", "r_nSdfUAqUU", "sLKe2IGyI-g", "swHxiOPZ7KU", "vQ1BLys7544", "vb4jZDsoxo4", "wpJacvtxDpg", "xExUJkrU5ck", "yy1UoL_N7wY", "zN_SinQO-nQ", "zWgZ5eYo6Vw", "-A9MaL0Rzl8", "-VvvVL0Aa9g", "0_kgS_2iX5E", "2te4q1YONbU", "3qt7Vh4Fjkg", "4JHFizKYU3U", "52csTdEfhos", "5_SAzdtUyCY", "5xv3bVKu7GY", "6QSbqxw9Akg", "8ypn5ORnk6I", "AClOSp-cX18", "Bz8vnVy8Pbk", "CT4sJ0KERJE", "ENBERmNJosk", "F7CimrjAeI8", "FdXgm3k8aPc", "FgZUWFrAH48", "Fsar4oxluFU", "FxiFGMkp55s", "GHKb3l43wSM", "HECP5DSBe68", "H_C1_TQtq3w", "HnbE_ml8nww", "J0hId4uzr_k", "K5w30Yjr9Os", "K9Fap1rX67U", "KYeq8gMxocE", "LQFkEI0hnVs", "Nh0zjHWZadc", "Nrnds7hoEQU", "OFnJlBl6Is4", "PHxqdTtbuvE", "PVe_yC76cZ4", "PY7q8zzxiFY", "QpT4WLDg_Vc", "RsZfrqUOh6I", "ScwMutbzDs0", "SfCmc7QlnJ4", "T7pZQqGpv7E", "Tqc1BOgjYE4", "UDDL0iWT3yM", "VEEQMH3j-Ww", "VmvDr8Zwjts", "WMEMLjtYJ7k", "WgpEsHcK9vw", "WwHioT-2w6A", "XgrmoWfM-w0", "Y9bMywYk4Gg", "YVyA6lQ9JIY", "YWtC_nGBzN0", "ab3WOW9s_SQ", "absgXopFkok", "c2CgCTEL-X0", "cMpTYMIuJ3E", "cO5anPdNN5I", "c_wRrDGKxME", "cxr4iEqV4KA", "d2y0xPsUYD8", "dBh91r17IL0", "ejwxk5OoUqk", "fVtkeB5gDXk", "fqnLU9LOzeA", "hjloEGbf19E", "jYwgCeJky3U", "kKb54aRkKVA", "kONplJXEkAQ", "kZPCYhaLpSs", "l8os-akKWOM", "lDjXLB4ytXo", "ldjn_s-MuVg", "n1_cW6sCid4", "nKJWfVHk5FY", "ngeh16pAL5s", "nufaCCyqrSQ", "pPz3mSDcsEU", "rg4Hern9pRE", "slyq09cyZCg", "tLGTrADI-J4", "tYOg5Lryr4k", "wUMT_dg-13k", "wV34DiEdv3I", "wrxYWlmMKc0", "xZpRLgLtZQU", "zD7zIlvfjNg", "zoNJBSjeHxA", "3S86MvDITm8", "8AQk_nqzMnQ", "BDGnkmngnmU", "IM5QHUgz2Jc", "JdjFvgGsgNo", "Kjrd3fQpUjU", "LwFcAxW3_S8", "Smfz_CGr8o4", "WW7YjyzZ_-g", "X1kOE88TggU", "dMo9iK8lD60", "mPtwWwyLGJ8", "tkt5sw9WwGg", "uZuGHSuiSGI", "up2aj6BYxAw", "xNw6-lD9FDw", "00ZXAgTCquE", "1nU-jbV3vEk", "43YL1kl9Ls0", "5V3nstSb2_o", "7vdBpvuDbF4", "9AR2hobfg2Y", "BjXFYwjZLIM", "HLqVs0k0wFI", "IJQxDnuIbFc", "IqItR21mvDg", "M7rgsyL_irA", "N9VP445FiEM", "OctgRuczdBc", "U1WoEZ_2Yo4", "cV4DFw54AM8", "edLohfQumeY", "fMpGhuES7dc", "o3PJ7PUs5Bs", "tSM40GlyFvk", "vUlxzHAkj84", "03IlBKcLrQA", "3ALARlVZ02I", "DNLw59q517w", "JttlFZzL814", "KtiA-8FRroU", "KyEKjmP4dCA", "QwCmqS-5nqw", "VAKeR10ZM68", "Wpj9at5vFoE", "YYIsiJEPguA", "cMPPAaUQvn8", "cROPdFygVW0", "g1dNr9OI_hI", "jpPYVD5Fi2o", "pNuZZfnkgFk", "qmVUtFjPB7M", "tkjAKLyXA4U", "uPU9WL-rCBY", "xH3DY-8VXdQ", "z3WKNoMJMIQ", "-Qgqqk4YUC8", "0qN2VVHKWs4", "0rV1cU5kbEs", "0tMRdN9AGsc", "1bhVibhKISc", "1zYVsjgZXQc", "22XIA4gXugQ", "32BTdWZ8EbE", "4GkSIoCx7Ms", "6q0aqSGPFmg", "7vIdfhOir9A", "9_DxDZbg83w", "9boO1j0jnt8", "B9cWKHqeIL4", "BMoiM-MgZB4", "BzihFvQM6vo", "CgFzLtTwRnc", "CunIAg6_Fac", "EKPjIt9GzX0", "EYZRCnB2TyE", "GM3kCdyZjBc", "HnLE4tEbzno", "ITyfGa6tqnQ", "JhC6auw2DFs", "KS5nLVTTVkU", "Kd1wxzUcmes", "Kg8s_HbUtsE", "LgrYif0xBn0", "MDbCt9eLheE", "MR2tvj-I6FQ", "OFTxT2ATyQU", "OTsBIok2vDc", "PB4sT7A_FIM", "QjSPtsIzcuY", "RJVc-wCAN9w", "S6PNrr65lqw", "T1YemJBjRUE", "TcK4zq2kLl8", "ThT9jZI_IjU", "Tl4B7_Efq3Y", "Tq0irXFZjXc", "UBm1crUCOCU", "UEkHdonJgX4", "UYCSD9BIpY4", "VjKS0ehEDWg", "XCaaFUWGk5s", "XIYyqR-tteQ", "XLqqa7BXP7U", "XM2c6bCVns0", "Xz2nLPuR67U", "YDWa5FrzRYg", "_VZu8wcAaDc", "aMwhYYI_EN4", "agyaA6NIRZU", "atWPGfQjMrE", "b9U3tFQnJ1Q", "bBGmJ5Bh4GY", "bIxH_giwR9I", "bQFll1yPmng", "bdBgQXnZpwo", "cpE2XmK5EpY", "cwVItfG7U54", "dQZEmXaiyCM", "djfxebk_8IY", "eR7CsunyjX0", "ebsPNqcUsv8", "eo_MonnRh1M", "f75WkC0LMoc", "gD_rPXjWapg", "gTZbnJbwVdc", "hhiZ4BkkIQ8", "hr0zYSxqoX4", "huTQdzn3_50", "i65AsqFqZFg", "jLdAOZTucso", "jd3EsdDG-r0", "kTPVAFLdKWQ", "lBDkBzkDrLE", "lrjjsfXEYac", "ly2Fg0E4OzQ", "lyVNneQWA00", "ny9_D_agudY", "nzISIsv5Hnw", "oNdpNBL2p2w", "p0NBe2a_1-E", "plsPprc1Xno", "rD05Dil9Wi8", "seJH-R8Zw7E", "sec9lBIqv1s", "t4xv-X8iE6A", "t_3F1dgrfgc", "uftexnSatnY", "v0FT_naxkrk", "vFJx2pDYb9w", "w5CIX3W-xyE", "wV8mK17Z6lw", "wbxgv4CZ3BQ", "x_0OW5Vc8Gg", "yDzBV_Pbarg", "yGs6oEb5gNs", "yxyXaOrEH9M", "yy0PxTDSQi0", "-7ojh5EpNy4", "4qyjXE_hUYg", "6IqJO8cPkcI", "9tvEl9XF_Nw", "ApQfBP1_8-c", "BNHsQF0bDRk", "E7DKcBbO8Aw", "ETKhY2OE8d4", "FWbVmM9UQ6Q", "Flg9fXNUFv4", "JX_oZ-G5z-s", "LejWA77c-nQ", "LnVp51ELDVQ", "MfVDMPArTiM", "QEAeR5KxDVs", "Wk8Oyjs8QnI", "gGjdYaRH3EQ", "les8UNEl8cg", "m0UnhdjXWao", "mZDAh_nYpiM", "qy69oEC7TDc", "rQRxgiwa2Tk", "8MOv09d2198", "HBTSXNynPZQ", "QdDRv6hyNUE", "UCfJ7icN-FU", "cM68HcaMqZs", "oxyF6W-Fa9s", "Hr-GVwGiuUY", "IwcVhDE5_NU", "OWbkkoAc6Wg", "RQFuPEp8gAI", "TyYUWC3ZIXk", "XBKayeHjk6s", "Xhgbvc_tyEg", "laS3onyPFCw", "ntouFNFyPtM", "nxyyIk2p5Lg", "v2Lc-bOvFu8", "wYvlkZIbsMk", "1tNBIB9FcYo", "Av3U2HtZm5s", "CW1jXIjAH8o", "VDKk6GTjWtA", "X_ScRVm-FCM", "ZXyY9W5E6UQ", "b_ObbYMoVrA", "fRa7U1hFyuc", "qqK-xqLFMxI", "sypW1ALvwcU", "wGnfis4gizo", "z476Esby8cs", "0q8t-InGmhA", "1vE24WLn1qU", "UYU5gThfVCc", "kufibbSFug0", "lnYsQ5mIxz8", "yH1PNLz1rts", "2JgvJaARLI0", "5P27N3mOMLQ", "7tPQwTT1hlw", "QKVYIrnHIQw", "jAjQkIjVhUQ", "nYk1jncKt_A", "0Tk5I6V5FOk", "0rDOC1BiY_Q", "1Rm6c8-IV9U", "2GtB-XEaffE", "2pje4hlSGQA", "5yiCtI0wYto", "6BOV9a5kXEw", "8v33GrNZpjE", "As_DRyO5CRM", "CGlth_o9QVk", "IVf7LnIv1Gg", "KvvH3wOHMUI", "QgcrxmU2TfE", "TOoteA3w9ok", "VUf9j28AKHA", "dtQ6Az5qVtk", "gRxwWho3JQs", "n5IK7Wwp1c4", "pI4v5mUeGHc", "paKLgIreyco", "t24DvAQIPj8", "vA4VQhNOVTg", "z19pqyYoFnM", "1qEt62srHhA", "31kwz-ZmAAY", "384FrKU9n3c", "7ZootldcwNY", "GglEPdqqPag", "JK6yE7dDiHw", "Yrz2BqQvVeo", "n9c5tcGtuAQ", "w6Ev6rDmY48", "-LzULlEiye4", "-_bs7bYAg3E", "-jabyfqlAqY", "-svMWkM9Y4g", "-vU3RtWuSUs", "0BN4-MlCaPM", "0QERCmY8FQ4", "0VE2iWMU3Fs", "0eU-bE3PY-Y", "0qrvM62dHuI", "1BLJ2fcCf10", "1VvX88ZtRno", "1a5sD6BMS9c", "2Mvg_rdraGg", "2cQwxsa4-Xw", "2sJn99qfvpw", "2uwmVRHivGs", "3xI5AahTjmw", "473vM31cK0w", "4LIot2YeW2o", "4qoF4j5Qxgk", "59lPLDnljDw", "5bmeWb0nTmA", "5e0oSAIgpms", "5rit0EFfMjU", "5zV7jvkyeBw", "6bmCxpWWSkw", "6fXH330Nw-w", "6x_Brso2Lt8", "75wmfJGi-NM", "7buyTnkgV50", "7diNV8lPpXI", "7mHMYX7oSyo", "7qTZH5Zznzs", "8J5XqEEEjFo", "8LRoIkRWq10", "8cDtSMVidhA", "8g5Mox133wA", "8nGJk7fpf48", "9I6pGI8Ii2Q", "9Px0mQTMJJQ", "9_ILAtaOHjA", "9r_C_tJa3zk", "ACiJ4ZsvqAg", "AzlAoLN7ofI", "BD4gvY6bTpQ", "BEuMADjEoXg", "BI4XAfzfxgw", "BRZpInbUSdA", "C4mqS3mBvt0", "CGZRTapogpE", "CGunpDWClyA", "CHW7LJ0Ufeg", "CK7g5UGsmJ0", "CXJsHtMc_Wk", "DIOmA6Fl_hM", "DLxFr_vpC50", "DYk9WQ_98A0", "De8ROlsZ1KI", "DgKvEFQXCDk", "DhWbpOeVlYw", "DqCQyiY_fS8", "E-kBbesys0k", "EcP6EgRmFXE", "FE9c_QHc0LY", "FQiJbO08ndU", "FTy-G8N4-oo", "FXu5Q8oXyAc", "FaO5LMNAVGY", "FaqNRUilcCU", "Fw-IMpUWv9s", "GAa6gVucMcw", "GEXHD2cf80s", "GF6s2PBDZrY", "GS6vDLWW0Ak", "GUk4qfh33Hc", "Gi7GTAjdxlY", "Gkh65fpg714", "GuF92cNyW6Y", "Hjd5rkk83Rw", "HsNUl9om0EI", "IBTeXEb0yAU", "IGkHjMLsFr8", "IhIUkIgiWvE", "Ii4sfgJG_vg", "IjJw1qmX9HM", "Ir8IrSYFyoU", "IvaH0wbc6fU", "IzoBWlz8FE4", "JyNfSm78avk", "K5eTeXE12No", "KBX80l3Uhns", "KBlEe5qOt4I", "KUl-QqFLUYM", "KVdZn2SiXaU", "KbxekEOfUx8", "KpCRCoJIsJ8", "LHonNejdwmU", "LT3kgE492oc", "LWP0KDFWUfQ", "L__hKT5aUVU", "Ll22yZzcI8U", "LpWua3wjyNc", "Lw5CSucKZwg", "MgLMFVeiZ_o", "MhCI9_lvIR8", "MkrpTIkNpQ4", "Mz_B4rkZzwU", "N1EwBgjYaPw", "N3l72TQdOJI", "NI4WixQkZzs", "NSj82TV1qq8", "O0LoUFHy3UM", "O0lrEVI5AZM", "OcxNKI2rfiw", "OfHZ3Kn5SQE", "OrhFp_SPQeM", "P-muGKsO-Qs", "PCCShwn5UfA", "PMLjBDavQFc", "Pdo1nIKm7MI", "Q-Uyr6g-bnI", "Q1zbE5QPj0I", "QAgSXt9sN5Y", "QNkQmBjaIDc", "QSQTIiFqT6s", "Qf5qqZWSdzo", "R7oM4Wjsk5A", "RLrzl4OANL0", "RS3gl32LzJs", "Rfk-wa0JYkw", "SImh1ohh0qc", "SqxMv7Myqls", "StG33gjuF_g", "T46qmV92skk", "T8n2hTSylWY", "TS7iFOlPUro", "TeHtvjj-gKc", "U6XbTWSwFTo", "UIiRqJgYxIU", "Um0q_5lyFik", "Utn7ym0qqfA", "V8gFVwIlh8A", "VV9T3Xkdc54", "V_v5GY-tL9c", "VfH2cX7HaIA", "Vg6-ohILtAA", "VgtgMpFJfxs", "WN-pqsQ5yXg", "Ww8mNUG9CWk", "WzLY7eeqSHQ", "XCU3Py3KloQ", "XZ_DLtvu2lQ", "XtJvLlxSZtY", "Y6_ORW15emY", "Y8txTM316kY", "YI6R-m09EOU", "YRSWJRsJ06M", "YigIV2pIUj0", "Z5U_pcACONY", "ZAoIw69_IAs", "ZF6CKoti3tE", "ZFYDVSjwXhs", "ZHxXSmJkPkY", "ZcNnGT51dqc", "Zo3GCNsUaj0", "ZxbMeaybS04", "_Xqmv2zQUq4", "a18rtkwaYGs", "a_SeEmRtLHQ", "b5-f82xEpdM", "bUxwRC5mIcY", "bX1qUKLiDwA", "bvgWWCDades", "cdS-xSf1E4Q", "ce4m3tYplno", "cjtJnC2bfTE", "cu2xNh6Hec0", "cuJ9QfDUa1c", "dTSvI54w74Q", "dTXN9dN3OmY", "dX79EWoVn_Q", "dd3qezyAh3M", "e-vq3YLzobs", "eB5c3A0D1sU", "ei5VUYvxrYA", "fMNVW7XG-zQ", "gACtyjkVr38", "gRGwoMBPhLU", "gbD2cu29784", "gse8c5SJjZE", "h-U8Swqk8yA", "h5s9qG6oXCM", "h8D8zFKR8Bc", "hPmAnJ9ZtIQ", "hU8ZstbHGDc", "i3BmJfEYa_0", "iElCnVJR-Wc", "iZHuPSLkvcc", "ifat-vHzvGE", "igqXy9N3EiA", "iqk4jreNvvo", "isfPFX27cqc", "jCTpWBrpZwI", "jG3Hq9wnHc0", "jHu1SZpiyAA", "jUY2dLSllhA", "jZ1wfjn3294", "jmhy34YwuN4", "joSGxOWYkHg", "k5LkInDJbjk", "k5vYJpi0vAk", "kRmob6UST5o", "kfvD0Wltrys", "km_deYuDvng", "kq_KJ3TPkF0", "kthM2wYDD2M", "lEw5A6pQjT0", "lO9YQ9enrBw", "mt9wQJhgZ20", "muOY8pe0SrI", "n3olNYgXMO8", "nNuhWjLHX7A", "nSBZeswKFSE", "nX24Um9PDCE", "nhd1ZmW6BxU", "njTOPWn124Y", "np3o7Jg_syc", "o6x5L6_R8bA", "o7vGRxSGwks", "o8cAFOeX_4A", "oK_ody2FmGs", "oONXueKWqfo", "oYTi3LnCwTI", "orUsH2o1SLE", "p54cma_IviI", "pQBTfQfaYOY", "pk9PZx25U_Q", "qEkrKbcI0hU", "qgLRGVZn8xQ", "qvo3OaB7LZI", "r0Q8ObHlkDo", "r3HuT75LYeE", "rMc6Pp4X0e8", "rZuVGuaJ_08", "rnKmjf7GG5M", "s3A9LpR_INU", "sRH4M9ahqBU", "sk5L-iHTvuY", "tPPG0Bba63E", "tpqkxbdxSWQ", "tzZeAdyK7Z0", "u0hFi65uYg8", "uHEG2dIeiQU", "uTGhrAV2Dl0", "v7X-PRKLnt8", "vetWTAcsWbM", "vxP-BzXJkFA", "wb4k5cTYfCc", "woBgF3y9tFc", "xEfqTs6BxBg", "xMc7GjmL4JU", "xXHM3RdbQzo", "xhQdXjWMKac", "y5eDuc3ekq0", "y9daXsCl0qk", "yBG4NGchJBE", "yRh6ZWLzyzs", "ye9asFFpV2E", "z6CgJLPkC24", "zJw19sCHesg", "zeOdGwzPPqU", "zewHDDKGPvI", "zmav41YAIqE", "7UGVgGP-Mbc", "7_bo2zs50bg", "9ecwAAoIMho", "9gHPkTuUb8s", "Bka3ITTOvC8", "CAkv9p3JwmA", "DX0GU3Egb0o", "GNM9AWwZzuU", "Grzg57Ofkg0", "J9Hang5lMcY", "LWFx-yVcbu4", "a5EQfpChSy8", "dxXN1bOU1DE", "hOoF95PkDHw", "p2yzplax2No", "pSTon6cyz3Y", "taHB4K_FaUc", "4YAiTSMbGqY", "5za19mYYXpg", "6vdwqvWfa5w", "6yGDsWxnsCI", "9GtPdYuJ1Cw", "FAkKYFlOR04", "Ss_mu2xU7RU", "SxSOpR1B1Ng", "T7o9E3RoKgw", "UTbTZrM6jSg", "VZmnVCRLu78", "fNFJhmGlLm8", "fl7QbEs3Psw", "kXLzqJx7lLg", "lbT39uR44qw", "mTO4eDp_-NE", "n9DoG9xLutg", "oGssnfejxXA", "pUsKz2kQrno", "qf-CB7c0ROI", "tMTANajugQo", "ybkRXq6oOng", "90-DDFPLTfQ", "DsJgVVM0RdE", "QsNHo1pz9Wk", "dtngfyK0M8E", "wG793CFubFQ", "3cV2m5ZxAEc", "DgXu6RZrYo0", "Z1jVql87Djo", "v3xh5xDfzsE", "amfJl3g3NBU", "lHuARm9ozbk", "w0ntVcBIsiU", "-jxH7B0_o44", "-vdV0MJGQ3s", "1QSQCt65dtE", "2HYOdPSaIhs", "2ouV7lV9CAs", "4u9KN7izoHU", "4xjxvLn3B8o", "52v3S2klDGE", "5pRKZrHSG5s", "7Uz3NqnpKO0", "8BVpvbAzNWE", "9fnNBl6yL9Y", "B-GSJoxmmzk", "B42AK2aSPJ4", "BRC_ZAkf1bk", "BcPNsGH8ZrI", "D5xOm07akVo", "DAebaGrClIE", "DodK1fkDz6c", "EOqGYAK_UsE", "Fl1llDSfcUk", "FyjTrwkUzBw", "G4BJUx5uwHU", "Gmz7qWwHh2c", "GybZIguqNKs", "HZJFo-8NhS8", "IIQmx8zLFZg", "JGVjBYwjh80", "Jov9n3MOTJE", "K4-pCGk9cRk", "KlIi1pY3QWM", "LtcxE-YRGvQ", "MwlDgE_SbBA", "Nbcu6S6v7MU", "OHBCxzAjA0E", "PKTCYysvvh8", "PXsnknF-CRg", "Qh2Y1E1k1dU", "RbCNfYlEE3o", "S_IsG5YdOno", "TF0yy-_HtRU", "VSDN9C1wVig", "VejzBC9ue2E", "X30NAZiKfDU", "ZADauCfZpLo", "_keS8K2gdNg", "_t9djK8rtOs", "c0-lZJPBAjM", "c9txbpLYhtc", "caR53vLdc-k", "dKEC2EcMkLQ", "dwquC7pKxFA", "e5IIrt2iRj0", "e9dWLcQxyww", "eiaDC4LtX8k", "i6WWmSpRJww", "iZRpfTwpK4c", "jAFwnA446YE", "l9wqcSRhAtI", "lakXFMEo-PU", "m7PBLjappU4", "nyu9_NBQ5Ug", "pR8eq_CnnRI", "rY-2ZgFV3b4", "rgKWjEWnHaE", "rvilMphpsxk", "sZsAlfJIWz0", "u7nISlZHF5k", "v78c0hdckNM", "wjdOE67vKOg", "xHM6L_RiL84", "xSYM6a6LhWc", "xu9YVZ4zFcU", "y0Cvtq18TCU", "yddv2XZEGqY", "z1L7O49tXIk", "zWjCpUdI1Bc", "zfo4FlQReG0", "0sVPLF1kQXY", "Is5j0iq44z4", "gTXMyxaMS3g", "-RboURikix8", "2tdNzHRTEXU", "3URo-qJZ9WE", "3b7U5blpLwU", "3twY96Re_0A", "4LKtTuCYp88", "6QTxzrplKX0", "6uv1ECKGMDM", "6yL9pEq8F9E", "7kytmqLa-lw", "8aTBmKCGS5w", "9BHNrZQJp8I", "AASWfMHtkYg", "AD6OByYeeIg", "A_BB_RR1N2Q", "CQXNgTzvbfQ", "DE27qi1PUW4", "DM613f6WkBI", "EHEn3-Ayl-c", "EPEUWxLuqqs", "Ez06RksvfkA", "FGG9pnn3EvY", "Ft6uDXvhzAw", "Gk9qxAFk5rs", "GtOIMVQRTWU", "Hdx6mFjC6xk", "Hj7DqnTvhzE", "IN5o9GD6I7U", "JC3eJDHZF2M", "KqIEWLu1igI", "LqFQJM-xs_g", "Lwrwq2GrBj8", "ONs0ayBSkhk", "PM9sofc1z68", "Pa4mUgFDCNg", "RAN3vJXEjO4", "RmqMClUnvmo", "STp8brh-Sy4", "TccrMbzAZRU", "UX6Tc1ZwOdY", "UdYx47lAky8", "XFQwOEPYRM8", "XiWatRgoVB8", "XtyGlP6141A", "YkUIBv8EzuY", "Ze9lkcRncKE", "_FQbyGmxzpg", "_Z9I9xkt7zQ", "boeUNOYoQzQ", "cGhZx1Zu5F0", "doGifST2XUg", "fNWzWQCHJT4", "fnY2RpJ1w_o", "gIxy-u-XDGs", "gVSD0g1GOLw", "ja4Bn5PNi2A", "jyG7E8_cyP4", "m9PDm4UBH2E", "mfF4-KrJzbA", "nWw9n0E_3tA", "oAR3u0LtKec", "oa-c1HbZmAo", "q-JzXldlCNg", "raZBJ2D9vU0", "sdsHtQVYK68", "t2Lhcaiq3G4", "v-gplZYb8b8", "vPyVcAq8HCU", "vtfenjSF6To", "yMeZhQ_xjug", "zH5IvVBlf74", "-8fRoEnaqJM", "9JriJ9Uh9gQ", "W8haGlfpzvA", "WaoSNMuc0R8", "foPi62aJCgU", "muPcS_g5wkU", "6JXNLM3zu9s", "I_NI20UKeok", "dZ4Py1ues4o", "phMSNE8fq0A", "xTF8QR1pOJk", "3DJGFbFjsYk", "3VhQjABzHjI", "4widxVq0o-w", "5Bcg1csNXlQ", "6Feq6oxezSM", "9RlA4d1V3vk", "BZAojdVxY1w", "BaQBKwmy8kY", "CyAaubPiQ3o", "HdQBPDdQwfI", "Jr_4z2dnZGs", "R1fyAq2TW8U", "RzY1gJa1gdY", "S4CbbtNysZc", "TSSCoa-eUf4", "UXTaW_0HKcE", "bULdUQK8Yxo", "bUWrr7aSyEc", "dGo5IEBTK8s", "jxJ5aAjuGUQ", "ktHGds7Nbx0", "mLqf31uzQRI", "o8dDjmSyy4M", "pUe6ZlbpUHA", "tjVxKT6L2s8", "vPA4n78XbQg", "xaoyV9N9De0", "yMky4MkBL-k", "yaDCXMRJ7J0", "-7xYoFhilMc", "-JcqEv3OF-k", "-kkiP7WUB74", "-rR0lKrD1MU", "1JL2BhqRk6c", "5OmZS5naDhY", "68ICGoahgLU", "9QM7slu1D_4", "AN4zq7FgREQ", "JataR3u-4ew", "LUv0yTHqZL4", "NC6wVtc2LYU", "NxvEdHAA0XE", "OrBzB5LVkz8", "P0QrKTg58TU", "PN56UT1RzJE", "PXOdqrE6z-0", "Q5USMZoEV3I", "SyiD6yAO6Pg", "T3ZuDuXQVfA", "X8otk8aGq30", "Z3IRxxfX388", "ZFWbRAmKMKo", "_Mtgcw4UngE", "bmIvRZUyg_Y", "cVguCdRczEI", "c_pu5dRyhWY", "d6RvkeGcCVM", "dMNocvCNBfc", "dcGgGMaJLoE", "eMUCsEnNwds", "gEkZxcvjoqA", "glptuOLMVPU", "h-6KIBtivao", "iVOTbjP8mng", "irB8VUoBCo8", "knqY1JJVGeM", "kyoNQW2_V2E", "lj6vh5k7bkc", "lpJz7j7zK2g", "lua7ea-KMAQ", "ndzVikcXOB0", "nrQ7bxWXdGo", "oGDrw0s3ioc", "p96IEIw5A8k", "qdRo6JyRQ-Y", "uA5Tnk_kGr8", "vUtz8Qa4-Qc", "vjmJ394AT9c", "w3YDeVcI6Co", "wPd1YKE07Us", "wg6TtINzewo", "xwD3eiVTwOw", "2BuyJF_iehg", "8cCqPkkhiME", "AYCuJiHsCHw", "VNH9vi5i1tk", "_MFlVDMzI_M", "ct74oN3Hv0k", "rPDpTceD0ig", "-Te_EMMUyEA", "-ej3M_1Xf7w", "093tEIJubcw", "0t70_afgG4g", "1k7kE-q8OZ4", "2OdXVtEo7RI", "3fg3YFtpYUg", "3zbxdjgSkfg", "44tH5opqfjQ", "4FJDzZzjPHU", "5ZbBbHdGTQE", "6uBPk1u2NkI", "7eC2Ip_Vgr8", "8HmUjDAJPgk", "8_ho7SiTQ94", "AilHHa1X7TM", "AyQmVkRqWg0", "BDkX73AoxGo", "CD-AM27UwPU", "FHUvZQrgBFw", "FNX_E3b7I-E", "Ft8kaF7sHw0", "GsD_ciAtrr0", "K6rtxWsjWts", "M8Rc2OLO2HU", "MCnXJVq4Po8", "MHBkK3KteNY", "Nj19C2Pj5rc", "PEM17cZJSjQ", "R1mWr2FesQo", "TO8e1VrgdWc", "UvwMM4dnpxU", "V44VgW0ve-U", "VU5NGpI0HtA", "WJ4UBUglW_0", "Wb9g2t9gcLw", "Y2RnnYunlcU", "aejLq5gpAUM", "c6Xg2rI51QI", "cfmWrcD7-M8", "d0BqxlIcZXE", "e4_GHp4lWgg", "gdBbTjJ1yeU", "i8AmH8do3ZM", "jKbTeFyEs34", "jq3pUTfdFKk", "kOO5sgUKzwY", "lwqGBfNLu8k", "mhYx4o7jzQU", "nOz8Ygvqn48", "ngE4Xrs9IgA", "ozi6198qvoc", "pNg06clCUz0", "qCg5CLqSpY4", "qOMukxWse0M", "rOhIP12Vqc0", "rZT_yjSdpdI", "rs3-TeCPzUg", "rv6N9mbrXOE", "tSGhnjwVvKM", "tzaUcmvuhpA", "v0eJndzN044", "vBnZ7pMMC70", "vG-9Y6aWzO4", "xA19maRYWUo", "xMKpJvBHvyo", "xYSX-DAoGXs", "y3jz6No_L50", "-Px1wKli9m4", "0FUwh_N5giU", "1NH4-z4SX6s", "1l5aZpNSJ_o", "1vo5esOqC5M", "2RplS1y2ops", "3qlyYFuNJzc", "45-DRzffgiI", "4xR7YVlOelo", "5lnmr3kbYAc", "5x-7Esl1GSM", "6LBiglJ0BRk", "7HcjJJhVh8U", "7KvpyZrQvsQ", "81UDGWIX9JY", "8OourQEU8ws", "8QSa9rCIr8s", "8ln7Tp7PY-g", "8yUsLaON6Wg", "9wS1cYDP8w4", "AeccV3vXpGE", "Ag3Dkwk9UQI", "Atgmw7idfRA", "BDo4ttySo2I", "BbpLbN897us", "BmXc5ORSAxY", "BtesZHSydRs", "CVjTDycVG3w", "DDpAnW6aH9Y", "DIG74e2tWO4", "DxIEaf-6l60", "F6dU7uT3t2g", "FZrM57ChgS0", "GYfZlyq6E5g", "GfQQ68qo5ns", "HGrvF6rXptQ", "HRfg80zqf5U", "Hay_Z1aIwUw", "HcT_0omZF2k", "IDLUAzHoK9w", "IRn_wcmKHGs", "IrP9QCR2FqY", "JSSrkfJOuPo", "JbhZckIobZA", "Jik97JyewO0", "JuBSuzPa0g8", "K4ixxqaK0rk", "Lst78X_GWfQ", "Lv3qwWUxYME", "MagCy6YdKnE", "MdLyOs9nG5c", "MzoK418q66k", "N3bES7S3r7U", "NC4NMDG2TnY", "O-_HFH57fA8", "PT9cX5OSycE", "PkBzOHaJDpE", "QTgNn67p1SM", "RB9sYa8kjc4", "RILbNnEXk5k", "Sx0DTHi0eKQ", "T59q3izmlXQ", "TFGPo35GLvk", "Vj2_ozVQu6c", "W6df1YXykhA", "WGWeasPQ3LA", "WiJTMdPs46I", "XGytWPzvE8A", "XlUY5FfDZr0", "YUOnlpBomSw", "YkevlDd-ZVA", "YsXNIx6wpCI", "ZV7t8U5dE64", "Zo1cWst2s6E", "affVncyJSPo", "ahnPag523_g", "apdNWPoeIho", "b3cRWHtIVnQ", "bKsOgOHmpl4", "bT4Tfnm0FBk", "cD400dMhAAw", "cGV54G--FsU", "dtfgv-u4nXs", "em1CXbIf2eA", "fYL2iblK5yk", "garbLgzE0GA", "h6IsixdY7-U", "jBfcmOW_VQk", "k042rkiczzE", "k2KN_Jw4IpA", "mJDhSjLTTD0", "mLjyoLkdqNA", "mgezRWgi-HI", "mosefK7NEzM", "njEasSO4_Ho", "oQ64Q_AhX64", "pH0cev_U7nU", "pX5nfaIXLNU", "pXZtkmV_qTk", "pwN8Vy2qknQ", "qDWxJVGJiAk", "qXxkCHdl1zc", "qpzxnu8bJj8", "rHUAOJJkXEo", "rKP1ygHyXO4", "rgoBuJOldM0", "rhK5_37FmZw", "rxDM3j3FYHo", "sZNn1LtlRmM", "sscWlYj6JKk", "tFhb7CmNPQ0", "tXrRVgG6dQM", "tgGk8hyKGhw", "tmvzmaoNSMI", "tyQJ_YGhHmQ", "uFOrsFXH2nk", "uYx4FEZclFc", "vQl7iT-k4lo", "wTW1V4va4-k", "wYWzIQa6O2M", "wivz6jWDrzw", "wrmuaO_2rjg", "wy_r4NxoIcg", "x3WxyBDmIBk", "xILWdd1AEe8", "xNEWARkJiWc", "yB0hvoZiQHY", "yPWNRGHSkpg", "ykBRKPjdy-4", "z4PPCAmNg4Y", "1ePkTwvUMFw", "3sR5RX5EPv4", "8F5sUWiyVKc", "xoM20NSEqHY", "yhDAzEg1wU4", "2ktBL6Fn4Kc", "36cDDhn37fQ", "3MIMS949AIM", "5a4qpmX8H4E", "7bCLwV6SIIA", "AAmZybmQx88", "APKftokYM7c", "BKrxgI2UmPo", "BuSvU6SnPoo", "CCC7UVT-Lv4", "Cz3MmzoyMQs", "D37CO8pZ7xY", "DM2PJUmcfwQ", "FjAqXMaLJLg", "H1Jfqn9byHQ", "JOZZh9hvUbg", "JYZvd-_L0Qs", "J_JzYIcqqD8", "KT_3i_AJ-Lk", "LJg3G1tjdIU", "Loqm_Sw6D_k", "N7hhcbr5ZT4", "NhE1saZpVdE", "PjZhLj2Rfpc", "Px8-2b2DZHo", "RAxVYOlta3E", "Rr4bWhxEC48", "SVs_gPE0048", "U7o5N1p_H5E", "UoAUqr38Oqo", "VrWoGOY9Hxw", "WJJMH3O6euY", "Y_PRjhhIwd0", "_EQOTpXM8fQ", "_PtDeDKdna4", "aga350w4gD4", "aholMFPOPC8", "cQmGKXu-glM", "ciQ5TiGcELQ", "d3evTsETxts", "d7l0Xd_ffPA", "fPBiJEpUmnE", "fjY9wUp-mdU", "h0Sadn-mtHc", "hJbP3qOLdwc", "l-3XiEMrsAI", "pOg8S8AWP4w", "raZqHqOy-p0", "sDpmwGy-93k", "t51hWmPEpK8", "tOwHMcbIy28", "wEl-qxFui3w", "Afo11qC9p6A", "Whikj6GMESs", "mDiY9i-ajb4", "qvlI_XleQSg", "uucIbX036jI", "--dtGOZ_wO0", "-JFNZNa4tDI", "S8-w9JXQjco", "hPGHUpdiYu8", "hZWNb5AZDWk", "hdi8S0zC5r4", "jOyZk_7Vrks", "pvAtxNa8ZvI", "t-u29N1utz0", "wI7jaqMDB6k", "4o7RcMI-suk", "ByCJ0UiGfpA", "PFptVc2umT8", "TOe0hUx-GXM", "m6dvQdmewC0", "mBtjwexZkg8", "nhfai8mxk10", "5g2dmVhGoP0", "BUBeEG_iUgk", "Cs2k20nNKpo", "LUG9FBHwjso", "MaOp1r7kJ9Q", "PDbP4ZToMq4", "VoIGrjjTnAY", "XExdOCYgoT8", "XV_7hfWt5qw", "ferFeZrzcXo", "mejWFbP3Y9Q", "rK6hu9mcUJw", "yH2c5G-ec20", "-SU4i_OizCE", "03qj9ya6d7o", "1VaXQG3ysJg", "1qlfg7d3kaU", "2baC_yXC60k", "2q3uX5Zu2Qc", "2x0vTVSfBaM", "3Uvm2w7xU8U", "3l4ZywGkewY", "4Bb4pSnfpaw", "4OV1Mzshos4", "4k9qzZrA-pk", "5-4uBasXjzI", "5FnEBRCkb8o", "5cUzFkoWYNU", "5qOPlsugfcA", "5tNyKTFiUuU", "5tRj7vontD0", "5vJPw_uOkI0", "5wIXvlhLBNc", "5wI_OvZrL-4", "62bLa44DKbo", "7q__zfPkC04", "7r9Zmj4eNM4", "9JLG8W_hJNs", "9f7F9uQvJ8s", "9h3--sgtzyM", "9mOKg3ph20E", "BIb-j5R5AyE", "BWmk6qAZqmY", "BksE_8OLeA8", "D5DNImnWwDU", "DWn2kuxx2f8", "ESgw7SjYQko", "EcVdgxbDI1M", "FKqoT16aUF4", "GG-ej3KEAH4", "HnPGAcfl4Dg", "IXCHUpdQfXo", "IXx7XMm7w1o", "IxVWEmFWL7Q", "J2tQOBj9OAo", "JEnm-R1DC58", "K4PGrea9bzE", "K8gRlESFzpY", "KDnRsbjVSzA", "LbrUtfj1Iwc", "M7bispcPQ4U", "M88p_gaFiCg", "N38TJkcghLg", "N9VNKDTQxtk", "N_ih1Hkh0Hk", "OAFw3tGE9Vw", "OMohpJAqBTE", "Od1cTyylHUY", "P3hGknDnhwQ", "P7JDLpcXiW0", "PlayfTI_EjE", "QkPQbAXnKOk", "QnzHIGIqc20", "RSy2DzxnjlE", "RflTQQHgmnA", "SK0Vbn8nXkM", "SaEQBYeDj5c", "Tkc-Ye8XQI0", "Tv74by7xJS8", "UYkTFd0Cz9U", "Uuzvc1tXqDY", "V-AZVX22Nh0", "VIIeaLhKlME", "VTEN9ySoVwA", "X5ELFg8LGzQ", "XAYvl3i7hWc", "XW916fLimps", "Ye_atDDasiY", "ZCBEzNSSkKw", "ZuTIR5SQIkM", "_8-SPbNn9Zs", "_juh6wD6gPs", "_kLaZTnvqRc", "_p5-Zm1zEMk", "amcUWmpeHOU", "bFb2-xnaB9Y", "bQZNsopvqLA", "c1j6ZKAlfak", "c6Oi6YhKbUo", "cSKpC_E_vWk", "dInbjPu0yFQ", "dOTpVSw4eZw", "dS0mxzjq59Y", "duDoQewGghA", "dupJcwjUpQM", "e4tgxvi5k94", "eSdNUJeQRug", "el-1KUrz9_A", "f1VXB8tjeqw", "fghFe22gROQ", "gWEqCQLjVQM", "ghojo2fWsEE", "hljJEdpTmlI", "hnqYliKrXPE", "hotmdHmj6jo", "jOc9PW-uk20", "jTnYAwbvWMY", "kEvp7ej4pWw", "kkc-nmyLIEQ", "l0CIK02RcNU", "l2EneEJnCl8", "l3D41kxonEc", "laWXzx2X8aM", "m6vGf2ObFTI", "mGXHZJtskms", "nIjjbwBS9ug", "oJdiJS6e4pk", "oS8AhSadK_0", "owUBxXQYCrg", "q2HQO8TsVMg", "qDJ2UiF4mic", "rIUwmPuPkrA", "rjK0_uVmm94", "s4GIFkXX8tc", "s62EQmVEIy0", "sAbMcGXLSW0", "tV2DOLaHcWc", "t_2uik4_VPM", "tahPG_5Y0lI", "tlk1epk4mgc", "u5GCCQJtqrk", "v1KhUNhHPwg", "v3cfyE2T1XQ", "vV-9ZFnnu6E", "vffIMSNvwfQ", "vlDB57a0IwY", "wLRpnth18ns", "xmf6Q2QzUys", "xnVb24XG87k", "ypVoHonio0w", "zWzZQQMFF_M", "zZCw8HxyQK4", "zxsl3cBeeVY", "4Tsr_mBG_Ao", "DXRaULq3zsc", "JVNRtOmgUzo", "JbJ7o8TwnwY", "TvTQPfhdvKI", "_8-UXlJxiVI", "m1Km4B8R8k0", "kGmoDDDaKAY", "8f5K0SpgJkc", "9PpjF3o-tGU", "HWJxF3w4a6I", "JeVB4iPqXEA", "OUeMh9-nq_s", "Q-5SWoyTBtc", "Vbr904-kLmI", "XLAs8iuEAvo", "ZXFSf9z4Y2c", "a9ywyrX7nlU", "d8VdiQgLax0", "nvIahS0w2l0", "oNh-mLPn2PA", "qb1Ur0Tdq_4", "t9kmjPVAq2I", "y4bhIsVcM4U", "-vjokSMPC3k", "1uZl6U-rN24", "2C4SKGv8wk0", "2EoqzNR68pY", "2QEQwYYbtfA", "3oOXBjAEnAQ", "3stpZKNF_jQ", "5dpH6XsDXKk", "78-nA8U6Rj8", "83Zo-68ALv4", "8MICr7b972I", "955XMKjSvIA", "9wxW60VA98A", "AX4use0u6cc", "BNVbWDIT4eo", "BdmkSYezfmo", "C2xfS1u6hzg", "D-zE-s2rrSI", "F3mTZ3edzjc", "FTBdimPd-Hg", "Fu9iNKtjIM4", "GTaTAdfVIFk", "HoR-wGfJU08", "Ieu0elFHuv4", "KTqvo9PjSUs", "LSHlJranurg", "LxwQK7zMGd4", "NhKJqs5QAqE", "O2noDNfFcRs", "PScKXFZhnCY", "QPw3OtXdLD4", "QlLT1JuaJzs", "STRK0Mf30U0", "TRdiAHeG7RM", "TWEq_sjFtfg", "TwoPskf0uek", "VlN6EGuN84Q", "YA8dDz3chrs", "Zt6TQIvUYlk", "a-u6Q4Vzhjg", "bRWCfJcSpEU", "bsOGyORiRRE", "d9iTlGeK3G0", "dPnppATZoMc", "gwoHnmcCuH4", "i4vK-x69AJ8", "izsPnFS1H_Q", "lKZ8eZlCzbY", "mEQKPAvXZRI", "mEo6ivE0AnI", "nEEsfU-PC74", "oKOwjy7Qo6g", "oUz7nNRPSRw", "onHXpBhXk8s", "pnXwzRbYyvo", "rTm_QUSgDOA", "t_xJrrANFTI", "vQJEPT6Awvc", "wKbg6iDSXJQ", "wKpVDLfxcdU", "wQ9G0JOstds", "-0rWhEQDwDQ", "-7oyc5zbl4M", "-IF-diJdkH4", "-Rht_M5oXSc", "-SIBZqoswj0", "-_2tcDr3PuU", "-_KjUjybX6A", "-gtyBAO5yMk", "-uQ8AVXDf1c", "08T3iLC8oSM", "0QWo9kLO5gA", "0ed9aBj7tWQ", "0mPthOT9LRE", "0mnAcgsRkys", "0mzjIo8_qfA", "0qvNA9OpPsc", "11C8XFDZIJM", "1BPFrMAHWPQ", "1g8BJVKN3JE", "1yDkQB2YJIo", "2COUdfMc8mk", "2_yq7TrmCcQ", "2b9qeKl43dc", "2mas7AQXadE", "2n0-0V2sG7Y", "2n1lHMgPfGk", "2rTVyJum8lM", "2viFp6czg7U", "3FqrRobqBqo", "3S3SHNMuJvo", "3UNdLxZVhlQ", "3fZsZfwzWXs", "3hDdiw8lS48", "3tgmpiYeZEo", "4B3t_c9ydTk", "4KZHDS2XrVQ", "4R_rtKxhAOY", "4f3CBlbLG0g", "4hUsQjum9DM", "4jcGnylkC54", "5IPC7DcNQMo", "5O-7nKbhyHc", "5OOvBjweeX0", "5Stp1vgi_Ww", "5lTuv9WBYtk", "5r91TEwwPbw", "68M0KezRq8I", "692Syku8knI", "6Ljsd4a6Wqk", "6Sors6yC8W4", "6VQC9GB57FI", "76tiKvIDVoQ", "7Cpn4K1NHxY", "7DUUUpHi6MA", "7HjTskjdwng", "7KrLrcfemrI", "7N4JHjukN3U", "7RIp3BtHbVw", "7WTfVxhNZ-I", "7XHpVogQex8", "7Y6r94v-0eg", "7fBlTPuxMFA", "7gqgNhfDoUg", "7jCEty32wSw", "7sfU8tMAucM", "84LtuI3c5LA", "8P0MC3u6Bx0", "8XpuPJ1j53c", "8_C8lC1C6K4", "8lTCMlMKsCA", "8sWgTtdACaQ", "9-MO6MtOKoE", "9CQTbbpgwQ0", "9EDi_nbOfsM", "9tBMm-DFK6g", "A8YN-xM6MnE", "A9l_pJ8jO1M", "AEncgXMylXc", "AFCSXzPq4mc", "AGqYcefwmbI", "AgDJmt545VI", "Apt7Y2YZdP4", "AuwyFPw-qjI", "B2-sJ_xdWwo", "BCQ43vDUtY8", "BWU7S9Ca6Tk", "BZeO4Is_Q2w", "B_P76-TLb24", "BdLZ9FviUX8", "BkfI49zQk7U", "Bs3Lw7bhDNc", "BxCf55uCBv4", "BxMnZFr2SRY", "CiPU16Jz--g", "Cwsh7Hr6PuI", "CzcYTp8E2VM", "D9diohOC9Fk", "DTXCnsrnEuo", "DVfyuPOqjR8", "Dh6C-aIPNH8", "Dkf512kNQBg", "DwmEeWBYrew", "EIVH_m2UznE", "EV43LA1_AP0", "EdI4HQEepHA", "EtvT_DESdFE", "EuFBHnzmkFQ", "F1JopzRwfqU", "FM05DQErgW8", "FQl939fze4E", "FUjL0AXHwGY", "FWoSbPxlbYE", "FtqHxJQDkdw", "FvkBilV26C4", "G0jIRTW2d-I", "GCPQe_f2Ook", "GDtt_z4qAII", "GEGcIgvFnA8", "GUkXb40ZN7Q", "GqdoHSeS0sc", "GrDUWG4fHSM", "GzubFCdhM0w", "HKmn-CGEHqI", "HeslTwqo4eI", "HuS5IfOxH8A", "IESij8pz-GA", "IIkZvu5mWAY", "IKd9pzb71RE", "ISBsiwZ8aAo", "IjfCxmsMAvE", "IoUClU9dOHQ", "Iq2Dhy9ooDI", "IwzFfUHT2h8", "JWl1BZdkBGo", "Jw-VwYYIJ1Y", "KSr5sYFXzm0", "KdrSDElI4nQ", "Kx8xCf3se38", "LONmzT07jic", "LVk5OEsyeS8", "LcJkmrPgA_Q", "LlrvTzmCLik", "LvIetV1LBJI", "MHr0Cwsi3Pk", "MnYhsKr54P4", "Mv5U2qbmo_8", "MvLDFNyboCU", "MzDpgrCyv_o", "NW-CjWoqwbY", "NbHITOQUhMA", "NfX_BkAL-qQ", "NgxYOFVjCNM", "NhnkECoo-So", "O8hIDK7H1UE", "OLxmfPRXebs", "OmbWAsQaxzA", "Oq3we4Wd6rA", "OuU6yny4jtI", "PChYlLo3XSo", "Q1nuyjsXYKA", "QX0CeYwa6Fg", "QZ5RUqlOP-E", "QhMrbq46E1U", "QjlXUNMBwos", "QqipP1enB8s", "Qr8AzWGlzBM", "RrCVLm4doxw", "RvLXKdTN8-E", "S8kthBLntns", "S8lU_d_Nf3o", "SCT6OwzK3Mw", "SIhpzPtgOVA", "SMEh0gCdm7s", "SOweJ19cPOU", "SSB5v-tcKDA", "T3uFkaydkT8", "T4vzyVWsFE4", "TL7v37LYY9A", "TZZZXxSeVy8", "TsBflmlyfkc", "U_FU0_OWpyo", "UkS_UXeX4Lk", "UvAmdhv8LlQ", "V9FVeEJVlA4", "VDLU6PEBIt8", "VOaYuRAl31M", "VeNDU-jDxRM", "Vn808-WOZ2E", "VrHRAM5Op5k", "VxU5HKfGHks", "WJdN6rzjVcE", "WPhE4Iu2uqc", "WgaZxhjcvXg", "XD2HBR1MMzI", "XT1KL6jdiNo", "XgjgpUu-0R0", "XnfYT-_x6qU", "XxrcBYM-BoY", "Y5H7dORtDyo", "Y7r3wyaauJ8", "YMWN-xF7UUU", "Yb5Fkja3euk", "YdypX84L8v4", "YtMcjihUIzA", "YxRs1DJTLSI", "YzwpNOxPUJQ", "ZJMEBVEf7gM", "ZO8WKT-9gzw", "ZY__ehgsq1A", "ZawdQa3ZQGQ", "ZbHP0cWc5v4", "ZkwweZQN0AA", "Zywlhm8ofwU", "_1BuJ-7VhLE", "_3dRMkB9Rn8", "_43f_obhPI8", "_55dnhSGtQs", "_EwPU2o30Ss", "_FinePs2d3M", "_KMf3x6uAHE", "_YS2hBc81Lw", "_pVitemxtu0", "_rVMxXzsi0g", "_y3i3Ys7RMw", "aPNoXAg4Xbg", "aTfgCKQFaAc", "aY-XfGiPLLs", "ab2IpXEawEI", "auxpOuA37_g", "b0vlM7KZP54", "bLF6xJdPzWw", "bLVqmTuGza8", "bLodSh_erpU", "bY6KMOlRKFY", "bYwkG-tYEx0", "bdLiuwiZIWM", "bs5xGDetxgk", "c3DwuEDZuZ8", "cTF5sgokCfo", "cesqGYA11NE", "cfCcVdvX7Wg", "cmqmJKKxh5Y", "cxG3xrdo2bE", "d8rCxm3ertY", "dDIRxML1YFs", "dG1B25uquB0", "dPj1IbXvSBM", "dx6mCfgAi3w", "dzs2fqJ4spg", "e3SnnFJzRJU", "e5d9zLx1uGY", "e7sV_HAdbQI", "e96vbstrGac", "eErh2xE3Fg8", "eHJ7E1F_lu8", "eNmTifNC7Kc", "eenupCTbqJQ", "f5Lxf_LxQAo", "f7tzWJR_9Ok", "f8titEnWbQY", "fAUtAE4Zf8w", "fMQ08G14FN8", "fZKGLZu1LWA", "fdnPYxcMUCs", "fqkuEOSk2Sk", "fs9ajfCmbXg", "g1q7cg52loU", "gRpsUzh02Zo", "gSBKac9GUXo", "goiknZMWvgk", "gppich5oTSg", "gwly-JDK4Dc", "gxXPQF7ApNM", "gxw9mV5kmVI", "hIfnEYx58Q8", "hWG4Z97mN9I", "h_6Ln5af18k", "hbcxTlHr5WQ", "hv4FS-ZiCAk", "iD_lTRnEL4I", "iUw1LhaVGBk", "iXIT0eILbbo", "iZ4bR_mBcRQ", "ij_3g6Kwnzg", "ivakx2AXqZY", "jDibW07KZZs", "jFkGNTlWLoQ", "jLuGC6yhpkM", "jP6y-u-Yhvk", "jVRKR718IgE", "jnhwjEe8vuo", "js0cf1b1Xno", "jzUtz9_VEMc", "kHDmF_DVgqk", "kO3GAT-0Gxk", "k_OJtd9LhcI", "kcYYkmkagu0", "kixQDTU8dvA", "kmdZ3Ld70-E", "kv4eDm94qOc", "lOl6JI48l-s", "lSNUdmFmfs8", "lnP7Pk-4FLE", "luMXpWH7wUw", "lxvG9dVdb00", "mGX0YwEdHz8", "mK4mC5YhWAk", "m_lGbpGEw5E", "mbssA4JdQt4", "mmcZuYBe320", "mpSMAaJl4CA", "muhhS7f4M1Q", "n9cDkdSNboE", "nGXsaHECF9s", "nQglTEG-6a8", "nRHl4W_DqcU", "nVQYyGilhuk", "njGYT3009sY", "njuvCd7pbBE", "nlVQ7Hfu8wg", "o2zrShoRmbg", "oBXBKNMH2dQ", "oBf5Kq2Xjpk", "oObAlI4Wud0", "ow47rfK4LTo", "ozLvjEDFisg", "p96dtRiniQU", "pDVMV10i2PM", "pT9GMBDMEUU", "pcBUYfvIAeM", "pfmf3Ldh8II", "pyfCfwKUilM", "q3H51N-Bbo8", "qHV1MEna0xE", "qQPNQmTJiQU", "qfpC6Q6_fmQ", "qgTRFW3qZLw", "qhmoENawmjk", "qka_o2IHFik", "r5eFVdwCaMc", "rBHod_4lRpg", "rCnCMvdiCZk", "rQZLG-XoKoU", "rzCDcFgdJWo", "rzUbe3tQUvw", "s-EUrT_MrIw", "s0S3ZUlfR98", "sPiRKs7bFAQ", "sTFn-rSnWeM", "seNhSNwb1wc", "t1I9wrNXBlU", "tB3GqbSS67c", "tBPWViHk8wI", "tKi8JvhZM0Y", "tbQAjnFVMME", "tp9Uu8zCpxw", "u-x1tklGpx4", "uDrIVAHZ6rs", "uDv5W7MWB6A", "uNCNZW5cfQo", "uajoAQq0qXk", "ueYELXJ0rWs", "upuAMLhmexE", "v-K0UOsRAOw", "v1-pMtvO8uU", "v2D2tj8SPDM", "vCxuLkl1MKc", "vLL5Sx4bYq4", "vSYlDh9cPg4", "vVK853Y0IC4", "vlkDkpArwIU", "w-6BnOcArBc", "w2pAXUgHSIc", "w6ljZ5OFwUk", "wFRfdZiX1Vo", "wG-F7SWtaEc", "wMNcBYRWHnA", "wQJ150bkzJA", "wRdFM0KDzp8", "wVuf_eCUQEI", "wWyXS8vRLaE", "w_3XtswZ9Wk", "wg2JOpP3TN4", "wjIO-R7tnGY", "wuI-dtoCU5o", "wv6Ja8vbfKY", "x0RgF8lalf0", "x2sLEXYNZ30", "x5sH-KOqhbI", "xHWCVEkstrw", "xOh2eJ1zIt4", "xawPDmHyiIg", "xfDRB8eEZp0", "xmEnbZW8xR0", "y4WZ3O_egb0", "yF6TnW2nqRM", "yL75POPtiAg", "yP8bWmMWv0g", "ych6PTHaxuM", "yvi5Sbmv3m0", "yz74NKhLpQM", "z7qttfrnP9g", "zaLDovGe94I", "zeIzYpsIBVY", "-HbB3k38iF8", "0sbqc_QTlP0", "165WDm5XO-I", "1agVoy2-pso", "2dN5kTqbM_c", "2qNCUVfvAwg", "31SHialb1Nk", "5PrEIlu7Djk", "6HmBjgrndXg", "6YLIUeOZaKo", "6YbSViipULc", "6gRDCz3UmmM", "7a6YQGZalTk", "8z8LeXroogs", "9t6nSAefsqs", "ASsPER2nZz4", "AyhHGeDHcLo", "BZghqpZo__s", "BdVLdHPgpso", "BfdT38m46Sc", "BuxFgT8lo-U", "CTjUlCuNv0Y", "EcJf2VACAb0", "I9XtHjpvmTo", "ISaRBWmBKIk", "J1_xgl2Z75g", "J_dVLtoie3w", "KwwiQd0zu1U", "Le-zH6IXcZw", "MCXBsnmSN2g", "MTvbuSlUWCM", "MWwzrxlNfTM", "Mb8NozJngTw", "N-IZp1HM1zQ", "NE9LyYVYti8", "NrYyrBfx_vw", "Od2zJVkmtqw", "Qb5_l2iWSck", "R_1YA5CltlU", "SetEKKac0IY", "Uve52cFsw2s", "VRRUHv4B7Rc", "VUcxu9XbByM", "VoAfAj7MmW0", "WRqGfBH6bLU", "Wf3kE7N7Bww", "XGAGsB7QtfE", "XoVdWb0GVRg", "Y3ov2_Exgaw", "_8PaaoxIRCo", "_GzjXn5eQso", "_vnOTxkWFx4", "bdKGLRojQz8", "bsIPziGIgQM", "buaPJ_0g2b4", "ddn9---CKNg", "f9kt4g3a56I", "fNZzk3fAo-g", "fcf_u-v9wFA", "fis83Z89V-s", "hKBn9HYsSrw", "kXPTjH8z2rk", "lFPQU8wtOSk", "n71vPsNG9DY", "neaEoH73meM", "ohDF8rpzhDU", "pQ-LAjJiO3E", "pjK05vL3eIM", "pkbVdj4gkbY", "pm6hwOijo1M", "q3SuBNVJggI", "rYkMWodUSV4", "sSPlsPcqcxQ", "steIO6yJnFw", "tepLChQg89w", "vncnxOFMGas", "w6fQ-z1Jm9g", "wi03ay4QkaU", "xUopSPn7Nro", "z-XMWNvwOfE", "zbPleoSqGYc", "-30qCwz_LTA", "-mX6lBTznJY", "-rg3FlE5qEs", "0-wAHXzBuyM", "01qAq5af7iA", "0PPqK3PSAmI", "0dDlgXPCcgM", "0k6PDX-mKNk", "1X7pjdGYH44", "1o1Iytkx7gM", "2614xfGXlrU", "295YWc-Z6BQ", "2LJ_ImtnVGk", "2Ou7iqLuU6Y", "2WsRMhyFdYY", "2YJL0SCDuLc", "2yI8a0YMsAI", "3muN3Mw7y6A", "3yK_JDP71GY", "4KAJrlIO3bs", "4NfYL7dzJQk", "4gSxT2Mcfz4", "4hgcGMr2AbM", "4qW9ZgGterE", "4vpbagQic8k", "5HzOPg71FLQ", "5LL-amu6tKE", "5OqX2opK0XQ", "5bV-4VXigEg", "5tgXqh0B15k", "65GXaMLYvzg", "6bbOoABV0oo", "7_5WRMdpnA0", "7emb1m1QaIw", "88ddeGq4Srk", "8bngpXgitAo", "8dmtZwl7buM", "8wuICam9MSE", "90lzeBO4oik", "9PRIR1eviCE", "9qh_EyfEEUo", "AZ-Mr1CiKVk", "AZKAOUflJGg", "Adc90N-uWks", "AiDSS-lBuu8", "B-kZZA5onC0", "Ba4EmY9272Y", "Bo3GW5j1NOs", "BtZY-d8xF5U", "CI9zk8165dU", "CIf9B2W5o3M", "CMgQr3FYngM", "CXrfeotp0T0", "D3jomGdzVx0", "DhQLzTuqQiI", "DkEfPbJgY7U", "EL83M5zykn8", "F2ezhH95E4E", "FFTYMozAMRk", "Fih7zGuuoO4", "FzpVnl8raGo", "G4lMuzJ8hLA", "G5p6-HXWFn4", "GW1faA3Sm4U", "GviNW9DCyJU", "GwyhOqm4nTs", "HfuDJxu7JeY", "IAtb2w9HzQY", "IJrVYfYAHfI", "IrsNE1mXVC4", "IvwFdi7NZns", "J9bH_SmVHHk", "J_OaaMengko", "JjN_-tB8uxs", "KGaM1NwRnt0", "KZFFmzAbyK0", "KvDqALt9u9E", "L-gVbo0pcKk", "LJvEsOD2n7o", "LentQZ1GFwc", "M1cN6sIiD14", "M3omQAaodCE", "M5o1i9QapV8", "MVXy7luS0Ac", "MaStO9moQMw", "MosHmwW61RY", "MyIM_jKRPm8", "Nj1ksnEfAf4", "NqDzI7RCaQA", "O1i3HUkHZBw", "OLlCXRPreK0", "P25GxBIwZbk", "PGoD8ZXbNz4", "PVTP28MUwIA", "PgzKXrrvg0k", "PtXQ32D3HTA", "PyzNBPSVQ6Q", "QIcO6Nkyx8Y", "RA86vKg5Dpo", "RFeAm2KIPVQ", "RZK81q_QARA", "RaTSgi5nv2g", "ReDcvM_I8nI", "RnqVpkKe-fE", "S3y9sw67mv8", "SNGaRyVj30o", "SZ2Q8OrUA7U", "SabBYWSqWrg", "SqPVkdawYiY", "SuJ2V7fO95c", "T2iKWzFyRd4", "T4MggQ8RiEo", "THjlXFaGAJo", "TRXMHmotd1Q", "Tc7E3PKKq0Q", "Tn5VTIhwMkY", "Tnwsxnzdsms", "U5ngYWEmzJY", "UYgIE_uiOmk", "UpVB2u_gHCA", "Uz6vL5PWDRQ", "V-xU9YCPKWk", "V0TSurdrf7I", "V4UI_N7wGyU", "VAKbXWi2-bI", "VEQrt3ZNpgs", "VHm71XA3AXA", "VYCFrELZQJA", "Ve0WkUprWa8", "VgtfNhZYCFI", "VjfY6dnTwpE", "WFzU8QvHA0E", "W_DBwAEClsI", "WipxqFg9FIs", "WruVe7zsw5w", "Wvzopi_TdkM", "X3fpM14Hy8w", "XIU5IDTz1W8", "XXEaYUJfY8M", "XmkV6d2Wi1E", "XsIA-WXhwKE", "YPIt4-shLNw", "YUFavJqn_Rs", "YhdiAQS5ZyY", "Yrl1SR4UELw", "Z0ciFayQtzU", "Z25TKoG9ixY", "ZTRe-cv7iRQ", "ZqT0JAK1HPs", "_0g07FpvErc", "_ARV8F-zBSs", "_Pt6Z-6oOas", "_jGrXLhi7yE", "ajDW04pjEEE", "avfMrVNUgD4", "b9GM2GU-bok", "bU4HsMniOkM", "bZsYKWXdP8k", "cIdjJnck404", "caKl7wiffAM", "cbaKBI5YNqA", "ce_KeBYgDyQ", "cegdV-4PSQY", "cp1Ch5GUnLI", "drZMQ6cSFkc", "duFPmY2vry0", "dxA0ybwXKyo", "e-e3TYuLJF4", "eURl9gslFo0", "eriDJxYOv-s", "fBFF7_AE7N0", "fEhKmmLZQEo", "fXFtoj93YVo", "fiDFy196Tcg", "fp-gt7OAhos", "fxfyfF-2g3w", "gDh91UgnJlQ", "gMFiz5kZF9s", "gf5JzeN4944", "h0PAw677sLU", "h0zJOL7Uo9Q", "hHiFaszTczY", "hWsxP4-EQow", "hbbX6Z5xSH4", "hmkH3_zXh80", "iOYYpzIlJo8", "i_-ijidD0Pc", "ibCfEyfeyyE", "inZ06uVjLkM", "j-UJKhhm7CU", "j1Nwp-onNy0", "j5AdEiDyZ0s", "jAdXxX2BqIQ", "jf4OM8FVT70", "jtVGTbjO33w", "kE1adHw5u4g", "kMzR90O59Os", "karn2uNZ7kw", "kfrHqUt_EGQ", "krY1c6cbcoc", "ks4mKqwCZbs", "l-S6xwz6vic", "lDDP_7DGtjo", "lheHPn17gi8", "lq4CXTzq6v0", "luayP4EHLuo", "m3pjEeyhbK8", "mIYXXWljm5U", "mfCeo-QoY1A", "mhGKzCkyVFY", "n0X0GvPy3hk", "n7f_KD9J6Ys", "nBKOddZyiDY", "nQedIAt_ABE", "nl4OBnX7w8o", "oATT4je2wD8", "oJziYVXOKE0", "oVJv-N-URlM", "pNiBgiZaQtg", "peCCENSNz5A", "qSos0ENoXFU", "qhAAVI7j9dE", "qjs4oGh-mOc", "qs8mdoehtn8", "r3gQY1sS30w", "rD7-CxgRM80", "rN9CSlDDT8M", "rgW9bp9Qu3A", "s90Sw2GjdoI", "sCBj5G5Stlw", "sMe8aN9v8kc", "sQuGQ4l8q_0", "sVqUPBoiLIQ", "stSS-MBLXkk", "t3K0Nz6w7iM", "t3gd46vM30g", "t86WFUGagwA", "tZ-R5Hgmf_U", "tbbJlIESbMo", "tqRYyma_gdI", "ttItckCefTM", "uVJEHXD1pvA", "ue-zhlMcxpM", "umB-wfuwOvs", "unpuuLcztk4", "uoLHcR0IEYA", "utxwfKpSDas", "uujeRdj2XMs", "v1fke9WET3I", "vQF8OVPA_Po", "vY-cnnKjfbI", "veYRaJaBRUw", "vtqOR_cynYY", "vu0Zd0RbzZc", "w7WCRe4VzOI", "wG5GYpJxxzU", "wG8FTIjMTM0", "wHS8mL-4IZ4", "wqLIvLNyLcg", "wt-looMgvLY", "x32JffqsbaA", "xzHK4K_CFGA", "y57yf9oV2ng", "yYk8t09m1Sc", "yihl4QaKKLI", "yjBirw1SeG0", "yu2_eVHNoXs", "zcbVFKdXeS4", "zrO2nu3uPt4", "-P_eQiNlAVs", "As83r9WeL9Q", "MfJ1kaw4Fk8", "Mhs9HQilRvM", "UU-EX-DmCuc", "XraG0iS51RM", "eRdhBMMJnY0", "gb1w5n9pl8Y", "hOQZY-u5RAY", "itYsjtpfVBw", "qnvMPfAYOdw", "xmUP6wbaDl8", "3z02f1Jr7Y4", "5qgPczKp0u0", "F5RL9rHn52Y", "S_mdCDW1dzg", "_18Le_3yG8s", "hBYfCuLOZQo", "0YqmvIH9Owc", "1Zi168qr-90", "3Pmbx8a1fjY", "3cjwOXLM7jQ", "41huNrEprHA", "5zbFdgnV7js", "6Xhh52lssvw", "6s2lSNo_kuI", "7HGseDr2AVA", "7RoHlV2fF9I", "9V9tSud_VaM", "9gKwJj_e_ZU", "BGlKOCZ1Qa8", "Bolqf9iNjmE", "CBYdZtQ07Zo", "DcfEJ899gk8", "EbsbfUbawKs", "EkF5reonMvU", "EyiKbhAEAuQ", "G-QzO7zSea4", "H-DadWZ0N80", "IXLwwAMCSys", "J9bDvy2Puvk", "JP6xgQeKiEU", "LJQdInaqNJY", "LRREx_w5oD0", "Leag3jXbLiw", "LifLuQz_nfw", "MJeUBmKZVYI", "NWMoUcn36zc", "O6FWDOQCJr4", "Q0AzmCqBhnw", "QY-GiY36RcA", "StRAIIwRiYk", "U3fM7JWnPTw", "U8NICSIJzKY", "VUr4bbF9LTo", "YZD8L4WU1r0", "_WB_AZLDr-k", "aqrlFq014MY", "b90e4jMhNfQ", "es0OpLgekbI", "f5fko1bDvi8", "h94WRImCbHw", "hdmZTh_v5cE", "hh6dNAQp350", "kDuIphs6ll8", "lKvyznIaHhQ", "lVLWaaGirrc", "mBPdseaK_6w", "mF1-f8MWFfM", "mt1vaczNk1c", "n2qk-Llfdy8", "oduQuQOQNZ0", "pfevjQsNjyw", "pfryiNeyby0", "qBpVRX_Js6k", "q_6CmLW183M", "qesY18kHwz8", "s3QX2rZasSU", "sGW4rRj2cbE", "sqX6HSkyOAY", "x1q8UQflfyQ", "xmdqP85Wiv0", "xzlcDkzQV9k", "z2Liqo6PykE", "8b5roi_SmCs", "9aG4c5H_SkU", "AQ4jaSydCh0", "BNKKKfE0VBs", "Ctbi3auHGog", "Owa9LbpIfSk", "ZxeplnWj0Sw", "iAM5b8QFw70", "rHdQhoY1mas", "stP0vWL2REc", "v48mCq3A3_A", "-NHENYwC4J8", "2_IWzYG5UfU", "8ZwIFYgyOXo", "9FEjIfe4T60", "A3Ei66b1HhU", "ApiQ2D7-Oew", "H6u1KLW4T9o", "L3tXMf1QfpA", "MlgRzVJXfdk", "TqQ65ivWZ94", "U3BRETJFfUQ", "UnhoF-VXOGE", "X2AaOmzOSbM", "hiYQ10O2iPE", "k-ElD-a1ChQ", "k3gQLlu3vuo", "qDC_md6GadA", "rQwnZk8ZCX0", "tN-oVv6fxsg", "tfMGXdj6okI", "uLUMw07KhhM", "0-CuFFjCezE", "1u_wkHNdwyM", "1zz-Wgf2r7c", "9rUzsQKH9Ss", "DqaPTbK5BMY", "G5O-I1nQq_A", "GS6vJZ3-ttU", "NVKcMNxeX64", "SvGLkSYDcdA", "UP9OW0pKCEU", "Vja2bx7AR5s", "bGnNWtg9wEc", "eeWXrLV7qpk", "oa9MBjlbPgI", "pi4ZnvAcjxw", "qiC8mC_lwO4", "qmSx1CfvNNA", "rWIJfxnp9YQ", "t_goa2oVRlc", "0sracjKx1Dw", "1LDvUsKmAvA", "442P2tYx5Bc", "4kvB71cMX7o", "4pogUamzbt8", "6TPk7c1wwTg", "7X4fR2JFbIU", "8q8Ss9g7Kjs", "9GHTAK6BFHU", "CAwbO3LZS7A", "D09QCaYLJZQ", "DAByZj0tD0c", "EnnEMd_VmgI", "EzmqWOsGMXc", "GiQC_AiWWIQ", "I4Q7dyfkv5o", "IPoA_kQg_B4", "KHe3gYhtg50", "NSE2wTrFnkg", "NiA1MhcnF8k", "OdBtgPWXIpw", "SE1DP9H8444", "VqeLVQ4R2sE", "XiNPEZGQYwA", "a4-njmx8lW0", "aC4Z9ly58fw", "cVP23zKRZ78", "dPPL91j96_U", "eO5RaJ2g44Q", "empx0mCLxEQ", "fAM6fO7EPf0", "fkN9nsO1TfY", "gRA-mC7eBAI", "h53S4ZtZJn8", "jGoMuv3ZE-4", "jSVa5cc8cHc", "lD3zJZzvYb8", "lDtG5zBPdN4", "lS0JirhBC7Q", "mBT2z1qC5uc", "n0lqYK9U_DU", "ofIJdUfaIDI", "r_IobpTguj4", "sxfWvD1Vr6Y", "uKMMqF9vGOc", "wY5rZloIA8M", "wdg8NlRvkQY", "-FQiGlsU7mg", "-LaUw1Wwf2I", "-gri_QsK37E", "-qmWHj0vF4g", "00Eo_E7J2Ec", "0AIfKIfIb6o", "0TEOQ3X6URw", "11u8rVEEBK0", "1GlpkuTN67I", "1rHqGKGrth4", "218CEzXVk88", "2GshQaGDhzU", "2aST8Qukh4E", "2c5nHl11ty0", "2zepSDprahc", "41PNNNhhJKY", "4Uv-OIJhsco", "4ZXpqvDzMEo", "4ZjbsyRMsAU", "4eFp0JgBBNc", "5Lvld5Dc0Fs", "5lb59ejd3Ds", "5yYnSCZZcA8", "7Zu6flXSTag", "7eCs-0_RzYQ", "7mn8FLX8co0", "86qrzK_6SMk", "8NZTOCx1vXM", "8Pn3EVnd3xI", "9WYjlXjLFZ4", "AEyLpHYeHoc", "APz4-vWKdG0", "AW_IpKr3gwk", "AralJX4zJkw", "BaWV3sOCHHc", "Dbng0_hXnek", "EsvmfHvG8T0", "FXbb-H6Eb0M", "GT0AtbE-_zU", "H95hiAkYKc8", "HNbBTp4z97w", "HW9lOcHh_yQ", "IG3gW79vRoI", "IWYiv4UxWfk", "J1bC_6VDEu8", "Jv9McDAeAbg", "KDurzv7JzVA", "KEgZ1wrCGx4", "KIYqpAHts3c", "KgyRs5f4X0E", "KmKU98HxzFg", "KmdzIa4f-iQ", "KwLKbniPzg0", "LCap16-CAIc", "LErnd3lNpdI", "LccEU3j5wq0", "LdFTxaVrs4U", "LgQd_oB9u4o", "Lmy2gl1mJRE", "MPFJlTQKwSw", "MaM2b3KHjQI", "Mug7ARHd6Wo", "MxTU4WfR29s", "NMLYow7OkUo", "Nro0W3xW1kI", "OeJbVh7NGr8", "PAeZgSR7M3g", "PqkZg6RtLsY", "Pw80nuVK41g", "PxP5X7xHDZw", "Q6irG-j0zLU", "QcvScfB4HM0", "Rsye8wQadWU", "ST4Ry8cLfFc", "T0k-neygnVU", "T3G0SycREk0", "T48gJvnRwB4", "Wjk8U32wVNg", "XCm1siFij50", "XKJy2UMK-eE", "XT4aCJJQPk0", "XbJEYBxvrxw", "XrNtT5QwYn8", "YY9EqEKXtRA", "YrIIVSov3N0", "ZK_6nBZCZec", "ZRn0hck8HSM", "_c3bt5fS9EU", "_oiYqnZIwB0", "aNyjJe81flI", "aVmkhFHnJuE", "bTanWu5dquA", "cBBBPhPEPQY", "cJcdorwE8-U", "dfXYlUzvf-Q", "dno0r9EUBEQ", "drpLGZEvAGs", "e6ZrtimDxYE", "eObM-tAgeG0", "e_u7mKSkbzk", "f9awWk5ZhiQ", "fPK-c1ZP46U", "g8-L6kSRoiM", "gCb73uJXSiE", "gb1FlsPkN1Y", "gjBvT1cFZK4", "iTSFeEvLCao", "iwU1leyaG9M", "jT-HHiAnOXU", "jcx4Yf9EQLo", "jpO1J0W3Fgw", "k4SrKGH_LZ0", "k8CNzJUAASk", "lNMdGlMkuXo", "lvcYhUzB7rM", "m3ghty4c9TU", "mXbgzE2Q0BA", "m_b1wRLT4EM", "n0wmU0CrS0E", "n1pCPbde79I", "n1xeqsj_DfI", "nIEW0ppdvPc", "nflWz95z4V8", "oA9im1Jx0DE", "oLD8V-wNP9A", "osPPrhlAUTk", "pkT4FFw3P0Q", "pqMOCth-UU4", "qo7N1LuGnO8", "qop59bN83S8", "r66wIQQVP9w", "r6xw6jlFlJg", "rJA5dvRVz7I", "rKlfr3CtFlo", "rSjh1aZKjbw", "rvjxRL7I7xI", "syWU676lCJk", "t9KRjdvYFm8", "ty9htVJuzMA", "ueHJNdW54A4", "v8PDV6sE7ZI", "vvHM6sn9Ak4", "wLlGJWmRm8k", "xymxgrFlZOg", "y1LqzX-unDM", "y6Ht2ws317s", "yIN9yvnYVwI", "yx-XdtWkoxs", "ztpJ-Vh2iHs", "-g-2KJ3gZ4c", "-gz5p9NbOGg", "-rHDuqJ0qNg", "0AeCGb03fsQ", "0cfUdmkgGAI", "1XAgU8ckuTQ", "1eOIGxr8sdQ", "2YMLDrj1UgM", "3CTJdc0vCJ0", "3IoRlMR8UXs", "3vgdyldGLGs", "3w4k2KfGcbo", "4-KozfXfA2M", "44a6cjvmE8A", "5hkei1KQhak", "622xQqMqcc8", "6IMmb2ySv-o", "6jsAC26BeIw", "78aBAM9LRVg", "7XgvZcGTh2A", "7c4X3PjwXFI", "825Y0QM1ujo", "9zZGqSXya64", "A6kK1cy5tLE", "ALzcEFGOa4E", "AWo6SFshdNE", "BXhCvuo6myU", "CbAoH0tDtFY", "Cn2-Jurk7fc", "D5NbUlEmQz8", "E1s0fEtIwnA", "EzbNiidTBfQ", "F0HE8x_qBjU", "FlVVAaWZZgg", "FxGi4rr4ACw", "G18lxEMxa80", "GjmirMFiYIs", "IsF7X_oKLBo", "J4SLGYccPIs", "JLEBZB4OQc0", "JMYl7Kt61jo", "JQVWT99Ngfs", "KRkAEfCTxSg", "KTUrNr993bA", "Kf17Mnm8r4k", "L-vi1XAzSEQ", "L0o7wHusoiU", "LYnqJVk9SBk", "LuAMJaVcv6U", "MFWyFDzcYxk", "MR7NuMCCO8Y", "N9HSID-uxZI", "NNa3ThaWn_A", "NPgn5uvuTlc", "NYB_gBJBm10", "NYlhxFaaa10", "NnvREZLiRXc", "O-vJ-OsmOvQ", "O6KBzMzQ4fY", "OQiyg7iu3x8", "OYl0Q-_Spp0", "OuPnhzzCQJc", "OvtXFjtTmqE", "PCKO3MaqaZg", "PKZcPbWGjkE", "Pa3XUwnPvAE", "Q7-Q00IgUiA", "QgtVRooEQLU", "RFdqmZ8YY7Q", "RozuJ1H7hhk", "RtuD8T0nXbU", "S69myavuIdg", "TIU8aoR_O74", "Tb0PqpzdgnQ", "UVsQB3_SL4w", "UfpxxmQBYiM", "UiDAEGvarnI", "VbiF_1-B96U", "WiMCallhXHc", "Ww5culbzHw0", "XPFRgBKV-w8", "YpOMpXehyQw", "ZCiIovOt9oM", "ZKNv_JPGPuE", "ZWGmgq7WC2A", "_Bmi62vQLZE", "_Gw8OX_dMw0", "__FmBJkXlHE", "aYbqgfT-R_U", "bdml_6huq7g", "bklIO94r8pA", "cHzVUGigo7w", "cV4b9Qf6Mu8", "eDmIZf7Ty38", "ePlA5RICcHs", "euWcfPViJSo", "ft-CDgiXp8U", "gRiJ1GjgYTg", "iG-dRWmNHF0", "iR7Wcfsrsuk", "iqui4CdZPUk", "k2aMgRUTCRQ", "kbsKjH76-JI", "ko5rVzxJ_yI", "koCuJo86co4", "l3ABvIXoPxY", "lNWLRAkwJxg", "lftdK0QkwAg", "liCuTkOFM38", "m19Rsfido1w", "mCSgDeSx50Q", "mKjg1b7OqgY", "nAdmam7Ib-w", "nTKuZxfNJFU", "nWDG5tMmRMU", "o6K0jw7WiUg", "oSYCcp4FpLQ", "p25d0yst5i4", "pObO4x4gVPg", "q5md7L5ju5E", "qPVumAdBsp0", "qk16uOmtTbs", "roqZTKPR3co", "sMlKF4lIzlM", "sZn5iViUkGg", "sq4v7ZsST4k", "t4NnhzT6zFA", "t6eLebI1ibE", "to-xD3cuhzc", "u1QTYEDfVRI", "uFxjHq_o9M8", "v6S9rpvhl3g", "w-f65dq4sVE", "wt00XjaI0EE", "xESjwJ87oys", "xQjycc-v6hg", "xlmb2_16Olw", "xqKM_jo1s3E", "yGvgxtS2tiw", "yxZcNZKr5Os", "zAoD7e5jXJY", "zI-Fz1Ej9a0", "zfsiGLtuOG8", "zk2MveecGqU", "-3I6TMKT3mk", "-JKiJ4Dp6QA", "-U46wFPZJmA", "0a4HakXyhFw", "0qkC7OquI68", "12iErqJ-Y3Y", "1XfHEtCDx6o", "1q0AuidvReI", "1wr7QfkziXI", "2Rj-q159yk8", "3ILNNzPSu7M", "3bd9M79qTxQ", "4AQ4KvkRcF4", "4OEBTKH7Flc", "4qZztvtjGZ8", "51OvaVryBJc", "67e-v3Qlg8E", "6ALlexU370c", "6Fvc2jrQpQc", "6Szu9XydTLE", "72b7yBeWvbk", "7lbZDpC0lzg", "868s5ESD1II", "86jJxzHi46o", "8JdAryUl5xo", "8ddqG-Jw5r0", "8tRqaoUeb0U", "90zJFkadzpQ", "9450t1MRMD0", "9McyxP212jE", "A9a4ndDlEh8", "ABOlnx0u6ps", "ANzZDgM-hYA", "AjCkALt1Ilc", "AntAcSggrXA", "B6629NTLukU", "BckB4n-dcf0", "Br7H6GXw2MY", "C1bPcIj1JY8", "C6DV3rkgbWY", "DY-7n9RgWXw", "EYrzzTjtf0E", "FFIB3MmSz44", "G30WjJ4-isk", "G9jPqo04bt0", "GQ8WmfpAhkA", "Gxju-wNCzCY", "HjhdlSELSuk", "Hm0PJMXW6YA", "HyaWMB_T38g", "JryZ7LSyw_U", "KS40yOLTthE", "KcXjXULG4b0", "KoIX3puLy3o", "Kwt6h0hnyWQ", "LMIBtooKgyE", "L_Rl6_zwOA8", "Lob9gSuw-y4", "LvI67_G1oOY", "LxqCKqXLA6o", "MJyldPf4zao", "MLAyeJwRt9Y", "MNi7fY4M4kk", "MrYkZFh-i-4", "O2RuyfwIC04", "O4L8kjE2SSc", "OeTEHmmQ8pI", "Ovhfjdow8-o", "P-bce5JM0VM", "QUpzCTD6Zok", "QWDUgAnwtNo", "R5cDilZVNXY", "RMuLHtFchi4", "RO8W62A1vxo", "RgXCLbNg2e8", "SBHeXzrExdE", "TPO_XZOKKRA", "U74IxdUQ2bc", "UKchGKuHefs", "Uy3FDPH8QYM", "VETJZzXd1q8", "VMOQSTgxjxQ", "VY9ocAN-vb4", "VjkRA_9pwCg", "VkVC5F0Xajg", "VyhUrgIRn7g", "W5aZQw8449c", "WAg0RzzZPUU", "X2pCFSnKBAY", "XjishY73-pE", "ZJPUu1wev3o", "_2mBmzeNxwE", "_BHESSgC13U", "_IfT12e7kgE", "_ML9z8FNtVM", "_hpqMbJJdp4", "_wmBXuggE_U", "_z83vrZgIgI", "a9qstwrMJNQ", "aIAULYUWta0", "aMAtiE95JPM", "aQlRl5GpUzs", "ak9Vc0rG__E", "aoqPFKBuDSs", "bFPLoNCRx2Q", "bN3e-nbBqHM", "cOVNeKI-Gy0", "dBAf9ctMVpo", "dIVjgLxvrNc", "dPLtLoIkcRQ", "e6xT8ljOaPs", "exJtpW4g86o", "fSojv-KqLsY", "fhV7NInxPzc", "fouPdnOo-4E", "gKySokLIbMc", "gPr9hU49ffo", "h28pmKsBY_8", "hdtMFtEiBT4", "hwSendSrLDc", "hx6MR-dbHOk", "hzkExbOm_Gc", "ioWJgLRNYoU", "j-hZ6bRevfo", "jWZcayJNnd4", "k100SP3Gk04", "kClkklT_SiA", "kb1X-0gnLjI", "l0f4-UG4vUs", "l0mcmqnXqYk", "lmNYbj8DnqY", "luHUjQNg90E", "mNXyVh5qhGM", "mzrSS6KmGbk", "norE7lWonbw", "oYtpWR2HbvQ", "pHiHVaE08cY", "pfvuFXbET-s", "pnwxqBb8SM8", "qLLBfp7TnzA", "qo1UZBAyL4k", "qx7Z1lY47A4", "r-OgwwFwhtc", "sMRalL_VPUU", "s_2k7-p2aVo", "sfbJvci_nP8", "tXhB5lnnwbA", "uGgIpFElxY4", "vhXJZN_JaNM", "vuED5hMksKE", "wfTKMX5BOII", "wo5qwcrgLlk", "x0ZmFMXhmhM", "xh8-CeNQjbk", "y_0dc5ffFbE", "yscappIjILg", "zxUYbGq3SE0", "0_owF6u2lg0", "0wtjUfbVisM", "3nx0pWOViiU", "6FFdh3JvvjQ", "AAHEWBH7Af4", "mJwhMvjaU-o", "x0mCqy0yaVs", "zLfw1x7MQ7U", "-2DLt5c5Fms", "0xYOQI4xJHc", "6OWRgzLCcAM", "8OC5bv5Swoo", "9zgM1INlMss", "CBSfr1nhQbI", "DmZnyWg3_8o", "Gk19VE1zA9g", "HuwPDPJTTYA", "JCS3HX45YlY", "Lu8Y9-5lN3w", "RBcYiGSPfFc", "Ug0Nu-lskx4", "Vgc2Dd8lNIg", "Wa-Bh9mYjAI", "XlKj2xwKL3Y", "ZyGsNcfPIPY", "bwkA9i4m4k0", "cnJ6jg8YA-Q", "dH__2Zn-TP4", "lvwh3zXnT9U", "nEW9Q1coblo", "p59P9MA2bFM", "rrFI4Lg1r6I", "xKSPemydSX0", "y82b-tCyZsw", "1f9OaHgK4Pw", "9OhxKXPyw-k", "9x7z54Edi1A", "A_LZg1MV-ZE", "IFRfOyJbSRM", "JanMR3n5NEQ", "MFeI4KV8bpM", "NBX_JO1riYY", "PsIeZKDGaR4", "RRzM0Ld1A3Q", "R_CL71b__Go", "awQJiIFzsDI", "f2kUJl5TNNA", "jE89DIxTKJg", "jFGvtPsO2KI", "mROTLIN9kDQ", "qEkNW1PFpWY", "t2OHGtpyq3c", "tQbRivdeoP0", "tvqxB4I_xPs", "xA5QUHLFq9s", "xrgRu_Ijt8k", "1VCToGBxxRs", "6Q-Z3vMvbzU", "7sgx449l9Dk", "BnIp16skJSw", "In_oH_jih30", "JdQT6BQpqBQ", "LtSB61bAABE", "NTx-f4zNSAg", "S1nsVQrUlqI", "YjbQqP34org", "nt_b6IWZa_o", "tBr992cUfoE", "x5q582mYHVQ", "zBkbDmhd4as", "-K3Dmps0g20", "-jmvkSiPDbI", "0si5Jf0he1g", "1jWd0whKg88", "24ONln1lqos", "2BIK3wV7aLU", "2OrDtLa9IFQ", "2QerAzoC0t0", "2SoqUMf-zAM", "3mk9R6FAPpo", "3ubBeALjMo8", "4HMrHIv_dTE", "4ef2qpAdc_A", "5J4nxxm_mDc", "5SOScpXVuPw", "5SVoMYEDOFc", "5_LxfsGrJRg", "5hmcWAjaCZY", "6DiaUNwHzRw", "6WP5KWrsKWU", "6mS8qrIaSUg", "6rq3xx7CQbk", "7V1D6v8tKr4", "7Zy2SK-fgic", "913FLVGqATQ", "977Ycyj5soc", "9BhIqGfr44k", "9jMEHfp_nCQ", "AHkOZ2wpReE", "Bw7B6BWwIqg", "C8M1CHFpg5w", "CCGcM3nqnEM", "CD4ca4QsZQ4", "Cdp4Ic276eo", "D8scT-LWOeU", "DbMhcvmrbAo", "DfaEKPOUn0k", "Dm5TC61PRsk", "Drb5OJsga9c", "E3x1Hk0zMh8", "FJdGNCSjriw", "FmLZJTPxtMM", "G7tTgKASEYg", "GpahS_Na2V0", "Gx54bzZC2to", "H-7smAVdDpY", "HGO5Q0a19NU", "IDOSdOUjuEE", "IYcemgihHS4", "J422h_Ibmy4", "JXpBOzxja_E", "Jm8kSSx6Hgo", "JvTsjvnVqoE", "Jyo5xljqq1Y", "KT4YQc1DqPE", "L-6GxURUXu4", "LBXh1xR6fX8", "Ln8YT3BVEAM", "MjsawSHyaao", "MmHR7JqQm3Y", "NGSuw-X77LA", "NKhENgMeDIc", "NQHfdoRdj2g", "NX4y71wSgLs", "OXviWkdj-js", "P4UZAisSDm0", "QnKeXXvLSb8", "QrOK9-pG8rQ", "R0MqtIGJb0g", "R0Qo7iskV6w", "REB8GKCBwLA", "RNn4s0cno9E", "Rb032C8656E", "Rk8QbDrPKds", "Rwibc-fF8MM", "RzJ_L_s7ndk", "SVPsKUpj9t0", "T8LggPmpN30", "ThJkzeNMUZA", "TrV4iNYlizQ", "TwqS1StZIRg", "USP28HVseTk", "VM5Sr5-DnNk", "V_2UItroRNE", "VlFXmhvNq7I", "W_wCe3oxSn8", "XWu_JHNXHWo", "XtMquPhiojU", "XwuSD4JlNmY", "Y0Rry3Yp7V0", "YSalzhQhTRg", "Ya_zBp2Dk_Y", "YlHRmTQkpto", "ZufHEfhr9ag", "_5kLf_cflA4", "_MIdBJiyQNY", "_R0M45eRXDA", "aGqngWMc0pM", "aNPN0VObwSY", "alZXqvsFmEc", "bLEh1bQUDI4", "cDPjM6DbkF4", "d5dT15Gg9fY", "eq7iKpe_pKA", "fFVnSbCPPUY", "fPkn9PcqEm8", "fVZmVGcatUs", "fa903dv04gM", "fuaeovBHrLU", "gj1AgmLRYQs", "hB0FduXXoUI", "i-oDgwXeUFA", "ixLaykoaK-M", "jBWJ3PLj044", "jCyLmQPkyjs", "jDzbu4NvNno", "jJe2zwa6lW4", "jLYPJCFgoEE", "kXr5tyWL5sw", "kkG6lq6YmzE", "licK_wSciBQ", "lrO-05vgMTc", "lrgIWbLLBxk", "mNmoy4IKnZY", "mZ7n0YJn7_4", "mypi_Cto6dg", "n-K66zgFe6Q", "nKIyKZcC8Vg", "nuyI8nTOoJs", "o3IOKi_YPbE", "oCzBqI4mgnA", "pUujpkDHlkY", "pfy2oxPSGAY", "prneoG8-Om4", "r87T_ioN9Bk", "rOEhjMwieqQ", "rfptYNUgPPE", "tTBeNmSVsOA", "tWailffX36w", "tlea0Tn2lt4", "u9SbFrUqNXY", "uA7QLVXiGxc", "v6wprZGtd8Y", "vAicIKz7Xe0", "vOxE5QibzNA", "wFNF_P-24pE", "wIEEDomqguU", "wX-z21OFWcM", "wlWVvfgsyTw", "xiGXQCD8I-Q", "yGog1AJVoZQ", "z4FFLFpeKWc", "zSiDQx51ae0", "zcR_9_SE4jo", "2MCZ-7gbWfg", "3DSWmxGnZgQ", "E1PWYyPJaZg", "L-22RdOkccY", "NHJ59VI5LQM", "QgI0Vrsh8GI", "c7afXspGcTw", "etywPwkAKUs", "vaRQ90ggH8E", "-b7dlJV2dp8", "02kW6ZqButw", "030QZHSlOoA", "0A1mil2QtaI", "0tp1EL8LtsM", "19vKtBOjAmw", "1Rss9qZn8VI", "44q7U9aA2Z0", "70W8cd6H68Q", "86fBoHdS7rs", "8qDdP8H7IB0", "9YAG2VB4Aes", "C5nUjdljYsM", "CP26Lhia3kg", "CQNcXsx-f_8", "D_qvu-q20lA", "DyZ1IPrpvLg", "EJE80cf00J0", "EzG1umf1FaY", "FkrPUJS2wIQ", "Fo1QIKbGgiQ", "HY97iYlH_FU", "HeHFpaYUUzg", "J7_KdfSWw7E", "JRGycBlrxoA", "JX8SLpFHHhQ", "KRD9pqwpdio", "KTCrMscoqoY", "L0WB4trEULE", "M8f0F9iKtTU", "MgnElaCWIIk", "Mwc2q-3kcfA", "PCP21WUSRK4", "Q8Bz8hI9meM", "QH9_84gcJRQ", "R-wQCtLH4jY", "RRuL6f75Hsw", "Rk72m9bLrBo", "SsmZXFnVw8Y", "T54A_C6WfxQ", "THUFe2zvqG4", "TLVIskpPf6Y", "ULu3wC6TMuE", "VQYI_9JILi8", "WOutyNLzXSw", "Z9381KNWWCM", "ZoV3-Gegmtc", "ZsQGvjQTTjE", "_Gb-8klYS6o", "_Qlor4PSeCE", "_dwwDbHJ1f0", "_tL2_5pMk6M", "aXzmzkFXTVY", "acAsQJ2s388", "bAJQAMFcW8M", "bEPfsks8-js", "bSmph1RAjzs", "bzqvSxvFfBo", "eK3nITiyHMQ", "fSkLfXntv20", "gjmzXe4SfWg", "gjtB9tIlQHU", "ibWi9Q7P6ZM", "kQ851L4vzS0", "koTla4YHV3c", "lutp9GQYsnI", "mM-xYN7CW3k", "o2P601YpyAs", "oVRJv_GTFmE", "q-0TRT779cU", "qVk7bkiS4Do", "r0vN1vZ1F6E", "r5STZ95Ceps", "rHTXsDu9MwQ", "tQLv3cjZ0vc", "u-dGPFGiwTk", "uMzSd9N0-E4", "uyCD_Dgsa2g", "vxtxSSuQNLY", "weSPy2_jMjM", "yP_Tp2heG98", "-ffEoEozwpc", "0IamUYOHHD0", "1YWKL9bwbBQ", "1_MOO2yndVA", "28L_3lse6SQ", "2aoDn7kxASQ", "3-guZM_SzdI", "4t4pCKXH1O8", "5G0hHIhWZQo", "5y2C-rSRPts", "6GscufRLlMk", "6rcimupk14k", "8ryhxIUejSk", "AIpPVfdz3b0", "C4Ognl2k7yM", "Cup_tQiqCtY", "DOhy-LRkCeg", "EKjXmt7uC8E", "FjmxfRD1Drw", "GAQ0DA4WwtE", "IFcq-JfiSEs", "JgWkTp881C8", "KvfNn0oRDWY", "L-CMFb-D0dQ", "MmVPv5sm87Y", "OTUlGJpXaKE", "QQvUCNtJhJQ", "R3E_fIV5vY0", "S9ffMlzAd1s", "SPXPmmWDnds", "TJSjxeYf34Y", "YJ1wt8JfB0w", "_EAyjdWqeNo", "cju87dyOBpQ", "ePrPBcmFgTs", "fXpo-z0lYL8", "iK9eNS42v1Q", "kmal_kqO-0Q", "lEHHjOb4FVI", "lKnTPZDuX7g", "nHu6LaW0IFo", "nPTMz8gXYX0", "oqJILtVCVgc", "pPHRWwNrrNE", "sPKZjj1zv2g", "svdz8fxSePc", "vFIkfpHlerQ", "xHnEDFlgEmM", "xutQUBkb3N4", "ywNIbWRSwVQ", "zDRZ-LdSSow", "-9deES-H2K8", "-fGAReNjiEo", "-uyirTCwrQM", "-yM6cmpnmes", "06ce1_138uE", "1GCmzuna5rw", "1QJ7k65a3PM", "1YWO27bgEuE", "1YtcAYD7IFE", "2hC7pQaxp-s", "2pZnt_zv0gY", "3HOA6IknsrI", "4QUM9JaEhQc", "4jHncdQJW6Y", "4r_zysFFOTU", "4w5dcpfDLFA", "5dPpKlQxphI", "6D_mmUt3dBc", "6JHhIbmr9QU", "6bPIa21_Mn4", "87BJw48yuQM", "8t8sLIFVRAA", "93BByeN03BY", "9MejXhz393c", "9YIRDk96QBI", "9YiBsBn8k7c", "9lYxaD-b_AI", "A7XD8yVZtec", "ACCAM5gc3co", "ARTMJhGmjiY", "AuYCeUDq5C4", "BYwAuy48jyg", "CPp2WajJrT8", "C_5XClGLTsU", "Ced2GsTaoJk", "CnVsESI9TaA", "Cri-B_cwNm8", "CyXJH--7vRk", "D2V22owPVBU", "D8Rp4DqPhrE", "DJIZ15jOxIU", "DJsS-3C0Jas", "DvAdCkYfndo", "EM_x1H00uA4", "EXTkUntxK1Y", "EcrvRjMy56A", "Eei9RhVt3UM", "F8qL1urswMY", "F_dx_2KZj60", "FbxLU0dQlEI", "FlyOoOnBg9s", "G4xXiRA6pUI", "Gc3CvQYEcF4", "HIa2Z9gOAiY", "HR6YFDaAsQY", "HSH58drr848", "HlORUBP15oc", "IFqRolV5z2Q", "IGmHQNFS42o", "InCFQmgVkF0", "JXGhnmWzMqg", "Ja5af0qgFWY", "Jj6LxjZ9YXI", "JuT8yXwdIRg", "K67jC1PsKDU", "KKJH-K6_CcQ", "Kuyc01x5tbo", "LiQpyg7ndlc", "LipYFjARDLs", "Loi9f_J4yok", "LrRzEoj8Nd4", "LrsZbuhBjzg", "M0NJdwO5ng8", "MZXBHLqDT0M", "Mltyel7pb5A", "N2oLzPWn12o", "N3oykJOQctQ", "NCR71pCFyJM", "O4js_37OHOQ", "OPGpxaW3jXM", "OZ6RzTMl62s", "PMNn1lz40rw", "PcUgV2mjel4", "Pv_Dl_Iauhw", "QGC91SHS6sM", "QvUQ2m3BYU8", "RiXmU7kl_lI", "S7VDZgctA3Q", "SEq1YdQQ8ek", "SOR_YEeDsYo", "SS999MI-ztw", "Sv3fO-obFEM", "SyT3FbIcUMU", "TOBGx3w2tDk", "TbFP9coNZqg", "TrO_2hYZs4Q", "Tt4t1fmP4ks", "UR_eYnzPR9s", "Ug2aI5vpWrc", "VIClrkT2yrM", "VPUt8i91J7U", "VkdqsZJvSk4", "Vn2a7sX8mCw", "WEPZ-4GaFUg", "WGko04QoLbA", "WL1n6lXJEYs", "WO-rTWegidA", "WSCe8zlnk88", "Ww_N-raKGkA", "X2KILE4422A", "X2wLz3yHv-g", "XJFh7DS5T3A", "XJRPdt-Zdhs", "XM4pIMbeDW8", "XUMxL4OtifY", "XUnyN3LJvsM", "XmX-TUmmXJw", "YHMVG_vv-60", "Ypyl3dEE7gM", "YumB47jc-oc", "YxiJGc02Z_s", "ZgAdMqlvAI4", "_MRyVhiyScA", "_xdtcpqMenQ", "aVXdWqALTKc", "aXpRZq1GGDc", "b5GxQ--6YkQ", "bNDKfOi1lHw", "bP3xfDwwDf0", "bTZ1S8xR6C8", "bt5MnHdzbMU", "c-9KTIY1tKI", "c4nEaSLq1h8", "c7z7msLYHyU", "cFJK6GxPGrI", "cFnzso0iBoM", "cFsUKUayrY0", "cLWhsPbURqs", "ceybnmGQCbQ", "chro0NjL_ZA", "crsXpGT4tfU", "dD8lYJXekOM", "dVHuD-BuLyA", "dZOt8iv-TDc", "dpxg-V3zw1c", "dxYqstdCkqY", "e6cCrmeERUs", "elCdMI6Uj_c", "ep2OZ_H3VfE", "evunmLaQOjg", "fLlU9FVbwls", "fTNDI8JVevk", "fstPduPa8gE", "glkY2jaTR4E", "gshY8w-IVy8", "h5QRCpvT7cY", "hVk_5GGxqWQ", "honkFMTBD3E", "iAQeA3_Yhlg", "iSygswE9kmU", "iWdXUVHg0Lc", "ibWAh744fKA", "imb46A8PJjY", "iqiQ8KHFR94", "ircIN09XJ3w", "j3OEa2aIUNE", "jaFm5McLR7I", "jaPxS18p4nI", "jg_cfrs5XWk", "jpWaH5P23Go", "kFUKptufimI", "k_1ZPbVsPx0", "kvwm6qmqtGs", "ky7hrWtsKMU", "lA1QNw2IdUA", "lCiMlxfLjcU", "lTmKDWI2MM8", "luuX8N5Iqkw", "m6q14h_GzLo", "mT4FEyDLFx8", "m_x2JVkn-YU", "mfIWpy7Np9s", "n1j58kPDq4E", "nRUHp3tKDMQ", "nbLXWNzoWs4", "ncc9NdOk-Qo", "oR0Ab1gO8BA", "oVYH2pV37hE", "on_dHyEyHdU", "otmeAhdyizQ", "p8CIqAKnYsM", "pFMNo8_VJC8", "pJcyMhcQGXc", "povSzPaAumc", "qDdViFBozy4", "qNvn_M2ia60", "qVPwshr9cJ0", "qckx-vaX0YQ", "r3yJU39jIas", "r5YrLB_BIMM", "rAyuvEkNtAI", "rhBMnyUOWa8", "rpDF0HA-tAg", "rr-cxo1QMTM", "sjnI0HSMsRM", "stbqTC8_UgU", "t1i4p3axUSM", "tCChlZt6Q7g", "tIrj_FLoc2I", "tQUCXs3cdHM", "ta8scdSGcy0", "u2j2clDgf_A", "uSqRejX7_cY", "v0xn4AQB4Rc", "v8iQYpkCOsE", "vQaIJ_c4YXk", "vWpB8qIqUM4", "vaHIFdTa_NI", "vnjKWFWNUEM", "vsCfjYR8X9o", "vy0lrnmcciQ", "wL30gF_qbac", "wa8emuCDMSM", "whQxYilG90g", "x-9pdAK9kxI", "xWoMyK6qBrk", "y3DTSp850DU", "y3_baAwcdEQ", "yY-se4RSj9U", "ys8NyD_Me14", "yvqmQVHd7Js", "zPL1d7Upxig", "zghL5T3DCJI", "zp7KLKGTI8Y", "zq8ixuM4Zis", "zteslt_Izaw", "zv8fXLAMXek", "19igH3wXcNI", "3bw6veSnzec", "3wL3gn6y1FA", "BVTRMKkYkvo", "Bo0lRLBEfio", "Bw5ILQLsQNU", "CSoiD78KoKg", "Cvc8UVZZNUk", "IQ8trlyJPSI", "JIENXOr8Vp0", "LboH2PBHyIs", "XcGgze97Jg0", "YnSpICcpqso", "_utkeyE93Qk", "bA17Kwyx3uA", "bxyV6v761H0", "iWjPVuV_ChY", "j7sRQm7rJdc", "m0A7lxNKW_A", "nQ1sQV_FeYA", "nesihoregIU", "nvjZzYefhZM", "nyZsyjnKuUc", "ti7mrbpDIHU", "u6Te2rZV8Bs", "vDZbXzzZV9w", "vVnn-jQU4Us", "yHTWneAY2ZU", "zk77vyStxsk", "-9i5RUMdtR8", "-j5GckxBNCQ", "0e6CC57CYOY", "1DYDx2F6fHw", "23fsN9oF4Q8", "2_QEpy_drH0", "2c-MV-GlaBE", "2cQ-juq5Fj8", "2jzA9cG4E2g", "3_Xs7tH6qNA", "3yurtYcjRyk", "5S0bZVIktT0", "6dvf7A2bzPE", "6e3WsWIQAzs", "767xuTdftfY", "7ThnkKrsiL0", "9WMD6R5SQw0", "ACnz2tNyths", "AGvq1baclis", "BODjOf6R_n8", "Blu9Xz-qkeA", "CTHJGuzqUb0", "C_2YvzF9E_4", "CbGFxky5s9o", "D58IAfJZUf8", "D8w5Bp4Gvvk", "DkH2KlmYk5g", "E3TU0wpgRPo", "EPCi663IYs0", "E_-KI5wYXks", "FcYlAh0Y4iU", "H8Zx9p2_9Do", "Hk1Qt7r7KDA", "HsBm4yDMLhQ", "I2AxqjHVsuM", "ITNr26T3P1U", "IkE9RfK0PMM", "IzBFaoKvGoU", "J-0bgwvGm5A", "JGvqBOknZB4", "JHxLQLyr5HA", "Lbp0gaZBEdM", "NT2-v-72tyA", "NtU8Muek1_c", "OJfSqJ-2ABQ", "OUry2HEbrPE", "OyugKwZfoH8", "P43Oz99eljk", "Q00NFoe-2N8", "R2kEmmbtENo", "RQ86K-IAB9Y", "S8jYarW16Tw", "SJKyODN62oA", "S_EzB9qp9jQ", "TU7J-_0ln34", "U7HaLdK3uPY", "UklkUUHKzz8", "VQSqC7mn2Hw", "VyOzizbnV28", "VzxTRvvlnWQ", "W_-6GXoqba8", "XBVnegvGjZQ", "YWepmQhRNTQ", "YoxwWAdWRfQ", "Yssa8qzJ6EU", "ZNWldhVMW7w", "_bg0UB2bBQU", "a7_kDwga3fc", "b0f9I8Iy71E", "bh23P_Nbj3A", "cfZZUgWfbKg", "dTozEZ_y1uA", "dyk7ETbiPUc", "emeymIsMeAo", "fHtOF1fiuMk", "fIyIAxj7J9c", "fNwplqrtX3Y", "fidQi02Iu5I", "fz5ypBjDWg0", "gAxyEgPBeg8", "gLHKVxGDVFk", "gYwU84QAegk", "hE2YMTW2iME", "hbnZmGa-BrA", "if36Jg-PXoQ", "j7YGo7laaKE", "k4MH_MKqUjU", "l01GALwqy3c", "l4rq1g7F3ks", "mu0Kl2nPofM", "nR0rNUN9ufk", "ni_L7p_TtO0", "nwJlbhy76jo", "nzha33Q4av4", "ofC9ZB9PcqU", "ozx-daqsxzE", "p2K60hKiOtQ", "pCHZ-NN5Ndc", "pFguJh_qAYw", "qDRG8o1kiXI", "qvKPCVzi1Lk", "rBJ4MpNU78U", "rN_w00MjjDk", "rlA-9OsN9mc", "sObMTrgmZMk", "u-_DrAJA5zE", "uTECxHw9ql0", "uuIafjTDzvA", "veBBfgABc5Q", "vjBGVj8-FuQ", "wAXQ_a4Hpxw", "wcpR0jjWtKk", "x3YphJpkVXI", "xMjV6OimeeI", "xU2yR9TCmhs", "yK3k77f_Jgc", "yUC8zLhEfEw", "ywYrgKnav2o", "isOCnUWIqLM", "1fprWu73zlo", "1taEKBhoqdM", "3Gj7xI7uWDQ", "59P4bXSmW8Q", "5kxa1gE70-4", "DZn9_m78N-M", "FjHqK7YIbS4", "MH3SfDgjwww", "O9CUXNeGSHc", "RUi6_0hg_Yg", "RXh9c8GrGZg", "UYvdiSjhOnQ", "_MWtIg25WNE", "e9CRRjvDkxo", "geuSAgZdd1M", "hVvQVXLyG4g", "jzxVYqPYn_o", "l7MxehypKHI", "lVsbHc8CfpI", "unUnWPuZPl8", "yx7q0ct4cMk", "z_ZLKswD6mM", "-6Sh8zo78vI", "35h5-rqp53E", "AnaHm4T79yE", "sqCUtv7-Jxo", "2P2dKJptGFY", "3_TVwmTbBWo", "BhdjbuvlDBE", "IG2YKWsBPn0", "P2WiX1uo40g", "TWpryywXWSQ", "YtlnkNu4OPE", "eJcqh4Wb8Bg", "gxXwOClkGuU", "rm7e3w3HJMI", "tZdC8ZjgGKs", "wyl2GFJ4vPA", "1I2X03fljas", "1MbIBlio328", "33N5KI54eWg", "7rJJccOD6fQ", "C4pdyS2pHwU", "FBRPJ2T0CGk", "HTfTBlaOic8", "Ia3wUBUmZ9w", "Kp5wEJi6oeU", "L9fOchg7R5g", "MxSUBm3CaZw", "P67TX3VRZ9I", "PXkLJ26dS-o", "RQucsIMjUug", "Ryl04h6mCwU", "T1u9su925k8", "U8jKS3vIAXQ", "VCQATf8gcRQ", "VINVrD2PEr8", "Vi5kle5DXws", "WS1KwkQ9cj0", "XrsZUZi34Js", "Zw0lGr6SIeU", "_ZLVbf9l-eg", "bt3HlP7iS2U", "djgfv_eb6mg", "fEWXyRS1FMA", "fgSqYYSXUH4", "g17tM_8mGVY", "iVsTjovQHpo", "jYoyUzjXwuw", "mO9iL0I2KnA", "nRfPzn4XKbM", "nY7u3oVibpE", "o63uBJ4tFCI", "qQxRahLDzxg", "qs6LIQ1_66Y", "regus4wD8HY", "sBU5-IA9MWI", "wSbo-kzN1lg", "wyCNiq5lyfE", "zZdcfHyaWak", "zyrXM9F_Bno", "vbwojT-o5wU", "0Q6VRkrMRRg", "1iUzDRbov_w", "1mCu2pxElfQ", "29GdIPnTEp4", "43UENbUKHAg", "7e-L7SuMphM", "BnGvvKA9BmY", "FRKoHXLVm0s", "Fnkls6-NG60", "HR_qJ6jgVwQ", "ITbKCNncTG0", "KSw1ByeDtQo", "agSI4PIqz4Q", "bCc6WuKyhVE", "ciXy37-m35k", "ikzKiRzmkR4", "jzN7RxtL2-c"], "Blues": ["0QBQfojgH0w", "0Z33WQfbqhY", "0ejRqaZ2Jqc", "1kmUEzgt5WQ", "4Y3FB1xwX-c", "B3oiye153gs", "CM8y1bZpNio", "Ce4zGktL_KM", "D_MR6OQr6Io", "MJSPZO-CUcU", "PYjE_4ckVcc", "PYpxyhoSgkE", "Rhsgud45mXE", "RnAzaEN_idE", "TGQ-u_4H89E", "TQ99deYC8Nw", "TaDfpgRpK8o", "fC9RjtMPk2Q", "if2uzFaWats", "kTf2473IHhU", "nUPOQM3cw8Y", "q_9x2fOL3Xw", "tbHHfRk55-A", "v_57rH4XJDo", "yXzdtmyq5Hw", "-ae_tBkCqeQ", "1N6HOC781tg", "1iwb6egbvk8", "1r2kl_Dl7s8", "2Un6OnxLCV8", "2kQ3FJMZzU8", "36X3wecT2z8", "4VfRJD-0hCY", "5l0Go4dck_Q", "5sUloHPGXaI", "6KIalRgY93E", "7o_Q111ErHQ", "8AFrIILb4cE", "9K6DeIZCGdY", "9RIaUg12HUU", "9SivA_uJOAc", "AmAraX3nXTk", "Axrzkrk08pg", "DPklvOknz0E", "Enw--zJCsnY", "FuSZ16ucXqU", "GeEYLgqGtjc", "HcGrD1nw4YE", "IuhxIcVejBY", "JIy0xoMB_aI", "JwFBQAkmnn0", "Ls3dvzMF7tk", "N8J9Oj-ytCI", "NGkXQchhEto", "NZR13PrTz7g", "QJm-aSl8g7o", "RImDzS-cdOY", "U-K-cr4DSXk", "UbyG2nu_sRw", "UtdzMfSwXi8", "V5sDt07V-Aw", "VzaNyqkL5ZM", "W9shk67MYdc", "WGC2H1D76TA", "XbszO_3RvPc", "Y743WrbVi8U", "YzuMzRhRTPE", "Z3euxRLOQec", "_153g4A3pTo", "aqK-IzYvLxs", "arcmJBW9GAg", "bCXILhgRXG4", "b_njZHbFs1Q", "cOz-5crTF1Q", "eAZAVAEgiY0", "eMxzPNblRqk", "fS9LF1cw6-M", "fbyVoRT3zVc", "fe-SBrc-ULE", "flzcDTYW624", "k4aMm_EjxsQ", "ksU7yy4Q6q4", "muaaG2fl6-o", "ndCyYRBfZYQ", "o-nzuNOhylU", "o5xCVtkOPVc", "opaKycA6PXM", "pTje2lTQlPI", "pv_rQKoFfCM", "sCyu8O1LxJU", "wF03wYPeKQM", "wj5UYDONmn8", "xTlXUsBOzxE", "yMKBR6MNtTA", "y_3OWcdxRR8", "yx6jiuDtmII", "9XGFoEbg5ew", "xs5Z6zV0Xc8", "0-Bf7kKIWTw", "2o4pkKiHmg8", "32U8eZyteSk", "6b31UygRYDI", "8oxzZV_Svvo", "BQWzJVfVf9k", "Bzs9y7lfoqA", "C0WZHBT-_Ls", "E01BTL2_2v4", "IBro9BjJa90", "LusSiXuM-Zs", "M4_eiMF7zt4", "MW-d08--1iE", "Mc3TtoKwZtA", "OyBvHWi8kP4", "SJz9VnfLQ6A", "WD7GwRxxBX4", "Z9DfJgs3IS4", "aDnDsJwUp7c", "aPNwKjOGhR4", "dMEe-7kkTF0", "fPhAcBiOq34", "gCXuoJzWt-A", "nBB6-hX_RyI", "pStV8bJZ6-8", "r3ixlwSwB_Y", "vXX4e3RHptk", "wj1T-s74aRQ", "xahLz8-2M3U", "yNbjJXiQwV4", "086q4ic4l6Q", "0ShAJarfhI0", "0TpbUK-cZ-w", "9tM_aLpNxxQ", "DkwiJYETf2A", "F2HdJK5IPC8", "FQVEUZCbv70", "ILbd8ORJZZA", "J1iE5LV-M18", "JYPcFa_Rrkw", "JmbTk9iVQYk", "MI0k7UDaXWg", "NpkrWFHNCNU", "VDqWjxUPdNg", "Y3AcOjU9GbQ", "ZOQ9ISlQC2c", "elfGtRffRDM", "faaoU9l28e0", "gC8OVNzdm7o", "iQYotyY8VCg", "pJ3J_hl1xjY", "q4UMk652CDw", "qJw8gJm6srw", "uLhT9FC1Wgo", "yMmgt3XbxxM", "yXIpp0hOS0s", "yrQkTYod95M", "-9EqaIP0018", "-dFnMw5KOiw", "-l2NnhuUrME", "-lXXRYVlL8k", "06tXMEZHc5A", "06xPuc1_xRk", "0AKJh9FTzxA", "0B6v9AnUtRM", "0IDQm2V7lrU", "0WKlstH1cNg", "0uQAIh-p5yE", "0xQ7vHUmYsc", "1jrDVC1IPuc", "2ACVNZlL0qs", "2LrNuagMmUs", "2U8IFfQWUXg", "2rMCT3wh46A", "37CkQ6otWeA", "38zjuVG_4zE", "397AHQf32-8", "3V5tZjq72-s", "3WjFCCeRWq4", "4Pjuf6j3vy4", "4Wl9ChuH6S8", "53KuTRN83e8", "5Jyc_SKCFY4", "5Q30ktJCFCQ", "5TccMzchmUk", "5bka-QZg4Ks", "5c-UugFDZug", "5mdcnihVW9Q", "64__AI_4spw", "6JrrsTySYaQ", "6jYyta-knnU", "709JsxZ8oXY", "7JIum5bKPTk", "82Djz2gYkq4", "8Al0ToSJyBo", "8_Wh1Z89PTs", "8e6F_y5Ik4g", "8lxqfBkXcJE", "9WKFlYXlpXQ", "9WWmsEqMY3M", "9moviGObRVE", "9t2n2lFUP00", "9tqbnfNY5Co", "A42iTJvJb8Y", "ARVmSICuN2Q", "AWsP5t1I5bg", "Ad2s6YdFhI8", "AsidsvBKy98", "B1pfEb2pujA", "B5Z3VmU1Mq4", "B6DaDPSBnoQ", "BHIC6FQjO5A", "BHw9Dy1tXS4", "BN4XiHTUnzc", "BNkYJZfVoOI", "BUXoRXisBnw", "BW9WBSqph2g", "BZ9c7_ePkL4", "BZD9p-W3Xds", "BZzyG-i-aCY", "BetGdlRBock", "BnVPf_rzOYI", "C3d_Ueu9_7w", "C7sJhHMB8WQ", "CEJSBAEY2MY", "CGz_agiEDYQ", "CPHsrsNqOAo", "CU60fdChTbc", "Cr3L07b-jrY", "D1KaK5ZenR8", "DmDJqHViEOw", "DvphFwud7JQ", "E0iVFNTRDzk", "E2HoTEEhRJ4", "EAq13vYyXEQ", "EVGgRRBGn6I", "EZJqAqa7WY4", "EqM7itbG02U", "FFLafsBMsD8", "FFzs08UpXDU", "FHUJWnn9mbw", "FgGWutSTLqo", "G3T0_1a02hc", "G4T4Mk-Aatg", "GCJpwCfAP6g", "GKl0ICn_Y9M", "G_OSf4jIUAs", "GeotEJhzAQ4", "GtxRF4tTspE", "GwPbcLoTGJI", "H77u4cVtj0s", "H8UfdNbvXcQ", "Hgiw68hfSfc", "I-NCqGUr8dk", "ICt3ykTS5EI", "Iba3kjzXDag", "Ic08MJVHFLE", "Iq_tLwVJOQc", "JBdZ-AawXqY", "JHBr5_Rd174", "JNR9Rf3La0w", "JS5JhKhKUrE", "JUZF5WyteSU", "JeRlb6rSXIQ", "JneXqkLKumk", "Jqu9DkzDRw0", "KQkY1W8x_p0", "KWYa-1dlr44", "KYfD7z-lzKE", "KuqexVjE0UE", "LeedBsMsOA8", "Lgt2hsSkTG0", "Lm9wrh0C0no", "LrOKVW5272U", "LzYZeKRhhHw", "M4KUegRVC_M", "MGonZBVsSp8", "MYHIr9UasJI", "MySLVMMiPBk", "N0kDn4eX38g", "N6sYjkj-ZDA", "NDIH-_Cv1KQ", "NFe0UUXlgrE", "NOswSjrhWSE", "NaGxeAbdJos", "NoS426QjwF0", "Nqom4lRar-A", "O2e2_FbOIRU", "OAanFInwnL0", "OQNsPXFehKQ", "ORG1aL-jYZ0", "OTnX_TBpyXw", "Ok35mphBlB0", "OvUeMni_EtA", "OxsSnmOZhqQ", "P-By_v8p_dU", "P43YWib8jY4", "PH853n9kV1M", "PNE9blgnK9k", "PSf4ievcFDg", "PbBQBHV2CRc", "PgsWlW5cztM", "QQj2bi98SW0", "R-PegjSeR7s", "R1ooT7JxR7s", "R72RzsXbCV4", "R96nd3ePv60", "RnW6kY51qLs", "Ru2QlLI2AlU", "SaMm2I2LDF8", "Sp3zpGFkmjI", "SqcuZ-XW0cM", "Stexuhn_baA", "T2cuR0TL0Uw", "T9ztUzNk3YQ", "TBtDQke5Sl0", "TIldKNg6ozg", "TLI80kKS-Jg", "TzILXLtS_U4", "U6bJnydMr3M", "UPZw6FZMDnE", "UV-c__Vsb-k", "UWNdOrRh7Rc", "UfGVR2RvOtM", "UhYcTnaWcQU", "UjA1cDCVXVo", "UnIePDIRXIE", "V5ORryIG3Ww", "VM287qk40bc", "VRCMaQiHHqM", "VXXgg0PgzKc", "VYH4E9JRSJ8", "VZM9NCpnZ9w", "VaVRGAy5x00", "VcOTj8LgoAQ", "VhmSEY-QErI", "W-p6hjR8NmU", "W0x1aybhvps", "WRxGozcWBHI", "WbmX88hcxzg", "WsmvulbWvt0", "WwhTThufck0", "X1Kb2vgcvE0", "X1SEEdYqp28", "X6P9LcfYjxk", "XAGxzbn14a0", "XBtRgIa0oDs", "XCsxhfzxVGc", "XFOmkbvYvlw", "XQRMJ59u6eg", "XZxOu7L2yvQ", "XwmJzzpYNKY", "YJdcTBDomEg", "YOgmJDHGPJA", "YUDvLHLxLU0", "YW0Ve7AjT30", "YbQbOTqzARY", "Z-CIxl0__mA", "Z3Vfj1aqANQ", "Z9BUrS-Bbaw", "ZZMro7dzQAI", "Zqt-eafSiWA", "ZvCQXnG4Kd4", "_7m7W390Y3U", "_Q2ZI_EJ_1w", "_b9C5mX39wk", "_vJZfaGl5sg", "aCNldnTJw8M", "aFmn8GIWd2c", "aGktlp9QA98", "aJmcT24ZuNc", "bBzI1Btd_8Y", "bUwOBvYRI6A", "bqYBlHGnldI", "bwhX6X5DoaY", "bx8D_U_8Fpc", "bxEcvFCbD-4", "cD33TatlFXY", "cQERfbkstMA", "cVqu99Dxhng", "cYJvYHJ-ooE", "crxbsS1_gKA", "cwQUt_GgYPc", "d9A0XgjzyuE", "dmSRqLAX-rI", "e5L807wfDH0", "eHI3wr6fXFw", "enbF3E2OWog", "fUFsuNbT6ME", "fYqx1qrAYH4", "fy7BkYWPz4s", "g_HGC13Bi2A", "h79t-Rp7xgE", "h908FGQsXAE", "hDoiyLFyTgc", "hEenAxOlQ1s", "hQHQ9UgMGJw", "hWdi6sP7QBk", "hZ74Re3gYig", "hfl6-22Gc38", "hqByQfgKDg8", "hz4noXJLgQk", "i0lirijXmZE", "iFeVrXnGWH8", "iGdyVCNgW5U", "iJhLFzz8NWw", "iYw7Lfw8v3M", "itpLvPgeylI", "jBmcNPHgkbs", "jJ1pyu83Bmc", "jKJAWG7S5zQ", "jR3qrKvrO6Q", "kKvZehFACOM", "kOloZVmxbsU", "kq-BqaJ5nH0", "lFE--jKmq20", "lPKdtIjF8HY", "lTIAwgndU8s", "lXJ-HTQ0rws", "lwvMg9Nj_Mc", "m8uTskX0v4g", "mQH9iUkcreQ", "mQnUD_xFyA8", "mnW_1XZomWg", "moMPJOQ8_R8", "n1Kr99IrECc", "n2YT1BVd5ck", "nBw3mBKcQYg", "nIXuYEQTcd4", "nWqc4EmIJtI", "oGAtNAtFrwY", "oXfZBP2D1D0", "pH08OVFkCuQ", "pi75NWhae1w", "qAivpy_Su1c", "qEzccbsFAr4", "qT_2UY9EjbU", "qcYff5A1Hws", "qjTDUDe8nlQ", "r-G2rA0CL2k", "r2OBepKnN9c", "rBFZcEkUS0s", "rH_v047yaPI", "rLzHfh6eB94", "rf6E0SzB79w", "rmWL0e-Y2NA", "rocCrAOJLKA", "rwdmlJsM8f0", "s2AdL8TYpgw", "sAaeMjbYPM4", "sR8uEGqk7rk", "sUEqoy2n0Ls", "sao-lUYPlxE", "sc_rH7Y9Z1E", "snA_8GtnySw", "tPVJXIPpmiA", "tYgXboS9Sfc", "ta2CBmevze4", "u6sNmbnjaWY", "uUIooXEqHcY", "uV2hTmNVzMI", "ui8ePB1D6mw", "unBvw7yRZq0", "uxjzPRqc7us", "v1HZc1Zefjw", "v3oLD6a_JwE", "vITHmzs-hag", "vmvd758chac", "wAeQ3hMPq6I", "wG4VZrcdmgA", "wG7DriWIntA", "wUvWjSY7TSk", "w_4rdp9g6pA", "wiOlsGo0CIE", "x16Gqa73-ts", "x3lVts5wBeE", "xH7a406AyDQ", "xMy4mGXRRjA", "xTYm8W5CSuk", "xjDgcXO9aTo", "xpMfgLq0wK8", "xwhRFcr2rks", "yPRVAnOC_UU", "z0SCPRBCv8E", "zSXCEskol40", "zaGFPg9JOTk", "zc_ZLGb4xO0", "0-2I7V90UqE", "4YzTI2rK61M", "7XLtpRWZgNM", "Hcg9vZKri10", "I4FBv1H-HkY", "IRKBdwen1kY", "Mty8MEVWLAg", "N6DpMHfVF2o", "Pq-MjZ2Z6m0", "QRnb1vqH1yc", "VvARvOJ1Q6A", "Y-5nF8tkVE8", "fQmMpRQnn9o", "fX7GyraeNaA", "mhqnbCXVUDQ", "nbAj20DMS1c", "oH8-bNr4Al8", "tbLls7XPxWk", "OyfvQhF3fOA", "0lLYNchiuds", "1bNYzxnphxI", "1hsAmUPZUzk", "2QEtYTt-zLU", "4iqP8n3Hq2g", "99iJ_TKFlNs", "9burLneU3z8", "9o9c6Nu8Q_Q", "9y2KcIXsuSE", "E5qLYEU6DdM", "EmURzpnjkoQ", "FxeIJk5cos8", "IdJVtxXaRGk", "IhkILT4TDII", "JWpDMDpufdM", "Jdh6_gPfZZg", "Khc3HuvReS4", "MwOUqXw8M-8", "Pt-oBbs84TE", "ShfF4uPLzDE", "Sx9SShHOYiU", "VLwppqtMjhg", "VnF2g_LYOac", "WFqTUyHV5RU", "WLRW3KFqlkk", "ZS3zHxS8UqM", "ZsDXNK5iFL4", "aWC4szrEzkU", "divwKHeEGOc", "e-Hrh2ljJdY", "jms5f8wKGyM", "k85bd2q_Kfk", "lv3JUoIIEs0", "o2bQSWxUi0Y", "obIWUfFBf5Y", "pOhrWQQnfHM", "qOfrVJrB0J4", "qsmURXIHQlY", "sV8X8GTcDEI", "sWohgt1Tcfg", "tZ1oG8am2Ig", "v9QX2Bfwn-o", "wjf3APe2uYU", "x0AHqedy7f8", "4htaDV8db5Q", "FP7YO-zU6k4", "kZlkgjI_CW4", "08L8pO4sPwc", "0Cb5QfJfHk8", "0E4Ey3GO8i0", "1q1qK-kmKHE", "1sPSBc2TNt4", "25bZhl57Bas", "2Buxk9lPLxw", "365PVsnajoY", "48d-FRpcMvU", "4rWbi0kmG94", "4v2hTtoPgwE", "5b51fEX0sRY", "5m7KKslyeWU", "5mDcSuAyZvI", "5sTjpM5_mwo", "6Z9J5lk9RL0", "6g7zq5-zoYU", "7yBJOw4NR5g", "8m0z91uhL8A", "9G4UA91v_JM", "9dI3wp5Qudw", "ADIArUefTwE", "AGjh_OkMczU", "AsvNYaVZzyQ", "B7oNYkSfP3Q", "BqLYhDXWRQw", "BwTsTijDkGg", "CZ-EHSD7AmI", "CnNLxBjLP1U", "DYTL1Qo1z3k", "DzWgBr8k8FY", "ECd51UsP7OM", "FqoKdDbQkb8", "G4uogmoitOM", "GcuxP038W78", "Hf2x8Md2gxc", "Hp6O4QmLPHI", "ICPydA4Y5ro", "IFbxNKMITbg", "JAc3hMtTzyo", "JdI0uRcjxZA", "L4dLET-1eSo", "LAkUnfFOoYc", "LFSZJEgmqfs", "LO5dMwWN1Kc", "M-7dvsBoKoM", "M-px7a4yDO8", "MlOLjD7B3oA", "NAdy2TfdgDM", "Nz3hEwMbnDs", "O1bFlL3nPo8", "OB7aluNjtoA", "O_fB6sSGwYI", "PCOUNX2hUeo", "Q-RUFshn58Q", "QEVswTWDczk", "QFHmpHCEp1c", "QPirUHu7eHc", "QzjahyZC8S4", "R4rmK_9L-XA", "RJNzsC861FQ", "S5aGOydLNI8", "SYePYwf3ExY", "SzFSIqwWmRw", "TCAdssVnmsw", "UDYYacN0SDs", "UhRaH5rH5ro", "UsWmWxm-JCg", "VNNnM7sNdAA", "VyUU9STxVF4", "WGjMx0EinxQ", "WnVbvvk286Y", "XOPzgoo2_l8", "XpUhnGo0lac", "Y5HeZ8r5Ejk", "YZR0bG3IQ9s", "_CENJ96y2vo", "anb5A1o80_I", "bTcvRR0Jrz4", "bln3Dj-dVT4", "bltCijNqs2Y", "bpvvgwchDKc", "ckfrxE0NGcY", "cp3vnSFNN64", "cspzRlCK8wQ", "cuhGDHCxW5M", "dYn3IXC7mU0", "eXeCzGYB6Y8", "ehpJIvjivEw", "fGW-3NlbCD0", "fI3efUUYguM", "gU2AS_4VnW0", "hJyLYoWBcPU", "htkWVP8aejY", "iF3Rba3Tszw", "iXDsCciPEb0", "jV4nt7Misr4", "k7HmzikfmF4", "kEIZ1n9Q-rw", "kRnuCCqDEyM", "kg8TbVaD230", "ki0tJCvCNs4", "lgoFcaiDPVU", "ljg7MFw6Ops", "mV2U6ah7Ht0", "mxb9SH8q2Us", "nFCgS4hlclY", "nq8mVPuAUeE", "oRl4m5u4muk", "opHvi1ZSca0", "pKg_wZo7VEs", "qAF3ONhGKt0", "qCfbrgts0fw", "qJPvDa0wPQg", "rrxQLrBGvM8", "s0WtyM44OB4", "sZ6sBELD0qE", "sZVYjqA2VPY", "seSvUkzfwf4", "shxiQmQTM4k", "siyY0up4J2U", "tOXMr9uiHiw", "tg5V9t5_-vQ", "u1e4CfbImSA", "u_Ssiwhhxy0", "vblzQ5RnUJs", "wK1OI-s6YT4", "xAiLQsvE8Vw", "xGnZvpm71Fs", "xMHAa0DM26E", "ySifAppq37I", "zVyM4v1tCjc", "zelne5LNFGg", "2K1PxRrGmU8", "5V2ZSiEXwr0", "6NxtL6QdJHY", "8T27NiI9CNw", "LGVUacG6CBE", "SW7edecvqPo", "TEYpc3roX28", "UdK97kT2NsU", "iEGCec8tCqk", "-gi1pwya3jQ", "-hS8q3x3x8k", "5XAyg0-YHKo", "D7SDavUCTJU", "Ka0iTXmrdrI", "Nhuqa0IE3T8", "XQLHZN7ANd0", "a53kK1IkqZg", "k80l0754fr8", "kMJS3w-z-hM", "lf2B6zgvyBs", "mRFqHECrhn0", "n5pZHo9t4qw", "oVo5PVpMNkA", "q_RLXQFeVnk", "rPAR0UE0nX0", "xqRnT9A7drk", "CuzPhd6nV10", "EFOdCVOVu5w", "ZZEJeTbeIGU", "bSlS8QuehfE", "deYu03nBgCI", "-0UQpzQaghI", "-A0M35Gf-94", "-JfMGo4Nymc", "-vbfpW1-B4M", "0EvCjI36RJs", "0F_q7QDVdBA", "0Vpc1izRP00", "0Y7O6MTSZXA", "0_BPMCYcl8Y", "0fJ29wqHQyU", "0jZgjmFH2Z0", "0t0TbDVx-dA", "0wp6YRH_O4Y", "0ytPDb--9jY", "1-_cw66Y7S4", "10GL9CQW0gc", "17oxsl4sKNc", "1BQLeUgbJyc", "1UJDnZ6-m3g", "1lm0BRzxd8s", "1q7Vu-0cEs0", "1wVuruSiudI", "22Zri9taNIQ", "2J6jjHvo8Ek", "2VculWlQYXc", "2aQRF47hJa0", "2dU2_WtqyjM", "37hqI7sqkXQ", "3_Ykx-7_6IE", "3cNzDY0jxPg", "3cmfo2uQ1S0", "3gA14T-IggM", "3nM_MsXdm6E", "3tguD8At-zM", "4O3lhbixJZw", "4kZKgrM8ka8", "4wRior4Wkfg", "5YWKuiUZ904", "5bORp7ACqqk", "5mQ3MQqvUWg", "65hm7eudStE", "6G9vE5-hHdA", "6_GStBm7_-U", "6kj010y5pmU", "71IQU3NK4NA", "73_sNlLmGRQ", "7MEIhoeLUKs", "7c0jQBpiKjA", "7hxtXftJitU", "7rYft5TYkSc", "7znjwg8e_Wo", "87lA5MAY9vQ", "8C2wn1g8UUI", "8MODoozycQg", "8bti7y2YVf0", "8s0DRk-YJL8", "92_VzvbDa1w", "95eJeHCYYF4", "99bj3paEZzM", "9K_E9-yj3R4", "9KzEOaG1TlQ", "9OXGSe5Ijyk", "9TOQMAQKYls", "9bjmb4B3V1s", "9o5YJXBbERE", "9pIIUz0Tevo", "AAPul1F6a9A", "AkXl6rm5YJw", "ApqOK1bHCG8", "AuqbNouPH7c", "B4TtUKT3XQs", "BE790lQlOlg", "BIbDKG3Hb4k", "BYvtflXKOc0", "BaQmbLqOszg", "BfxegnRWBrw", "BjWA-g9XY3w", "BvRv3TQIKOE", "Bzz4DNd26E8", "C5y_y7g9Bzw", "CTpNKSMHt54", "CfAHYQlLMWg", "Cz3GmzKK5ww", "DEa02MGSFRk", "DRWotbIOg-Q", "DYIXkNwcHyM", "Dw7bx8QW32A", "E2tJ9SGH0jc", "E4IEsbcatoc", "EESmw1SvsF0", "EEtU5OmUVkI", "EZz57KNayVw", "Eb_h1B1IhnU", "EnmtDAVpNsc", "EtfUTWoQai8", "F1l5m9H6nN4", "FM9DU5QnZhM", "FO1BH9Xj2xo", "FU87W5C-yCM", "FkZLJJalL9E", "FrGM0y3rTl0", "FsB8hvRsOiQ", "GI8Qbxh7PD8", "GNwBzKD3m9w", "GRlfQ3q40c4", "GZNa0_6JsUs", "GaOg9FBYnQM", "Gz-pHaoT7KY", "HIRUemhlPO4", "HOeyOgxUyPE", "HR9Gns5VmiI", "HU06aBdkUmU", "Hhhup0LOI0M", "HuUHKzBH3cM", "IUV_DxB1NEk", "IXYCDEeqKRE", "IiwmVauxDS8", "IlUxUkIQcH0", "Io2J3XhRu5w", "J0Mau5TTSvs", "J5MyA0v_l0o", "JCulGRRkUhk", "JFptlTySDHk", "JKCiy16Ezoo", "JQ3NjWGDIXk", "Jo9L3Axsus8", "K3u9IZHeCRA", "KASgnQXLZrw", "KJt1Qdz_xgE", "KUWQZXLv-9I", "KkbJa3F45SI", "Kmvq_1PK2Tg", "KtHzkTtHMZo", "LArFL9WHQDw", "LOSwiD70W_k", "LWnWc9A2UCc", "MPaUx8CoQwE", "M_VS0ulDaw0", "N-yU613ma1w", "N2CmXg4nGWY", "NBRbwTXUW8c", "NP9OU8exHhE", "NXb7ZYFHAnQ", "Nh77itlR5CI", "Nkzp72LYMOc", "Ny2i-Rnjfys", "NygbKcWqtew", "O2ptzRZTgyc", "O6fDIa1XHyA", "OCVAKVSUP-c", "OIqpDhOJGS0", "Oi3Z-rXtVuo", "P5rRR__pXiE", "Q4gN1f5HAKg", "QLNGUV4vi4U", "QaulkL-WkCc", "Qu5fFQNopIc", "Qzqm9fbDSzs", "RAxlvU42RNQ", "RR2EqhD6nlM", "RcOFt6UjGnM", "RnBP3mcT2bY", "RpoBTzryxkc", "SCAJbSnQZXs", "STbAERknA5Q", "SkACUgk6wFw", "StKWQkU2jLk", "T1XbGtZtFoQ", "TBUOk1reC-0", "TI6-Kgjtxt8", "TN5t6FWzZ7I", "TStVxdtz6dQ", "TpuFEOr-4jg", "UC3q1cl8tu4", "UGBQ2pBEVCk", "UQSU0RrfZsI", "UluGA7vbTCY", "VE8E5v7Fh4A", "VFAFohRT7jA", "VO1EdoDTJHk", "VQwlCXIDJiA", "VW6vp7YRvHk", "Va1tfDUjr74", "Vl4bN9vAgxw", "VlSwi6VsLMA", "WO0joCvt8UQ", "WU9L6EQwmyc", "WgQ2YZ6MxIY", "WrQEAbwBxU8", "WwAOCW_QTHk", "XYTld86u45A", "Xh42n1xCmGE", "Xh9D3kmhuIw", "YOLWC6gH8qI", "YZoaGy8dL_0", "YlANLHpqLlE", "Z22SYbGRGrU", "ZBXbnvHS_0A", "ZQsYZC7h9dk", "Za0v00PzuKI", "Zl-MNlp9pKY", "ZxMCTwtMfL8", "_POYkEwdtBQ", "_XObcDe2-W8", "_deCp3rIVM4", "_k639ByXIyQ", "_oRmkUCP7wQ", "a8RyBQE3g74", "aDE324kZHH0", "aGbqo-bvoEU", "aReAiYj5ucw", "aSRv0ztpOD0", "aZLVqxAOdKg", "a_DhJdBP3U4", "aeB3vGlkuM4", "avf_akrp6uE", "b08LsVKheQI", "bXDSIL5AXcU", "blD8-92U5ic", "bsAdH3A13s4", "c9Jh97jiIOM", "cJ0c5p5N7v8", "cVf1mkPvFHY", "cddMuH3WYJk", "cgOX9t0mysQ", "ch_vD_090jk", "cv0H30sEOgE", "dVOnE8ZhITg", "dXreXWyXR-o", "drutgMvJkJk", "dwd7BFOcZWc", "e2EGQD37W_g", "eSrasMIzbSc", "f3j4rKi5huE", "fMRpbdFA-U4", "fWiXpjxAB1U", "fe4GAACOous", "fv-IanWTDfI", "fyI1WkOMXto", "g23SyKxTUak", "gF84qFS8Oho", "gQHI6xTzgh4", "gTJ50suNxpU", "gi4lMaWy9O4", "goRLWldZE5U", "godQKkrwjpc", "gyrbvYaYwm0", "h0I3-eRYmQo", "hCJ9gq-u3Pc", "hT_FrFuL4kM", "hf9--LynOZY", "hhXT0cE4haU", "hrG3LtBIk74", "i7C6Dkh_mgs", "i9_wE3sQ4Zs", "iLTdHasc_go", "iaCsPw3RmkY", "idbCNY4rKyk", "inaGuXV0s1c", "ioMo2iJO4pc", "irxxgz6t3mo", "itzPhku0vDg", "iy0GSYAIxy0", "j_xx37PZUAE", "jjhhrs6EfeQ", "kIJZe7Em9OI", "kLMe2iT_UZg", "ksc2I_d5XcQ", "kuZEx3n9RSc", "l6_rL3oMfOM", "lGjGcxF32Bw", "lM0xtKiu4gw", "l_G87DjGuOM", "ll9r1bYGN_k", "lt2IFNaXadw", "lvC334MH10s", "m1Njcx6c8Mc", "m1zQe5wRNrA", "m7fuJgsqXTM", "m9T_Yel3DxQ", "mLfQ2RI2qoI", "mowelkPcisg", "msPU7xGjEPI", "nFWZZT3d4No", "nN4RccTVFF0", "nd1FOR59K_w", "nhh3Nnmpz90", "nz-IiIi--0o", "oK6SkGWa4HM", "o_KbE5kNKBY", "onwleQz8YNU", "or8TtuFI3-k", "pP2q6LPYsXQ", "qJ6AjHE4SLk", "qZhePKhJ05o", "qanXvcskvM8", "qknS2tEc9qg", "qyrVBm3mMoE", "qzr2X0dahOk", "r0UV2b6ZpTg", "rCYHNslnA98", "rSktAbYSkms", "rTmJH8bwKOM", "reLwtqZQWxs", "romvVIfeWaY", "rwnEIWJG_g4", "sJFR96Yawxk", "seG6kSX5A6g", "sggJqZzEJkM", "shVniVvDoB8", "spTPczcmwWo", "spihWIlS6_Q", "tDKrXG8XuJo", "tIdZ7cwWTfo", "tJ_0r7BQYpQ", "tUTWHwMHHIQ", "tmi1Uj2GoTE", "u0NYLqX3x5E", "u8Nx1-ZKlwc", "u_huNb7Jp4Y", "uaQa6plzIxE", "utYGqqQfdIo", "vACyGBET3yI", "vCkl99Rw_bY", "vHEkwgELSf0", "vSuo46JkUhI", "vVcOmmGbhJo", "vvs6M5fKXpw", "vzn3_Oi5pr8", "w6Zgr83lHcM", "wVlxDzfEoJc", "wWDPN1_QK_I", "wZuq0ivEIVA", "x3rAwKekdXE", "x7TxEn99wJU", "xVCPeG6phqk", "xcX1EfiFhME", "xcmTwlIg6H8", "xf4V7L9csG4", "y1ueWz4-Hkk", "yKyvb5KgzxI", "yUMClMdg3JA", "yudOt4xJ4xg", "yvPowvrxaWM", "zdDiMHfKVIo", "zmQyZ9sLgbk", "znpvCw7YSpE", "4MYclhN0TwM", "BaHDXwLEsLk", "HAOnX0q_k4E", "Q5imRimWKmc", "aAcXs-l8BNQ", "cF1fQkObIfU", "e7IwVBf4oAg", "fc5YVStlYzo", "hjuFIC6CL58", "k7bcmOeYgAU", "vI5j3iJVlF0", "wzHiOUL6988", "-8DQd-sYjZs", "-95gZPk06rI", "0HQ4d9WH_Ss", "2ayYomymrP0", "33jE2OlsCqM", "55wkd5RpVq0", "5mspUVTY-yc", "7meWak4-D3U", "8VfGv8Wrb6o", "9NBI1CkMdMA", "FnLRtp2rnGY", "HOA-_zOIhRE", "I821q5uGlg8", "IB0BFM0kd9o", "POpt-NjLywo", "PtaTq54IHCY", "RaU42SttV9Y", "TTtatDZJgEI", "ZBTIaUBy6Ug", "_9FctKvzbMo", "dotyRC8qTtM", "edJkYGwFkHM", "iEYgfiC4ZEc", "ijJ0aSDhDK4", "l406U2VmIiI", "lBOu8HECMyQ", "nTgLA7RgMb0", "oXu6yIDzYqc", "pC3LcCI7nCo", "pE1XI_1bzy8", "pvkXnUWqbgE", "rFKfL5UxdJE", "rHd_alcCPag", "rXIOs7e7KcI", "rywDegUQHX4", "tlMmgLx9ShI", "uPxdY9O8j6w", "vwkMLkzCIvc", "wRxOgIhkrIA", "wcAkw6v5X5o", "zVTPWx3tzsY", "zi7hNxR7iw0", "rjDr-5xVfGU", "-C2mjqhCa8E", "4v90pMTn38c", "553tQocipbQ", "CUUZpKWfY-U", "DRiEu7meL2c", "G3eNga_DE14", "GFxR6gcCZ8E", "H4G0i67tRtc", "K88KjSkZfvA", "L67M4xSqjvU", "NGC8RrzJZX8", "PrdmF7A_Dis", "QdFRYgJ_3mA", "Qnn8FmMipdE", "YZK4msOQAl0", "a2gqe13mqWQ", "eNHJ2W7uuPs", "j3xxDScDUVE", "kqaW5-MCtm0", "lHUa5mNNUpw", "uJjsS3lc9co", "vgLZVLWSRp0", "zwMnRD7f1mU", "hK0pjNDf_Jk", "4xXReOV5jRA", "5YWBKzR5Ugw", "6OACGVTEPrI", "73tnMrjumuc", "Ah9P3fMYPuA", "FtXC-vxwdR4", "ORsaXUdPgaM", "RR_TOJMcgaY", "SKS0V91NsIo", "TvZ07Yl8J2g", "gWig9Sdfe0E", "jqHn_WoPd6I", "lBYSwbFR9eE", "mzwC2qBf_Wo", "oZ26kz2pmNA", "2fAf6VRZEFI", "2rVg_Y7rwTY", "3E6dWbyqr-M", "ALj_dUn1usk", "AyAWMcxKlqs", "HppplIIVGec", "IoDX6FgJgQY", "KaHfbgJrj_A", "M_EBD44t52o", "PVsLv6u4FHw", "QZ_Il5cjI58", "TPGibXQg_6g", "TXjuk6oM0XQ", "WEu_QQdiHZI", "aZjQ8hU_r9Y", "bsfX8uV9c9c", "coXSAOi9uyk", "e7N0E7Z_jD4", "fGLDWyHn6wE", "gx-xe0Gu6cU", "hjkNP0fphME", "iGK6lPzjico", "jeHwH2ws5CE", "omZ_mPEVk1o", "tFQh5i7btBk", "y3PnzLyvJ-o", "yA1zH5CP0Vc", "ySjYmd5Lvyg", "-OcLg16Hj5s", "0r_4h9BVrgE", "28ZF3I9HAL0", "2VcAhrGWWmc", "6jpXvdOPdfs", "7-stybkfluU", "8r7MP7zwUTE", "ADIDAZnBYzk", "BUkQZfyZPZc", "Cqc_5ENHY90", "DlXLnFTh0go", "GyC2OEoh9hs", "HY1Y1F5NBu4", "LaVf9upswu8", "MPYZHxdeTsc", "OMIKmHjUo6g", "QRhcKaHF1rw", "TUdfH6Aus4k", "Uk2T3hQI-UE", "UvyXIq3qJuM", "VHoKslXvP6k", "VxREI8WPdnU", "YXJkZkqbWb0", "a3QhQVRvEes", "aHKuzJ17QZ4", "cayU6qiMEHc", "mVEHlCLjm6o", "uXnAvfKPk0o", "urhxeHEpncA", "vdGBZwuAeNo", "y32Kb7haJm8", "z-W8RQ8zAGA", "-5r_KSLxYkg", "-9rcD0Nc7Fc", "-JUlgY_tndI", "1er6cEH4G-A", "3LYGsxyurAw", "6U9rN2r-CH8", "7eQWik0b-xI", "F20fhnzlQ50", "FczgAnq5v1s", "M2VvBaPAx18", "RpnjiouBphM", "TxGLwEYnku0", "UPOI8Y8FIJs", "UV2pIlhA0cM", "Xpy05nISlo8", "_bvVN4lTchc", "axM69w5ZDpE", "gW9Cif0vc3A", "iW9qKqpi0IQ", "pakxHuOOyYc", "tM2xmrhF3n8", "v5a9gdR_BN0", "w2DO5rqwDMc", "wmqQIuaWBIg", "xKBKPviYlzs", "-9a7MkXPoNA", "-JjBQZp2SFs", "-MPLynnL97w", "-WF6rxVoEvE", "-p4APRBGytg", "-qJzDHuuwig", "059AaNVNxXk", "09LtlJ4A-hs", "0Jvh0spor9w", "0SYEDTbHMos", "0YXU_hV3Hjk", "0ZFBqGwQmU8", "0aYs2y7yuB0", "0ientavmEH4", "0ikyZdlkVDU", "0ldd9f2pEPQ", "0u1AJwi3y08", "1DnV-y_mQ9c", "1IoQROtYvTI", "1JKK0pP8V68", "1RmI7Y1GqiY", "1Z6zPnufBXE", "1lnnBGvqmjg", "1ntBep2MOdE", "1qjufzkEtuM", "1qqPAHFsPVo", "1tQfMCnUxXs", "2-V2UgKJgDE", "2PvrIXfdBjU", "2hMswn-hymQ", "2tP3zCCdAv0", "3-p7WormQUY", "32HI6QDriww", "36bcshLBUrg", "3NV6o7WZ4Yk", "3U7yCsPWRLE", "3UVtKB2SzX0", "3pZCzmpza5Y", "46k9UUuENME", "46tQTZAR1LY", "4_xTNZzwp80", "53nA74MdKPU", "569G92Dd4z8", "58yWHw_uptg", "5Le5K0W_m2w", "5okZz0t-QfA", "5wceA1TZ8-Q", "63Ep1kOoyg8", "63p-hlpV0Q0", "7-Uk5t0LUDM", "7640EbVrYoU", "7GKVowX2NoQ", "7IPDoYTGEK4", "7OcvN71UbNQ", "7OvYaHxWEX8", "7QOD7NkXdCo", "7ajEsyT9ieo", "7cq4eXAqjMY", "7lg47IUnevw", "7sVeJG39YSc", "86Jeq0mumF0", "8Aa9S1zmfi4", "8N5bCKCLx-U", "8TilLhM0JCM", "8ZJzAW8rTCg", "8dKHHx0NduM", "93tyy48Fv4Q", "96NSH8rrJcI", "9LD7WjnqucU", "9bYV6STjE40", "9opjB65MddY", "9yAvwWmFb0c", "ADoW3S8Iows", "AO8Oj3BT9xc", "AUi3oiCb-w0", "AX4hPVJn5bo", "AnB3dKtzOaw", "B2aK5uQtxQs", "BA2auXeRxPY", "BBWjwh8TD4o", "BOjOntKDPks", "BRD5nCXi9GI", "BhpoIFZx1aY", "BvkWOmLvnDU", "CPr5FWxI8mQ", "CWBi13mgo4o", "CjwKALQvKVU", "Cp_-q2JFQ18", "Csiij6uIeYw", "D645NsATF5Y", "DBTufOELGLg", "DYObvQ7yGa8", "DgIPbUjtIpk", "DxE2fyo8baM", "E113xS7oRzM", "EGlVMLnEf_Y", "ENDusvF8El0", "ETukjk_XEGs", "EZYiU21fMbs", "F6Eo6-PYQdI", "FEztk2f45cM", "FHn9xgNt2zo", "FOIOXN4pjts", "FPLVNQFeO-Q", "FTkr9niP7wI", "FeGsRUfxLa0", "G0YXsVBuGN8", "G2DaW0jCItM", "GOQ5TUgz3Ks", "GYPdF6Lqy54", "Gn81-rBzpQU", "Gu0_xlDuMJs", "H4sEPaGR8XM", "H4vUK34teYo", "HJonN7kboR4", "HZ3Vl0skBfY", "Hc7C4P1-ceI", "Hf1c0YnLsqc", "I0fDRMMXTQo", "I1NPUI_3pmo", "IIamGkh0wNY", "IPSMiNXmSSY", "IQK4LeavAzM", "ISIO2MpjDCE", "IhhnEAbmlUU", "IsX-jkMfskM", "Iww0DUDOxz8", "IyyqAX8IpXk", "IzVAyl2A7YA", "J-FiIi-nlh0", "J7ef3breGic", "J8trF4T3aws", "J9epKzMXvZ0", "JEi8yquV9CA", "JHJDSL-5MoA", "JJRBpsviX9c", "JMZftV6qyCo", "JNj31KyIZ-U", "JSNka8vK3Kg", "JSieqQlsH8c", "K3wv-NZSB-Q", "K6V6vWRd08U", "K7017eNSUsc", "K8yNqtN8s84", "KEI6Ed8SW4s", "KVXaM7nOtHw", "Kle2u00w9KE", "L2tacaI_QwI", "L3QPfAnuJto", "LG_svIZ_E7c", "LL0o9CFUOFo", "LLVvyYkI5qU", "L_IGdoHV6GA", "LoKcIvuVktU", "M0M4CmSE52M", "MWNBC-0_y0U", "Ma5rA5Ia6J4", "MbCG92BZmQs", "Mfu3R5IakZE", "MiKLo-TgePo", "MikFMZ2qp6Q", "Mmx4LrRNkY8", "MptsH_GiU5g", "Mvp-cMZGnug", "Mxb3lCoU-PM", "N-kLqz5en0Q", "NR8TeIe-Oj8", "NU6jblqYklo", "NlubUlKhADA", "No8f6POdO7E", "NqksDyXO97c", "NuEH6MAl9Hk", "OATTq-SzMSI", "OD0hz4xFaMg", "OOeL4GOl2q0", "OUhx7dA1zVg", "OZ3iPXof0Qk", "ObZ4O4NqHyU", "Oj1ss_uUNlo", "OlE6ipmizNM", "Ouvcoh-UuiY", "OxrN6dwIglo", "OzLr0zO7Puc", "PAwn77Akl7s", "PDFpn6DsieM", "P_uI_Eh6NXw", "Pd7xpo6vFFQ", "Qa7ckml-niU", "RBD5X2FU93o", "RPVHHg5euvA", "RQyoLgv7Lhk", "RUdrP25QX20", "RXDlBa4SLq8", "RabU3dBD2dc", "RbUnl0ZQVtQ", "RhPr74ea4PA", "RiInuZEaIBU", "RvQEBY1-vcs", "RzMeRhCZ_Do", "S2AEXK9S3k8", "S80bGOWWago", "SCPKuiAMIe0", "SGpoG2mzzwo", "SI65PpEcb-c", "STmQRfgbHRo", "STtlgwFNjd4", "S_QkmZ8b2VI", "ScNGNBrOnB0", "Sn-1bBdSglU", "StAn0DJa_Dc", "T2BM8Re6MCs", "T3fSiFexkwE", "T7cSSmdWtT0", "T8tlPcDMdkw", "T9ptNmVABCU", "TNXW8l66R2Q", "TWP8GCgG3LI", "TeCUvYfaR3s", "TmRM1nRo7r0", "Toub6R4xDE8", "TvPKTF31H-c", "U6dVw4MSyos", "ULFoOd_BcYI", "ULTqb9xnuac", "URWMNodsZ-M", "USsWSMt4Idk", "V-Z-vqpDz3U", "V7hE2e0gz90", "VGTO2jOJSYc", "VTD5yCy6sHI", "Va-2zsbou6Q", "Vc8KpIt-P6I", "VrfvWQIq6dc", "VtBK5mwdGQA", "W4olwUOvy5U", "W6zLu7X8VOg", "W8PCh7O0fWc", "WFLTYBgZdEs", "WHxJJuD7Aoo", "WL5I6UYkEqY", "Wfi8fgxcF9o", "WxKmDJ3zkt0", "WyPW_F5r6gU", "XFx5unbsuvU", "XKENlqWAnH8", "XPgoLBCaTP8", "XXfY6O-O5Cc", "XlYuqlh-DRs", "Y4hj90xj4BA", "Y6R2UiYvNdk", "YljCWGYk5Hk", "YxRSFP4KcHk", "Z4pa0Ippjms", "ZCT3ggeuF0U", "Zb0e7cEMeNU", "ZsIWpsiwtOA", "_s3wGV2Rfog", "a9-aXkjjXtQ", "aAbJSJt8JXs", "aBJlaPWiNi8", "aC8es8FeLt4", "aRe33IjXPUg", "aVTlEo13EVw", "aaEX99gQPEU", "aiNym4O1Y9k", "alByS9QL4mM", "anz1CEmYj-U", "auKadYDL8W8", "awi8p7fB1kg", "b1JU7GHBz_U", "b2F5GjQRNag", "bI0896iB57Y", "bIXe8OMxXNg", "bUfnYcjnpl4", "bUisWDBvFcc", "bkPo2NJTWVo", "bpOgF-VlFQk", "cArwYv-4AvU", "cIcYODz0sfc", "cNH6hK6R_Zk", "cypRrnGEU-c", "d05btncQsQ0", "dJlsHHcgQGo", "dt2UUzvBojM", "e2Fm72KqNAs", "e4PXkmqmevg", "eBPMf706KOM", "eC0ZLF5tKwo", "eDhyH8JDVkg", "eEllWr8KjeM", "eF8X0NiGYAY", "eGAK6x6O4Kw", "eLdgAPbrPNk", "eWyafSN-sgY", "ewkhXE_nPRY", "f-4PTm5meTU", "fprWzt5ymlA", "fqnQcGZZjA8", "g1-W5KHk03g", "gB8_wE0jGpg", "gDFUYg30-i8", "gP8h72gsDmA", "gPHcKivyxxI", "gTGAoguPOnE", "gcH5crhbTFs", "ghcCrqnjxs4", "gjKakN6gWlg", "gkHDm5cNSfY", "gtjrk8wyH_s", "gvgHOyglQc8", "gxQVoHo6DoI", "h-ePAWqOzI4", "h1ObQ2k1y4g", "h40f8ePAlCY", "h8TZzxlhVF0", "hAH3wrigtbE", "hLCnDRJJH_w", "hakDNgdFVac", "i1RNnnqwWvc", "iLsFM0_1a-w", "ifiJMjGgffM", "ik5eIfaDLuw", "io8qKOC5U38", "jD85TvQ8FHg", "jIpkdmHteEM", "jW8wVzBZAjI", "jY93M6wCuts", "jbrexnRKsj0", "jf0IoDypGz0", "kLFIJUr_8u0", "kOIZP1SKeXo", "kRJK-yNNT1g", "kRVq3U2-DBk", "kTk7DfOsOys", "kUlDWEPwzl8", "kz9_E1lvsv8", "l43W6avQefc", "l4EbkFVDKEw", "lL5t1bpyMKA", "lMSfEJf45Js", "lUlQuKi-t8A", "lVDyEBWviXI", "lYPZGI-R8xU", "lc3Zj8Uo0NI", "lf1J4fupGq4", "lh8r-KrENac", "m7HZlmqFsig", "m96R23UmsPM", "mM4vbFMmdkA", "mRL7hhd_Wl8", "mS-IdGYAOcc", "m_fgP4qyrL4", "mdAd2iIIYcw", "mhJyM1j2UgQ", "mmcoJfNg5CM", "mqFkId7RA9s", "mzTHjAM4AuM", "nFNlFZdaAnE", "nReOzD1nNPc", "nZ9qO6z5DrI", "oBigQmmh890", "oIHUiu3bKdw", "oLS1Xn0fvNM", "oLfT90i1WSw", "oMXaIsQ7WTs", "o_0kD5Bzb3c", "of4ucZ2aTno", "onmG8F7Iot8", "opuNd3xvPmo", "ortq4q_YKAQ", "pDVLoRq0ffg", "pENmqPiQCMw", "pY_M9JCoCMY", "q4-zmF7vHFU", "q7H4VTeEsbc", "qQJf9s1L81I", "qVTtfpkRbyk", "qkH6PIi-fqQ", "qml0faKPxqA", "qtivSKpPg_I", "r6gJAsyYXsM", "rAYJYMXW3qg", "rMrIfgZiTYI", "rOfEHGZLFGk", "rvpFLNaqNHM", "rwXioEnRxRY", "s8K7EA57WPk", "sI74O3VrJ9w", "sU1qPe_5X-s", "sfOCFnq9BYg", "tAZbBMX5d0A", "tArPEwm_XMk", "tekl2Lrt2_A", "uOm5hlrjWMU", "uRe3Ud0EPvs", "utF_Is2wcCc", "vGhO3AwcENQ", "vTh37Gfyb6s", "vaHevckK6wA", "viyh3daQWjs", "vk-n0IIk94Q", "vnFqp9djjEo", "w5TsDVQFE_Y", "w5ch5fI-b3A", "wM7j7P2Opek", "wOM5ZUiXYvg", "wbL7kadcM5w", "wnti_7CuEnc", "wvB4iPBMeo0", "wzgCDf1iMgg", "xGwwg4UWOaU", "xPjR98OR-mA", "xUzBBdxWo-Q", "xYx0o9bORH4", "xbmywBZAmzA", "xcIEfIzYPEA", "xhkD8g2t8os", "xlaQheROohQ", "xmdJMy9PAa0", "xySUHLwSUws", "yFIB5DGdTXE", "yG-0fbiGrFg", "yXXYx67w7RA", "z2afO5ch9ZU", "zBu3tP6hpJc", "zLz5TjpW-50", "zPoti4vyg3k", "zTKlLL_95c4", "zVBccPzYOL4", "z_5D6Rj_Mb0", "zhme39DfccY", "zjLhum0XIck", "zoesK1WWyOM", "zwWIqbQ5OH0", "zxm7xY6NyFY", "zzJJc8hTouk", "8iGyJ29I8bo", "AQhkP43D0ng", "Afq8tZIwaos", "E3w3KcO0JKU", "IhSeRpb2CyE", "UP3AUm8GD0s", "VwHjiLNHELU", "fXmy1aBLbIo", "rElo8JI0uIc", "xMnJf1qE3AU", "0AZR2pW85dg", "2VLdEzQlP1w", "5BZKCLQosow", "6r2AX44Jbes", "7b0mZtWhafk", "93IVcKPCzq0", "9PozuentIK4", "ANfCywBSqmo", "GKunIMfoUAI", "I30UNnMh3M8", "ItRAgKYVA60", "LAgXDaP4NO4", "M46ugbYV5U8", "N4Glo1NmPw4", "NsV-MOs3Fls", "PBbuq1gyfbw", "PVcziwl9uxU", "X2h_PnAG6Yc", "XAge8tsqYlU", "c47Z0tRBXog", "d6pl3Kl5JU4", "hmZL_m8xCyA", "jqMcXl1xkls", "puWfWnFuDik", "u9lGlN7E7NA", "y-qrBqdSc-E", "oFVxIjn0rIY", "-a07GT74tHk", "08wIP5Uc4mI", "ADP36e_-2Xg", "BDVRe9WOmcA", "G9HAGsV7cRM", "PIcJLfNh7t4", "TSm2EhPZHFo", "TbkXyFr1S8w", "Uujxqe_Ph30", "alRQIBEUO7E", "bDjmbhG9rKA", "gujc2Hufcus", "hALQfutdiIo", "hDyow4pODlY", "sce_jHbJy5A", "-5Pg1t4OOw4", "-VrtOnsX_FM", "04u9NlFAbvw", "4OxZ8srRIa4", "5BlcZiR1SqI", "6ZBVZWRRj6Y", "7ONFyvyXnWs", "8t8wh1S5weE", "8yMKoB2A9GM", "ARpR4-iDeOc", "EfmFugbgodk", "I7qPDskU1kY", "KJI0L-6I0VQ", "OTr52crOgGc", "Qwep9bL0eJo", "SEPj3P0WfSw", "SYlan6Qb_qw", "ajh8WXVVYUA", "feki3687le4", "gHuFcyHpaVs", "iBJG3yE2LgA", "jRL6E80u6Mo", "mDEf8VMBsu4", "mG0W0v2hBUU", "nMEmo6QqiNE", "pCayytHaZTs", "paFukJXMc6w", "rI_M3OCQ_1w", "raDsmhKx-9I", "s8nAPIuMEkQ", "tgMHbDh9dCw", "udKysMAfOU4", "zRA5L0HHoV4", "3NE9g7NkMe4", "3jPmaSGUSd8", "7jXzgjPZjiw", "FD44ewRaBTQ", "SKW4RxkxTx8", "dQJtWF4SXM8", "ea-bgWiDJng", "gr7dhMsBY_Y", "qEvOO7lyblo", "quZGy1_miz4", "uWz2SjMDeu0", "ursCbasOJt8", "wu2naS_AGjE", "zlslOpw5OrQ", "-cdblmF-TOg", "0-o2-_tViA0", "03wyaYSaN-I", "0PBINyW8iqQ", "1OD_t2O6Gr4", "21G900lKUV4", "2OyEJ1sEZsI", "2Z6fZP5O6S8", "41IZAEaqsBM", "8ofq1UoFZBY", "99BHCb_fqig", "9beWvVKfJu0", "ALJOpy-kY54", "Ck6l4lhUEps", "Cy9fS_aOLho", "D5ZAP_EHqXQ", "EGinlB_TisI", "FjoqKiuH7V0", "JAquBrDVeaw", "KKQgWtDIRWc", "KgUtai-Frl4", "KtwFZ61k-fE", "MtFiz0tilTM", "Njkgva_k6fE", "OiyaaSdPW1w", "QI5qISKJHjA", "T9fQ5DR1fAo", "TCoMvZoE-Fs", "T_wI6luwa7Y", "Tc1X5qxmLVk", "Trhb8Y1su6A", "UhP14XXpwGU", "YtpkBPMBTPs", "Z5E7KgG-vZM", "_6Y-nZGT9bE", "_Wn3YpHTkMU", "asd487AczcA", "cKctO-dl_bI", "eoUMnmVlTXE", "feDinMgVA-g", "gCa1-0CZgGM", "hZq2fheNDlI", "j5r8KVLSraM", "j9KmueZkfVU", "kVFU8IXgRYA", "lIQBux1_8Gw", "mDJOiVS5t8c", "muC_edzFeDg", "n7iK_47UxFA", "rAI2UESNPHk", "rRne7BwpKos", "rbFUs5sbw3Q", "shy-Dd_wz-U", "tLCMpOh5FHs", "tjvbuTKS1d8", "u6_5ZUGMO_Y", "uKB8TBVEn1M", "v4s4nAeqT1c", "vcI1Q-_wFi8", "vpXXlCYXudY", "vz1zxDor66E", "wMS6dBEZscw", "wdMRphefSX4", "yMpvmk7csfw", "yuxj9sbnR_w", "6SjdNsrc5s8", "IpMLtCPMNtI", "PuzOLlEQfr8", "SlC0uevsnWs", "X6CVMVxXAa8", "Yfyymjm24Ro", "g-Su9nseHVY", "gYCWqEv8prs", "h06BBd7YJyw", "hCXnGeGWANY", "hvuAitdCPp4", "pEa-GrbALH8", "pISGTZSlQQo", "wNF8wQxSmLk", "xIzNC_8mKZk", "-06ZdNEwMak", "-ccaB7EY8f4", "4KqaIndshFU", "6UeZa7rzQ2g", "BSQr3rxf1nI", "KwnNFtnnblc", "Zie6MB5MWUE", "0apuO07Aksw", "1ajxqWXaXZ4", "2CADuLqkyHA", "3RxhC6xR5qU", "47K7SAc2J5Q", "5QfehUtPpJk", "5jTc4moBwnA", "6ewz0ntK654", "6ii9W2WFXPY", "9660zs9G4fM", "9O9OMdHWYzQ", "AMXLQ9ohLAk", "AW05ZCZX5iA", "An2psAd9cTk", "Ca_2FY_df3o", "D3FdjBaej8k", "FZjsE6rhIxM", "GKcXY4LgSes", "GMzZemepeUg", "GeeYQOc70M4", "Gvj1Xpta99I", "HeqYcVcj_C8", "HwOiBzyhfBQ", "J_CVaPsM-ww", "JeEB4lMf1C4", "JjXcISWKM4E", "K760yn3ziLk", "KbcsD8Suo34", "LFSQYIWDtAY", "Lh7IOLX2MhY", "Nui3-E32noc", "OBVPwB0aP5Y", "OwAXhVwFBn8", "PHRYZB7516g", "PeUbSrIUDfY", "PtOV9TIqznU", "RZZkUIiGA90", "RrWgXvIs1gQ", "SHv2f5woU3U", "SbhCkGZeb-g", "UNsqdiQvBoE", "W2B4lsFFzdU", "YgyyX2P9Ud4", "ZQMXipi3z2A", "_JjarfIW3HU", "_Pta_xjoT2k", "e7Jm9klwM6o", "e91g9McLkxE", "eKo25mgkB1E", "f8Ludm7gxr0", "g-J12NcKe0c", "ht99U88rWe8", "i-LXhMEAY7I", "kME57_Z-Tew", "lxU1sTbUAZU", "niYvSg5uqQI", "nsM8ShaLrpo", "pMKa_zhxF1I", "pqdpS8A331E", "qi8UgOBlSfE", "r44WszHQjLM", "rPMiQqamWJI", "rlTmgVDQwsw", "sbDmIZyRDlw", "skf1pqKjesY", "tBwOIglkYzg", "vuKkQ62_ab4", "wdtUQXxZOLM", "wvNbleNflzU", "zPiECWy09uQ", "pqKYY1vqYFU", "NZht6wLcNTA", "NaPxxKdOJdo", "0DvpjI7RtmQ", "17XanNdbl3E", "FXq7Ti9Pjj4", "Hwcwk62cI18", "df7_-2Vb_wU", "gERC0gYldTY", "iNmzZS03kxA", "rd1-JoJg1Ag", "t3ovlep9Nq4", "-ywNN6cd9Sk", "54jJLHQxRVQ", "Xm4IudkbDjo", "_BTi2Jf6V48", "_kWPTD8tR5Q", "fY-aXu99qNY", "0gUDa9bSFM4", "4AIqkTrtSYQ", "672UK6XGh5I", "8V5_YW4SDJ4", "9AhJ6zm5o_I", "E7N8KagF6no", "EJUOlieZbC8", "ElHuf-uaXLg", "KIKJ19Zgpw4", "KaTHjm_hSKw", "LPWEpZKkqRI", "LnEJGL_Dnns", "NUOhfdFnMSc", "NgKGLNBvkK4", "PVGPl8k7L9g", "PkxLUJw9yQY", "RLFu8jKISjw", "SOID4KfF6no", "Y_qH_E3Sa_s", "bCh08iz5EJo", "dB3ZIXaqctM", "dVPsAgjhVQs", "eWs23xXDIlM", "gX4N7DNARZw", "gcmuVoHl5Us", "i97j6yb-JD0", "iHqqXFfVKVw", "qd47b-0V6MU", "qnnpePkiLzA", "qsSAEHSQbZ8", "rIoqPBtiYmk", "rL7-Y-d7hF0", "s4ZJ4HIsmP4", "xWw-uU-cwBk", "yDb0Mpn0H_Y", "z9yYVhBhzvM", "zN5kQUDiwYU", "-9-tqID63Ew", "1kEHt1aS9ks", "81vZNoDZ1q8", "EVUelLWvn7M", "GGaG4WOKu6g", "Me_BAbwxgsc", "N07t1W6Nnsk", "NQDMAaZG3Cg", "NkK7MgPY3do", "OnF7ohdpsyc", "TLhYB1YO08c", "YPM2IciOins", "_YUZSLNJ7SI", "cCfHEHoeurw", "fa-LyfKvZVQ", "gRbEe6kic9A", "iDLlvVtONrg", "iV6C6TsVZyw", "j9TeIj_L8OI", "jepldOVx2vQ", "jw7P19E5irA", "k9C6lmp1r0o", "k_INg9MlabU", "msvmi0xwadg", "n_FBRQhn5ps", "t6oSWfB1EGo", "yc1S899Cl3c", "z-GggDxbCSQ", "-5IlYk4Sy-E", "-CQ7y283xvI", "-MjYaJdCgrM", "-RTRHn-ArK4", "-Ray9WDI8og", "-eDnYFBFLZY", "-y1csCWor40", "0ABErHeJgDM", "0V6SvHOo7OQ", "0YvAXvrfO5M", "0_0Kw2V81io", "0vyvkyiy0is", "0xq_s-TS15M", "10pGvQxiVs0", "1CdP1VIM9Mw", "1F4AqZVKhEc", "1HpKG3qObU0", "1L1d3hmPsKE", "1MZfywTQjss", "1N5UaFYWIMQ", "1U85F9BBlHY", "1X8GxYTQ9Ys", "1Y4USUWs60Q", "1_pyOxzKt2M", "1u0kIU_d3ts", "21relMydYgA", "2EYM1_sMQJM", "2Qf8JnKcmrY", "2acEt2i7N4o", "2ftFWP6qvRc", "31deNzgcfzg", "37iffmM-grw", "38J6uP7wOIY", "39bCWG7XKCg", "3AtasUlui_Y", "3DndRle-4yE", "3XWZGcuvBYQ", "3Xz7TNAH8us", "3ZH4sZC0ZVw", "3gGcF8V6iSo", "3igv0ajl-N0", "3kJ4gQU5WvU", "3pLJPUTUjzM", "3su9DNO9zK4", "3v7-4lTPICo", "41oHsl0fH3E", "460Q2H0UGao", "46I3ICpL9KM", "48oeURuZKhI", "4IdnwktXuRE", "4Z36kCgIMlQ", "4bIy43FsIQ8", "5-vRiMHdBBM", "57_vKcXidtA", "5QuwC1HPYAA", "5a_wGfxx4Y0", "5d_nq_9catc", "5fghiwM5bA8", "5peoaw4oyto", "5uqN4nmhRqc", "64PTNBDgSbQ", "6E6-RNIF7S0", "6IBPs5oKv4I", "6SHsbbDYQJA", "6YHPARdxz2c", "6ZnPAzAXTv4", "6c-kUqdCgAg", "6g2Q0u8rBuI", "6iZtQLcb1o8", "6oq63BsGsY0", "6pqBpCOzePc", "6x7w-s-7xvY", "70AbRZVvpfs", "7CXGDvzW5Ko", "7I3Qgsf9okM", "7PACM6vdv9Y", "7WmZ3UQnAk4", "7bsTqDNbTTs", "7j68NAN_kBE", "7jqrCdwZUp0", "7mVBKE7PeN4", "7uY-slsfnWg", "82mmq4s-ieg", "82pt3t--thw", "83c-Zh_YmIo", "8BMqmWL2npI", "8BbK1VnEg7E", "8D4UHxbN6io", "8HNtVMbZO5Y", "8N7Ugj8E_Po", "8NPKHwrZ8j4", "8NqyqpG81wU", "8TlJN_QRGmE", "8WW45G9FePI", "8fQ3WyE7zbs", "8gqIXdQbAQs", "8ungd1LH4o4", "8v1T-ge450w", "9XtKCF5OivU", "9Y9PAMZT1Bc", "9oBemgsG8DQ", "9xOKEAMn8Vo", "A6Fk1rcYMzw", "ADuM7ZMhvvY", "AGXeeviOFjc", "AI5ZY8EucK4", "AgspGNK5AP4", "AnnrTVzhrro", "AtGnpHve5ak", "AtWshIdmHK0", "B9-KSOL-mf8", "BKF9V-LwxTg", "BUEr6mrkrD8", "BXquOlTbb_U", "BetJwNRNz7U", "BsJYc6YTwe4", "C4kbGIvWsic", "C5uwg5I8Sb4", "CCciGYU6_WY", "CJlhPk_ZETY", "CdPs2io2nOk", "CqLqaRl9YKU", "CrmrA7p8DLI", "Cu8WE2VRaQs", "D8ZlN5cpE-g", "DCb5N5xUu-c", "DDBda3EvOsE", "DDRApgKxvXQ", "DFxtnidYqx4", "DUaGiisOehg", "DbJDOGoRIek", "DibZLBscyhQ", "DidvxGg3GzU", "E5cZxDev88Q", "EKUaFNvlC_w", "ELL_g2iPRFY", "EPLnaLwJaIk", "EbFMnVXf8Bc", "Ek59LIf99JU", "EkdTHklnTxU", "F6EhEesK08w", "FBYIb5VV_T8", "FH-Ly3dJObM", "FHEeqjoQ9UM", "FJHeKRloPrc", "FSmKckOc8eg", "Foi0wPXYnzo", "G-8X7fI0p5A", "G2qN906-lEY", "G41Jna3cDPY", "GAVocUUy5YY", "GFvLMTI-DTU", "GIGqHacjhSQ", "GpYSlEtgzmk", "H2TFmqurm-M", "HHFwFqFCoa0", "HINppTOKseo", "HL8YrJ12_5U", "HSI83xYbrp4", "HSeeWz-yVo0", "Hb9ac-bWYaY", "HqXK0nBM9wE", "I1ZW2jbD85c", "IUGEY0U8Mwo", "I_Bl9t3X0Ts", "Ic4Qrj1cC3U", "IlOypcq7Fks", "ItkvRiT7Kqw", "J4PGXw8jnY8", "J9eVwqQNWJs", "JFueiqcgp-k", "JGndcnOcyl8", "JHLolj8QNao", "JOixc40qAAE", "JXJMvSJUhA8", "JlDgRmw_hOs", "K44-OxNZqmc", "KakqoIwLIcE", "Kb4Nbx-9T08", "KdU7TNsgHnM", "Kn0OoA8sSHE", "KuujwgNn19I", "Kv-rzc4q3l4", "L74kq7wzX1M", "LbNWKgo2pN4", "Lg48nzzSg9E", "Lh2cHyPwO0s", "LirL2SrLmzc", "LvtUfRMHuOo", "M8LuSilZ_go", "M8UV1bX08Yk", "M8_unF_BXk0", "MBfDxCOvnF4", "MK1Z6Y0q7vE", "ML56H-D_IOg", "Mqht53agJuo", "Mxv73vBCU8Q", "N-1RUkxg3Dk", "NHHswNYdBgc", "NNqVnOvv3ek", "N_cYj37295c", "NcIY0ch89_s", "NisSSpcesx4", "Njj7Eqxoh1M", "NkfVhNd1nkc", "NowNFUja7Qk", "NssVJD7NADQ", "OS_9ic2Mhaw", "OlD1hK7kszA", "OnOWBmiEKrA", "PB0VMfHxOEQ", "PhRi2P5ErtE", "PmH-NmFoaRs", "PrmMEeMXhKw", "PwNrwcpOlEI", "PwYf5bd6loc", "QCCxPoNQC0o", "QEy8KYZXUEM", "QJmfPnP_SYU", "QSYwzFP2jKg", "QTXYYJlw1i4", "QxW8bh0YQ4A", "R7cp33m0fmg", "R7uCXieqRh4", "RT1FEkyf-eU", "RT2hlAWR3sQ", "RTSEzeD8JF4", "RU-phNky4NM", "ReFjXtJdCdk", "ReLA3v0Pyl8", "RoUVrUEyBH0", "S1aYnYZjlN8", "S4ue6hTbRAE", "SAIJkkFGX5c", "SGXe6m5NB2Q", "SM77Y5Lcum4", "SQdKhncB88I", "SV-C0x4QbEY", "SY2XO1D4iHk", "SpPzUWPuScM", "SwRfFtmX06g", "T17m-wvUiTA", "T6yoiiSi1ko", "TBFif9YPT-w", "TKgh5Nr7sr8", "TP_Yu3a8DG0", "TUg46zAZ-3c", "TcTpYRP0oEs", "TfS61FDzbo4", "U22JM40DRXc", "UGUnMZudZMI", "UNDkgURkNIc", "Uc7F1Q4VQ0I", "UqDKD4ErVOE", "V1HgPu4sU5w", "V31TBrG05ek", "V3eC6bWJaDk", "VE92KJLqAWM", "VFICSTiuZCU", "VLQUmqyvw1M", "VXGLxIFMdvc", "VcZeL6bEtuw", "W8iXZ1dvAZo", "WKxVwL-CzaM", "WV9w8koeU0o", "WYZWyQaWCDo", "WeMoTkQ8D4k", "WkLbKZGJSUQ", "WnJJm5BZ45s", "WtBRp2SYU20", "X0yjfWL7X3o", "XA37fbl-hiI", "XAW2mFJ8IO8", "XMG4Q5xx84s", "XMwa3uFPBLY", "XdSjsiqBFws", "Y3yQVQxUUlk", "YOmT5FtqgxQ", "YP3DC5Bk_2U", "YT_e7-_ovCk", "Y_fkfs0AX7M", "YbwSWJ3AFY4", "YdqRKfsqdCU", "YrvFnWMVBEA", "Z9ZgPp0bd4Y", "ZTLmvceEXu4", "ZUHqgDSf3qc", "Z_5ylO8t2wk", "Zgvw_8FRxlw", "ZkVgUWLcYxE", "ZrKp3qLfEGg", "ZrbL6aCzRLU", "ZvqmB66MHzM", "_2blVY1SbnA", "_A5O1kYdSlE", "_AxD-ijU5us", "_LKGIn0Jnzc", "_Pkkb1sBvho", "_T9yNtBgl3Y", "_o-_4wamv9E", "_z7jVkFcvr4", "aAFD4diiU7k", "aNsA6DfaCYs", "aQ9lHTe5G2Q", "aeg4vJHqmvQ", "ajXQ2dS2ppY", "aoIP6hRdHRk", "as-ex4wjNeY", "b4FY3tbr4vw", "b8DaO5ni05w", "b9wKsmFiQQM", "bC4iqnpY88o", "bCDIq68ndqk", "bCX1QQVwI3w", "bK1sjho2LVY", "bRaH0_tllfg", "bUM2sq0QZg8", "bUoQ7gG4lBY", "bWQ5aYXRbXU", "biE43lfyTXk", "bipEcp-OhEQ", "buQkyk_z284", "c0fSoZ16KpM", "cQlQz3ad6Co", "cbWmbe0Zkf4", "ccHihpNnl98", "cjCeHhNrEf4", "cjras9JKRM4", "cu4LnVOA4vw", "czA88dH9-xo", "d4ObSSIj8zM", "d9Me85R-_1o", "dB68s8MGme0", "dIhcSIFHBbc", "dR63pL3AIsE", "dUPG-vyfZUg", "dn2evejLGNA", "dqHjjlnAiow", "e2lORl9FHwo", "eCmqkqYP88I", "eG_SvAJxnic", "eUDNebbApCU", "eVydlJr9ZBc", "eY7weqbBAyE", "eZB8Zd-jATI", "edR6UWNTLiw", "egArK6wg1EM", "ek39zVRn94k", "f1bh_5qlw-s", "f6AiRsMZJ7c", "fEOOcai3Zhc", "fF6dkWQvRWY", "fiy1h1aXMYQ", "fo2L294290E", "fuw2r5BLqaQ", "g2gZJj9anIE", "g3tsSJ5gvoA", "gKw9HoBTk98", "gOf2iaG6Myo", "g_whFCVfBXE", "ggxAUfff2aY", "glSGwZLqehY", "gndQdJimHNY", "gtSgOPKIiQ0", "h4iJHhLyFYE", "hRf7PttLRtQ", "hWN2PHYvtuo", "hoU1UmmBatc", "hsIqcnarF2U", "i3Dw2mnx7KQ", "iKM81bGiRbY", "iO5nOXnRGvQ", "iWvibhV2sOI", "iYXKbRhkLsY", "ibGzh7Dz6Ec", "icyi51_DFQE", "imNt_1Rt2FY", "iozUMqIqSqg", "isI6nCLlzxE", "iyNoHGxkBt8", "j1W3hFQindw", "j1vII7TUvUE", "jEO2EqQLAhA", "jOTwRcNNB6M", "jU1abpL--cU", "jclyRwZUdR8", "jj_AyuBYQys", "jsFnj2QDxi0", "k3r_D4-_Ns4", "kOiXN_kQZwc", "kOr2k1BjzoQ", "kQd3uGK6b38", "kcqX3QwWzDs", "ke3m-C5D_W4", "lfMMZY_1l_g", "lv-UUWx5oEo", "lwoao9UnShs", "lx_oCYT4fhs", "mDoVysdqcyc", "mE3c9NlXwgo", "mIuqOqIrI2s", "mYWZihNpmuQ", "mc_v4qaoL6Q", "mhbcxM9WXdY", "miyIPEv-xFY", "mnEWiEOTTzc", "moZgswh1K3g", "mqexBFBfW_0", "nAMbk2C1VGw", "nBGxxB1dVFg", "nH5uIPLVN0k", "nKcoRhzZ8wY", "nj_QqVRGOvg", "nsaR6OFBAKE", "nvK-73vdcFE", "o-PJt-ytreA", "oEjB4FZWWVE", "oLOttganfMk", "oNjaztq_MYQ", "oPCxnpQ19F8", "o_OD1DbBMp8", "oe3i7ND5z10", "oy2c_8-iZ3A", "p7PHUJR2yyQ", "pAe9lTY-KLU", "pJFNbtOLOms", "pR-GFcF9NIk", "pTckt2AkRdU", "pVHtAzVGniY", "pdv-0lGIEFQ", "pkm-EGOCB4M", "qFcY-TqdLYo", "rFNyV9IAtEQ", "rZjr_uwVQpY", "rcwXRtsSAcE", "rl2HtEFnIGY", "rvQsAkJNP90", "rztTuaPEfGU", "s3urR1c_UUo", "s5jLg_HqvhU", "sD3t3jJyfis", "sF3rRjhv7rs", "sIVkuplU9Q0", "sJSgBVKn774", "sNXNOYnSieQ", "sR_wZHHjvXA", "sSnBlNd_6ik", "sSsnvOdNqFs", "sUTyLpLRTKc", "sn5utNRPWKc", "so3rmW7IjHI", "t0Ik_bJ16hg", "t4KT_Huz144", "t9ohaUD1l_I", "tVFNgsIEdko", "tsRkJctCmq8", "tvGt4cAzXas", "tvfj4zyZTVU", "tyhKmPdPVac", "u0XXuXgBarA", "u3eeLUN2ktg", "u62pmviMtLo", "u7HkMeoo53c", "u7rzJlLwOto", "u91bNuXE4h4", "u99zr48H7ak", "ue9UYCx8tRY", "v4tGHfkIeiQ", "vA58Lf75u5c", "vFxs8WJpCco", "vQL-G_ZS5wE", "vSxj8v4YPWM", "vaue8H-0PS8", "vj6a1cu5ZEk", "vpvRyFzN_ok", "w0iOoj18nnM", "wJ1raRfWQ9o", "wRpoLlcHQWY", "wXJauw_GiPY", "wjWZiLURM-k", "wtDBt_BXK_Y", "wyKX99oyNlY", "x2k1s_A5RQ4", "x3Djd2dG9Ac", "xgDY4VkbFlw", "xlsupKeWwzU", "xpSZNFQCZcA", "xpnQFKxPGyk", "yEK35AUVg-o", "yGUu1QGGyAw", "ydUexovNnkU", "yd_zfKUMtAQ", "yiP6iJ9n_I0", "ylebiZD89RI", "ypFR-n9HtUU", "z8man7AanY0", "zGH3enDSQAY", "zHoX5J0pbQ0", "zXGvPFM7snQ", "ziGcIggcDYs", "zrhlX9g-IyQ", "zyyuZ27BR-k", "zzfF0yAwz8U", "-9QumIh7NWE", "-NxgE3_BjlM", "1-e3pmUhMuk", "4gmAsEYH_us", "4nz2iCMbUwQ", "5GbUindsLm4", "5Wcsh6RID2c", "70kD7gKzYQ8", "7EkdZPMSfIE", "GNkhF9EKB6k", "GrRMV4SBjY8", "JfLaAjtAEkc", "Jk3Pb8dXr3I", "LRErH6BkC-A", "NKGL1EY8PHk", "Oq4Zl2C6CFc", "TKI72f2cD7Q", "Tjcwet_9ZO4", "WetclCF6cvA", "YjqY9x211LQ", "ZtZAdT3gozE", "_MoqyPsui2Y", "ajDPkUDAk5A", "cOW5nNhA3AM", "cehf-vAXry8", "eq-kqkPqv44", "exP4IsBoHAM", "fqvt5IKzisI", "jA2PYb7htJU", "koBTsC0q6xc", "l2pRTo5osgE", "lOucjKKqLdo", "oqW2eHsOgWI", "poxHRXcCiiA", "rWXPP7RBMkk", "sjIttoiEak8", "t9ANrp9nfhE", "uUQ1X7Qoz_Q", "wPdyM-le_yw", "z-nCpYMGr1o", "0dYpZjr-fhQ", "0qhfpgs_wow", "302EE0-0fLs", "3wZTl1Fuimo", "5veTIBk_Ovw", "636Qo0lyvbk", "6K6Pyi8W8JM", "6_faL0IEJKs", "6ih4qk9V2gM", "7N6-YUesVsc", "7y-O3TRD63A", "8YQuZSAZyjg", "8kdvID5shEM", "ByOaW1n6IX8", "Byga2ETGluU", "ChFCMrIzAkE", "DGP6c5ZXUN4", "DN2_H-8Nf1M", "DUPL0suSPpk", "IvuX1eH3_a8", "KawL6ATqV4c", "KsesmigtZbY", "M3lOx1kTvGQ", "NJIaDUIYAmo", "PuST67gnWBc", "REp8CbrC5D4", "VDPV7T82OIc", "ZJ3nuYQGtNc", "a5XaI96CPeg", "aS6cytpdXl0", "bf8TKvAh1XY", "d22TMUyLEdk", "dTJ2MavIbJY", "gzWjZEh62_A", "jRPtzPi6yjk", "jXLMNTRPAX4", "lBOt9wSwsfQ", "ldlQ2KsGUDU", "nBAdHIuFAOU", "pFmoCA-jRkg", "qaWvr_6aDIM", "rB_AtY1DvWs", "rMrztIym8U0", "uohrqgaykLY", "uzG6eRmiaE0", "vrlvEMe7aE4", "wAzT0EsJ29E", "xXE2gYEjT9M", "GMeC9xfHQkU", "bpc-dnAEeZ4", "j0xATCr9ndE", "jYkQwZOcslM", "-HfnGM_K5Mk", "12gqwPvgI9w", "3LGZd74Y7a8", "63Hm_EDWwIY", "H8SbyRlT68M", "QycwPoj6oIo", "U8Ai5ASJ_fw", "ap-QqNClC-4", "b6jQco4fJPg", "bXdYzjogiCw", "djTuzhW4PWg", "ebmETP_MkE8", "iwRKnschWq8", "nSxgGrwtm1E", "oG0EjItJg4A", "vasyIAR7X6Q", "y6bwKNMMA60", "zbrDhUJVXsc", "BAxB67yX1iY", "DMq1JcrfSvM", "DOzr046_7dI", "htNueioQVLM", "u1Gn7-NRSzQ", "yy40c4yqWEY", "BQNpCYBwa1E", "Owi6zWvKv6M", "1zsF5_D7B7w", "763NH411nIA", "7EcKH4hhpgE", "CAtmBO0pYH0", "Hl-G_AmRLz8", "K8amiUcsqhU", "OwOXUeqHqwQ", "jyMCYKuOnno", "ln9fYQ2V5h4", "mgZUgtZawxU", "nCx-eNKidhI", "nRmTmRkWnU4", "tzRm1a-RFy0", "yjCN5BU84tg", "0HBPW2PSP_E", "27wC-QmYAkw", "2KQyv7lE478", "2RYTit1apWU", "3YhUOJ6pJYA", "5CQyRjBf4yg", "5Mo54hmmdTs", "6-ywppCbQcw", "6E3GM6fITns", "6J433FIB61Y", "800CahWQ6Iw", "85qUbMTE5Qo", "94kwXHx6K7M", "C-97LyOlICA", "C6pYeTOwxlo", "DKHQ6-Vu0V8", "DfSoHRgMumE", "E8LrKMa_g7k", "H1377n_QttM", "HFx0YbTA8eE", "HrUXEzd3GZ4", "IpgUKKYCQzI", "JrhChirjzxE", "KS3-hapC6QI", "Kr9jkbS4u-8", "LH4GtT7tjc0", "MGBECNuTk3E", "MZCTbv2N_e0", "NAjhwye2OXc", "NJ7023UFQuk", "Nef2GPaXnxo", "NkJpALjjux8", "OKd8emp-OnU", "PEVsUvIKguQ", "P_yL8Hq7wgI", "Rc3qWJLY3y0", "Rx5v8vcT-0k", "STNuy-ZipNE", "T2-1gFQm0Co", "TpyTJkjW8rs", "ULVrfn0RKnM", "UrM777JMx-c", "VtGXWo62buQ", "VuSlFSltfUg", "ZCSyXm4_CS0", "ZtTQjKJiN2g", "a3D4Uy6c_2M", "aSe0N3bn5VQ", "aozRgtm4t_8", "b7e-G41FrLQ", "bgstp8c34lY", "dgbiPq-uAjk", "dpeup2wiZDI", "emGo0JYKkiE", "eovw3uUpuFk", "ewa_zJonzH0", "ezWXYizHbG8", "fSnOvyqMO6M", "giYLDMiGGg0", "i5_DdFy4vZs", "i6EOtcJ10K4", "i7fbLV-T3Gk", "jIYsAp2ghBU", "l23YxROPeCY", "lMxGORUYaic", "neZrmFPY1NU", "oYm8tjUNXAs", "p309exy6rXs", "plrdM_epzJE", "qwi_wbuImAc", "r0PD0RsbSdU", "rJM0BpP2Pak", "t7vmcw-uF-Q", "v9c3V3KhFM4", "vxXo13muv7g", "wvxXJAtTJuc", "zmSEODCg0ZU", "zroO07LG23A", "-2SfPD6cIek", "0h_PwJlnG4w", "1Tu_oGYQN14", "1ZO6jc6hcDg", "1muV1tfCg-U", "2lUG8gEiQQE", "33OS0h-SHeQ", "3irKNo-wuEs", "459bfZYx3lg", "511wT66lswg", "59mDfVPGjnE", "5XhNcmuNXR8", "6FL65Fbnvww", "6qI0yajkDKw", "7tdWAflX83Y", "8USPh6cQVMw", "9wbMxyji6C4", "ADrAoZn33_U", "AMD5n45I624", "AVMWKzxqRFI", "AiQnPqO4Zt4", "B81ALluca8g", "BlY6nSXzgEs", "DHddyv1myNQ", "Ef04L8mazmw", "FGVO0b66AMk", "GCOw1G_E1iU", "HV1zS1-KnLc", "HcRNZyv-lNQ", "I5aBSKOQ_-I", "IGfyzzcMOKk", "IU6r7Ybojtc", "ItZkn9C8e8s", "JWkhYlsZNfY", "Jvp8oDWyqII", "K6NdqwH51Is", "Kanx-8mntQ4", "LzyKk1lIKZM", "MSQwUKe--sI", "MVLgh4nBxwg", "MVdYkgjD2RM", "MWhZ3mfKysQ", "MxMQlJpQSGY", "OWqXlagD4jM", "OxzFnG8Ri10", "PF9cewm816M", "Q34rBYrJbeE", "QGbwonSjeUs", "RUJsPR0jzuU", "SDNjCNPm6Ws", "TXZUtODOjHE", "U-lrkh18hyY", "U4yw3XTvjeU", "Vk8yv9jNy3s", "WjR5feKpG6c", "WncPmYpMfwc", "XFLy8PpynVg", "Xmk-FP26eFo", "XsCwxzOeiwk", "Z-Vf6Znq9pA", "ZnSDhH3inG4", "_GhLxu5DIww", "_uiAeKxYy7g", "aNJ7OtBI-kg", "avs-QAhlp7I", "bT2uMHgjHMo", "bqi_iAvLLNY", "bs802M_jqtk", "cLoMs3CGSA4", "dChOc19CENU", "dSX19Sgop_w", "eF1Vs7S5LaM", "eSIqAiOcXzA", "f-iai5Wfnec", "fI70Y5xBXWQ", "g-U-CTwVUAs", "gTBg7BZKkK8", "icgrRymHcuo", "ims26KcTAz4", "iuwq4R4el38", "jNAclfWxC7g", "jvsqXTH-mjs", "kBYy0r_ycVg", "kIseHDMFyrk", "kYz6yfj1BW4", "leMczPCHLDI", "mbZB1OWPr3U", "nKMlZ2upJmI", "nsGUGswrSBg", "o7zWYGQhPRE", "oQPE8b6LOsQ", "oQy3OXOk9RM", "ooDEA1WLE2Q", "pzjG4Am7vKE", "qDBDmFWcUvM", "qVGdIVByTTs", "qy_lXNYKECA", "rIJrRdqRX0w", "ssGOpUSPgqs", "tveAHHdRJ9w", "uA5szPH2gqo", "w2xyM_Vngyk", "wDLoFaKtmYM", "wIY8ODFFVUs", "wt0rnjtky-Q", "ysYQuFapoRI", "z2knNVgbkJo", "z4vQ9uAyYM0", "z_-x28WJwio", "zvxv9TZ7G8Q", "0ptyCiV_D0o", "2-llUUTh5o4", "20-VcLze9fs", "2Own0olyjVU", "32kvCUX9hYM", "66EZFfHE0Ms", "6bYYt4I_UVM", "6epxuhFKuzY", "8ylmx3CzdcU", "A1TGh2i_YFA", "ABsRIFFyaZI", "AwqQfJthoiY", "BlOHvtj4FOU", "Dg6onmCr7_E", "DylOngmn9XA", "E1Xuuq6Zt8w", "ERCuotbgIy0", "FbLeJRipMEc", "JWQaKqX2gNw", "KQ7Cy6e_AJA", "Kyoaew3illc", "MLkVz9MUCIk", "N4slrj9Dy4s", "NselXAdw7J0", "OyqeVMY3hcg", "PO5hpQB0ntI", "Q2ip0Bed8Qw", "QSHjvh2VsaA", "RcB974Elggc", "RkXBncXUSGY", "S6UkVNI-bQo", "WhBBPFOqaSM", "X5dwNXz3FM8", "_aS7MLBgR2M", "a97fn2Wdb4E", "aZ70-sKFiWo", "aujmIovxn9Y", "d9qY8x9cuRo", "dsfD1eAmt1k", "dv7KB-TT4EY", "fYIWaaDcQR0", "fg_IwcFCFhY", "fxKyYZrwLRY", "iXWykHO95mk", "ibSzhTUAwME", "itwueAktMF8", "jBMAtay12tc", "ke-mz_EPQXA", "nWvbi9Y4dBI", "nh09borFt8o", "njogXTXewJQ", "tAz1tVVQ6dc", "tp-sUkr6cMc", "u03mN0ylOpM", "wqkdkCsy8I8", "yEkUACzDYSM", "yy7DHPPXhBw", "3Oqjha3zYDA", "7ko4tyuFn1c", "alAxZqjNF1w", "sLhLwBWOSiI", "wzGUcoAzWz0", "y8J0xuMmorE", "-9rSx1f0--c", "0s-b2PsGLqk", "22mWUkAi0PI", "2oFsj82yY7Q", "6iHxyKi653g", "91uw_HrIO6U", "DOhYwhA_Yyg", "G_8byyU6Zno", "JtWGt_Gxsf4", "PaOzbKCGrOo", "RceIgPLsveQ", "TEv_vJClbVg", "WhzVE_d_SZ8", "ZtD6wqsyZTo", "_gewmR9CldI", "aXSeb8Dx8YE", "gCH0dSTOESU", "n9unogEb2SQ", "nfy6YM9axHo", "olm9Djn9ojc", "p1WH8NTaVvE", "tYc4XXXtuHg", "xaW5MZXSj1I", "1-I_bFfgn10", "6ODms8z3i5I", "8L2mqkk28sE", "DnJXldBETJY", "LvQ0W5c4fbM", "Nq6Kq4Q1asc", "PQD6WFCgado", "VNfM3zgBQN8", "c4TT0eVYUB0", "dSfNhGJp5X4", "fNFX65nyY50", "mE5ob09ineo", "oXDd5msIclc", "vt30TC3tOkI", "yGQ5jNE6n0A", "-3BDaBVj_bs", "-9ITKFiJKEY", "-L9X3EXRNLM", "-UfCfCXUCWY", "-m4DJr6zZx8", "-s-8-79ifUA", "08KjiMEYhBU", "0LsH7KqrnI0", "0ba8a9SzLc8", "1YZYttIixe8", "1emi21U68eE", "2ZTH8SXdIJM", "2ycFjTmnk-E", "430s5RfeDek", "4N8FRDfRHlc", "4nvXd42j88s", "5qO44ZleCfk", "5s6ODe8mYHc", "6347qPo3p0g", "6gNL1blwul8", "74_gJeie7j0", "7CLMGvbEyjM", "7lwHlyQ-ZuY", "7ra0M2GFNO8", "8F-jIvhCl_A", "8x3IX_jlCA4", "9-XxtzFiOu4", "9i6k-4YrZtA", "9ruFCsuCKu0", "B7QTKTTEcN0", "BSY08viNmPI", "C3uOMTDHK3o", "C49OqZa0ok8", "CnmPgUX739o", "DSKXzT7vjKw", "De3UCOqUZw0", "ES9sTaE3-eE", "F9i83OJBDag", "Fi06wzBfKbQ", "FrBkzDC7jaI", "G5ilkT7uuUk", "GOQmZSyMras", "GjoztYA7Fzc", "H1sVsnRA-bY", "IbG7hOzl7Gw", "JP5jbZPsvf0", "JwaacHua9c4", "KOfRmhdssYw", "KXBBEKzb30Y", "KbKqMbTC0X0", "LAPZklZEY7M", "MJxMa_YDL5E", "MXgwihDEvz4", "MdhtgOCjLTM", "MeuqqLHDjW8", "Mq7TjINBmxU", "N2qY8uhy7rg", "NvNVqow6pkA", "OUQBZrFnsU8", "OoEcXPMHF1U", "P1h0Ohk1WN8", "PlgFdWyQHKs", "Qaw5Iez7jYQ", "QnPIHNef-3Q", "Qrdv4gADxqc", "RhSWa1AXypU", "RhevfikCE0o", "SUcbXhRT6G4", "T4zR2RJIxHE", "TNPLkiNSXNE", "U9BLCGo29bU", "Uaw-t1ZeLOs", "VoZrkkUdPfo", "WQzVSRLjDBs", "X48T9wECQr8", "YN2TJFjknGk", "ZAOtNxCaQUE", "ZzAIbCgW-IM", "_DgIUKmYRvA", "_MniqF9zjCM", "_fDiFq5D_Ao", "aRfim2ijovI", "aj8ITYfWu7g", "aroGWpVAwQQ", "as2LpNBtluk", "ayqjIx3QLRc", "b3UlfRHww3E", "b7kWdTBClOI", "b86qA9j4jxE", "bt-ILJRj3Nw", "bupp5sgNZ7g", "bzJxZN98fpk", "cjCvw2LqK1E", "cm5imT86HNs", "dLI9Q_XDUzc", "fwUoTo-JiAc", "h4307uKCCOE", "hZzBL4Vvj-o", "haMAD3zYD00", "hbfU43dNbQ0", "hng5F260Iv4", "htHX73rcGbM", "ipjSbPZ958k", "jFvkLnccCJY", "jThqxFyG-TA", "jxE-7N9g8sw", "l6hqJQqLLFM", "lh0hqqPk2Qk", "lqZ3RlE-Mbs", "n-sw8x9V4pA", "n-uPQRHXyjg", "nDQeQlAS52o", "nEu1Dcl3X44", "nGe9dQb_qKI", "o25QJCCCq5c", "p5EMPKCLQUo", "p6hrzQBfiSk", "pn5j5xGQhiY", "q6kEriMfaJI", "qP6nwIXTuLc", "qQj6Slo_reU", "qk-ChenghsI", "r1qXeE14sRI", "r8tsfRqWqgI", "rI7UdVQ1Opc", "rIKW8Tre9b8", "rgkjftGTYbg", "t0uIckMN03g", "tYPRBeyU9TA", "vSakrkrv7rE", "wAdn325yNig", "wrcNCFZSdnw", "xYptgLzIKuo", "xui0mNktixw", "yXV2bANtqlk", "1fDUkKCVMYA", "3QySmgewiRQ", "3TLnHUotSis", "6TxTK8CNKrw", "DFvVZ-QZA_M", "OJY2LhD5nns", "SBQnR_tSWpg", "ZKwownZxyxM", "ZNlsI1FtP10", "ZoIJlDVXiMQ", "hERWnXCJ_Zo", "mextLpng-Y0", "sMJ4bA8FCRU", "sQ-aqUuETDI", "szGCVf19y00", "u3hguYaqiQo", "vaPROeB7qkY", "zIC7bGV2dq8", "LnqdRban1y4", "3KdN1KEvn70", "7SlRZJcavGk", "8Kky_mOdtyo", "97Amj1adxgE", "AFbg4SgEwBg", "ATv9gSbjYEE", "C-mqmBfN-3o", "EeE47GOQY0g", "FSqSUSXIqR4", "FfWu6nrvkWc", "GObCkXwTbSE", "HVKibjt3H0g", "J25rFPYXBos", "JUWQ6cdPoBU", "JcPq6osN4fI", "KWEkooA6VcA", "L5-UUqoLcFc", "LLovWNWqLag", "NSEtVYm-DTo", "NV5EEMJKLMA", "ORaGhJbpmO8", "Ox5m2mu_NRk", "Qhriz4HpXCc", "S5hmzvFQ2TU", "To0YeYoM9-M", "UjUM-p5Fg0k", "YVC-e5KlSFI", "YjeE-xG0Gbc", "YkjMIqm36jQ", "ZYr1OTCGnmE", "_-gSaO4jFIY", "_pHhJhqfZcc", "_qXVaOmnuEU", "bW9PEpBTLeE", "dYzJlC1JuOs", "drOKf9MlfqM", "ehFvwetMo-w", "g33WD2RHbas", "hDoooOgzZvo", "hOnrUQfxVh8", "hi94fRNo8PE", "i8GexWQExKI", "j_8UZ3XFYhI", "kx3l7QDImo4", "l8DrNuyewYk", "lcEEIaKTyO8", "lxljxc9urPs", "n8Y6NDq1vM8", "nruLbexQMyY", "o4hk3YO6DBY", "oDqfm9UMbsU", "oWTCyZp6yqM", "oiDKioS4bO4", "siVO97J2pBk", "t69rThduI7c", "tiS7QMO7jHk", "uNHxb8_JU98", "wHnS8KR6UmI", "yChKkkkv2CE", "yny7-X9mhmo", "0mCrj07o3do", "OGvrgfMXCFc", "eGVtnqLJVv0", "kCZuwZHg9N4", "--igl9k6Izo", "-RdNER1Pk1U", "-T8ko63Nszg", "-X0PDT8wSjQ", "-XAr2eCTDG4", "-ZKUDcCBDWA", "-hMCDeSC7Wk", "-huO7_6tqSc", "-nZCjk4pkmE", "0GIkEnvhbXs", "0HZYo5RV9PU", "0L1rfcZ2zHs", "0P2cN92mETw", "0XY7kkMQcAo", "0b571CM9AtA", "0bPMMrT3Shk", "0qgA_wRRDfQ", "13CYZo660Uk", "16kYPvDls0Q", "1AyUyqxIJEs", "1CHOKkOHmAA", "1LQi4gEcPdk", "1LdorHmnMwM", "1TYFaboSn0M", "1X_sVjfjl1Y", "1af3IIizj8w", "1jJEzY2BlYw", "1wJQrRrUhvE", "1zImuKg1hpg", "2514ifANlno", "26dxKUjvsNg", "27VKcL-Z4jY", "2I-8nX4yQJM", "2ZjxHriiOOY", "2hKf9qIkxbA", "2hdxbFSHgYg", "2r_Mpu4Wo2M", "2vCfvmHEBm0", "3BAeLA_rGeg", "3IHZI6TNe6M", "3LCCdFMwEvg", "3_Ugz7EqSQg", "3jKhXKUMmvM", "3p66uR4HAU4", "3utzfg9VYoU", "3znDxaKsu4g", "49QUo4aRH6o", "4SEshb-5sJs", "4YflYgE2stU", "4qKHyAiGylg", "50KU1R0pQAQ", "51hs9bOSns4", "57Nm1_f4RNs", "57dmdkye_zo", "59Mn0I7zGHM", "59xWR7yU55A", "5FQpSH5rXHw", "5JK-Z7gGpcY", "5MOiVCVPAi8", "5UddYiQ9ctI", "5i8ohAjkY90", "5lsNDHJk3gE", "5vW37BaWyNM", "6Asih1EXsIw", "6RRA8xHYbFY", "6Sa6S4cF3vA", "6YH_h53NSbU", "6aHFp4Wsg8o", "6oRPBVlXgY4", "6ouB90LY8yA", "6sXk5JVF2ck", "6x3I7iwxu88", "6xawx1s7qEU", "6ywNHkSeYiE", "70KydxgZjlM", "71ESBrup1ZY", "7DhlCbBWPms", "7KzOo4rbHuw", "7M47HnjIf6I", "7hXsY9TAsFg", "7klV2pJObc0", "7n21c4QC3zA", "808njZuYNj0", "82VHsHfbl3Q", "88GBTPAOa_o", "8CD1G7RxuU8", "8NXxwIt_LM8", "8T0QedEQS8E", "8gykpN_tJWI", "8lATwCY4zVk", "8tDMuPxjLro", "8wY3-q1XAEg", "97AewbhlAhc", "9HJ06F6zs80", "9IvC3n_cJwg", "9LJ6a07Know", "9gBfXg8cpVw", "9gCZhhcRxcA", "9uzrPzMeYjk", "A0ohahUprX4", "A3PEbsbEgqM", "A6NYy67mhSo", "ACw9HtYL0MQ", "AGXHYq4TsGc", "ATlwXMB3Pw4", "AUFFrP4zTqA", "AW_rt5yJVUc", "AYJKBIC0JUw", "AfN7tkLFd8E", "AiPQkBFPg1A", "B0ClpZdzRJs", "B0WCWTPDf7g", "B2GrsOkaJls", "B4BqJn719Qc", "B6GuPUMwyn4", "B6IMEz4n_5s", "BBDxwQl1Se4", "BNHXZ-t3JCA", "BTOkK1qrBSM", "BYFoKS9pkIU", "BcwGzeLwpjM", "Bevn_ddTSgg", "BkC9XFTtwMM", "Bq3MhjmxV4s", "BsdDm4-ZBfI", "Bu6W67wEHWs", "Bv2V8JREUT8", "C3jBx7NTAf0", "CXnSGrkTIuE", "CY_Rq_iew2U", "CZRx_sTow1Y", "CtyU5CjoC5Y", "Cz0Sfd_TSPA", "DBeHQfvqO6k", "DFN9heqsjMA", "DHG4ySpOD5A", "DIudkEONZOU", "DTI9EMCaK3E", "DZ3zxrltUpc", "DcfDxPnBUQk", "DclD1sSi51E", "DoV17Ff5YWw", "DogCw5gVByc", "DpVSQthgZo8", "DyA34RfH7gA", "DzbEEBCZFLU", "ECUxnQ2cJeU", "EXuShDNr6ic", "EZ5sgL3FEsQ", "EpLuwgttz9I", "EqRUxXsLVlk", "FHtImBrGIs4", "FIpxHvHlzSs", "FTcQhRrXbJk", "FZZ6KWW1vlA", "FaCRYA7l37k", "FfwUgGASGvE", "FjlE5dyiPlw", "FjlaOIoJ1AE", "FwG0iT4YMic", "Fwz7Mlz9htM", "G0pISzKyiBs", "G1QIXREYBEE", "G1WMxwZWH3g", "G8bWpI5QaU8", "GRr46pXjqBk", "GSigJPMuhOM", "GSvSIbNBaHw", "GiSf7-YfTzY", "GxerYtTyHAg", "Gy4zrLM0miI", "H6OiIzgQydo", "HBKV-QEK-XQ", "HQ7iJqFsL_s", "HQTExIToQQM", "HUxMicWiUrI", "HapKeNfrNDY", "Hf-Cx_M-QKo", "HhQtKFvepbk", "HxtVyTGdDgE", "I2QxqjEE1V4", "I7_szd9OuRE", "IAwgv3KEkAY", "IPAx1qsZWhw", "IRO-SC7gfZA", "IV72IWik8OY", "IffH1Fez2aw", "InHXarwIk08", "Ix4nu70e6dk", "J69k2RN-gI8", "JIqahXOmeOY", "JJgyrClqukA", "Jot-21CKlW8", "JzIYRgy33Mw", "K0JdQ688CcI", "K1j-IH1aEjs", "K3mrhZ27KWg", "K60ebY_2v1g", "KC65tYKxoLc", "KMeOeFGhWX0", "KOK9AgW9jcU", "KUNsuAANrHA", "Keoy_QL9cRM", "Kh6LyxoP-qk", "KhJVnxZ07fU", "KrYGbWmvQVE", "L0QB9tPOOK4", "L8uC_1jXggA", "LSaU4BQ5glw", "LScnWec_g0w", "LTj9KpRRN78", "LUJBELP_-Mg", "LVs-_o7r3yw", "LWUEr-v3nh0", "LZne25lqwnE", "LaFXvMhGBL8", "LeVD_Z4oc7Q", "Lq2ZiazVpQc", "M0JwoOZkNhM", "M0t-jibVe54", "M6p-5lSS0bQ", "MDZP26i0f6c", "MP6IrUrkapM", "MPVNBv_qhB4", "MRSiJaYYu1o", "MuWtihdlQdU", "MwRwI_Q22qE", "MyQwZC7vgQ0", "N1LyUTEMawc", "NSI9j7XMfNM", "NS_Eol_VHYc", "NW5o601Dsic", "Nb8dCTCyfVw", "NmA553vHlBg", "NmFjF-0g4Pw", "NonxxSMZ1j4", "Npuj_oEhtKo", "Ny3FwtH8A3U", "O326_kOSM9I", "O8dd-29bnxc", "OGbJ6hceu3E", "OP0gpjBqoUg", "OQFCB86i1e8", "OVDxwBPuS64", "OWz7rLARAdw", "O_y44KOYPiM", "Oq3bKivnzJE", "Oumsow9RKMk", "OvER6F3zLa0", "OzLUu9MTBNs", "OzY10rgQml0", "PKC0RVVk85M", "PKqI62ziKXY", "PfOB04_kb-w", "Pi8W5I6dozU", "PkmI9-AmhmY", "Po0VOpFYuJU", "PobkK6GvybE", "PpqHXId1q1I", "PyKaqaxfAoY", "Pz1pSO4XrtY", "Q-Ic_Ex-F_U", "Q-OOvQKCMmo", "Q0pG8DRLLEg", "Q7s75HTpmRc", "QQadXySR_Zg", "Qf-tKejq3XI", "QhRmnsQyhFc", "QtHWI0A80hU", "QwndKWG7YcQ", "R0Itg-x4XjA", "RAmqXiqLH5M", "RDZeq_-nZa4", "RFlcxo2_3fg", "RHGCDwEuH6c", "RToSrqZMdLA", "RVIrC1jrepQ", "RXt3LL8Usuw", "RbrLE8lB8Wc", "RdlFJY-OJRo", "Rf59czgsrq4", "RnZSLoN4rFc", "S2FPAIeDZcM", "S388UE36C3k", "S7H0Dm4szAY", "SJeU_-zlaqY", "SOc0fOEVzIA", "SZqKj5fNdCs", "SdK3WR4mJUQ", "TNnyslOK_mM", "Tx6c4T0oaoM", "UA3jPuq1hj8", "UDN5AQ4KF_k", "UEKy7soOPh8", "UQULSQXTwGc", "UTk4uYj7SSE", "Uc1rCB38ti8", "UhkgurtqY1U", "UlzoKrPKqnk", "UnarcKQpYVM", "UqxP6zyKcKY", "UsvJKuaf1p8", "Uw8F8XSe4uw", "Uwrrq1eq_-w", "V8iUqBm6xw8", "VGJ_wbtu9_I", "VHMtBDu9tcI", "VJipIjapOdA", "VNNDH2FrPcg", "VRw0QS2OO0Y", "VWCIM5jqTJ0", "VrHs5Q7lmEw", "Vul254mLWPU", "W-g0BAsRQNE", "W0I4LTFPbuM", "W9se817EuA4", "WD_itzoe40s", "WFqVcCinycQ", "WNpX3Fv4edI", "WQMkEiXiSYo", "WrOIo2vhCK0", "X0lZ78t--34", "X9Cd1OXNeSU", "XDA3BskyPzs", "XKQFvvItIE4", "XNTOiQaKq4Y", "XQW1U76-s88", "XSp9a_Qr0uA", "XVkk1suAtSE", "Xgz4iIo4P78", "XlFm-nH9ucI", "XmDo7HvxG1E", "Xt2l3HvtVxQ", "XuXoTqFogGA", "YBjtSt8eUVc", "YW-n4HukuQY", "YZk6DXI7zjo", "Yf95TQCOHBU", "YgLOVf29-gU", "YvhRg1nGak4", "YwWxkYw6UYU", "Z3h89xZANAM", "ZEmzDlaFtuo", "ZEtDdEPANCI", "ZPa6D__QdZ4", "ZPyMGaOzOSY", "ZQj_6Nl4Pm0", "ZY2a-rBzwYc", "Zee-3VfY5Ws", "ZemqUQaKUM8", "ZlI5hRsx4Tg", "Zll29ziiNKU", "Zs_FbNk1xdI", "ZxReUqpnA8k", "_03NpPjbB6I", "_6kc0GC_FDo", "_CmUZLlD2jw", "_EBbAGanRcQ", "_GhqQbxlFVw", "_PLf5m80Kh0", "_VfuVB_VRJc", "_XuWdmm2vG8", "_bqsrEviygU", "_iUQYPG_A8k", "_rLNbK3P9WE", "aLsEbi2pG9k", "aSiaKUywbHw", "aWjplesuzyw", "aheyJAUBrdk", "ari1jL9pWms", "b9e-B8MM6zI", "bIvuJghUsdY", "bL3eifaMhK4", "bNphiaUWwWQ", "bVC9QvUsNRw", "bcrvD24-asY", "biU47oG02TI", "btAvJJT9kys", "c1p9Y8tld9E", "c2Zc_SM8mmI", "c5CRE3BaAQM", "cITqNByQW2M", "cUAou999JrQ", "cafOHmo5Z44", "cqOP18LnSYY", "d3u9FdsgACw", "d61N5tOult8", "d6BUJoLaKF8", "d6WS3heSHkA", "d80UFWWQNGI", "dDwufYkX0ZI", "dRAWc9_ZxLs", "daBHOUrgBAE", "db87xBeeDGc", "doPHaWYJCzM", "dwJGA6lR2vI", "e-3QC3oymaQ", "e-zPyYmsHTE", "e8G-zjl9GN4", "eGOUP9SF714", "eat1pxlQGB4", "ebuGZmX5NxY", "ekxSIGNlTng", "eqE03EW-2Ro", "f-3kPTjmn1c", "f-QcyPJAibM", "f9aWShOFTqI", "fMnQN1ZgIxM", "fVyHcPUr3I8", "fdofXQ-ZsYg", "frZDKKEW_eI", "fvofZ-2nUhg", "fwJrj0ntJQ8", "fzs-YG1JArk", "g-yaL_avPNs", "g-zgv6vWDfI", "g5X77nehSTk", "gGSwzogT718", "gLvvsXea8w4", "gOSzb2DZS6M", "gfCT7QVXfxI", "gs_ZZ2lF-M8", "gvj0qBVdOOE", "h3VeDR0vBcI", "hES9iV5BYZY", "hGYz-QvcHCE", "h_bBTMtwJ2U", "hasYKSMey6M", "hbHFafKIR_s", "hg10c3rfSnk", "hpkd_KmMBvg", "iJaqxwiLj-g", "iUhWLkJsiCU", "ieKg0pOLpEo", "iiqhUBrBn5o", "ik_YZmFfGt4", "ipV7Nhr66go", "iyAQJhGiM-k", "j5IbvtDourE", "j78V21sy8lM", "jMBZS9JTW5g", "jVjsk3HDqPM", "jasKy_YEKow", "jdG5Xkk5b0k", "jiiRMCVm_54", "jkJwJxG-6SE", "jxFEjl2LTCU", "jx_vajcCLRY", "k7bKKzxJqPU", "kKTgKhb0Eag", "kR3w0Jl94gI", "kdgI4EDegf8", "kuXdH_sOWfU", "kzWtCmhD7_M", "l6IKsXdgKgw", "lCxY50eE97o", "lD45hAuWr7c", "lFA5WRs2iEI", "lK4JC6FxwuE", "lObtJm9MASo", "lVlYZrhqbyQ", "lgHAI_eXFJU", "lp_MpDzcGrs", "lszpKnGW2_g", "lwWoWUpwbNA", "lwjFvM_gQFw", "m-KmdtnOauo", "m5lBtyFoiXw", "mCTNTboHPbo", "mK0wWkXASKA", "mNFmiM9VsJE", "mR9S8z7ykE0", "mYdabOk9tyk", "mbY6_HS4ai8", "mdmva68ERUQ", "mgm2IiADnD4", "mtQU88VrCLI", "mxaAWOLjBP4", "n-FfdpNLbe0", "n9kNZKbst3I", "nK4DsStnBR0", "nKZqxvKtGms", "nVsPQtegQfE", "ncmTpG0uKGs", "nkV-cD_yI-s", "nqARjvs5GDE", "nrcdLvwjKu4", "nv96R7jVZUM", "nxnzNuOCIJk", "o2NYyxPC6u8", "o6TVuzEVsig", "o9-lvPa8y8Q", "oKxLoZ_6dHE", "oWTPbBevksM", "ocRzN638y5k", "orRTUkLa_7o", "ot5vUjmpwys", "owg2K-WbwhI", "ozEIs2t6rSY", "ozQ8Y2tK734", "p-bvcmxXdxE", "p1bdL_beIz4", "p3uDpehs0ns", "p5xmosYjceU", "p6W-NmSzfns", "p6_s2QS83bU", "pPvKBdAKbJM", "pT5FPW5GcGE", "pVbwK3bRKxU", "paGef4VDCXo", "pbui--kf1_8", "qXhCnIF3SzU", "qg4uYctqNEw", "qqzZlxLp3zo", "r2ZgqMLw40M", "r65UzbqrkTs", "r7cS8PDLvyg", "rC3TSY3ODVg", "rGOmtdQq0Z4", "rJ7tZvj9Ee8", "r_9uocaGz-k", "rq1hs0nyT78", "rq6pTWqg3LE", "sPmqk3vSHNg", "sgXdCbCLUe8", "smW7EVK8L3A", "sxeEZz0mhyc", "syQZTduPbUg", "syaxuDyYvgc", "t0yJK9h17bE", "tBDA_5JHQ60", "tD5FQ84kygw", "tDRqD6ENOTA", "tHUK9caIOBY", "tJ3yoDTPZmY", "tLtApwnZSd4", "tMrrf7Zi0s8", "tUti5HyQbZs", "tWBslx884ZQ", "tY2XUjf_x2Q", "tjlZpdcVAJ0", "u2YGmbTLSzA", "uAkVdm79iI0", "uDZGGrNMrto", "uINtNcOlJYs", "uK9-E_oTX7k", "uRaQV61rlqo", "uTZJuETfTfQ", "ua864v0g0cI", "ub7KjGCqwjY", "ubJG2tQ1mmw", "ur-h83mGURo", "uzFpAbHRyu0", "v4p__nRghjo", "v5bwKA1P28s", "v9DgyOTuBm0", "vNbMSVuLgQI", "vSvk2UcAZnE", "vdtGeXdLyIQ", "wFXoTiG8wBQ", "wFc5r37iTLM", "wHJHUB49WE0", "wMWjCqT0_-w", "wcClGvL0eGc", "wcJ1LlcFmw4", "wfisNruX8S4", "wnROPLL_D-0", "wq0nZnT4CK8", "wu2KuLNzDcQ", "wucE5UBIhww", "wukvHuq26yQ", "x2XNVh-w-0w", "xDTkiG1mprM", "xEQRbO9sJu8", "xHDmYw1M7xI", "xc5StatDfbs", "xcJiFaTwewI", "xdxmIQHaW3w", "xhKYlMifLok", "xmuqruLU3yA", "xyGYis6ozhc", "xzqwQxucSVI", "y1V-2DxZWW0", "y5P56dPmFOs", "yFhp5IycCHk", "yKs8G0CEcZc", "yPF8H72DizU", "yTI4j7YpHCY", "yY8tGMWO1B4", "yn1HpfIEF-c", "ywUeBAywl9k", "z9FQJEE2Xp8", "zCJHlUZ6pxY", "zGepKTvNDqM", "zPzFfJbZqBw", "zhAqkKRs7-s", "zsRmfkS_ZsI", "-0zX8N-5a_k", "AjP4ykl1R14", "-EwEVrm_qD8", "-SNa5Ho5KL8", "-Tpy8QSABDs", "-WVnrntStIU", "0GVcGsirO2M", "0MInOWF4osM", "0glyiZsuT6I", "0u2MgY3iSFU", "0vdjUzxMNrw", "1rCudbOKqNI", "2EeYNE8GDIk", "3cRRwE2hPVY", "4Inh3ysiSO0", "5Z-IbaPqN0I", "61cWbuqXD30", "6RdiMPlz2wc", "6StSY-8UwWI", "6_VIBACQF4c", "6gP4ZGM24tw", "6lasATLxyac", "6uqUNmsy4oY", "6yuInmo383s", "8XzdiYk2FGw", "8lTOi9WArX8", "8mZuoMZbmJU", "93VbkmqN_b0", "ABgNjGXmsRk", "AFCtMw77qDU", "B8hLaTQjWI4", "BSoCk6tGD2w", "BezC-NPPzho", "CIGY0689tsQ", "D7kshs-212s", "DH9DmOtm9Jg", "EQGEIdGIvV0", "EgG3hMoCbgE", "GOE-G-3wuKE", "GuHhLR_ikQ8", "H6GIulG57y4", "H7NsWI1rZTc", "IosnDJ-unI8", "Jal0Ir4bFXI", "JdrMrfiA-_w", "KgTS1k1dkAQ", "KjsCAdJJ2fw", "KokTm2YqXyM", "Kuz6wj4YsLk", "KxKpBDvkwfU", "LWRLJsmPviw", "LfT1ttQwSqk", "Lugsmy59_pg", "M4oROFguEAw", "M6v3_kcbQrQ", "MIYnfQ-mBN8", "O-2pTPTvaQM", "Og5DNr3Cyzc", "PEsfoBs1Z_s", "Qa7Exdv9TqM", "RY9--b1Dy0I", "S5tLTZ2Gez0", "S9smK6NLUak", "S_equsnqZFY", "SozdcoHDpUg", "TCJrT3zg4z0", "TUpLS4z1FCw", "Tqg9EeAZf8s", "TrSCNIcZHf0", "TxoVJc9Ildw", "UZLU1GJyM5s", "VtZYwVflJAo", "VvGWDVvKBrk", "W-WIkin_pRw", "WHXS6UomjJc", "Wj_xj9-NnS4", "XYssPCBzosM", "XtGiapO3wew", "XwOFCh8Pzts", "Yy7udrDkC2A", "Z81RC-pW4ic", "ZHCBCGOnofg", "_XyOPQjW6p8", "a5P0WCWlsCk", "alPuYU4vC1k", "b9seDJqXbwo", "bpITBdIEFyg", "c79iQoMp0vo", "cI-aJRbXnTw", "cbmA8539OFI", "cjxThyox_U4", "dYKdk7I_iyw", "emi_ccHGMuQ", "euTzyeXUlSw", "fd_jbM0xnQo", "gFUQ5HQx9X8", "gHKmdCkvVUs", "gpC9KuBlUt8", "hNvo1E6w_iA", "jEuHgU-rsR4", "jXbQBzvFMMk", "k9aIgvNJsoA", "kPPbUQIIlsU", "m54WPDUldWw", "mGWGZyh9TBo", "mide1J8pevw", "mvdehiLI8FQ", "oXuob_SnxJs", "ok039CbgDEo", "pP45eFAWT_k", "q4MaquCPixc", "qhxUYX66lzw", "qnWdfGjVQoQ", "rBQi5SLNVnI", "rOfC-ndBfZU", "rakAJLlfoMU", "rzGt8yaV51Y", "s__tL8EcIhg", "skuFq-jIldY", "tWUuyHUXBP8", "tpCelrCRyVQ", "ty3iJFtAfHs", "u39p_W1XXDw", "uT3q8fhl8O0", "ujRYIrFyGqw", "upW_4_xmd9A", "v2jJTFvfdY8", "vGayBA8O68A", "vdgUgzuOAxA", "w8FS5pk0V9M", "w8huK1OS73w", "wFTA9ZfgnpQ", "w_A7ahfeMe0", "wgH6mkVi2W8", "wiW0si_db5o", "wocoJxHZi60", "x4RzuQc7sxI", "xO15rtn4ZVA", "xyioSOz65wk", "ygMmEpcAFwg", "zZv9Sv_FxcI", "E-Oq4Gy36hs", "UEEShOe3vSk", "JPoDmrWGXWQ", "uY0n-OtxaHE", "8VU_dxOFc1c", "9_kqZaelo1w", "GW-Xzk04MEc", "edgZdXO0TUI", "kClYYAA85Ok", "m5klykYS3es", "maRZoZjNqts", "q8hHuOeDrj0", "0Yu9sN7E194", "0vUwbInM1dU", "1yURXpa0p4M", "2Q1WmwhEiAg", "5uZdMhvmbZc", "64obf8hXM5o", "6x8PTKErf7s", "9mWf63JW2s0", "AScAdIwX02c", "AugnUBH53h8", "E56sJsdkKRQ", "MAdVPc4kwCY", "NAI4I16nMXw", "NGg0SBwtTIU", "O3wwPdrLsbo", "OhfljwQ4cig", "P9ZX4RxoY40", "ZfcxMqbb1gI", "_GQh3YTI4mY", "aR_fcp00VzM", "mwZCI0Qfjg0", "ojCojlLfkzI", "p_Z-aSQnUho", "pp6zbDPT1yQ", "-5YXhboFD-s", "-QdpkibYaYY", "-feGkGE1cb4", "00LtFW1MoOw", "02iQj_ja8Vo", "0Gd08biVuRc", "0sY6trjcvY0", "0vgNQu0Qgv4", "0yeilyJagFY", "12jvEcOFoQE", "16LQKshS2fg", "1vPvel345Ok", "2HPgspAnEMk", "2OuNPDZg8mI", "2Tr87lhSHGQ", "30AGyVYbO6Y", "3O4Lg7Y8u7U", "3SWm6AmAlLg", "3_9pDNtrRLg", "40GIwl8tkIE", "4AM2m-8yaeE", "4BM4iBic-vQ", "4o7YF-fu0sg", "4sahheAWXXE", "5ZGIjpD9ZDM", "5i7KRHgIjh4", "67kKx0oPtlQ", "6GDo7dSJFb4", "7OwXnogcafg", "7P0wJ-CZ1ms", "7Zz9ouL3joQ", "7ooAsWFvodY", "8IG-wdFW35o", "8UqO77NBKLU", "8V9SZQuwzto", "8dfWNU-Mows", "8eifmDxS2jM", "8rZMpYHj1w0", "90LXbqPB7XE", "9q6_DboXkyY", "9xexvJzTe_0", "A2g6DHqX4AU", "AAs4D7IEYBg", "AEW52nZEue8", "ASG-O2lBG0Q", "AUQQqUMT_X4", "AhsBpZGM4yY", "BdSDHLmgZ0Q", "Bg4NqSzzJ5A", "CPqutzaak44", "DRqcpAWeWK4", "DrT5xZ4Ezyw", "DwTGz6NOlTU", "EPjlcFNJD8o", "ESkue2cnMyc", "EYGwGnmVZsk", "EdV83yJOem4", "EsC3qGlD6jg", "FEvHzLm1SDw", "FYQOGacgrrA", "FcuD6eQATXM", "Fdr8gKigllE", "Fnbf99bKnmw", "GE_exxszUy8", "GTDybPiurv4", "GZ_ghtSRDg0", "GnHQeWTKEp4", "HXNghQ309Nc", "HaEVqni1aA0", "HhJMjyXTLXs", "I7VLu4wi6NA", "IazcE-viwd0", "IfWKIuZCz4I", "Ivck36RyCQg", "IyitiWJVJD0", "IzAyW91RJyo", "J1kyH_528VE", "J66oFi61dMM", "JHgyZticsag", "KSacNQ7TqNs", "KTIAXYVDAus", "Kgutev_I7mE", "LLvKYIRvz1o", "LOK7qfDGMPg", "LXwJamjm46U", "LcTwpn_xIPU", "LkIcGifbLQQ", "Lk_OXWgCVRg", "M3XtEl_sMuA", "M59o70oAm2A", "M7WRaOQo-F8", "MVM5EtQGKGE", "MqpmCyPdJEA", "MyeNoHqPWDQ", "N-OhxG56XRo", "N9Eb6WRJXzk", "NAAT3FIHEBs", "NEPG1WKMMLg", "NSGklnHYcHQ", "NZeuIlHffUI", "NxZzYUiIGDc", "OVMLQJFqiyg", "OtBRdrD9vdo", "Ox2XsUu3vgo", "PD4E8CrSg38", "PLSWlkOjKCg", "PNPIaFYQAXw", "Q44gUasLk2g", "Q6Ev-FLTp3s", "QeV-1o1gAE0", "QigLE6g937g", "QkuzUiQo_Xo", "QvZoWH_dzOo", "RXDWeBZudD4", "SAEfqt8n_gQ", "SI72iEePlgY", "SP7b9qqcT4M", "Sjj_wT9mu6E", "SqO_5xu9c6A", "Srd7iiyPfUc", "SwgnfpN3WH8", "T60YgMt6izk", "TC9W57-T-qQ", "TCR410XFl2Q", "TVOReJkAODY", "Tqcm2AAW55U", "U1c0v_1rRmY", "UHMCLcF4u2A", "UJAaImectRw", "Um50ycZ4lU0", "UnWV-Ub9Nxg", "Uq729j81tSg", "VDO6EadZGBI", "WCNyEBqwKxQ", "WjQU5cs1iwI", "WpBJ5lWtGPg", "Wquc5FLbEtY", "XEPmL9VJXyo", "XQ3njpuvgVc", "XU8hLXVAuwo", "XZMTHIg5uZM", "Xd_29NB96Yo", "Xp-qmOpNo78", "XqEVqg-YfhQ", "YBDF2BE035w", "YK5aKj4nu2o", "YmBoIYkniDo", "YxGmp7DPxSM", "YxJoigGvWw0", "Z5ZM92YSWDM", "ZAyxc-mmIEY", "ZBmGYK_PKWU", "ZlshfOXn7XY", "_1p6CRs_W4o", "_5MWpxEBnmQ", "_NNBllMvQlE", "_PQy2cmQ9Mg", "_hSj_dMMGyM", "_k6KT6C1hmI", "aydtrFANWLU", "bp_Kbq7n1I4", "bqyZ2QvoWnY", "cD7KkLgeOzM", "cZHcdM6bRDs", "cZnlCn3aXiM", "d3jM38k-JSI", "d3kaLiPr7Do", "dK500bR-dLs", "dKb5Mm_HzWg", "dYZCmrgCK-o", "e4CSjqeOUTE", "eNjT2EmdUmY", "eUpLB0Vn8W4", "ea3VTHa--qk", "fKyTh6vXxPo", "fNYj_lsjZk0", "g6hZRPkFCDA", "gCExB42lE80", "gf5xb3FNdro", "hAhQjiA5fiw", "hgZPUO0_g3M", "hloVopBAevI", "hnrpUhmUFBk", "i1GFTm9zcHc", "i5o5KAtOdQ4", "iJQ_UqMxH-Q", "iPjeFE3hrng", "i_7Je9M5zDg", "jexl5c7S8mo", "jnsSN6nflgI", "kEJTb0xp4l8", "kKhI8cVMe7c", "kfv79Gi8YiE", "kjhrwTjv7Vo", "kxi1nZ93-I4", "l88GXCfZaTo", "lNZjlx8EM3s", "leDOefyjSv0", "mFQT-9jBQtQ", "mPonFpdEKiU", "mtIxHTUtSx4", "nMI0Kj09f24", "nehOj2MOFig", "nnyNitNsSI8", "np5XefPf1j4", "nrc8q4bmAMU", "oDiTN9e6spg", "oHi9qNPhwKs", "oYXtC9eH4KI", "o_wI3DR4SNo", "ogdm2nF2s_g", "pNtoJR0a1qg", "pP9LBTqqb-M", "pSWVpTtW9jg", "pvEptOeLD2s", "qGoWcDFCRzY", "qJ5U8QwLop4", "qakGcT8YhoU", "qfH2VZvSp_M", "qyoQ5Rp2abE", "rG0QztKxq-g", "rL1raz3b8OA", "ra5Ew79B64Q", "rbxfHm_6e-k", "romJPjGtecA", "rsA5IFJPAhk", "rszihFbGoak", "rwLXQcB6wYA", "sk1MnW6UjVM", "stNxUHDCIBc", "tEgBWqdABTY", "tVfSpS1jdIo", "tcr9jHCq7Ps", "tnBlWq3IRzc", "tnklg4Ue6oo", "tp6oSODfbFw", "tpZoZp_wJzo", "u5ALO9HkLIw", "uDiMzwh_4-s", "uMbnRaRpZi4", "uPMQKRRiTDc", "uQsq5MEqLCc", "uYRkj-WA2qs", "v-uAtm-til8", "vGwSOMYjZOI", "vhA6IwgE4KI", "vnXVrMbnyLU", "vx48B-5aFMk", "w9yIQ5WMutk", "wG6q9JJb-CY", "wlNvsDjJB8g", "wvIWbU4qvyE", "x1iLQndxYTI", "xAdNyrj6G0Q", "xKh6HmCKfGc", "xMcUQgBULkQ", "xbi4DYZXTTE", "xhEcYe7TK4Y", "y7Tc7IFhqdQ", "yEOzYcr5p_g", "yL7CmNXnypk", "yMT44jSL_4U", "z-MreTdcjjw", "z7bhYHiVhk4", "zGbOFtUc29I", "8kqmr6raeas", "DmiET0O_wdA", "JYukxC9bmTc", "Ut4EfpncK3Y", "f5Vq1c6HC_A", "hkBuwNl3J_Q", "iyvrQPepIPM", "lCKRv985Z5Q", "lO3038iaqmo", "zn_lcBACtOY", "-1SM1dzBap0", "-H0xpiGWJ3M", "-YgUoEM3pXY", "-sOOSyVQ3ow", "-wTGwDm8lp4", "03aEt1eZuTE", "095UjcP_pT0", "0FUFNG7VxWY", "0KO1T2RJfRE", "0NOoiTVpsHA", "0vKkZpF1Ldo", "0z7BcQkNOpA", "1-6iRvt7hBo", "12iUl7q7oq4", "1JsydGzM38M", "1pionya7v2o", "1x0tq4Tv4ts", "22QX2Oed85E", "2FV5f0hoIQ0", "4kqEiBXE6_0", "4vGNmzJZQf0", "58EBYJv2huw", "5I0Duqd4Fa8", "5VUKjgcs0u0", "5cImLs2bm2Q", "6-vIKve4LtE", "65VjhjA4ZTs", "675v9MuGKa0", "6A8ltkCfZaE", "6NqmaDPN7qE", "6P1LIAG12hY", "6cpPBTs3cL8", "6v0pEHrRbTM", "6yjwM4JAKJI", "73RaQPLkphc", "7A9WtUO_iBM", "7IlyDpnmyUQ", "7bDXFieSkdI", "7nOFosB1eZA", "81C7lPmvb2M", "85wjHWr03xk", "8JZrMQKcyP4", "8xvD-Np6uRI", "9FLlFpJ3k5I", "9gWqGv_PUis", "9mAIapmyThM", "9wmMWBMO9Yo", "A6JROubV1Io", "AGC7ICvJ0YQ", "AHeIpLENNSg", "AO-g3ED1Sp0", "AY_Fk3Z9SZk", "AqUie_Rhj88", "B83fdRR4HFs", "BawOeC142e0", "BhT9E0_cvEY", "BlHOH-pmIP4", "BoJExgKlv7Y", "C8yteEo0cyc", "CLZtgR5JVDA", "CQcRiqoJzjY", "CmU7KCTHQbY", "D2kQL1IM9vM", "DD8YK2dFS54", "DLyBM21UHAI", "DXeA55pJnUc", "Ehg9dpZhpLE", "ElA-_SuuAAw", "EvFddSfGfU8", "EwZA5PjpGuw", "F-jasKuWv30", "FDg_OV8dXXk", "Fy1xS_GMcWY", "G5avIyGBYs8", "GKWjaV0NLYo", "Gu7asxy7d34", "GxN7Yttx6FI", "HG8aTQtqX0g", "HW4O8covNrY", "HeUyIBdQTNA", "HtIvX2RqDDM", "I3ejIFlqwrc", "I3g14RE3nis", "I8yohTOMe5Q", "I92eLM04e5E", "IGzMQJaF7bU", "IRcNyTDBxV0", "IaPLqDvxuLg", "IbzhmwJB2_A", "IgbsLQT8YEk", "IuXKiFLq_60", "JBCckCN-e5k", "JKlqTVMWS3I", "JKvadMmEvPY", "JMRLOXHF4BE", "JgIfNFHZ7WI", "JhgyyVQDP1w", "KazduasVx54", "KvzYsiqoH1c", "LY1vLBK9w7U", "Ltmzakax59Q", "MN7mfR8Vs3U", "MQhX4IPuCqI", "MW67kTY8JZE", "MqvJ3pGc8hY", "N9sv4Rn5PPE", "NKoLuFl04WA", "NManhwowsTI", "NZ6xPV4Yhf8", "Nf1eVEVc-dw", "Nq1eQdEgdOc", "NxnxmKp4RjU", "O5ueJ9DGNk4", "O9x0EMWV2iU", "OA5PufFDZag", "OmJcQgUHcZ4", "Omr03J7KqYo", "QDJBXQrX6vQ", "QKMijXzDejw", "QiUT_w2Q_SA", "Qk78jIUWMzk", "Qtya3NMH0jM", "QzvKn1haAnE", "RIY84PVhcWM", "ROYan5wi5mg", "RnsU_8hVags", "RvMp91YS5Mk", "S3MQQuSFfBg", "SDJUJJ8RQBM", "SWIRE7LyPgA", "SioLFIYx2xw", "Sp1i1oc1h_4", "SsUsgqwfh9U", "TLsc9nenF6Y", "TRXNUtstlFU", "TWnGYTVyI2s", "Tw39661Pzo8", "UAkW-aKEduQ", "USYZ-09UzGU", "UWhZb4Qs1SI", "UWmIEIp6erY", "U_tU1QcnZcg", "V2UZp5G5MiQ", "VJZVaUd3Ldk", "VrHWNesUh2s", "VsXH56VpQAo", "W8BeX6KhzQw", "WT5S7qIbomI", "WWezfld4TMY", "WXH4hwFyH2M", "WXn2b43wilY", "WetwMco7m5Y", "X8Fa974Qt58", "XB-Gu0e3Hs0", "XFQtjZZjIbg", "Ydf2XheuLmg", "ZOGltXCSD_U", "ZVYvyaI2zhc", "Zf6jqv8ShhA", "_4kOSVBWVx0", "_E8yIqh1ZNo", "_vviiGf2nJ4", "aEMmoHnO93M", "aPY0m-fXp8M", "aTjr8rpaK1o", "aaNuzfFiCMo", "b0BGjfiqcCo", "bGNrquChjGU", "bOYNh0OO18U", "bOwoJfbJTGY", "cKbWXZNr77U", "csM1bMahCTE", "dsPMwBMK7DM", "e7MGW2b15Gk", "eEt-23bjv6s", "eVmVl4Tsiz8", "eWbOCxUGFjs", "epB7EFJheII", "ergoFh_uGkQ", "fLKQmEtYvO8", "fOCKy38EqBA", "fRv2Bxbngws", "fYN0VqYB7s0", "fl51o1Oz05k", "g6hP4K0cJhA", "g9BVHFvpJfE", "gQDF7o62XSg", "grmSnjiCQUE", "h1PqmDYEkpc", "h3oVljoSpjs", "h6x-gHUkB_Q", "hFDlXMxifUQ", "hQ78hhUK_Zo", "hbq8-7k4T40", "hlsRjfnFZa4", "hup3D6QfBZo", "iIddZonGx3k", "iL59yu4XuBk", "idFSon0QsPg", "ifDgenu-izM", "ilblC9U7wf0", "iu3w2Ei3Uik", "jCrkHuVILaQ", "jUJtZDaFIDg", "jVNy3N3ZW_E", "jylFyhQ4fDU", "k4Zx-iLqWBM", "kLH71eCvnWk", "kWe7P6f8eAQ", "ktqQ1j_JXEw", "l7xPG_Xd4Vo", "lB5Q74iVelU", "lJimGJriv14"], "Jazz": ["3j_Iy5ODn18", "Y7NQ9-071W0", "gEfj14rjzlM", "lv299ssm4n0", "-2Njekj-SQ0", "-CL3BH4CyME", "-Eqjxpvzyfk", "-KQ1Nr0A30g", "-LxJJ-ZLz6Y", "-jVWvg-GESo", "-kMSHiuC-9U", "-n1_RsOnuTw", "-sBU9efQeI0", "0BJpOQ76ft0", "0NXnJpjijNs", "0OnNPe-802Q", "0SrdNAuYrVo", "0VPXJHTfCos", "0X_fv3Ornb8", "0iDfQ5h_rNw", "0pDQigbUqjo", "0pGoL5H9OZs", "0zJx0Rh56Ow", "1qyVqPqkWXM", "1ugjsyNpPRI", "1vGBEi4Rui4", "2P7-67Wwqlg", "2eibVyU6lGs", "2k68C--XCYY", "2mUOHO1AHUQ", "2uo_Xmi-mdU", "3ErBZxNItpo", "3LXN9oxO-8I", "3L_XC_Cinew", "3SsahUUJTlA", "3cijhDwjMcc", "3dU7ctxKA8o", "3koVJ4dE0es", "3sOpSc4N4-k", "40HcJx1Ps-Q", "4GY4pqMHPOQ", "4M0sihnoirw", "4aPG91qm71w", "4awtZOUVio0", "4ctNTFIGnJs", "4dQdAan2nhc", "4dxLfwpe2FI", "4nL4ZsMQq-Y", "4qIA8C8X_xA", "50kq2cmcSzk", "54ypt677HT4", "5L6Ux1i6PyU", "5_t0Cps5sOo", "5f3Um8_VRIY", "5gb4nYAcstE", "5s_TP_bD-kU", "63hUWUcYP94", "65wSPsYFsLg", "6NO9M8fmTcg", "6Z4523hT0GM", "6gY_gOIq3jw", "6ggVm0UVEiQ", "6nz6oMJBl6o", "6rHPY2sVbbc", "6uulHDJKCH8", "73wBqc5zaSA", "75lXm31Z-yo", "78GIIn83lQc", "7OJvOkJadcs", "7cheIp1ja_8", "7d6LH4Dbn8M", "7dvFku-HvWM", "7gpMdzfzVRE", "7rpI71NuRHs", "8HpfYO9GXi8", "8N9LuNfrHJI", "8QOGuxtI5AY", "8TEG0oXB2C4", "8WvT_8TrCOM", "8wZQJlIbkpE", "93dqOoXITSI", "96nLOB5aeJA", "97k8jD9Bgd4", "9JW0pylK5mw", "9K6L-vDXIMk", "9T0U-DsN8Jk", "9VUnBGRMhX4", "9Yfr0NcN2g4", "9_XENZjpXF8", "9h9mW12sK1I", "9i-QefQ2By0", "A8V0y5RKCUU", "AEjWAhkwCu8", "AMAImDN43sM", "BCfzznjyoBU", "BTuTYmpOy5w", "BVxcfGZMJK0", "BZ9C0q26E6E", "BZZSkJLYVBg", "BproWNrc7SI", "BqG2O17rkKc", "BxW0H-hcXxM", "Bz5K3OQzzVc", "C4kteljHnWU", "C7jE89ZVsE4", "CLY9YG844NE", "CmLHAUkQYYo", "CqKy23-vFcA", "CyZUN7wEAhU", "D4XRMi_QDbA", "DOWAaukVfTg", "DffvxyBWazc", "DxDY7VTFwmk", "E7RHQwP7LGo", "E98umwiNeyI", "EFvAiO1obVA", "EGnoa0s8wT8", "EX7fRJE7cwg", "EXtQXN_905o", "EmgK9l26c70", "EvM58CdgaKE", "EyvMW_2BVyk", "Ezhj9YqBY0w", "F-H-eDtpul4", "F4qjrf7KGVU", "FO2Ab6rXNbw", "G2bBpon0UuI", "G3_HUjRJkC4", "GFcdJkEpneU", "G_XRY9nh2NM", "GmcL9KpAIBA", "GpsDU9Rkr10", "GyJZ5sjDHFo", "H01W1gejh4Q", "H68YlEHdwEA", "HEPW10N0YS0", "HVxnLS-lbpw", "HZqvOOvE-w0", "Hcv7f_KYa38", "HqrSh2ox2hE", "HqtyylYFnas", "HvD1KN-nZyQ", "IF2CJ6U_8CY", "ILX-h8-iPdU", "IPDAB4IUXw0", "Ihz9xDIPJSY", "IubneslvwYk", "Iud_2ykBWwo", "Ivs7iGW54Wg", "IwwtSfTUOJQ", "J4Os9vf8vG0", "JQpkxBysjP4", "JlM2bJMXZAE", "JpC5hXefoV4", "K6Hf0vt88zg", "KIbaIjNfQBE", "KJvFM1uS8ew", "KL7UKvy8Tak", "KMaav6YSh2I", "KSKPi59E6Yk", "KYjgQNhq3N0", "K_JVBE9GxIs", "KaOJyncGiBk", "LKc4rytdndQ", "LVIw8TAYURc", "Lcx7nV9ntaw", "LfzynCJ_Lv4", "LhDHrbrcchI", "LhM2dlDij88", "Lm1mWkNwDDE", "M46i3p5xBy8", "MbOpQZcXYoY", "MielV5R9kbk", "MmVBbFbK96A", "MxUce-psxHE", "N69BU14XgLE", "N9CWIdnMims", "NcjMbA6SXbs", "NdiuNgrZdpg", "Ng1gKfZF31s", "NiF11WvFKX8", "Noa1i8XnHzk", "O1fXdFy2z7A", "OOypsPN8bDY", "OVCRBA_RbRs", "OeaC9nsLsj4", "Op7Ig_tOmvM", "OpmKqwA3-yk", "OsXPI6RbG8w", "P9-6Vl32aOA", "PZ-lLZeT0Fg", "Px9HA3KUQeM", "QAOIg4BijuA", "QUrzZxCd1mM", "QW62VUnoS1A", "QdR7JsZ4DIQ", "QlTQkppM1r8", "Qq1RlXOoWAg", "QqQbPY72rFA", "RLyHtjzxRCc", "RPB8HQOd4Uk", "RloZRL4KAZo", "SGLNasbjhe0", "SZMRuK4KLgg", "Slglc1M8V4g", "TUioWmgL84c", "Tmke7EIi3JE", "TrXJHI-O6EQ", "Ty7DVCrS-P0", "UFaHji-0C1M", "UGCxrt7Gcb4", "URfE7yAjgE4", "UfM_UjekxGw", "UkHZD_8TprU", "UnP04NP9CbQ", "Uq3LkF_9RHU", "UwoVqJfO0Ns", "VAQAylEJvs0", "VAUBX8Xgkdg", "VFDq8_PRyiE", "VGnmnMGtdMc", "VvbJ7GvOYtk", "WSUM_okyP3s", "WdVjyZg0W64", "WeygKRbBV14", "WjChmQcB130", "Wt-7j9h2TOc", "X-ckmm0iEuQ", "XGqzBW6vlWU", "XNqfxPt78qs", "XVCmP-Xrjt4", "XZOOj0NE7cc", "XjPdqu1o8XI", "XtHqtSGS4AY", "Y0mv4NC-7mo", "Y0tSACoCCrQ", "YLNw40pROmY", "YT0wueGPZNU", "YU0gzkgTXro", "YVTe7tAP46I", "YX_fPlkKhRI", "Z3APAFgw5as", "Z6WyIBEA-9g", "ZC_xwtSkMQs", "ZJ2PxrkO07I", "ZYxjmaEofeU", "_Cl4_ZBk0MY", "_RfsKDCch1g", "_aj5UZ3mb00", "_bBLEbsWxo8", "a3oVoKvBv7k", "aDZ9cB8LS_c", "aSureWP-syM", "adBfa7xLvck", "aiwG2kOK8hk", "alEi3HUNd2E", "auUuLLfAz0I", "axJt6jvuWos", "b2Lo1uyg2xQ", "b2T9TZgrH24", "b83f_7vo6rI", "b83sA-ALIeM", "bBr-Tpf10DU", "bTax-yBO4S4", "bWCEvVDagAs", "boMvMHFFsnU", "bqzLUiPtwN8", "bsBS-XS_wxg", "buFbWR0IgBo", "c3s9zmXEDiM", "c4WDB9xOe8Q", "c9Y-ybRN0Tc", "cGScGSvzZ0c", "cWq4bLAjgeo", "ce79bWJjQWE", "cjqweYsSSxM", "cnxmIB-Pdq0", "cwaNSUutLN4", "d12dkVlCbPA", "dSfF0bV8Pxk", "dURZb7Z7AHE", "dVa_0Mf8HSg", "dXQ970j7c7w", "dcouq8aKjiM", "dlz-dyuYTN0", "eHnhRS1pAe8", "ePsGYNHCexc", "ehsWoykPZHw", "emZb7oA_PLA", "eu3C6VAkIIU", "eude3Ib_6q0", "ew0zezZ7I4w", "f-Aqme0h7o4", "f-vPfJPX0to", "fEw09AglDcA", "fMtRPvKQjTs", "fWS78RcU9ik", "fYY34m60qDs", "fe9APnSCbIA", "g1IOaYUBrcE", "gNOr-bDVH0Y", "gpkVMH3YmVA", "grTZe56D6HM", "h1mpiKXDCW4", "hE0O-bCmAis", "hEzhyt5ZiA4", "hGGYA5vrTNw", "hHIhf3H3hYA", "hL67Dh0apC0", "hXF6cYLd3W0", "haWZT92VbUs", "hauwoWMDBFQ", "hjPGTw8q2OA", "hvBSueepKaE", "hvgMGUb2X2g", "i2uv7-9ZZvc", "i3liZqzEkLQ", "iFNuTcb7V8Q", "iGZL4Nb44yE", "iL9uDrbOz5I", "iO3HJnYvbKo", "ij6IF8k6cTs", "jDqAmK8Et0k", "jR9iLR8s9cA", "jZ4dO6JYv_M", "jdEGVdgpSik", "jeOiD07S5Po", "kNXOb7kzC6c", "kOdNzcOvGVs", "kPh2S5ObsNk", "kVR2QXl_wog", "kkp5YP-LlLE", "kmAey_nmZ6k", "kvuYCve6hKs", "lR8ghxilMOc", "lUu1I-rlyxI", "lat8OqE8kuY", "llh6bcVyXDs", "m5oDTUV4U-A", "m7oKOrCHSQE", "mUJkJLWbh6Q", "m_m2oxLgqns", "meBoH_MXUng", "mlHMw2oCjek", "mvb8F0RI7nU", "mxfnennYk_8", "n60eeP1HnqQ", "n8c8Y7lc2rI", "nB0zBDiXkrA", "nSJge5uOqjs", "nVLU-TSoQQg", "nZmxvNfgi1Q", "ncuclvCK2vg", "nifmh6CFmR4", "o77jvII7Qmo", "oPy5gA97hgk", "odDD1f-iymc", "on8HKJ4DdOw", "oqWLMZdJo5s", "pOifkZ2q5V8", "pS3Bgk4AMlg", "pSIPCYCwNEg", "pVtZ7aFPXKM", "paQx4w2idY8", "pbNmk8JwN-s", "pv0roQ3okYs", "qHPc5edPKH4", "qTlNPQmou7k", "qfC50VPnQoY", "qhB5KAnMW8o", "qqPmZaMCzQQ", "qxdVBFtAr9Q", "qzPkRVn4uXw", "r23B2m1VMF8", "rUxxGGnqHGI", "s-e8Rwg5tks", "s2fgZo9wpeo", "sDgb_7eG2Oo", "sHb9idw0PPA", "sHkJsax6YPA", "sM6uuyeCDCA", "sNMbLPAY9kI", "sQoz8LWiFGQ", "sgDDF9kfydI", "snroFNokFZE", "sy2MFnHQJIo", "t5WYl6HIYPA", "tHjT9YwW0lw", "tOIfTI1JqPE", "tRAThtQ-OTM", "ti4WVj8vZm0", "tlTXeK3hC-k", "tvxnPO0gzYA", "u9ikbPjMB_c", "uYq6ZYh2Lh0", "uc7nl9ebNtA", "ufM9WsK4oZw", "v4sUBPRRxLc", "vBBggONVtgU", "vC0Rn04CuTo", "vVO1uf6SYLg", "vflNcwR48N0", "vk1mPbKzqUc", "w_oOeNETXQ8", "whOUapqB4ek", "wl1k_SDJNRo", "x2x4RQ1rsQA", "xLck-DA8YxA", "xU4zqg44INs", "y0oBSpdptHo", "y3QIVy-exs8", "yB8oOCt-kAs", "yTRvl0rDgGQ", "yU5FKWJue7g", "yb26UuTYP3Q", "zNFJhTGoB0k", "zWoFt_FAGng", "zYn2rRamkS0", "zbqo6XRnJvU", "zmQy3MTNw6k", "zqe8gvl7HCc", "J-IXJsyzMCo", "rnT-jsq3k2M", "41uNP3x2twg", "5bHSX-Ya3ps", "5lLJPyQTjRo", "61Ft3iOeskM", "7aX3zz3iELQ", "82VZXZjkfnw", "8Wqt8ZLlw7I", "9Z-WhKN38M4", "BfQsImsY2KQ", "C1Y-MlB1fgw", "CKDyuWzrSvU", "CYfW5EbPXbI", "Cpd-znpaakI", "GSvlBGeLU20", "GpKJdDDZvjg", "HMQS3ijDg-E", "IjHvXeVkBnw", "J_T1O4WgVm8", "K0pOzh_VHDs", "KmxJ5rs48t0", "KtGguUQbyOY", "M0HsXVhWwOk", "MdhVZXF1GEM", "NoJJckMSSu8", "RJ93vEZ2LV4", "SJr-8HY65aE", "THFIkwrJgZw", "TvpCD_dCPrg", "XPWPVOz77M8", "XVb9MRmFR7c", "Y7NUJ47QejI", "Z-MfucjvmkQ", "Z7GlZISrXDA", "_L0f2ZUsUCw", "_ZhuOzSVowg", "a3-Wrnp0CeM", "bIYlNVBN1AU", "bN9Wy9UoJEA", "dV2LDQmeoy4", "dkejerd5MN4", "epSJG_ahiBw", "fptp2Exl6Hc", "g-QZ1GI93i4", "gLJHHZRLdng", "iC3-ttDeeJ0", "j-O2fdWKrKk", "j2SIziCoohs", "jgBygq7uHjQ", "kFtlYvSnxG8", "kSh77CLKBLU", "l4wn3bec0RQ", "lKRb8gWPNcw", "mV31qqjfgi0", "nq-Ok9FfUSE", "oc0mLsSClww", "qVXRMDa5IOc", "r7xeGUdDBuk", "rbHpB6pn8m8", "reA9s46FmRk", "sJ2DsSrCm_o", "teKzWeKCBQU", "v-PnvuAR0n0", "ve5gtNzJd6E", "xY-0-jlYwXY", "xeYgghgc4Bc", "zgpRwNVzB6E", "2-d36CHQetA", "BQIAuJwJedI", "BdxBJL4r9yQ", "QP2rJFswyE4", "VH5sNzApdyM", "kwSqLWcHvY0", "-q2woZRTWhg", "0-FFykOx0g8", "2DA1-sM9X8I", "AO4FHnM1OIU", "Boc5zU_0Rts", "CpWvWijyChc", "F58F2o3WQok", "Izcedcy1qQY", "Mt4nUYL1OFY", "P9_h-i3_3Mw", "WpbbVj1iFlE", "fJ8GprcBBvc", "fngLFmOSY78", "3mYLha8sUY0", "4NGB_xD2k0Q", "EDCiAtlWjtk", "GK5eiironUQ", "V7nHeIxDrgU", "Y9Dxn4Ga_X8", "ZFNpmR9IJag", "cjMTOuNCCEQ", "dh8pFUIZW9Y", "mViugq2TysA", "nmoqn7FwvQ4", "uC25LwDJxpE", "dFHYIosyoE8", "-4UYvmhqMAI", "-C8eip1x51A", "-EXKOvTDPdE", "-EuyF-HhAuc", "-UlzqKpuvVE", "-cz0IVdj31c", "0FONkZ6gLfA", "14Wmy1AOfck", "1EEZHGn1Zjc", "1PbL7q_Noco", "1QOlXHQ1xlo", "1V9SMCEAfxw", "1giltwNTKeM", "2VXHHqE0sMI", "2e82UetMJMk", "3aeS1QScjXc", "4CwYrnjutDs", "4JxlxWQGZWk", "4SWtb005uq4", "4VmZwkqi0HU", "55-TDTUt2Dc", "5Q3SQZs_RYo", "5dfwiL3QSdU", "5o0Wc5Fm5mQ", "62Zu0Fmzg_c", "68ijOXV-MtE", "6I327Jzo--Y", "6PYhhtu9CUs", "6z9qwdK4XM4", "7Za1bI3a7Ic", "8OzLPUAJUiI", "8ZJSA_puVfE", "8fRX3TdemQw", "8qOePms2G3c", "8ws2wfGcRls", "91bPivDOqCE", "9U9OTvYq9o8", "9VQBh1D698U", "9wV1afxgN1k", "ACQSJHkAIDU", "AQHdf7z440E", "AZvjNijeXSA", "AlwSe3LmYX8", "Avhy23Mv8PY", "B6LrHwafC7A", "BEID7zlYavg", "BJ-JFsP0BEk", "BOXMI2m0LUY", "B_WDba8tk04", "BcvgbcHnxp0", "ByLjmShZGZo", "CC17BK6qrok", "CbXuPDbMryM", "ClIGRmpsbh0", "DASdOnNeFDg", "DJrYMQEKTro", "DNRhe1Jeljw", "DRlMy6xmvxw", "DW1yTvyh0mA", "DgSkCxNt0bM", "Dlx-eBUNrgc", "E-olXorXTMY", "ET3nxQ8sj-U", "EuY4XaRfMtE", "G086o8qAveo", "GNzmVThey7c", "GOSu1v1-bjc", "GQ04kPgY09Q", "I3PXHzy8Ryw", "IFrkYlPAZI8", "IJuysI4fCHQ", "IQpvVnHdqck", "I_fmfmivDZo", "Iaoh8YictCU", "IeGV9JQCp5Y", "Iepks3De1Io", "ImLc03Lu-hc", "J6pSrN2bwNE", "K-Sv1qya4YA", "LItwZG0mbsQ", "LWMI4s3zC9s", "LmlfAnuKE7k", "M07R-9Vytfs", "MJyPlecdckg", "MMww2BFczkU", "MeOXwf2HVec", "Mir959i7F2M", "NiUKKjzyBOU", "OPvUfl_RbMg", "PVyb5y7i8Yo", "PjblJFGiJk8", "Plp22gITo3Q", "QAeCF6mJOrA", "QXPKXsOK9FM", "QbyXmWkvqjU", "QcrqVFyLVYk", "Qxu3np_yINQ", "REvydL3xBYE", "RS5dWWY3IOo", "RaJFsHXkpo8", "SHJNhsC2II8", "TP-oYlkF2rY", "USOPCCFvNOE", "UaP0THTcsHY", "UuRjSlEpa5c", "V3NFuQWFU44", "VMWdOOrBxVo", "VgP-WGjY98c", "VjJYlMlOxDw", "W9ACslSf6_I", "WM38wmPWvTA", "XAX2nBBuNOA", "XvlFcowfTOw", "YYIrPXEx3Ys", "YswmyxLhV9Q", "_RqNcr4Kw5Y", "_fjhQsnlbUU", "_lsDYKNxguQ", "aDYtLYNQoHA", "aEZnmyTNKFQ", "aL5l86C7aX4", "b5awGE3UHkg", "bA2DgVozNHE", "bB35NWrWiww", "bH6UIZDonOI", "bvj0kQav4n8", "ccFwekl3JOg", "crf6XbWeZmQ", "d07TBtFc4tI", "d2yIsdymv-Q", "dP5YxB9f9ls", "dapaHMEG2ik", "eSI4AS_fBm8", "eUR_4KcGxGM", "eVaSQqJ37gQ", "f4LaBOkZaKY", "fHJzkfdF6is", "fb0PreFr0g0", "gd4C9QxGuy0", "gh_CY6b_3iw", "gslsQGcx5-M", "gtTKFJpDokg", "hFdZpqgbOCc", "hnBZjhR1N68", "hurAP07ZdcQ", "hyvPmKvxaqo", "hzFozbN3uk0", "i4L7FNK0siw", "i9YY_F67La4", "iwrh9hqK55k", "j6Ck1kaETIg", "jYs_dTPgKHo", "jxd3MyAA628", "k1_pOhheMpI", "kZ1yZ_nCn-s", "l55rGQf_vvE", "le28TVNPEbY", "lkXmh9DFzeI", "lp2FlxFAS_A", "mfMKm38Bkfc", "n8O2VPDH8rk", "neZ6xQ6WgnM", "niM93_GlFeY", "nyZnXOhfBzg", "p8agiZ6v30M", "pHYbc2k1UUI", "pKtaZkwLlaI", "p_db4OXYJVw", "ps1lFxLQ2P8", "qMPIcqXGUWo", "qX18KapXgTs", "rw-lBfzmNrQ", "sjOUFMasOQs", "szlcKLHvyUc", "t4WHHIEBZpU", "tbvuJ-JN28Y", "thtilhatc2Y", "tmdFA-bpNGg", "tvuNgQTYfCQ", "uaYKGdQa2wY", "umUCN5basoQ", "ux7UQ2_XXJw", "v7pJpl7q9sU", "vN48d6p3ekM", "vOLLGNtPtto", "vhy5IL9jQNI", "vvOwLleEiKM", "wPRJy0GRDl4", "wZltDku6Phg", "wnJj6PCo0zA", "xEMSy9LlwZY", "xMhtTF3leEA", "xiihiUrHJ8Y", "xmAD0TzR4C8", "xrqs5JFmw4E", "yBvGhgU7GwY", "ysIeGJwG03s", "zAnWR2kfjtI", "zq55mWIsFIw", "-JFDB9porJU", "UDm0nDRScvI", "3SOpsD3AxP0", "9I2QmedXIMU", "LKyn1pxICWs", "VLxAbGgYx3g", "WTocqiDvgyI", "WZYLZ0-c47c", "Wn43CxHXPY0", "fkKlrBHJnjE", "n7aOlUlkz1k", "Y3FVvNUPMuM", "aMaKTQKx3f0", "b5GlQ9V5GWo", "2VDiq9wfp3I", "9bsv0j7zMas", "AlcGSwjtRF4", "AmMaGgIuOio", "CG-lehF2BC4", "EF8qrrcyemo", "J3PPJgg5g1A", "MBL1UodmzvA", "U2Y8yUbnN70", "YAz2fqRTHm0", "_8KYXQ25fWM", "agPhy2qIqq4", "broQiG221M4", "jjVQzPidUjM", "-ieJ3_lW1L8", "02sYOUig680", "0Sl-CNOU6o8", "0dr-XRT_QzQ", "1CwdkpraPf4", "1E9Qk-mJXdA", "1a-33JpJBpA", "23AyH6vPfm0", "2yF3E1iZy9U", "3YDBfeKZb5o", "4bspQdxr9Aw", "5DueRszdHjE", "5GFren1Fyfo", "5JQkGIs0VSM", "5XdjRYtHe2M", "5jC-hRtbzDQ", "6523dhBOus4", "6El7HmFo7Mk", "6Xmwb-zcg_c", "6b3wWgRqEDM", "6jxkevSSbMo", "7GImaVhPbs0", "7_xX_OZ2QBU", "8YKmgsgMiNk", "8YiEMnzzvVc", "8mOE1geVu4k", "A2NH8itT9ts", "A47LTW9Bptw", "AI7jcfUfoAs", "ASNqn2fd6aQ", "AXdPQIuBRQo", "A_VoE0wqj3o", "AqE-eQdXCiw", "BH8wc15C-WQ", "BHL3raGTJ5s", "BLx6fnxQw_k", "BSWRUfmoWys", "BmKJyVRXD5A", "CSLuNPdkFgY", "CTIDDq-fYv8", "D3J5-9lffTo", "D4V7uVQJlpE", "DPu9uk_zuGo", "D_sWjEQJGJc", "EaoyB2tRro4", "F7KmkeDOTnM", "FNCvMwROTlU", "Fh2Be2wane0", "G4RL3BEsrfs", "GjLBGOIv6jw", "GzpQ_ftu5Fg", "HCZ0c171pVQ", "H_eKQY5hCU8", "HgEVfao9Sok", "HgtcEaPQwnU", "HvAobCXY7u0", "I3fp-_7YAyg", "IFirn5FjHAk", "IOLnZWkvCYQ", "ImT-XkRq9Cw", "JQRJ3GNd-xk", "Jldtj2mPWcY", "KGImIh8NJzQ", "KOO6K-JKhRY", "Kv249Vmu4BA", "LN8YRYm2RyI", "LkzYe7eo8mI", "M2GAlMyTvXI", "MTwYtDU8JTU", "MUu2h_0DBSg", "MY6cKA2U2js", "O7yxkKoMmLs", "Ok8usRTKlnc", "PLFKze6dgC0", "RCnDFMura4g", "RNRoiAetfRw", "SS0x6K0zAA8", "TZiHWf0eIUs", "UTTeUvtJU9A", "UZ0aqq32Cpo", "VPVWcSK4Ecw", "WSo_pGbFcC8", "Woa69cVAhXA", "YKoxw-gs9RU", "YdRu1Pd1pyo", "YkpG4gBShxM", "YmHG09ZXXz0", "Yq3zsX13nqs", "Z3mRa1468zI", "ZR_6uZucEVY", "ZaOkix48VbI", "ZhbgKdjmg0U", "ZkCKUUalHdw", "Zr44hUqCs18", "_-JbXkblm-w", "_alvEvonT90", "adbAFy_LaXs", "aqYSt0z6ZjU", "azva2gXDtsw", "cSyHqc05FgU", "cfNqBDbdrbI", "cjJaXMB6LJs", "d93c9sr0z4o", "dU2hC5qEyiA", "e54tZYHYun0", "ePSdc90wFxE", "eXHmdgsRnso", "hFn59SdFraM", "hN3VGQGistM", "hfWtGE3xc14", "hov6XXtZgho", "jw4oSF4ApZ4", "khZ05GKipjw", "l8n1lxE3YCc", "lA3usCTFccQ", "lEQwXZRd2Ew", "mP4Ykghv-cs", "nLxRQmAvdhs", "nrxXcs183KA", "nu-ckUdERdc", "ocaej9hFWD8", "p4yc5zdAgpE", "p8QCcc78PuQ", "q8fOLqZk51o", "qKO4fgjOy2c", "r5-bL4wDCS0", "rhyGdLEFVZI", "rmm6dmdikMI", "sYnQEUjZov8", "slCXBVoSuKo", "uL7I3IEaxLg", "ucIf10aeFXA", "v0DqTZBaPwA", "v3ii2OzJObU", "v9e8jMhysnY", "vI2Lwq9o3Dc", "vJ64h3QtAHI", "vMzVJlKZLPA", "vSpb9skf_RY", "vTQaUOhWeJA", "vghXLBY0378", "vtrFXUd0Hxs", "w9gemWuY2P8", "w_FvXgNaWoE", "wgUExOOdEZY", "wxd0EH3sm30", "y2rXJMeklUk", "y5nmQ6jFiMw", "yQrz_WlpXk0", "-aL_E3gdZXE", "3RWcv_x72MI", "4tH5ieZ43is", "5aLGGA6kmCo", "8i0X8FgZ3OA", "9yAsDW7rBYM", "AkfOM3M7AQ4", "B_LOHmFxbOM", "DgbocPVu1pE", "HXiQFN1G8vA", "LgbhsXEnlvE", "QQWavEbk21k", "Qz-qc9iCAPY", "Z-16q2AcCh4", "_u7XXTmz9es", "ediQWGggGtU", "epeKTHiZeLo", "fOnBjzoiy9U", "fWUF-GQIpd4", "jh38U4Laju0", "mgqFCNXEJHc", "pRRxF7Sio9A", "rhUe50jthe4", "sy1ocCVkCQM", "ucmOtt7rmp4", "2r2QnXFVFsE", "AntnkzfC6q8", "H6-4WtDBd0s", "OECFG0lET0M", "YpHXKulomHk", "-zDSBWspTVs", "0cvOwKkA-Gc", "0nyQoVmLWbc", "3yTI2c-H54M", "4y8yCJQzmYk", "5ZkwWp2QACo", "7C7bNJZIziU", "8VCXvYSMAYA", "ALeRVICBDvw", "BfaAlhgNReQ", "BlLq6PkV9M0", "BvL_oTKqD5g", "CA31Ntz2AbI", "CEmvgvByD_g", "CNq1567NMB0", "DGQcSPSxpok", "IoAShNtfzWI", "JX9jdETI8Zk", "KhW9udJeMNk", "MCEfApIlZfg", "NNg-MbuWShg", "QbNlZSx8Ha4", "R7mlWQx6YUI", "RdDPMrj-u2g", "RrvAvqZVpgo", "V39H0yPzqmI", "VH0xpDWHFN4", "WldAlAiV7PQ", "X2M3hJf4YpA", "XkUs3z6jnX0", "Yhvdf355uzk", "YrgKZUSUyug", "ZBWFcg_R_aI", "_wyzBXz6o6Q", "aQ9XVhbz2CA", "cHpbXFxjhIs", "cREn4kLqcYk", "eduVFKf2IM4", "ewJWjxDfM_0", "f9e94AdJxF0", "gl7-dV85WkI", "is8RPHitgqU", "jDWoiRbxmrg", "jTcA5AfChYs", "kyb7jARIHPs", "lREnkgx-mU4", "lYxNdXBnEPw", "m-LSDTCVvu8", "n_qWqINN_Zg", "o4Uold77cuA", "q2MMTVjWLk8", "rG2XVeLDJos", "rjFtLbVUhEo", "seP_iKAtqck", "v_FKWgRR8qo", "xgPQqat1Hb0", "xj4Inv3fynU", "xkXLZ2k0Abg", "yJdgmF5J6C4", "z4H7XQ6-0QQ", "zOVwDUpkIBk", "zdaApyQvThY", "-38DGPONJ0Y", "-vLR-GP0Jjs", "4yAi6umY7v8", "9Mof2L3CGzw", "DBXBIopPRB4", "G7_igemtFPo", "GNKHePOgy5M", "R-1DyFtd6gw", "TcjF7Lvsthk", "UXKlLiI-WAk", "XtE1YoyzWHY", "i-ap1yxzqME", "tMz7jCTkuF0", "vOubgyqZ3YM", "wqkc3VWzRIM", "jCh_z_i1NXI", "6BWrH3pWFRI", "DYIr3MwhODo", "HuzR_BMbty0", "Ib_7hVUcPro", "OIVKmSufuUM", "Puuv-R6ffq4", "Q2LQzZfbdwc", "gXFGspShgEk", "lPJ4fNm2bcY", "oG4gIbmzF2w", "zGtwYBFOKFs", "5BYv14UgqnY", "7pzT1uvnMIc", "E9qDcWxMSnE", "EItACQjGJ68", "KE78NAiTmC8", "MFSb65PXE8o", "MOBm8_Gl_z8", "OJKJ_6aMeYk", "TTRbOnYLSLk", "UBeqRODpPSE", "V5lgQT8CBTY", "XudYhl-It0M", "_O9x39wJ9Is", "_i2brL4jNag", "bHL4LMEbrlk", "lyP5RJ3qtNA", "mgSmL4Mq-L8", "nva0_GLd9Oc", "xddaA1XGIY0", "Me7EEmWcMkQ", "Z8XM5B1Muy4", "ZxpXg3OKrJc", "_nErVqDT_Kc", "kLELvMfDk1U", "-BBGuFQ6IVc", "-NS6MkbFCMc", "051yxxaMmaQ", "05v5sxZqP34", "0VIHJVv8iCw", "0WMQQ7epgz0", "0mDLElhmCZ4", "0s2bLRQnOgM", "2A5Vzr9lmys", "3DVT5BAYgKc", "3F571pvYL7s", "3gxL97GF48I", "3qI7dW0DA7Y", "4f0a6Rf7Ors", "4jn8sRYh60Q", "53nThwglS6w", "5CJU6unp8ms", "5Cu9_vJAgiE", "5lfZn177Wog", "5myl7BCmnx0", "6BUAb30FpNM", "6q3Smo7BF70", "6xIV5GO0V4U", "7Ar1X41qm-Y", "7Ax4ODLR-0o", "7Bl6DI7DPWs", "7IFRLn4EEV4", "7M7tWfk8GDc", "7xMpFg56qXU", "8IcUhyM9LI4", "8tuZ1clq4I4", "8urgDv586Rc", "93ni-mRK120", "9KN-NqCz-Xc", "AiMiX90uKlU", "B0RlvVGY_Kc", "BFuzBnIJ6IM", "BGgrdqbtsZE", "BTK4MyDdhEc", "Bxi8CEvF3l0", "C6znSqPK1qc", "CDUlRnyudqY", "CEGVjasOzuU", "CIv8knFdvSE", "E5OQsjQ-nUU", "EHy9o7NDmoE", "ETw3HcyL5Jc", "EY1ErCw6BvQ", "EbK5YOxBpzM", "Ez34vrYSN4c", "FKB69HslCTY", "FW5sTjlTL8c", "Fp3FtVTymVg", "G41lXTeOUdQ", "GF8teflA55M", "GKDyeEYRQX4", "GkEe_CAmtyo", "GtxepX0TxSo", "H2GA8sCjwaQ", "H4lFaMULEpw", "HJj1Qo7o8Hs", "HV-X8deidXk", "HXs5dz_oxTc", "I3taWuOAM3Y", "IA4ocxG-hsA", "ICZS1O6Divg", "IfBGYYwEBn4", "Ig5sgcdaPho", "IwIiGN8beJs", "IyoA907Cd0M", "JHmyBmM5wXI", "JN15gkn5sso", "Jn6PUFRNcPI", "JqPDHE3Mdrk", "KVdCQKJDOo0", "KrEso6LcctM", "L0v99boWAhU", "LATp59p5aM8", "LqYcQIzJZn0", "Lv_HpsUynEA", "Lxzq2tR_FUc", "MC31UK30W90", "MLwXXAbWfNs", "MnQwausoBbo", "MyY9MQQ6bOI", "N-e5-iCcbfA", "NAWZw4bwJlI", "NvABSWkyJNw", "O3az9_0Q4Yg", "OAm6kaPPye8", "OTPVEhbLp0I", "O_UCfU9ploY", "P-m95bbN9ww", "P53zg_x8EPw", "Q02_x-Qy6Tg", "QBNMdMziJ1M", "QJ9n9xPzzEk", "R04N9W8luo8", "RAkOwOyzPt4", "Rw6wOnAkTMA", "Ry26GjnpXj8", "SF5je3aY-Ak", "SGQRz8eEXeM", "TCeJZIgymbg", "TDVhRc_cmb4", "TEJ8LHcHEEA", "UEH024Cws_w", "UMWsJrm2CSk", "UVIxLkqVhu8", "UaVvlvr-h7o", "UsTl9mWq7uE", "VBltmV-Qnww", "WG8rj7p_MJE", "WaHmHYYlSUc", "Wb_bjD3ntFM", "WbrT7CDJKWw", "XQo-lSmQWbw", "XRIkb_8o5fk", "XaE2GZjTIYc", "XfwVuJ4jRUE", "Xk83a8RNCvw", "Y1dpV-ERGmc", "Y2vvvq4cn38", "Y47IlTAAJo4", "Y79L8-tEoPc", "Y8qrC1mt7iU", "YnmiTxUMF6U", "ZuCtA9CTMWo", "_7FfAk1T4Qw", "_dwN0JAK_EQ", "_yyNdbLBifY", "a0_fi3YdKZQ", "aoPBbweNL5k", "bHQFe1Jt-KM", "br6JQ4Nun28", "cezlJv04R4Q", "ciaIToens2s", "dAZEszhbYPw", "dzUGI2hfZzw", "eSKoKwy18hQ", "e_2zx8bTOcE", "fVKq9nP7fU0", "fnM1CI9xp7U", "fs7B9XLW0bw", "g16G7IOza3s", "g1P5Bk08_tY", "gRNnjbD20Lc", "gYXVYQvQA7I", "gp7R1-hrRig", "gyHVLWgXd6k", "h-VGtPnbpQc", "ht1jLKm4Fc4", "htGY-9ApFW0", "inmu3Iw55Gc", "kPfLPEeNg8k", "lUhxCcIAkYM", "lvjarg0owN0", "mDb-rd-peBM", "mFgq1vHFn08", "mVfEsqQ9hTI", "mb-batnCNbU", "ncuQ58p7rs4", "ndqEv-JEawU", "nmMywqcJxRk", "o-KLYEivNF0", "oPkJkdPxJ9A", "oako6SCTX0w", "ogsqR3O_E3Y", "pAypjwGrx30", "pCHYFWt-q0U", "pIZ62tWs97g", "pMhDbd55HpM", "q7-GNon_zjs", "qWHHDwKMhRY", "qnH44INpfX8", "qto6FNdXnso", "rHsV8XTYxfE", "rdmX3s3-el8", "rjcMzrQj3jk", "s3nB7moETrY", "s_ZiWlWnM30", "suc4SfXNrzg", "t3Fr19x68x4", "tSPV5oyOS1Q", "tgBo-CJZukk", "tjmfZgp6p_Q", "tou8Hl-l6Qo", "uJiI0hpfz9o", "uxHOUSSVlV8", "v0_hBkZZTz8", "vh0NNkE44hA", "vpYa6LuJxGw", "vsshZ1XqVgQ", "vuW-Ae5MdSk", "wjKiy_q86-I", "wvJcy07cMvc", "x7OhKxwASfc", "xZGd3i4-jYY", "xeVG8HHIs58", "ypD9d7zZZSc", "zH6kRuzkHHA", "zbQtOonu_YI", "zhW1TMCxFnY", "8V-XQKlWqw8", "Fqztp4r_9Jg", "3rdiioYN96M", "lTesKVYcI_A", "lehHiDpXN44", "aKd9KLnGg_g", "17ax-jVWQb0", "2rWWWXbpL7Y", "3ccJSGfr-xs", "3n49q7K_Sy0", "4-Oa-xbWdMU", "6_p6Zk9HgSQ", "7tqrFSrVHTk", "BCvPNILaqEE", "ChLnC9YYZDA", "CnGZZijN8SU", "DwfGxWFCXSI", "EEBiboLX6sQ", "Efo5NnXYl2I", "EwndeqyO19A", "FA4bderAEH8", "FTdHWKn_fJ0", "IA-2Ei23bHc", "J5T22PMNguQ", "PXj0NA7dURw", "QX3Dbgpf6cM", "RC05tKfr96A", "SARtwKBSMu8", "SxTaQDQHbag", "UJJumGxeaBw", "WXrOJvymiNo", "YhaQW27qAUU", "_1cYzYTtcSA", "aIElaEmCd4o", "b4P6kEoBLlc", "cCu0ihksetA", "ccbRqp0T7Q4", "fPrrOlaPdpk", "fdIxucqA4tk", "ggDmsftfdEY", "jID38A3Q_pU", "oMplrTumJfE", "rJ8MAlr56wQ", "uiikC8NQo7o", "yhIdwKAhLL4", "yw-iNXz4ij8", "-OEVUARmfkg", "1TFKyU_CMGM", "5uC6vYPrjSM", "9dgaf8bzffw", "A5tpdlMYJrY", "DOqEPnPbvKU", "EBAq06UeKOg", "Jmc3XujPryQ", "LUbzvP9rCFQ", "b6ZR1BRZuy8", "bVTjaUAVq4k", "gFHEg8PO1i8", "g_O1AYxE0rM", "jeroal3CCzc", "ke3Tf7VuiC8", "pw79oQ7H-zk", "tza4yf3aH8U", "uFdDWzN7HjY", "v1F3dXdc8Vk", "vsJ7llY199E", "wR02QZR6hq8", "x1CUSh0YUIg", "2aSPnaqefxE", "CRTwhWuJAeo", "IWiSCvIW3AU", "OJOnkhAtgOo", "lparByhYBcs", "rtSbrb-YUnE", "z_wPKVM-KVw", "orKNoDcM6Kw", "9DchQDy25-w", "3VEvNUP9RYw", "9b6KaekG7so", "DsiQbugHpY0", "ROrFlhCl5-M", "fF1U3zBgBqo", "2i13W7wtGq8", "2pvqb1r0cUQ", "4KWaO46fn2I", "8Nms1RAMi7U", "Gu-BfSOpln0", "IrcyC4Y99aE", "Pc-0fgum5F0", "VNyVVtcToUk", "YDYld0yHiQ0", "b6Qzi6xtzTk", "gAlJvBk4Aw8", "hWrOjp8AvXI", "jsYsXTlIhAM", "jtytDJo0oe8", "q8XkecQUMDI", "qnam6P9XEd4", "vR03itK2H6k", "vipmZksx9wQ", "xOBvPd1CjKk", "xZdwY-HcX6c", "EM9v_yDlYME", "Pa7B3-evx0E", "twf8RnghgVs", "usyEde6d9nc", "8hCndP22BUI", "lMr4-NlDiNQ", "rkiSFYDMQ3o", "Gu4KDyS3I1U", "MF_6MKH889Y", "NPVbTNGlRWc", "VJ4g2O5nq8I", "fWJcesWwf7o", "mGDIGK9FZeQ", "msmdYD1RKgw", "ImWOJhUU1xE", "P18zwFajy9Y", "dp-6xgmgSYA", "-fyHpKgx2HM", "105_OQh_jWs", "16uQng3l9aA", "2Kgc0sNL8_U", "2TED61IE1xA", "35SumNZy11o", "447_6hmPb2I", "4pvnVHfllTM", "5RdCmjz9xk0", "5Rhfa_v4q-g", "5fOIagEW33Q", "6VfbH1Awp38", "6Vx-hcAqijM", "6lujhqaa6Xs", "8zuOLSf-76g", "9BUIk9XDyXQ", "APmIo8flhos", "AZm5s6yXOOw", "B22OV7ejR2Y", "DcvP98bCjDc", "EiKjPFz-sFw", "EqWSHW19o6g", "FFcvuXv736E", "GLm8nxtEtJ8", "IHygptXRLo4", "OsK8t-zG7HE", "PDixXg2AXcI", "PQE91DKo6c8", "Sex8z_0KFD8", "T5Z9jRY19To", "TA3Luu92C5g", "W-uAHhkQk30", "XcaF4LuYkCM", "Y6NKhxFHbi8", "Z26916jrMAE", "c4c6duaxwKc", "cT0P53AwuBw", "ckCYAqJdUF4", "d4ZLbvQoVcM", "eATJuzE7dM0", "i3XI4S_GYRQ", "j5f06R2yy6k", "j_YMR6gok20", "m1-KvfvO_hw", "mFXA6cT6hMo", "nmXsaoCTdSw", "oKV8IVBVk-M", "oX1f0fMlW9U", "po5hNjlMNxI", "r3qt1A01rIY", "slnr_DcGD_A", "t4s_Gti22lY", "vBX2DgEJOLw", "xNWRaS8scJ8", "xil_5MH2hXI", "xyF2lb8FjIk", "ygugs1N2lpU", "yuJZR6liKx0", "mJn90uUVMMU", "sq4kFP-vIhc", "0xgIo0hqX14", "3nrXaaEvhoU", "4erd6j7W1GQ", "6vTEHzkndNQ", "8esQ0KjSRiM", "B2y30snlNTo", "BFaH8aNPprE", "CpC99lT_TU0", "G5GpYek_Xyg", "GaIIozEYHaA", "K3NfbCe-bok", "Lfjmwlf2ftk", "QvkuRNyfQEM", "R3omvL290dE", "V_wZlr4XRH0", "WFBGe3-ByOk", "ZIgk2aoZMX4", "eRjcZc1-NjQ", "iY2wAlAQ0Gc", "jyWB88p854A", "kRIfVU-fykM", "mK1FxNCTAZY", "p35wkfNXAjo", "qIPgkQPm78I", "snQEmtdiTjg", "uBR8p0rW3Kc", "z9EdCLdcn1U", "ApLCAbDyjMg", "BMkcx8VVUPA", "BV6r7wC5P-I", "Hs0TlXl8pzQ", "MR3sOdt_ekk", "Sf-Yh_ZUnuk"], "Metal": ["5GVDkEIsmkM", "hWtGc7Ksl94", "n7mThiRQlhQ", "ymjG_IZIC9k", "-9jxSjl-kdU", "-OW71HXRDnI", "-cC1kFliBFA", "0GKKVa4S16w", "0Nj9eoaNX-A", "1I00-kkGvIA", "1kAuwYr-J4E", "221XP8dKdaE", "2QpBH1eerqA", "2S2DSwJeVnA", "2dcpA7ejNEc", "3UOv5pWYCHk", "3yAB93bvSdQ", "4aw_3qFh8tA", "5MFthNnDgtk", "5NEaLeJisnA", "5QaegA41VO8", "5TiYsiF6hX8", "69vqoJJDWoY", "7hjU_LMGamE", "8Ledzji5n58", "8bkbIMRi_MA", "9HsxsLaRh1w", "9q6PFaPlgWw", "9yArVNPsMog", "AQ88uUZE5dw", "AiIQ1AXWYVY", "Araj_vbz5Rk", "BJoUMXS4cno", "Bz058naWv3c", "C5lCegKI-Og", "CV18tCcwx6k", "D6Rc_OxpIjk", "Dbla8KnWml8", "E19IgBDGPqY", "FWJ74AbBTOI", "FY23Vwdu7Ts", "G1nyNaSGvYA", "GJayp4C-lMc", "GgOiMAIj4K0", "GngXN0gK6z0", "HGTBypksKXU", "HVWjh7zehd4", "Hhysx11PJOA", "IqsGfqZuV6M", "JbJmFuWHUGg", "K-hXYUKkfYA", "KBHW51WN8I0", "KeNsecVdEUo", "Ljjog7fRkiY", "M8nSWUvf394", "NhwcHYnMKJM", "PGadj80cqJc", "PQlqTF1kPeU", "Pt7BW3qsrUs", "Pz1kmgyNPcM", "QiqHFg1uJYE", "Ql-5MGSPG68", "RFubltFPkOQ", "RocEEu1ysq8", "TydLgWLWy0g", "UcOQhr8R9Nk", "VBc9OPsTdp0", "VFtQsQ6QRvk", "VnzLvJ0BZoM", "WOaDf9RL27M", "XOix9bXmrxs", "XRAKHBhW96E", "XVTugGT6msQ", "Xew2AdpwLO0", "Z32owkC2oAs", "Zq9D03bKeE8", "_ZXHpRFZDps", "b35vaBG7zx4", "c2xQb-9hRu4", "c5e9a9HpDB8", "cZue13y7GrM", "e1bz3REXqSw", "eRo_yAa9zOE", "eg7Ii8djBNM", "fTeCs69dmIE", "hjxwFsNbkmc", "i08gZbvns3w", "i0MOS46_mLI", "i5-IxnqQVnc", "i5iV7CvQ0Y4", "ieSrhc_maQE", "ig_zrCMNbZc", "is_wLsz_FcI", "khD5Ri8rC_U", "l9NxWU68gtc", "lWagqVRo64Y", "l_eDgGPKf-o", "lxPwdEVsP08", "mOiwD-olKE0", "mtQuj5G_M-U", "n1Mb4PcA-jE", "nXU-YqzlF8Y", "nco3Zp4PPSQ", "nfXDSZSqRIc", "oVplP-XmLww", "onKTZHN3mEw", "qPzmwn-RetU", "sBk6q22HHZQ", "uhslan95UYg", "uiQpDpH6sTo", "uvrS2LLwgCs", "v3wUFMeTsF0", "v9FRjhrQ1qw", "wPRg9qE_eNU", "x-s8__E8o50", "x1Fk9ehsnk4", "xrwn6ffO874", "xx0M6kP_kv4", "yIyEORa0GPU", "zMXy01d4Fnw", "ztMmnxjbX_M", "-B4FY4Zsv1Y", "-P26syVyKKI", "-aJWoA-atm8", "0DuRxuLQy04", "0QjLAiSNo4s", "1mdWZ9ImArk", "1qnhPVurmXE", "28oiHGeeT9U", "2Vixn_JOv-Y", "2ospzZ8Ot4s", "2sRxTwdx4DE", "3NOhMBhVtbA", "3PDAzOJDS20", "3V8zuj4pH0M", "3WJs7bIjMlc", "3Y87VcHas4k", "3pGFg7j4B5U", "4-PC86qWGFI", "426ozBk3CaA", "4ig3DdprAqQ", "4kXRyc7q1sc", "4xf0-5RpPvY", "50xhrfJmVe8", "5ZEdA8IVHlU", "5bsn75BJXQM", "5kmwmYNrjpg", "5qiYTDtP-uI", "61110mCqZ2U", "6KmnakNz9Y0", "6LQ694zbJKI", "6coiXLpgjGE", "6tjO4u1S_qY", "70R35a28Dnk", "78jJACMH1qU", "7BPbquqOtBM", "7PDKfoAUPEU", "7a0X4k8ItuA", "7iIO0-oy8OU", "8Ag6ek_8XdU", "8DviW3yhFnk", "8Sph3UGDrg4", "8el1_vkXby0", "8yOrNW6aSZw", "8zkdpQDtF5Y", "94eGi-brlf4", "99NcpCK_Qyc", "9X7SkPbsF-Q", "9bpa64GIu5Q", "9unvgTzbYiI", "9zCuZW9UEx4", "ANRcLbMPOL8", "A_8bKpuZMdc", "AeSbCWIq2Io", "BJlV_aKrkHk", "BLVAz7IPnKg", "BW73rDPVAU0", "CNYbM-DWMHw", "ClZL5fM8kjU", "D1lmxvZJXGo", "D86tLOt6jdE", "DDRTnRvFaFc", "DJHlz7hZS-8", "DL0Z9pp-5ZI", "DNgDWrqIxfo", "DOP9dyd-4-g", "DYQYLPnMjeU", "DhYBTnDnDrA", "DtDC8J6j5fQ", "Dw-QiEtpmm4", "EIGk_5JACJ4", "EmvaygMJQ_g", "F9dBuFa3rNY", "FFyYVR-3cAk", "FiIv-8fNOJ0", "FmzMPjRKfLE", "Fv8PMQ5wMeU", "G0t7x5XfQVY", "GEo722c4u1Y", "H3_-7s6bBaQ", "H85ewZl7udk", "HS1U1XTpSjY", "HjzDVAzYxzo", "HsM2FT5BDO4", "HtAFPtDQPu0", "IED5xGaOtrE", "IMHjUVAhUe4", "IpK7IJN_0wg", "JHMajGSfJQ0", "JpmI4n4zi68", "Jq8atZthdLk", "Ju8GjIuu9HU", "JvvzBTJ13NE", "JwEM8-z-JDw", "KLTj2WcKNCo", "K_xdHqnRqqQ", "Kp46fhWubLA", "L4dJNVRK2nA", "L5IwFuAxTUk", "L9_L8iiJJAk", "LBsKO3P_rzQ", "LgFuwmuEFaw", "LsmIChm7ez8", "M67RDB2ydK8", "MgpSPXpSUVE", "MjAsfEeIdEQ", "Mkg4Zck3tAY", "NPb9f6wOI5E", "NgBH3QePNis", "NwCFwFe3aZ0", "O1m36krXRRw", "O1vrtQpRl_U", "OQdGo1gRQ-g", "OZbTJW876xw", "POnl1GWt53g", "PpdsziFmhu4", "PwLIOi9QScg", "QT2NpZ8sa1c", "QitngbyPjCI", "Qwl3cxalq1Y", "RJ_gC4HrAR4", "SEMkiTnXOhw", "SFi2P2b-VYY", "Scfn9px0dDw", "SpyNrQWyz7o", "SwmsWmS9aO4", "UNTGwQsfUWo", "V9zy5Q9cZ2Q", "VAHH-LCy1wo", "VI_rVEPAwwU", "VMgQum4Lrqg", "VePp8AKsQoI", "WAPHMPXN9gc", "WEsMbtQJ5QY", "WTy2-eBqOAc", "XNBttDKFN4c", "XU7FoD98Ft8", "YP5n-LuBIVo", "Yu_UsO4SskM", "YyLhhQOqBIQ", "Z3KeHfJlUcA", "ZBYKduG-PGM", "ZHtGdsBlG0A", "Zn3gLkW1Z08", "ZnkiIHDXW-M", "ZsfS_voXrZ4", "Zzlykns0ZeI", "_0dA30vfvqw", "_Em35habC7k", "_QpGuq3lRJU", "_ri20yZvaWU", "a6wkez9egPE", "aIjRG8yv4YY", "aNDBUqW1V1w", "aO0SgTl2faM", "b5uyaNtB8wo", "b6-GbhN0TIc", "bIyPp0tM-84", "bSgC4Cv0RhU", "beXJLDVNk-M", "bzeQSnriEF8", "cHfYwEopZJw", "caeX_FCwIDI", "cfF4G16ez0M", "cxRSwTnei3w", "dEcxR9zTelU", "dL7Bm-VcNEU", "dXQkXzUyILk", "dnQG18iVqwg", "e1IhEmHxnmU", "e4Mk8VNzZxo", "e4Y2TIN4M4U", "ek4IRL-L4P4", "f2CMqohuX0E", "fKu0bPbv658", "fMSIdcYbrHk", "g0-KQznUAfM", "gTgTyW0Lhn0", "gptdy0IRW3U", "h6GBU0E38as", "hDX2mABYJDg", "hJrFI7tTh-8", "hZfLKc0roBE", "hsdeSGl1lts", "iIQRHN0ZZi0", "iNNFz2PhGjI", "iYdleMybIzM", "idsS0DKvUBM", "jaEC0BMjuzo", "jrvI2tVCqVc", "kTAgTGbo-Eg", "kXUxatMORYs", "kt6DNVBbe6Q", "kwdCf6LTYwc", "l9SoHwS3jkk", "l9x4rtnHgoY", "lZiIVCZib3k", "lgXP2rs7Sm8", "liROTPySlL0", "ljnsZEM5YBs", "mB_uSV0Vj7Q", "mD7jOeY6qro", "mKczNZNNDCg", "mZBLiQ027Xs", "mhz5dBej3kw", "n02KNh7j40k", "n_pkgQkvBr8", "niJNwoh7QSM", "ntuArxrv3wo", "oJ8j4pWDnzM", "oUCVVWPNDK8", "ocTwMqoHujQ", "p7ViVRs5bII", "pRovGYhJSdM", "pb5nhZlLiSk", "psg4AjHReXo", "q9ACV8S5mNw", "qcxdY8ByEko", "qwBXm1CoMVk", "r-dv0YRU6Ps", "r04LxUpt284", "r2wtAAYTW3w", "r4WemJH6u5U", "rO_0h2d8JdY", "rSBrjnZV30I", "rvZOGL9bk4g", "rzldbUEH8IA", "sGNCm-uqXzM", "sNEDUS5seyE", "sRiVHIvFrg8", "sUUrG5bluhk", "t6iysFxGQeg", "tAdApHicNzg", "tTAOgAGbWB4", "tUVxOPd5ShA", "tomTmZWPzxQ", "uNm3kipwj0M", "uXizcFK9RyI", "uZJc58cKHsE", "vBlMGtcdeTs", "vVoX39dTe8Q", "vcfX8tpBmCo", "vqbI_G5hZ34", "w35gFBu8h04", "wU2uaN4YNLw", "x--Goef09Lg", "x5tdJwbPeV0", "xUaAJEdn--U", "xdv1NKREJKE", "xzCGK4CckGA", "y5MYPG-aDw0", "yEE8tFInIRQ", "yFskZbxWTe8", "yPGw0h-6DqY", "zuWPhYhKX9s", "LW5Wqh3tzvU", "dhSnnnRpQ8A", "y1vXD6SeipI", "-Bg0C6DVVSQ", "00_bKJwZhM8", "0HWTz0oGIOY", "0PV4Z1D1WFo", "0U0Ys7EF2cE", "0b5Arrblrzg", "0bTM15eE3gs", "0eu4NugvLNw", "1JCeHwZGYmM", "1hASvBalccg", "1l9E8pqRgAs", "1vmVeniKdnA", "251v8PchGts", "29R7sK55IAo", "2XRmdwj2y4c", "2dDoq55y9ag", "2xEd6qADcoA", "3BiT703PzlA", "3wPaRp2A190", "48N0GzMtFjw", "4FmHPPle5iY", "4LapaNWpwq8", "4jExgfuPEnk", "6UMg7dM8ndc", "8-npkHlzcKI", "8JW6S9CPz5U", "8TtXIF52XeE", "8mgznO8hqoE", "9DYXMLqPWNc", "9g6_RIefJgU", "AHC_JKF9oM0", "AiGRQ4CLLJ8", "AkDCMLXGQkM", "B05TeBN_Pgg", "BAG2LVvfsQ4", "CNDF31qmJKE", "CUeYQw3XW84", "CVWn2ZpX7I0", "DoiZN3syU9I", "Du_OfLFsY0M", "EL8Sl-N0rA0", "EMA2jQ_dSaY", "EQvliX9SHXA", "FREAMGlqEDM", "FVorYgl6Byc", "FgdnirsnKpE", "FlR4hWTYIlc", "FqYTNv7T9sg", "H4VWQP98v5A", "HfGVdcUe_iY", "IBg_sLjQKj0", "IC5-2N0vVgc", "IKsF_2ZvufM", "JTugL-r-M74", "J_Hdc0gejMk", "JtbhW4Tkq6Q", "K6nqqNnHxnM", "KQ608bpPVdo", "KoEQQO2IuXo", "KrUOUWA9Wig", "KxZAmgq7saU", "Kz5jsgFa6Ds", "LDzwfi_bYy0", "LRJfuyhpeHs", "LljpFHNCpyQ", "M91hmbKVx_A", "NIW5YQSTN_Y", "Ncwjc9puSzM", "O0DzYBZnibc", "O27mnQV9i_g", "O5j4-U7Ul1U", "O6FRiEY8K0w", "OEXOXKaPgQw", "ONW4RG95zms", "OPNvjp7pBKg", "OQ771qYJ7dM", "Ozt28xW5M-4", "PCHMy0H1j_w", "PQ5nGk06tBM", "PZgQhBBlogE", "Pwj7aoJ9_sk", "PymsfJNQ478", "QJA5shc9bew", "QLcTEadmBBc", "QYy9QjOb9Ts", "QjXdhp5f8DU", "QxVY4tQk19g", "R8ZjTjoidi8", "RFsp7FtkeJY", "RK_MCsgUcaM", "SMUokEsSCcw", "Slbyg-4hlq0", "SzTBs0YZI3k", "TyR5Y7PvXdE", "UBU4Ywycpts", "UBkDKyGR46k", "UPiqhpsNRlM", "UvMSM38Nn4c", "V3ySSyw6zH0", "VSkWQw2GSKw", "VZ49pl4TZXU", "VemkBOUc3c4", "VkrgB0V4zsY", "VqdbvjZjJPs", "VwEMs-JyPfA", "W3L2DDiTplI", "WZSxk4SbiaA", "WecSLZKdexc", "WpmsZKOuN30", "X4hHmRBZwX8", "X7cSbJjUkWc", "XG0LnVXNpRE", "XJjoAKYhnuk", "XkFyGqMgWBE", "Xvt5dyPK-So", "YEhLfKZ6KQ8", "YHKPJMU6UGY", "YzJhMWV6GyI", "ZjADs_XqCQk", "ZksVazjLLnY", "Zp2suRJI3uA", "_2rK771ZLTY", "_4F9Bn0A53U", "_zW1mUuCSyY", "a8qhGMtUPXQ", "aOmiGV6VFoA", "aYaC15Hc1M8", "ba2eX8rAuv8", "c7xekNLZBT4", "cAbNak6ZldA", "cUUYE2eWfM0", "cXStV20-ccU", "cZ_C8ctdDAk", "ctuFQNfKeio", "dAWL2klXc7s", "dbe22RKHQwk", "doy-3cCRMLc", "duHk7iwdEdI", "eOSObrXgoBU", "ecK8J_U8pvg", "eeCeccciF6Y", "gHFIm25KunY", "gLXg8V3z42o", "gLY-Z2n2lLE", "hk-7G3a6E24", "hu3uiv3j-gQ", "ioKe_-pdqLY", "ipbZchFkMjY", "iwiZvitdtVE", "j8VyukxucF8", "jRRvnkP2rM4", "jToGz-gTkYE", "jVJJJkUQkPY", "j_bmNgnpAQ4", "kpJxVlNq43E", "l2eirmSDxnY", "l4iLs755MqE", "lNmW0ZB99hU", "ldOfLMAdRyU", "nWfrLVq4KY0", "nX1MDorQelE", "nzBpbUbWsNs", "o9jyfBnWThw", "oIgDE9KR3HI", "oRpqgiv_GDI", "olcrSRu4TwA", "oykWh9NIxks", "pPDnw6hXLY8", "punmseIsnXI", "qAvP9qJKqlc", "qMMDVswEIU0", "qQYCK9kJOTc", "qqp9CKBhQQc", "rEu2guRJeEI", "rGnw-N6OHtM", "rLYXP2pJR9I", "rNGAqQC6AYc", "rqmAQZRba9s", "rwkjWzlEL6A", "sS9Woq5wCOE", "sTOHT-Ptmfo", "sdj1etI9HCk", "siZNDzTtDWc", "sopxnN9NG7c", "t6hehmhNTUw", "txxkj-fRrrU", "udTu0Q7cSCM", "ui0OXFmPbd4", "uqIY72qHjk4", "v1fCyxs8Y48", "v9fo-jBFUsU", "vGjRjLex4tA", "vJCKxFxeVJ8", "vWcyERXVIhw", "vakntu988yA", "vgvg3VHWcgk", "w2xpO_VrHz4", "wZVJjRXQMOs", "wfqsgAXxJLM", "wrLMXWNGjz4", "xDEWO_7MOuc", "xs7G7H5_l2o", "xuvwvX3CwzI", "yGtol7F4owE", "yTGoJY_LWNQ", "ypAeddwUTfY", "ytgeG288zGc", "yuGXRwcGcXI", "z-INABDaH3s", "zKq9yWCSlmI", "zSEFKVkctIg", "zZC5Bg6PIfk", "zw3IF5d-4MM", "-WQDvxFCnrI", "0J4VG4xrrM4", "0k0h_8en_yw", "1-yo3etLSRo", "15luyFIn5SI", "1B4f2n2cQlo", "1HMlQUml44M", "1PqoBAcO1M8", "24iMiUPDrCM", "2OriWsBgZ24", "2g_7bw7i9Ns", "3K4QXdyJG-w", "3re3vLS5T1k", "4-pcC5kPcVM", "4HxrqTBZips", "4yWFyWGZ4Z8", "55pW1NkcMNk", "5HosbfLKhmY", "5iH4oXYrNuU", "5pJd2EJ02MY", "5u08Y_3HTQg", "5zz99OY_aWY", "6QCTZcSBKA4", "6qY805kUpgk", "73rEU-0Uq44", "7BXWKGytx0M", "7i_1UVqlc9I", "7jhpiQFyTUw", "7uTJv-6WUps", "8-JWml9KXJM", "8Gno5WrtSxQ", "8LB-ru7htDQ", "8YPUBUmYQ_g", "9-6e2K5S-Lk", "95c8Vxb7JsY", "9D4MwqIlLGI", "9gbfm3VYvGw", "AW5FJK1oFKA", "Ah41u4eYfQI", "B0IKUsbMwcc", "BGOQb25PceQ", "CXWNK5Qngcw", "Cnmju9v0al8", "D7I_K2H6n8U", "DAmZyxrEO28", "Dfc5TD43xtc", "DhF_0EAKW_c", "DkY0P-pzmOI", "E4TbH5SH6rM", "ELk7DpySMjI", "EMQdZcOoQUk", "EWv9mUY_Bek", "FbBD-4M4E-Y", "FqU3IVZGgOQ", "GE9QUVG-MJk", "GSXFLBnyV7k", "GcmUgdNneFc", "Gk0hbLpwFpQ", "Ha0zzzXsN5U", "Hjg5uPklnUo", "HmULC6Nd2uc", "I8loa3_659A", "IW9loUo3k4I", "J-uztn6GHm0", "J1QyY96kv8M", "JSf0-08IUeM", "JTduDCsPjcs", "JWUN-4U-OUM", "JfUCpkSI_Rk", "KT0DTWiDT4c", "LOtS1OaCxvA", "LlhYEBWhQYY", "LvxFYW_4xEk", "MrhqM1kRDXQ", "OaVBM9Dniak", "OqUwd8hyGts", "PAO8mgVIauk", "QmaR-7D3mQo", "QrbTK0dPWx8", "RM_P9D1IoKE", "RNLfNOLwkEQ", "RuADETImQrw", "S3OR16c3NyY", "SBmXAcNMSM8", "SNikL-QHkfs", "SnUyexpfRNU", "StWtOmzymfU", "T-E4QqVYmLA", "T3A7wQdTqYI", "T3hYzMt5KMo", "TCrsSECHn9w", "TLbL1L4L9YU", "T_zcna6dNUA", "UF-IDlHMmiQ", "UxQXuFZdVYI", "V-Wr8atYpyU", "VDUQUASzrSA", "VVGwuPF1S4U", "VdLbtV21pBY", "VhXUyWSJWVs", "VkZZC0ldi6k", "VrBu3-cQqFQ", "WHCZxDuWiic", "WjFLOoVmZd0", "Wvs-G4n1Op0", "XRctbCsGHww", "Xbw9u0jr1Ns", "XmaZU74LSW4", "Y4pGBMljtnE", "ZNOukX1OvXo", "ZlkhHOKtvDI", "_4CkfnyZTvA", "_Qtp9nUgIAc", "_TN_SVAN-9s", "_f8prt6L6NI", "_gj1BX9Xn4M", "aFfcOFcBUdQ", "aWB_0N5Zs3s", "ahFAgBUn2HE", "bSt-ze2_26g", "cAUQ9RMLc_U", "cxOggfbuTMc", "dfilzJKCN1c", "eEsKnTGOdw4", "eT7VmQGrBxk", "eqXkp4NPV3g", "ex_NNbVHKLQ", "fCpfteyydyM", "fLZFEQVSqxE", "fd1MogB9ECw", "fogoiB047G4", "fu7ozh-Hip8", "ge8YzPlP9PY", "gz8cAgGoWTk", "h0Q9LYYxvEI", "h1exofhthf8", "h2Pr--es5Ms", "hQov29HPO7Y", "hUA1oOSXXIc", "hZDESh4psAA", "iKozUhB0xFI", "j1uJ-1aQQTo", "jTS9yac8vt4", "jeRAsHA706w", "kVd9UXB8FIg", "kp0LXAmdQHY", "l9zmz77Tkec", "lFpYOcz8ufU", "lPOWUdmRz_E", "lQpc1gy7HLA", "lYMvO4RtnnQ", "lpiJNcRbREA", "m1R4al8cWmM", "m6XGHeRkwU0", "mbD2230NHyQ", "n9QtBEy20fY", "nlfqFZoGnh8", "ot07eudv-zI", "p4GMRMIdftM", "ppbDnIv3JME", "pzL83TH3nO0", "qtszlJbcnkE", "qw5Jl9VFSxE", "qzqwXBjqNTk", "r0IEeRdOWnM", "r1LMAa5dg4Y", "raBd-1SQEN0", "sBQY81skduQ", "sB_pAidT2_4", "skdRVIcsM_U", "t10Ajcd_G-w", "t1NtS_VJPaA", "tFWrSmi32vc", "tGACpLyA9cU", "tLCg7ONuIl4", "tVGmhmI09SQ", "tusxb71lWNg", "txWuTuvuXW8", "tyseL22u2Qk", "uKACeBdRSME", "uWhbZlTLG5w", "vira9s3sebw", "vqQpFz2Hr6w", "vtR_j6hq3Bc", "wuxHf4GdzQs", "wxspNjyOGec", "x64MX4ZDJ6Y", "xDSLKxQM8Ws", "xN4JF9wItD8", "xPtGAM5vNCQ", "xUnSRiq5w9I", "xhT1Ak7JJeo", "yCnfgqPecWY", "yncEPB1z_es", "zBXtBPVpy1M", "zJK-_6AXoc0", "eDOFo7Fux6s", "u2HrKqBExx4", "4J_BKcM6s1E", "-w54SgTfEdw", "2uUGMf_cjTo", "3MiM_uixrTQ", "5RrqAMglVJo", "7fkXmWtSVKA", "7g3KNqtixaM", "7xykcoms77M", "9MgXUquiWlw", "9Y6z-76BWi0", "A5vGXDthKfg", "ConhjBsBNbA", "FHIE9Clh0yM", "GXFCRaKYSMg", "GtoafmmfY3o", "H56qkoVFISE", "HypAQDlcN60", "IRiDnBBppog", "IpDk06POego", "J7SOLE6qjPE", "JYUMojg9auY", "JvtDXpiDY0I", "KRjgQy0Oi6Q", "KjCt914r6BI", "KtqrMema8oE", "LkAmFiJXs-Y", "O-_IUsj25Eg", "OsXHSRzZpTU", "P-_OQIS0ayY", "QVDAtuDo82Y", "QuyDJ1tgKSo", "RJaYHqJft18", "Sswu7WROXXA", "TU-2r1Fd3Zk", "TbZgVd1mapI", "UH3QGkxL8sM", "UibmphfgyOA", "Wq9M9S-eeyY", "XiQ4I_4NywM", "_vflFaQ-QME", "aPhLVA-iw2w", "ajXC-xPB6yA", "cX4Xc2LGisE", "fkrrojsRzgM", "gJefB6e0ojM", "g_HdLFTD2zY", "i6qYhtIz3U4", "ixBRYkjvpVw", "kbGW3zry--k", "mYBheoLxD3s", "nFHOxJY7z60", "nq6BSI-9v_w", "oPVBKTnL8oQ", "oV90Djr_gy4", "od5YB8xWf0I", "p2sBMbQQ2RE", "pBAtCSnbZHs", "pVbMRPWWcl4", "pgLeOkP8JY0", "pt2eW_aX3KQ", "qdRkyZbV9Ok", "rjQ12PE6zw0", "sGjbmaGH9SE", "tV6UT5zATRk", "thc2QINtd5Q", "tw6tpvX2CSw", "uzOAjmrWGCM", "xoOEJJfWbSw", "y-h5nFCTqhM", "zOCPnbPQDl8", "DIs0GjxQCfo", "VumXNu4lkVw", "a36xYl1nsRQ", "a6DE0JXiBbw", "cp4KS9zOfvo", "xTULv3YfzWo", "-0ew6xgvjYE", "058GYb73h_8", "5IKUjBiE5jk", "6qlVhCu2s5k", "9li8PK_HKok", "B5c0PWxSacI", "C7iZx1YaLqs", "CLUu9VyqFgk", "CTL5cUfLI3w", "HCrvbCa0DR8", "KFYh_q5bG4E", "KrdkGB1NJN0", "NWvLvch9Iew", "Ok-btb4-Qxs", "OzMGPg_fc8w", "PuFhIZExr3Q", "QARMqLpCQXI", "QCWgUK7bc1c", "S4DFE80Coow", "TVV042lX6ws", "_YeTryUG8a0", "_rXbrKI98IM", "fBKfFHag8Iw", "fL-sCUMI754", "jcUL-rDx730", "kWsraLImqgo", "kYe5ijBaenQ", "l2UsWYJmNw0", "n3opnXPX28w", "oRpuKcvwL_0", "pSGn7ksZFxE", "prNpoy2ZIHg", "qRGnyhaJsoE", "sDlsdivAVq0", "staEpyY9F5c", "tLuA_hqfSuk", "tM2gk3kQBmM", "vo_PR2vF4vk", "wXYhd6N9Jxw", "yXLjNdQa6AI", "zn6POoNyE0g", "ZHMqxyRj51w", "VUHZ-o4JFqc", "08W7P9nXkjM", "0Rn0hx0lje4", "0m1GyJwFKT8", "5FQdtIaD8eU", "79velsG8JXU", "7JKVhNDRKSQ", "8xgBrlyZ5uM", "9BpKcrO9NVU", "A1zoGuYNTOU", "B0kb5Db-9Vw", "BDY0imohlP0", "CPDtSjX1BkU", "CgygfRPMZ-U", "Fku8KUZZ-lk", "Hdf1oNSOhwY", "HtK5tml3eII", "ISl5YX3a5AQ", "JYslIdDg3Gg", "JsI9-130BgQ", "KVVyI3S1vkU", "LSggBeUkujo", "MVS3PcZ3CMM", "NFhXg0EmH_w", "PsTMcMHhryE", "Wk20-waKTfk", "afp94idJMpE", "bdIjrTaSH14", "cTgofJAlIpc", "dPzoSauHoM8", "eXfeVzExMXQ", "fpQJ9FSdWv4", "gM1P9JlNL3k", "jDkD7MdOVBQ", "k6P6Aok1w_A", "lTxbJyn71yA", "nK1UxNExBhc", "rDmK9g31H88", "sKhQPhuTyTk", "tvGvePX50Ck", "xQOUM7j7Bqk", "y9yL1jBEU1I", "yNfT2fC6BHE", "z3v--LomTt0", "jpXOkyg78SA", "kWTxGBLlaUs", "nebVflkCVyA", "-Sb03B7rbqI", "05hSW7RC_VE", "7XQsJ1o0EUE", "7xDBRqNeEqw", "ArNNABHsuPg", "BsUb0KGBwHA", "CbEv31tyVJY", "Ev56OBSmPYY", "FWm7qRD5YF0", "JTttZhh8WFg", "JqZQIPBR78k", "N-rfrj84AfI", "OHm9cy4vzwA", "SfaC4wMQ7Vc", "YK_VIeSVx0c", "biD_2Mzknhc", "kSs_e_Ywdtw", "kikHjMYg8U0", "oVUkrs2Fh1w", "pBeEoPWU7mM", "pzZspfye_zg", "tq1n_tTLHwM", "0nSQmoXiIu4", "1q02l3yd704", "2B52N4SoBK8", "37Zux5-qsTg", "3odMn-HEkdg", "4BtWqL2qIik", "5ioCg4X7jRQ", "9l0dfen1Mxo", "ArvDRQrjU-Y", "Bk23nsmGrMk", "C9dFdlYYKaE", "CiPVbkXcKzo", "EtjAQlOyZ5g", "FqvVtqzH2fs", "GMl1RWZhvgU", "IEDlHCAdej0", "JfZa1LXsj5Q", "K3-0WkYcNCs", "KoM5H0gU-jE", "M-F9Hqc7jSM", "N2j79MQ6zQw", "NSoVpUKPkbg", "NUX8BleuG4w", "OIFYobSf7BU", "Oj5bY3eSMGI", "OxypOMxkyXc", "T2HSVzDPzIo", "TXOs-oCm890", "TqtYBAgO3Ww", "U9kQUkAAcyM", "UnNXfWAk8ig", "Uw-K9YWXeUE", "VBBwemBFoc0", "XAuvyJaqVgs", "XBhrCH7keEo", "XdDjxicggXA", "XnBL-8K2yx8", "Y-mS7VHqNS8", "Y8kMOTb30Sk", "Yql1E2T6LMU", "YraITVo5WSQ", "YzfgOkU3YfQ", "ZMZ7Sz0Jm44", "ZaWCBUsf7iY", "_62cI_oOa3s", "_JDSevPmWIY", "_PRhGbMu9-g", "aZNYAsqCeVU", "bgiOhG2qz6I", "cJqBWSiBCAM", "cdEoKt1ayCc", "dPvs1vZ09M4", "e35Dm7wNr2I", "ftpeVnnONj0", "fwEpe6z1kCs", "g8TcuOvmK6Q", "hgDsTS5IcTg", "i0tnRbPGxt0", "iR1TZ6DlLZA", "lJmWJVVYdnk", "mo2g6zfogpI", "nJWCOzi-EBE", "nuDJRsfVZgY", "o40P35UII6U", "oAfPuE4DhxM", "r4xs5noDF8s", "rD3FVNrg-i0", "tMiUj8he19U", "uhZrtqnHJ2w", "uzN6pWtwJuA", "wdnXNw6Dkvw", "xeyr0ZysWwY", "0ge7b5OlGxk", "0zAQZxuIb7E", "7b2J3DRmh_E", "NMzj0K97eU4", "OePheXvpHfs", "TYo8SDsVORs", "TwulAgL-1pI", "Z0dgh9DB70Y", "_HRg83OJCwU", "aLkaLJ8TlUY", "aMfjoYyrYAQ", "bS-ax1ZrnX8", "czRdHqAlPAQ", "d7qCUaeT5iw", "drAEFlSlksk", "f4g3VS4wkVs", "fJmoG8ud1qg", "heFhwlIoAxU", "jMPH7qvKnZ0", "n8bj9YKMMw4", "o46XYgV-JBQ", "pVJKrxxZPWQ", "qIRZP84_hoU", "vWaUkd8olXY", "xX8HA-RLKTw", "y0vcKBxvFi0", "5RZ8h2jtrRQ", "QGk37DHnD5A", "SzMX2QPFhWo", "g4O4kqwvPJM", "mI5itWcLvmo", "miydrMk7NaU", "qH1_xgKmZLQ", "wkUVRt8SCkM", "2XEmqYz7pV8", "7-nHdCmX2k8", "D887cmvJysg", "SLku0GJDk1w", "eqV841_0qSU", "nYD22NtbTwA", "al59gbqNJtU", "rjSOpuxwoVg", "lpydbLv76WU", "IThjEDxRYx4", "P8OjGX5LjSs", "XUaXyW8lUpQ", "cfkHCztyfXU", "zl3KbKlLZCs", "-Ti1lni2XPA", "-mhRsfUUgeQ", "-r3MZRlpkII", "1Z3OXMhcvZc", "1pPl4KvfCIk", "2IBYSSu7Yy4", "2kOn-domMfY", "35AjDzw-9NU", "3nhKYzW6pTk", "4XHZCfHVmuo", "5NrzbvbJ9sI", "5XWnUcUw440", "5gs86hDATV4", "7Sqi_A_43IE", "7_kFjmscGyk", "7lT5gUcaXnw", "7z-O35J62ek", "9-495cIRtV0", "AnrgRsiXGjE", "BhUB2b-5Oys", "BxxeUcRtdio", "CHYEKTKf4nY", "CasDTNCwswI", "D6yb-d_TjRw", "DKRVYcKX1xM", "DyAQw6xCobM", "EP3jtptdGYY", "Fw1VTTzE-BI", "GHRCQ4s4vxU", "GVr-0osyBxI", "HHazHmAyYpE", "HQxMP-2GrU8", "HsmpDsWNC8g", "I8mNNgjh8mY", "IVVSAKWur7U", "IYR10FusyB0", "JRwQ4ZHr6i8", "JanYww5slNA", "Jx4P3uHZ-Mw", "K7w-JdZz2AM", "KEvI9M8ITr8", "Lad8VJhv3gQ", "M3Rypidt0xk", "MEmuuxjFMdI", "MYxJ55ljVJo", "MsgT2bfDiHA", "NeCCgURptuE", "NfYUObBdp8I", "NgwP6S33eiQ", "PY0_-_QObvM", "PaLEWuN8veU", "PrL6o8V0TUQ", "R09O8owVSXA", "Rc1j-dO4v5Q", "SEL4-NK4cuk", "TdSDzXj_mjo", "TzxrSdYsR04", "Waxg8qsAv8k", "X-yfkDvB2BI", "XCqVB3t8d5c", "XQr9xxaXvlA", "Y9mvTTZFwRw", "YHEYWZXvEs4", "YuRpAgp-k4A", "ZXHNI5DwAqs", "ZoOIx-frrqc", "_anYRABsx2U", "_yqRSs3kdSo", "aGdenF64IkM", "aV0DZ8MKqME", "bHz6pkL5jkE", "bKgxI71mwds", "cviYlkCj8BU", "ddU-Z2ZmgXA", "duVu74_vFQE", "eia-ouLlqzQ", "g-yZ2bj601M", "g93hBwXunhQ", "gYFVbE3ewpQ", "gi-Er7LEgIs", "iI7VurCdMXw", "iO7nx5Dg0kE", "iRtUvyOGHqQ", "iWrbCVaPzio", "ia8T6UGFPs8", "jBoUTmwXqj0", "kZtKkPDIuCw", "l3eXT9KGKiQ", "l5jVlYQoe7o", "lO6yZMcacNg", "m6dBOK8xP-g", "mkhpd5obThs", "n4CSKogNo7A", "nX1wgwE-kjI", "nhIdXsjcEv4", "oyBbo3XFy6I", "peE-6p1qZOA", "pjmqEYV00YY", "qEK01sHmVis", "rJFL6c5rSNs", "rQTGzyfMmR4", "rU9cozJTQjk", "s-fR1ypCyyc", "s06nlPF3rgk", "sFadZE_IAkI", "sSUoakra9aE", "titgoKnSQ90", "u8mpR2WI4tA", "uFePUOiDHXU", "x1i_nphD9kI", "xCTOC-ueJ2c", "z2Ds5RuyErA", "znz-_PIcu7E", "5ntxDfunHSU", "PUC5SzSeyIo", "UZ8KbLtMGio", "UuAHw5oLddQ", "XHyutEstgVI", "XkqHfFZWAZo", "knkGT2eb_Eo", "vmUdqnxpJf0", "5bTR9mdp0D8", "0ISeblieA10", "1Wu6VBan0SA", "F1Z7PzUdzFU", "IHWx-Yjlbi4", "MBqBAeyolMQ", "UxytW7y-Mkc", "Y9ugHJo14wU", "fVVzOtWCEvk", "mkTC6JE0Rlo", "r0ZJlJ3WmqY", "v4GDDAlTe18", "zVpXa9NWiyw", "drxM3ldLVpY", "-kWcjLZrvFI", "4b_xaMakW1U", "7wduHIWnJVI", "8j0puFAEQAA", "FX_85FFP4Jo", "FfSv_noVsNA", "UXhCaWjw-ro", "UZaSgalUUZQ", "UxEm2ZaFEjw", "Wz379TfBmDU", "Zi4c8C_0WEw", "dFME_uxB_Pc", "mwfSo_P9ouI", "oEyl6qdIJ-Q", "oKv1SMk2pHk", "16ZMT1x0RD0", "36uqTkNcWjo", "89JWBB9zLLA", "BpVMrgWX2PU", "X1b1QPQ7h5Q", "XEFbV7cJIcs", "aFMh7IZd8qI", "dOFQfhUP2gU", "kT95aOMV2yA", "nm0merGwoWw", "o9omHtgSji8", "wTdfYfIW2Hw", "wUpE6pwoktI", "y1LrWN_CImc", "0V8ys6Oyq5g", "0YWz4LaCWTQ", "2rrZtea9N_U", "5aeqdmN0tOg", "5zLIiff3lQ0", "6O2wh9s9lBw", "7-WwbGMw0eM", "7AhDeQv2oDo", "8BR5z7gSEiM", "8_FGgD-wwPI", "AdPa4EU7k8Q", "BkMrLUaBS6M", "C34v3VV9RV0", "CAf1HAfBz5Y", "CNaQHc3d8z8", "CkIeAEEsu68", "DmQvrEqSXP0", "E0DvHRmcgWo", "EgF_URs3nNg", "EwO7ibkyKdE", "Fhxrw0PNTzU", "J7xjo4pSnds", "K0x9BpBywmE", "KTq8gFF4b0o", "Kcrp5SaNlac", "Ks9PkPSRt4I", "Kz21CXWvxO0", "LEn-cCpFj9I", "Ms15E0r7ie0", "Nd20bgmRYYM", "P4_BzORKROQ", "PDcSIrKy9i0", "Qz-nL1nC5gw", "R5xwTGYEIuk", "R89OiOlHOjg", "RYwEN5OHTys", "SpbjiT_myr8", "TEgAAudtSEA", "TX-LyK3_wYg", "TzrrrfZ8he0", "U-FK9tFKF8g", "UheOBnutW1g", "UkJSfjS0UaA", "UyLn6Y5rSa4", "VZYyqlIH0Qo", "YuzhauiGMKI", "Yvy7-3GYbYo", "ZN053POITy0", "_honssc3UZc", "_o07P6YnqUs", "a2uWxgGDv24", "auGeW_T0zHs", "bBy_1fNyIUs", "c4IF0awC8pM", "cjXc6hxkeV0", "d-88CdHRdbs", "dMphC7M8D4w", "dTbxwiuSiUY", "eCwtbDlI_4c", "eano2J40WfE", "efzv2PlStu0", "fCzBbeOnz1c", "fTK2UwjG3cI", "g0GuOgvZVDs", "gGCJgjGBN9U", "hlRm2QrHkSM", "iYEDxnuK3wM", "ispx1eAqPro", "k39ivTyct80", "kEWsc4yzWNo", "lGddBrrZqcc", "lZqjfD7rOyo", "mKm9Prr4PkQ", "mztnwrsEY9w", "nfm4_7tplnk", "nk5fqinm5q4", "noBz81THaAA", "oYyXowzAN6M", "rTm1Rixd3qo", "sZ08RMo6ulU", "sitU3xasLMc", "slgXDFI7yiQ", "t7crpYK-r9U", "uKJ-hDvm564", "xZ5TMf5k400", "xsgYP8dlDZM", "xwv7bbeCGzk", "y9vNBpNL-ps", "zVlD-zEQtTI", "zko-jmlK9wE", "zxUelDEK8ws", "7hod6poViIo", "2FZ2Ts1KURs", "4zlkOIWYLwM", "Hc82PBgXiSE", "HeBLDObjNbQ", "OFxwoOO0Ci8", "VeaFnDI6O6M", "nQtLftoy-fI", "vtDZD_JXXdE", "Bsz2MCqO3aM", "T8XiRa-KXwI", "ZYJ5yoTlOEY", "mo-HLLDybIo", "-QMHgdpZOSM", "I_nMC7ZdDx8", "VIyk1ONf564", "q4JVU5k3G5Q", "qhYgoIJD2Bk", "FVRtcjn36nw", "FafACxSROTU", "OpoTCocdQuc", "UwEOmzyINPk", "YPJkJemYHLs", "btfbGUacmYk", "lojQpsDFSNk", "pyGZ9R6nHn0", "qTOlC2qaB5g", "0KRj9JdRSks", "9EF66AU2gZM", "_qxHHLsdKKc", "xAvZtg3sx8M", "yelQifYgq_Y", "0FibX5ovaMM", "-aoJ0qk-e70", "0QRWG5Fz7Tw", "4uIoZmVe7dI", "9lfdDjqliW4", "BIC_pEbbrhc", "BLEAh2d_8x8", "HwBeXSY8elc", "Kk6W6NBL7xY", "O8pVJxOfIB8", "P02Ql1xtGG8", "Q-Rfx3XPgKk", "Q-wsaKhef5s", "QsTN-wkzMYM", "SbuUu2hxR5c", "Uo1BYQEeGjE", "aTL7Pi6lRyU", "gEj0rXctgIc", "iUYbvepVNoo", "kAIAEgB-kto", "ll858xB1Zck", "rfk1Qwt8Zqc", "uz1CB68k5nk", "vDzAUWSWLE8", "wGMjndpLXU4", "bgNLkaRqTVI", "vH54yeDbYUE", "vRb4prxQcVU", "LwjcS3Pzmos", "zLxuYOKPI-I", "-F_4YycBK3Q", "3j_pd-XOUZE", "ArZvDIlD9hs", "EVtPUS9OWE4", "FaNwZMUehNc", "GF6k5VaKiyY", "Kye57dqupLc", "MFNl4dP9fis", "NXB2TEbzd_Q", "TdaIR4SxVAc", "Tuk0AtA4fRw", "U_-749XATQw", "UoR_n6wuc9c", "a7op5sfhLcQ", "f_wMdQd2mNs", "jCr0j04xuF4", "mzb6BTW9-rY", "or2U-zB2avg", "pTkGiJBT3zk", "qmlHtyS_P8k", "rn07fBYqo9A", "tu8Iz82xNAw", "02-J6zu2JE0", "0u_MBcii8p8", "1Jf-CGnaK-I", "3jErSW1xjao", "4Euslnqr0Ik", "4sKEe242CkI", "54pSpSNElqw", "5Z0nMaFX8V4", "6rsW8XxLCn0", "7j3e7JNQ6QM", "9DZMPC16sPI", "9DxYf9CESko", "9YJDMurzrFk", "9jVDy0MRPhA", "BCrfocnwWYA", "FV5KTc7vaQM", "FZHSrJKuql4", "IhepDSvzCgA", "MttDmgwGyHk", "O6xA_OTApUw", "OUAPO-16CW8", "P2Eo25QnwX8", "P7sZxWo5qWY", "Q2H8z28NWyQ", "QlJ4M7N60TU", "QokOzQcjuKo", "SA4-6DfpGPs", "X3bEF34fZXc", "Yo3VQ67HTdY", "YuJeBbTb_DA", "_hmpnyELEvY", "_p29eTJH4Lw", "aYJxHaFuqns", "dXa5xN0i2c0", "e9i3xGPIbc4", "el5m2BMlQ5M", "fYDwVsmBEP8", "im3EddHAruQ", "jgv3pumkOxI", "jrDmRkyqdw0", "l56VCCRHX6Q", "lc9-6PdgyPQ", "m3OhCkeSuEU", "mokcNyLIwvA", "n3jxibNHxz8", "sidHrL4jut8", "tSyBG7sd8Q8", "tzZjXxIuj6Q", "uOFozrSyR7Q", "uZglhivfGrM", "unsYG6kUijs", "vV2_5L_nIz8", "vs4GRUmvgVo", "xauUeoss9xI", "ytg9OCBorQs", "zGqtz51v1hc", "0-Vj2V9h3TI", "56ObaVeqpv0", "6w5HiHAzM78", "FeyulOqzyXw", "ForAQT31teA", "R0jwsidG4wU", "Z2lxuGLb5Qw", "bMvLAef3HBw", "ccbGjh7b4hI", "dGhUaQ_12kM", "eW403WGTEAM", "eu5b_goaTU8", "f8mMHGx0-ps", "kOK30WqtvUk", "npG5c9qvXnc", "psilImabXs0", "xuQZg9tMxY8", "zFhLsAcry6I", "5cUPBdO1oak", "F5P8q6J_PI0", "M-N03zUh4ak", "XERxX58DEQA", "aFUDD9BMNXg", "dGOGqAsfEXY", "jJxYUmcwMBY", "2UfXErFRT_o", "2_XB95RlrGY", "6SVF47Q0hKk", "A6NYoY-Cegw", "E4oKrdX3h9Q", "IG2HFmtgo7Y", "J-0HulBt4DM", "JCIoKgq1YAI", "KbmmLGJ0-sY", "NFANWbi8T1Q", "PXRuuEb22-4", "QSI_jgR0vjo", "YAabw2TCahs", "aGtgfTlvt_4", "buj7cSgfFVA", "hCvHZ44KtQs", "khQV8VWs8aQ", "sA3bl4AGGks", "x58RXX80cNM", "zkXml-Oz6iU", "w64TEcQ5sZg", "1FL9syfCxsc", "1ozy6Ubgl5w", "38rvMB_0qFg", "4XJVl7onTsg", "Dih5PfKvejI", "HXpz2lrpCVM", "Hha4M_8ADOM", "KaOvMd8PE0U", "OuwvvwpU10c", "PPde5bwmfhw", "PpR0MEw9iMM", "RGJyvOL0wxo", "SngNXoDuTCg", "Wkp-TkN6Qdw", "_8oE_graFA8", "bDgHJpKaoZc", "dOaWLVClIjA", "f2GONC41MsI", "k-IE-G78zfE", "lLKW0zezBk4", "oTjV94DSMOA", "s2xeSsK8KUk", "tsq5j-guP9s", "tuH8-H_bolk", "uToBScu8fpY", "vZEJVSQX22w", "xJ5eRRhk44M", "8bzqaA6fo-c", "9n378x71pdk", "C7AbJ8dzhxU", "DS-RL9rIdFM", "E3Yr4KTi0iI", "L8yO4e0Lpxs", "UvCyc1pvSEU", "VLrzDxUnC6k", "WgilRWocDbc", "sZyDCMqU29s", "zd6YlrgVbyI", "RRLWpoZUzUs", "SGrtU_IXJHk", "ZW4G-Q0GMcw", "3yD4FKD2TC0", "EzDT2Zkm2hk"]}
\ No newline at end of file
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..7fb05c4b04dd2a32aa5fcd2e9244275e3c189c90
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Cyan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/audiocraft/.DS_Store b/audiocraft/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..3d025d56d612ef32c6cc5d24b478f0051a99b453
Binary files /dev/null and b/audiocraft/.DS_Store differ
diff --git a/audiocraft/audiocraft/__init__.py b/audiocraft/audiocraft/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6ab346075f1b35366e7231054513097b87552c6f
--- /dev/null
+++ b/audiocraft/audiocraft/__init__.py
@@ -0,0 +1,26 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+AudioCraft is a general framework for training audio generative models.
+At the moment we provide the training code for:
+
+- [MusicGen](https://arxiv.org/abs/2306.05284), a state-of-the-art
+    text-to-music and melody+text autoregressive generative model.
+    For the solver, see `audiocraft.solvers.musicgen.MusicGenSolver`, and for the model,
+    `audiocraft.models.musicgen.MusicGen`.
+- [AudioGen](https://arxiv.org/abs/2209.15352), a state-of-the-art
+    text-to-general-audio generative model.
+- [EnCodec](https://arxiv.org/abs/2210.13438), efficient and high fidelity
+    neural audio codec which provides an excellent tokenizer for autoregressive language models.
+    See `audiocraft.solvers.compression.CompressionSolver`, and `audiocraft.models.encodec.EncodecModel`.
+- [MultiBandDiffusion](TODO), alternative diffusion-based decoder compatible with EnCodec that
+    improves the perceived quality and reduces the artifacts coming from adversarial decoders.
+"""
+
+# flake8: noqa
+from . import data, modules, models
+
+__version__ = '1.0.0'
diff --git a/audiocraft/audiocraft/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..86fa306b5e9a2d1640efc285f194a4467aa43c56
Binary files /dev/null and b/audiocraft/audiocraft/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/__pycache__/environment.cpython-311.pyc b/audiocraft/audiocraft/__pycache__/environment.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1d5e9b099c329be39daed775b20be484b9eb96cc
Binary files /dev/null and b/audiocraft/audiocraft/__pycache__/environment.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/__pycache__/train.cpython-311.pyc b/audiocraft/audiocraft/__pycache__/train.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..40e12c00bca52906ac5864e5b50eddd3008f0207
Binary files /dev/null and b/audiocraft/audiocraft/__pycache__/train.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/__init__.py b/audiocraft/audiocraft/adversarial/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..864058706fbfae13d7f7dc850cc411a2f27d1510
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/__init__.py
@@ -0,0 +1,22 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Adversarial losses and discriminator architectures."""
+
+# flake8: noqa
+from .discriminators import (
+    MultiPeriodDiscriminator,
+    MultiScaleDiscriminator,
+    MultiScaleSTFTDiscriminator
+)
+from .losses import (
+    AdversarialLoss,
+    AdvLossType,
+    get_adv_criterion,
+    get_fake_criterion,
+    get_real_criterion,
+    FeatLossType,
+    FeatureMatchingLoss
+)
diff --git a/audiocraft/audiocraft/adversarial/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/adversarial/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8351a050986017eb21b892eb45fe3048b4d9e100
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/__pycache__/losses.cpython-311.pyc b/audiocraft/audiocraft/adversarial/__pycache__/losses.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc78db435ee9a3d5ba7a14b96fb715a1e8350a6b
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/__pycache__/losses.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/discriminators/__init__.py b/audiocraft/audiocraft/adversarial/discriminators/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f9e5ff59950ee0b1d1a67c9b3831d67d08048148
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/discriminators/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# flake8: noqa
+from .mpd import MultiPeriodDiscriminator
+from .msd import MultiScaleDiscriminator
+from .msstftd import MultiScaleSTFTDiscriminator
diff --git a/audiocraft/audiocraft/adversarial/discriminators/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..609a6a962bb6b29c1da747c8f0d396752582776b
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/discriminators/__pycache__/base.cpython-311.pyc b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/base.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..9f24c63486bef279219b1ecb0c713c1f595b55dc
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/base.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/discriminators/__pycache__/mpd.cpython-311.pyc b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/mpd.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ca3f4f1eaed8ef2179ccbe1b3032a7b2f6de82f3
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/mpd.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/discriminators/__pycache__/msd.cpython-311.pyc b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/msd.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..20c29000d813fbf6e4adc10d20b255b30c0b1689
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/msd.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/discriminators/__pycache__/msstftd.cpython-311.pyc b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/msstftd.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0bb5f8d391ad5e488cebd3e0c63865d4bf13f127
Binary files /dev/null and b/audiocraft/audiocraft/adversarial/discriminators/__pycache__/msstftd.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/adversarial/discriminators/base.py b/audiocraft/audiocraft/adversarial/discriminators/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9d517e9f5bf0f4e18252c45c8db3a35a7255f69
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/discriminators/base.py
@@ -0,0 +1,34 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from abc import ABC, abstractmethod
+import typing as tp
+
+import torch
+import torch.nn as nn
+
+
+FeatureMapType = tp.List[torch.Tensor]
+LogitsType = torch.Tensor
+MultiDiscriminatorOutputType = tp.Tuple[tp.List[LogitsType], tp.List[FeatureMapType]]
+
+
+class MultiDiscriminator(ABC, nn.Module):
+    """Base implementation for discriminators composed of sub-discriminators acting at different scales.
+    """
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def forward(self, x: torch.Tensor) -> MultiDiscriminatorOutputType:
+        ...
+
+    @property
+    @abstractmethod
+    def num_discriminators(self) -> int:
+        """Number of discriminators.
+        """
+        ...
diff --git a/audiocraft/audiocraft/adversarial/discriminators/mpd.py b/audiocraft/audiocraft/adversarial/discriminators/mpd.py
new file mode 100644
index 0000000000000000000000000000000000000000..8debd1fa72d77ca03df680facb60bdf79638cade
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/discriminators/mpd.py
@@ -0,0 +1,106 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from ...modules import NormConv2d
+from .base import MultiDiscriminator, MultiDiscriminatorOutputType
+
+
+def get_padding(kernel_size: int, dilation: int = 1) -> int:
+    return int((kernel_size * dilation - dilation) / 2)
+
+
+class PeriodDiscriminator(nn.Module):
+    """Period sub-discriminator.
+
+    Args:
+        period (int): Period between samples of audio.
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        n_layers (int): Number of convolutional layers.
+        kernel_sizes (list of int): Kernel sizes for convolutions.
+        stride (int): Stride for convolutions.
+        filters (int): Initial number of filters in convolutions.
+        filters_scale (int): Multiplier of number of filters as we increase depth.
+        max_filters (int): Maximum number of filters.
+        norm (str): Normalization method.
+        activation (str): Activation function.
+        activation_params (dict): Parameters to provide to the activation function.
+    """
+    def __init__(self, period: int, in_channels: int = 1, out_channels: int = 1,
+                 n_layers: int = 5, kernel_sizes: tp.List[int] = [5, 3], stride: int = 3,
+                 filters: int = 8, filters_scale: int = 4, max_filters: int = 1024,
+                 norm: str = 'weight_norm', activation: str = 'LeakyReLU',
+                 activation_params: dict = {'negative_slope': 0.2}):
+        super().__init__()
+        self.period = period
+        self.n_layers = n_layers
+        self.activation = getattr(torch.nn, activation)(**activation_params)
+        self.convs = nn.ModuleList()
+        in_chs = in_channels
+        for i in range(self.n_layers):
+            out_chs = min(filters * (filters_scale ** (i + 1)), max_filters)
+            eff_stride = 1 if i == self.n_layers - 1 else stride
+            self.convs.append(NormConv2d(in_chs, out_chs, kernel_size=(kernel_sizes[0], 1), stride=(eff_stride, 1),
+                                         padding=((kernel_sizes[0] - 1) // 2, 0), norm=norm))
+            in_chs = out_chs
+        self.conv_post = NormConv2d(in_chs, out_channels, kernel_size=(kernel_sizes[1], 1), stride=1,
+                                    padding=((kernel_sizes[1] - 1) // 2, 0), norm=norm)
+
+    def forward(self, x: torch.Tensor):
+        fmap = []
+        # 1d to 2d
+        b, c, t = x.shape
+        if t % self.period != 0:  # pad first
+            n_pad = self.period - (t % self.period)
+            x = F.pad(x, (0, n_pad), 'reflect')
+            t = t + n_pad
+        x = x.view(b, c, t // self.period, self.period)
+
+        for conv in self.convs:
+            x = conv(x)
+            x = self.activation(x)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        # x = torch.flatten(x, 1, -1)
+
+        return x, fmap
+
+
+class MultiPeriodDiscriminator(MultiDiscriminator):
+    """Multi-Period (MPD) Discriminator.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        periods (Sequence[int]): Periods between samples of audio for the sub-discriminators.
+        **kwargs: Additional args for `PeriodDiscriminator`
+    """
+    def __init__(self, in_channels: int = 1, out_channels: int = 1,
+                 periods: tp.Sequence[int] = [2, 3, 5, 7, 11], **kwargs):
+        super().__init__()
+        self.discriminators = nn.ModuleList([
+            PeriodDiscriminator(p, in_channels, out_channels, **kwargs) for p in periods
+        ])
+
+    @property
+    def num_discriminators(self):
+        return len(self.discriminators)
+
+    def forward(self, x: torch.Tensor) -> MultiDiscriminatorOutputType:
+        logits = []
+        fmaps = []
+        for disc in self.discriminators:
+            logit, fmap = disc(x)
+            logits.append(logit)
+            fmaps.append(fmap)
+        return logits, fmaps
diff --git a/audiocraft/audiocraft/adversarial/discriminators/msd.py b/audiocraft/audiocraft/adversarial/discriminators/msd.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4e67e29b46ab22f6ffeec85ffc64d8b99800b1b
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/discriminators/msd.py
@@ -0,0 +1,126 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import numpy as np
+import torch
+import torch.nn as nn
+
+from ...modules import NormConv1d
+from .base import MultiDiscriminator, MultiDiscriminatorOutputType
+
+
+class ScaleDiscriminator(nn.Module):
+    """Waveform sub-discriminator.
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        kernel_sizes (Sequence[int]): Kernel sizes for first and last convolutions.
+        filters (int): Number of initial filters for convolutions.
+        max_filters (int): Maximum number of filters.
+        downsample_scales (Sequence[int]): Scale for downsampling implemented as strided convolutions.
+        inner_kernel_sizes (Sequence[int] or None): Kernel sizes for inner convolutions.
+        groups (Sequence[int] or None): Groups for inner convolutions.
+        strides (Sequence[int] or None): Strides for inner convolutions.
+        paddings (Sequence[int] or None): Paddings for inner convolutions.
+        norm (str): Normalization method.
+        activation (str): Activation function.
+        activation_params (dict): Parameters to provide to the activation function.
+        pad (str): Padding for initial convolution.
+        pad_params (dict): Parameters to provide to the padding module.
+    """
+    def __init__(self, in_channels=1, out_channels=1, kernel_sizes: tp.Sequence[int] = [5, 3],
+                 filters: int = 16, max_filters: int = 1024, downsample_scales: tp.Sequence[int] = [4, 4, 4, 4],
+                 inner_kernel_sizes: tp.Optional[tp.Sequence[int]] = None, groups: tp.Optional[tp.Sequence[int]] = None,
+                 strides: tp.Optional[tp.Sequence[int]] = None, paddings: tp.Optional[tp.Sequence[int]] = None,
+                 norm: str = 'weight_norm', activation: str = 'LeakyReLU',
+                 activation_params: dict = {'negative_slope': 0.2}, pad: str = 'ReflectionPad1d',
+                 pad_params: dict = {}):
+        super().__init__()
+        assert len(kernel_sizes) == 2
+        assert kernel_sizes[0] % 2 == 1
+        assert kernel_sizes[1] % 2 == 1
+        assert (inner_kernel_sizes is None or len(inner_kernel_sizes) == len(downsample_scales))
+        assert (groups is None or len(groups) == len(downsample_scales))
+        assert (strides is None or len(strides) == len(downsample_scales))
+        assert (paddings is None or len(paddings) == len(downsample_scales))
+        self.activation = getattr(torch.nn, activation)(**activation_params)
+        self.convs = nn.ModuleList()
+        self.convs.append(
+            nn.Sequential(
+                getattr(torch.nn, pad)((np.prod(kernel_sizes) - 1) // 2, **pad_params),
+                NormConv1d(in_channels, filters, kernel_size=np.prod(kernel_sizes), stride=1, norm=norm)
+            )
+        )
+
+        in_chs = filters
+        for i, downsample_scale in enumerate(downsample_scales):
+            out_chs = min(in_chs * downsample_scale, max_filters)
+            default_kernel_size = downsample_scale * 10 + 1
+            default_stride = downsample_scale
+            default_padding = (default_kernel_size - 1) // 2
+            default_groups = in_chs // 4
+            self.convs.append(
+                NormConv1d(in_chs, out_chs,
+                           kernel_size=inner_kernel_sizes[i] if inner_kernel_sizes else default_kernel_size,
+                           stride=strides[i] if strides else default_stride,
+                           groups=groups[i] if groups else default_groups,
+                           padding=paddings[i] if paddings else default_padding,
+                           norm=norm))
+            in_chs = out_chs
+
+        out_chs = min(in_chs * 2, max_filters)
+        self.convs.append(NormConv1d(in_chs, out_chs, kernel_size=kernel_sizes[0], stride=1,
+                                     padding=(kernel_sizes[0] - 1) // 2, norm=norm))
+        self.conv_post = NormConv1d(out_chs, out_channels, kernel_size=kernel_sizes[1], stride=1,
+                                    padding=(kernel_sizes[1] - 1) // 2, norm=norm)
+
+    def forward(self, x: torch.Tensor):
+        fmap = []
+        for layer in self.convs:
+            x = layer(x)
+            x = self.activation(x)
+            fmap.append(x)
+        x = self.conv_post(x)
+        fmap.append(x)
+        # x = torch.flatten(x, 1, -1)
+        return x, fmap
+
+
+class MultiScaleDiscriminator(MultiDiscriminator):
+    """Multi-Scale (MSD) Discriminator,
+
+    Args:
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        downsample_factor (int): Downsampling factor between the different scales.
+        scale_norms (Sequence[str]): Normalization for each sub-discriminator.
+        **kwargs: Additional args for ScaleDiscriminator.
+    """
+    def __init__(self, in_channels: int = 1, out_channels: int = 1, downsample_factor: int = 2,
+                 scale_norms: tp.Sequence[str] = ['weight_norm', 'weight_norm', 'weight_norm'], **kwargs):
+        super().__init__()
+        self.discriminators = nn.ModuleList([
+            ScaleDiscriminator(in_channels, out_channels, norm=norm, **kwargs) for norm in scale_norms
+        ])
+        self.downsample = nn.AvgPool1d(downsample_factor * 2, downsample_factor, padding=downsample_factor)
+
+    @property
+    def num_discriminators(self):
+        return len(self.discriminators)
+
+    def forward(self, x: torch.Tensor) -> MultiDiscriminatorOutputType:
+        logits = []
+        fmaps = []
+        for i, disc in enumerate(self.discriminators):
+            if i != 0:
+                self.downsample(x)
+            logit, fmap = disc(x)
+            logits.append(logit)
+            fmaps.append(fmap)
+        return logits, fmaps
diff --git a/audiocraft/audiocraft/adversarial/discriminators/msstftd.py b/audiocraft/audiocraft/adversarial/discriminators/msstftd.py
new file mode 100644
index 0000000000000000000000000000000000000000..81a9100961c7a89a39df2643b24268fb90bfeaa4
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/discriminators/msstftd.py
@@ -0,0 +1,134 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import torchaudio
+import torch
+from torch import nn
+from einops import rearrange
+
+from ...modules import NormConv2d
+from .base import MultiDiscriminator, MultiDiscriminatorOutputType
+
+
+def get_2d_padding(kernel_size: tp.Tuple[int, int], dilation: tp.Tuple[int, int] = (1, 1)):
+    return (((kernel_size[0] - 1) * dilation[0]) // 2, ((kernel_size[1] - 1) * dilation[1]) // 2)
+
+
+class DiscriminatorSTFT(nn.Module):
+    """STFT sub-discriminator.
+
+    Args:
+        filters (int): Number of filters in convolutions.
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        n_fft (int): Size of FFT for each scale.
+        hop_length (int): Length of hop between STFT windows for each scale.
+        kernel_size (tuple of int): Inner Conv2d kernel sizes.
+        stride (tuple of int): Inner Conv2d strides.
+        dilations (list of int): Inner Conv2d dilation on the time dimension.
+        win_length (int): Window size for each scale.
+        normalized (bool): Whether to normalize by magnitude after stft.
+        norm (str): Normalization method.
+        activation (str): Activation function.
+        activation_params (dict): Parameters to provide to the activation function.
+        growth (int): Growth factor for the filters.
+    """
+    def __init__(self, filters: int, in_channels: int = 1, out_channels: int = 1,
+                 n_fft: int = 1024, hop_length: int = 256, win_length: int = 1024, max_filters: int = 1024,
+                 filters_scale: int = 1, kernel_size: tp.Tuple[int, int] = (3, 9), dilations: tp.List = [1, 2, 4],
+                 stride: tp.Tuple[int, int] = (1, 2), normalized: bool = True, norm: str = 'weight_norm',
+                 activation: str = 'LeakyReLU', activation_params: dict = {'negative_slope': 0.2}):
+        super().__init__()
+        assert len(kernel_size) == 2
+        assert len(stride) == 2
+        self.filters = filters
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.win_length = win_length
+        self.normalized = normalized
+        self.activation = getattr(torch.nn, activation)(**activation_params)
+        self.spec_transform = torchaudio.transforms.Spectrogram(
+            n_fft=self.n_fft, hop_length=self.hop_length, win_length=self.win_length, window_fn=torch.hann_window,
+            normalized=self.normalized, center=False, pad_mode=None, power=None)
+        spec_channels = 2 * self.in_channels
+        self.convs = nn.ModuleList()
+        self.convs.append(
+            NormConv2d(spec_channels, self.filters, kernel_size=kernel_size, padding=get_2d_padding(kernel_size))
+        )
+        in_chs = min(filters_scale * self.filters, max_filters)
+        for i, dilation in enumerate(dilations):
+            out_chs = min((filters_scale ** (i + 1)) * self.filters, max_filters)
+            self.convs.append(NormConv2d(in_chs, out_chs, kernel_size=kernel_size, stride=stride,
+                                         dilation=(dilation, 1), padding=get_2d_padding(kernel_size, (dilation, 1)),
+                                         norm=norm))
+            in_chs = out_chs
+        out_chs = min((filters_scale ** (len(dilations) + 1)) * self.filters, max_filters)
+        self.convs.append(NormConv2d(in_chs, out_chs, kernel_size=(kernel_size[0], kernel_size[0]),
+                                     padding=get_2d_padding((kernel_size[0], kernel_size[0])),
+                                     norm=norm))
+        self.conv_post = NormConv2d(out_chs, self.out_channels,
+                                    kernel_size=(kernel_size[0], kernel_size[0]),
+                                    padding=get_2d_padding((kernel_size[0], kernel_size[0])),
+                                    norm=norm)
+
+    def forward(self, x: torch.Tensor):
+        fmap = []
+        z = self.spec_transform(x)  # [B, 2, Freq, Frames, 2]
+        z = torch.cat([z.real, z.imag], dim=1)
+        z = rearrange(z, 'b c w t -> b c t w')
+        for i, layer in enumerate(self.convs):
+            z = layer(z)
+            z = self.activation(z)
+            fmap.append(z)
+        z = self.conv_post(z)
+        return z, fmap
+
+
+class MultiScaleSTFTDiscriminator(MultiDiscriminator):
+    """Multi-Scale STFT (MS-STFT) discriminator.
+
+    Args:
+        filters (int): Number of filters in convolutions.
+        in_channels (int): Number of input channels.
+        out_channels (int): Number of output channels.
+        sep_channels (bool): Separate channels to distinct samples for stereo support.
+        n_ffts (Sequence[int]): Size of FFT for each scale.
+        hop_lengths (Sequence[int]): Length of hop between STFT windows for each scale.
+        win_lengths (Sequence[int]): Window size for each scale.
+        **kwargs: Additional args for STFTDiscriminator.
+    """
+    def __init__(self, filters: int, in_channels: int = 1, out_channels: int = 1, sep_channels: bool = False,
+                 n_ffts: tp.List[int] = [1024, 2048, 512], hop_lengths: tp.List[int] = [256, 512, 128],
+                 win_lengths: tp.List[int] = [1024, 2048, 512], **kwargs):
+        super().__init__()
+        assert len(n_ffts) == len(hop_lengths) == len(win_lengths)
+        self.sep_channels = sep_channels
+        self.discriminators = nn.ModuleList([
+            DiscriminatorSTFT(filters, in_channels=in_channels, out_channels=out_channels,
+                              n_fft=n_ffts[i], win_length=win_lengths[i], hop_length=hop_lengths[i], **kwargs)
+            for i in range(len(n_ffts))
+        ])
+
+    @property
+    def num_discriminators(self):
+        return len(self.discriminators)
+
+    def _separate_channels(self, x: torch.Tensor) -> torch.Tensor:
+        B, C, T = x.shape
+        return x.view(-1, 1, T)
+
+    def forward(self, x: torch.Tensor) -> MultiDiscriminatorOutputType:
+        logits = []
+        fmaps = []
+        for disc in self.discriminators:
+            logit, fmap = disc(x)
+            logits.append(logit)
+            fmaps.append(fmap)
+        return logits, fmaps
diff --git a/audiocraft/audiocraft/adversarial/losses.py b/audiocraft/audiocraft/adversarial/losses.py
new file mode 100644
index 0000000000000000000000000000000000000000..be293e739bdc2d91273f30fb789befe7c8b49a43
--- /dev/null
+++ b/audiocraft/audiocraft/adversarial/losses.py
@@ -0,0 +1,228 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Utility module to handle adversarial losses without requiring to mess up the main training loop.
+"""
+
+import typing as tp
+
+import flashy
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+ADVERSARIAL_LOSSES = ['mse', 'hinge', 'hinge2']
+
+
+AdvLossType = tp.Union[nn.Module, tp.Callable[[torch.Tensor], torch.Tensor]]
+FeatLossType = tp.Union[nn.Module, tp.Callable[[torch.Tensor, torch.Tensor], torch.Tensor]]
+
+
+class AdversarialLoss(nn.Module):
+    """Adversary training wrapper.
+
+    Args:
+        adversary (nn.Module): The adversary module will be used to estimate the logits given the fake and real samples.
+            We assume here the adversary output is ``Tuple[List[torch.Tensor], List[List[torch.Tensor]]]``
+            where the first item is a list of logits and the second item is a list of feature maps.
+        optimizer (torch.optim.Optimizer): Optimizer used for training the given module.
+        loss (AdvLossType): Loss function for generator training.
+        loss_real (AdvLossType): Loss function for adversarial training on logits from real samples.
+        loss_fake (AdvLossType): Loss function for adversarial training on logits from fake samples.
+        loss_feat (FeatLossType): Feature matching loss function for generator training.
+        normalize (bool): Whether to normalize by number of sub-discriminators.
+
+    Example of usage:
+        adv_loss = AdversarialLoss(adversaries, optimizer, loss, loss_real, loss_fake)
+        for real in loader:
+            noise = torch.randn(...)
+            fake = model(noise)
+            adv_loss.train_adv(fake, real)
+            loss, _ = adv_loss(fake, real)
+            loss.backward()
+    """
+    def __init__(self,
+                 adversary: nn.Module,
+                 optimizer: torch.optim.Optimizer,
+                 loss: AdvLossType,
+                 loss_real: AdvLossType,
+                 loss_fake: AdvLossType,
+                 loss_feat: tp.Optional[FeatLossType] = None,
+                 normalize: bool = True):
+        super().__init__()
+        self.adversary: nn.Module = adversary
+        flashy.distrib.broadcast_model(self.adversary)
+        self.optimizer = optimizer
+        self.loss = loss
+        self.loss_real = loss_real
+        self.loss_fake = loss_fake
+        self.loss_feat = loss_feat
+        self.normalize = normalize
+
+    def _save_to_state_dict(self, destination, prefix, keep_vars):
+        # Add the optimizer state dict inside our own.
+        super()._save_to_state_dict(destination, prefix, keep_vars)
+        destination[prefix + 'optimizer'] = self.optimizer.state_dict()
+        return destination
+
+    def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
+        # Load optimizer state.
+        self.optimizer.load_state_dict(state_dict.pop(prefix + 'optimizer'))
+        super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
+
+    def get_adversary_pred(self, x):
+        """Run adversary model, validating expected output format."""
+        logits, fmaps = self.adversary(x)
+        assert isinstance(logits, list) and all([isinstance(t, torch.Tensor) for t in logits]), \
+            f'Expecting a list of tensors as logits but {type(logits)} found.'
+        assert isinstance(fmaps, list), f'Expecting a list of features maps but {type(fmaps)} found.'
+        for fmap in fmaps:
+            assert isinstance(fmap, list) and all([isinstance(f, torch.Tensor) for f in fmap]), \
+                f'Expecting a list of tensors as feature maps but {type(fmap)} found.'
+        return logits, fmaps
+
+    def train_adv(self, fake: torch.Tensor, real: torch.Tensor) -> torch.Tensor:
+        """Train the adversary with the given fake and real example.
+
+        We assume the adversary output is the following format: Tuple[List[torch.Tensor], List[List[torch.Tensor]]].
+        The first item being the logits and second item being a list of feature maps for each sub-discriminator.
+
+        This will automatically synchronize gradients (with `flashy.distrib.eager_sync_model`)
+        and call the optimizer.
+        """
+        loss = torch.tensor(0., device=fake.device)
+        all_logits_fake_is_fake, _ = self.get_adversary_pred(fake.detach())
+        all_logits_real_is_fake, _ = self.get_adversary_pred(real.detach())
+        n_sub_adversaries = len(all_logits_fake_is_fake)
+        for logit_fake_is_fake, logit_real_is_fake in zip(all_logits_fake_is_fake, all_logits_real_is_fake):
+            loss += self.loss_fake(logit_fake_is_fake) + self.loss_real(logit_real_is_fake)
+
+        if self.normalize:
+            loss /= n_sub_adversaries
+
+        self.optimizer.zero_grad()
+        with flashy.distrib.eager_sync_model(self.adversary):
+            loss.backward()
+        self.optimizer.step()
+
+        return loss
+
+    def forward(self, fake: torch.Tensor, real: torch.Tensor) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        """Return the loss for the generator, i.e. trying to fool the adversary,
+        and feature matching loss if provided.
+        """
+        adv = torch.tensor(0., device=fake.device)
+        feat = torch.tensor(0., device=fake.device)
+        with flashy.utils.readonly(self.adversary):
+            all_logits_fake_is_fake, all_fmap_fake = self.get_adversary_pred(fake)
+            all_logits_real_is_fake, all_fmap_real = self.get_adversary_pred(real)
+            n_sub_adversaries = len(all_logits_fake_is_fake)
+            for logit_fake_is_fake in all_logits_fake_is_fake:
+                adv += self.loss(logit_fake_is_fake)
+            if self.loss_feat:
+                for fmap_fake, fmap_real in zip(all_fmap_fake, all_fmap_real):
+                    feat += self.loss_feat(fmap_fake, fmap_real)
+
+        if self.normalize:
+            adv /= n_sub_adversaries
+            feat /= n_sub_adversaries
+
+        return adv, feat
+
+
+def get_adv_criterion(loss_type: str) -> tp.Callable:
+    assert loss_type in ADVERSARIAL_LOSSES
+    if loss_type == 'mse':
+        return mse_loss
+    elif loss_type == 'hinge':
+        return hinge_loss
+    elif loss_type == 'hinge2':
+        return hinge2_loss
+    raise ValueError('Unsupported loss')
+
+
+def get_fake_criterion(loss_type: str) -> tp.Callable:
+    assert loss_type in ADVERSARIAL_LOSSES
+    if loss_type == 'mse':
+        return mse_fake_loss
+    elif loss_type in ['hinge', 'hinge2']:
+        return hinge_fake_loss
+    raise ValueError('Unsupported loss')
+
+
+def get_real_criterion(loss_type: str) -> tp.Callable:
+    assert loss_type in ADVERSARIAL_LOSSES
+    if loss_type == 'mse':
+        return mse_real_loss
+    elif loss_type in ['hinge', 'hinge2']:
+        return hinge_real_loss
+    raise ValueError('Unsupported loss')
+
+
+def mse_real_loss(x: torch.Tensor) -> torch.Tensor:
+    return F.mse_loss(x, torch.tensor(1., device=x.device).expand_as(x))
+
+
+def mse_fake_loss(x: torch.Tensor) -> torch.Tensor:
+    return F.mse_loss(x, torch.tensor(0., device=x.device).expand_as(x))
+
+
+def hinge_real_loss(x: torch.Tensor) -> torch.Tensor:
+    return -torch.mean(torch.min(x - 1, torch.tensor(0., device=x.device).expand_as(x)))
+
+
+def hinge_fake_loss(x: torch.Tensor) -> torch.Tensor:
+    return -torch.mean(torch.min(-x - 1, torch.tensor(0., device=x.device).expand_as(x)))
+
+
+def mse_loss(x: torch.Tensor) -> torch.Tensor:
+    if x.numel() == 0:
+        return torch.tensor([0.0], device=x.device)
+    return F.mse_loss(x, torch.tensor(1., device=x.device).expand_as(x))
+
+
+def hinge_loss(x: torch.Tensor) -> torch.Tensor:
+    if x.numel() == 0:
+        return torch.tensor([0.0], device=x.device)
+    return -x.mean()
+
+
+def hinge2_loss(x: torch.Tensor) -> torch.Tensor:
+    if x.numel() == 0:
+        return torch.tensor([0.0])
+    return -torch.mean(torch.min(x - 1, torch.tensor(0., device=x.device).expand_as(x)))
+
+
+class FeatureMatchingLoss(nn.Module):
+    """Feature matching loss for adversarial training.
+
+    Args:
+        loss (nn.Module): Loss to use for feature matching (default=torch.nn.L1).
+        normalize (bool): Whether to normalize the loss.
+            by number of feature maps.
+    """
+    def __init__(self, loss: nn.Module = torch.nn.L1Loss(), normalize: bool = True):
+        super().__init__()
+        self.loss = loss
+        self.normalize = normalize
+
+    def forward(self, fmap_fake: tp.List[torch.Tensor], fmap_real: tp.List[torch.Tensor]) -> torch.Tensor:
+        assert len(fmap_fake) == len(fmap_real) and len(fmap_fake) > 0
+        feat_loss = torch.tensor(0., device=fmap_fake[0].device)
+        feat_scale = torch.tensor(0., device=fmap_fake[0].device)
+        n_fmaps = 0
+        for (feat_fake, feat_real) in zip(fmap_fake, fmap_real):
+            assert feat_fake.shape == feat_real.shape
+            n_fmaps += 1
+            feat_loss += self.loss(feat_fake, feat_real)
+            feat_scale += torch.mean(torch.abs(feat_real))
+
+        if self.normalize:
+            feat_loss /= n_fmaps
+
+        return feat_loss
diff --git a/audiocraft/audiocraft/data/__init__.py b/audiocraft/audiocraft/data/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3c9447208f3b3e620c1ee5ea3f68e49d43b8ef33
--- /dev/null
+++ b/audiocraft/audiocraft/data/__init__.py
@@ -0,0 +1,10 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Audio loading and writing support. Datasets for raw audio
+or also including some metadata."""
+
+# flake8: noqa
+from . import audio, audio_dataset, info_audio_dataset, music_dataset, sound_dataset, btc_chords
diff --git a/audiocraft/audiocraft/data/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..33b8903e49c71c8b16938b2ee0673913e7dfe698
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/audio.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/audio.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dd295262702b5a0eccc7b7389109f48d4217bb29
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/audio.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/audio_dataset.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/audio_dataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6cbe6141d9db316dbc205aea721e7f24affca540
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/audio_dataset.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/audio_utils.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/audio_utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e78660c508e6d09fc7ba697de4e96bb6d09513f8
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/audio_utils.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/btc_chords.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/btc_chords.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f04bcb553fae5367ca4250ff45b5a35bfd59c925
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/btc_chords.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/chords.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/chords.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cfd648bd86e1b8436aa9d892a0a750655f631e6f
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/chords.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/info_audio_dataset.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/info_audio_dataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..70748fcce561ab3f15f55dcb5460709484130fa2
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/info_audio_dataset.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/music_dataset.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/music_dataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0c3e8ec044d21279b85d7bdefc741e1c7bcdb9ac
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/music_dataset.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/sound_dataset.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/sound_dataset.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8ead434b04295cc05ccc5d8b669b0ffc6d2e1f67
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/sound_dataset.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/__pycache__/zip.cpython-311.pyc b/audiocraft/audiocraft/data/__pycache__/zip.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..52ef5c4c9c7ace559374cc08fef3b865049cee8c
Binary files /dev/null and b/audiocraft/audiocraft/data/__pycache__/zip.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/data/audio.py b/audiocraft/audiocraft/data/audio.py
new file mode 100644
index 0000000000000000000000000000000000000000..8348791b63a19685f163136c0eccb7bc04e503d0
--- /dev/null
+++ b/audiocraft/audiocraft/data/audio.py
@@ -0,0 +1,257 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Audio IO methods are defined in this module (info, read, write),
+We rely on av library for faster read when possible, otherwise on torchaudio.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+import logging
+import typing as tp
+
+import numpy as np
+import soundfile
+import torch
+from torch.nn import functional as F
+import torchaudio as ta
+
+import av
+
+from .audio_utils import f32_pcm, i16_pcm, normalize_audio
+
+
+_av_initialized = False
+
+
+def _init_av():
+    global _av_initialized
+    if _av_initialized:
+        return
+    logger = logging.getLogger('libav.mp3')
+    logger.setLevel(logging.ERROR)
+    _av_initialized = True
+
+
+@dataclass(frozen=True)
+class AudioFileInfo:
+    sample_rate: int
+    duration: float
+    channels: int
+
+
+def _av_info(filepath: tp.Union[str, Path]) -> AudioFileInfo:
+    _init_av()
+    with av.open(str(filepath)) as af:
+        stream = af.streams.audio[0]
+        sample_rate = stream.codec_context.sample_rate
+        duration = float(stream.duration * stream.time_base)
+        channels = stream.channels
+        return AudioFileInfo(sample_rate, duration, channels)
+
+
+def _soundfile_info(filepath: tp.Union[str, Path]) -> AudioFileInfo:
+    info = soundfile.info(filepath)
+    return AudioFileInfo(info.samplerate, info.duration, info.channels)
+
+
+def audio_info(filepath: tp.Union[str, Path]) -> AudioFileInfo:
+    # torchaudio no longer returns useful duration informations for some formats like mp3s.
+    filepath = Path(filepath)
+    if filepath.suffix in ['.flac', '.ogg']:  # TODO: Validate .ogg can be safely read with av_info
+        # ffmpeg has some weird issue with flac.
+        return _soundfile_info(filepath)
+    else:
+        return _av_info(filepath)
+
+
+def _av_read(filepath: tp.Union[str, Path], seek_time: float = 0, duration: float = -1.) -> tp.Tuple[torch.Tensor, int]:
+    """FFMPEG-based audio file reading using PyAV bindings.
+    Soundfile cannot read mp3 and av_read is more efficient than torchaudio.
+
+    Args:
+        filepath (str or Path): Path to audio file to read.
+        seek_time (float): Time at which to start reading in the file.
+        duration (float): Duration to read from the file. If set to -1, the whole file is read.
+    Returns:
+        tuple of torch.Tensor, int: Tuple containing audio data and sample rate
+    """
+    _init_av()
+    with av.open(str(filepath)) as af:
+        stream = af.streams.audio[0]
+        sr = stream.codec_context.sample_rate
+        num_frames = int(sr * duration) if duration >= 0 else -1
+        frame_offset = int(sr * seek_time)
+        # we need a small negative offset otherwise we get some edge artifact
+        # from the mp3 decoder.
+        af.seek(int(max(0, (seek_time - 0.1)) / stream.time_base), stream=stream)
+        frames = []
+        length = 0
+        for frame in af.decode(streams=stream.index):
+            current_offset = int(frame.rate * frame.pts * frame.time_base)
+            strip = max(0, frame_offset - current_offset)
+            buf = torch.from_numpy(frame.to_ndarray())
+            if buf.shape[0] != stream.channels:
+                buf = buf.view(-1, stream.channels).t()
+            buf = buf[:, strip:]
+            frames.append(buf)
+            length += buf.shape[1]
+            if num_frames > 0 and length >= num_frames:
+                break
+        assert frames
+        # If the above assert fails, it is likely because we seeked past the end of file point,
+        # in which case ffmpeg returns a single frame with only zeros, and a weird timestamp.
+        # This will need proper debugging, in due time.
+        wav = torch.cat(frames, dim=1)
+        assert wav.shape[0] == stream.channels
+        if num_frames > 0:
+            wav = wav[:, :num_frames]
+        return f32_pcm(wav), sr
+
+
+def audio_read(filepath: tp.Union[str, Path], seek_time: float = 0.,
+               duration: float = -1., pad: bool = False) -> tp.Tuple[torch.Tensor, int]:
+    """Read audio by picking the most appropriate backend tool based on the audio format.
+
+    Args:
+        filepath (str or Path): Path to audio file to read.
+        seek_time (float): Time at which to start reading in the file.
+        duration (float): Duration to read from the file. If set to -1, the whole file is read.
+        pad (bool): Pad output audio if not reaching expected duration.
+    Returns:
+        tuple of torch.Tensor, int: Tuple containing audio data and sample rate.
+    """
+    fp = Path(filepath)
+    if fp.suffix in ['.flac', '.ogg']:  # TODO: check if we can safely use av_read for .ogg
+        # There is some bug with ffmpeg and reading flac
+        info = _soundfile_info(filepath)
+        frames = -1 if duration <= 0 else int(duration * info.sample_rate)
+        frame_offset = int(seek_time * info.sample_rate)
+        wav, sr = soundfile.read(filepath, start=frame_offset, frames=frames, dtype=np.float32)
+        assert info.sample_rate == sr, f"Mismatch of sample rates {info.sample_rate} {sr}"
+        wav = torch.from_numpy(wav).t().contiguous()
+        if len(wav.shape) == 1:
+            wav = torch.unsqueeze(wav, 0)
+    elif (
+        fp.suffix in ['.wav', '.mp3'] and fp.suffix[1:] in ta.utils.sox_utils.list_read_formats()
+        and duration <= 0 and seek_time == 0
+    ):
+        # Torchaudio is faster if we load an entire file at once.
+        wav, sr = ta.load(fp)
+    else:
+        wav, sr = _av_read(filepath, seek_time, duration)
+    if pad and duration > 0:
+        expected_frames = int(duration * sr)
+        wav = F.pad(wav, (0, expected_frames - wav.shape[-1]))
+    return wav, sr
+
+
+def audio_write(stem_name: tp.Union[str, Path],
+                wav: torch.Tensor, sample_rate: int,
+                format: str = 'wav', mp3_rate: int = 320, normalize: bool = True,
+                strategy: str = 'peak', peak_clip_headroom_db: float = 1,
+                rms_headroom_db: float = 18, loudness_headroom_db: float = 14,
+                loudness_compressor: bool = False,
+                log_clipping: bool = True, make_parent_dir: bool = True,
+                add_suffix: bool = True) -> Path:
+    """Convenience function for saving audio to disk. Returns the filename the audio was written to.
+
+    Args:
+        stem_name (str or Path): Filename without extension which will be added automatically.
+        format (str): Either "wav" or "mp3".
+        mp3_rate (int): kbps when using mp3s.
+        normalize (bool): if `True` (default), normalizes according to the prescribed
+            strategy (see after). If `False`, the strategy is only used in case clipping
+            would happen.
+        strategy (str): Can be either 'clip', 'peak', or 'rms'. Default is 'peak',
+            i.e. audio is normalized by its largest value. RMS normalizes by root-mean-square
+            with extra headroom to avoid clipping. 'clip' just clips.
+        peak_clip_headroom_db (float): Headroom in dB when doing 'peak' or 'clip' strategy.
+        rms_headroom_db (float): Headroom in dB when doing 'rms' strategy. This must be much larger
+            than the `peak_clip` one to avoid further clipping.
+        loudness_headroom_db (float): Target loudness for loudness normalization.
+        loudness_compressor (bool): Uses tanh for soft clipping when strategy is 'loudness'.
+         when strategy is 'loudness' log_clipping (bool): If True, basic logging on stderr when clipping still
+            occurs despite strategy (only for 'rms').
+        make_parent_dir (bool): Make parent directory if it doesn't exist.
+    Returns:
+        Path: Path of the saved audio.
+    """
+    assert wav.dtype.is_floating_point, "wav is not floating point"
+    if wav.dim() == 1:
+        wav = wav[None]
+    elif wav.dim() > 2:
+        raise ValueError("Input wav should be at most 2 dimension.")
+    assert wav.isfinite().all()
+    wav = normalize_audio(wav, normalize, strategy, peak_clip_headroom_db,
+                          rms_headroom_db, loudness_headroom_db, loudness_compressor,
+                          log_clipping=log_clipping, sample_rate=sample_rate,
+                          stem_name=str(stem_name))
+    kwargs: dict = {}
+    if format == 'mp3':
+        suffix = '.mp3'
+        kwargs.update({"compression": mp3_rate})
+    elif format == 'wav':
+        wav = i16_pcm(wav)
+        suffix = '.wav'
+        kwargs.update({"encoding": "PCM_S", "bits_per_sample": 16})
+    else:
+        raise RuntimeError(f"Invalid format {format}. Only wav or mp3 are supported.")
+    if not add_suffix:
+        suffix = ''
+    path = Path(str(stem_name) + suffix)
+    if make_parent_dir:
+        path.parent.mkdir(exist_ok=True, parents=True)
+    try:
+        ta.save(path, wav, sample_rate, **kwargs)
+    except Exception:
+        if path.exists():
+            # we do not want to leave half written files around.
+            path.unlink()
+        raise
+    return path
+
+def audio_postproc(wav: torch.Tensor, sample_rate: int, normalize: bool = True,
+                strategy: str = 'peak', peak_clip_headroom_db: float = 1,
+                rms_headroom_db: float = 18, loudness_headroom_db: float = 14,
+                loudness_compressor: bool = False, log_clipping: bool = True) -> Path:
+    """Convenience function for saving audio to disk. Returns the filename the audio was written to.
+
+    Args:
+        wav (torch.Tensor): Audio data to save.
+        sample_rate (int): Sample rate of audio data.
+        format (str): Either "wav" or "mp3".
+        mp3_rate (int): kbps when using mp3s.
+        normalize (bool): if `True` (default), normalizes according to the prescribed
+            strategy (see after). If `False`, the strategy is only used in case clipping
+            would happen.
+        strategy (str): Can be either 'clip', 'peak', or 'rms'. Default is 'peak',
+            i.e. audio is normalized by its largest value. RMS normalizes by root-mean-square
+            with extra headroom to avoid clipping. 'clip' just clips.
+        peak_clip_headroom_db (float): Headroom in dB when doing 'peak' or 'clip' strategy.
+        rms_headroom_db (float): Headroom in dB when doing 'rms' strategy. This must be much larger
+            than the `peak_clip` one to avoid further clipping.
+        loudness_headroom_db (float): Target loudness for loudness normalization.
+        loudness_compressor (bool): Uses tanh for soft clipping when strategy is 'loudness'.
+         when strategy is 'loudness' log_clipping (bool): If True, basic logging on stderr when clipping still
+            occurs despite strategy (only for 'rms').
+        make_parent_dir (bool): Make parent directory if it doesn't exist.
+    Returns:
+        Path: Path of the saved audio.
+    """
+    assert wav.dtype.is_floating_point, "wav is not floating point"
+    if wav.dim() == 1:
+        wav = wav[None]
+    elif wav.dim() > 2:
+        raise ValueError("Input wav should be at most 2 dimension.")
+    assert wav.isfinite().all()
+    wav = normalize_audio(wav, normalize, strategy, peak_clip_headroom_db,
+                          rms_headroom_db, loudness_headroom_db, loudness_compressor,
+                          log_clipping=log_clipping, sample_rate=sample_rate,
+                          stem_name=None)
+    
+    return wav
diff --git a/audiocraft/audiocraft/data/audio_dataset.py b/audiocraft/audiocraft/data/audio_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..b508538f6b9cd4d0d9bd611ac24d9df36bbdba88
--- /dev/null
+++ b/audiocraft/audiocraft/data/audio_dataset.py
@@ -0,0 +1,614 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""AudioDataset support. In order to handle a larger number of files
+without having to scan again the folders, we precompute some metadata
+(filename, sample rate, duration), and use that to efficiently sample audio segments.
+"""
+import argparse
+import copy
+from concurrent.futures import ThreadPoolExecutor, Future
+from dataclasses import dataclass, fields
+from contextlib import ExitStack
+from functools import lru_cache
+import gzip
+import json
+import logging
+import os
+from pathlib import Path
+import random
+import sys
+import typing as tp
+
+import torch
+import torch.nn.functional as F
+
+from .audio import audio_read, audio_info
+from .audio_utils import convert_audio
+from .zip import PathInZip
+
+try:
+    import dora
+except ImportError:
+    dora = None  # type: ignore
+
+
+@dataclass(order=True)
+class BaseInfo:
+
+    @classmethod
+    def _dict2fields(cls, dictionary: dict):
+        return {
+            field.name: dictionary[field.name]
+            for field in fields(cls) if field.name in dictionary
+        }
+
+    @classmethod
+    def from_dict(cls, dictionary: dict):
+        _dictionary = cls._dict2fields(dictionary)
+        return cls(**_dictionary)
+
+    def to_dict(self):
+        return {
+            field.name: self.__getattribute__(field.name)
+            for field in fields(self)
+            }
+
+
+@dataclass(order=True)
+class AudioMeta(BaseInfo):
+    path: str
+    duration: float
+    sample_rate: int
+    bpm: float
+    # meter: int
+    amplitude: tp.Optional[float] = None
+    weight: tp.Optional[float] = None
+    phr_start: tp.List[tp.Optional[float]] = None
+    # info_path is used to load additional information about the audio file that is stored in zip files.
+    info_path: tp.Optional[PathInZip] = None
+
+    @classmethod
+    def from_dict(cls, dictionary: dict):
+        base = cls._dict2fields(dictionary)
+        if 'info_path' in base and base['info_path'] is not None:
+            base['info_path'] = PathInZip(base['info_path'])
+        return cls(**base)
+
+    def to_dict(self):
+        d = super().to_dict()
+        if d['info_path'] is not None:
+            d['info_path'] = str(d['info_path'])
+        return d
+
+
+@dataclass(order=True)
+class SegmentInfo(BaseInfo):
+    meta: AudioMeta
+    seek_time: float
+    # The following values are given once the audio is processed, e.g.
+    # at the target sample rate and target number of channels.
+    n_frames: int      # actual number of frames without padding
+    total_frames: int  # total number of frames, padding included
+    sample_rate: int   # actual sample rate
+    channels: int      # number of audio channels.
+
+
+DEFAULT_EXTS = ['.wav', '.mp3', '.flac', '.ogg', '.m4a']
+
+logger = logging.getLogger(__name__)
+
+
+def _get_audio_meta(file_path: str, minimal: bool = True) -> AudioMeta:
+    """AudioMeta from a path to an audio file.
+
+    Args:
+        file_path (str): Resolved path of valid audio file.
+        minimal (bool): Whether to only load the minimal set of metadata (takes longer if not).
+    Returns:
+        AudioMeta: Audio file path and its metadata.
+    """
+    info = audio_info(file_path)
+    amplitude: tp.Optional[float] = None
+    if not minimal:
+        wav, sr = audio_read(file_path)
+        amplitude = wav.abs().max().item()
+    
+    # load json info
+    json_file = file_path.replace('.wav', '.json')
+    with open(json_file ,'r') as f:
+        json_str = f.read()
+    info_json = json.loads(json_str)
+    
+    if "phr_start" not in info_json.keys():
+        info_json["phr_start"] = None
+
+    # return AudioMeta(file_path, info.duration, info.sample_rate, info_json["bpm"], info_json["meter"], amplitude, None, info_json["phr_start"])
+    return AudioMeta(file_path, info.duration, info.sample_rate, info_json["bpm"], amplitude, None, info_json["phr_start"])
+
+def _resolve_audio_meta(m: AudioMeta, fast: bool = True) -> AudioMeta:
+    """If Dora is available as a dependency, try to resolve potential relative paths
+    in list of AudioMeta. This method is expected to be used when loading meta from file.
+
+    Args:
+        m (AudioMeta): Audio meta to resolve.
+        fast (bool): If True, uses a really fast check for determining if a file
+            is already absolute or not. Only valid on Linux/Mac.
+    Returns:
+        AudioMeta: Audio meta with resolved path.
+    """
+    def is_abs(m):
+        if fast:
+            return str(m)[0] == '/'
+        else:
+            os.path.isabs(str(m))
+
+    if not dora:
+        return m
+
+    if not is_abs(m.path):
+        m.path = dora.git_save.to_absolute_path(m.path)
+    if m.info_path is not None and not is_abs(m.info_path.zip_path):
+        m.info_path.zip_path = dora.git_save.to_absolute_path(m.path)
+    return m
+
+
+def find_audio_files(path: tp.Union[Path, str],
+                     exts: tp.List[str] = DEFAULT_EXTS,
+                     resolve: bool = True,
+                     minimal: bool = True,
+                     progress: bool = False,
+                     workers: int = 0) -> tp.List[AudioMeta]:
+    """Build a list of AudioMeta from a given path,
+    collecting relevant audio files and fetching meta info.
+
+    Args:
+        path (str or Path): Path to folder containing audio files.
+        exts (list of str): List of file extensions to consider for audio files.
+        minimal (bool): Whether to only load the minimal set of metadata (takes longer if not).
+        progress (bool): Whether to log progress on audio files collection.
+        workers (int): number of parallel workers, if 0, use only the current thread.
+    Returns:
+        list of AudioMeta: List of audio file path and its metadata.
+    """
+    audio_files = []
+    futures: tp.List[Future] = []
+    pool: tp.Optional[ThreadPoolExecutor] = None
+    with ExitStack() as stack:
+        if workers > 0:
+            pool = ThreadPoolExecutor(workers)
+            stack.enter_context(pool)
+
+        if progress:
+            print("Finding audio files...")
+        for root, folders, files in os.walk(path, followlinks=True):
+            for file in files:
+                full_path = Path(root) / file
+                if full_path.suffix.lower() in exts:
+                    audio_files.append(full_path)
+                    if pool is not None:
+                        futures.append(pool.submit(_get_audio_meta, str(audio_files[-1]), minimal))
+                    if progress:
+                        print(format(len(audio_files), " 8d"), end='\r', file=sys.stderr)
+
+        if progress:
+            print("Getting audio metadata...")
+        meta: tp.List[AudioMeta] = []
+        for idx, file_path in enumerate(audio_files):
+            try:
+                if pool is None:
+                    m = _get_audio_meta(str(file_path), minimal)
+                else:
+                    m = futures[idx].result()
+                if resolve:
+                    m = _resolve_audio_meta(m)
+            except Exception as err:
+                print("Error with", str(file_path), err, file=sys.stderr)
+                continue
+            meta.append(m)
+            if progress:
+                print(format((1 + idx) / len(audio_files), " 3.1%"), end='\r', file=sys.stderr)
+    meta.sort()
+    return meta
+
+
+def load_audio_meta(path: tp.Union[str, Path],
+                    resolve: bool = True, fast: bool = True) -> tp.List[AudioMeta]:
+    """Load list of AudioMeta from an optionally compressed json file.
+
+    Args:
+        path (str or Path): Path to JSON file.
+        resolve (bool): Whether to resolve the path from AudioMeta (default=True).
+        fast (bool): activates some tricks to make things faster.
+    Returns:
+        list of AudioMeta: List of audio file path and its total duration.
+    """
+    open_fn = gzip.open if str(path).lower().endswith('.gz') else open
+    with open_fn(path, 'rb') as fp:  # type: ignore
+        lines = fp.readlines()
+    meta = []
+    for line in lines:
+        d = json.loads(line)
+        m = AudioMeta.from_dict(d)
+        if resolve:
+            m = _resolve_audio_meta(m, fast=fast)
+        meta.append(m)
+    return meta
+
+
+def save_audio_meta(path: tp.Union[str, Path], meta: tp.List[AudioMeta]):
+    """Save the audio metadata to the file pointer as json.
+
+    Args:
+        path (str or Path): Path to JSON file.
+        metadata (list of BaseAudioMeta): List of audio meta to save.
+    """
+    Path(path).parent.mkdir(exist_ok=True, parents=True)
+    open_fn = gzip.open if str(path).lower().endswith('.gz') else open
+    with open_fn(path, 'wb') as fp:  # type: ignore
+        for m in meta:
+            json_str = json.dumps(m.to_dict()) + '\n'
+            json_bytes = json_str.encode('utf-8')
+            fp.write(json_bytes)
+
+
+class AudioDataset:
+    """Base audio dataset.
+
+    The dataset takes a list of AudioMeta and create a dataset composed of segments of audio
+    and potentially additional information, by creating random segments from the list of audio
+    files referenced in the metadata and applying minimal data pre-processing such as resampling,
+    mixing of channels, padding, etc.
+
+    If no segment_duration value is provided, the AudioDataset will return the full wav for each
+    audio file. Otherwise, it will randomly sample audio files and create a segment of the specified
+    duration, applying padding if required.
+
+    By default, only the torch Tensor corresponding to the waveform is returned. Setting return_info=True
+    allows to return a tuple containing the torch Tensor and additional metadata on the segment and the
+    original audio meta.
+
+    Note that you can call `start_epoch(epoch)` in order to get
+    a deterministic "randomization" for `shuffle=True`.
+    For a given epoch and dataset index, this will always return the same extract.
+    You can get back some diversity by setting the `shuffle_seed` param.
+
+    Args:
+        meta (list of AudioMeta): List of audio files metadata.
+        segment_duration (float, optional): Optional segment duration of audio to load.
+            If not specified, the dataset will load the full audio segment from the file.
+        shuffle (bool): Set to `True` to have the data reshuffled at every epoch.
+        sample_rate (int): Target sample rate of the loaded audio samples.
+        channels (int): Target number of channels of the loaded audio samples.
+        sample_on_duration (bool): Set to `True` to sample segments with probability
+            dependent on audio file duration. This is only used if `segment_duration` is provided.
+        sample_on_weight (bool): Set to `True` to sample segments using the `weight` entry of
+            `AudioMeta`. If `sample_on_duration` is also True, the actual weight will be the product
+            of the file duration and file weight. This is only used if `segment_duration` is provided.
+        min_segment_ratio (float): Minimum segment ratio to use when the audio file
+            is shorter than the desired segment.
+        max_read_retry (int): Maximum number of retries to sample an audio segment from the dataset.
+        return_info (bool): Whether to return the wav only or return wav along with segment info and metadata.
+        min_audio_duration (float, optional): Minimum audio file duration, in seconds, if provided
+            audio shorter than this will be filtered out.
+        max_audio_duration (float, optional): Maximal audio file duration in seconds, if provided
+            audio longer than this will be filtered out.
+        shuffle_seed (int): can be used to further randomize
+        load_wav (bool): if False, skip loading the wav but returns a tensor of 0
+            with the expected segment_duration (which must be provided if load_wav is False).
+        permutation_on_files (bool): only if `sample_on_weight` and `sample_on_duration`
+            are False. Will ensure a permutation on files when going through the dataset.
+            In that case the epoch number must be provided in order for the model
+            to continue the permutation across epochs. In that case, it is assumed
+            that `num_samples = total_batch_size * num_updates_per_epoch`, with
+            `total_batch_size` the overall batch size accounting for all gpus.
+    """
+    def __init__(self,
+                 meta: tp.List[AudioMeta],
+                 segment_duration: tp.Optional[float] = None,
+                 shuffle: bool = True,
+                 num_samples: int = 10_000,
+                 sample_rate: int = 48_000,
+                 channels: int = 2,
+                 pad: bool = True,
+                 sample_on_duration: bool = True,
+                 sample_on_weight: bool = True,
+                 min_segment_ratio: float = 1,
+                 max_read_retry: int = 10,
+                 return_info: bool = False,
+                 min_audio_duration: tp.Optional[float] = None,
+                 max_audio_duration: tp.Optional[float] = None,
+                 shuffle_seed: int = 0,
+                 load_wav: bool = True,
+                 permutation_on_files: bool = False,
+                 ):
+        assert len(meta) > 0, "No audio meta provided to AudioDataset. Please check loading of audio meta."
+        assert segment_duration is None or segment_duration > 0
+        assert segment_duration is None or min_segment_ratio >= 0
+        self.segment_duration = segment_duration
+        self.min_segment_ratio = min_segment_ratio
+        self.max_audio_duration = max_audio_duration
+        self.min_audio_duration = min_audio_duration
+        if self.min_audio_duration is not None and self.max_audio_duration is not None:
+            assert self.min_audio_duration <= self.max_audio_duration
+        self.meta: tp.List[AudioMeta] = self._filter_duration(meta)
+        assert len(self.meta)  # Fail fast if all data has been filtered.
+        self.total_duration = sum(d.duration for d in self.meta)
+
+        if segment_duration is None:
+            num_samples = len(self.meta)
+        self.num_samples = num_samples
+        self.shuffle = shuffle
+        self.sample_rate = sample_rate
+        self.channels = channels
+        self.pad = pad
+        self.sample_on_weight = sample_on_weight
+        self.sample_on_duration = sample_on_duration
+        self.sampling_probabilities = self._get_sampling_probabilities()
+        self.max_read_retry = max_read_retry
+        self.return_info = return_info
+        self.shuffle_seed = shuffle_seed
+        self.current_epoch: tp.Optional[int] = None
+        self.load_wav = load_wav
+        if not load_wav:
+            assert segment_duration is not None
+        self.permutation_on_files = permutation_on_files
+        if permutation_on_files:
+            assert not self.sample_on_duration
+            assert not self.sample_on_weight
+            assert self.shuffle
+
+    def start_epoch(self, epoch: int):
+        self.current_epoch = epoch
+
+    def __len__(self):
+        return self.num_samples
+
+    def _get_sampling_probabilities(self, normalized: bool = True):
+        """Return the sampling probabilities for each file inside `self.meta`."""
+        scores: tp.List[float] = []
+        for file_meta in self.meta:
+            score = 1.
+            if self.sample_on_weight and file_meta.weight is not None:
+                score *= file_meta.weight
+            if self.sample_on_duration:
+                score *= file_meta.duration
+            scores.append(score)
+        probabilities = torch.tensor(scores)
+        if normalized:
+            probabilities /= probabilities.sum()
+        return probabilities
+
+    @staticmethod
+    @lru_cache(16)
+    def _get_file_permutation(num_files: int, permutation_index: int, base_seed: int):
+        # Used to keep the most recent files permutation in memory implicitely.
+        # will work unless someone is using a lot of Datasets in parallel.
+        rng = torch.Generator()
+        rng.manual_seed(base_seed + permutation_index)
+        return torch.randperm(num_files, generator=rng)
+
+    def sample_file(self, index: int, rng: torch.Generator) -> AudioMeta:
+        """Sample a given file from `self.meta`. Can be overridden in subclasses.
+        This is only called if `segment_duration` is not None.
+
+        You must use the provided random number generator `rng` for reproducibility.
+        You can further make use of the index accessed.
+        """
+        if self.permutation_on_files:
+            assert self.current_epoch is not None
+            total_index = self.current_epoch * len(self) + index
+            permutation_index = total_index // len(self.meta)
+            relative_index = total_index % len(self.meta)
+            permutation = AudioDataset._get_file_permutation(
+                len(self.meta), permutation_index, self.shuffle_seed)
+            file_index = permutation[relative_index]
+            return self.meta[file_index]
+
+        if not self.sample_on_weight and not self.sample_on_duration:
+            file_index = int(torch.randint(len(self.sampling_probabilities), (1,), generator=rng).item())
+        else:
+            file_index = int(torch.multinomial(self.sampling_probabilities, 1, generator=rng).item())
+
+        return self.meta[file_index]
+
+    def _audio_read(self, path: str, seek_time: float = 0, duration: float = -1):
+        # Override this method in subclass if needed.
+        if self.load_wav:
+            return audio_read(path, seek_time, duration, pad=False)
+        else:
+            assert self.segment_duration is not None
+            n_frames = int(self.sample_rate * self.segment_duration)
+            return torch.zeros(self.channels, n_frames), self.sample_rate
+
+    def __getitem__(self, index: int) -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, SegmentInfo]]:
+        if self.segment_duration is None:
+            file_meta = self.meta[index]
+            out, sr = audio_read(file_meta.path)
+            out = convert_audio(out, sr, self.sample_rate, self.channels)
+            n_frames = out.shape[-1]
+            segment_info = SegmentInfo(file_meta, seek_time=0., n_frames=n_frames, total_frames=n_frames,
+                                       sample_rate=self.sample_rate, channels=out.shape[0])
+        else:
+            rng = torch.Generator()
+            if self.shuffle:
+                # We use index, plus extra randomness, either totally random if we don't know the epoch.
+                # otherwise we make use of the epoch number and optional shuffle_seed.
+                if self.current_epoch is None:
+                    rng.manual_seed(index + self.num_samples * random.randint(0, 2**24))
+                else:
+                    rng.manual_seed(index + self.num_samples * (self.current_epoch + self.shuffle_seed))
+            else:
+                # We only use index
+                rng.manual_seed(index)
+
+            for retry in range(self.max_read_retry):
+                file_meta = self.sample_file(index, rng)
+                # We add some variance in the file position even if audio file is smaller than segment
+                # without ending up with empty segments
+
+                # sample with phrase
+                if file_meta.phr_start is not None:
+                    # max_seek = max(0, len(file_meta.phr_start[:-1]))
+                    max_seek = max(0, len([start for start in file_meta.phr_start if start + self.segment_duration <= file_meta.duration])) # sample with time
+                    seek_time = file_meta.phr_start[int(torch.rand(1, generator=rng).item() * max_seek)] # choose from phrase
+                
+                else:
+                    max_seek = max(0, file_meta.duration - self.segment_duration * self.min_segment_ratio)
+                    seek_time = torch.rand(1, generator=rng).item() * max_seek # can be change to choose phrase start
+
+                    if file_meta.duration == self.segment_duration:
+                        seek_time = 0
+                
+                # phr_dur = 60./file_meta.bpm * (file_meta.meter * 4.) # if meter=4 then 16 beats per phrase
+                try:
+                    out, sr = audio_read(file_meta.path, seek_time, self.segment_duration, pad=False)
+                    # out, sr = audio_read(file_meta.path, seek_time, phr_dur, pad=False) # use phrase trunk as input
+                    out = convert_audio(out, sr, self.sample_rate, self.channels)
+                    n_frames = out.shape[-1]
+                    target_frames = int(self.segment_duration * self.sample_rate)
+                    if self.pad:
+                        out = F.pad(out, (0, target_frames - n_frames))
+                    segment_info = SegmentInfo(file_meta, seek_time, n_frames=n_frames, total_frames=target_frames,
+                                               sample_rate=self.sample_rate, channels=out.shape[0])
+                except Exception as exc:
+                    logger.warning("Error opening file %s: %r", file_meta.path, exc)
+                    if retry == self.max_read_retry - 1:
+                        raise
+                else:
+                    break
+
+        if self.return_info:
+            # Returns the wav and additional information on the wave segment
+            return out, segment_info
+        else:
+            return out
+
+    def collater(self, samples):
+        """The collater function has to be provided to the dataloader
+        if AudioDataset has return_info=True in order to properly collate
+        the samples of a batch.
+        """
+        if self.segment_duration is None and len(samples) > 1:
+            assert self.pad, "Must allow padding when batching examples of different durations."
+
+        # In this case the audio reaching the collater is of variable length as segment_duration=None.
+        to_pad = self.segment_duration is None and self.pad
+        if to_pad:
+            max_len = max([wav.shape[-1] for wav, _ in samples])
+
+            def _pad_wav(wav):
+                return F.pad(wav, (0, max_len - wav.shape[-1]))
+
+        if self.return_info:
+            if len(samples) > 0:
+                assert len(samples[0]) == 2
+                assert isinstance(samples[0][0], torch.Tensor)
+                assert isinstance(samples[0][1], SegmentInfo)
+
+            wavs = [wav for wav, _ in samples]
+            segment_infos = [copy.deepcopy(info) for _, info in samples]
+
+            if to_pad:
+                # Each wav could be of a different duration as they are not segmented.
+                for i in range(len(samples)):
+                    # Determines the total length of the signal with padding, so we update here as we pad.
+                    segment_infos[i].total_frames = max_len
+                    wavs[i] = _pad_wav(wavs[i])
+
+            wav = torch.stack(wavs)
+            return wav, segment_infos
+        else:
+            assert isinstance(samples[0], torch.Tensor)
+            if to_pad:
+                samples = [_pad_wav(s) for s in samples]
+            return torch.stack(samples)
+
+    def _filter_duration(self, meta: tp.List[AudioMeta]) -> tp.List[AudioMeta]:
+        """Filters out audio files with audio durations that will not allow to sample examples from them."""
+        orig_len = len(meta)
+
+        # Filter data that is too short.
+        if self.min_audio_duration is not None:
+            meta = [m for m in meta if m.duration >= self.min_audio_duration]
+
+        # Filter data that is too long.
+        if self.max_audio_duration is not None:
+            meta = [m for m in meta if m.duration <= self.max_audio_duration]
+
+        filtered_len = len(meta)
+        removed_percentage = 100*(1-float(filtered_len)/orig_len)
+        msg = 'Removed %.2f percent of the data because it was too short or too long.' % removed_percentage
+        if removed_percentage < 10:
+            logging.debug(msg)
+        else:
+            logging.warning(msg)
+        return meta
+
+    @classmethod
+    def from_meta(cls, root: tp.Union[str, Path], **kwargs):
+        """Instantiate AudioDataset from a path to a directory containing a manifest as a jsonl file.
+
+        Args:
+            root (str or Path): Path to root folder containing audio files.
+            kwargs: Additional keyword arguments for the AudioDataset.
+        """
+        root = Path(root)
+        if root.is_dir():
+            if (root / 'data.jsonl').exists():
+                root = root / 'data.jsonl'
+            elif (root / 'data.jsonl.gz').exists():
+                root = root / 'data.jsonl.gz'
+            else:
+                raise ValueError("Don't know where to read metadata from in the dir. "
+                                 "Expecting either a data.jsonl or data.jsonl.gz file but none found.")
+        meta = load_audio_meta(root)
+        return cls(meta, **kwargs)
+
+    @classmethod
+    def from_path(cls, root: tp.Union[str, Path], minimal_meta: bool = True,
+                  exts: tp.List[str] = DEFAULT_EXTS, **kwargs):
+        """Instantiate AudioDataset from a path containing (possibly nested) audio files.
+
+        Args:
+            root (str or Path): Path to root folder containing audio files.
+            minimal_meta (bool): Whether to only load minimal metadata or not.
+            exts (list of str): Extensions for audio files.
+            kwargs: Additional keyword arguments for the AudioDataset.
+        """
+        root = Path(root)
+        if root.is_file():
+            meta = load_audio_meta(root, resolve=True)
+        else:
+            meta = find_audio_files(root, exts, minimal=minimal_meta, resolve=True)
+        return cls(meta, **kwargs)
+
+
+def main():
+    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
+    parser = argparse.ArgumentParser(
+        prog='audio_dataset',
+        description='Generate .jsonl files by scanning a folder.')
+    parser.add_argument('root', help='Root folder with all the audio files')
+    parser.add_argument('output_meta_file',
+                        help='Output file to store the metadata, ')
+    parser.add_argument('--complete',
+                        action='store_false', dest='minimal', default=True,
+                        help='Retrieve all metadata, even the one that are expansive '
+                             'to compute (e.g. normalization).')
+    parser.add_argument('--resolve',
+                        action='store_true', default=False,
+                        help='Resolve the paths to be absolute and with no symlinks.')
+    parser.add_argument('--workers',
+                        default=10, type=int,
+                        help='Number of workers.')
+    args = parser.parse_args()
+    meta = find_audio_files(args.root, DEFAULT_EXTS, progress=True,
+                            resolve=args.resolve, minimal=args.minimal, workers=args.workers)
+    save_audio_meta(args.output_meta_file, meta)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/audiocraft/audiocraft/data/audio_utils.py b/audiocraft/audiocraft/data/audio_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..e9fb715f9801ace1fb2d510f59c161f5ffbe8695
--- /dev/null
+++ b/audiocraft/audiocraft/data/audio_utils.py
@@ -0,0 +1,385 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Various utilities for audio convertion (pcm format, sample rate and channels),
+and volume normalization."""
+import sys
+import typing as tp
+
+import julius
+import torch
+import torchaudio
+import numpy as np
+
+from .chords import Chords
+chords = Chords() # initiate object
+
+
+def convert_audio_channels(wav: torch.Tensor, channels: int = 2) -> torch.Tensor:
+    """Convert audio to the given number of channels.
+
+    Args:
+        wav (torch.Tensor): Audio wave of shape [B, C, T].
+        channels (int): Expected number of channels as output.
+    Returns:
+        torch.Tensor: Downmixed or unchanged audio wave [B, C, T].
+    """
+    *shape, src_channels, length = wav.shape
+    if src_channels == channels:
+        pass
+    elif channels == 1:
+        # Case 1:
+        # The caller asked 1-channel audio, and the stream has multiple
+        # channels, downmix all channels.
+        wav = wav.mean(dim=-2, keepdim=True)
+    elif src_channels == 1:
+        # Case 2:
+        # The caller asked for multiple channels, but the input file has
+        # a single channel, replicate the audio over all channels.
+        wav = wav.expand(*shape, channels, length)
+    elif src_channels >= channels:
+        # Case 3:
+        # The caller asked for multiple channels, and the input file has
+        # more channels than requested. In that case return the first channels.
+        wav = wav[..., :channels, :]
+    else:
+        # Case 4: What is a reasonable choice here?
+        raise ValueError('The audio file has less channels than requested but is not mono.')
+    return wav
+
+
+def convert_audio(wav: torch.Tensor, from_rate: float,
+                  to_rate: float, to_channels: int) -> torch.Tensor:
+    """Convert audio to new sample rate and number of audio channels."""
+    wav = julius.resample_frac(wav, int(from_rate), int(to_rate))
+    wav = convert_audio_channels(wav, to_channels)
+    return wav
+
+
+def normalize_loudness(wav: torch.Tensor, sample_rate: int, loudness_headroom_db: float = 14,
+                       loudness_compressor: bool = False, energy_floor: float = 2e-3):
+    """Normalize an input signal to a user loudness in dB LKFS.
+    Audio loudness is defined according to the ITU-R BS.1770-4 recommendation.
+
+    Args:
+        wav (torch.Tensor): Input multichannel audio data.
+        sample_rate (int): Sample rate.
+        loudness_headroom_db (float): Target loudness of the output in dB LUFS.
+        loudness_compressor (bool): Uses tanh for soft clipping.
+        energy_floor (float): anything below that RMS level will not be rescaled.
+    Returns:
+        torch.Tensor: Loudness normalized output data.
+    """
+    energy = wav.pow(2).mean().sqrt().item()
+    if energy < energy_floor:
+        return wav
+    transform = torchaudio.transforms.Loudness(sample_rate)
+    input_loudness_db = transform(wav).item()
+    # calculate the gain needed to scale to the desired loudness level
+    delta_loudness = -loudness_headroom_db - input_loudness_db
+    gain = 10.0 ** (delta_loudness / 20.0)
+    output = gain * wav
+    if loudness_compressor:
+        output = torch.tanh(output)
+    assert output.isfinite().all(), (input_loudness_db, wav.pow(2).mean().sqrt())
+    return output
+
+
+def _clip_wav(wav: torch.Tensor, log_clipping: bool = False, stem_name: tp.Optional[str] = None) -> None:
+    """Utility function to clip the audio with logging if specified."""
+    max_scale = wav.abs().max()
+    if log_clipping and max_scale > 1:
+        clamp_prob = (wav.abs() > 1).float().mean().item()
+        print(f"CLIPPING {stem_name or ''} happening with proba (a bit of clipping is okay):",
+              clamp_prob, "maximum scale: ", max_scale.item(), file=sys.stderr)
+    wav.clamp_(-1, 1)
+
+
+def normalize_audio(wav: torch.Tensor, normalize: bool = True,
+                    strategy: str = 'peak', peak_clip_headroom_db: float = 1,
+                    rms_headroom_db: float = 18, loudness_headroom_db: float = 14,
+                    loudness_compressor: bool = False, log_clipping: bool = False,
+                    sample_rate: tp.Optional[int] = None,
+                    stem_name: tp.Optional[str] = None) -> torch.Tensor:
+    """Normalize the audio according to the prescribed strategy (see after).
+
+    Args:
+        wav (torch.Tensor): Audio data.
+        normalize (bool): if `True` (default), normalizes according to the prescribed
+            strategy (see after). If `False`, the strategy is only used in case clipping
+            would happen.
+        strategy (str): Can be either 'clip', 'peak', or 'rms'. Default is 'peak',
+            i.e. audio is normalized by its largest value. RMS normalizes by root-mean-square
+            with extra headroom to avoid clipping. 'clip' just clips.
+        peak_clip_headroom_db (float): Headroom in dB when doing 'peak' or 'clip' strategy.
+        rms_headroom_db (float): Headroom in dB when doing 'rms' strategy. This must be much larger
+            than the `peak_clip` one to avoid further clipping.
+        loudness_headroom_db (float): Target loudness for loudness normalization.
+        loudness_compressor (bool): If True, uses tanh based soft clipping.
+        log_clipping (bool): If True, basic logging on stderr when clipping still
+            occurs despite strategy (only for 'rms').
+        sample_rate (int): Sample rate for the audio data (required for loudness).
+        stem_name (str, optional): Stem name for clipping logging.
+    Returns:
+        torch.Tensor: Normalized audio.
+    """
+    scale_peak = 10 ** (-peak_clip_headroom_db / 20)
+    scale_rms = 10 ** (-rms_headroom_db / 20)
+    if strategy == 'peak':
+        rescaling = (scale_peak / wav.abs().max())
+        if normalize or rescaling < 1:
+            wav = wav * rescaling
+    elif strategy == 'clip':
+        wav = wav.clamp(-scale_peak, scale_peak)
+    elif strategy == 'rms':
+        mono = wav.mean(dim=0)
+        rescaling = scale_rms / mono.pow(2).mean().sqrt()
+        if normalize or rescaling < 1:
+            wav = wav * rescaling
+        _clip_wav(wav, log_clipping=log_clipping, stem_name=stem_name)
+    elif strategy == 'loudness':
+        assert sample_rate is not None, "Loudness normalization requires sample rate."
+        wav = normalize_loudness(wav, sample_rate, loudness_headroom_db, loudness_compressor)
+        _clip_wav(wav, log_clipping=log_clipping, stem_name=stem_name)
+    else:
+        assert wav.abs().max() < 1
+        assert strategy == '' or strategy == 'none', f"Unexpected strategy: '{strategy}'"
+    return wav
+
+
+def f32_pcm(wav: torch.Tensor) -> torch.Tensor:
+    """Convert audio to float 32 bits PCM format.
+    """
+    if wav.dtype.is_floating_point:
+        return wav
+    elif wav.dtype == torch.int16:
+        return wav.float() / 2**15
+    elif wav.dtype == torch.int32:
+        return wav.float() / 2**31
+    raise ValueError(f"Unsupported wav dtype: {wav.dtype}")
+
+
+def i16_pcm(wav: torch.Tensor) -> torch.Tensor:
+    """Convert audio to int 16 bits PCM format.
+
+    ..Warning:: There exist many formula for doing this conversion. None are perfect
+    due to the asymmetry of the int16 range. One either have possible clipping, DC offset,
+    or inconsistencies with f32_pcm. If the given wav doesn't have enough headroom,
+    it is possible that `i16_pcm(f32_pcm)) != Identity`.
+    """
+    if wav.dtype.is_floating_point:
+        assert wav.abs().max() <= 1
+        candidate = (wav * 2 ** 15).round()
+        if candidate.max() >= 2 ** 15:  # clipping would occur
+            candidate = (wav * (2 ** 15 - 1)).round()
+        return candidate.short()
+    else:
+        assert wav.dtype == torch.int16
+        return wav
+
+def convert_txtchord2chroma_orig(text_chords, bpms, meters, gen_sec):
+    chromas = []
+    # total_len = int(gen_sec * 44100 / 512)
+    total_len = int(gen_sec * 32000 / 640)
+    for chord, bpm, meter in zip(text_chords, bpms, meters):
+        phr_len = int(60. / bpm * (meter * 4) * 32000 / 640)
+        # phr_len = int(60. / bpm * (meter * 4) * 44100 / 2048)
+        chroma = torch.zeros([total_len, 12])
+        count = 0
+        offset = 0
+        
+        stext = chord.split(" ")
+        timebin = phr_len // 4 # frames per bar
+        while count < total_len:
+            for tokens in stext:
+                if count >= total_len: 
+                    break
+                stoken = tokens.split(',')
+                for token in stoken:
+                    off_timebin = timebin + offset
+                    rounded_timebin = round(off_timebin)
+                    offset = off_timebin - rounded_timebin
+                    offset = offset/len(stoken)
+                    add_step = rounded_timebin//len(stoken)
+                    mhot = chords.chord(token)
+                    rolled = np.roll(mhot[2], mhot[0])
+                    for i in range(count, count + add_step):
+                        if count >= total_len: 
+                            break
+                        chroma[i] = torch.Tensor(rolled)
+                        count += 1
+        chromas.append(chroma)
+    chroma = torch.stack(chromas)
+    return chroma
+
+def convert_txtchord2chroma(chord, bpm, meter, gen_sec):
+    total_len = int(gen_sec * 32000 / 640)
+
+    phr_len = int(60. / bpm * (meter * 4) * 32000 / 640)
+    # phr_len = int(60. / bpm * (meter * 4) * 44100 / 2048)
+    chroma = torch.zeros([total_len, 12])
+    count = 0
+    offset = 0
+    
+    stext = chord.split(" ")
+    timebin = phr_len // 4 # frames per bar
+    while count < total_len:
+        for tokens in stext:
+            if count >= total_len: 
+                break
+            stoken = tokens.split(',')
+            for token in stoken:
+                off_timebin = timebin + offset
+                rounded_timebin = round(off_timebin)
+                offset = off_timebin - rounded_timebin
+                offset = offset/len(stoken)
+                add_step = rounded_timebin//len(stoken)
+                mhot = chords.chord(token)
+                rolled = np.roll(mhot[2], mhot[0])
+                for i in range(count, count + add_step):
+                    if count >= total_len: 
+                        break
+                    chroma[i] = torch.Tensor(rolled)
+                    count += 1
+    return chroma
+
+
+
+def convert_txtchord2chroma_24(chord, bpm, meter, gen_sec):
+    total_len = int(gen_sec * 32000 / 640)
+
+    phr_len = int(60. / bpm * (meter * 4) * 32000 / 640)
+    # phr_len = int(60. / bpm * (meter * 4) * 44100 / 2048)
+    chroma = torch.zeros([total_len, 24])
+    count = 0
+    offset = 0
+    
+    stext = chord.split(" ")
+    timebin = phr_len // 4 # frames per bar
+    while count < total_len:
+        for tokens in stext:
+            if count >= total_len: 
+                break
+            stoken = tokens.split(',')
+            for token in stoken:
+                off_timebin = timebin + offset
+                rounded_timebin = round(off_timebin)
+                offset = off_timebin - rounded_timebin
+                offset = offset/len(stoken)
+                add_step = rounded_timebin//len(stoken)
+
+                root, bass, ivs_vec, _ = chords.chord(token)
+                root_vec = torch.zeros(12)
+                root_vec[root] = 1
+                final_vec = np.concatenate([root_vec, ivs_vec]) # [C]
+                for i in range(count, count + add_step):
+                    if count >= total_len: 
+                        break
+                    chroma[i] = torch.Tensor(final_vec)
+                    count += 1
+    return chroma
+
+def get_chroma_chord_from_lab(chord_path, gen_sec):
+    total_len = int(gen_sec * 32000 / 640)
+    feat_hz = 32000/640
+    intervals = []
+    labels = []
+    feat_chord = np.zeros((12, total_len)) # root| ivs
+    with open(chord_path, 'r') as f:
+        for line in f.readlines():
+            splits = line.split()
+            if len(splits) == 3:
+                st_sec, ed_sec, ctag = splits
+                st_sec = float(st_sec)
+                ed_sec = float(ed_sec)
+
+                st_frame = int(st_sec*feat_hz)
+                ed_frame = int(ed_sec*feat_hz)
+
+                mhot = chords.chord(ctag)
+                final_vec = np.roll(mhot[2], mhot[0])
+
+                final_vec = final_vec[..., None] # [C, T]
+                feat_chord[:, st_frame:ed_frame] = final_vec
+    feat_chord = torch.from_numpy(feat_chord)
+    return feat_chord
+
+
+def get_chroma_chord_from_text(text_chord, bpm, meter, gen_sec):
+    total_len = int(gen_sec * 32000 / 640)
+
+    phr_len = int(60. / bpm * (meter * 4) * 32000 / 640)
+    chroma = np.zeros([12, total_len])
+    count = 0
+    offset = 0
+    
+    stext = chord.split(" ")
+    timebin = phr_len // 4 # frames per bar
+    while count < total_len:
+        for tokens in stext:
+            if count >= total_len: 
+                break
+            stoken = tokens.split(',')
+            for token in stoken:
+                off_timebin = timebin + offset
+                rounded_timebin = round(off_timebin)
+                offset = off_timebin - rounded_timebin
+                offset = offset/len(stoken)
+                add_step = rounded_timebin//len(stoken)
+                mhot = chords.chord(token)
+                final_vec = np.roll(mhot[2], mhot[0])
+                final_vec = final_vec[..., None] # [C, T]
+
+                for i in range(count, count + add_step):
+                    if count >= total_len: 
+                        break
+                    chroma[:, i] = final_vec
+                    count += 1
+    feat_chord = torch.from_numpy(feat_chord)
+    return feat_chord
+
+def get_beat_from_npy(beat_path, gen_sec):
+    total_len = int(gen_sec * 32000 / 640) 
+
+    beats_np = np.load(beat_path, allow_pickle=True)
+    feat_beats = np.zeros((2, total_len))
+    meter = int(max(beats_np.T[1]))
+    beat_time = beats_np[:, 0]
+    bar_time = beats_np[np.where(beats_np[:, 1] == 1)[0], 0]
+
+    beat_frame = [int((t)*feat_hz) for t in beat_time if (t >= 0 and t < duration)]
+    bar_frame =[int((t)*feat_hz) for t in bar_time if (t >= 0 and t < duration)]
+
+    feat_beats[0, beat_frame] = 1
+    feat_beats[1, bar_frame] = 1
+    kernel = np.array([0.05, 0.1, 0.3, 0.9, 0.3, 0.1, 0.05])
+    feat_beats[0] = np.convolve(feat_beats[0] , kernel, 'same') # apply soft kernel
+    beat_events = feat_beats[0] + feat_beats[1]
+    beat_events = torch.tensor(beat_events).unsqueeze(0) # [T] -> [1, T]
+
+    bpm = 60 // np.mean([j-i for i, j in zip(beat_time[:-1], beat_time[1:])])
+    return beat_events, bpm, meter
+
+def get_beat_from_bpm(bpm, meter, gen_sec):
+    total_len = int(gen_sec * 32000 / 640)
+
+    feat_beats = np.zeros((2, total_len))
+
+    beat_time_gap = 60 / bpm
+    beat_gap = 60 / bpm * feat_hz
+    
+    beat_time = np.arange(0, duration, beat_time_gap)
+    beat_frame = np.round(np.arange(0, n_frames_feat, beat_gap)).astype(int)
+    if beat_frame[-1] == n_frames_feat:
+        beat_frame = beat_frame[:-1]
+    bar_frame = beat_frame[::meter]
+    
+    feat_beats[0, beat_frame] = 1
+    feat_beats[1, bar_frame] = 1
+    kernel = np.array([0.05, 0.1, 0.3, 0.9, 0.3, 0.1, 0.05])
+    feat_beats[0] = np.convolve(feat_beats[0] , kernel, 'same') # apply soft kernel
+    beat_events = feat_beats[0] + feat_beats[1]
+    beat_events = torch.tensor(beat_events).unsqueeze(0) # [T] -> [1, T]
+    return beat_events, beat_time, meter
\ No newline at end of file
diff --git a/audiocraft/audiocraft/data/btc_chords.py b/audiocraft/audiocraft/data/btc_chords.py
new file mode 100644
index 0000000000000000000000000000000000000000..1208be9a2d22bb470550c3129fc930eece99ca87
--- /dev/null
+++ b/audiocraft/audiocraft/data/btc_chords.py
@@ -0,0 +1,524 @@
+# encoding: utf-8
+"""
+This module contains chord evaluation functionality.
+
+It provides the evaluation measures used for the MIREX ACE task, and
+tries to follow [1]_ and [2]_ as closely as possible.
+
+Notes
+-----
+This implementation tries to follow the references and their implementation
+(e.g., https://github.com/jpauwels/MusOOEvaluator for [2]_). However, there
+are some known (and possibly some unknown) differences. If you find one not
+listed in the following, please file an issue:
+
+ - Detected chord segments are adjusted to fit the length of the annotations.
+   In particular, this means that, if necessary, filler segments of 'no chord'
+   are added at beginnings and ends. This can result in different segmentation
+   scores compared to the original implementation.
+
+References
+----------
+.. [1] Christopher Harte, "Towards Automatic Extraction of Harmony Information
+       from Music Signals." Dissertation,
+       Department for Electronic Engineering, Queen Mary University of London,
+       2010.
+.. [2] Johan Pauwels and Geoffroy Peeters.
+       "Evaluating Automatically Estimated Chord Sequences."
+       In Proceedings of ICASSP 2013, Vancouver, Canada, 2013.
+
+"""
+
+import numpy as np
+import pandas as pd
+
+
+CHORD_DTYPE = [('root', np.int_),
+               ('bass', np.int_),
+               ('intervals', np.int_, (12,)),
+               ('is_major',np.bool_)]
+
+CHORD_ANN_DTYPE = [('start', np.float32),
+                   ('end', np.float32),
+                   ('chord', CHORD_DTYPE)]
+
+NO_CHORD = (-1, -1, np.zeros(12, dtype=np.int_), False)
+UNKNOWN_CHORD = (-1, -1, np.ones(12, dtype=np.int_) * -1, False)
+
+PITCH_CLASS = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']
+
+
+def idx_to_chord(idx):
+    if idx == 24:
+        return "-"
+    elif idx == 25:
+        return u"\u03B5"
+
+    minmaj = idx % 2
+    root = idx // 2
+
+    return PITCH_CLASS[root] + ("M" if minmaj == 0 else "m")
+
+class Chords:
+
+    def __init__(self):
+        self._shorthands = {
+            'maj': self.interval_list('(1,3,5)'),
+            'min': self.interval_list('(1,b3,5)'),
+            'dim': self.interval_list('(1,b3,b5)'),
+            'aug': self.interval_list('(1,3,#5)'),
+            'maj7': self.interval_list('(1,3,5,7)'),
+            'min7': self.interval_list('(1,b3,5,b7)'),
+            '7': self.interval_list('(1,3,5,b7)'),
+            '6': self.interval_list('(1,6)'),  # custom
+            '5': self.interval_list('(1,5)'),
+            '4': self.interval_list('(1,4)'),  # custom
+            '1': self.interval_list('(1)'),
+            'dim7': self.interval_list('(1,b3,b5,bb7)'),
+            'hdim7': self.interval_list('(1,b3,b5,b7)'),
+            'minmaj7': self.interval_list('(1,b3,5,7)'),
+            'maj6': self.interval_list('(1,3,5,6)'),
+            'min6': self.interval_list('(1,b3,5,6)'),
+            '9': self.interval_list('(1,3,5,b7,9)'),
+            'maj9': self.interval_list('(1,3,5,7,9)'),
+            'min9': self.interval_list('(1,b3,5,b7,9)'),
+            'add9': self.interval_list('(1,3,5,9)'), # custom
+            'sus2': self.interval_list('(1,2,5)'),
+            'sus4': self.interval_list('(1,4,5)'),
+            '7sus2': self.interval_list('(1,2,5,b7)'), # custom
+            '7sus4': self.interval_list('(1,4,5,b7)'), # custom
+            '11': self.interval_list('(1,3,5,b7,9,11)'),
+            'min11': self.interval_list('(1,b3,5,b7,9,11)'),
+            '13': self.interval_list('(1,3,5,b7,13)'),
+            'maj13': self.interval_list('(1,3,5,7,13)'),
+            'min13': self.interval_list('(1,b3,5,b7,13)')
+        }
+
+    def chords(self, labels):
+
+        """
+        Transform a list of chord labels into an array of internal numeric
+        representations.
+
+        Parameters
+        ----------
+        labels : list
+            List of chord labels (str).
+
+        Returns
+        -------
+        chords : numpy.array
+            Structured array with columns 'root', 'bass', and 'intervals',
+            containing a numeric representation of chords.
+
+        """
+        crds = np.zeros(len(labels), dtype=CHORD_DTYPE)
+        cache = {}
+        for i, lbl in enumerate(labels):
+            cv = cache.get(lbl, None)
+            if cv is None:
+                cv = self.chord(lbl)
+                cache[lbl] = cv
+            crds[i] = cv
+
+        return crds
+
+    def label_error_modify(self, label):
+        if label == 'Emin/4': label = 'E:min/4'
+        elif label == 'A7/3': label = 'A:7/3'
+        elif label == 'Bb7/3': label = 'Bb:7/3'
+        elif label == 'Bb7/5': label = 'Bb:7/5'
+        elif label.find(':') == -1:
+            if label.find('min') != -1:
+                label = label[:label.find('min')] + ':' + label[label.find('min'):]
+        return label
+
+    def chord(self, label):
+        """
+        Transform a chord label into the internal numeric represenation of
+        (root, bass, intervals array).
+
+        Parameters
+        ----------
+        label : str
+            Chord label.
+
+        Returns
+        -------
+        chord : tuple
+            Numeric representation of the chord: (root, bass, intervals array).
+
+        """
+
+        
+        is_major = False
+
+        if label == 'N':
+            return NO_CHORD
+        if label == 'X':
+            return UNKNOWN_CHORD
+
+        label = self.label_error_modify(label)
+
+        c_idx = label.find(':')
+        s_idx = label.find('/')
+
+        if c_idx == -1:
+            quality_str = 'maj'
+            if s_idx == -1:
+                root_str = label
+                bass_str = ''
+            else:
+                root_str = label[:s_idx]
+                bass_str = label[s_idx + 1:]
+        else:
+            root_str = label[:c_idx]
+            if s_idx == -1:
+                quality_str = label[c_idx + 1:]
+                bass_str = ''
+            else:
+                quality_str = label[c_idx + 1:s_idx]
+                bass_str = label[s_idx + 1:]
+
+        root = self.pitch(root_str)
+        bass = self.interval(bass_str) if bass_str else 0
+        ivs = self.chord_intervals(quality_str)
+        ivs[bass] = 1
+
+        if 'min' in quality_str:
+            is_major = False
+        else:
+            is_major = True
+
+
+        return root, bass, ivs, is_major
+
+    _l = [0, 1, 1, 0, 1, 1, 1]
+    _chroma_id = (np.arange(len(_l) * 2) + 1) + np.array(_l + _l).cumsum() - 1
+
+    def modify(self, base_pitch, modifier):
+        """
+        Modify a pitch class in integer representation by a given modifier string.
+
+        A modifier string can be any sequence of 'b' (one semitone down)
+        and '#' (one semitone up).
+
+        Parameters
+        ----------
+        base_pitch : int
+            Pitch class as integer.
+        modifier : str
+            String of modifiers ('b' or '#').
+
+        Returns
+        -------
+        modified_pitch : int
+            Modified root note.
+
+        """
+        for m in modifier:
+            if m == 'b':
+                base_pitch -= 1
+            elif m == '#':
+                base_pitch += 1
+            else:
+                raise ValueError('Unknown modifier: {}'.format(m))
+        return base_pitch
+
+    def pitch(self, pitch_str):
+        """
+        Convert a string representation of a pitch class (consisting of root
+        note and modifiers) to an integer representation.
+
+        Parameters
+        ----------
+        pitch_str : str
+            String representation of a pitch class.
+
+        Returns
+        -------
+        pitch : int
+            Integer representation of a pitch class.
+
+        """
+        return self.modify(self._chroma_id[(ord(pitch_str[0]) - ord('C')) % 7],
+                      pitch_str[1:]) % 12
+
+    def interval(self, interval_str):
+        """
+        Convert a string representation of a musical interval into a pitch class
+        (e.g. a minor seventh 'b7' into 10, because it is 10 semitones above its
+        base note).
+
+        Parameters
+        ----------
+        interval_str : str
+            Musical interval.
+
+        Returns
+        -------
+        pitch_class : int
+            Number of semitones to base note of interval.
+
+        """
+        for i, c in enumerate(interval_str):
+            if c.isdigit():
+                return self.modify(self._chroma_id[int(interval_str[i:]) - 1],
+                              interval_str[:i]) % 12
+
+    def interval_list(self, intervals_str, given_pitch_classes=None):
+        """
+        Convert a list of intervals given as string to a binary pitch class
+        representation. For example, 'b3, 5' would become
+        [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0].
+
+        Parameters
+        ----------
+        intervals_str : str
+            List of intervals as comma-separated string (e.g. 'b3, 5').
+        given_pitch_classes : None or numpy array
+            If None, start with empty pitch class array, if numpy array of length
+            12, this array will be modified.
+
+        Returns
+        -------
+        pitch_classes : numpy array
+            Binary pitch class representation of intervals.
+
+        """
+        if given_pitch_classes is None:
+            given_pitch_classes = np.zeros(12, dtype=np.int_)
+        for int_def in intervals_str[1:-1].split(','):
+            int_def = int_def.strip()
+            if int_def[0] == '*':
+                given_pitch_classes[self.interval(int_def[1:])] = 0
+            else:
+                given_pitch_classes[self.interval(int_def)] = 1
+        return given_pitch_classes
+
+    # mapping of shorthand interval notations to the actual interval representation
+
+    def chord_intervals(self, quality_str):
+        """
+        Convert a chord quality string to a pitch class representation. For
+        example, 'maj' becomes [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0].
+
+        Parameters
+        ----------
+        quality_str : str
+            String defining the chord quality.
+
+        Returns
+        -------
+        pitch_classes : numpy array
+            Binary pitch class representation of chord quality.
+
+        """
+        list_idx = quality_str.find('(')
+        if list_idx == -1:
+            return self._shorthands[quality_str].copy()
+        if list_idx != 0:
+            ivs = self._shorthands[quality_str[:list_idx]].copy()
+        else:
+            ivs = np.zeros(12, dtype=np.int_)
+
+
+        return self.interval_list(quality_str[list_idx:], ivs)
+
+    def load_chords(self, filename):
+        """
+        Load chords from a text file.
+
+        The chord must follow the syntax defined in [1]_.
+
+        Parameters
+        ----------
+        filename : str
+            File containing chord segments.
+
+        Returns
+        -------
+        crds : numpy structured array
+            Structured array with columns "start", "end", and "chord",
+            containing the beginning, end, and chord definition of chord
+            segments.
+
+        References
+        ----------
+        .. [1] Christopher Harte, "Towards Automatic Extraction of Harmony
+               Information from Music Signals." Dissertation,
+               Department for Electronic Engineering, Queen Mary University of
+               London, 2010.
+
+        """
+        start, end, chord_labels = [], [], []
+        with open(filename, 'r') as f:
+            for line in f:
+                if line:
+
+                    splits = line.split()
+                    if len(splits) == 3:
+
+                        s = splits[0]
+                        e = splits[1]
+                        l = splits[2]
+
+                        start.append(float(s))
+                        end.append(float(e))
+                        chord_labels.append(l)
+
+        crds = np.zeros(len(start), dtype=CHORD_ANN_DTYPE)
+        crds['start'] = start
+        crds['end'] = end
+        crds['chord'] = self.chords(chord_labels)
+
+        return crds
+
+    def reduce_to_triads(self, chords, keep_bass=False):
+        """
+        Reduce chords to triads.
+
+        The function follows the reduction rules implemented in [1]_. If a chord
+        chord does not contain a third, major second or fourth, it is reduced to
+        a power chord. If it does not contain neither a third nor a fifth, it is
+        reduced to a single note "chord".
+
+        Parameters
+        ----------
+        chords : numpy structured array
+            Chords to be reduced.
+        keep_bass : bool
+            Indicates whether to keep the bass note or set it to 0.
+
+        Returns
+        -------
+        reduced_chords : numpy structured array
+            Chords reduced to triads.
+
+        References
+        ----------
+        .. [1] Johan Pauwels and Geoffroy Peeters.
+               "Evaluating Automatically Estimated Chord Sequences."
+               In Proceedings of ICASSP 2013, Vancouver, Canada, 2013.
+
+        """
+        unison = chords['intervals'][:, 0].astype(bool)
+        maj_sec = chords['intervals'][:, 2].astype(bool)
+        min_third = chords['intervals'][:, 3].astype(bool)
+        maj_third = chords['intervals'][:, 4].astype(bool)
+        perf_fourth = chords['intervals'][:, 5].astype(bool)
+        dim_fifth = chords['intervals'][:, 6].astype(bool)
+        perf_fifth = chords['intervals'][:, 7].astype(bool)
+        aug_fifth = chords['intervals'][:, 8].astype(bool)
+        no_chord = (chords['intervals'] == NO_CHORD[-1]).all(axis=1)
+
+        reduced_chords = chords.copy()
+        ivs = reduced_chords['intervals']
+
+        ivs[~no_chord] = self.interval_list('(1)')
+        ivs[unison & perf_fifth] = self.interval_list('(1,5)')
+        ivs[~perf_fourth & maj_sec] = self._shorthands['sus2']
+        ivs[perf_fourth & ~maj_sec] = self._shorthands['sus4']
+
+        ivs[min_third] = self._shorthands['min']
+        ivs[min_third & aug_fifth & ~perf_fifth] = self.interval_list('(1,b3,#5)')
+        ivs[min_third & dim_fifth & ~perf_fifth] = self._shorthands['dim']
+
+        ivs[maj_third] = self._shorthands['maj']
+        ivs[maj_third & dim_fifth & ~perf_fifth] = self.interval_list('(1,3,b5)')
+        ivs[maj_third & aug_fifth & ~perf_fifth] = self._shorthands['aug']
+
+        if not keep_bass:
+            reduced_chords['bass'] = 0
+        else:
+            # remove bass notes if they are not part of the intervals anymore
+            reduced_chords['bass'] *= ivs[range(len(reduced_chords)),
+                                          reduced_chords['bass']]
+        # keep -1 in bass for no chords
+        reduced_chords['bass'][no_chord] = -1
+
+        return reduced_chords
+
+    def convert_to_id(self, root, is_major):
+        if root == -1:
+            return 24
+        else:
+            if is_major:
+                return root * 2
+            else:
+                return root * 2 + 1
+
+    def get_converted_chord(self, filename):
+        loaded_chord = self.load_chords(filename)
+        triads = self.reduce_to_triads(loaded_chord['chord'])
+
+        df = self.assign_chord_id(triads)
+        df['start'] = loaded_chord['start']
+        df['end'] = loaded_chord['end']
+
+        return df
+
+    def assign_chord_id(self, entry):
+        # maj, min chord only
+        # if you want to add other chord, change this part and get_converted_chord(reduce_to_triads)
+        df = pd.DataFrame(data=entry[['root', 'is_major']])
+        df['chord_id'] = df.apply(lambda row: self.convert_to_id(row['root'], row['is_major']), axis=1)
+        return df
+
+    def convert_to_id_voca(self, root, quality):
+        if root == -1:
+            return 169
+        else:
+            if quality == 'min':
+                return root * 14
+            elif quality == 'maj':
+                return root * 14 + 1
+            elif quality == 'dim':
+                return root * 14 + 2
+            elif quality == 'aug':
+                return root * 14 + 3
+            elif quality == 'min6':
+                return root * 14 + 4
+            elif quality == 'maj6':
+                return root * 14 + 5
+            elif quality == 'min7':
+                return root * 14 + 6
+            elif quality == 'minmaj7':
+                return root * 14 + 7
+            elif quality == 'maj7':
+                return root * 14 + 8
+            elif quality == '7':
+                return root * 14 + 9
+            elif quality == 'dim7':
+                return root * 14 + 10
+            elif quality == 'hdim7':
+                return root * 14 + 11
+            elif quality == 'sus2':
+                return root * 14 + 12
+            elif quality == 'sus4':
+                return root * 14 + 13
+            else:
+                return 168
+
+
+    def lab_file_error_modify(self, ref_labels):
+        for i in range(len(ref_labels)):
+            if ref_labels[i][-2:] == ':4':
+                ref_labels[i] = ref_labels[i].replace(':4', ':sus4')
+            elif ref_labels[i][-2:] == ':6':
+                ref_labels[i] = ref_labels[i].replace(':6', ':maj6')
+            elif ref_labels[i][-4:] == ':6/2':
+                ref_labels[i] = ref_labels[i].replace(':6/2', ':maj6/2')
+            elif ref_labels[i] == 'Emin/4':
+                ref_labels[i] = 'E:min/4'
+            elif ref_labels[i] == 'A7/3':
+                ref_labels[i] = 'A:7/3'
+            elif ref_labels[i] == 'Bb7/3':
+                ref_labels[i] = 'Bb:7/3'
+            elif ref_labels[i] == 'Bb7/5':
+                ref_labels[i] = 'Bb:7/5'
+            elif ref_labels[i].find(':') == -1:
+                if ref_labels[i].find('min') != -1:
+                    ref_labels[i] = ref_labels[i][:ref_labels[i].find('min')] + ':' + ref_labels[i][ref_labels[i].find('min'):]
+        return ref_labels
+
diff --git a/audiocraft/audiocraft/data/chords.py b/audiocraft/audiocraft/data/chords.py
new file mode 100644
index 0000000000000000000000000000000000000000..1208be9a2d22bb470550c3129fc930eece99ca87
--- /dev/null
+++ b/audiocraft/audiocraft/data/chords.py
@@ -0,0 +1,524 @@
+# encoding: utf-8
+"""
+This module contains chord evaluation functionality.
+
+It provides the evaluation measures used for the MIREX ACE task, and
+tries to follow [1]_ and [2]_ as closely as possible.
+
+Notes
+-----
+This implementation tries to follow the references and their implementation
+(e.g., https://github.com/jpauwels/MusOOEvaluator for [2]_). However, there
+are some known (and possibly some unknown) differences. If you find one not
+listed in the following, please file an issue:
+
+ - Detected chord segments are adjusted to fit the length of the annotations.
+   In particular, this means that, if necessary, filler segments of 'no chord'
+   are added at beginnings and ends. This can result in different segmentation
+   scores compared to the original implementation.
+
+References
+----------
+.. [1] Christopher Harte, "Towards Automatic Extraction of Harmony Information
+       from Music Signals." Dissertation,
+       Department for Electronic Engineering, Queen Mary University of London,
+       2010.
+.. [2] Johan Pauwels and Geoffroy Peeters.
+       "Evaluating Automatically Estimated Chord Sequences."
+       In Proceedings of ICASSP 2013, Vancouver, Canada, 2013.
+
+"""
+
+import numpy as np
+import pandas as pd
+
+
+CHORD_DTYPE = [('root', np.int_),
+               ('bass', np.int_),
+               ('intervals', np.int_, (12,)),
+               ('is_major',np.bool_)]
+
+CHORD_ANN_DTYPE = [('start', np.float32),
+                   ('end', np.float32),
+                   ('chord', CHORD_DTYPE)]
+
+NO_CHORD = (-1, -1, np.zeros(12, dtype=np.int_), False)
+UNKNOWN_CHORD = (-1, -1, np.ones(12, dtype=np.int_) * -1, False)
+
+PITCH_CLASS = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B']
+
+
+def idx_to_chord(idx):
+    if idx == 24:
+        return "-"
+    elif idx == 25:
+        return u"\u03B5"
+
+    minmaj = idx % 2
+    root = idx // 2
+
+    return PITCH_CLASS[root] + ("M" if minmaj == 0 else "m")
+
+class Chords:
+
+    def __init__(self):
+        self._shorthands = {
+            'maj': self.interval_list('(1,3,5)'),
+            'min': self.interval_list('(1,b3,5)'),
+            'dim': self.interval_list('(1,b3,b5)'),
+            'aug': self.interval_list('(1,3,#5)'),
+            'maj7': self.interval_list('(1,3,5,7)'),
+            'min7': self.interval_list('(1,b3,5,b7)'),
+            '7': self.interval_list('(1,3,5,b7)'),
+            '6': self.interval_list('(1,6)'),  # custom
+            '5': self.interval_list('(1,5)'),
+            '4': self.interval_list('(1,4)'),  # custom
+            '1': self.interval_list('(1)'),
+            'dim7': self.interval_list('(1,b3,b5,bb7)'),
+            'hdim7': self.interval_list('(1,b3,b5,b7)'),
+            'minmaj7': self.interval_list('(1,b3,5,7)'),
+            'maj6': self.interval_list('(1,3,5,6)'),
+            'min6': self.interval_list('(1,b3,5,6)'),
+            '9': self.interval_list('(1,3,5,b7,9)'),
+            'maj9': self.interval_list('(1,3,5,7,9)'),
+            'min9': self.interval_list('(1,b3,5,b7,9)'),
+            'add9': self.interval_list('(1,3,5,9)'), # custom
+            'sus2': self.interval_list('(1,2,5)'),
+            'sus4': self.interval_list('(1,4,5)'),
+            '7sus2': self.interval_list('(1,2,5,b7)'), # custom
+            '7sus4': self.interval_list('(1,4,5,b7)'), # custom
+            '11': self.interval_list('(1,3,5,b7,9,11)'),
+            'min11': self.interval_list('(1,b3,5,b7,9,11)'),
+            '13': self.interval_list('(1,3,5,b7,13)'),
+            'maj13': self.interval_list('(1,3,5,7,13)'),
+            'min13': self.interval_list('(1,b3,5,b7,13)')
+        }
+
+    def chords(self, labels):
+
+        """
+        Transform a list of chord labels into an array of internal numeric
+        representations.
+
+        Parameters
+        ----------
+        labels : list
+            List of chord labels (str).
+
+        Returns
+        -------
+        chords : numpy.array
+            Structured array with columns 'root', 'bass', and 'intervals',
+            containing a numeric representation of chords.
+
+        """
+        crds = np.zeros(len(labels), dtype=CHORD_DTYPE)
+        cache = {}
+        for i, lbl in enumerate(labels):
+            cv = cache.get(lbl, None)
+            if cv is None:
+                cv = self.chord(lbl)
+                cache[lbl] = cv
+            crds[i] = cv
+
+        return crds
+
+    def label_error_modify(self, label):
+        if label == 'Emin/4': label = 'E:min/4'
+        elif label == 'A7/3': label = 'A:7/3'
+        elif label == 'Bb7/3': label = 'Bb:7/3'
+        elif label == 'Bb7/5': label = 'Bb:7/5'
+        elif label.find(':') == -1:
+            if label.find('min') != -1:
+                label = label[:label.find('min')] + ':' + label[label.find('min'):]
+        return label
+
+    def chord(self, label):
+        """
+        Transform a chord label into the internal numeric represenation of
+        (root, bass, intervals array).
+
+        Parameters
+        ----------
+        label : str
+            Chord label.
+
+        Returns
+        -------
+        chord : tuple
+            Numeric representation of the chord: (root, bass, intervals array).
+
+        """
+
+        
+        is_major = False
+
+        if label == 'N':
+            return NO_CHORD
+        if label == 'X':
+            return UNKNOWN_CHORD
+
+        label = self.label_error_modify(label)
+
+        c_idx = label.find(':')
+        s_idx = label.find('/')
+
+        if c_idx == -1:
+            quality_str = 'maj'
+            if s_idx == -1:
+                root_str = label
+                bass_str = ''
+            else:
+                root_str = label[:s_idx]
+                bass_str = label[s_idx + 1:]
+        else:
+            root_str = label[:c_idx]
+            if s_idx == -1:
+                quality_str = label[c_idx + 1:]
+                bass_str = ''
+            else:
+                quality_str = label[c_idx + 1:s_idx]
+                bass_str = label[s_idx + 1:]
+
+        root = self.pitch(root_str)
+        bass = self.interval(bass_str) if bass_str else 0
+        ivs = self.chord_intervals(quality_str)
+        ivs[bass] = 1
+
+        if 'min' in quality_str:
+            is_major = False
+        else:
+            is_major = True
+
+
+        return root, bass, ivs, is_major
+
+    _l = [0, 1, 1, 0, 1, 1, 1]
+    _chroma_id = (np.arange(len(_l) * 2) + 1) + np.array(_l + _l).cumsum() - 1
+
+    def modify(self, base_pitch, modifier):
+        """
+        Modify a pitch class in integer representation by a given modifier string.
+
+        A modifier string can be any sequence of 'b' (one semitone down)
+        and '#' (one semitone up).
+
+        Parameters
+        ----------
+        base_pitch : int
+            Pitch class as integer.
+        modifier : str
+            String of modifiers ('b' or '#').
+
+        Returns
+        -------
+        modified_pitch : int
+            Modified root note.
+
+        """
+        for m in modifier:
+            if m == 'b':
+                base_pitch -= 1
+            elif m == '#':
+                base_pitch += 1
+            else:
+                raise ValueError('Unknown modifier: {}'.format(m))
+        return base_pitch
+
+    def pitch(self, pitch_str):
+        """
+        Convert a string representation of a pitch class (consisting of root
+        note and modifiers) to an integer representation.
+
+        Parameters
+        ----------
+        pitch_str : str
+            String representation of a pitch class.
+
+        Returns
+        -------
+        pitch : int
+            Integer representation of a pitch class.
+
+        """
+        return self.modify(self._chroma_id[(ord(pitch_str[0]) - ord('C')) % 7],
+                      pitch_str[1:]) % 12
+
+    def interval(self, interval_str):
+        """
+        Convert a string representation of a musical interval into a pitch class
+        (e.g. a minor seventh 'b7' into 10, because it is 10 semitones above its
+        base note).
+
+        Parameters
+        ----------
+        interval_str : str
+            Musical interval.
+
+        Returns
+        -------
+        pitch_class : int
+            Number of semitones to base note of interval.
+
+        """
+        for i, c in enumerate(interval_str):
+            if c.isdigit():
+                return self.modify(self._chroma_id[int(interval_str[i:]) - 1],
+                              interval_str[:i]) % 12
+
+    def interval_list(self, intervals_str, given_pitch_classes=None):
+        """
+        Convert a list of intervals given as string to a binary pitch class
+        representation. For example, 'b3, 5' would become
+        [0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0].
+
+        Parameters
+        ----------
+        intervals_str : str
+            List of intervals as comma-separated string (e.g. 'b3, 5').
+        given_pitch_classes : None or numpy array
+            If None, start with empty pitch class array, if numpy array of length
+            12, this array will be modified.
+
+        Returns
+        -------
+        pitch_classes : numpy array
+            Binary pitch class representation of intervals.
+
+        """
+        if given_pitch_classes is None:
+            given_pitch_classes = np.zeros(12, dtype=np.int_)
+        for int_def in intervals_str[1:-1].split(','):
+            int_def = int_def.strip()
+            if int_def[0] == '*':
+                given_pitch_classes[self.interval(int_def[1:])] = 0
+            else:
+                given_pitch_classes[self.interval(int_def)] = 1
+        return given_pitch_classes
+
+    # mapping of shorthand interval notations to the actual interval representation
+
+    def chord_intervals(self, quality_str):
+        """
+        Convert a chord quality string to a pitch class representation. For
+        example, 'maj' becomes [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0].
+
+        Parameters
+        ----------
+        quality_str : str
+            String defining the chord quality.
+
+        Returns
+        -------
+        pitch_classes : numpy array
+            Binary pitch class representation of chord quality.
+
+        """
+        list_idx = quality_str.find('(')
+        if list_idx == -1:
+            return self._shorthands[quality_str].copy()
+        if list_idx != 0:
+            ivs = self._shorthands[quality_str[:list_idx]].copy()
+        else:
+            ivs = np.zeros(12, dtype=np.int_)
+
+
+        return self.interval_list(quality_str[list_idx:], ivs)
+
+    def load_chords(self, filename):
+        """
+        Load chords from a text file.
+
+        The chord must follow the syntax defined in [1]_.
+
+        Parameters
+        ----------
+        filename : str
+            File containing chord segments.
+
+        Returns
+        -------
+        crds : numpy structured array
+            Structured array with columns "start", "end", and "chord",
+            containing the beginning, end, and chord definition of chord
+            segments.
+
+        References
+        ----------
+        .. [1] Christopher Harte, "Towards Automatic Extraction of Harmony
+               Information from Music Signals." Dissertation,
+               Department for Electronic Engineering, Queen Mary University of
+               London, 2010.
+
+        """
+        start, end, chord_labels = [], [], []
+        with open(filename, 'r') as f:
+            for line in f:
+                if line:
+
+                    splits = line.split()
+                    if len(splits) == 3:
+
+                        s = splits[0]
+                        e = splits[1]
+                        l = splits[2]
+
+                        start.append(float(s))
+                        end.append(float(e))
+                        chord_labels.append(l)
+
+        crds = np.zeros(len(start), dtype=CHORD_ANN_DTYPE)
+        crds['start'] = start
+        crds['end'] = end
+        crds['chord'] = self.chords(chord_labels)
+
+        return crds
+
+    def reduce_to_triads(self, chords, keep_bass=False):
+        """
+        Reduce chords to triads.
+
+        The function follows the reduction rules implemented in [1]_. If a chord
+        chord does not contain a third, major second or fourth, it is reduced to
+        a power chord. If it does not contain neither a third nor a fifth, it is
+        reduced to a single note "chord".
+
+        Parameters
+        ----------
+        chords : numpy structured array
+            Chords to be reduced.
+        keep_bass : bool
+            Indicates whether to keep the bass note or set it to 0.
+
+        Returns
+        -------
+        reduced_chords : numpy structured array
+            Chords reduced to triads.
+
+        References
+        ----------
+        .. [1] Johan Pauwels and Geoffroy Peeters.
+               "Evaluating Automatically Estimated Chord Sequences."
+               In Proceedings of ICASSP 2013, Vancouver, Canada, 2013.
+
+        """
+        unison = chords['intervals'][:, 0].astype(bool)
+        maj_sec = chords['intervals'][:, 2].astype(bool)
+        min_third = chords['intervals'][:, 3].astype(bool)
+        maj_third = chords['intervals'][:, 4].astype(bool)
+        perf_fourth = chords['intervals'][:, 5].astype(bool)
+        dim_fifth = chords['intervals'][:, 6].astype(bool)
+        perf_fifth = chords['intervals'][:, 7].astype(bool)
+        aug_fifth = chords['intervals'][:, 8].astype(bool)
+        no_chord = (chords['intervals'] == NO_CHORD[-1]).all(axis=1)
+
+        reduced_chords = chords.copy()
+        ivs = reduced_chords['intervals']
+
+        ivs[~no_chord] = self.interval_list('(1)')
+        ivs[unison & perf_fifth] = self.interval_list('(1,5)')
+        ivs[~perf_fourth & maj_sec] = self._shorthands['sus2']
+        ivs[perf_fourth & ~maj_sec] = self._shorthands['sus4']
+
+        ivs[min_third] = self._shorthands['min']
+        ivs[min_third & aug_fifth & ~perf_fifth] = self.interval_list('(1,b3,#5)')
+        ivs[min_third & dim_fifth & ~perf_fifth] = self._shorthands['dim']
+
+        ivs[maj_third] = self._shorthands['maj']
+        ivs[maj_third & dim_fifth & ~perf_fifth] = self.interval_list('(1,3,b5)')
+        ivs[maj_third & aug_fifth & ~perf_fifth] = self._shorthands['aug']
+
+        if not keep_bass:
+            reduced_chords['bass'] = 0
+        else:
+            # remove bass notes if they are not part of the intervals anymore
+            reduced_chords['bass'] *= ivs[range(len(reduced_chords)),
+                                          reduced_chords['bass']]
+        # keep -1 in bass for no chords
+        reduced_chords['bass'][no_chord] = -1
+
+        return reduced_chords
+
+    def convert_to_id(self, root, is_major):
+        if root == -1:
+            return 24
+        else:
+            if is_major:
+                return root * 2
+            else:
+                return root * 2 + 1
+
+    def get_converted_chord(self, filename):
+        loaded_chord = self.load_chords(filename)
+        triads = self.reduce_to_triads(loaded_chord['chord'])
+
+        df = self.assign_chord_id(triads)
+        df['start'] = loaded_chord['start']
+        df['end'] = loaded_chord['end']
+
+        return df
+
+    def assign_chord_id(self, entry):
+        # maj, min chord only
+        # if you want to add other chord, change this part and get_converted_chord(reduce_to_triads)
+        df = pd.DataFrame(data=entry[['root', 'is_major']])
+        df['chord_id'] = df.apply(lambda row: self.convert_to_id(row['root'], row['is_major']), axis=1)
+        return df
+
+    def convert_to_id_voca(self, root, quality):
+        if root == -1:
+            return 169
+        else:
+            if quality == 'min':
+                return root * 14
+            elif quality == 'maj':
+                return root * 14 + 1
+            elif quality == 'dim':
+                return root * 14 + 2
+            elif quality == 'aug':
+                return root * 14 + 3
+            elif quality == 'min6':
+                return root * 14 + 4
+            elif quality == 'maj6':
+                return root * 14 + 5
+            elif quality == 'min7':
+                return root * 14 + 6
+            elif quality == 'minmaj7':
+                return root * 14 + 7
+            elif quality == 'maj7':
+                return root * 14 + 8
+            elif quality == '7':
+                return root * 14 + 9
+            elif quality == 'dim7':
+                return root * 14 + 10
+            elif quality == 'hdim7':
+                return root * 14 + 11
+            elif quality == 'sus2':
+                return root * 14 + 12
+            elif quality == 'sus4':
+                return root * 14 + 13
+            else:
+                return 168
+
+
+    def lab_file_error_modify(self, ref_labels):
+        for i in range(len(ref_labels)):
+            if ref_labels[i][-2:] == ':4':
+                ref_labels[i] = ref_labels[i].replace(':4', ':sus4')
+            elif ref_labels[i][-2:] == ':6':
+                ref_labels[i] = ref_labels[i].replace(':6', ':maj6')
+            elif ref_labels[i][-4:] == ':6/2':
+                ref_labels[i] = ref_labels[i].replace(':6/2', ':maj6/2')
+            elif ref_labels[i] == 'Emin/4':
+                ref_labels[i] = 'E:min/4'
+            elif ref_labels[i] == 'A7/3':
+                ref_labels[i] = 'A:7/3'
+            elif ref_labels[i] == 'Bb7/3':
+                ref_labels[i] = 'Bb:7/3'
+            elif ref_labels[i] == 'Bb7/5':
+                ref_labels[i] = 'Bb:7/5'
+            elif ref_labels[i].find(':') == -1:
+                if ref_labels[i].find('min') != -1:
+                    ref_labels[i] = ref_labels[i][:ref_labels[i].find('min')] + ':' + ref_labels[i][ref_labels[i].find('min'):]
+        return ref_labels
+
diff --git a/audiocraft/audiocraft/data/info_audio_dataset.py b/audiocraft/audiocraft/data/info_audio_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..47ab4b1594faf1e9f1ce962fb980d80295b1f079
--- /dev/null
+++ b/audiocraft/audiocraft/data/info_audio_dataset.py
@@ -0,0 +1,110 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Base classes for the datasets that also provide non-audio metadata,
+e.g. description, text transcription etc.
+"""
+from dataclasses import dataclass
+import logging
+import math
+import re
+import typing as tp
+
+import torch
+
+from .audio_dataset import AudioDataset, AudioMeta
+from ..environment import AudioCraftEnvironment
+from ..modules.conditioners import SegmentWithAttributes, ConditioningAttributes
+
+
+logger = logging.getLogger(__name__)
+
+
+def _clusterify_meta(meta: AudioMeta) -> AudioMeta:
+    """Monkey-patch meta to match cluster specificities."""
+    meta.path = AudioCraftEnvironment.apply_dataset_mappers(meta.path)
+    if meta.info_path is not None:
+        meta.info_path.zip_path = AudioCraftEnvironment.apply_dataset_mappers(meta.info_path.zip_path)
+    return meta
+
+
+def clusterify_all_meta(meta: tp.List[AudioMeta]) -> tp.List[AudioMeta]:
+    """Monkey-patch all meta to match cluster specificities."""
+    return [_clusterify_meta(m) for m in meta]
+
+
+@dataclass
+class AudioInfo(SegmentWithAttributes):
+    """Dummy SegmentInfo with empty attributes.
+
+    The InfoAudioDataset is expected to return metadata that inherits
+    from SegmentWithAttributes class and can return conditioning attributes.
+
+    This basically guarantees all datasets will be compatible with current
+    solver that contain conditioners requiring this.
+    """
+    audio_tokens: tp.Optional[torch.Tensor] = None  # populated when using cached batch for training a LM.
+
+    def to_condition_attributes(self) -> ConditioningAttributes:
+        return ConditioningAttributes()
+
+
+class InfoAudioDataset(AudioDataset):
+    """AudioDataset that always returns metadata as SegmentWithAttributes along with the audio waveform.
+
+    See `audiocraft.data.audio_dataset.AudioDataset` for initialization arguments.
+    """
+    def __init__(self, meta: tp.List[AudioMeta], **kwargs):
+        super().__init__(clusterify_all_meta(meta), **kwargs)
+
+    def __getitem__(self, index: int) -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, SegmentWithAttributes]]:
+        if not self.return_info:
+            wav = super().__getitem__(index)
+            assert isinstance(wav, torch.Tensor)
+            return wav
+        wav, meta = super().__getitem__(index)
+        return wav, AudioInfo(**meta.to_dict())
+
+
+def get_keyword_or_keyword_list(value: tp.Optional[str]) -> tp.Union[tp.Optional[str], tp.Optional[tp.List[str]]]:
+    """Preprocess a single keyword or possible a list of keywords."""
+    if isinstance(value, list):
+        return get_keyword_list(value)
+    else:
+        return get_keyword(value)
+
+
+def get_string(value: tp.Optional[str]) -> tp.Optional[str]:
+    """Preprocess a single keyword."""
+    if value is None or (not isinstance(value, str)) or len(value) == 0 or value == 'None':
+        return None
+    else:
+        return value.strip()
+
+
+def get_keyword(value: tp.Optional[str]) -> tp.Optional[str]:
+    """Preprocess a single keyword."""
+    if value is None or (not isinstance(value, str)) or len(value) == 0 or value == 'None':
+        return None
+    else:
+        return value.strip().lower()
+
+
+def get_keyword_list(values: tp.Union[str, tp.List[str]]) -> tp.Optional[tp.List[str]]:
+    """Preprocess a list of keywords."""
+    if isinstance(values, str):
+        values = [v.strip() for v in re.split(r'[,\s]', values)]
+    elif isinstance(values, float) and math.isnan(values):
+        values = []
+    if not isinstance(values, list):
+        logger.debug(f"Unexpected keyword list {values}")
+        values = [str(values)]
+
+    kws = [get_keyword(v) for v in values]
+    kw_list = [k for k in kws if k is not None]
+    if len(kw_list) == 0:
+        return None
+    else:
+        return kw_list
diff --git a/audiocraft/audiocraft/data/music_dataset.py b/audiocraft/audiocraft/data/music_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..0d31516ddc3efa7669a946500932991be892a6e2
--- /dev/null
+++ b/audiocraft/audiocraft/data/music_dataset.py
@@ -0,0 +1,349 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Dataset of music tracks with rich metadata.
+"""
+from dataclasses import dataclass, field, fields, replace
+import gzip
+import json
+import logging
+from pathlib import Path
+import random
+import typing as tp
+import pretty_midi
+import numpy as np
+
+import torch
+import torch.nn.functional as F
+from .btc_chords import Chords
+
+from .info_audio_dataset import (
+    InfoAudioDataset,
+    AudioInfo,
+    get_keyword_list,
+    get_keyword,
+    get_string
+)
+from ..modules.conditioners import (
+    ConditioningAttributes,
+    JointEmbedCondition,
+    WavCondition,
+    ChordCondition,
+    BeatCondition
+)
+from ..utils.utils import warn_once
+
+
+logger = logging.getLogger(__name__)
+
+CHORDS = Chords()
+
+
+@dataclass
+class MusicInfo(AudioInfo):
+    """Segment info augmented with music metadata.
+    """
+    # music-specific metadata
+    title: tp.Optional[str] = None
+    artist: tp.Optional[str] = None  # anonymized artist id, used to ensure no overlap between splits
+    key: tp.Optional[str] = None
+    bpm: tp.Optional[float] = None
+    genre: tp.Optional[str] = None
+    moods: tp.Optional[list] = None
+    keywords: tp.Optional[list] = None
+    description: tp.Optional[str] = None
+    name: tp.Optional[str] = None
+    instrument: tp.Optional[str] = None
+    chord: tp.Optional[ChordCondition] = None
+    beat: tp.Optional[BeatCondition] = None
+    # original wav accompanying the metadata
+    self_wav: tp.Optional[WavCondition] = None
+    # dict mapping attributes names to tuple of wav, text and metadata
+    joint_embed: tp.Dict[str, JointEmbedCondition] = field(default_factory=dict)
+
+    @property
+    def has_music_meta(self) -> bool:
+        return self.name is not None
+
+    def to_condition_attributes(self) -> ConditioningAttributes:
+        out = ConditioningAttributes()
+        for _field in fields(self):
+            key, value = _field.name, getattr(self, _field.name)
+            if key == 'self_wav':
+                out.wav[key] = value
+            elif key == 'chord':
+                out.chord[key] = value
+            elif key == 'beat':
+                out.beat[key] = value
+            elif key == 'joint_embed':
+                for embed_attribute, embed_cond in value.items():
+                    out.joint_embed[embed_attribute] = embed_cond
+            else:
+                if isinstance(value, list):
+                    value = ' '.join(value)
+                out.text[key] = value
+        return out
+
+    @staticmethod
+    def attribute_getter(attribute):
+        if attribute == 'bpm':
+            preprocess_func = get_bpm
+        elif attribute == 'key':
+            preprocess_func = get_musical_key
+        elif attribute in ['moods', 'keywords']:
+            preprocess_func = get_keyword_list
+        elif attribute in ['genre', 'name', 'instrument']:
+            preprocess_func = get_keyword
+        elif attribute in ['title', 'artist', 'description']:
+            preprocess_func = get_string
+        else:
+            preprocess_func = None
+        return preprocess_func
+
+    @classmethod
+    def from_dict(cls, dictionary: dict, fields_required: bool = False):
+        _dictionary: tp.Dict[str, tp.Any] = {}
+
+        # allow a subset of attributes to not be loaded from the dictionary
+        # these attributes may be populated later
+        post_init_attributes = ['self_wav', 'chord', 'beat', 'joint_embed']
+        optional_fields = ['keywords']
+
+        for _field in fields(cls):
+            if _field.name in post_init_attributes:
+                continue
+            elif _field.name not in dictionary:
+                if fields_required and _field.name not in optional_fields:
+                    raise KeyError(f"Unexpected missing key: {_field.name}")
+            else:
+                preprocess_func: tp.Optional[tp.Callable] = cls.attribute_getter(_field.name)
+                value = dictionary[_field.name]
+                if preprocess_func:
+                    value = preprocess_func(value)
+                _dictionary[_field.name] = value
+        return cls(**_dictionary)
+
+
+def augment_music_info_description(music_info: MusicInfo, merge_text_p: float = 0.,
+                                   drop_desc_p: float = 0., drop_other_p: float = 0.) -> MusicInfo:
+    """Augment MusicInfo description with additional metadata fields and potential dropout.
+    Additional textual attributes are added given probability 'merge_text_conditions_p' and
+    the original textual description is dropped from the augmented description given probability drop_desc_p.
+
+    Args:
+        music_info (MusicInfo): The music metadata to augment.
+        merge_text_p (float): Probability of merging additional metadata to the description.
+            If provided value is 0, then no merging is performed.
+        drop_desc_p (float): Probability of dropping the original description on text merge.
+            if provided value is 0, then no drop out is performed.
+        drop_other_p (float): Probability of dropping the other fields used for text augmentation.
+    Returns:
+        MusicInfo: The MusicInfo with augmented textual description.
+    """
+    def is_valid_field(field_name: str, field_value: tp.Any) -> bool:
+        valid_field_name = field_name in ['key', 'bpm', 'genre', 'moods', 'instrument', 'keywords']
+        valid_field_value = field_value is not None and isinstance(field_value, (int, float, str, list))
+        keep_field = random.uniform(0, 1) < drop_other_p
+        return valid_field_name and valid_field_value and keep_field
+
+    def process_value(v: tp.Any) -> str:
+        if isinstance(v, (int, float, str)):
+            return str(v)
+        if isinstance(v, list):
+            return ", ".join(v)
+        else:
+            raise ValueError(f"Unknown type for text value! ({type(v), v})")
+
+    description = music_info.description
+
+    metadata_text = ""
+    # metadata_text = "rock style music, consistent rhythm, catchy song."
+    if random.uniform(0, 1) < merge_text_p:
+        meta_pairs = [f'{_field.name}: {process_value(getattr(music_info, _field.name))}'
+                      for _field in fields(music_info) if is_valid_field(_field.name, getattr(music_info, _field.name))]
+        random.shuffle(meta_pairs)
+        metadata_text = ". ".join(meta_pairs)
+        description = description if not random.uniform(0, 1) < drop_desc_p else None
+        logger.debug(f"Applying text augmentation on MMI info. description: {description}, metadata: {metadata_text}")
+
+    if description is None:
+        description = metadata_text if len(metadata_text) > 1 else None
+    else:
+        description = ". ".join([description.rstrip('.'), metadata_text])
+    description = description.strip() if description else None
+
+    music_info = replace(music_info)
+    music_info.description = description
+    return music_info
+
+
+class Paraphraser:
+    def __init__(self, paraphrase_source: tp.Union[str, Path], paraphrase_p: float = 0.):
+        self.paraphrase_p = paraphrase_p
+        open_fn = gzip.open if str(paraphrase_source).lower().endswith('.gz') else open
+        with open_fn(paraphrase_source, 'rb') as f:  # type: ignore
+            self.paraphrase_source = json.loads(f.read())
+        logger.info(f"loaded paraphrasing source from: {paraphrase_source}")
+
+    def sample_paraphrase(self, audio_path: str, description: str):
+        if random.random() >= self.paraphrase_p:
+            return description
+        info_path = Path(audio_path).with_suffix('.json')
+        if info_path not in self.paraphrase_source:
+            warn_once(logger, f"{info_path} not in paraphrase source!")
+            return description
+        new_desc = random.choice(self.paraphrase_source[info_path])
+        logger.debug(f"{description} -> {new_desc}")
+        return new_desc
+
+
+class MusicDataset(InfoAudioDataset):
+    """Music dataset is an AudioDataset with music-related metadata.
+
+    Args:
+        info_fields_required (bool): Whether to enforce having required fields.
+        merge_text_p (float): Probability of merging additional metadata to the description.
+        drop_desc_p (float): Probability of dropping the original description on text merge.
+        drop_other_p (float): Probability of dropping the other fields used for text augmentation.
+        joint_embed_attributes (list[str]): A list of attributes for which joint embedding metadata is returned.
+        paraphrase_source (str, optional): Path to the .json or .json.gz file containing the
+            paraphrases for the description. The json should be a dict with keys are the
+            original info path (e.g. track_path.json) and each value is a list of possible
+            paraphrased.
+        paraphrase_p (float): probability of taking a paraphrase.
+
+    See `audiocraft.data.info_audio_dataset.InfoAudioDataset` for full initialization arguments.
+    """
+    def __init__(self, *args, info_fields_required: bool = True,
+                 merge_text_p: float = 0., drop_desc_p: float = 0., drop_other_p: float = 0.,
+                 joint_embed_attributes: tp.List[str] = [],
+                 paraphrase_source: tp.Optional[str] = None, paraphrase_p: float = 0,
+                 **kwargs):
+        kwargs['return_info'] = True  # We require the info for each song of the dataset.
+        super().__init__(*args, **kwargs)
+        self.info_fields_required = info_fields_required
+        self.merge_text_p = merge_text_p
+        self.drop_desc_p = drop_desc_p
+        self.drop_other_p = drop_other_p
+        self.joint_embed_attributes = joint_embed_attributes
+        self.paraphraser = None
+        self.downsample_rate = 640
+        self.sr = 32000
+        if paraphrase_source is not None:
+            self.paraphraser = Paraphraser(paraphrase_source, paraphrase_p)
+
+    def __getitem__(self, index):
+        wav, info = super().__getitem__(index) # wav_seg and seg_info
+        info_data = info.to_dict()
+
+        # unpack info
+        target_sr = self.sr
+        n_frames_wave = info.n_frames
+        n_frames_feat = int(info.n_frames // self.downsample_rate)
+
+        music_info_path = str(info.meta.path).replace('no_vocal.wav', 'tags.json')
+        chord_path = str(info.meta.path).replace('no_vocal.wav', 'chord.lab')
+        beats_path = str(info.meta.path).replace('no_vocal.wav', 'beats.npy')
+
+        if all([
+            not Path(music_info_path).exists(),
+            not Path(beats_path).exists(),
+            not Path(chord_path).exists(),
+        ]):
+            raise FileNotFoundError
+
+        ### music info        
+        with open(music_info_path, 'r') as json_file:
+            music_data = json.load(json_file)
+            music_data.update(info_data)
+            music_info = MusicInfo.from_dict(music_data, fields_required=self.info_fields_required)
+        if self.paraphraser is not None:
+                music_info.description = self.paraphraser.sample(music_info.meta.path, music_info.description)
+        if self.merge_text_p:
+            music_info = augment_music_info_description(
+                music_info, self.merge_text_p, self.drop_desc_p, self.drop_other_p)
+        
+
+        ### load features to tensors ###
+        feat_hz = target_sr/self.downsample_rate
+        ## beat&bar: 2 x T
+        feat_beats = np.zeros((2, n_frames_feat))
+        
+        beats_np = np.load(beats_path)
+        beat_time = beats_np[:, 0]
+        bar_time = beats_np[np.where(beats_np[:, 1] == 1)[0], 0]
+        beat_frame = [
+            int((t-info.seek_time)*feat_hz) for t in beat_time
+                if (t >= info.seek_time and t < info.seek_time + self.segment_duration)]
+        bar_frame =[
+            int((t-info.seek_time)*feat_hz) for t in bar_time
+                if (t >= info.seek_time and t < info.seek_time + self.segment_duration)]
+        feat_beats[0, beat_frame] = 1
+        feat_beats[1, bar_frame] = 1
+        kernel = np.array([0.05, 0.1, 0.3, 0.9, 0.3, 0.1, 0.05])
+        feat_beats[0] = np.convolve(feat_beats[0] , kernel, 'same') # apply soft kernel
+        beat_events = feat_beats[0] + feat_beats[1]
+        beat_events = torch.tensor(beat_events).unsqueeze(0) # [T] -> [1, T]
+
+        music_info.beat = BeatCondition(beat=beat_events[None], length=torch.tensor([n_frames_feat]),
+                                        bpm=[music_data["bpm"]], path=[music_info_path], seek_frame=[info.seek_time*target_sr//self.downsample_rate])
+
+        ## chord: 12 x T
+        feat_chord = np.zeros((12, n_frames_feat)) # root| ivs
+        with open(chord_path, 'r') as f:
+            for line in f.readlines():
+                splits = line.split()
+                if len(splits) == 3:
+                    st_sec, ed_sec, ctag = splits
+                    st_sec = float(st_sec) - info.seek_time
+                    ed_sec = float(ed_sec) - info.seek_time
+                    st_frame = int(st_sec*feat_hz)
+                    ed_frame = int(ed_sec*feat_hz)
+                    
+                    # 12 chorma
+                    mhot = CHORDS.chord(ctag)
+                    final_vec = np.roll(mhot[2], mhot[0])
+
+                    final_vec = final_vec[..., None]
+                    feat_chord[:, st_frame:ed_frame] = final_vec
+        feat_chord = torch.from_numpy(feat_chord)
+        
+        music_info.chord = ChordCondition(
+                chord=feat_chord[None], length=torch.tensor([n_frames_feat]),
+                bpm=[music_data["bpm"]], path=[chord_path], seek_frame=[info.seek_time*self.sr//self.downsample_rate])
+            
+        music_info.self_wav = WavCondition(
+            wav=wav[None], length=torch.tensor([info.n_frames]),
+            sample_rate=[info.sample_rate], path=[info.meta.path], seek_time=[info.seek_time])
+
+        for att in self.joint_embed_attributes:
+            att_value = getattr(music_info, att)
+            joint_embed_cond = JointEmbedCondition(
+                wav[None], [att_value], torch.tensor([info.n_frames]),
+                sample_rate=[info.sample_rate], path=[info.meta.path], seek_time=[info.seek_time])
+            music_info.joint_embed[att] = joint_embed_cond
+
+        return wav, music_info 
+
+
+def get_musical_key(value: tp.Optional[str]) -> tp.Optional[str]:
+    """Preprocess key keywords, discarding them if there are multiple key defined."""
+    if value is None or (not isinstance(value, str)) or len(value) == 0 or value == 'None':
+        return None
+    elif ',' in value:
+        # For now, we discard when multiple keys are defined separated with comas
+        return None
+    else:
+        return value.strip().lower()
+
+
+def get_bpm(value: tp.Optional[str]) -> tp.Optional[float]:
+    """Preprocess to a float."""
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except ValueError:
+        return None
diff --git a/audiocraft/audiocraft/data/sound_dataset.py b/audiocraft/audiocraft/data/sound_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..8b88cbe8016b4bd28c2de749177c9af29f7755fc
--- /dev/null
+++ b/audiocraft/audiocraft/data/sound_dataset.py
@@ -0,0 +1,330 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Dataset of audio with a simple description.
+"""
+
+from dataclasses import dataclass, fields, replace
+import json
+from pathlib import Path
+import random
+import typing as tp
+
+import numpy as np
+import torch
+
+from .info_audio_dataset import (
+    InfoAudioDataset,
+    get_keyword_or_keyword_list
+)
+from ..modules.conditioners import (
+    ConditioningAttributes,
+    SegmentWithAttributes,
+    WavCondition,
+)
+
+
+EPS = torch.finfo(torch.float32).eps
+TARGET_LEVEL_LOWER = -35
+TARGET_LEVEL_UPPER = -15
+
+
+@dataclass
+class SoundInfo(SegmentWithAttributes):
+    """Segment info augmented with Sound metadata.
+    """
+    description: tp.Optional[str] = None
+    self_wav: tp.Optional[torch.Tensor] = None
+
+    @property
+    def has_sound_meta(self) -> bool:
+        return self.description is not None
+
+    def to_condition_attributes(self) -> ConditioningAttributes:
+        out = ConditioningAttributes()
+
+        for _field in fields(self):
+            key, value = _field.name, getattr(self, _field.name)
+            if key == 'self_wav':
+                out.wav[key] = value
+            else:
+                out.text[key] = value
+        return out
+
+    @staticmethod
+    def attribute_getter(attribute):
+        if attribute == 'description':
+            preprocess_func = get_keyword_or_keyword_list
+        else:
+            preprocess_func = None
+        return preprocess_func
+
+    @classmethod
+    def from_dict(cls, dictionary: dict, fields_required: bool = False):
+        _dictionary: tp.Dict[str, tp.Any] = {}
+
+        # allow a subset of attributes to not be loaded from the dictionary
+        # these attributes may be populated later
+        post_init_attributes = ['self_wav']
+
+        for _field in fields(cls):
+            if _field.name in post_init_attributes:
+                continue
+            elif _field.name not in dictionary:
+                if fields_required:
+                    raise KeyError(f"Unexpected missing key: {_field.name}")
+            else:
+                preprocess_func: tp.Optional[tp.Callable] = cls.attribute_getter(_field.name)
+                value = dictionary[_field.name]
+                if preprocess_func:
+                    value = preprocess_func(value)
+                _dictionary[_field.name] = value
+        return cls(**_dictionary)
+
+
+class SoundDataset(InfoAudioDataset):
+    """Sound audio dataset: Audio dataset with environmental sound-specific metadata.
+
+    Args:
+        info_fields_required (bool): Whether all the mandatory metadata fields should be in the loaded metadata.
+        external_metadata_source (tp.Optional[str]): Folder containing JSON metadata for the corresponding dataset.
+            The metadata files contained in this folder are expected to match the stem of the audio file with
+            a json extension.
+        aug_p (float): Probability of performing audio mixing augmentation on the batch.
+        mix_p (float): Proportion of batch items that are mixed together when applying audio mixing augmentation.
+        mix_snr_low (int): Lowerbound for SNR value sampled for mixing augmentation.
+        mix_snr_high (int): Upperbound for SNR value sampled for mixing augmentation.
+        mix_min_overlap (float): Minimum overlap between audio files when performing mixing augmentation.
+        kwargs: Additional arguments for AudioDataset.
+
+    See `audiocraft.data.info_audio_dataset.InfoAudioDataset` for full initialization arguments.
+    """
+    def __init__(
+        self,
+        *args,
+        info_fields_required: bool = True,
+        external_metadata_source: tp.Optional[str] = None,
+        aug_p: float = 0.,
+        mix_p: float = 0.,
+        mix_snr_low: int = -5,
+        mix_snr_high: int = 5,
+        mix_min_overlap: float = 0.5,
+        **kwargs
+    ):
+        kwargs['return_info'] = True  # We require the info for each song of the dataset.
+        super().__init__(*args, **kwargs)
+        self.info_fields_required = info_fields_required
+        self.external_metadata_source = external_metadata_source
+        self.aug_p = aug_p
+        self.mix_p = mix_p
+        if self.aug_p > 0:
+            assert self.mix_p > 0, "Expecting some mixing proportion mix_p if aug_p > 0"
+            assert self.channels == 1, "SoundDataset with audio mixing considers only monophonic audio"
+        self.mix_snr_low = mix_snr_low
+        self.mix_snr_high = mix_snr_high
+        self.mix_min_overlap = mix_min_overlap
+
+    def _get_info_path(self, path: tp.Union[str, Path]) -> Path:
+        """Get path of JSON with metadata (description, etc.).
+        If there exists a JSON with the same name as 'path.name', then it will be used.
+        Else, such JSON will be searched for in an external json source folder if it exists.
+        """
+        info_path = Path(path).with_suffix('.json')
+        if Path(info_path).exists():
+            return info_path
+        elif self.external_metadata_source and (Path(self.external_metadata_source) / info_path.name).exists():
+            return Path(self.external_metadata_source) / info_path.name
+        else:
+            raise Exception(f"Unable to find a metadata JSON for path: {path}")
+
+    def __getitem__(self, index):
+        wav, info = super().__getitem__(index)
+        info_data = info.to_dict()
+        info_path = self._get_info_path(info.meta.path)
+        if Path(info_path).exists():
+            with open(info_path, 'r') as json_file:
+                sound_data = json.load(json_file)
+                sound_data.update(info_data)
+                sound_info = SoundInfo.from_dict(sound_data, fields_required=self.info_fields_required)
+                # if there are multiple descriptions, sample one randomly
+                if isinstance(sound_info.description, list):
+                    sound_info.description = random.choice(sound_info.description)
+        else:
+            sound_info = SoundInfo.from_dict(info_data, fields_required=False)
+
+        sound_info.self_wav = WavCondition(
+            wav=wav[None], length=torch.tensor([info.n_frames]),
+            sample_rate=[sound_info.sample_rate], path=[info.meta.path], seek_time=[info.seek_time])
+
+        return wav, sound_info
+
+    def collater(self, samples):
+        # when training, audio mixing is performed in the collate function
+        wav, sound_info = super().collater(samples)  # SoundDataset always returns infos
+        if self.aug_p > 0:
+            wav, sound_info = mix_samples(wav, sound_info, self.aug_p, self.mix_p,
+                                          snr_low=self.mix_snr_low, snr_high=self.mix_snr_high,
+                                          min_overlap=self.mix_min_overlap)
+        return wav, sound_info
+
+
+def rms_f(x: torch.Tensor) -> torch.Tensor:
+    return (x ** 2).mean(1).pow(0.5)
+
+
+def normalize(audio: torch.Tensor, target_level: int = -25) -> torch.Tensor:
+    """Normalize the signal to the target level."""
+    rms = rms_f(audio)
+    scalar = 10 ** (target_level / 20) / (rms + EPS)
+    audio = audio * scalar.unsqueeze(1)
+    return audio
+
+
+def is_clipped(audio: torch.Tensor, clipping_threshold: float = 0.99) -> torch.Tensor:
+    return (abs(audio) > clipping_threshold).any(1)
+
+
+def mix_pair(src: torch.Tensor, dst: torch.Tensor, min_overlap: float) -> torch.Tensor:
+    start = random.randint(0, int(src.shape[1] * (1 - min_overlap)))
+    remainder = src.shape[1] - start
+    if dst.shape[1] > remainder:
+        src[:, start:] = src[:, start:] + dst[:, :remainder]
+    else:
+        src[:, start:start+dst.shape[1]] = src[:, start:start+dst.shape[1]] + dst
+    return src
+
+
+def snr_mixer(clean: torch.Tensor, noise: torch.Tensor, snr: int, min_overlap: float,
+              target_level: int = -25, clipping_threshold: float = 0.99) -> torch.Tensor:
+    """Function to mix clean speech and noise at various SNR levels.
+
+    Args:
+        clean (torch.Tensor): Clean audio source to mix, of shape [B, T].
+        noise (torch.Tensor): Noise audio source to mix, of shape [B, T].
+        snr (int): SNR level when mixing.
+        min_overlap (float): Minimum overlap between the two mixed sources.
+        target_level (int): Gain level in dB.
+        clipping_threshold (float): Threshold for clipping the audio.
+    Returns:
+        torch.Tensor: The mixed audio, of shape [B, T].
+    """
+    if clean.shape[1] > noise.shape[1]:
+        noise = torch.nn.functional.pad(noise, (0, clean.shape[1] - noise.shape[1]))
+    else:
+        noise = noise[:, :clean.shape[1]]
+
+    # normalizing to -25 dB FS
+    clean = clean / (clean.max(1)[0].abs().unsqueeze(1) + EPS)
+    clean = normalize(clean, target_level)
+    rmsclean = rms_f(clean)
+
+    noise = noise / (noise.max(1)[0].abs().unsqueeze(1) + EPS)
+    noise = normalize(noise, target_level)
+    rmsnoise = rms_f(noise)
+
+    # set the noise level for a given SNR
+    noisescalar = (rmsclean / (10 ** (snr / 20)) / (rmsnoise + EPS)).unsqueeze(1)
+    noisenewlevel = noise * noisescalar
+
+    # mix noise and clean speech
+    noisyspeech = mix_pair(clean, noisenewlevel, min_overlap)
+
+    # randomly select RMS value between -15 dBFS and -35 dBFS and normalize noisyspeech with that value
+    # there is a chance of clipping that might happen with very less probability, which is not a major issue.
+    noisy_rms_level = np.random.randint(TARGET_LEVEL_LOWER, TARGET_LEVEL_UPPER)
+    rmsnoisy = rms_f(noisyspeech)
+    scalarnoisy = (10 ** (noisy_rms_level / 20) / (rmsnoisy + EPS)).unsqueeze(1)
+    noisyspeech = noisyspeech * scalarnoisy
+    clean = clean * scalarnoisy
+    noisenewlevel = noisenewlevel * scalarnoisy
+
+    # final check to see if there are any amplitudes exceeding +/- 1. If so, normalize all the signals accordingly
+    clipped = is_clipped(noisyspeech)
+    if clipped.any():
+        noisyspeech_maxamplevel = noisyspeech[clipped].max(1)[0].abs().unsqueeze(1) / (clipping_threshold - EPS)
+        noisyspeech[clipped] = noisyspeech[clipped] / noisyspeech_maxamplevel
+
+    return noisyspeech
+
+
+def snr_mix(src: torch.Tensor, dst: torch.Tensor, snr_low: int, snr_high: int, min_overlap: float):
+    if snr_low == snr_high:
+        snr = snr_low
+    else:
+        snr = np.random.randint(snr_low, snr_high)
+    mix = snr_mixer(src, dst, snr, min_overlap)
+    return mix
+
+
+def mix_text(src_text: str, dst_text: str):
+    """Mix text from different sources by concatenating them."""
+    if src_text == dst_text:
+        return src_text
+    return src_text + " " + dst_text
+
+
+def mix_samples(wavs: torch.Tensor, infos: tp.List[SoundInfo], aug_p: float, mix_p: float,
+                snr_low: int, snr_high: int, min_overlap: float):
+    """Mix samples within a batch, summing the waveforms and concatenating the text infos.
+
+    Args:
+        wavs (torch.Tensor): Audio tensors of shape [B, C, T].
+        infos (list[SoundInfo]): List of SoundInfo items corresponding to the audio.
+        aug_p (float): Augmentation probability.
+        mix_p (float): Proportion of items in the batch to mix (and merge) together.
+        snr_low (int): Lowerbound for sampling SNR.
+        snr_high (int): Upperbound for sampling SNR.
+        min_overlap (float): Minimum overlap between mixed samples.
+    Returns:
+        tuple[torch.Tensor, list[SoundInfo]]: A tuple containing the mixed wavs
+            and mixed SoundInfo for the given batch.
+    """
+    # no mixing to perform within the batch
+    if mix_p == 0:
+        return wavs, infos
+
+    if random.uniform(0, 1) < aug_p:
+        # perform all augmentations on waveforms as [B, T]
+        # randomly picking pairs of audio to mix
+        assert wavs.size(1) == 1, f"Mix samples requires monophonic audio but C={wavs.size(1)}"
+        wavs = wavs.mean(dim=1, keepdim=False)
+        B, T = wavs.shape
+        k = int(mix_p * B)
+        mixed_sources_idx = torch.randperm(B)[:k]
+        mixed_targets_idx = torch.randperm(B)[:k]
+        aug_wavs = snr_mix(
+            wavs[mixed_sources_idx],
+            wavs[mixed_targets_idx],
+            snr_low,
+            snr_high,
+            min_overlap,
+        )
+        # mixing textual descriptions in metadata
+        descriptions = [info.description for info in infos]
+        aug_infos = []
+        for i, j in zip(mixed_sources_idx, mixed_targets_idx):
+            text = mix_text(descriptions[i], descriptions[j])
+            m = replace(infos[i])
+            m.description = text
+            aug_infos.append(m)
+
+        # back to [B, C, T]
+        aug_wavs = aug_wavs.unsqueeze(1)
+        assert aug_wavs.shape[0] > 0, "Samples mixing returned empty batch."
+        assert aug_wavs.dim() == 3, f"Returned wav should be [B, C, T] but dim = {aug_wavs.dim()}"
+        assert aug_wavs.shape[0] == len(aug_infos), "Mismatch between number of wavs and infos in the batch"
+
+        return aug_wavs, aug_infos  # [B, C, T]
+    else:
+        # randomly pick samples in the batch to match
+        # the batch size when performing audio mixing
+        B, C, T = wavs.shape
+        k = int(mix_p * B)
+        wav_idx = torch.randperm(B)[:k]
+        wavs = wavs[wav_idx]
+        infos = [infos[i] for i in wav_idx]
+        assert wavs.shape[0] == len(infos), "Mismatch between number of wavs and infos in the batch"
+
+        return wavs, infos  # [B, C, T]
diff --git a/audiocraft/audiocraft/data/zip.py b/audiocraft/audiocraft/data/zip.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0b17849d36991e7def35a14d3d518b9d867ce36
--- /dev/null
+++ b/audiocraft/audiocraft/data/zip.py
@@ -0,0 +1,76 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Utility for reading some info from inside a zip file.
+"""
+
+import typing
+import zipfile
+
+from dataclasses import dataclass
+from functools import lru_cache
+from typing_extensions import Literal
+
+
+DEFAULT_SIZE = 32
+MODE = Literal['r', 'w', 'x', 'a']
+
+
+@dataclass(order=True)
+class PathInZip:
+    """Hold a path of file within a zip file.
+
+    Args:
+        path (str): The convention is <path_to_zip>:<relative_path_inside_zip>.
+            Let's assume there is a zip file /some/location/foo.zip
+            and inside of it is a json file located at /data/file1.json,
+            Then we expect path = "/some/location/foo.zip:/data/file1.json".
+    """
+
+    INFO_PATH_SEP = ':'
+    zip_path: str
+    file_path: str
+
+    def __init__(self, path: str) -> None:
+        split_path = path.split(self.INFO_PATH_SEP)
+        assert len(split_path) == 2
+        self.zip_path, self.file_path = split_path
+
+    @classmethod
+    def from_paths(cls, zip_path: str, file_path: str):
+        return cls(zip_path + cls.INFO_PATH_SEP + file_path)
+
+    def __str__(self) -> str:
+        return self.zip_path + self.INFO_PATH_SEP + self.file_path
+
+
+def _open_zip(path: str, mode: MODE = 'r'):
+    return zipfile.ZipFile(path, mode)
+
+
+_cached_open_zip = lru_cache(DEFAULT_SIZE)(_open_zip)
+
+
+def set_zip_cache_size(max_size: int):
+    """Sets the maximal LRU caching for zip file opening.
+
+    Args:
+        max_size (int): the maximal LRU cache.
+    """
+    global _cached_open_zip
+    _cached_open_zip = lru_cache(max_size)(_open_zip)
+
+
+def open_file_in_zip(path_in_zip: PathInZip, mode: str = 'r') -> typing.IO:
+    """Opens a file stored inside a zip and returns a file-like object.
+
+    Args:
+        path_in_zip (PathInZip): A PathInZip object representing the file to return a file-like object of.
+        mode (str): The mode in which to open the file with.
+    Returns:
+        A file-like object for PathInZip.
+    """
+    zf = _cached_open_zip(path_in_zip.zip_path)
+    return zf.open(path_in_zip.file_path)
diff --git a/audiocraft/audiocraft/environment.py b/audiocraft/audiocraft/environment.py
new file mode 100644
index 0000000000000000000000000000000000000000..adc7819305758bb50a9984928bfa7f13eabef5f5
--- /dev/null
+++ b/audiocraft/audiocraft/environment.py
@@ -0,0 +1,176 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Provides cluster and tools configuration across clusters (slurm, dora, utilities).
+"""
+
+import logging
+import os
+from pathlib import Path
+import re
+import typing as tp
+
+import omegaconf
+
+from .utils.cluster import _guess_cluster_type
+
+
+logger = logging.getLogger(__name__)
+
+
+class AudioCraftEnvironment:
+    """Environment configuration for teams and clusters.
+
+    AudioCraftEnvironment picks compute cluster settings (slurm, dora) from the current running environment
+    or declared variable and the loaded team configuration. Additionally, the AudioCraftEnvironment
+    provides pointers to a reference folder resolved automatically across clusters that is shared across team members,
+    allowing to share sigs or other files to run jobs. Finally, it provides dataset mappers to automatically
+    map dataset file paths to new locations across clusters, allowing to use the same manifest of files across cluters.
+
+    The cluster type is identified automatically and base configuration file is read from config/teams.yaml.
+    Use the following environment variables to specify the cluster, team or configuration:
+
+        AUDIOCRAFT_CLUSTER (optional): Cluster type to enforce. Useful if the cluster type
+            cannot be inferred automatically.
+        AUDIOCRAFT_CONFIG (optional): Path to yaml config holding the teams configuration.
+            If not set, configuration is read from config/teams.yaml.
+        AUDIOCRAFT_TEAM (optional): Name of the team. Recommended to set to your own team.
+            Cluster configuration are shared across teams to match compute allocation,
+            specify your cluster configuration in the configuration file under a key mapping
+            your team name.
+    """
+    _instance = None
+    DEFAULT_TEAM = "default"
+
+    def __init__(self) -> None:
+        """Loads configuration."""
+        self.team: str = os.getenv("AUDIOCRAFT_TEAM", self.DEFAULT_TEAM)
+        cluster_type = _guess_cluster_type()
+        cluster = os.getenv(
+            "AUDIOCRAFT_CLUSTER", cluster_type.value
+        )
+        logger.info("Detecting cluster type %s", cluster_type)
+
+        self.cluster: str = cluster
+
+        config_path = os.getenv(
+            "AUDIOCRAFT_CONFIG",
+            Path(__file__)
+            .parent.parent.joinpath("config/teams", self.team)
+            .with_suffix(".yaml"),
+        )
+        self.config = omegaconf.OmegaConf.load(config_path)
+        self._dataset_mappers = []
+        cluster_config = self._get_cluster_config()
+        if "dataset_mappers" in cluster_config:
+            for pattern, repl in cluster_config["dataset_mappers"].items():
+                regex = re.compile(pattern)
+                self._dataset_mappers.append((regex, repl))
+
+    def _get_cluster_config(self) -> omegaconf.DictConfig:
+        assert isinstance(self.config, omegaconf.DictConfig)
+        return self.config[self.cluster]
+
+    @classmethod
+    def instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    @classmethod
+    def reset(cls):
+        """Clears the environment and forces a reload on next invocation."""
+        cls._instance = None
+
+    @classmethod
+    def get_team(cls) -> str:
+        """Gets the selected team as dictated by the AUDIOCRAFT_TEAM env var.
+        If not defined, defaults to "labs".
+        """
+        return cls.instance().team
+
+    @classmethod
+    def get_cluster(cls) -> str:
+        """Gets the detected cluster.
+        This value can be overridden by the AUDIOCRAFT_CLUSTER env var.
+        """
+        return cls.instance().cluster
+
+    @classmethod
+    def get_dora_dir(cls) -> Path:
+        """Gets the path to the dora directory for the current team and cluster.
+        Value is overridden by the AUDIOCRAFT_DORA_DIR env var.
+        """
+        cluster_config = cls.instance()._get_cluster_config()
+        dora_dir = os.getenv("AUDIOCRAFT_DORA_DIR", cluster_config["dora_dir"])
+        logger.warning(f"Dora directory: {dora_dir}")
+        return Path(dora_dir)
+
+    @classmethod
+    def get_reference_dir(cls) -> Path:
+        """Gets the path to the reference directory for the current team and cluster.
+        Value is overridden by the AUDIOCRAFT_REFERENCE_DIR env var.
+        """
+        cluster_config = cls.instance()._get_cluster_config()
+        return Path(os.getenv("AUDIOCRAFT_REFERENCE_DIR", cluster_config["reference_dir"]))
+
+    @classmethod
+    def get_slurm_exclude(cls) -> tp.Optional[str]:
+        """Get the list of nodes to exclude for that cluster."""
+        cluster_config = cls.instance()._get_cluster_config()
+        return cluster_config.get("slurm_exclude")
+
+    @classmethod
+    def get_slurm_partitions(cls, partition_types: tp.Optional[tp.List[str]] = None) -> str:
+        """Gets the requested partitions for the current team and cluster as a comma-separated string.
+
+        Args:
+            partition_types (list[str], optional): partition types to retrieve. Values must be
+                from ['global', 'team']. If not provided, the global partition is returned.
+        """
+        if not partition_types:
+            partition_types = ["global"]
+
+        cluster_config = cls.instance()._get_cluster_config()
+        partitions = [
+            cluster_config["partitions"][partition_type]
+            for partition_type in partition_types
+        ]
+        return ",".join(partitions)
+
+    @classmethod
+    def resolve_reference_path(cls, path: tp.Union[str, Path]) -> Path:
+        """Converts reference placeholder in path with configured reference dir to resolve paths.
+
+        Args:
+            path (str or Path): Path to resolve.
+        Returns:
+            Path: Resolved path.
+        """
+        path = str(path)
+
+        if path.startswith("//reference"):
+            reference_dir = cls.get_reference_dir()
+            logger.warn(f"Reference directory: {reference_dir}")
+            assert (
+                reference_dir.exists() and reference_dir.is_dir()
+            ), f"Reference directory does not exist: {reference_dir}."
+            path = re.sub("^//reference", str(reference_dir), path)
+
+        return Path(path)
+
+    @classmethod
+    def apply_dataset_mappers(cls, path: str) -> str:
+        """Applies dataset mapping regex rules as defined in the configuration.
+        If no rules are defined, the path is returned as-is.
+        """
+        instance = cls.instance()
+
+        for pattern, repl in instance._dataset_mappers:
+            path = pattern.sub(repl, path)
+
+        return path
diff --git a/audiocraft/audiocraft/grids/__init__.py b/audiocraft/audiocraft/grids/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..70643517cd1a8b4e712eca90e23411ae89937795
--- /dev/null
+++ b/audiocraft/audiocraft/grids/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Dora Grids."""
diff --git a/audiocraft/audiocraft/grids/_base_explorers.py b/audiocraft/audiocraft/grids/_base_explorers.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3f26666aa596f7bd2e8695c4f00e7963e978ceb
--- /dev/null
+++ b/audiocraft/audiocraft/grids/_base_explorers.py
@@ -0,0 +1,80 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from abc import ABC, abstractmethod
+import time
+import typing as tp
+from dora import Explorer
+import treetable as tt
+
+
+def get_sheep_ping(sheep) -> tp.Optional[str]:
+    """Return the amount of time since the Sheep made some update
+    to its log. Returns a str using the relevant time unit."""
+    ping = None
+    if sheep.log is not None and sheep.log.exists():
+        delta = time.time() - sheep.log.stat().st_mtime
+        if delta > 3600 * 24:
+            ping = f'{delta / (3600 * 24):.1f}d'
+        elif delta > 3600:
+            ping = f'{delta / (3600):.1f}h'
+        elif delta > 60:
+            ping = f'{delta / 60:.1f}m'
+        else:
+            ping = f'{delta:.1f}s'
+    return ping
+
+
+class BaseExplorer(ABC, Explorer):
+    """Base explorer for AudioCraft grids.
+
+    All task specific solvers are expected to implement the `get_grid_metrics`
+    method to specify logic about metrics to display for a given task.
+
+    If additional stages are used, the child explorer must define how to handle
+    these new stages in the `process_history` and `process_sheep` methods.
+    """
+    def stages(self):
+        return ["train", "valid", "evaluate"]
+
+    def get_grid_meta(self):
+        """Returns the list of Meta information to display for each XP/job.
+        """
+        return [
+            tt.leaf("index", align=">"),
+            tt.leaf("name", wrap=140),
+            tt.leaf("state"),
+            tt.leaf("sig", align=">"),
+            tt.leaf("sid", align="<"),
+        ]
+
+    @abstractmethod
+    def get_grid_metrics(self):
+        """Return the metrics that should be displayed in the tracking table.
+        """
+        ...
+
+    def process_sheep(self, sheep, history):
+        train = {
+            "epoch": len(history),
+        }
+        parts = {"train": train}
+        for metrics in history:
+            for key, sub in metrics.items():
+                part = parts.get(key, {})
+                if 'duration' in sub:
+                    # Convert to minutes for readability.
+                    sub['duration'] = sub['duration'] / 60.
+                part.update(sub)
+                parts[key] = part
+        ping = get_sheep_ping(sheep)
+        if ping is not None:
+            for name in self.stages():
+                if name not in parts:
+                    parts[name] = {}
+                # Add the ping to each part for convenience.
+                parts[name]['ping'] = ping
+        return parts
diff --git a/audiocraft/audiocraft/grids/audiogen/__init__.py b/audiocraft/audiocraft/grids/audiogen/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a0a2688450ce120088b79c3314a2f267394dc11
--- /dev/null
+++ b/audiocraft/audiocraft/grids/audiogen/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""AudioGen grids."""
diff --git a/audiocraft/audiocraft/grids/audiogen/audiogen_base_16khz.py b/audiocraft/audiocraft/grids/audiogen/audiogen_base_16khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..190cc1d0a1e316347e8ebbdfc8de7e2942c1b3d7
--- /dev/null
+++ b/audiocraft/audiocraft/grids/audiogen/audiogen_base_16khz.py
@@ -0,0 +1,23 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ..musicgen._explorers import LMExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@LMExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=64, partition=partitions)
+    launcher.bind_(solver='audiogen/audiogen_base_16khz')
+    # replace this by the desired environmental sound dataset
+    launcher.bind_(dset='internal/sounds_16khz')
+
+    fsdp = {'autocast': False, 'fsdp.use': True}
+    medium = {'model/lm/model_scale': 'medium'}
+
+    launcher.bind_(fsdp)
+    launcher(medium)
diff --git a/audiocraft/audiocraft/grids/audiogen/audiogen_pretrained_16khz_eval.py b/audiocraft/audiocraft/grids/audiogen/audiogen_pretrained_16khz_eval.py
new file mode 100644
index 0000000000000000000000000000000000000000..12f6d402a3c4a113d4c37be062790fa435b72104
--- /dev/null
+++ b/audiocraft/audiocraft/grids/audiogen/audiogen_pretrained_16khz_eval.py
@@ -0,0 +1,68 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Evaluation with objective metrics for the pretrained AudioGen models.
+This grid takes signature from the training grid and runs evaluation-only stage.
+
+When running the grid for the first time, please use:
+REGEN=1 dora grid audiogen.audiogen_pretrained_16khz_eval
+and re-use the REGEN=1 option when the grid is changed to force regenerating it.
+
+Note that you need the proper metrics external libraries setup to use all
+the objective metrics activated in this grid. Refer to the README for more information.
+"""
+
+import os
+
+from ..musicgen._explorers import GenerationEvalExplorer
+from ...environment import AudioCraftEnvironment
+from ... import train
+
+
+def eval(launcher, batch_size: int = 32):
+    opts = {
+        'dset': 'audio/audiocaps_16khz',
+        'solver/audiogen/evaluation': 'objective_eval',
+        'execute_only': 'evaluate',
+        '+dataset.evaluate.batch_size': batch_size,
+        '+metrics.fad.tf.batch_size': 32,
+    }
+    # binary for FAD computation: replace this path with your own path
+    metrics_opts = {
+        'metrics.fad.tf.bin': '/data/home/jadecopet/local/usr/opt/google-research'
+    }
+    opt1 = {'generate.lm.use_sampling': True, 'generate.lm.top_k': 250, 'generate.lm.top_p': 0.}
+    opt2 = {'transformer_lm.two_step_cfg': True}
+
+    sub = launcher.bind(opts)
+    sub.bind_(metrics_opts)
+
+    # base objective metrics
+    sub(opt1, opt2)
+
+
+@GenerationEvalExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=4, partition=partitions)
+
+    if 'REGEN' not in os.environ:
+        folder = train.main.dora.dir / 'grids' / __name__.split('.', 2)[-1]
+        with launcher.job_array():
+            for sig in folder.iterdir():
+                if not sig.is_symlink():
+                    continue
+                xp = train.main.get_xp_from_sig(sig.name)
+                launcher(xp.argv)
+        return
+
+    audiogen_base = launcher.bind(solver="audiogen/audiogen_base_16khz")
+    audiogen_base.bind_({'autocast': False, 'fsdp.use': True})
+
+    audiogen_base_medium = audiogen_base.bind({'continue_from': '//pretrained/facebook/audiogen-medium'})
+    audiogen_base_medium.bind_({'model/lm/model_scale': 'medium'})
+    eval(audiogen_base_medium, batch_size=128)
diff --git a/audiocraft/audiocraft/grids/compression/__init__.py b/audiocraft/audiocraft/grids/compression/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b688528f1f3e4efc0c2a1e9d490f33c4158b3f0
--- /dev/null
+++ b/audiocraft/audiocraft/grids/compression/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""EnCodec grids."""
diff --git a/audiocraft/audiocraft/grids/compression/_explorers.py b/audiocraft/audiocraft/grids/compression/_explorers.py
new file mode 100644
index 0000000000000000000000000000000000000000..eed30d5b8a1c14676503148ddf133c79ed2e33bf
--- /dev/null
+++ b/audiocraft/audiocraft/grids/compression/_explorers.py
@@ -0,0 +1,55 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import treetable as tt
+
+from .._base_explorers import BaseExplorer
+
+
+class CompressionExplorer(BaseExplorer):
+    eval_metrics = ["sisnr", "visqol"]
+
+    def stages(self):
+        return ["train", "valid", "evaluate"]
+
+    def get_grid_meta(self):
+        """Returns the list of Meta information to display for each XP/job.
+        """
+        return [
+            tt.leaf("index", align=">"),
+            tt.leaf("name", wrap=140),
+            tt.leaf("state"),
+            tt.leaf("sig", align=">"),
+        ]
+
+    def get_grid_metrics(self):
+        """Return the metrics that should be displayed in the tracking table.
+        """
+        return [
+            tt.group(
+                "train",
+                [
+                    tt.leaf("epoch"),
+                    tt.leaf("bandwidth", ".2f"),
+                    tt.leaf("adv", ".4f"),
+                    tt.leaf("d_loss", ".4f"),
+                ],
+                align=">",
+            ),
+            tt.group(
+                "valid",
+                [
+                    tt.leaf("bandwidth", ".2f"),
+                    tt.leaf("adv", ".4f"),
+                    tt.leaf("msspec", ".4f"),
+                    tt.leaf("sisnr", ".2f"),
+                ],
+                align=">",
+            ),
+            tt.group(
+                "evaluate", [tt.leaf(name, ".3f") for name in self.eval_metrics], align=">"
+            ),
+        ]
diff --git a/audiocraft/audiocraft/grids/compression/debug.py b/audiocraft/audiocraft/grids/compression/debug.py
new file mode 100644
index 0000000000000000000000000000000000000000..5612ff5688d85fede0e605b244919e8081cb1da9
--- /dev/null
+++ b/audiocraft/audiocraft/grids/compression/debug.py
@@ -0,0 +1,31 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Grid search file, simply list all the exp you want in `explorer`.
+Any new exp added there will be scheduled.
+You can cancel and experiment by commenting its line.
+
+This grid is a minimal example for debugging compression task
+and how to override parameters directly in a grid.
+Learn more about dora grids: https://github.com/facebookresearch/dora
+"""
+
+from ._explorers import CompressionExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@CompressionExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=2, partition=partitions)
+    launcher.bind_(solver='compression/debug')
+
+    with launcher.job_array():
+        # base debug task using config from solver=compression/debug
+        launcher()
+        # we can override parameters in the grid to launch additional xps
+        launcher({'rvq.bins': 2048, 'rvq.n_q': 4})
diff --git a/audiocraft/audiocraft/grids/compression/encodec_audiogen_16khz.py b/audiocraft/audiocraft/grids/compression/encodec_audiogen_16khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..c9b41f684045594bb264cfb7f4f15d1da439382c
--- /dev/null
+++ b/audiocraft/audiocraft/grids/compression/encodec_audiogen_16khz.py
@@ -0,0 +1,29 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Grid search file, simply list all the exp you want in `explorer`.
+Any new exp added there will be scheduled.
+You can cancel and experiment by commenting its line.
+
+This grid shows how to train the new AudioGen EnCodec model at 16 kHz.
+"""
+
+from ._explorers import CompressionExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@CompressionExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=8, partition=partitions)
+    # use configuration for AudioGen's EnCodec model trained on monophonic audio sampled at 16 kHz
+    # AudioGen's EnCodec is trained with a total stride of 320 leading to a frame rate of 50 hz
+    launcher.bind_(solver='compression/encodec_audiogen_16khz')
+    # replace this by the desired sound dataset
+    launcher.bind_(dset='internal/sounds_16khz')
+    # launch xp
+    launcher()
diff --git a/audiocraft/audiocraft/grids/compression/encodec_base_24khz.py b/audiocraft/audiocraft/grids/compression/encodec_base_24khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..117b2b1e496ca31b3d614672b472c9213cedb4ad
--- /dev/null
+++ b/audiocraft/audiocraft/grids/compression/encodec_base_24khz.py
@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Grid search file, simply list all the exp you want in `explorer`.
+Any new exp added there will be scheduled.
+You can cancel and experiment by commenting its line.
+
+This grid shows how to train a base causal EnCodec model at 24 kHz.
+"""
+
+from ._explorers import CompressionExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@CompressionExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=8, partition=partitions)
+    # base causal EnCodec trained on monophonic audio sampled at 24 kHz
+    launcher.bind_(solver='compression/encodec_base_24khz')
+    # replace this by the desired dataset
+    launcher.bind_(dset='audio/example')
+    # launch xp
+    launcher()
diff --git a/audiocraft/audiocraft/grids/compression/encodec_musicgen_32khz.py b/audiocraft/audiocraft/grids/compression/encodec_musicgen_32khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..9da31daa5f009f46e753601a51a06391594b8f9b
--- /dev/null
+++ b/audiocraft/audiocraft/grids/compression/encodec_musicgen_32khz.py
@@ -0,0 +1,34 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Grid search file, simply list all the exp you want in `explorer`.
+Any new exp added there will be scheduled.
+You can cancel and experiment by commenting its line.
+
+This grid shows how to train a MusicGen EnCodec model at 32 kHz.
+"""
+
+from ._explorers import CompressionExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@CompressionExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=8, partition=partitions)
+    # use configuration for MusicGen's EnCodec model trained on monophonic audio sampled at 32 kHz
+    # MusicGen's EnCodec is trained with a total stride of 640 leading to a frame rate of 50 hz
+    launcher.bind_(solver='compression/encodec_musicgen_32khz')
+    # replace this by the desired music dataset
+    launcher.bind_(dset='internal/music_400k_32khz')
+    # launch xp
+    launcher()
+    launcher({
+        'metrics.visqol.bin': '/data/home/jadecopet/local/usr/opt/visqol',
+        'label': 'visqol',
+        'evaluate.metrics.visqol': True
+    })
diff --git a/audiocraft/audiocraft/grids/diffusion/4_bands_base_32khz.py b/audiocraft/audiocraft/grids/diffusion/4_bands_base_32khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7e67bcc89dd0c8e50d770e600b55f179fe19588
--- /dev/null
+++ b/audiocraft/audiocraft/grids/diffusion/4_bands_base_32khz.py
@@ -0,0 +1,27 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Training of the 4 diffusion models described in
+"From Discrete Tokens to High-Fidelity Audio Using Multi-Band Diffusion"
+(paper link).
+"""
+
+from ._explorers import DiffusionExplorer
+
+
+@DiffusionExplorer
+def explorer(launcher):
+    launcher.slurm_(gpus=4, partition='learnfair')
+
+    launcher.bind_({'solver': 'diffusion/default',
+                    'dset': 'internal/music_10k_32khz'})
+
+    with launcher.job_array():
+        launcher({'filter.use': True, 'filter.idx_band': 0, "processor.use": False, 'processor.power_std': 0.4})
+        launcher({'filter.use': True, 'filter.idx_band': 1, "processor.use": False, 'processor.power_std': 0.4})
+        launcher({'filter.use': True, 'filter.idx_band': 2, "processor.use": True, 'processor.power_std': 0.4})
+        launcher({'filter.use': True, 'filter.idx_band': 3, "processor.use": True, 'processor.power_std': 0.75})
diff --git a/audiocraft/audiocraft/grids/diffusion/__init__.py b/audiocraft/audiocraft/grids/diffusion/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e5737294ae16c0de52085b8dcf6825c348f617e4
--- /dev/null
+++ b/audiocraft/audiocraft/grids/diffusion/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Diffusion grids."""
diff --git a/audiocraft/audiocraft/grids/diffusion/_explorers.py b/audiocraft/audiocraft/grids/diffusion/_explorers.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bf4ca57b63f5f9308bd1178ddbde5d8f06748e5
--- /dev/null
+++ b/audiocraft/audiocraft/grids/diffusion/_explorers.py
@@ -0,0 +1,66 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import treetable as tt
+
+from .._base_explorers import BaseExplorer
+
+
+class DiffusionExplorer(BaseExplorer):
+    eval_metrics = ["sisnr", "visqol"]
+
+    def stages(self):
+        return ["train", "valid", "valid_ema", "evaluate", "evaluate_ema"]
+
+    def get_grid_meta(self):
+        """Returns the list of Meta information to display for each XP/job.
+        """
+        return [
+            tt.leaf("index", align=">"),
+            tt.leaf("name", wrap=140),
+            tt.leaf("state"),
+            tt.leaf("sig", align=">"),
+        ]
+
+    def get_grid_metrics(self):
+        """Return the metrics that should be displayed in the tracking table.
+        """
+        return [
+            tt.group(
+                "train",
+                [
+                    tt.leaf("epoch"),
+                    tt.leaf("loss", ".3%"),
+                ],
+                align=">",
+            ),
+            tt.group(
+                "valid",
+                [
+                    tt.leaf("loss", ".3%"),
+                    # tt.leaf("loss_0", ".3%"),
+                ],
+                align=">",
+            ),
+            tt.group(
+                "valid_ema",
+                [
+                    tt.leaf("loss", ".3%"),
+                    # tt.leaf("loss_0", ".3%"),
+                ],
+                align=">",
+            ),
+            tt.group(
+                "evaluate", [tt.leaf("rvm", ".4f"), tt.leaf("rvm_0", ".4f"),
+                             tt.leaf("rvm_1", ".4f"), tt.leaf("rvm_2", ".4f"),
+                             tt.leaf("rvm_3", ".4f"), ], align=">"
+            ),
+            tt.group(
+                "evaluate_ema", [tt.leaf("rvm", ".4f"), tt.leaf("rvm_0", ".4f"),
+                                 tt.leaf("rvm_1", ".4f"), tt.leaf("rvm_2", ".4f"),
+                                 tt.leaf("rvm_3", ".4f")], align=">"
+            ),
+        ]
diff --git a/audiocraft/audiocraft/grids/musicgen/__init__.py b/audiocraft/audiocraft/grids/musicgen/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d3f101f5a29ff85271e44e4f27545168a8f27baa
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""MusicGen grids."""
diff --git a/audiocraft/audiocraft/grids/musicgen/_explorers.py b/audiocraft/audiocraft/grids/musicgen/_explorers.py
new file mode 100644
index 0000000000000000000000000000000000000000..334836b72559a120feb8a15eef3fe96ce88a4edb
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/_explorers.py
@@ -0,0 +1,93 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import treetable as tt
+
+from .._base_explorers import BaseExplorer
+
+
+class LMExplorer(BaseExplorer):
+    eval_metrics: tp.List[str] = []
+
+    def stages(self) -> tp.List[str]:
+        return ['train', 'valid']
+
+    def get_grid_metrics(self):
+        """Return the metrics that should be displayed in the tracking table."""
+        return [
+            tt.group(
+                'train',
+                [
+                    tt.leaf('epoch'),
+                    tt.leaf('duration', '.1f'),  # duration in minutes
+                    tt.leaf('ping'),
+                    tt.leaf('ce', '.4f'),  # cross entropy
+                    tt.leaf("ppl", '.3f'),  # perplexity
+                ],
+                align='>',
+            ),
+            tt.group(
+                'valid',
+                [
+                    tt.leaf('ce', '.4f'),
+                    tt.leaf('ppl', '.3f'),
+                    tt.leaf('best_ppl', '.3f'),
+                ],
+                align='>',
+            ),
+        ]
+
+    def process_sheep(self, sheep, history):
+        parts = super().process_sheep(sheep, history)
+
+        track_by = {'ppl': 'lower'}  # values should be in ['lower', 'higher']
+        best_metrics = {k: (1 if v == 'lower' else -1) * float('inf') for k, v in track_by.items()}
+
+        def comparator(mode, a, b):
+            return a < b if mode == 'lower' else a > b
+
+        for metrics in history:
+            for key, sub in metrics.items():
+                for metric in track_by:
+                    # for the validation set, keep track of best metrics (ppl in this example)
+                    # this is so we can conveniently compare metrics between runs in the grid
+                    if key == 'valid' and metric in sub and comparator(
+                        track_by[metric], sub[metric], best_metrics[metric]
+                    ):
+                        best_metrics[metric] = sub[metric]
+
+        if 'valid' in parts:
+            parts['valid'].update({f'best_{k}': v for k, v in best_metrics.items()})
+        return parts
+
+
+class GenerationEvalExplorer(BaseExplorer):
+    eval_metrics: tp.List[str] = []
+
+    def stages(self) -> tp.List[str]:
+        return ['evaluate']
+
+    def get_grid_metrics(self):
+        """Return the metrics that should be displayed in the tracking table."""
+        return [
+            tt.group(
+                'evaluate',
+                [
+                    tt.leaf('epoch', '.3f'),
+                    tt.leaf('duration', '.1f'),
+                    tt.leaf('ping'),
+                    tt.leaf('ce', '.4f'),
+                    tt.leaf('ppl', '.3f'),
+                    tt.leaf('fad', '.3f'),
+                    tt.leaf('kld', '.3f'),
+                    tt.leaf('text_consistency', '.3f'),
+                    tt.leaf('chroma_cosine', '.3f'),
+                ],
+                align='>',
+            ),
+        ]
diff --git a/audiocraft/audiocraft/grids/musicgen/musicgen_base_32khz.py b/audiocraft/audiocraft/grids/musicgen/musicgen_base_32khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e364614537e426f21c18a2c2a9d94b3babce051
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/musicgen_base_32khz.py
@@ -0,0 +1,43 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ._explorers import LMExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@LMExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=32, partition=partitions)
+    launcher.bind_(solver='musicgen/musicgen_base_32khz')
+    # replace this by the desired music dataset
+    launcher.bind_(dset='internal/music_400k_32khz')
+
+    fsdp = {'autocast': False, 'fsdp.use': True}
+    medium = {'model/lm/model_scale': 'medium'}
+    large = {'model/lm/model_scale': 'large'}
+
+    cfg_low = {'classifier_free_guidance.training_dropout': 0.2}
+    wd_low = {'conditioners.description.t5.word_dropout': 0.2}
+
+    adam = {'optim.optimizer': 'adamw', 'optim.lr': 1e-4}
+
+    launcher.bind_(fsdp)
+
+    launcher.slurm_(gpus=32).bind_(label='32gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub()
+
+    launcher.slurm_(gpus=64).bind_(label='64gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub(medium, adam)
+
+    launcher.slurm_(gpus=96).bind_(label='96gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub(large, cfg_low, wd_low, adam, {'optim.max_norm': 3})
diff --git a/audiocraft/audiocraft/grids/musicgen/musicgen_base_cached_32khz.py b/audiocraft/audiocraft/grids/musicgen/musicgen_base_cached_32khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9a43f37d7369b5de4542fba87c4c8739d58b1e8
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/musicgen_base_cached_32khz.py
@@ -0,0 +1,67 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ._explorers import LMExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@LMExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=32, partition=partitions)
+    launcher.bind_(solver='musicgen/musicgen_base_32khz')
+    # replace this by the desired music dataset
+    launcher.bind_(dset='internal/music_400k_32khz')
+
+    fsdp = {'autocast': False, 'fsdp.use': True}
+    medium = {'model/lm/model_scale': 'medium'}
+    large = {'model/lm/model_scale': 'large'}
+
+    cfg_low = {'classifier_free_guidance.training_dropout': 0.2}
+    wd_low = {'conditioners.description.t5.word_dropout': 0.2}
+
+    adam = {'optim.optimizer': 'adamw', 'optim.lr': 1e-4}
+
+    # BEGINNING OF CACHE WRITING JOBS.
+    cache_write = {
+        'cache.path': '/fsx-codegen/defossez/cache/interleave_stereo_nv_32k',
+        'cache.write': True,
+        'generate.every': 500,
+        'evaluate.every': 500,
+        'logging.log_updates': 50,
+    }
+
+    cache_sub = launcher.bind({'model/lm/model_scale': 'xsmall', 'conditioner': 'none'})
+    cache_sub.bind_({'deadlock.use': True})
+    cache_sub.slurm_(gpus=8)
+    with launcher.job_array():
+        num_shards = 10  # total number of jobs running in parallel.
+        for shard in range(0, num_shards):
+            launcher(cache_write, {'cache.write_num_shards': num_shards, 'cache.write_shard': shard})
+
+    # REMOVE THE FOLLOWING RETURN STATEMENT ONCE THE ABOVE JOBS ARE DONE,
+    # OR SUFFICIENTLY AHEAD.
+    return
+
+    cache = {
+        'cache.path': '/fsx-codegen/defossez/cache/interleave_stereo_nv_32k',
+    }
+    launcher.bind_(fsdp, cache)
+
+    launcher.slurm_(gpus=32).bind_(label='32gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub()
+
+    launcher.slurm_(gpus=64).bind_(label='64gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub(medium, adam)
+
+    launcher.slurm_(gpus=96).bind_(label='96gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub(large, cfg_low, wd_low, adam, {'optim.max_norm': 3})
diff --git a/audiocraft/audiocraft/grids/musicgen/musicgen_clapemb_32khz.py b/audiocraft/audiocraft/grids/musicgen/musicgen_clapemb_32khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..64ad3f8c77afe1ab5908e407ad14d4879e1b1ad1
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/musicgen_clapemb_32khz.py
@@ -0,0 +1,32 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ._explorers import LMExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@LMExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=32, partition=partitions)
+    launcher.bind_(solver='musicgen/musicgen_base_32khz')
+    # replace this by the desired music dataset
+    launcher.bind_(dset='internal/music_400k_32khz')
+    launcher.bind_(conditioner='clapemb2music')
+
+    fsdp = {'autocast': False, 'fsdp.use': True}
+    cache_path = {'conditioners.description.clap.cache_path':
+                  '/fsx-audio-craft-llm/jadecopet/experiments/audiocraft/caches/clap_embed_music'}
+    text_wav_training_opt = {'conditioners.description.clap.text_p': 0.5}
+
+    launcher.bind_(fsdp)
+
+    launcher.slurm_(gpus=32).bind_(label='32gpus')
+    with launcher.job_array():
+        launcher()
+        launcher(text_wav_training_opt)
+        launcher(cache_path)
+        launcher(cache_path, text_wav_training_opt)
diff --git a/audiocraft/audiocraft/grids/musicgen/musicgen_melody_32khz.py b/audiocraft/audiocraft/grids/musicgen/musicgen_melody_32khz.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0d6710a23c117406e9724057a62eccab88ce907
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/musicgen_melody_32khz.py
@@ -0,0 +1,65 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from ._explorers import LMExplorer
+from ...environment import AudioCraftEnvironment
+
+
+@LMExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=32, partition=partitions)
+    launcher.bind_(solver='musicgen/musicgen_melody_32khz')
+    # replace this by the desired music dataset
+    launcher.bind_(dset='internal/music_400k_32khz')
+
+    fsdp = {'autocast': False, 'fsdp.use': True}
+    medium = {'model/lm/model_scale': 'medium'}
+    large = {'model/lm/model_scale': 'large'}
+
+    cfg_low = {'classifier_free_guidance.training_dropout': 0.2}
+    wd_low = {'conditioners.description.t5.word_dropout': 0.2}
+
+    adam = {'optim.optimizer': 'adamw', 'optim.lr': 1e-4}
+
+    cache_path = {'conditioners.self_wav.chroma_stem.cache_path':
+                  '/fsx-audio-craft-llm/jadecopet/experiments/audiocraft/caches/chroma_stem'}
+
+    # CACHE GENERATION JOBS
+    n_cache_gen_jobs = 4
+    gen_sub = launcher.slurm(gpus=1)
+    gen_sub.bind_(
+        cache_path, {
+            # the cache is always computed over the whole file, so duration doesn't matter here.
+            'dataset.segment_duration': 2.,
+            'dataset.batch_size': 8,
+            'dataset.train.permutation_on_files': True,  # try to not repeat files.
+            'optim.epochs': 10,
+            'model/lm/model_scale': 'xsmall',
+
+        })
+    with gen_sub.job_array():
+        for gen_job in range(n_cache_gen_jobs):
+            gen_sub({'dataset.train.shuffle_seed': gen_job})
+
+    # ACTUAL TRAINING JOBS.
+    launcher.bind_(fsdp)
+
+    launcher.slurm_(gpus=32).bind_(label='32gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub()
+        sub(cache_path)
+
+    launcher.slurm_(gpus=64).bind_(label='64gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub(medium, adam)
+
+    launcher.slurm_(gpus=96).bind_(label='96gpus')
+    with launcher.job_array():
+        sub = launcher.bind()
+        sub(large, cfg_low, wd_low, adam, {'optim.max_norm': 3})
diff --git a/audiocraft/audiocraft/grids/musicgen/musicgen_pretrained_32khz_eval.py b/audiocraft/audiocraft/grids/musicgen/musicgen_pretrained_32khz_eval.py
new file mode 100644
index 0000000000000000000000000000000000000000..39ceaf7dab15ec3f0f669cfe57ca9e932a9ab40d
--- /dev/null
+++ b/audiocraft/audiocraft/grids/musicgen/musicgen_pretrained_32khz_eval.py
@@ -0,0 +1,99 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Evaluation with objective metrics for the pretrained MusicGen models.
+This grid takes signature from the training grid and runs evaluation-only stage.
+
+When running the grid for the first time, please use:
+REGEN=1 dora grid musicgen.musicgen_pretrained_32khz_eval
+and re-use the REGEN=1 option when the grid is changed to force regenerating it.
+
+Note that you need the proper metrics external libraries setup to use all
+the objective metrics activated in this grid. Refer to the README for more information.
+"""
+
+import os
+
+from ._explorers import GenerationEvalExplorer
+from ...environment import AudioCraftEnvironment
+from ... import train
+
+
+def eval(launcher, batch_size: int = 32, eval_melody: bool = False):
+    opts = {
+        'dset': 'audio/musiccaps_32khz',
+        'solver/musicgen/evaluation': 'objective_eval',
+        'execute_only': 'evaluate',
+        '+dataset.evaluate.batch_size': batch_size,
+        '+metrics.fad.tf.batch_size': 16,
+    }
+    # chroma-specific evaluation
+    chroma_opts = {
+        'dset': 'internal/music_400k_32khz',
+        'dataset.evaluate.segment_duration': 30,
+        'dataset.evaluate.num_samples': 1000,
+        'evaluate.metrics.chroma_cosine': True,
+        'evaluate.metrics.fad': False,
+        'evaluate.metrics.kld': False,
+        'evaluate.metrics.text_consistency': False,
+    }
+    # binary for FAD computation: replace this path with your own path
+    metrics_opts = {
+        'metrics.fad.tf.bin': '/data/home/jadecopet/local/usr/opt/google-research'
+    }
+    opt1 = {'generate.lm.use_sampling': True, 'generate.lm.top_k': 250, 'generate.lm.top_p': 0.}
+    opt2 = {'transformer_lm.two_step_cfg': True}
+
+    sub = launcher.bind(opts)
+    sub.bind_(metrics_opts)
+
+    # base objective metrics
+    sub(opt1, opt2)
+
+    if eval_melody:
+        # chroma-specific metrics
+        sub(opt1, opt2, chroma_opts)
+
+
+@GenerationEvalExplorer
+def explorer(launcher):
+    partitions = AudioCraftEnvironment.get_slurm_partitions(['team', 'global'])
+    launcher.slurm_(gpus=4, partition=partitions)
+
+    if 'REGEN' not in os.environ:
+        folder = train.main.dora.dir / 'grids' / __name__.split('.', 2)[-1]
+        with launcher.job_array():
+            for sig in folder.iterdir():
+                if not sig.is_symlink():
+                    continue
+                xp = train.main.get_xp_from_sig(sig.name)
+                launcher(xp.argv)
+        return
+
+    with launcher.job_array():
+        musicgen_base = launcher.bind(solver="musicgen/musicgen_base_32khz")
+        musicgen_base.bind_({'autocast': False, 'fsdp.use': True})
+
+        # base musicgen models
+        musicgen_base_small = musicgen_base.bind({'continue_from': '//pretrained/facebook/musicgen-small'})
+        eval(musicgen_base_small, batch_size=128)
+
+        musicgen_base_medium = musicgen_base.bind({'continue_from': '//pretrained/facebook/musicgen-medium'})
+        musicgen_base_medium.bind_({'model/lm/model_scale': 'medium'})
+        eval(musicgen_base_medium, batch_size=128)
+
+        musicgen_base_large = musicgen_base.bind({'continue_from': '//pretrained/facebook/musicgen-large'})
+        musicgen_base_large.bind_({'model/lm/model_scale': 'large'})
+        eval(musicgen_base_large, batch_size=128)
+
+        # melody musicgen model
+        musicgen_melody = launcher.bind(solver="musicgen/musicgen_melody_32khz")
+        musicgen_melody.bind_({'autocast': False, 'fsdp.use': True})
+
+        musicgen_melody_medium = musicgen_melody.bind({'continue_from': '//pretrained/facebook/musicgen-melody'})
+        musicgen_melody_medium.bind_({'model/lm/model_scale': 'medium'})
+        eval(musicgen_melody_medium, batch_size=128, eval_melody=True)
diff --git a/audiocraft/audiocraft/losses/__init__.py b/audiocraft/audiocraft/losses/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..d55107b2c11822cab749ed3683cf19020802898a
--- /dev/null
+++ b/audiocraft/audiocraft/losses/__init__.py
@@ -0,0 +1,21 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Loss related classes and functions. In particular the loss balancer from
+EnCodec, and the usual spectral losses."""
+
+# flake8: noqa
+from .balancer import Balancer
+from .sisnr import SISNR
+from .stftloss import (
+    LogSTFTMagnitudeLoss,
+    MRSTFTLoss,
+    SpectralConvergenceLoss,
+    STFTLoss
+)
+from .specloss import (
+    MelSpectrogramL1Loss,
+    MultiScaleMelSpectrogramLoss,
+)
diff --git a/audiocraft/audiocraft/losses/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/losses/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..291f80307bdaf60691dccdb14f940574abe9b8d7
Binary files /dev/null and b/audiocraft/audiocraft/losses/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/losses/__pycache__/balancer.cpython-311.pyc b/audiocraft/audiocraft/losses/__pycache__/balancer.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..010d41c0c3a20d6b230bc6fb75ae25aa2e2e7eeb
Binary files /dev/null and b/audiocraft/audiocraft/losses/__pycache__/balancer.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/losses/__pycache__/sisnr.cpython-311.pyc b/audiocraft/audiocraft/losses/__pycache__/sisnr.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a8e3f0383c9fde685859b1a0e2948a6fa6dd4bfc
Binary files /dev/null and b/audiocraft/audiocraft/losses/__pycache__/sisnr.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/losses/__pycache__/specloss.cpython-311.pyc b/audiocraft/audiocraft/losses/__pycache__/specloss.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..246667d116ba5e49f48ee9a508f47eced417c5aa
Binary files /dev/null and b/audiocraft/audiocraft/losses/__pycache__/specloss.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/losses/__pycache__/stftloss.cpython-311.pyc b/audiocraft/audiocraft/losses/__pycache__/stftloss.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..a99f58760fea3330a18e08e0ebdef59e5a41456b
Binary files /dev/null and b/audiocraft/audiocraft/losses/__pycache__/stftloss.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/losses/balancer.py b/audiocraft/audiocraft/losses/balancer.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a0ac8adebab8cdee8f82351965195dc02800d18
--- /dev/null
+++ b/audiocraft/audiocraft/losses/balancer.py
@@ -0,0 +1,136 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import flashy
+import torch
+from torch import autograd
+
+
+class Balancer:
+    """Loss balancer.
+
+    The loss balancer combines losses together to compute gradients for the backward.
+    Given `y = f(...)`, and a number of losses `l1(y, ...)`, `l2(y, ...)`, with `...`
+    not having any dependence on `f`, the balancer can efficiently normalize the partial gradients
+    `d l1 / d y`, `d l2 / dy` before summing them in order to achieve a desired ratio between
+    the losses. For instance if `weights = {'l1': 2, 'l2': 1}`, 66% of the gradient
+    going into `f(...)` will come from `l1` on average, and 33% from `l2`. This allows for an easy
+    interpration of the weights even if the intrisic scale of `l1`, `l2` ... is unknown.
+
+    Noting `g1 = d l1 / dy`, etc., the balanced gradient `G` will be
+    (with `avg` an exponential moving average over the updates),
+
+        G = sum_i total_norm * g_i / avg(||g_i||) * w_i / sum(w_i)
+
+    If `balance_grads` is False, this is deactivated, and instead the gradient will just be the
+    standard sum of the partial gradients with the given weights.
+
+    A call to the backward method of the balancer will compute the the partial gradients,
+    combining all the losses and potentially rescaling the gradients,
+    which can help stabilize the training and reason about multiple losses with varying scales.
+    The obtained gradient with respect to `y` is then back-propagated to `f(...)`.
+
+    Expected usage:
+
+        weights = {'loss_a': 1, 'loss_b': 4}
+        balancer = Balancer(weights, ...)
+        losses: dict = {}
+        losses['loss_a'] = compute_loss_a(x, y)
+        losses['loss_b'] = compute_loss_b(x, y)
+        if model.training():
+            effective_loss = balancer.backward(losses, x)
+
+    Args:
+        weights (dict[str, float]): Weight coefficient for each loss. The balancer expect the losses keys
+            from the backward method to match the weights keys to assign weight to each of the provided loss.
+        balance_grads (bool): Whether to rescale gradients so that weights reflect the fraction of the
+            overall gradient, rather than a constant multiplier.
+        total_norm (float): Reference norm when rescaling gradients, ignored otherwise.
+        emay_decay (float): EMA decay for averaging the norms.
+        per_batch_item (bool): Whether to compute the averaged norm per batch item or not. This only holds
+            when rescaling the gradients.
+        epsilon (float): Epsilon value for numerical stability.
+        monitor (bool): If True, stores in `self.metrics` the relative ratio between the norm of the gradients
+            coming from each loss, when calling `backward()`.
+    """
+    def __init__(self, weights: tp.Dict[str, float], balance_grads: bool = True, total_norm: float = 1.,
+                 ema_decay: float = 0.999, per_batch_item: bool = True, epsilon: float = 1e-12,
+                 monitor: bool = False):
+        self.weights = weights
+        self.per_batch_item = per_batch_item
+        self.total_norm = total_norm or 1.
+        self.averager = flashy.averager(ema_decay or 1.)
+        self.epsilon = epsilon
+        self.monitor = monitor
+        self.balance_grads = balance_grads
+        self._metrics: tp.Dict[str, tp.Any] = {}
+
+    @property
+    def metrics(self):
+        return self._metrics
+
+    def backward(self, losses: tp.Dict[str, torch.Tensor], input: torch.Tensor) -> torch.Tensor:
+        """Compute the backward and return the effective train loss, e.g. the loss obtained from
+        computing the effective weights. If `balance_grads` is True, the effective weights
+        are the one that needs to be applied to each gradient to respect the desired relative
+        scale of gradients coming from each loss.
+
+        Args:
+            losses (Dict[str, torch.Tensor]): dictionary with the same keys as `self.weights`.
+            input (torch.Tensor): the input of the losses, typically the output of the model.
+                This should be the single point of dependence between the losses
+                and the model being trained.
+        """
+        norms = {}
+        grads = {}
+        for name, loss in losses.items():
+            # Compute partial derivative of the less with respect to the input.
+            grad, = autograd.grad(loss, [input], retain_graph=True)
+            if self.per_batch_item:
+                # We do not average the gradient over the batch dimension.
+                dims = tuple(range(1, grad.dim()))
+                norm = grad.norm(dim=dims, p=2).mean()
+            else:
+                norm = grad.norm(p=2)
+            norms[name] = norm
+            grads[name] = grad
+
+        count = 1
+        if self.per_batch_item:
+            count = len(grad)
+        # Average norms across workers. Theoretically we should average the
+        # squared norm, then take the sqrt, but it worked fine like that.
+        avg_norms = flashy.distrib.average_metrics(self.averager(norms), count)
+        # We approximate the total norm of the gradient as the sums of the norms.
+        # Obviously this can be very incorrect if all gradients are aligned, but it works fine.
+        total = sum(avg_norms.values())
+
+        self._metrics = {}
+        if self.monitor:
+            # Store the ratio of the total gradient represented by each loss.
+            for k, v in avg_norms.items():
+                self._metrics[f'ratio_{k}'] = v / total
+
+        total_weights = sum([self.weights[k] for k in avg_norms])
+        assert total_weights > 0.
+        desired_ratios = {k: w / total_weights for k, w in self.weights.items()}
+
+        out_grad = torch.zeros_like(input)
+        effective_loss = torch.tensor(0., device=input.device, dtype=input.dtype)
+        for name, avg_norm in avg_norms.items():
+            if self.balance_grads:
+                # g_balanced = g / avg(||g||) * total_norm * desired_ratio
+                scale = desired_ratios[name] * self.total_norm / (self.epsilon + avg_norm)
+            else:
+                # We just do regular weighted sum of the gradients.
+                scale = self.weights[name]
+            out_grad.add_(grads[name], alpha=scale)
+            effective_loss += scale * losses[name].detach()
+        # Send the computed partial derivative with respect to the output of the model to the model.
+        input.backward(out_grad)
+        return effective_loss
diff --git a/audiocraft/audiocraft/losses/sisnr.py b/audiocraft/audiocraft/losses/sisnr.py
new file mode 100644
index 0000000000000000000000000000000000000000..30f1fa1de9aca22758b6665609a1eacc0bd992ca
--- /dev/null
+++ b/audiocraft/audiocraft/losses/sisnr.py
@@ -0,0 +1,92 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+import typing as tp
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+def _unfold(a: torch.Tensor, kernel_size: int, stride: int) -> torch.Tensor:
+    """Given input of size [*OT, T], output Tensor of size [*OT, F, K]
+    with K the kernel size, by extracting frames with the given stride.
+    This will pad the input so that `F = ceil(T / K)`.
+    see https://github.com/pytorch/pytorch/issues/60466
+    """
+    *shape, length = a.shape
+    n_frames = math.ceil(length / stride)
+    tgt_length = (n_frames - 1) * stride + kernel_size
+    a = F.pad(a, (0, tgt_length - length))
+    strides = list(a.stride())
+    assert strides[-1] == 1, "data should be contiguous"
+    strides = strides[:-1] + [stride, 1]
+    return a.as_strided([*shape, n_frames, kernel_size], strides)
+
+
+def _center(x: torch.Tensor) -> torch.Tensor:
+    return x - x.mean(-1, True)
+
+
+def _norm2(x: torch.Tensor) -> torch.Tensor:
+    return x.pow(2).sum(-1, True)
+
+
+class SISNR(nn.Module):
+    """SISNR loss.
+
+    Input should be [B, C, T], output is scalar.
+
+    Args:
+        sample_rate (int): Sample rate.
+        segment (float or None): Evaluate on chunks of that many seconds. If None, evaluate on
+            entire audio only.
+        overlap (float): Overlap between chunks, i.e. 0.5 = 50 % overlap.
+        epsilon (float): Epsilon value for numerical stability.
+    """
+    def __init__(
+        self,
+        sample_rate: int = 16000,
+        segment: tp.Optional[float] = 20,
+        overlap: float = 0.5,
+        epsilon: float = torch.finfo(torch.float32).eps,
+    ):
+        super().__init__()
+        self.sample_rate = sample_rate
+        self.segment = segment
+        self.overlap = overlap
+        self.epsilon = epsilon
+
+    def forward(self, out_sig: torch.Tensor, ref_sig: torch.Tensor) -> torch.Tensor:
+        B, C, T = ref_sig.shape
+        assert ref_sig.shape == out_sig.shape
+
+        if self.segment is None:
+            frame = T
+            stride = T
+        else:
+            frame = int(self.segment * self.sample_rate)
+            stride = int(frame * (1 - self.overlap))
+
+        epsilon = self.epsilon * frame  # make epsilon prop to frame size.
+
+        gt = _unfold(ref_sig, frame, stride)
+        est = _unfold(out_sig, frame, stride)
+        if self.segment is None:
+            assert gt.shape[-1] == 1
+
+        gt = _center(gt)
+        est = _center(est)
+        dot = torch.einsum("bcft,bcft->bcf", gt, est)
+
+        proj = dot[:, :, :, None] * gt / (epsilon + _norm2(gt))
+        noise = est - proj
+
+        sisnr = 10 * (
+            torch.log10(epsilon + _norm2(proj)) - torch.log10(epsilon + _norm2(noise))
+        )
+        return -1 * sisnr[..., 0].mean()
diff --git a/audiocraft/audiocraft/losses/specloss.py b/audiocraft/audiocraft/losses/specloss.py
new file mode 100644
index 0000000000000000000000000000000000000000..11f2eb3e5c44b542a02f13db64bfb22fa0d3d212
--- /dev/null
+++ b/audiocraft/audiocraft/losses/specloss.py
@@ -0,0 +1,149 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import numpy as np
+from torchaudio.transforms import MelSpectrogram
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+from ..modules import pad_for_conv1d
+
+
+class MelSpectrogramWrapper(nn.Module):
+    """Wrapper around MelSpectrogram torchaudio transform providing proper padding
+    and additional post-processing including log scaling.
+
+    Args:
+        n_mels (int): Number of mel bins.
+        n_fft (int): Number of fft.
+        hop_length (int): Hop size.
+        win_length (int): Window length.
+        n_mels (int): Number of mel bins.
+        sample_rate (int): Sample rate.
+        f_min (float or None): Minimum frequency.
+        f_max (float or None): Maximum frequency.
+        log (bool): Whether to scale with log.
+        normalized (bool): Whether to normalize the melspectrogram.
+        floor_level (float): Floor level based on human perception (default=1e-5).
+    """
+    def __init__(self, n_fft: int = 1024, hop_length: int = 256, win_length: tp.Optional[int] = None,
+                 n_mels: int = 80, sample_rate: float = 22050, f_min: float = 0.0, f_max: tp.Optional[float] = None,
+                 log: bool = True, normalized: bool = False, floor_level: float = 1e-5):
+        super().__init__()
+        self.n_fft = n_fft
+        hop_length = int(hop_length)
+        self.hop_length = hop_length
+        self.mel_transform = MelSpectrogram(n_mels=n_mels, sample_rate=sample_rate, n_fft=n_fft, hop_length=hop_length,
+                                            win_length=win_length, f_min=f_min, f_max=f_max, normalized=normalized,
+                                            window_fn=torch.hann_window, center=False)
+        self.floor_level = floor_level
+        self.log = log
+
+    def forward(self, x):
+        p = int((self.n_fft - self.hop_length) // 2)
+        if len(x.shape) == 2:
+            x = x.unsqueeze(1)
+        x = F.pad(x, (p, p), "reflect")
+        # Make sure that all the frames are full.
+        # The combination of `pad_for_conv1d` and the above padding
+        # will make the output of size ceil(T / hop).
+        x = pad_for_conv1d(x, self.n_fft, self.hop_length)
+        self.mel_transform.to(x.device)
+        mel_spec = self.mel_transform(x)
+        B, C, freqs, frame = mel_spec.shape
+        if self.log:
+            mel_spec = torch.log10(self.floor_level + mel_spec)
+        return mel_spec.reshape(B, C * freqs, frame)
+
+
+class MelSpectrogramL1Loss(torch.nn.Module):
+    """L1 Loss on MelSpectrogram.
+
+    Args:
+        sample_rate (int): Sample rate.
+        n_fft (int): Number of fft.
+        hop_length (int): Hop size.
+        win_length (int): Window length.
+        n_mels (int): Number of mel bins.
+        f_min (float or None): Minimum frequency.
+        f_max (float or None): Maximum frequency.
+        log (bool): Whether to scale with log.
+        normalized (bool): Whether to normalize the melspectrogram.
+        floor_level (float): Floor level value based on human perception (default=1e-5).
+    """
+    def __init__(self, sample_rate: int, n_fft: int = 1024, hop_length: int = 256, win_length: int = 1024,
+                 n_mels: int = 80, f_min: float = 0.0, f_max: tp.Optional[float] = None,
+                 log: bool = True, normalized: bool = False, floor_level: float = 1e-5):
+        super().__init__()
+        self.l1 = torch.nn.L1Loss()
+        self.melspec = MelSpectrogramWrapper(n_fft=n_fft, hop_length=hop_length, win_length=win_length,
+                                             n_mels=n_mels, sample_rate=sample_rate, f_min=f_min, f_max=f_max,
+                                             log=log, normalized=normalized, floor_level=floor_level)
+
+    def forward(self, x, y):
+        self.melspec.to(x.device)
+        s_x = self.melspec(x)
+        s_y = self.melspec(y)
+        return self.l1(s_x, s_y)
+
+
+class MultiScaleMelSpectrogramLoss(nn.Module):
+    """Multi-Scale spectrogram loss (msspec).
+
+    Args:
+        sample_rate (int): Sample rate.
+        range_start (int): Power of 2 to use for the first scale.
+        range_stop (int): Power of 2 to use for the last scale.
+        n_mels (int): Number of mel bins.
+        f_min (float): Minimum frequency.
+        f_max (float or None): Maximum frequency.
+        normalized (bool): Whether to normalize the melspectrogram.
+        alphas (bool): Whether to use alphas as coefficients or not.
+        floor_level (float): Floor level value based on human perception (default=1e-5).
+    """
+    def __init__(self, sample_rate: int, range_start: int = 6, range_end: int = 11,
+                 n_mels: int = 64, f_min: float = 0.0, f_max: tp.Optional[float] = None,
+                 normalized: bool = False, alphas: bool = True, floor_level: float = 1e-5):
+        super().__init__()
+        l1s = list()
+        l2s = list()
+        self.alphas = list()
+        self.total = 0
+        self.normalized = normalized
+        for i in range(range_start, range_end):
+            l1s.append(
+                MelSpectrogramWrapper(n_fft=2 ** i, hop_length=(2 ** i) / 4, win_length=2 ** i,
+                                      n_mels=n_mels, sample_rate=sample_rate, f_min=f_min, f_max=f_max,
+                                      log=False, normalized=normalized, floor_level=floor_level))
+            l2s.append(
+                MelSpectrogramWrapper(n_fft=2 ** i, hop_length=(2 ** i) / 4, win_length=2 ** i,
+                                      n_mels=n_mels, sample_rate=sample_rate, f_min=f_min, f_max=f_max,
+                                      log=True, normalized=normalized, floor_level=floor_level))
+            if alphas:
+                self.alphas.append(np.sqrt(2 ** i - 1))
+            else:
+                self.alphas.append(1)
+            self.total += self.alphas[-1] + 1
+
+        self.l1s = nn.ModuleList(l1s)
+        self.l2s = nn.ModuleList(l2s)
+
+    def forward(self, x, y):
+        loss = 0.0
+        self.l1s.to(x.device)
+        self.l2s.to(x.device)
+        for i in range(len(self.alphas)):
+            s_x_1 = self.l1s[i](x)
+            s_y_1 = self.l1s[i](y)
+            s_x_2 = self.l2s[i](x)
+            s_y_2 = self.l2s[i](y)
+            loss += F.l1_loss(s_x_1, s_y_1) + self.alphas[i] * F.mse_loss(s_x_2, s_y_2)
+        if self.normalized:
+            loss = loss / self.total
+        return loss
diff --git a/audiocraft/audiocraft/losses/stftloss.py b/audiocraft/audiocraft/losses/stftloss.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ad4b7d3324ee5b0e6064b6f71cf8caf0fdc3be7
--- /dev/null
+++ b/audiocraft/audiocraft/losses/stftloss.py
@@ -0,0 +1,207 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+# Adapted from MIT code under the original license
+# Copyright 2019 Tomoki Hayashi
+# MIT License (https://opensource.org/licenses/MIT)
+import typing as tp
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+# TODO: Replace with torchaudio.STFT?
+def _stft(x: torch.Tensor, fft_size: int, hop_length: int, win_length: int,
+          window: tp.Optional[torch.Tensor], normalized: bool) -> torch.Tensor:
+    """Perform STFT and convert to magnitude spectrogram.
+
+    Args:
+        x: Input signal tensor (B, C, T).
+        fft_size (int): FFT size.
+        hop_length (int): Hop size.
+        win_length (int): Window length.
+        window (torch.Tensor or None): Window function type.
+        normalized (bool): Whether to normalize the STFT or not.
+
+    Returns:
+        torch.Tensor: Magnitude spectrogram (B, C, #frames, fft_size // 2 + 1).
+    """
+    B, C, T = x.shape
+    x_stft = torch.stft(
+        x.view(-1, T), fft_size, hop_length, win_length, window,
+        normalized=normalized, return_complex=True,
+    )
+    x_stft = x_stft.view(B, C, *x_stft.shape[1:])
+    real = x_stft.real
+    imag = x_stft.imag
+
+    # NOTE(kan-bayashi): clamp is needed to avoid nan or inf
+    return torch.sqrt(torch.clamp(real ** 2 + imag ** 2, min=1e-7)).transpose(2, 1)
+
+
+class SpectralConvergenceLoss(nn.Module):
+    """Spectral convergence loss.
+    """
+    def __init__(self, epsilon: float = torch.finfo(torch.float32).eps):
+        super().__init__()
+        self.epsilon = epsilon
+
+    def forward(self, x_mag: torch.Tensor, y_mag: torch.Tensor):
+        """Calculate forward propagation.
+
+        Args:
+            x_mag: Magnitude spectrogram of predicted signal (B, #frames, #freq_bins).
+            y_mag: Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins).
+        Returns:
+            torch.Tensor: Spectral convergence loss value.
+        """
+        return torch.norm(y_mag - x_mag, p="fro") / (torch.norm(y_mag, p="fro") + self.epsilon)
+
+
+class LogSTFTMagnitudeLoss(nn.Module):
+    """Log STFT magnitude loss.
+
+    Args:
+        epsilon (float): Epsilon value for numerical stability.
+    """
+    def __init__(self, epsilon: float = torch.finfo(torch.float32).eps):
+        super().__init__()
+        self.epsilon = epsilon
+
+    def forward(self, x_mag: torch.Tensor, y_mag: torch.Tensor):
+        """Calculate forward propagation.
+
+        Args:
+            x_mag (torch.Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins).
+            y_mag (torch.Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins).
+        Returns:
+            torch.Tensor: Log STFT magnitude loss value.
+        """
+        return F.l1_loss(torch.log(self.epsilon + y_mag), torch.log(self.epsilon + x_mag))
+
+
+class STFTLosses(nn.Module):
+    """STFT losses.
+
+    Args:
+        n_fft (int): Size of FFT.
+        hop_length (int): Hop length.
+        win_length (int): Window length.
+        window (str): Window function type.
+        normalized (bool): Whether to use normalized STFT or not.
+        epsilon (float): Epsilon for numerical stability.
+    """
+    def __init__(self, n_fft: int = 1024, hop_length: int = 120, win_length: int = 600,
+                 window: str = "hann_window", normalized: bool = False,
+                 epsilon: float = torch.finfo(torch.float32).eps):
+        super().__init__()
+        self.n_fft = n_fft
+        self.hop_length = hop_length
+        self.win_length = win_length
+        self.normalized = normalized
+        self.register_buffer("window", getattr(torch, window)(win_length))
+        self.spectral_convergenge_loss = SpectralConvergenceLoss(epsilon)
+        self.log_stft_magnitude_loss = LogSTFTMagnitudeLoss(epsilon)
+
+    def forward(self, x: torch.Tensor, y: torch.Tensor) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        """Calculate forward propagation.
+
+        Args:
+            x (torch.Tensor): Predicted signal (B, T).
+            y (torch.Tensor): Groundtruth signal (B, T).
+        Returns:
+            torch.Tensor: Spectral convergence loss value.
+            torch.Tensor: Log STFT magnitude loss value.
+        """
+        x_mag = _stft(x, self.n_fft, self.hop_length,
+                      self.win_length, self.window, self.normalized)  # type: ignore
+        y_mag = _stft(y, self.n_fft, self.hop_length,
+                      self.win_length, self.window, self.normalized)  # type: ignore
+        sc_loss = self.spectral_convergenge_loss(x_mag, y_mag)
+        mag_loss = self.log_stft_magnitude_loss(x_mag, y_mag)
+
+        return sc_loss, mag_loss
+
+
+class STFTLoss(nn.Module):
+    """Single Resolution STFT loss.
+
+    Args:
+        n_fft (int): Nb of FFT.
+        hop_length (int): Hop length.
+        win_length (int): Window length.
+        window (str): Window function type.
+        normalized (bool): Whether to use normalized STFT or not.
+        epsilon (float): Epsilon for numerical stability.
+        factor_sc (float): Coefficient for the spectral loss.
+        factor_mag (float): Coefficient for the magnitude loss.
+    """
+    def __init__(self, n_fft: int = 1024, hop_length: int = 120, win_length: int = 600,
+                 window: str = "hann_window", normalized: bool = False,
+                 factor_sc: float = 0.1, factor_mag: float = 0.1,
+                 epsilon: float = torch.finfo(torch.float32).eps):
+        super().__init__()
+        self.loss = STFTLosses(n_fft, hop_length, win_length, window, normalized, epsilon)
+        self.factor_sc = factor_sc
+        self.factor_mag = factor_mag
+
+    def forward(self, x: torch.Tensor, y: torch.Tensor) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        """Calculate forward propagation.
+
+        Args:
+            x (torch.Tensor): Predicted signal (B, T).
+            y (torch.Tensor): Groundtruth signal (B, T).
+        Returns:
+            torch.Tensor: Single resolution STFT loss.
+        """
+        sc_loss, mag_loss = self.loss(x, y)
+        return self.factor_sc * sc_loss + self.factor_mag * mag_loss
+
+
+class MRSTFTLoss(nn.Module):
+    """Multi resolution STFT loss.
+
+    Args:
+        n_ffts (Sequence[int]): Sequence of FFT sizes.
+        hop_lengths (Sequence[int]): Sequence of hop sizes.
+        win_lengths (Sequence[int]): Sequence of window lengths.
+        window (str): Window function type.
+        factor_sc (float): Coefficient for the spectral loss.
+        factor_mag (float): Coefficient for the magnitude loss.
+        normalized (bool): Whether to use normalized STFT or not.
+        epsilon (float): Epsilon for numerical stability.
+    """
+    def __init__(self, n_ffts: tp.Sequence[int] = [1024, 2048, 512], hop_lengths: tp.Sequence[int] = [120, 240, 50],
+                 win_lengths: tp.Sequence[int] = [600, 1200, 240], window: str = "hann_window",
+                 factor_sc: float = 0.1, factor_mag: float = 0.1,
+                 normalized: bool = False, epsilon: float = torch.finfo(torch.float32).eps):
+        super().__init__()
+        assert len(n_ffts) == len(hop_lengths) == len(win_lengths)
+        self.stft_losses = torch.nn.ModuleList()
+        for fs, ss, wl in zip(n_ffts, hop_lengths, win_lengths):
+            self.stft_losses += [STFTLosses(fs, ss, wl, window, normalized, epsilon)]
+        self.factor_sc = factor_sc
+        self.factor_mag = factor_mag
+
+    def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+        """Calculate forward propagation.
+
+        Args:
+            x (torch.Tensor): Predicted signal (B, T).
+            y (torch.Tensor): Groundtruth signal (B, T).
+        Returns:
+            torch.Tensor: Multi resolution STFT loss.
+        """
+        sc_loss = torch.Tensor([0.0])
+        mag_loss = torch.Tensor([0.0])
+        for f in self.stft_losses:
+            sc_l, mag_l = f(x, y)
+            sc_loss += sc_l
+            mag_loss += mag_l
+        sc_loss /= len(self.stft_losses)
+        mag_loss /= len(self.stft_losses)
+
+        return self.factor_sc * sc_loss + self.factor_mag * mag_loss
diff --git a/audiocraft/audiocraft/metrics/__init__.py b/audiocraft/audiocraft/metrics/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..3474bdc4f1c88b21904d2a21ba077c93a8a70c8b
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/__init__.py
@@ -0,0 +1,14 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Metrics like CLAP score, FAD, KLD, Visqol, Chroma similarity, etc.
+"""
+# flake8: noqa
+from .clap_consistency import CLAPTextConsistencyMetric, TextConsistencyMetric
+from .chroma_cosinesim import ChromaCosineSimilarityMetric
+from .fad import FrechetAudioDistanceMetric
+from .kld import KLDivergenceMetric, PasstKLDivergenceMetric
+from .rvm import RelativeVolumeMel
+from .visqol import ViSQOL
diff --git a/audiocraft/audiocraft/metrics/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..37df2fed20227e2b2b56aab3c7ac89ea25a35264
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/__pycache__/chroma_cosinesim.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/chroma_cosinesim.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..51a9ac4e76cbaf6b0702f181f280ba7241b8f1a5
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/chroma_cosinesim.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/__pycache__/clap_consistency.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/clap_consistency.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..cfaa4d19047a3cd0807f1c02c5fd688634139f9a
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/clap_consistency.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/__pycache__/fad.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/fad.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8f4a9e8b4ee3bdaea2ee0529cbd7dc98ff27047f
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/fad.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/__pycache__/kld.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/kld.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..513c56207cbf5af77c3491cee7749fcac8170af9
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/kld.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/__pycache__/rvm.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/rvm.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..c8b2757666530770f23dd12368717da0e9c97a1a
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/rvm.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/__pycache__/visqol.cpython-311.pyc b/audiocraft/audiocraft/metrics/__pycache__/visqol.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..885016eda4c3b75b47457a03bb767bf97a5d8be9
Binary files /dev/null and b/audiocraft/audiocraft/metrics/__pycache__/visqol.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/metrics/chroma_cosinesim.py b/audiocraft/audiocraft/metrics/chroma_cosinesim.py
new file mode 100644
index 0000000000000000000000000000000000000000..40c26081b803c2017fae1b6d7d086f0b0e074cef
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/chroma_cosinesim.py
@@ -0,0 +1,72 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+import torchmetrics
+
+from ..data.audio_utils import convert_audio
+from ..modules.chroma import ChromaExtractor
+
+
+class ChromaCosineSimilarityMetric(torchmetrics.Metric):
+    """Chroma cosine similarity metric.
+
+    This metric extracts a chromagram for a reference waveform and
+    a generated waveform and compares each frame using the cosine similarity
+    function. The output is the mean cosine similarity.
+
+    Args:
+        sample_rate (int): Sample rate used by the chroma extractor.
+        n_chroma (int): Number of chroma used by the chroma extractor.
+        radix2_exp (int): Exponent for the chroma extractor.
+        argmax (bool): Whether the chroma extractor uses argmax.
+        eps (float): Epsilon for cosine similarity computation.
+    """
+    def __init__(self, sample_rate: int, n_chroma: int, radix2_exp: int, argmax: bool, eps: float = 1e-8):
+        super().__init__()
+        self.chroma_sample_rate = sample_rate
+        self.n_chroma = n_chroma
+        self.eps = eps
+        self.chroma_extractor = ChromaExtractor(sample_rate=self.chroma_sample_rate, n_chroma=self.n_chroma,
+                                                radix2_exp=radix2_exp, argmax=argmax)
+        self.add_state("cosine_sum", default=torch.tensor(0.), dist_reduce_fx="sum")
+        self.add_state("weight", default=torch.tensor(0.), dist_reduce_fx="sum")
+
+    def update(self, preds: torch.Tensor, targets: torch.Tensor,
+               sizes: torch.Tensor, sample_rates: torch.Tensor) -> None:
+        """Compute cosine similarity between chromagrams and accumulate scores over the dataset."""
+        if preds.size(0) == 0:
+            return
+
+        assert preds.shape == targets.shape, (
+            f"Preds and target shapes mismatch: preds={preds.shape}, targets={targets.shape}")
+        assert preds.size(0) == sizes.size(0), (
+            f"Number of items in preds ({preds.shape}) mismatch ",
+            f"with sizes ({sizes.shape})")
+        assert preds.size(0) == sample_rates.size(0), (
+            f"Number of items in preds ({preds.shape}) mismatch ",
+            f"with sample_rates ({sample_rates.shape})")
+        assert torch.all(sample_rates == sample_rates[0].item()), "All sample rates are not the same in the batch"
+
+        device = self.weight.device
+        preds, targets = preds.to(device), targets.to(device)  # type: ignore
+        sample_rate = sample_rates[0].item()
+        preds = convert_audio(preds, from_rate=sample_rate, to_rate=self.chroma_sample_rate, to_channels=1)
+        targets = convert_audio(targets, from_rate=sample_rate, to_rate=self.chroma_sample_rate, to_channels=1)
+        gt_chroma = self.chroma_extractor(targets)
+        gen_chroma = self.chroma_extractor(preds)
+        chroma_lens = (sizes / self.chroma_extractor.winhop).ceil().int()
+        for i in range(len(gt_chroma)):
+            t = int(chroma_lens[i].item())
+            cosine_sim = torch.nn.functional.cosine_similarity(
+                gt_chroma[i, :t], gen_chroma[i, :t], dim=1, eps=self.eps)
+            self.cosine_sum += cosine_sim.sum(dim=0)  # type: ignore
+            self.weight += torch.tensor(t)  # type: ignore
+
+    def compute(self) -> float:
+        """Computes the average cosine similarty across all generated/target chromagrams pairs."""
+        assert self.weight.item() > 0, "Unable to compute with total number of comparisons <= 0"  # type: ignore
+        return (self.cosine_sum / self.weight).item()  # type: ignore
diff --git a/audiocraft/audiocraft/metrics/clap_consistency.py b/audiocraft/audiocraft/metrics/clap_consistency.py
new file mode 100644
index 0000000000000000000000000000000000000000..d2a6c61ae177533ca2fb17e25bc77d2acbbe3791
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/clap_consistency.py
@@ -0,0 +1,84 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from pathlib import Path
+import typing as tp
+
+import torch
+import torchmetrics
+from transformers import RobertaTokenizer  # type: ignore
+
+from ..data.audio_utils import convert_audio
+from ..environment import AudioCraftEnvironment
+from ..utils.utils import load_clap_state_dict
+
+try:
+    import laion_clap  # type: ignore
+except ImportError:
+    laion_clap = None
+
+
+class TextConsistencyMetric(torchmetrics.Metric):
+    """Text consistency metric measuring consistency between audio and text pairs."""
+
+    def update(self, audio: torch.Tensor, text: tp.List[str], sizes: torch.Tensor, sample_rates: torch.Tensor) -> None:
+        raise NotImplementedError("implement how to update the metric from the audio and text pairs.")
+
+    def compute(self):
+        raise NotImplementedError("implement how to compute the final metric score.")
+
+
+class CLAPTextConsistencyMetric(TextConsistencyMetric):
+    """Text consistency metric relying on Contrastive Language-Audio Pretraining (CLAP).
+
+    This metric is similar to the MuLan Cycle Consistency from MusicLM (https://arxiv.org/pdf/2301.11325.pdf)
+    or the CLAP score used in Make-An-Audio (https://arxiv.org/pdf/2301.12661v1.pdf).
+
+    As a joint audio-text embedding model, a pretrained CLAP model can be used to quantify the
+    similarity between audio-text pairs. We compute the CLAP embeddings from the text descriptions as
+    well as the generated audio based on them, and define the MCC metric as the average cosine similarity
+    between these embeddings.
+
+    Model implementation & pre-trained checkpoints: https://github.com/LAION-AI/CLAP
+    """
+    def __init__(self, model_path: tp.Union[str, Path], model_arch: str = 'HTSAT-tiny', enable_fusion: bool = False):
+        super().__init__()
+        if laion_clap is None:
+            raise ImportError("Please install CLAP to compute text consistency: 'pip install laion_clap'")
+        self.add_state("cosine_sum", default=torch.tensor(0.), dist_reduce_fx="sum")
+        self.add_state("weight", default=torch.tensor(0.), dist_reduce_fx="sum")
+        self._initialize_model(model_path, model_arch, enable_fusion)
+
+    def _initialize_model(self, model_path: tp.Union[str, Path], model_arch: str, enable_fusion: bool):
+        model_path = AudioCraftEnvironment.resolve_reference_path(model_path)
+        self.tokenize = RobertaTokenizer.from_pretrained('roberta-base')
+        self.model = laion_clap.CLAP_Module(enable_fusion=enable_fusion, amodel=model_arch)
+        self.model_sample_rate = 48_000
+        load_clap_state_dict(self.model, model_path)
+        self.model.eval()
+
+    def _tokenizer(self, texts: tp.Union[str, tp.List[str]]) -> dict:
+        # we use the default params from CLAP module here as well
+        return self.tokenize(texts, padding="max_length", truncation=True, max_length=77, return_tensors="pt")
+
+    def update(self, audio: torch.Tensor, text: tp.List[str], sizes: torch.Tensor, sample_rates: torch.Tensor) -> None:
+        """Compute cosine similarity between audio and text pairs and accumulate scores over the dataset."""
+        assert audio.size(0) == len(text), "Number of audio and text samples should match"
+        assert torch.all(sample_rates == sample_rates[0].item()), "All items in batch should have the same sample rate"
+        sample_rate = int(sample_rates[0].item())
+        # convert audio batch to 48kHz monophonic audio with no channel dimension: [B, C, T] -> [B, T]
+        audio = convert_audio(audio, from_rate=sample_rate, to_rate=self.model_sample_rate, to_channels=1).mean(dim=1)
+        audio_embeddings = self.model.get_audio_embedding_from_data(audio, use_tensor=True)
+        text_embeddings = self.model.get_text_embedding(text, tokenizer=self._tokenizer, use_tensor=True)
+        # cosine similarity between the text and the audio embedding
+        cosine_sim = torch.nn.functional.cosine_similarity(audio_embeddings, text_embeddings, dim=1, eps=1e-8)
+        self.cosine_sum += cosine_sim.sum(dim=0)
+        self.weight += torch.tensor(cosine_sim.size(0))
+
+    def compute(self):
+        """Computes the average cosine similarty across all audio/text pairs."""
+        assert self.weight.item() > 0, "Unable to compute with total number of comparisons <= 0"  # type: ignore
+        return (self.cosine_sum / self.weight).item()  # type: ignore
diff --git a/audiocraft/audiocraft/metrics/fad.py b/audiocraft/audiocraft/metrics/fad.py
new file mode 100644
index 0000000000000000000000000000000000000000..de66138dbb14fd4246bbfe590bddfd5beaf1ed8c
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/fad.py
@@ -0,0 +1,329 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+from pathlib import Path
+import os
+import subprocess
+import tempfile
+import typing as tp
+
+from audiocraft.data.audio import audio_write
+from audiocraft.data.audio_utils import convert_audio
+import flashy
+import torch
+import torchmetrics
+
+from ..environment import AudioCraftEnvironment
+
+
+logger = logging.getLogger(__name__)
+
+VGGISH_SAMPLE_RATE = 16_000
+VGGISH_CHANNELS = 1
+
+
+class FrechetAudioDistanceMetric(torchmetrics.Metric):
+    """Fréchet Audio Distance computation based on official TensorFlow implementation from Google Research.
+
+    From: D.C. Dowson & B.V. Landau The Fréchet distance between
+    multivariate normal distributions
+    https://doi.org/10.1016/0047-259X(82)90077-X
+    The Fréchet distance between two multivariate gaussians,
+    `X ~ N(mu_x, sigma_x)` and `Y ~ N(mu_y, sigma_y)`, is `d^2`.
+    d^2 = (mu_x - mu_y)^2 + Tr(sigma_x + sigma_y - 2 * sqrt(sigma_x*sigma_y))
+        = (mu_x - mu_y)^2 + Tr(sigma_x) + Tr(sigma_y)
+                        - 2 * Tr(sqrt(sigma_x*sigma_y)))
+
+    To use this FAD computation metric, you need to have the proper Frechet Audio Distance tool setup
+    from: https://github.com/google-research/google-research/tree/master/frechet_audio_distance
+    We provide the below instructions as reference but we do not guarantee for further support
+    in frechet_audio_distance installation. This was tested with python 3.10, cuda 11.8, tensorflow 2.12.0.
+
+        We recommend installing the frechet_audio_distance library in a dedicated env (e.g. conda).
+
+        1. Get the code and models following the repository instructions. We used the steps below:
+                git clone git@github.com:google-research/google-research.git
+                git clone git@github.com:tensorflow/models.git
+                mkdir google-research/tensorflow_models
+                touch google-research/tensorflow_models/__init__.py
+                cp -r models/research/audioset google-research/tensorflow_models/
+                touch google-research/tensorflow_models/audioset/__init__.py
+                echo "from .vggish import mel_features, vggish_params, vggish_slim" > \
+                    google-research/tensorflow_models/audioset/__init__.py
+                # we can now remove the tensorflow models repository
+                # rm -r models
+                cd google-research
+           Follow the instructions to download the vggish checkpoint. AudioCraft base configuration
+           assumes it is placed in the AudioCraft reference dir.
+
+           Note that we operate the following changes for the code to work with TensorFlow 2.X and python 3:
+           - Update xrange for range in:
+             https://github.com/google-research/google-research/blob/master/frechet_audio_distance/audioset_model.py
+           - Update `tf_record = tf.python_io.tf_record_iterator(filename).next()` to
+             `tf_record = tf.python_io.tf_record_iterator(filename).__next__()` in
+              https://github.com/google-research/google-research/blob/master/frechet_audio_distance/fad_utils.py
+           - Update `import vggish_params as params` to `from . import vggish_params as params` in:
+             https://github.com/tensorflow/models/blob/master/research/audioset/vggish/vggish_slim.py
+           - Add flag to provide a given batch size for running the AudioSet model in:
+             https://github.com/google-research/google-research/blob/master/frechet_audio_distance/create_embeddings_main.py
+             ```
+             flags.DEFINE_integer('batch_size', 64,
+                                  'Number of samples in the batch for AudioSet model.')
+             ```
+             Ensure you pass the flag to the create_embeddings_beam.create_pipeline function, adding:
+             `batch_size=FLAGS.batch_size` to the provided parameters.
+
+        2. Follow instructions for the library installation and a valid TensorFlow installation
+           ```
+           # e.g. instructions from: https://www.tensorflow.org/install/pip
+           conda install -c conda-forge cudatoolkit=11.8.0
+           python3 -m pip install nvidia-cudnn-cu11==8.6.0.163 tensorflow==2.12.*
+           mkdir -p $CONDA_PREFIX/etc/conda/activate.d
+           echo 'CUDNN_PATH=$(dirname $(python -c "import nvidia.cudnn;print(nvidia.cudnn.__file__)"))' \
+             >> $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+           echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CONDA_PREFIX/lib/:$CUDNN_PATH/lib' \
+             >> $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+           source $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+           # Verify install: on a machine with GPU device
+           python3 -c "import tensorflow as tf; print(tf.config.list_physical_devices('GPU'))"
+           ```
+
+           Now install frechet_audio_distance required dependencies:
+           ```
+           # We assume we already have TensorFlow installed from the above steps
+           pip install apache-beam numpy scipy tf_slim
+           ```
+
+           Finally, follow remaining library instructions to ensure you have a working frechet_audio_distance setup
+           (you may want to specify --model_ckpt flag pointing to the model's path).
+
+        3. AudioCraft's FrechetAudioDistanceMetric requires 2 environment variables pointing to the python executable
+           and Tensorflow library path from the above installation steps:
+            export TF_PYTHON_EXE="<PATH_TO_THE_ENV_PYTHON_BINARY>"
+            export TF_LIBRARY_PATH="<PATH_TO_THE_ENV_CUDNN_LIBRARY>"
+
+            e.g. assuming we have installed everything in a dedicated conda env
+            with python 3.10 that is currently active:
+            export TF_PYTHON_EXE="$CONDA_PREFIX/bin/python"
+            export TF_LIBRARY_PATH="$CONDA_PREFIX/lib/python3.10/site-packages/nvidia/cudnn/lib"
+
+            Finally you may want to export the following variable:
+            export TF_FORCE_GPU_ALLOW_GROWTH=true
+            See: https://www.tensorflow.org/guide/gpu#limiting_gpu_memory_growth
+
+            You can save those environment variables in your training conda env, when currently active:
+            `$CONDA_PREFIX/etc/conda/activate.d/env_vars.sh`
+            e.g. assuming the env with TensorFlow and frechet_audio_distance install is named ac_eval,
+            and the training conda env is named audiocraft:
+            ```
+            # activate training env
+            conda activate audiocraft
+            # get path to all envs
+            CONDA_ENV_DIR=$(dirname $CONDA_PREFIX)
+            # export pointers to evaluation env for using TensorFlow in FrechetAudioDistanceMetric
+            touch $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+            echo 'export TF_PYTHON_EXE="$CONDA_ENV_DIR/ac_eval/bin/python"' >> \
+                $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+            echo 'export TF_LIBRARY_PATH="$CONDA_ENV_DIR/ac_eval/lib/python3.10/site-packages/nvidia/cudnn/lib"' >> \
+                $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+            # optionally:
+            echo 'export TF_FORCE_GPU_ALLOW_GROWTH=true' >> $CONDA_PREFIX/etc/conda/activate.d/env_vars.sh
+            # you may need to reactivate the audiocraft env for this to take effect
+            ```
+
+    Args:
+        bin (Path or str): Path to installed frechet audio distance code.
+        model_path (Path or str): Path to Tensorflow checkpoint for the model
+            used to compute statistics over the embedding beams.
+        format (str): Audio format used to save files.
+        log_folder (Path or str, optional): Path where to write process logs.
+    """
+    def __init__(self, bin: tp.Union[Path, str], model_path: tp.Union[Path, str],
+                 format: str = "wav", batch_size: tp.Optional[int] = None,
+                 log_folder: tp.Optional[tp.Union[Path, str]] = None):
+        super().__init__()
+        self.model_sample_rate = VGGISH_SAMPLE_RATE
+        self.model_channels = VGGISH_CHANNELS
+        self.model_path = AudioCraftEnvironment.resolve_reference_path(model_path)
+        assert Path(self.model_path).exists(), f"Could not find provided model checkpoint path at: {self.model_path}"
+        self.format = format
+        self.batch_size = batch_size
+        self.bin = bin
+        self.tf_env = {"PYTHONPATH": str(self.bin)}
+        self.python_path = os.environ.get('TF_PYTHON_EXE') or 'python'
+        logger.info("Python exe for TF is  %s", self.python_path)
+        if 'TF_LIBRARY_PATH' in os.environ:
+            self.tf_env['LD_LIBRARY_PATH'] = os.environ['TF_LIBRARY_PATH']
+        if 'TF_FORCE_GPU_ALLOW_GROWTH' in os.environ:
+            self.tf_env['TF_FORCE_GPU_ALLOW_GROWTH'] = os.environ['TF_FORCE_GPU_ALLOW_GROWTH']
+        logger.info("Env for TF is %r", self.tf_env)
+        self.reset(log_folder)
+        self.add_state("total_files", default=torch.tensor(0.), dist_reduce_fx="sum")
+
+    def reset(self, log_folder: tp.Optional[tp.Union[Path, str]] = None):
+        """Reset torchmetrics.Metrics state."""
+        log_folder = Path(log_folder or tempfile.mkdtemp())
+        self.tmp_dir = log_folder / 'fad'
+        self.tmp_dir.mkdir(exist_ok=True)
+        self.samples_tests_dir = self.tmp_dir / 'tests'
+        self.samples_tests_dir.mkdir(exist_ok=True)
+        self.samples_background_dir = self.tmp_dir / 'background'
+        self.samples_background_dir.mkdir(exist_ok=True)
+        self.manifest_tests = self.tmp_dir / 'files_tests.cvs'
+        self.manifest_background = self.tmp_dir / 'files_background.cvs'
+        self.stats_tests_dir = self.tmp_dir / 'stats_tests'
+        self.stats_background_dir = self.tmp_dir / 'stats_background'
+        self.counter = 0
+
+    def update(self, preds: torch.Tensor, targets: torch.Tensor,
+               sizes: torch.Tensor, sample_rates: torch.Tensor,
+               stems: tp.Optional[tp.List[str]] = None):
+        """Update torchmetrics.Metrics by saving the audio and updating the manifest file."""
+        assert preds.shape == targets.shape, f"preds={preds.shape} != targets={targets.shape}"
+        num_samples = preds.shape[0]
+        assert num_samples == sizes.size(0) and num_samples == sample_rates.size(0)
+        assert stems is None or num_samples == len(set(stems))
+        for i in range(num_samples):
+            self.total_files += 1  # type: ignore
+            self.counter += 1
+            wav_len = int(sizes[i].item())
+            sample_rate = int(sample_rates[i].item())
+            pred_wav = preds[i]
+            target_wav = targets[i]
+            pred_wav = pred_wav[..., :wav_len]
+            target_wav = target_wav[..., :wav_len]
+            stem_name = stems[i] if stems is not None else f'sample_{self.counter}_{flashy.distrib.rank()}'
+            # dump audio files
+            try:
+                pred_wav = convert_audio(
+                    pred_wav.unsqueeze(0), from_rate=sample_rate,
+                    to_rate=self.model_sample_rate, to_channels=1).squeeze(0)
+                audio_write(
+                    self.samples_tests_dir / stem_name, pred_wav, sample_rate=self.model_sample_rate,
+                    format=self.format, strategy="peak")
+            except Exception as e:
+                logger.error(f"Exception occured when saving tests files for FAD computation: {repr(e)} - {e}")
+            try:
+                # for the ground truth audio, we enforce the 'peak' strategy to avoid modifying
+                # the original audio when writing it
+                target_wav = convert_audio(
+                    target_wav.unsqueeze(0), from_rate=sample_rate,
+                    to_rate=self.model_sample_rate, to_channels=1).squeeze(0)
+                audio_write(
+                    self.samples_background_dir / stem_name, target_wav, sample_rate=self.model_sample_rate,
+                    format=self.format, strategy="peak")
+            except Exception as e:
+                logger.error(f"Exception occured when saving background files for FAD computation: {repr(e)} - {e}")
+
+    def _get_samples_name(self, is_background: bool):
+        return 'background' if is_background else 'tests'
+
+    def _create_embedding_beams(self, is_background: bool, gpu_index: tp.Optional[int] = None):
+        if is_background:
+            input_samples_dir = self.samples_background_dir
+            input_filename = self.manifest_background
+            stats_name = self.stats_background_dir
+        else:
+            input_samples_dir = self.samples_tests_dir
+            input_filename = self.manifest_tests
+            stats_name = self.stats_tests_dir
+        beams_name = self._get_samples_name(is_background)
+        log_file = self.tmp_dir / f'fad_logs_create_beams_{beams_name}.log'
+
+        logger.info(f"Scanning samples folder to fetch list of files: {input_samples_dir}")
+        with open(input_filename, "w") as fout:
+            for path in Path(input_samples_dir).glob(f"*.{self.format}"):
+                fout.write(f"{str(path)}\n")
+
+        cmd = [
+            self.python_path, "-m",
+            "frechet_audio_distance.create_embeddings_main",
+            "--model_ckpt", f"{self.model_path}",
+            "--input_files", f"{str(input_filename)}",
+            "--stats", f"{str(stats_name)}",
+        ]
+        if self.batch_size is not None:
+            cmd += ["--batch_size", str(self.batch_size)]
+        logger.info(f"Launching frechet_audio_distance embeddings main method: {' '.join(cmd)} on {beams_name}")
+        env = os.environ
+        if gpu_index is not None:
+            env["CUDA_VISIBLE_DEVICES"] = str(gpu_index)
+        process = subprocess.Popen(
+            cmd, stdout=open(log_file, "w"), env={**env, **self.tf_env}, stderr=subprocess.STDOUT)
+        return process, log_file
+
+    def _compute_fad_score(self, gpu_index: tp.Optional[int] = None):
+        cmd = [
+            self.python_path, "-m", "frechet_audio_distance.compute_fad",
+            "--test_stats", f"{str(self.stats_tests_dir)}",
+            "--background_stats", f"{str(self.stats_background_dir)}",
+        ]
+        logger.info(f"Launching frechet_audio_distance compute fad method: {' '.join(cmd)}")
+        env = os.environ
+        if gpu_index is not None:
+            env["CUDA_VISIBLE_DEVICES"] = str(gpu_index)
+        result = subprocess.run(cmd, env={**env, **self.tf_env}, capture_output=True)
+        if result.returncode:
+            logger.error(
+                "Error with FAD computation from stats: \n %s \n %s",
+                result.stdout.decode(), result.stderr.decode()
+            )
+            raise RuntimeError("Error while executing FAD computation from stats")
+        try:
+            # result is "FAD: (d+).(d+)" hence we remove the prefix with (d+) being one digit or more
+            fad_score = float(result.stdout[4:])
+            return fad_score
+        except Exception as e:
+            raise RuntimeError(f"Error parsing FAD score from command stdout: {e}")
+
+    def _log_process_result(self, returncode: int, log_file: tp.Union[Path, str], is_background: bool) -> None:
+        beams_name = self._get_samples_name(is_background)
+        if returncode:
+            with open(log_file, "r") as f:
+                error_log = f.read()
+                logger.error(error_log)
+            os._exit(1)
+        else:
+            logger.info(f"Successfully computed embedding beams on {beams_name} samples.")
+
+    def _parallel_create_embedding_beams(self, num_of_gpus: int):
+        assert num_of_gpus > 0
+        logger.info("Creating embeddings beams in a parallel manner on different GPUs")
+        tests_beams_process, tests_beams_log_file = self._create_embedding_beams(is_background=False, gpu_index=0)
+        bg_beams_process, bg_beams_log_file = self._create_embedding_beams(is_background=True, gpu_index=1)
+        tests_beams_code = tests_beams_process.wait()
+        bg_beams_code = bg_beams_process.wait()
+        self._log_process_result(tests_beams_code, tests_beams_log_file, is_background=False)
+        self._log_process_result(bg_beams_code, bg_beams_log_file, is_background=True)
+
+    def _sequential_create_embedding_beams(self):
+        logger.info("Creating embeddings beams in a sequential manner")
+        tests_beams_process, tests_beams_log_file = self._create_embedding_beams(is_background=False)
+        tests_beams_code = tests_beams_process.wait()
+        self._log_process_result(tests_beams_code, tests_beams_log_file, is_background=False)
+        bg_beams_process, bg_beams_log_file = self._create_embedding_beams(is_background=True)
+        bg_beams_code = bg_beams_process.wait()
+        self._log_process_result(bg_beams_code, bg_beams_log_file, is_background=True)
+
+    @flashy.distrib.rank_zero_only
+    def _local_compute_frechet_audio_distance(self):
+        """Compute Frechet Audio Distance score calling TensorFlow API."""
+        num_of_gpus = torch.cuda.device_count() if torch.cuda.is_available() else 0
+        if num_of_gpus > 1:
+            self._parallel_create_embedding_beams(num_of_gpus)
+        else:
+            self._sequential_create_embedding_beams()
+        fad_score = self._compute_fad_score(gpu_index=0)
+        return fad_score
+
+    def compute(self) -> float:
+        """Compute metrics."""
+        assert self.total_files.item() > 0, "No files dumped for FAD computation!"  # type: ignore
+        fad_score = self._local_compute_frechet_audio_distance()
+        logger.warning(f"FAD score = {fad_score}")
+        fad_score = flashy.distrib.broadcast_object(fad_score, src=0)
+        return fad_score
diff --git a/audiocraft/audiocraft/metrics/kld.py b/audiocraft/audiocraft/metrics/kld.py
new file mode 100644
index 0000000000000000000000000000000000000000..ebbbcda09b0419be4d51ae6698292ff7221e47e6
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/kld.py
@@ -0,0 +1,220 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import contextlib
+from functools import partial
+import logging
+import os
+import typing as tp
+
+import torch
+import torchmetrics
+
+from ..data.audio_utils import convert_audio
+
+
+logger = logging.getLogger(__name__)
+
+
+class _patch_passt_stft:
+    """Decorator to patch torch.stft in PaSST."""
+    def __init__(self):
+        self.old_stft = torch.stft
+
+    def __enter__(self):
+        # return_complex is a mandatory parameter in latest torch versions
+        # torch is throwing RuntimeErrors when not set
+        torch.stft = partial(torch.stft, return_complex=False)
+
+    def __exit__(self, *exc):
+        torch.stft = self.old_stft
+
+
+def kl_divergence(pred_probs: torch.Tensor, target_probs: torch.Tensor, epsilon: float = 1e-6) -> torch.Tensor:
+    """Computes the elementwise KL-Divergence loss between probability distributions
+    from generated samples and target samples.
+
+    Args:
+        pred_probs (torch.Tensor): Probabilities for each label obtained
+            from a classifier on generated audio. Expected shape is [B, num_classes].
+        target_probs (torch.Tensor): Probabilities for each label obtained
+            from a classifier on target audio. Expected shape is [B, num_classes].
+        epsilon (float): Epsilon value.
+    Returns:
+        kld (torch.Tensor): KLD loss between each generated sample and target pair.
+    """
+    kl_div = torch.nn.functional.kl_div((pred_probs + epsilon).log(), target_probs, reduction="none")
+    return kl_div.sum(-1)
+
+
+class KLDivergenceMetric(torchmetrics.Metric):
+    """Base implementation for KL Divergence metric.
+
+    The KL divergence is measured between probability distributions
+    of class predictions returned by a pre-trained audio classification model.
+    When the KL-divergence is low, the generated audio is expected to
+    have similar acoustic characteristics as the reference audio,
+    according to the classifier.
+    """
+    def __init__(self):
+        super().__init__()
+        self.add_state("kld_pq_sum", default=torch.tensor(0.), dist_reduce_fx="sum")
+        self.add_state("kld_qp_sum", default=torch.tensor(0.), dist_reduce_fx="sum")
+        self.add_state("kld_all_sum", default=torch.tensor(0.), dist_reduce_fx="sum")
+        self.add_state("weight", default=torch.tensor(0), dist_reduce_fx="sum")
+
+    def _get_label_distribution(self, x: torch.Tensor, sizes: torch.Tensor,
+                                sample_rates: torch.Tensor) -> tp.Optional[torch.Tensor]:
+        """Get model output given provided input tensor.
+
+        Args:
+            x (torch.Tensor): Input audio tensor of shape [B, C, T].
+            sizes (torch.Tensor): Actual audio sample length, of shape [B].
+            sample_rates (torch.Tensor): Actual audio sample rate, of shape [B].
+        Returns:
+            probs (torch.Tensor): Probabilities over labels, of shape [B, num_classes].
+        """
+        raise NotImplementedError("implement method to extract label distributions from the model.")
+
+    def update(self, preds: torch.Tensor, targets: torch.Tensor,
+               sizes: torch.Tensor, sample_rates: torch.Tensor) -> None:
+        """Calculates running KL-Divergence loss between batches of audio
+        preds (generated) and target (ground-truth)
+        Args:
+            preds (torch.Tensor): Audio samples to evaluate, of shape [B, C, T].
+            targets (torch.Tensor): Target samples to compare against, of shape [B, C, T].
+            sizes (torch.Tensor): Actual audio sample length, of shape [B].
+            sample_rates (torch.Tensor): Actual audio sample rate, of shape [B].
+        """
+        assert preds.shape == targets.shape
+        assert preds.size(0) > 0, "Cannot update the loss with empty tensors"
+        preds_probs = self._get_label_distribution(preds, sizes, sample_rates)
+        targets_probs = self._get_label_distribution(targets, sizes, sample_rates)
+        if preds_probs is not None and targets_probs is not None:
+            assert preds_probs.shape == targets_probs.shape
+            kld_scores = kl_divergence(preds_probs, targets_probs)
+            assert not torch.isnan(kld_scores).any(), "kld_scores contains NaN value(s)!"
+            self.kld_pq_sum += torch.sum(kld_scores)
+            kld_qp_scores = kl_divergence(targets_probs, preds_probs)
+            self.kld_qp_sum += torch.sum(kld_qp_scores)
+            self.weight += torch.tensor(kld_scores.size(0))
+
+    def compute(self) -> dict:
+        """Computes KL-Divergence across all evaluated pred/target pairs."""
+        weight: float = float(self.weight.item())  # type: ignore
+        assert weight > 0, "Unable to compute with total number of comparisons <= 0"
+        logger.info(f"Computing KL divergence on a total of {weight} samples")
+        kld_pq = self.kld_pq_sum.item() / weight  # type: ignore
+        kld_qp = self.kld_qp_sum.item() / weight  # type: ignore
+        kld_both = kld_pq + kld_qp
+        return {'kld': kld_pq, 'kld_pq': kld_pq, 'kld_qp': kld_qp, 'kld_both': kld_both}
+
+
+class PasstKLDivergenceMetric(KLDivergenceMetric):
+    """KL-Divergence metric based on pre-trained PASST classifier on AudioSet.
+
+    From: PaSST: Efficient Training of Audio Transformers with Patchout
+    Paper: https://arxiv.org/abs/2110.05069
+    Implementation: https://github.com/kkoutini/PaSST
+
+    Follow instructions from the github repo:
+    ```
+    pip install 'git+https://github.com/kkoutini/passt_hear21@0.0.19#egg=hear21passt'
+    ```
+
+    Args:
+        pretrained_length (float, optional): Audio duration used for the pretrained model.
+    """
+    def __init__(self, pretrained_length: tp.Optional[float] = None):
+        super().__init__()
+        self._initialize_model(pretrained_length)
+
+    def _initialize_model(self, pretrained_length: tp.Optional[float] = None):
+        """Initialize underlying PaSST audio classifier."""
+        model, sr, max_frames, min_frames = self._load_base_model(pretrained_length)
+        self.min_input_frames = min_frames
+        self.max_input_frames = max_frames
+        self.model_sample_rate = sr
+        self.model = model
+        self.model.eval()
+        self.model.to(self.device)
+
+    def _load_base_model(self, pretrained_length: tp.Optional[float]):
+        """Load pretrained model from PaSST."""
+        try:
+            if pretrained_length == 30:
+                from hear21passt.base30sec import get_basic_model  # type: ignore
+                max_duration = 30
+            elif pretrained_length == 20:
+                from hear21passt.base20sec import get_basic_model  # type: ignore
+                max_duration = 20
+            else:
+                from hear21passt.base import get_basic_model  # type: ignore
+                # Original PASST was trained on AudioSet with 10s-long audio samples
+                max_duration = 10
+            min_duration = 0.15
+            min_duration = 0.15
+        except ModuleNotFoundError:
+            raise ModuleNotFoundError(
+                "Please install hear21passt to compute KL divergence: ",
+                "pip install 'git+https://github.com/kkoutini/passt_hear21@0.0.19#egg=hear21passt'"
+            )
+        model_sample_rate = 32_000
+        max_input_frames = int(max_duration * model_sample_rate)
+        min_input_frames = int(min_duration * model_sample_rate)
+        with open(os.devnull, 'w') as f, contextlib.redirect_stdout(f):
+            model = get_basic_model(mode='logits')
+        return model, model_sample_rate, max_input_frames, min_input_frames
+
+    def _process_audio(self, wav: torch.Tensor, sample_rate: int, wav_len: int) -> tp.List[torch.Tensor]:
+        """Process audio to feed to the pretrained model."""
+        wav = wav.unsqueeze(0)
+        wav = wav[..., :wav_len]
+        wav = convert_audio(wav, from_rate=sample_rate, to_rate=self.model_sample_rate, to_channels=1)
+        wav = wav.squeeze(0)
+        # we don't pad but return a list of audio segments as this otherwise affects the KLD computation
+        segments = torch.split(wav, self.max_input_frames, dim=-1)
+        valid_segments = []
+        for s in segments:
+            # ignoring too small segments that are breaking the model inference
+            if s.size(-1) > self.min_input_frames:
+                valid_segments.append(s)
+        return [s[None] for s in valid_segments]
+
+    def _get_model_preds(self, wav: torch.Tensor) -> torch.Tensor:
+        """Run the pretrained model and get the predictions."""
+        assert wav.dim() == 3, f"Unexpected number of dims for preprocessed wav: {wav.shape}"
+        wav = wav.mean(dim=1)
+        # PaSST is printing a lot of garbage that we are not interested in
+        with open(os.devnull, "w") as f, contextlib.redirect_stdout(f):
+            with torch.no_grad(), _patch_passt_stft():
+                logits = self.model(wav.to(self.device))
+                probs = torch.softmax(logits, dim=-1)
+                return probs
+
+    def _get_label_distribution(self, x: torch.Tensor, sizes: torch.Tensor,
+                                sample_rates: torch.Tensor) -> tp.Optional[torch.Tensor]:
+        """Get model output given provided input tensor.
+
+        Args:
+            x (torch.Tensor): Input audio tensor of shape [B, C, T].
+            sizes (torch.Tensor): Actual audio sample length, of shape [B].
+            sample_rates (torch.Tensor): Actual audio sample rate, of shape [B].
+        Returns:
+            probs (torch.Tensor, optional): Probabilities over labels, of shape [B, num_classes].
+        """
+        all_probs: tp.List[torch.Tensor] = []
+        for i, wav in enumerate(x):
+            sample_rate = int(sample_rates[i].item())
+            wav_len = int(sizes[i].item())
+            wav_segments = self._process_audio(wav, sample_rate, wav_len)
+            for segment in wav_segments:
+                probs = self._get_model_preds(segment).mean(dim=0)
+                all_probs.append(probs)
+        if len(all_probs) > 0:
+            return torch.stack(all_probs, dim=0)
+        else:
+            return None
diff --git a/audiocraft/audiocraft/metrics/rvm.py b/audiocraft/audiocraft/metrics/rvm.py
new file mode 100644
index 0000000000000000000000000000000000000000..2047b6c8d5b1d58a67090b947e7e2666c3104eca
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/rvm.py
@@ -0,0 +1,110 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+import torch
+from torch import nn
+import torchaudio
+
+
+def db_to_scale(volume: tp.Union[float, torch.Tensor]):
+    return 10 ** (volume / 20)
+
+
+def scale_to_db(scale: torch.Tensor, min_volume: float = -120):
+    min_scale = db_to_scale(min_volume)
+    return 20 * torch.log10(scale.clamp(min=min_scale))
+
+
+class RelativeVolumeMel(nn.Module):
+    """Relative volume melspectrogram measure.
+
+    Computes a measure of distance over two mel spectrogram that is interpretable in terms
+    of decibels. Given `x_ref` and `x_est` two waveforms of shape `[*, T]`, it will
+    first renormalize both by the ground truth of `x_ref`.
+
+    ..Warning:: This class returns the volume of the distortion at the spectrogram level,
+        e.g. low negative values reflects lower distortion levels. For a SNR (like reported
+        in the MultiBandDiffusion paper), just take `-rvm`.
+
+    Then it computes the mel spectrogram `z_ref` and `z_est` and compute volume of the difference
+    relative to the volume of `z_ref` for each time-frequency bin. It further adds some limits, e.g.
+    clamping the values between -25 and 25 dB (controlled by `min_relative_volume` and `max_relative_volume`)
+    with the goal of avoiding the loss being dominated by parts where the reference is almost silent.
+    Indeed, volumes in dB can take unbounded values both towards -oo and +oo, which can make the final
+    average metric harder to interpret. Besides, anything below -30 dB of attenuation would sound extremely
+    good (for a neural network output, although sound engineers typically aim for much lower attenuations).
+    Similarly, anything above +30 dB would just be completely missing the target, and there is no point
+    in measuring by exactly how much it missed it. -25, 25 is a more conservative range, but also more
+    in line with what neural nets currently can achieve.
+
+    For instance, a Relative Volume Mel (RVM) score of -10 dB means that on average, the delta between
+    the target and reference mel-spec is 10 dB lower than the reference mel-spec value.
+
+    The metric can be aggregated over a given frequency band in order have different insights for
+    different region of the spectrum. `num_aggregated_bands` controls the number of bands.
+
+    ..Warning:: While this function is optimized for interpretability, nothing was done to ensure it
+        is numerically stable when computing its gradient. We thus advise against using it as a training loss.
+
+    Args:
+        sample_rate (int): Sample rate of the input audio.
+        n_mels (int): Number of mel bands to use.
+        n_fft (int): Number of frequency bins for the STFT.
+        hop_length (int): Hop length of the STFT and the mel-spectrogram.
+        min_relative_volume (float): The error `z_ref - z_est` volume is given relative to
+            the volume of `z_ref`. If error is smaller than -25 dB of `z_ref`, then it is clamped.
+        max_relative_volume (float): Same as `min_relative_volume` but clamping if the error is larger than that.
+        max_initial_gain (float): When rescaling the audio at the very beginning, we will limit the gain
+            to that amount, to avoid rescaling near silence. Given in dB.
+        min_activity_volume (float): When computing the reference level from `z_ref`, will clamp low volume
+            bins to that amount. This is effectively our "zero" level for the reference mel-spectrogram,
+            and anything below that will be considered equally.
+        num_aggregated_bands (int): Number of bands to keep when computing the average RVM value.
+            For instance, a value of 3 would give 3 scores, roughly for low, mid and high freqs.
+    """
+    def __init__(self, sample_rate: int = 24000, n_mels: int = 80, n_fft: int = 512,
+                 hop_length: int = 128, min_relative_volume: float = -25,
+                 max_relative_volume: float = 25, max_initial_gain: float = 25,
+                 min_activity_volume: float = -25,
+                 num_aggregated_bands: int = 4) -> None:
+        super().__init__()
+        self.melspec = torchaudio.transforms.MelSpectrogram(
+            n_mels=n_mels, n_fft=n_fft, hop_length=hop_length,
+            normalized=True, sample_rate=sample_rate, power=2)
+        self.min_relative_volume = min_relative_volume
+        self.max_relative_volume = max_relative_volume
+        self.max_initial_gain = max_initial_gain
+        self.min_activity_volume = min_activity_volume
+        self.num_aggregated_bands = num_aggregated_bands
+
+    def forward(self, estimate: torch.Tensor, ground_truth: torch.Tensor) -> tp.Dict[str, torch.Tensor]:
+        """Compute RVM metric between estimate and reference samples.
+
+        Args:
+            estimate (torch.Tensor): Estimate sample.
+            ground_truth (torch.Tensor): Reference sample.
+
+        Returns:
+            dict[str, torch.Tensor]: Metrics with keys `rvm` for the overall average, and `rvm_{k}`
+            for the RVM over the k-th band (k=0..num_aggregated_bands - 1).
+        """
+        min_scale = db_to_scale(-self.max_initial_gain)
+        std = ground_truth.pow(2).mean().sqrt().clamp(min=min_scale)
+        z_gt = self.melspec(ground_truth / std).sqrt()
+        z_est = self.melspec(estimate / std).sqrt()
+
+        delta = z_gt - z_est
+        ref_db = scale_to_db(z_gt, self.min_activity_volume)
+        delta_db = scale_to_db(delta.abs(), min_volume=-120)
+        relative_db = (delta_db - ref_db).clamp(self.min_relative_volume, self.max_relative_volume)
+        dims = list(range(relative_db.dim()))
+        dims.remove(dims[-2])
+        losses_per_band = relative_db.mean(dim=dims)
+        aggregated = [chunk.mean() for chunk in losses_per_band.chunk(self.num_aggregated_bands, dim=0)]
+        metrics = {f'rvm_{index}': value for index, value in enumerate(aggregated)}
+        metrics['rvm'] = losses_per_band.mean()
+        return metrics
diff --git a/audiocraft/audiocraft/metrics/visqol.py b/audiocraft/audiocraft/metrics/visqol.py
new file mode 100644
index 0000000000000000000000000000000000000000..44f4b0a2c3c6c726857db8386491823dd85dde51
--- /dev/null
+++ b/audiocraft/audiocraft/metrics/visqol.py
@@ -0,0 +1,216 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import csv
+import json
+import logging
+from pathlib import Path
+import tempfile
+import typing as tp
+import subprocess
+import shutil
+
+import torch
+import torchaudio
+
+logger = logging.getLogger(__name__)
+
+
+class ViSQOL:
+    """ViSQOL wrapper to run ViSQOL from Python using a pre-installed binary.
+
+    To learn more about ViSQOL and how to build ViSQOL binary using bazel, please refer to the
+    instructions available in the open source repository: https://github.com/google/visqol
+
+    ViSQOL is capable of running in two modes:
+
+    Audio Mode:
+        When running in audio mode, input signals must have a 48kHz sample rate. Input should be resampled to 48kHz.
+        Input signals can be multi-channel, but they will be down-mixed to mono for performing the comparison.
+        Audio mode uses support vector regression, with the maximum range at ~4.75.
+
+    Speech Mode:
+        When running in speech mode, ViSQOL uses a wideband model. It therefore expects input sample rates of 16kHz.
+            Input should be resampled to 16kHz.
+        As part of the speech mode processing, a root mean square implementation for voice activity detection
+            is performed on the reference signal to determine what parts of the signal have voice activity and
+            should therefore be included in the comparison. The signal is normalized before performing the voice
+            activity detection.
+        Input signals can be multi-channel, but they will be down-mixed to mono for performing the comparison.
+        Speech mode is scaled to have a maximum MOS of 5.0 to match previous version behavior.
+
+    For more details, check the guidelines: https://github.com/google/visqol#general-guidelines-for-input
+
+    Args:
+        visqol_bin (str): Path to the ViSQOL binary.
+        mode (str): ViSQOL computation mode, expecting "audio" or "speech".
+        model (str): Name of the model to use for similarity to quality model.
+        debug (bool): Whether to also get debug metrics from ViSQOL or not.
+    """
+    SAMPLE_RATES_MODES = {"audio": 48_000, "speech": 16_000}
+    ALLOWED_SAMPLE_RATES = frozenset(SAMPLE_RATES_MODES.values())
+
+    def __init__(self, bin: tp.Union[Path, str], mode: str = "audio",
+                 model: str = "libsvm_nu_svr_model.txt", debug: bool = False):
+        assert bin is not None and Path(bin).exists(), f"Could not find ViSQOL binary in specified path: {bin}"
+        self.visqol_bin = str(bin)
+        self.visqol_mode = mode
+        self.target_sr = self._get_target_sr(self.visqol_mode)
+        self.model = model
+        self.debug = debug
+        assert Path(self.visqol_model).exists(), \
+            f"Could not find the specified model in ViSQOL install: {self.visqol_model}"
+
+    def _get_target_sr(self, mode: str) -> int:
+        # returns target sampling rate for the corresponding ViSQOL mode.
+        if mode not in ViSQOL.SAMPLE_RATES_MODES:
+            raise ValueError(
+                f"Unsupported mode! Allowed are: {', '.join(ViSQOL.SAMPLE_RATES_MODES.keys())}"
+            )
+        return ViSQOL.SAMPLE_RATES_MODES[mode]
+
+    def _prepare_files(
+        self, ref_sig: torch.Tensor, deg_sig: torch.Tensor, sr: int, target_sr: int, pad_with_silence: bool = False
+    ):
+        # prepare files for ViSQOL evaluation.
+        assert target_sr in ViSQOL.ALLOWED_SAMPLE_RATES
+        assert len(ref_sig) == len(deg_sig), (
+            "Expects same number of ref and degraded inputs",
+            f" but ref len {len(ref_sig)} != deg len {len(deg_sig)}"
+        )
+        # resample audio if needed
+        if sr != target_sr:
+            transform = torchaudio.transforms.Resample(sr, target_sr)
+            pad = int(0.5 * target_sr)
+            rs_ref = []
+            rs_deg = []
+            for i in range(len(ref_sig)):
+                rs_ref_i = transform(ref_sig[i])
+                rs_deg_i = transform(deg_sig[i])
+                if pad_with_silence:
+                    rs_ref_i = torch.nn.functional.pad(rs_ref_i, (pad, pad), mode='constant', value=0)
+                    rs_deg_i = torch.nn.functional.pad(rs_deg_i, (pad, pad), mode='constant', value=0)
+                rs_ref.append(rs_ref_i)
+                rs_deg.append(rs_deg_i)
+            ref_sig = torch.stack(rs_ref)
+            deg_sig = torch.stack(rs_deg)
+        # save audio chunks to tmp dir and create csv
+        tmp_dir = Path(tempfile.mkdtemp())
+        try:
+            tmp_input_csv_path = tmp_dir / "input.csv"
+            tmp_results_csv_path = tmp_dir / "results.csv"
+            tmp_debug_json_path = tmp_dir / "debug.json"
+            with open(tmp_input_csv_path, "w") as csv_file:
+                csv_writer = csv.writer(csv_file)
+                csv_writer.writerow(["reference", "degraded"])
+                for i in range(len(ref_sig)):
+                    tmp_ref_filename = tmp_dir / f"ref_{i}.wav"
+                    tmp_deg_filename = tmp_dir / f"deg_{i}.wav"
+                    torchaudio.save(
+                        tmp_ref_filename,
+                        torch.clamp(ref_sig[i], min=-0.99, max=0.99),
+                        sample_rate=target_sr,
+                        bits_per_sample=16,
+                        encoding="PCM_S"
+                    )
+                    torchaudio.save(
+                        tmp_deg_filename,
+                        torch.clamp(deg_sig[i], min=-0.99, max=0.99),
+                        sample_rate=target_sr,
+                        bits_per_sample=16,
+                        encoding="PCM_S"
+                    )
+                    csv_writer.writerow([str(tmp_ref_filename), str(tmp_deg_filename)])
+            return tmp_dir, tmp_input_csv_path, tmp_results_csv_path, tmp_debug_json_path
+        except Exception as e:
+            logger.error("Exception occurred when preparing files for ViSQOL: %s", e)
+            return tmp_dir, None, None, None
+
+    def _flush_files(self, tmp_dir: tp.Union[Path, str]):
+        # flush tmp files used to compute ViSQOL.
+        shutil.rmtree(str(tmp_dir))
+
+    def _collect_moslqo_score(self, results_csv_path: tp.Union[Path, str]) -> float:
+        # collect results for each evaluated pair and return averaged moslqo score.
+        with open(results_csv_path, "r") as csv_file:
+            reader = csv.DictReader(csv_file)
+            moslqo_scores = [float(row["moslqo"]) for row in reader]
+            if len(moslqo_scores) > 0:
+                return sum(moslqo_scores) / len(moslqo_scores)
+            else:
+                return 0.0
+
+    def _collect_debug_data(self, debug_json_path: tp.Union[Path, str]) -> dict:
+        # collect debug data for the visqol inference.
+        with open(debug_json_path, "r") as f:
+            data = json.load(f)
+            return data
+
+    @property
+    def visqol_model(self):
+        return f'{self.visqol_bin}/model/{self.model}'
+
+    def _run_visqol(
+        self,
+        input_csv_path: tp.Union[Path, str],
+        results_csv_path: tp.Union[Path, str],
+        debug_csv_path: tp.Optional[tp.Union[Path, str]],
+    ):
+        input_csv_path = str(input_csv_path)
+        results_csv_path = str(results_csv_path)
+        debug_csv_path = str(debug_csv_path)
+        cmd = [
+            f'{self.visqol_bin}/bazel-bin/visqol',
+            '--batch_input_csv', f'{input_csv_path}',
+            '--results_csv', f'{results_csv_path}'
+        ]
+        if debug_csv_path is not None:
+            cmd += ['--output_debug', f'{debug_csv_path}']
+        if self.visqol_mode == "speech":
+            cmd += ['--use_speech_mode']
+        cmd += ['--similarity_to_quality_model', f'{self.visqol_model}']
+        result = subprocess.run(cmd, capture_output=True)
+        if result.returncode:
+            logger.error("Error with visqol: \n %s \n %s", result.stdout.decode(), result.stderr.decode())
+            raise RuntimeError("Error while executing visqol")
+        result.check_returncode()
+
+    def __call__(
+        self,
+        ref_sig: torch.Tensor,
+        deg_sig: torch.Tensor,
+        sr: int,
+        pad_with_silence: bool = False,
+    ):
+        """Calculate the ViSQOL metric for a pair of audio signals at a given sample rate.
+        Args:
+            ref_sig (torch.Tensor): Reference signals as [B, C, T].
+            deg_sig (torch.Tensor): Degraded signals as [B, C, T].
+            sr (int): Sample rate of the two audio signals.
+            pad_with_silence (bool): Whether to pad the file with silences as recommended
+                in visqol guidelines (see: https://github.com/google/visqol#general-guidelines-for-input).
+        Returns:
+            float: The ViSQOL score or mean score for the batch.
+        """
+        logger.debug(f"Calculating visqol with mode={self.visqol_mode} on {len(ref_sig)} samples")
+        tmp_dir, input_csv, results_csv, debug_json = self._prepare_files(
+            ref_sig, deg_sig, sr, self.target_sr, pad_with_silence
+        )
+        try:
+            if input_csv and results_csv:
+                self._run_visqol(
+                    input_csv,
+                    results_csv,
+                    debug_json if self.debug else None,
+                )
+                mosqol = self._collect_moslqo_score(results_csv)
+                return mosqol
+            else:
+                raise RuntimeError("Something unexpected happened when running VISQOL!")
+        except Exception as e:
+            logger.error("Exception occurred when running ViSQOL: %s", e)
+        finally:
+            self._flush_files(tmp_dir)
diff --git a/audiocraft/audiocraft/models/__init__.py b/audiocraft/audiocraft/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..be6bfe4b787a132aeaabaed1c3437c9ecd5c656c
--- /dev/null
+++ b/audiocraft/audiocraft/models/__init__.py
@@ -0,0 +1,18 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Models for EnCodec, AudioGen, MusicGen, as well as the generic LMModel.
+"""
+# flake8: noqa
+from . import builders, loaders
+from .encodec import (
+    CompressionModel, EncodecModel, DAC,
+    HFEncodecModel, HFEncodecCompressionModel)
+from .audiogen import AudioGen
+from .lm import LMModel
+from .multibanddiffusion import MultiBandDiffusion
+from .musicgen import MusicGen
+from .unet import DiffusionUnet
diff --git a/audiocraft/audiocraft/models/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..da36453ed3573a6e1d198357130c356270ee5206
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/audiogen.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/audiogen.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..75d6a4610579722296550c8a17c602ffb1d7ddfc
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/audiogen.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/builders.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/builders.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d9a2220766f6586ceb09135edeb28733d9772274
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/builders.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/encodec.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/encodec.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..5b9ed907232b76840c1b299ea5fbcbb7da100d3b
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/encodec.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/lm.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/lm.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..571c3468eb4c452f0cb4cf4d7fd6bee703743b11
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/lm.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/loaders.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/loaders.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..002244067cbc34f0467a773d0c6888fd657902bc
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/loaders.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/multibanddiffusion.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/multibanddiffusion.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f7f19ed26afe9b48bdbc18db92e27295e5f9e4ed
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/multibanddiffusion.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/musicgen.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/musicgen.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7647d227c35c714e64488c4f20ff473094112d4e
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/musicgen.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/__pycache__/unet.cpython-311.pyc b/audiocraft/audiocraft/models/__pycache__/unet.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ee74b3c24f03f7c31cef168b98e97c0d56008c04
Binary files /dev/null and b/audiocraft/audiocraft/models/__pycache__/unet.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/models/audiogen.py b/audiocraft/audiocraft/models/audiogen.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cb889982ddc027e2588b7cfb8ef428b313ce88a
--- /dev/null
+++ b/audiocraft/audiocraft/models/audiogen.py
@@ -0,0 +1,263 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Main model for using AudioGen. This will combine all the required components
+and provide easy access to the generation API.
+"""
+
+import typing as tp
+
+import torch
+
+from .encodec import CompressionModel
+from .lm import LMModel
+from .builders import get_debug_compression_model, get_debug_lm_model
+from .loaders import load_compression_model, load_lm_model
+from ..data.audio_utils import convert_audio
+from ..modules.conditioners import ConditioningAttributes
+from ..utils.autocast import TorchAutocast
+
+
+class AudioGen:
+    """AudioGen main model with convenient generation API.
+
+    Args:
+        name (str): name of the model.
+        compression_model (CompressionModel): Compression model
+            used to map audio to invertible discrete representations.
+        lm (LMModel): Language model over discrete representations.
+        max_duration (float, optional): maximum duration the model can produce,
+            otherwise, inferred from the training params.
+    """
+    def __init__(self, name: str, compression_model: CompressionModel, lm: LMModel,
+                 max_duration: tp.Optional[float] = None):
+        self.name = name
+        self.compression_model = compression_model
+        self.lm = lm
+        if max_duration is None:
+            if hasattr(lm, 'cfg'):
+                max_duration = lm.cfg.dataset.segment_duration  # type: ignore
+            else:
+                raise ValueError("You must provide max_duration when building directly AudioGen")
+        assert max_duration is not None
+        self.max_duration: float = max_duration
+        self.device = next(iter(lm.parameters())).device
+        self.generation_params: dict = {}
+        self.set_generation_params(duration=5)  # 5 seconds by default
+        self._progress_callback: tp.Optional[tp.Callable[[int, int], None]] = None
+        if self.device.type == 'cpu':
+            self.autocast = TorchAutocast(enabled=False)
+        else:
+            self.autocast = TorchAutocast(
+                enabled=True, device_type=self.device.type, dtype=torch.float16)
+
+    @property
+    def frame_rate(self) -> float:
+        """Roughly the number of AR steps per seconds."""
+        return self.compression_model.frame_rate
+
+    @property
+    def sample_rate(self) -> int:
+        """Sample rate of the generated audio."""
+        return self.compression_model.sample_rate
+
+    @property
+    def audio_channels(self) -> int:
+        """Audio channels of the generated audio."""
+        return self.compression_model.channels
+
+    @staticmethod
+    def get_pretrained(name: str = 'facebook/audiogen-medium', device=None):
+        """Return pretrained model, we provide a single model for now:
+        - facebook/audiogen-medium (1.5B), text to sound,
+          # see: https://huggingface.co/facebook/audiogen-medium
+        """
+        if device is None:
+            if torch.cuda.device_count():
+                device = 'cuda'
+            else:
+                device = 'cpu'
+
+        if name == 'debug':
+            # used only for unit tests
+            compression_model = get_debug_compression_model(device, sample_rate=16000)
+            lm = get_debug_lm_model(device)
+            return AudioGen(name, compression_model, lm, max_duration=10)
+
+        compression_model = load_compression_model(name, device=device)
+        lm = load_lm_model(name, device=device)
+        assert 'self_wav' not in lm.condition_provider.conditioners, \
+            "AudioGen do not support waveform conditioning for now"
+        return AudioGen(name, compression_model, lm)
+
+    def set_generation_params(self, use_sampling: bool = True, top_k: int = 250,
+                              top_p: float = 0.0, temperature: float = 1.0,
+                              duration: float = 10.0, cfg_coef: float = 3.0,
+                              two_step_cfg: bool = False, extend_stride: float = 2):
+        """Set the generation parameters for AudioGen.
+
+        Args:
+            use_sampling (bool, optional): Use sampling if True, else do argmax decoding. Defaults to True.
+            top_k (int, optional): top_k used for sampling. Defaults to 250.
+            top_p (float, optional): top_p used for sampling, when set to 0 top_k is used. Defaults to 0.0.
+            temperature (float, optional): Softmax temperature parameter. Defaults to 1.0.
+            duration (float, optional): Duration of the generated waveform. Defaults to 10.0.
+            cfg_coef (float, optional): Coefficient used for classifier free guidance. Defaults to 3.0.
+            two_step_cfg (bool, optional): If True, performs 2 forward for Classifier Free Guidance,
+                instead of batching together the two. This has some impact on how things
+                are padded but seems to have little impact in practice.
+            extend_stride: when doing extended generation (i.e. more than 10 seconds), by how much
+                should we extend the audio each time. Larger values will mean less context is
+                preserved, and shorter value will require extra computations.
+        """
+        assert extend_stride < self.max_duration, "Cannot stride by more than max generation duration."
+        self.extend_stride = extend_stride
+        self.duration = duration
+        self.generation_params = {
+            'use_sampling': use_sampling,
+            'temp': temperature,
+            'top_k': top_k,
+            'top_p': top_p,
+            'cfg_coef': cfg_coef,
+            'two_step_cfg': two_step_cfg,
+        }
+
+    def set_custom_progress_callback(self, progress_callback: tp.Optional[tp.Callable[[int, int], None]] = None):
+        """Override the default progress callback."""
+        self._progress_callback = progress_callback
+
+    def generate(self, descriptions: tp.List[str], progress: bool = False) -> torch.Tensor:
+        """Generate samples conditioned on text.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions, None)
+        assert prompt_tokens is None
+        return self._generate_tokens(attributes, prompt_tokens, progress)
+
+    def generate_continuation(self, prompt: torch.Tensor, prompt_sample_rate: int,
+                              descriptions: tp.Optional[tp.List[tp.Optional[str]]] = None,
+                              progress: bool = False) -> torch.Tensor:
+        """Generate samples conditioned on audio prompts.
+
+        Args:
+            prompt (torch.Tensor): A batch of waveforms used for continuation.
+                Prompt should be [B, C, T], or [C, T] if only one sample is generated.
+            prompt_sample_rate (int): Sampling rate of the given audio waveforms.
+            descriptions (list of str, optional): A list of strings used as text conditioning. Defaults to None.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+        if prompt.dim() == 2:
+            prompt = prompt[None]
+        if prompt.dim() != 3:
+            raise ValueError("prompt should have 3 dimensions: [B, C, T] (C = 1).")
+        prompt = convert_audio(prompt, prompt_sample_rate, self.sample_rate, self.audio_channels)
+        if descriptions is None:
+            descriptions = [None] * len(prompt)
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions, prompt)
+        assert prompt_tokens is not None
+        return self._generate_tokens(attributes, prompt_tokens, progress)
+
+    @torch.no_grad()
+    def _prepare_tokens_and_attributes(
+            self,
+            descriptions: tp.Sequence[tp.Optional[str]],
+            prompt: tp.Optional[torch.Tensor],
+    ) -> tp.Tuple[tp.List[ConditioningAttributes], tp.Optional[torch.Tensor]]:
+        """Prepare model inputs.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            prompt (torch.Tensor): A batch of waveforms used for continuation.
+        """
+        attributes = [
+            ConditioningAttributes(text={'description': description})
+            for description in descriptions]
+
+        if prompt is not None:
+            if descriptions is not None:
+                assert len(descriptions) == len(prompt), "Prompt and nb. descriptions doesn't match"
+            prompt = prompt.to(self.device)
+            prompt_tokens, scale = self.compression_model.encode(prompt)
+            assert scale is None
+        else:
+            prompt_tokens = None
+        return attributes, prompt_tokens
+
+    def _generate_tokens(self, attributes: tp.List[ConditioningAttributes],
+                         prompt_tokens: tp.Optional[torch.Tensor], progress: bool = False) -> torch.Tensor:
+        """Generate discrete audio tokens given audio prompt and/or conditions.
+
+        Args:
+            attributes (list of ConditioningAttributes): Conditions used for generation (here text).
+            prompt_tokens (torch.Tensor, optional): Audio prompt used for continuation.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        Returns:
+            torch.Tensor: Generated audio, of shape [B, C, T], T is defined by the generation params.
+        """
+        total_gen_len = int(self.duration * self.frame_rate)
+        max_prompt_len = int(min(self.duration, self.max_duration) * self.frame_rate)
+        current_gen_offset: int = 0
+
+        def _progress_callback(generated_tokens: int, tokens_to_generate: int):
+            generated_tokens += current_gen_offset
+            if self._progress_callback is not None:
+                # Note that total_gen_len might be quite wrong depending on the
+                # codebook pattern used, but with delay it is almost accurate.
+                self._progress_callback(generated_tokens, total_gen_len)
+            else:
+                print(f'{generated_tokens: 6d} / {total_gen_len: 6d}', end='\r')
+
+        if prompt_tokens is not None:
+            assert max_prompt_len >= prompt_tokens.shape[-1], \
+                "Prompt is longer than audio to generate"
+
+        callback = None
+        if progress:
+            callback = _progress_callback
+
+        if self.duration <= self.max_duration:
+            # generate by sampling from LM, simple case.
+            with self.autocast:
+                gen_tokens = self.lm.generate(
+                    prompt_tokens, attributes,
+                    callback=callback, max_gen_len=total_gen_len, **self.generation_params)
+
+        else:
+            all_tokens = []
+            if prompt_tokens is None:
+                prompt_length = 0
+            else:
+                all_tokens.append(prompt_tokens)
+                prompt_length = prompt_tokens.shape[-1]
+
+            stride_tokens = int(self.frame_rate * self.extend_stride)
+            while current_gen_offset + prompt_length < total_gen_len:
+                time_offset = current_gen_offset / self.frame_rate
+                chunk_duration = min(self.duration - time_offset, self.max_duration)
+                max_gen_len = int(chunk_duration * self.frame_rate)
+                with self.autocast:
+                    gen_tokens = self.lm.generate(
+                        prompt_tokens, attributes,
+                        callback=callback, max_gen_len=max_gen_len, **self.generation_params)
+                if prompt_tokens is None:
+                    all_tokens.append(gen_tokens)
+                else:
+                    all_tokens.append(gen_tokens[:, :, prompt_tokens.shape[-1]:])
+                prompt_tokens = gen_tokens[:, :, stride_tokens:]
+                prompt_length = prompt_tokens.shape[-1]
+                current_gen_offset += stride_tokens
+
+            gen_tokens = torch.cat(all_tokens, dim=-1)
+
+        # generate audio
+        assert gen_tokens.dim() == 3
+        with torch.no_grad():
+            gen_audio = self.compression_model.decode(gen_tokens, None)
+        return gen_audio
diff --git a/audiocraft/audiocraft/models/builders.py b/audiocraft/audiocraft/models/builders.py
new file mode 100644
index 0000000000000000000000000000000000000000..2a427bc4f4a1925501d9eee54429e3f72eedb7f9
--- /dev/null
+++ b/audiocraft/audiocraft/models/builders.py
@@ -0,0 +1,267 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+All the functions to build the relevant models and modules
+from the Hydra config.
+"""
+
+import typing as tp
+
+import audiocraft
+import omegaconf
+import torch
+
+from .encodec import CompressionModel, EncodecModel
+from .lm import LMModel
+from ..modules.codebooks_patterns import (
+    CodebooksPatternProvider,
+    DelayedPatternProvider,
+    MusicLMPattern,
+    ParallelPatternProvider,
+    UnrolledPatternProvider,
+    VALLEPattern,
+)
+from ..modules.conditioners import (
+    BaseConditioner,
+    ChromaStemConditioner,
+    CLAPEmbeddingConditioner,
+    ConditionFuser,
+    ConditioningProvider,
+    LUTConditioner,
+    T5Conditioner,
+    ChordProgressionConditioner,
+    BeatConditioner
+)
+from .unet import DiffusionUnet
+from .. import quantization as qt
+from ..utils.utils import dict_from_config
+from ..modules.diffusion_schedule import MultiBandProcessor, SampleProcessor
+
+
+def get_quantizer(quantizer: str, cfg: omegaconf.DictConfig, dimension: int) -> qt.BaseQuantizer:
+    klass = {
+        'no_quant': qt.DummyQuantizer,
+        'rvq': qt.ResidualVectorQuantizer
+    }[quantizer]
+    kwargs = dict_from_config(getattr(cfg, quantizer))
+    if quantizer != 'no_quant':
+        kwargs['dimension'] = dimension
+    return klass(**kwargs)
+
+
+def get_encodec_autoencoder(encoder_name: str, cfg: omegaconf.DictConfig):
+    if encoder_name == 'seanet':
+        kwargs = dict_from_config(getattr(cfg, 'seanet'))
+        encoder_override_kwargs = kwargs.pop('encoder')
+        decoder_override_kwargs = kwargs.pop('decoder')
+        encoder_kwargs = {**kwargs, **encoder_override_kwargs}
+        decoder_kwargs = {**kwargs, **decoder_override_kwargs}
+        encoder = audiocraft.modules.SEANetEncoder(**encoder_kwargs)
+        decoder = audiocraft.modules.SEANetDecoder(**decoder_kwargs)
+        return encoder, decoder
+    else:
+        raise KeyError(f"Unexpected compression model {cfg.compression_model}")
+
+
+def get_compression_model(cfg: omegaconf.DictConfig) -> CompressionModel:
+    """Instantiate a compression model."""
+    if cfg.compression_model == 'encodec':
+        kwargs = dict_from_config(getattr(cfg, 'encodec'))
+        encoder_name = kwargs.pop('autoencoder')
+        quantizer_name = kwargs.pop('quantizer')
+        encoder, decoder = get_encodec_autoencoder(encoder_name, cfg)
+        quantizer = get_quantizer(quantizer_name, cfg, encoder.dimension)
+        frame_rate = kwargs['sample_rate'] // encoder.hop_length
+        renormalize = kwargs.pop('renormalize', False)
+        # deprecated params
+        kwargs.pop('renorm', None)
+        return EncodecModel(encoder, decoder, quantizer,
+                            frame_rate=frame_rate, renormalize=renormalize, **kwargs).to(cfg.device)
+    else:
+        raise KeyError(f"Unexpected compression model {cfg.compression_model}")
+
+
+def get_lm_model(cfg: omegaconf.DictConfig) -> LMModel:
+    """Instantiate a transformer LM."""
+    if cfg.lm_model == 'transformer_lm':
+        kwargs = dict_from_config(getattr(cfg, 'transformer_lm'))
+        n_q = kwargs['n_q']
+        q_modeling = kwargs.pop('q_modeling', None)
+        codebooks_pattern_cfg = getattr(cfg, 'codebooks_pattern')
+        attribute_dropout = dict_from_config(getattr(cfg, 'attribute_dropout'))
+        cls_free_guidance = dict_from_config(getattr(cfg, 'classifier_free_guidance'))
+        cfg_prob, cfg_coef = cls_free_guidance['training_dropout'], cls_free_guidance['inference_coef']
+        fuser = get_condition_fuser(cfg)
+        condition_provider = get_conditioner_provider(kwargs["dim"], cfg).to(cfg.device)
+        if len(fuser.fuse2cond['cross']) > 0:  # enforce cross-att programmatically
+            kwargs['cross_attention'] = True
+        if codebooks_pattern_cfg.modeling is None:
+            assert q_modeling is not None, \
+                "LM model should either have a codebook pattern defined or transformer_lm.q_modeling"
+            codebooks_pattern_cfg = omegaconf.OmegaConf.create(
+                {'modeling': q_modeling, 'delay': {'delays': list(range(n_q))}}
+            )
+        pattern_provider = get_codebooks_pattern_provider(n_q, codebooks_pattern_cfg)
+        return LMModel(
+            pattern_provider=pattern_provider,
+            condition_provider=condition_provider,
+            fuser=fuser,
+            cfg_dropout=cfg_prob,
+            cfg_coef=cfg_coef,
+            attribute_dropout=attribute_dropout,
+            dtype=getattr(torch, cfg.dtype),
+            device=cfg.device,
+            **kwargs
+        ).to(cfg.device)
+    else:
+        raise KeyError(f"Unexpected LM model {cfg.lm_model}")
+
+
+def get_conditioner_provider(output_dim: int, cfg: omegaconf.DictConfig) -> ConditioningProvider:
+    """Instantiate a conditioning model."""
+    device = cfg.device
+    duration = cfg.dataset.segment_duration
+    cfg = getattr(cfg, 'conditioners')
+    dict_cfg = {} if cfg is None else dict_from_config(cfg)
+    conditioners: tp.Dict[str, BaseConditioner] = {}
+    condition_provider_args = dict_cfg.pop('args', {})
+    condition_provider_args.pop('merge_text_conditions_p', None)
+    condition_provider_args.pop('drop_desc_p', None)
+
+    for cond, cond_cfg in dict_cfg.items():
+        model_type = cond_cfg['model']
+        model_args = cond_cfg[model_type]
+        if model_type == 't5':
+            conditioners[str(cond)] = T5Conditioner(output_dim=output_dim, device=device, **model_args)
+        elif model_type == 'lut':
+            conditioners[str(cond)] = LUTConditioner(output_dim=output_dim, **model_args)
+        elif model_type == 'chroma_stem':
+            conditioners[str(cond)] = ChromaStemConditioner(
+                output_dim=output_dim,
+                duration=duration,
+                device=device,
+                **model_args
+            )
+        elif model_type == 'beat':
+            conditioners[str(cond)] = BeatConditioner(
+                output_dim=output_dim,
+                device=device,
+                **model_args
+            )
+        elif model_type == 'chord':
+            conditioners[str(cond)] = ChordProgressionConditioner(
+                output_dim=output_dim,
+                device=device,
+                **model_args
+            )
+        elif model_type == 'clap':
+            conditioners[str(cond)] = CLAPEmbeddingConditioner(
+                output_dim=output_dim,
+                device=device,
+                **model_args
+            )
+        else:
+            raise ValueError(f"Unrecognized conditioning model: {model_type}")
+    conditioner = ConditioningProvider(conditioners, device=device, **condition_provider_args)
+    return conditioner
+
+
+def get_condition_fuser(cfg: omegaconf.DictConfig) -> ConditionFuser:
+    """Instantiate a condition fuser object."""
+    fuser_cfg = getattr(cfg, 'fuser')
+    fuser_methods = ['sum', 'cross', 'prepend', 'input_interpolate']
+    fuse2cond = {k: fuser_cfg[k] for k in fuser_methods}
+    kwargs = {k: v for k, v in fuser_cfg.items() if k not in fuser_methods}
+    print(f"==== use in-attention: {fuser_cfg['in_attn']} ====")
+    fuser = ConditionFuser(fuse2cond=fuse2cond, **kwargs)
+    return fuser
+
+
+def get_codebooks_pattern_provider(n_q: int, cfg: omegaconf.DictConfig) -> CodebooksPatternProvider:
+    """Instantiate a codebooks pattern provider object."""
+    pattern_providers = {
+        'parallel': ParallelPatternProvider,
+        'delay': DelayedPatternProvider,
+        'unroll': UnrolledPatternProvider,
+        'valle': VALLEPattern,
+        'musiclm': MusicLMPattern,
+    }
+    name = cfg.modeling
+    kwargs = dict_from_config(cfg.get(name)) if hasattr(cfg, name) else {}
+    klass = pattern_providers[name]
+    return klass(n_q, **kwargs)
+
+
+def get_debug_compression_model(device='cpu', sample_rate: int = 32000):
+    """Instantiate a debug compression model to be used for unit tests."""
+    assert sample_rate in [16000, 32000], "unsupported sample rate for debug compression model"
+    model_ratios = {
+        16000: [10, 8, 8],  # 25 Hz at 16kHz
+        32000: [10, 8, 16]  # 25 Hz at 32kHz
+    }
+    ratios: tp.List[int] = model_ratios[sample_rate]
+    frame_rate = 25
+    seanet_kwargs: dict = {
+        'n_filters': 4,
+        'n_residual_layers': 1,
+        'dimension': 32,
+        'ratios': ratios,
+    }
+    print(seanet_kwargs)
+    encoder = audiocraft.modules.SEANetEncoder(**seanet_kwargs)
+    decoder = audiocraft.modules.SEANetDecoder(**seanet_kwargs)
+    quantizer = qt.ResidualVectorQuantizer(dimension=32, bins=400, n_q=4)
+    init_x = torch.randn(8, 32, 128)
+    quantizer(init_x, 1)  # initialize kmeans etc.
+    compression_model = EncodecModel(
+        encoder, decoder, quantizer,
+        frame_rate=frame_rate, sample_rate=sample_rate, channels=1).to(device)
+    return compression_model.eval()
+
+
+def get_diffusion_model(cfg: omegaconf.DictConfig):
+    # TODO Find a way to infer the channels from dset
+    channels = cfg.channels
+    num_steps = cfg.schedule.num_steps
+    return DiffusionUnet(
+            chin=channels, num_steps=num_steps, **cfg.diffusion_unet)
+
+
+def get_processor(cfg, sample_rate: int = 24000):
+    sample_processor = SampleProcessor()
+    if cfg.use:
+        kw = dict(cfg)
+        kw.pop('use')
+        kw.pop('name')
+        if cfg.name == "multi_band_processor":
+            sample_processor = MultiBandProcessor(sample_rate=sample_rate, **kw)
+    return sample_processor
+
+
+def get_debug_lm_model(device='cpu'):
+    """Instantiate a debug LM to be used for unit tests."""
+    pattern = DelayedPatternProvider(n_q=4)
+    dim = 16
+    providers = {
+        'description': LUTConditioner(n_bins=128, dim=dim, output_dim=dim, tokenizer="whitespace"),
+    }
+    condition_provider = ConditioningProvider(providers)
+    fuser = ConditionFuser(
+        {'cross': ['description'], 'prepend': [],
+         'sum': [], 'input_interpolate': []})
+    lm = LMModel(
+        pattern, condition_provider, fuser,
+        n_q=4, card=400, dim=dim, num_heads=4, custom=True, num_layers=2,
+        cross_attention=True, causal=True)
+    return lm.to(device).eval()
+
+
+def get_wrapped_compression_model(
+        compression_model: CompressionModel,
+        cfg: omegaconf.DictConfig) -> CompressionModel:
+    # more to come.
+    return compression_model
diff --git a/audiocraft/audiocraft/models/encodec.py b/audiocraft/audiocraft/models/encodec.py
new file mode 100644
index 0000000000000000000000000000000000000000..40d133017c0a0eddaafb07d291b3845789775bc3
--- /dev/null
+++ b/audiocraft/audiocraft/models/encodec.py
@@ -0,0 +1,393 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Compression models or wrapper around existing models.
+Also defines the main interface that a model must follow to be usable as an audio tokenizer.
+"""
+
+from abc import ABC, abstractmethod
+import logging
+import math
+from pathlib import Path
+import typing as tp
+
+import numpy as np
+import torch
+from torch import nn
+from transformers import EncodecModel as HFEncodecModel
+
+from .. import quantization as qt
+
+
+logger = logging.getLogger()
+
+
+class CompressionModel(ABC, nn.Module):
+    """Base API for all compression model that aim at being used as audio tokenizers
+    with a language model.
+    """
+
+    @abstractmethod
+    def forward(self, x: torch.Tensor) -> qt.QuantizedResult:
+        ...
+
+    @abstractmethod
+    def encode(self, x: torch.Tensor) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
+        """See `EncodecModel.encode`."""
+        ...
+
+    @abstractmethod
+    def decode(self, codes: torch.Tensor, scale: tp.Optional[torch.Tensor] = None):
+        """See `EncodecModel.decode`."""
+        ...
+
+    @abstractmethod
+    def decode_latent(self, codes: torch.Tensor):
+        """Decode from the discrete codes to continuous latent space."""
+        ...
+
+    @property
+    @abstractmethod
+    def channels(self) -> int:
+        ...
+
+    @property
+    @abstractmethod
+    def frame_rate(self) -> float:
+        ...
+
+    @property
+    @abstractmethod
+    def sample_rate(self) -> int:
+        ...
+
+    @property
+    @abstractmethod
+    def cardinality(self) -> int:
+        ...
+
+    @property
+    @abstractmethod
+    def num_codebooks(self) -> int:
+        ...
+
+    @property
+    @abstractmethod
+    def total_codebooks(self) -> int:
+        ...
+
+    @abstractmethod
+    def set_num_codebooks(self, n: int):
+        """Set the active number of codebooks used by the quantizer."""
+        ...
+
+    @staticmethod
+    def get_pretrained(
+            name: str, device: tp.Union[torch.device, str] = 'cpu'
+            ) -> 'CompressionModel':
+        """Instantiate a CompressionModel from a given pretrained model.
+
+        Args:
+            name (Path or str): name of the pretrained model. See after.
+            device (torch.device or str): Device on which the model is loaded.
+
+        Pretrained models:
+            - dac_44khz (https://github.com/descriptinc/descript-audio-codec)
+            - dac_24khz (same)
+            - facebook/encodec_24khz (https://huggingface.co/facebook/encodec_24khz)
+            - facebook/encodec_32khz (https://huggingface.co/facebook/encodec_32khz)
+            - your own model on HugginFace. Export instructions to come...
+        """
+
+        from . import builders, loaders
+        model: CompressionModel
+        if name in ['dac_44khz', 'dac_24khz']:
+            model_type = name.split('_')[1]
+            logger.info("Getting pretrained compression model from DAC %s", model_type)
+            model = DAC(model_type)
+        elif name in ['debug_compression_model']:
+            logger.info("Getting pretrained compression model for debug")
+            model = builders.get_debug_compression_model()
+        elif Path(name).exists():
+            # We assume here if the paths exist that it is in fact an AC checkpoint
+            # that was exported using `audiocraft.utils.export` functions.
+            model = loaders.load_compression_model(name, device=device)
+        else:
+            logger.info("Getting pretrained compression model from HF %s", name)
+            hf_model = HFEncodecModel.from_pretrained(name)
+            model = HFEncodecCompressionModel(hf_model).to(device)
+        return model.to(device).eval()
+
+
+class EncodecModel(CompressionModel):
+    """Encodec model operating on the raw waveform.
+
+    Args:
+        encoder (nn.Module): Encoder network.
+        decoder (nn.Module): Decoder network.
+        quantizer (qt.BaseQuantizer): Quantizer network.
+        frame_rate (int): Frame rate for the latent representation.
+        sample_rate (int): Audio sample rate.
+        channels (int): Number of audio channels.
+        causal (bool): Whether to use a causal version of the model.
+        renormalize (bool): Whether to renormalize the audio before running the model.
+    """
+    # we need assignment to override the property in the abstract class,
+    # I couldn't find a better way...
+    frame_rate: float = 0
+    sample_rate: int = 0
+    channels: int = 0
+
+    def __init__(self,
+                 encoder: nn.Module,
+                 decoder: nn.Module,
+                 quantizer: qt.BaseQuantizer,
+                 frame_rate: int,
+                 sample_rate: int,
+                 channels: int,
+                 causal: bool = False,
+                 renormalize: bool = False):
+        super().__init__()
+        self.encoder = encoder
+        self.decoder = decoder
+        self.quantizer = quantizer
+        self.frame_rate = frame_rate
+        self.sample_rate = sample_rate
+        self.channels = channels
+        self.renormalize = renormalize
+        self.causal = causal
+        if self.causal:
+            # we force disabling here to avoid handling linear overlap of segments
+            # as supported in original EnCodec codebase.
+            assert not self.renormalize, 'Causal model does not support renormalize'
+
+    @property
+    def total_codebooks(self):
+        """Total number of quantizer codebooks available."""
+        return self.quantizer.total_codebooks
+
+    @property
+    def num_codebooks(self):
+        """Active number of codebooks used by the quantizer."""
+        return self.quantizer.num_codebooks
+
+    def set_num_codebooks(self, n: int):
+        """Set the active number of codebooks used by the quantizer."""
+        self.quantizer.set_num_codebooks(n)
+
+    @property
+    def cardinality(self):
+        """Cardinality of each codebook."""
+        return self.quantizer.bins
+
+    def preprocess(self, x: torch.Tensor) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
+        scale: tp.Optional[torch.Tensor]
+        if self.renormalize:
+            mono = x.mean(dim=1, keepdim=True)
+            volume = mono.pow(2).mean(dim=2, keepdim=True).sqrt()
+            scale = 1e-8 + volume
+            x = x / scale
+            scale = scale.view(-1, 1)
+        else:
+            scale = None
+        return x, scale
+
+    def postprocess(self,
+                    x: torch.Tensor,
+                    scale: tp.Optional[torch.Tensor] = None) -> torch.Tensor:
+        if scale is not None:
+            assert self.renormalize
+            x = x * scale.view(-1, 1, 1)
+        return x
+
+    def forward(self, x: torch.Tensor) -> qt.QuantizedResult:
+        assert x.dim() == 3
+        length = x.shape[-1]
+        x, scale = self.preprocess(x)
+
+        emb = self.encoder(x)
+        q_res = self.quantizer(emb, self.frame_rate)
+        out = self.decoder(q_res.x)
+
+        # remove extra padding added by the encoder and decoder
+        assert out.shape[-1] >= length, (out.shape[-1], length)
+        out = out[..., :length]
+
+        q_res.x = self.postprocess(out, scale)
+
+        return q_res
+
+    def encode(self, x: torch.Tensor) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
+        """Encode the given input tensor to quantized representation along with scale parameter.
+
+        Args:
+            x (torch.Tensor): Float tensor of shape [B, C, T]
+
+        Returns:
+            codes, scale (tuple of torch.Tensor, torch.Tensor): Tuple composed of:
+                codes a float tensor of shape [B, K, T] with K the number of codebooks used and T the timestep.
+                scale a float tensor containing the scale for audio renormalizealization.
+        """
+        assert x.dim() == 3
+        x, scale = self.preprocess(x)
+        emb = self.encoder(x)
+        codes = self.quantizer.encode(emb)
+        return codes, scale
+
+    def decode(self, codes: torch.Tensor, scale: tp.Optional[torch.Tensor] = None):
+        """Decode the given codes to a reconstructed representation, using the scale to perform
+        audio denormalization if needed.
+
+        Args:
+            codes (torch.Tensor): Int tensor of shape [B, K, T]
+            scale (torch.Tensor, optional): Float tensor containing the scale value.
+
+        Returns:
+            out (torch.Tensor): Float tensor of shape [B, C, T], the reconstructed audio.
+        """
+        emb = self.decode_latent(codes)
+        out = self.decoder(emb)
+        out = self.postprocess(out, scale)
+        # out contains extra padding added by the encoder and decoder
+        return out
+
+    def decode_latent(self, codes: torch.Tensor):
+        """Decode from the discrete codes to continuous latent space."""
+        return self.quantizer.decode(codes)
+
+
+class DAC(CompressionModel):
+    def __init__(self, model_type: str = "44khz"):
+        super().__init__()
+        try:
+            import dac.utils
+        except ImportError:
+            raise RuntimeError("Could not import dac, make sure it is installed, "
+                               "please run `pip install descript-audio-codec`")
+        self.model = dac.utils.load_model(model_type=model_type)
+        self.n_quantizers = self.total_codebooks
+        self.model.eval()
+
+    def forward(self, x: torch.Tensor) -> qt.QuantizedResult:
+        # We don't support training with this.
+        raise NotImplementedError("Forward and training with DAC not supported.")
+
+    def encode(self, x: torch.Tensor) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
+        codes = self.model.encode(x, self.n_quantizers)[1]
+        return codes, None
+
+    def decode(self, codes: torch.Tensor, scale: tp.Optional[torch.Tensor] = None):
+        assert scale is None
+        z_q = self.decode_latent(codes)
+        return self.model.decode(z_q)
+
+    def decode_latent(self, codes: torch.Tensor):
+        """Decode from the discrete codes to continuous latent space."""
+        return self.model.quantizer.from_codes(codes)[0]
+
+    @property
+    def channels(self) -> int:
+        return 1
+
+    @property
+    def frame_rate(self) -> float:
+        return self.model.sample_rate / self.model.hop_length
+
+    @property
+    def sample_rate(self) -> int:
+        return self.model.sample_rate
+
+    @property
+    def cardinality(self) -> int:
+        return self.model.codebook_size
+
+    @property
+    def num_codebooks(self) -> int:
+        return self.n_quantizers
+
+    @property
+    def total_codebooks(self) -> int:
+        return self.model.n_codebooks
+
+    def set_num_codebooks(self, n: int):
+        """Set the active number of codebooks used by the quantizer.
+        """
+        assert n >= 1
+        assert n <= self.total_codebooks
+        self.n_quantizers = n
+
+
+class HFEncodecCompressionModel(CompressionModel):
+    """Wrapper around HuggingFace Encodec.
+    """
+    def __init__(self, model: HFEncodecModel):
+        super().__init__()
+        self.model = model
+        bws = self.model.config.target_bandwidths
+        num_codebooks = [
+            bw * 1000 / (self.frame_rate * math.log2(self.cardinality))
+            for bw in bws
+        ]
+        deltas = [nc - int(nc) for nc in num_codebooks]
+        # Checking we didn't do some bad maths and we indeed have integers!
+        assert all(deltas) <= 1e-3, deltas
+        self.possible_num_codebooks = [int(nc) for nc in num_codebooks]
+        self.set_num_codebooks(max(self.possible_num_codebooks))
+
+    def forward(self, x: torch.Tensor) -> qt.QuantizedResult:
+        # We don't support training with this.
+        raise NotImplementedError("Forward and training with HF EncodecModel not supported.")
+
+    def encode(self, x: torch.Tensor) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
+        bandwidth_index = self.possible_num_codebooks.index(self.num_codebooks)
+        bandwidth = self.model.config.target_bandwidths[bandwidth_index]
+        res = self.model.encode(x, None, bandwidth)
+        assert len(res[0]) == 1
+        assert len(res[1]) == 1
+        return res[0][0], res[1][0]
+
+    def decode(self, codes: torch.Tensor, scale: tp.Optional[torch.Tensor] = None):
+        if scale is None:
+            scales = [None]  # type: ignore
+        else:
+            scales = scale  # type: ignore
+        res = self.model.decode(codes[None], scales)
+        return res[0]
+
+    def decode_latent(self, codes: torch.Tensor):
+        """Decode from the discrete codes to continuous latent space."""
+        return self.model.quantizer.decode(codes.transpose(0, 1))
+
+    @property
+    def channels(self) -> int:
+        return self.model.config.audio_channels
+
+    @property
+    def frame_rate(self) -> float:
+        hop_length = int(np.prod(self.model.config.upsampling_ratios))
+        return self.sample_rate / hop_length
+
+    @property
+    def sample_rate(self) -> int:
+        return self.model.config.sampling_rate
+
+    @property
+    def cardinality(self) -> int:
+        return self.model.config.codebook_size
+
+    @property
+    def num_codebooks(self) -> int:
+        return self._num_codebooks
+
+    @property
+    def total_codebooks(self) -> int:
+        return max(self.possible_num_codebooks)
+
+    def set_num_codebooks(self, n: int):
+        """Set the active number of codebooks used by the quantizer.
+        """
+        if n not in self.possible_num_codebooks:
+            raise ValueError(f"Allowed values for num codebooks: {self.possible_num_codebooks}")
+        self._num_codebooks = n
diff --git a/audiocraft/audiocraft/models/lm.py b/audiocraft/audiocraft/models/lm.py
new file mode 100644
index 0000000000000000000000000000000000000000..f21d61f198baf4fb88c0e9ebd400948e2277fcd6
--- /dev/null
+++ b/audiocraft/audiocraft/models/lm.py
@@ -0,0 +1,533 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from dataclasses import dataclass
+from functools import partial
+import logging
+import math
+import typing as tp
+
+import torch
+from torch import nn
+
+from ..utils import utils
+from ..modules.streaming import StreamingModule, State
+from ..modules.transformer import StreamingTransformer, create_norm_fn
+from ..modules.conditioners import (
+    ConditionFuser,
+    ClassifierFreeGuidanceDropout,
+    AttributeDropout,
+    ConditioningProvider,
+    ConditioningAttributes,
+    ConditionType,
+)
+from ..modules.codebooks_patterns import CodebooksPatternProvider
+from ..modules.activations import get_activation_fn
+
+
+logger = logging.getLogger(__name__)
+ConditionTensors = tp.Dict[str, ConditionType]
+CFGConditions = tp.Union[ConditionTensors, tp.Tuple[ConditionTensors, ConditionTensors]]
+
+
+def get_init_fn(method: str, input_dim: int, init_depth: tp.Optional[int] = None):
+    """LM layer initialization.
+    Inspired from xlformers: https://github.com/fairinternal/xlformers
+
+    Args:
+        method (str): Method name for init function. Valid options are:
+            'gaussian', 'uniform'.
+        input_dim (int): Input dimension of the initialized module.
+        init_depth (int, optional): Optional init depth value used to rescale
+            the standard deviation if defined.
+    """
+    # Compute std
+    std = 1 / math.sqrt(input_dim)
+    # Rescale with depth
+    if init_depth is not None:
+        std = std / math.sqrt(2 * init_depth)
+
+    if method == 'gaussian':
+        return partial(
+            torch.nn.init.trunc_normal_, mean=0.0, std=std, a=-3 * std, b=3 * std
+        )
+    elif method == 'uniform':
+        bound = math.sqrt(3) * std  # ensure the standard deviation is `std`
+        return partial(torch.nn.init.uniform_, a=-bound, b=bound)
+    else:
+        raise ValueError("Unsupported layer initialization method")
+
+
+def init_layer(m: nn.Module,
+               method: str,
+               init_depth: tp.Optional[int] = None,
+               zero_bias_init: bool = False):
+    """Wrapper around ``get_init_fn`` for proper initialization of LM modules.
+
+    Args:
+        m (nn.Module): Module to initialize.
+        method (str): Method name for the init function.
+        init_depth (int, optional): Optional init depth value used to rescale
+            the standard deviation if defined.
+        zero_bias_init (bool): Whether to initialize the bias to 0 or not.
+    """
+    if isinstance(m, nn.Linear):
+        init_fn = get_init_fn(method, m.in_features, init_depth=init_depth)
+        if m.weight.device.type == 'cpu' and m.weight.dtype == torch.float16:
+            weight = m.weight.float()
+            init_fn(weight)
+            m.weight.data[:] = weight.half()
+        else:
+            init_fn(m.weight)
+        if zero_bias_init and m.bias is not None:
+            nn.init.constant_(m.bias, 0)
+    elif isinstance(m, nn.Embedding):
+        init_fn = get_init_fn(method, m.embedding_dim, init_depth=None)
+        if m.weight.device.type == 'cpu' and m.weight.dtype == torch.float16:
+            weight = m.weight.float()
+            init_fn(weight)
+            m.weight.data[:] = weight.half()
+        else:
+            init_fn(m.weight)
+
+
+class ScaledEmbedding(nn.Embedding):
+    """Boost learning rate for embeddings (with `scale`).
+    """
+    def __init__(self, *args, lr=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.lr = lr
+
+    def make_optim_group(self):
+        group = {"params": list(self.parameters())}
+        if self.lr is not None:
+            group["lr"] = self.lr
+        return group
+
+
+@dataclass
+class LMOutput:
+    # The logits are already re-aligned with the input codes
+    # hence no extra shift is required, e.g. when computing CE
+    logits: torch.Tensor  # [B, K, T, card]
+    mask: torch.Tensor  # [B, K, T]
+
+
+class LMModel(StreamingModule):
+    """Transformer-based language model on multiple streams of codes.
+
+    Args:
+        pattern_provider (CodebooksPatternProvider): Pattern provider for codebook interleaving.
+        condition_provider (MusicConditioningProvider): Conditioning provider from metadata.
+        fuser (ConditionFuser): Fuser handling the fusing of conditions with language model input.
+        n_q (int): Number of parallel streams to model.
+        card (int): Cardinality, vocabulary size.
+        dim (int): Dimension of the transformer encoder.
+        num_heads (int): Number of heads for the transformer encoder.
+        hidden_scale (int): Scale for hidden feed forward dimension of the transformer encoder.
+        norm (str): Normalization method.
+        norm_first (bool): Use pre-norm instead of post-norm.
+        emb_lr (float, optional): Embedding-specific learning rate.
+        bias_proj (bool): Use bias for output projections.
+        weight_init (str, optional): Method for weight initialization.
+        depthwise_init (str, optional): Method for depthwise weight initialization.
+        zero_bias_init (bool): If true and bias in Linears, initialize bias to zeros.
+        cfg_dropout (float): Classifier-free guidance dropout.
+        cfg_coef (float): Classifier-free guidance coefficient.
+        attribute_dropout (dict): Attribute dropout probabilities.
+        two_step_cfg (bool): Whether to run classifier free-guidance with 2 distinct steps.
+        **kwargs: Additional parameters for the transformer encoder.
+    """
+    def __init__(self, pattern_provider: CodebooksPatternProvider, condition_provider: ConditioningProvider,
+                 fuser: ConditionFuser, n_q: int = 8, card: int = 1024, dim: int = 128, num_heads: int = 8,
+                 hidden_scale: int = 4, norm: str = 'layer_norm', norm_first: bool = False,
+                 emb_lr: tp.Optional[float] = None, bias_proj: bool = True,
+                 weight_init: tp.Optional[str] = None, depthwise_init: tp.Optional[str] = None,
+                 zero_bias_init: bool = False, cfg_dropout: float = 0, cfg_coef: float = 1.0,
+                 attribute_dropout: tp.Dict[str, tp.Dict[str, float]] = {}, two_step_cfg: bool = False,
+                 **kwargs):
+        super().__init__()
+        self.cfg_coef = cfg_coef
+        self.cfg_dropout = ClassifierFreeGuidanceDropout(p=cfg_dropout)
+        self.att_dropout = AttributeDropout(p=attribute_dropout)
+        self.condition_provider = condition_provider
+        self.fuser = fuser
+        self.card = card
+        embed_dim = self.card + 1
+        self.n_q = n_q
+        self.dim = dim
+        self.pattern_provider = pattern_provider
+        self.two_step_cfg = two_step_cfg
+        self.emb = nn.ModuleList([ScaledEmbedding(embed_dim, dim, lr=emb_lr) for _ in range(n_q)])
+        if 'activation' in kwargs:
+            kwargs['activation'] = get_activation_fn(kwargs['activation'])
+        self.transformer = StreamingTransformer(
+            d_model=dim, num_heads=num_heads, dim_feedforward=int(hidden_scale * dim),
+            norm=norm, norm_first=norm_first, **kwargs)
+        self.out_norm: tp.Optional[nn.Module] = None
+        if norm_first:
+            self.out_norm = create_norm_fn(norm, dim)
+        self.linears = nn.ModuleList([nn.Linear(dim, self.card, bias=bias_proj) for _ in range(n_q)])
+        self._init_weights(weight_init, depthwise_init, zero_bias_init)
+        self._fsdp: tp.Optional[nn.Module]
+        self.__dict__['_fsdp'] = None
+
+    def _init_weights(self, weight_init: tp.Optional[str], depthwise_init: tp.Optional[str], zero_bias_init: bool):
+        """Initialization of the transformer module weights.
+
+        Args:
+            weight_init (str, optional): Weight initialization strategy. See ``get_init_fn`` for valid options.
+            depthwise_init (str, optional): Depthwise initialization strategy. The following options are valid:
+                'current' where the depth corresponds to the current layer index or 'global' where the total number
+                of layer is used as depth. If not set, no depthwise initialization strategy is used.
+            zero_bias_init (bool): Whether to initialize bias to zero or not.
+        """
+        assert depthwise_init is None or depthwise_init in ['current', 'global']
+        assert depthwise_init is None or weight_init is not None, \
+            "If 'depthwise_init' is defined, a 'weight_init' method should be provided."
+        assert not zero_bias_init or weight_init is not None, \
+            "If 'zero_bias_init', a 'weight_init' method should be provided"
+
+        if weight_init is None:
+            return
+
+        for emb_layer in self.emb:
+            init_layer(emb_layer, method=weight_init, init_depth=None, zero_bias_init=zero_bias_init)
+
+        for layer_idx, tr_layer in enumerate(self.transformer.layers):
+            depth = None
+            if depthwise_init == 'current':
+                depth = layer_idx + 1
+            elif depthwise_init == 'global':
+                depth = len(self.transformer.layers)
+            init_fn = partial(init_layer, method=weight_init, init_depth=depth, zero_bias_init=zero_bias_init)
+            tr_layer.apply(init_fn)
+
+        for linear in self.linears:
+            init_layer(linear, method=weight_init, init_depth=None, zero_bias_init=zero_bias_init)
+
+    @property
+    def special_token_id(self) -> int:
+        return self.card
+
+    @property
+    def num_codebooks(self) -> int:
+        return self.n_q
+
+    def forward(self, sequence: torch.Tensor,
+                conditions: tp.List[ConditioningAttributes],
+                condition_tensors: tp.Optional[ConditionTensors] = None) -> torch.Tensor:
+        """Apply language model on sequence and conditions.
+        Given a tensor of sequence of shape [B, K, S] with K the number of codebooks and
+        S the sequence steps, return the logits with shape [B, card, K, S].
+
+        Args:
+            indices (torch.Tensor): Indices of the codes to model.
+            conditions (list of ConditioningAttributes): Conditions to use when modeling
+                the given codes. Note that when evaluating multiple time with the same conditioning
+                you should pre-compute those and pass them as `condition_tensors`.
+            condition_tensors (dict[str, ConditionType], optional): Pre-computed conditioning
+                tensors, see `conditions`.
+        Returns:
+            torch.Tensor: Logits.
+        """
+        B, K, S = sequence.shape
+        #assert K == self.num_codebooks, "Sequence shape must match the specified number of codebooks"
+        input_ = sum([self.emb[k](sequence[:, k]) for k in range(K)]) # [B, K, S] -> [B, K, S, dim] -(sum)> [B, S, dim]
+        if condition_tensors is None:
+            assert not self._is_streaming, "Conditions tensors should be precomputed when streaming."
+            # apply dropout modules
+            conditions = self.cfg_dropout(conditions)
+            conditions = self.att_dropout(conditions)
+            tokenized = self.condition_provider.tokenize(conditions)
+            # encode conditions and fuse, both have a streaming cache to not recompute when generating.
+            condition_tensors = self.condition_provider(tokenized)
+        else:
+            assert not conditions, "Shouldn't pass both conditions and condition_tensors."
+
+        # input_, cross_attention_input = self.fuser(input_, condition_tensors)
+        input_, in_attn_input, cross_attention_input = self.fuser(input_, condition_tensors)
+
+        # out = self.transformer(input_, cross_attention_src=cross_attention_input)
+        out = self.transformer(input_, in_attn_src=in_attn_input, cross_attention_src=cross_attention_input)
+        if self.out_norm:
+            out = self.out_norm(out)
+        logits = torch.stack([self.linears[k](out) for k in range(K)], dim=1)  # [B, K, S, card]
+
+        # remove the prefix from the model outputs
+        if len(self.fuser.fuse2cond['prepend']) > 0:
+            logits = logits[:, :, -S:]
+
+        return logits  # [B, K, S, card]
+
+    def compute_predictions(
+            self, codes: torch.Tensor,
+            conditions: tp.List[ConditioningAttributes],
+            condition_tensors: tp.Optional[ConditionTensors] = None) -> LMOutput:
+        """Given an input tensor of codes [B, K, T] and list of conditions, runs the model
+        forward using the specified codes interleaving pattern.
+
+        Args:
+            codes (torch.Tensor): Input codes of shape [B, K, T] with B the batch size,
+                K the number of codebooks and T the number of timesteps.
+            conditions (list of ConditioningAttributes): conditionings to use when modeling
+                the given codes. Note that when evaluating multiple time with the same conditioning
+                you should pre-compute those and pass them as `condition_tensors`.
+            condition_tensors (dict[str, ConditionType], optional): pre-computed conditioning
+                tensors, see `conditions`.
+        Returns:
+            LMOutput: Language model outputs
+                logits (torch.Tensor) of shape [B, K, T, card] corresponding to the provided codes,
+                    i.e. the first item corresponds to logits to predict the first code, meaning that
+                    no additional shifting of codes and logits is required.
+                mask (torch.Tensor) of shape [B, K, T], mask over valid and invalid positions.
+                    Given the specified interleaving strategies, parts of the logits and codes should
+                    not be considered as valid predictions because of invalid context.
+        """
+        B, K, T = codes.shape
+        codes = codes.contiguous()
+        # map codes [B, K, T] into pattern sequence [B, K, S] using special_token_id for masked tokens
+        pattern = self.pattern_provider.get_pattern(T)
+        sequence_codes, sequence_indexes, sequence_mask = pattern.build_pattern_sequence(
+            codes, self.special_token_id, keep_only_valid_steps=True
+        )
+        # apply model on pattern sequence
+        model = self if self._fsdp is None else self._fsdp
+        logits = model(sequence_codes, conditions, condition_tensors)  # [B, K, S, card]
+        # map back the logits on pattern sequence to logits on original codes: [B, K, S, card] -> [B, K, T, card]
+        # and provide the corresponding mask over invalid positions of tokens
+        logits = logits.permute(0, 3, 1, 2)  # [B, card, K, S]
+        # note: we use nans as special token to make it obvious if we feed unexpected logits
+        logits, logits_indexes, logits_mask = pattern.revert_pattern_logits(
+            logits, float('nan'), keep_only_valid_steps=True
+        )
+        logits = logits.permute(0, 2, 3, 1)  # [B, K, T, card]
+        logits_mask = logits_mask[None, :, :].expand(B, -1, -1)  # [K, T] -> [B, K, T]
+        return LMOutput(logits, logits_mask)
+
+    def _sample_next_token(self,
+                           sequence: torch.Tensor,
+                           cfg_conditions: CFGConditions,
+                           unconditional_state: State,
+                           use_sampling: bool = False,
+                           temp: float = 1.0,
+                           top_k: int = 0,
+                           top_p: float = 0.0,
+                           cfg_coef: tp.Optional[float] = None) -> torch.Tensor:
+        """Sample next token from the model given a sequence and a set of conditions. The model supports
+        multiple sampling strategies (greedy sampling, softmax, top-k, top-p...).
+
+        Args:
+            sequence (torch.Tensor): Current sequence of shape [B, K, S]
+                with K corresponding to the number of codebooks and S the number of sequence steps.
+                S = 1 in streaming mode, except for the first step that contains a bigger prompt.
+            condition_tensors (dict[str, ConditionType): Set of conditions. If CFG is used,
+                should be twice the batch size, being the concatenation of the conditions + null conditions.
+            use_sampling (bool): Whether to use a sampling strategy or not.
+            temp (float): Sampling temperature.
+            top_k (int): K for "top-k" sampling.
+            top_p (float): P for "top-p" sampling.
+            cfg_coef (float, optional): classifier free guidance coefficient
+        Returns:
+            next_token (torch.Tensor): Next token tensor of shape [B, K, 1].
+        """
+        B = sequence.shape[0]
+        cfg_coef = self.cfg_coef if cfg_coef is None else cfg_coef
+        model = self if self._fsdp is None else self._fsdp
+        if self.two_step_cfg and cfg_conditions != {}:
+            assert isinstance(cfg_conditions, tuple), type(cfg_conditions)
+            condition_tensors, null_condition_tensors = cfg_conditions
+            cond_logits = model(sequence, conditions=[], condition_tensors=condition_tensors)
+            state = self.get_streaming_state()
+            self.set_streaming_state(unconditional_state)
+            uncond_logits = model(sequence, conditions=[], condition_tensors=null_condition_tensors)
+            unconditional_state.update(self.get_streaming_state())
+            self.set_streaming_state(state)
+            logits = uncond_logits + (cond_logits - uncond_logits) * self.cfg_coef
+        else:
+            assert isinstance(cfg_conditions, dict)
+            condition_tensors = cfg_conditions
+            if condition_tensors:
+                # Preparing for CFG, predicting both conditional and unconditional logits.
+                sequence = torch.cat([sequence, sequence], dim=0)
+            all_logits = model(
+                sequence,
+                conditions=[], condition_tensors=condition_tensors)
+            if condition_tensors:
+                cond_logits, uncond_logits = all_logits.split(B, dim=0)  # [B, K, T, card]
+                logits = uncond_logits + (cond_logits - uncond_logits) * cfg_coef
+            else:
+                logits = all_logits
+
+        logits = logits.permute(0, 1, 3, 2)  # [B, K, card, T]
+        logits = logits[..., -1]  # [B x K x card]
+
+        # Apply softmax for sampling if temp > 0. Else, do greedy sampling to avoid zero division error.
+        if use_sampling and temp > 0.0:
+            probs = torch.softmax(logits / temp, dim=-1)
+            if top_p > 0.0:
+                next_token = utils.sample_top_p(probs, p=top_p)
+            elif top_k > 0:
+                next_token = utils.sample_top_k(probs, k=top_k)
+            else:
+                next_token = utils.multinomial(probs, num_samples=1)
+        else:
+            next_token = torch.argmax(logits, dim=-1, keepdim=True)
+
+        return next_token
+
+    @torch.no_grad()
+    def generate(self,
+                 prompt: tp.Optional[torch.Tensor] = None,
+                 conditions: tp.List[ConditioningAttributes] = [],
+                 num_samples: tp.Optional[int] = None,
+                 max_gen_len: int = 256,
+                 use_sampling: bool = True,
+                 temp: float = 1.0,
+                 top_k: int = 250,
+                 top_p: float = 0.0,
+                 cfg_coef: tp.Optional[float] = None,
+                 two_step_cfg: tp.Optional[bool] = None,
+                 remove_prompts: bool = False,
+                 check: bool = False,
+                 callback: tp.Optional[tp.Callable[[int, int], None]] = None) -> torch.Tensor:
+        """Generate tokens sampling from the model given a prompt or unconditionally. Generation can
+        be perform in a greedy fashion or using sampling with top K and top P strategies.
+
+        Args:
+            prompt (torch.Tensor, optional): Prompt tokens of shape [B, K, T].
+            conditions_tensors (list of ConditioningAttributes, optional): List of conditions.
+            num_samples (int, optional): Number of samples to generate when no prompt and no conditions are given.
+            max_gen_len (int): Maximum generation length.
+            use_sampling (bool): Whether to use a sampling strategy or not.
+            temp (float): Sampling temperature.
+            top_k (int): K for "top-k" sampling.
+            top_p (float): P for "top-p" sampling.
+            cfg_coeff (float, optional): Classifier-free guidance coefficient.
+            two_step_cfg (bool, optional): Whether to perform classifier-free guidance with two steps generation.
+            remove_prompts (bool): Whether to remove prompts from generation or not.
+            check (bool): Whether to apply further checks on generated sequence.
+            callback (Callback, optional): Callback function to report generation progress.
+        Returns:
+            torch.Tensor: Generated tokens.
+        """
+        assert not self.training, "generation shouldn't be used in training mode."
+        first_param = next(iter(self.parameters()))
+        device = first_param.device
+
+        # Checking all input shapes are consistent.
+        possible_num_samples = []
+        if num_samples is not None:
+            possible_num_samples.append(num_samples)
+        elif prompt is not None:
+            possible_num_samples.append(prompt.shape[0])
+        elif conditions:
+            possible_num_samples.append(len(conditions))
+        else:
+            possible_num_samples.append(1)
+        assert [x == possible_num_samples[0] for x in possible_num_samples], "Inconsistent inputs shapes"
+        num_samples = possible_num_samples[0]
+
+        # below we create set of conditions: one conditional and one unconditional
+        # to do that we merge the regular condition together with the null condition
+        # we then do 1 forward pass instead of 2.
+        # the reason for that is two-fold:
+        # 1. it is about x2 faster than doing 2 forward passes
+        # 2. avoid the streaming API treating the 2 passes as part of different time steps
+        # We also support doing two different passes, in particular to ensure that
+        # the padding structure is exactly the same between train and test.
+        # With a batch size of 1, this can be slower though.
+        cfg_conditions: CFGConditions
+        two_step_cfg = self.two_step_cfg if two_step_cfg is None else two_step_cfg
+        if conditions:
+            null_conditions = ClassifierFreeGuidanceDropout(p=1.0)(conditions)
+            if two_step_cfg:
+                cfg_conditions = (
+                    self.condition_provider(self.condition_provider.tokenize(conditions)),
+                    self.condition_provider(self.condition_provider.tokenize(null_conditions)),
+                )
+            else:
+                conditions = conditions + null_conditions
+                tokenized = self.condition_provider.tokenize(conditions)
+                cfg_conditions = self.condition_provider(tokenized)
+        else:
+            cfg_conditions = {}
+
+        if prompt is None:
+            assert num_samples > 0
+            prompt = torch.zeros((num_samples, self.num_codebooks, 0), dtype=torch.long, device=device)
+
+        B, K, T = prompt.shape
+        start_offset = T
+        assert start_offset < max_gen_len
+
+        pattern = self.pattern_provider.get_pattern(max_gen_len)
+        # this token is used as default value for codes that are not generated yet
+        unknown_token = -1
+
+        # we generate codes up to the max_gen_len that will be mapped to the pattern sequence
+        gen_codes = torch.full((B, K, max_gen_len), unknown_token, dtype=torch.long, device=device)
+        # filling the gen_codes with the prompt if needed
+        gen_codes[..., :start_offset] = prompt
+        # create the gen_sequence with proper interleaving from the pattern: [B, K, S]
+        gen_sequence, indexes, mask = pattern.build_pattern_sequence(gen_codes, self.special_token_id)
+        # retrieve the start_offset in the sequence:
+        # it is the first sequence step that contains the `start_offset` timestep
+        start_offset_sequence = pattern.get_first_step_with_timesteps(start_offset)
+        assert start_offset_sequence is not None
+
+        with self.streaming():
+            unconditional_state = self.get_streaming_state()
+            prev_offset = 0
+            gen_sequence_len = gen_sequence.shape[-1]  # gen_sequence shape is [B, K, S]
+            for offset in range(start_offset_sequence, gen_sequence_len):
+                # get current sequence (note that the streaming API is providing the caching over previous offsets)
+                curr_sequence = gen_sequence[..., prev_offset:offset]
+                curr_mask = mask[None, ..., prev_offset:offset].expand(B, -1, -1)
+                if check:
+                    # check coherence between mask and sequence
+                    assert (curr_sequence == torch.where(curr_mask, curr_sequence, self.special_token_id)).all()
+                    # should never happen as gen_sequence is filled progressively
+                    assert not (curr_sequence == unknown_token).any()
+                # sample next token from the model, next token shape is [B, K, 1]
+                next_token = self._sample_next_token(
+                    curr_sequence, cfg_conditions, unconditional_state, use_sampling, temp, top_k, top_p,
+                    cfg_coef=cfg_coef)
+                # ensure the tokens that should be masked are properly set to special_token_id
+                # as the model never output special_token_id
+                valid_mask = mask[..., offset:offset+1].expand(B, -1, -1)
+                next_token[~valid_mask] = self.special_token_id
+                # ensure we don't overwrite prompt tokens, we only write over unknown tokens
+                # (then mask tokens should be left as is as well, which is correct)
+                gen_sequence[..., offset:offset+1] = torch.where(
+                    gen_sequence[..., offset:offset+1] == unknown_token,
+                    next_token, gen_sequence[..., offset:offset+1]
+                )
+                prev_offset = offset
+                if callback is not None:
+                    callback(1 + offset - start_offset_sequence, gen_sequence_len - start_offset_sequence)
+        unconditional_state.clear()
+
+        # ensure sequence has been entirely filled
+        assert not (gen_sequence == unknown_token).any()
+        # ensure gen_sequence pattern and mask are matching
+        # which means the gen_sequence is valid according to the pattern
+        assert (
+            gen_sequence == torch.where(mask[None, ...].expand(B, -1, -1), gen_sequence, self.special_token_id)
+        ).all()
+        # get back the codes, trimming the prompt if needed and cutting potentially incomplete timesteps
+        out_codes, out_indexes, out_mask = pattern.revert_pattern_sequence(gen_sequence, special_token=unknown_token)
+
+        # sanity checks over the returned codes and corresponding masks
+        assert (out_codes[..., :max_gen_len] != unknown_token).all()
+        assert (out_mask[..., :max_gen_len] == 1).all()
+
+        out_start_offset = start_offset if remove_prompts else 0
+        out_codes = out_codes[..., out_start_offset:max_gen_len]
+
+        # ensure the returned codes are all valid
+        assert (out_codes >= 0).all() and (out_codes <= self.card).all()
+        return out_codes
diff --git a/audiocraft/audiocraft/models/loaders.py b/audiocraft/audiocraft/models/loaders.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c7808a0588bd1a8084157b072bae42aa7efaf84
--- /dev/null
+++ b/audiocraft/audiocraft/models/loaders.py
@@ -0,0 +1,141 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Utility functions to load from the checkpoints.
+Each checkpoint is a torch.saved dict with the following keys:
+- 'xp.cfg': the hydra config as dumped during training. This should be used
+    to rebuild the object using the audiocraft.models.builders functions,
+- 'model_best_state': a readily loadable best state for the model, including
+    the conditioner. The model obtained from `xp.cfg` should be compatible
+    with this state dict. In the case of a LM, the encodec model would not be
+    bundled along but instead provided separately.
+
+Those functions also support loading from a remote location with the Torch Hub API.
+They also support overriding some parameters, in particular the device and dtype
+of the returned model.
+"""
+
+from pathlib import Path
+from huggingface_hub import hf_hub_download
+import typing as tp
+import os
+
+from omegaconf import OmegaConf, DictConfig
+import torch
+
+from . import builders
+from .encodec import CompressionModel
+
+
+def get_audiocraft_cache_dir() -> tp.Optional[str]:
+    return os.environ.get('AUDIOCRAFT_CACHE_DIR', None)
+
+
+def _get_state_dict(
+    file_or_url_or_id: tp.Union[Path, str],
+    filename: tp.Optional[str] = None,
+    device='cpu',
+    cache_dir: tp.Optional[str] = None,
+):
+    if cache_dir is None:
+        cache_dir = get_audiocraft_cache_dir()
+    # Return the state dict either from a file or url
+    file_or_url_or_id = str(file_or_url_or_id)
+    assert isinstance(file_or_url_or_id, str)
+
+    if os.path.isfile(file_or_url_or_id):
+        return torch.load(file_or_url_or_id, map_location=device)
+
+    if os.path.isdir(file_or_url_or_id):
+        file = f"{file_or_url_or_id}/{filename}"
+        return torch.load(file, map_location=device)
+
+    elif file_or_url_or_id.startswith('https://'):
+        return torch.hub.load_state_dict_from_url(file_or_url_or_id, map_location=device, check_hash=True)
+
+    else:
+        assert filename is not None, "filename needs to be defined if using HF checkpoints"
+
+        file = hf_hub_download(repo_id=file_or_url_or_id, filename=filename, cache_dir=cache_dir)
+        return torch.load(file, map_location=device)
+
+
+def load_compression_model_ckpt(file_or_url_or_id: tp.Union[Path, str], cache_dir: tp.Optional[str] = None):
+    return _get_state_dict(file_or_url_or_id, filename="compression_state_dict.bin", cache_dir=cache_dir)
+
+
+def load_compression_model(file_or_url_or_id: tp.Union[Path, str], device='cpu', cache_dir: tp.Optional[str] = None):
+    pkg = load_compression_model_ckpt(file_or_url_or_id, cache_dir=cache_dir)
+    if 'pretrained' in pkg:
+        return CompressionModel.get_pretrained(pkg['pretrained'], device=device)
+    cfg = OmegaConf.create(pkg['xp.cfg'])
+    cfg.device = str(device)
+    model = builders.get_compression_model(cfg)
+    model.load_state_dict(pkg['best_state'])
+    model.eval()
+    return model
+
+
+def load_lm_model_ckpt(file_or_url_or_id: tp.Union[Path, str], cache_dir: tp.Optional[str] = None):
+    return _get_state_dict(file_or_url_or_id, filename="state_dict.bin", cache_dir=cache_dir)
+
+
+def _delete_param(cfg: DictConfig, full_name: str):
+    parts = full_name.split('.')
+    for part in parts[:-1]:
+        if part in cfg:
+            cfg = cfg[part]
+        else:
+            return
+    OmegaConf.set_struct(cfg, False)
+    if parts[-1] in cfg:
+        del cfg[parts[-1]]
+    OmegaConf.set_struct(cfg, True)
+
+
+def load_lm_model(file_or_url_or_id: tp.Union[Path, str], device='cpu', cache_dir: tp.Optional[str] = None):
+    pkg = load_lm_model_ckpt(file_or_url_or_id, cache_dir=cache_dir)
+    cfg = OmegaConf.create(pkg['xp.cfg'])
+    cfg.device = str(device)
+    if cfg.device == 'cpu':
+        cfg.dtype = 'float32'
+    else:
+        cfg.dtype = 'float16'
+    _delete_param(cfg, 'conditioners.self_wav.chroma_stem.cache_path')
+    _delete_param(cfg, 'conditioners.args.merge_text_conditions_p')
+    _delete_param(cfg, 'conditioners.args.drop_desc_p')
+    model = builders.get_lm_model(cfg)
+    model.load_state_dict(pkg['best_state'])
+    model.eval()
+    model.cfg = cfg
+    return model
+
+
+def load_mbd_ckpt(file_or_url_or_id: tp.Union[Path, str], cache_dir: tp.Optional[str] = None):
+    return _get_state_dict(file_or_url_or_id, filename="all_in_one.pt", cache_dir=cache_dir)
+
+
+def load_diffusion_models(file_or_url_or_id: tp.Union[Path, str], device='cpu', cache_dir: tp.Optional[str] = None):
+    pkg = load_mbd_ckpt(file_or_url_or_id, cache_dir=cache_dir)
+    models = []
+    processors = []
+    cfgs = []
+    sample_rate = pkg['sample_rate']
+    for i in range(pkg['n_bands']):
+        cfg = pkg[i]['cfg']
+        model = builders.get_diffusion_model(cfg)
+        model_dict = pkg[i]['model_state']
+        model.load_state_dict(model_dict)
+        model.to(device)
+        processor = builders.get_processor(cfg=cfg.processor, sample_rate=sample_rate)
+        processor_dict = pkg[i]['processor_state']
+        processor.load_state_dict(processor_dict)
+        processor.to(device)
+        models.append(model)
+        processors.append(processor)
+        cfgs.append(cfg)
+    return models, processors, cfgs
diff --git a/audiocraft/audiocraft/models/multibanddiffusion.py b/audiocraft/audiocraft/models/multibanddiffusion.py
new file mode 100644
index 0000000000000000000000000000000000000000..1121d2fc660ab2ceed7deaaf87edba5337ab5472
--- /dev/null
+++ b/audiocraft/audiocraft/models/multibanddiffusion.py
@@ -0,0 +1,194 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Multi Band Diffusion models as described in
+"From Discrete Tokens to High-Fidelity Audio Using Multi-Band Diffusion"
+(paper link).
+"""
+
+import typing as tp
+
+import torch
+import julius
+
+from .unet import DiffusionUnet
+from ..modules.diffusion_schedule import NoiseSchedule
+from .encodec import CompressionModel
+from ..solvers.compression import CompressionSolver
+from .loaders import load_compression_model, load_diffusion_models
+
+
+class DiffusionProcess:
+    """Sampling for a diffusion Model.
+
+    Args:
+        model (DiffusionUnet): Diffusion U-Net model.
+        noise_schedule (NoiseSchedule): Noise schedule for diffusion process.
+    """
+    def __init__(self, model: DiffusionUnet, noise_schedule: NoiseSchedule) -> None:
+        """
+        """
+        self.model = model
+        self.schedule = noise_schedule
+
+    def generate(self, condition: torch.Tensor, initial_noise: torch.Tensor,
+                 step_list: tp.Optional[tp.List[int]] = None):
+        """Perform one diffusion process to generate one of the bands.
+
+        Args:
+            condition (tensor): The embeddings form the compression model.
+            initial_noise (tensor): The initial noise to start the process/
+        """
+        return self.schedule.generate_subsampled(model=self.model, initial=initial_noise, step_list=step_list,
+                                                 condition=condition)
+
+
+class MultiBandDiffusion:
+    """Sample from multiple diffusion models.
+
+    Args:
+        DPs (list of DiffusionProcess): Diffusion processes.
+        codec_model (CompressionModel): Underlying compression model used to obtain discrete tokens.
+    """
+    def __init__(self, DPs: tp.List[DiffusionProcess], codec_model: CompressionModel) -> None:
+        self.DPs = DPs
+        self.codec_model = codec_model
+        self.device = next(self.codec_model.parameters()).device
+
+    @property
+    def sample_rate(self) -> int:
+        return self.codec_model.sample_rate
+
+    @staticmethod
+    def get_mbd_musicgen(device=None):
+        """Load our diffusion models trained for MusicGen."""
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        path = 'https://dl.fbaipublicfiles.com/encodec/Diffusion/mbd_musicgen_32khz.th'
+        name = 'facebook/musicgen-small'
+        codec_model = load_compression_model(name, device=device)
+        models, processors, cfgs = load_diffusion_models(path, device=device)
+        DPs = []
+        for i in range(len(models)):
+            schedule = NoiseSchedule(**cfgs[i].schedule, sample_processor=processors[i], device=device)
+            DPs.append(DiffusionProcess(model=models[i], noise_schedule=schedule))
+        return MultiBandDiffusion(DPs=DPs, codec_model=codec_model)
+
+    @staticmethod
+    def get_mbd_24khz(bw: float = 3.0, pretrained: bool = True,
+                      device: tp.Optional[tp.Union[torch.device, str]] = None,
+                      n_q: tp.Optional[int] = None):
+        """Get the pretrained Models for MultibandDiffusion.
+
+        Args:
+            bw (float): Bandwidth of the compression model.
+            pretrained (bool): Whether to use / download if necessary the models.
+            device (torch.device or str, optional): Device on which the models are loaded.
+            n_q (int, optional): Number of quantizers to use within the compression model.
+        """
+        if device is None:
+            device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        assert bw in [1.5, 3.0, 6.0], f"bandwidth {bw} not available"
+        if n_q is not None:
+            assert n_q in [2, 4, 8]
+            assert {1.5: 2, 3.0: 4, 6.0: 8}[bw] == n_q, \
+                f"bandwidth and number of codebooks missmatch to use n_q = {n_q} bw should be {n_q * (1.5 / 2)}"
+        n_q = {1.5: 2, 3.0: 4, 6.0: 8}[bw]
+        codec_model = CompressionSolver.model_from_checkpoint(
+            '//pretrained/facebook/encodec_24khz', device=device)
+        codec_model.set_num_codebooks(n_q)
+        codec_model = codec_model.to(device)
+        path = f'https://dl.fbaipublicfiles.com/encodec/Diffusion/mbd_comp_{n_q}.pt'
+        models, processors, cfgs = load_diffusion_models(path, device=device)
+        DPs = []
+        for i in range(len(models)):
+            schedule = NoiseSchedule(**cfgs[i].schedule, sample_processor=processors[i], device=device)
+            DPs.append(DiffusionProcess(model=models[i], noise_schedule=schedule))
+        return MultiBandDiffusion(DPs=DPs, codec_model=codec_model)
+
+        return MultiBandDiffusion(DPs, codec_model)
+
+    @torch.no_grad()
+    def get_condition(self, wav: torch.Tensor, sample_rate: int) -> torch.Tensor:
+        """Get the conditioning (i.e. latent reprentatios of the compression model) from a waveform.
+        Args:
+            wav (torch.Tensor): The audio that we want to extract the conditioning from
+            sample_rate (int): sample rate of the audio"""
+        if sample_rate != self.sample_rate:
+            wav = julius.resample_frac(wav, sample_rate, self.sample_rate)
+        codes, scale = self.codec_model.encode(wav)
+        assert scale is None, "Scaled compression models not supported."
+        emb = self.get_emb(codes)
+        return emb
+
+    @torch.no_grad()
+    def get_emb(self, codes: torch.Tensor):
+        """Get latent representation from the discrete codes
+        Argrs:
+            codes (torch.Tensor): discrete tokens"""
+        emb = self.codec_model.decode_latent(codes)
+        return emb
+
+    def generate(self, emb: torch.Tensor, size: tp.Optional[torch.Size] = None,
+                 step_list: tp.Optional[tp.List[int]] = None):
+        """Generate Wavform audio from the latent embeddings of the compression model
+        Args:
+            emb (torch.Tensor): Conditioning embeddinds
+            size (none torch.Size): size of the output
+                if None this is computed from the typical upsampling of the model
+            step_list (optional list[int]): list of Markov chain steps, defaults to 50 linearly spaced step.
+        """
+        if size is None:
+            upsampling = int(self.codec_model.sample_rate / self.codec_model.frame_rate)
+            size = torch.Size([emb.size(0), self.codec_model.channels, emb.size(-1) * upsampling])
+        assert size[0] == emb.size(0)
+        out = torch.zeros(size).to(self.device)
+        for DP in self.DPs:
+            out += DP.generate(condition=emb, step_list=step_list, initial_noise=torch.randn_like(out))
+        return out
+
+    def re_eq(self, wav: torch.Tensor, ref: torch.Tensor, n_bands: int = 32, strictness: float = 1):
+        """match the eq to the encodec output by matching the standard deviation of some frequency bands
+        Args:
+            wav (torch.Tensor): audio to equalize
+            ref (torch.Tensor):refenrence audio from which we match the spectrogram.
+            n_bands (int): number of bands of the eq
+            strictness (float): how strict the the matching. 0 is no matching, 1 is exact matching.
+        """
+        split = julius.SplitBands(n_bands=n_bands, sample_rate=self.codec_model.sample_rate).to(wav.device)
+        bands = split(wav)
+        bands_ref = split(ref)
+        out = torch.zeros_like(ref)
+        for i in range(n_bands):
+            out += bands[i] * (bands_ref[i].std() / bands[i].std()) ** strictness
+        return out
+
+    def regenerate(self, wav: torch.Tensor, sample_rate: int):
+        """Regenerate a wavform through compression and diffusion regeneration.
+        Args:
+            wav (torch.Tensor): Original 'ground truth' audio
+            sample_rate (int): sample rate of the input (and output) wav
+        """
+        if sample_rate != self.codec_model.sample_rate:
+            wav = julius.resample_frac(wav, sample_rate, self.codec_model.sample_rate)
+        emb = self.get_condition(wav, sample_rate=self.codec_model.sample_rate)
+        size = wav.size()
+        out = self.generate(emb, size=size)
+        if sample_rate != self.codec_model.sample_rate:
+            out = julius.resample_frac(out, self.codec_model.sample_rate, sample_rate)
+        return out
+
+    def tokens_to_wav(self, tokens: torch.Tensor, n_bands: int = 32):
+        """Generate Waveform audio with diffusion from the discrete codes.
+        Args:
+            tokens (torch.Tensor): discrete codes
+            n_bands (int): bands for the eq matching.
+        """
+        wav_encodec = self.codec_model.decode(tokens)
+        condition = self.get_emb(tokens)
+        wav_diffusion = self.generate(emb=condition, size=wav_encodec.size())
+        return self.re_eq(wav=wav_diffusion, ref=wav_encodec, n_bands=n_bands)
diff --git a/audiocraft/audiocraft/models/musicgen.py b/audiocraft/audiocraft/models/musicgen.py
new file mode 100644
index 0000000000000000000000000000000000000000..e04878cc794b53e6c8f67ee9d341550ccccf0bf3
--- /dev/null
+++ b/audiocraft/audiocraft/models/musicgen.py
@@ -0,0 +1,583 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Main model for using MusicGen. This will combine all the required components
+and provide easy access to the generation API.
+"""
+
+import typing as tp
+import warnings
+
+import torch
+import numpy as np
+
+from .encodec import CompressionModel
+from .lm import LMModel
+from .builders import get_debug_compression_model, get_debug_lm_model
+from .loaders import load_compression_model, load_lm_model
+from ..data.audio_utils import convert_audio, convert_txtchord2chroma, convert_txtchord2chroma_24
+from ..modules.conditioners import ConditioningAttributes, WavCondition, ChordCondition, BeatCondition
+from ..utils.autocast import TorchAutocast
+
+
+MelodyList = tp.List[tp.Optional[torch.Tensor]]
+MelodyType = tp.Union[torch.Tensor, MelodyList]
+
+
+# backward compatible names mapping
+_HF_MODEL_CHECKPOINTS_MAP = {
+    "small": "facebook/musicgen-small",
+    "medium": "facebook/musicgen-medium",
+    "large": "facebook/musicgen-large",
+    "melody": "facebook/musicgen-melody",
+}
+
+
+class MusicGen:
+    """MusicGen main model with convenient generation API.
+
+    Args:
+        name (str): name of the model.
+        compression_model (CompressionModel): Compression model
+            used to map audio to invertible discrete representations.
+        lm (LMModel): Language model over discrete representations.
+        max_duration (float, optional): maximum duration the model can produce,
+            otherwise, inferred from the training params.
+    """
+    def __init__(self, name: str, compression_model: CompressionModel, lm: LMModel,
+                 max_duration: tp.Optional[float] = None):
+        self.name = name
+        self.compression_model = compression_model
+        self.lm = lm
+        if max_duration is None:
+            if hasattr(lm, 'cfg'):
+                max_duration = lm.cfg.dataset.segment_duration  # type: ignore
+            else:
+                raise ValueError("You must provide max_duration when building directly MusicGen")
+        assert max_duration is not None
+        self.max_duration: float = max_duration
+        self.device = next(iter(lm.parameters())).device
+        self.generation_params: dict = {}
+        self.set_generation_params(duration=6, extend_stride=3)  # 6 seconds by default
+        self._progress_callback: tp.Optional[tp.Callable[[int, int], None]] = None
+        if self.device.type == 'cpu':
+            self.autocast = TorchAutocast(enabled=False)
+        else:
+            self.autocast = TorchAutocast(
+                enabled=True, device_type=self.device.type, dtype=torch.float16)
+
+    @property
+    def frame_rate(self) -> float:
+        """Roughly the number of AR steps per seconds."""
+        return self.compression_model.frame_rate
+
+    @property
+    def sample_rate(self) -> int:
+        """Sample rate of the generated audio."""
+        return self.compression_model.sample_rate
+
+    @property
+    def audio_channels(self) -> int:
+        """Audio channels of the generated audio."""
+        return self.compression_model.channels
+
+    @staticmethod
+    def get_pretrained(name: str = 'facebook/musicgen-melody', device=None):
+        """Return pretrained model, we provide four models:
+        - facebook/musicgen-small (300M), text to music,
+          # see: https://huggingface.co/facebook/musicgen-small
+        - facebook/musicgen-medium (1.5B), text to music,
+          # see: https://huggingface.co/facebook/musicgen-medium
+        - facebook/musicgen-melody (1.5B) text to music and text+melody to music,
+          # see: https://huggingface.co/facebook/musicgen-melody
+        - facebook/musicgen-large (3.3B), text to music,
+          # see: https://huggingface.co/facebook/musicgen-large
+        """
+        if device is None:
+            if torch.cuda.device_count():
+                device = 'cuda'
+            else:
+                device = 'cpu'
+
+        if name == 'debug':
+            # used only for unit tests
+            compression_model = get_debug_compression_model(device)
+            lm = get_debug_lm_model(device)
+            return MusicGen(name, compression_model, lm, max_duration=30)
+
+        if name in _HF_MODEL_CHECKPOINTS_MAP:
+            warnings.warn(
+                "MusicGen pretrained model relying on deprecated checkpoint mapping. " +
+                f"Please use full pre-trained id instead: facebook/musicgen-{name}")
+            name = _HF_MODEL_CHECKPOINTS_MAP[name]
+
+        lm = load_lm_model(name, device=device)
+        compression_model = load_compression_model(name, device=device)
+        if 'self_wav' in lm.condition_provider.conditioners:
+            lm.condition_provider.conditioners['self_wav'].match_len_on_eval = True
+
+        return MusicGen(name, compression_model, lm)
+
+    def set_generation_params(self, use_sampling: bool = True, top_k: int = 250,
+                              top_p: float = 0.0, temperature: float = 1.0,
+                              duration: float = 30.0, cfg_coef: float = 3.0,
+                              two_step_cfg: bool = False, extend_stride: float = 18):
+        """Set the generation parameters for MusicGen.
+
+        Args:
+            use_sampling (bool, optional): Use sampling if True, else do argmax decoding. Defaults to True.
+            top_k (int, optional): top_k used for sampling. Defaults to 250.
+            top_p (float, optional): top_p used for sampling, when set to 0 top_k is used. Defaults to 0.0.
+            temperature (float, optional): Softmax temperature parameter. Defaults to 1.0.
+            duration (float, optional): Duration of the generated waveform. Defaults to 30.0.
+            cfg_coef (float, optional): Coefficient used for classifier free guidance. Defaults to 3.0.
+            two_step_cfg (bool, optional): If True, performs 2 forward for Classifier Free Guidance,
+                instead of batching together the two. This has some impact on how things
+                are padded but seems to have little impact in practice.
+            extend_stride: when doing extended generation (i.e. more than 30 seconds), by how much
+                should we extend the audio each time. Larger values will mean less context is
+                preserved, and shorter value will require extra computations.
+        """
+        assert extend_stride < self.max_duration, "Cannot stride by more than max generation duration."
+        self.extend_stride = extend_stride
+        self.duration = duration
+        self.generation_params = {
+            'use_sampling': use_sampling,
+            'temp': temperature,
+            'top_k': top_k,
+            'top_p': top_p,
+            'cfg_coef': cfg_coef,
+            'two_step_cfg': two_step_cfg,
+        }
+
+    def set_custom_progress_callback(self, progress_callback: tp.Optional[tp.Callable[[int, int], None]] = None):
+        """Override the default progress callback."""
+        self._progress_callback = progress_callback
+
+    def generate_unconditional(self, num_samples: int, progress: bool = False,
+                               return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                                                                        tp.Tuple[torch.Tensor, torch.Tensor]]:
+        """Generate samples in an unconditional manner.
+
+        Args:
+            num_samples (int): Number of samples to be generated.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+        descriptions: tp.List[tp.Optional[str]] = [None] * num_samples
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions, None)
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+
+    def generate(self, descriptions: tp.List[str], progress: bool = False, return_tokens: bool = False) \
+            -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
+        """Generate samples conditioned on text.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions, None)
+        assert prompt_tokens is None
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+
+    def generate_with_chroma(self, descriptions: tp.List[str], melody_wavs: MelodyType,
+                             melody_sample_rate: int, progress: bool = False,
+                             return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                                                                      tp.Tuple[torch.Tensor, torch.Tensor]]:
+        """Generate samples conditioned on text and melody.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            melody_wavs: (torch.Tensor or list of Tensor): A batch of waveforms used as
+                melody conditioning. Should have shape [B, C, T] with B matching the description length,
+                C=1 or 2. It can be [C, T] if there is a single description. It can also be
+                a list of [C, T] tensors.
+            melody_sample_rate: (int): Sample rate of the melody waveforms.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+        if isinstance(melody_wavs, torch.Tensor):
+            if melody_wavs.dim() == 2:
+                melody_wavs = melody_wavs[None]
+            if melody_wavs.dim() != 3:
+                raise ValueError("Melody wavs should have a shape [B, C, T].")
+            melody_wavs = list(melody_wavs)
+        else:
+            for melody in melody_wavs:
+                if melody is not None:
+                    assert melody.dim() == 2, "One melody in the list has the wrong number of dims."
+
+        melody_wavs = [
+            convert_audio(wav, melody_sample_rate, self.sample_rate, self.audio_channels)
+            if wav is not None else None
+            for wav in melody_wavs]
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions=descriptions, prompt=None,
+                                                                        melody_wavs=melody_wavs)
+        assert prompt_tokens is None
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+    
+    def generate_with_chords(self, descriptions: tp.List[str], melody_chords: tp.Optional[tp.Union[MelodyList,tp.List[str]]] = None,
+                             bpms: tp.Optional[tp.Union[float,int,tp.List[float],tp.List[int]]] = [120.], 
+                             meters: tp.Optional[tp.Union[float,int,tp.List[float],tp.List[int]]] = [4.], 
+                             progress: bool = False, return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                                                                      tp.Tuple[torch.Tensor, torch.Tensor]]:
+        """Generate samples conditioned on text and melody.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            melody_chords: (torch.Tensor or list of Tensor): A list of chords in chormagram or string type
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+
+        if isinstance(melody_chords[0], str):
+            # check the bpm, meter length
+            if len(bpms) == 1:
+                bpms *= len(melody_chords)
+            if len(meters) == 1:
+                meters *= len(melody_chords)
+            assert len(bpms) == len(melody_chords), "bpm length is not equal to chord length"
+            assert len(meters) == len(melody_chords), "meter length is not equal to chord length"
+            # convert str to chromagram
+            melody_chromas = []
+            for melody_chord, bpm, meter in zip(melody_chords, bpms, meters):
+                melody_chroma = convert_txtchord2chroma(melody_chord, bpm, meter, self.duration).permute(1,0) # [C=12, T]
+                melody_chromas.append(melody_chroma)
+            melody_chromas = torch.stack(melody_chromas, dim=0)
+            assert melody_chromas.dim() == 3
+            melody_chords = list(melody_chromas)
+        else:
+            for melody in melody_chords:
+                if melody is not None:
+                    assert melody.dim() == 2, "One melody in the list has the wrong number of dims."
+
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions=descriptions, prompt=None,
+                                                                        melody_chords=melody_chords, bpms=bpms)
+        assert prompt_tokens is None
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+    
+    def generate_with_chords_and_beats(self, descriptions: tp.List[str], melody_chords: tp.Optional[tp.Union[MelodyList,tp.List[str]]] = None,
+                             bpms: tp.Optional[tp.Union[float,int,tp.List[float],tp.List[int]]] = [120.], 
+                             meters: tp.Optional[tp.Union[float,int,tp.List[float],tp.List[int]]] = [4.], 
+                             progress: bool = False, return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                                                                      tp.Tuple[torch.Tensor, torch.Tensor]]:
+        """Generate samples conditioned on text and melody.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            melody_chords: (torch.Tensor or list of Tensor): A list of chords in chormagram or string type
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+
+        if isinstance(melody_chords[0], str):
+            # check the bpm, meter length
+            if len(bpms) == 1:
+                bpms *= len(melody_chords)
+            if len(meters) == 1:
+                meters *= len(melody_chords)
+            assert len(bpms) == len(melody_chords), "bpm length is not equal to chord length"
+            assert len(meters) == len(melody_chords), "meter length is not equal to chord length"
+            # convert str to chromagram
+            melody_chromas = []
+            for melody_chord, bpm, meter in zip(melody_chords, bpms, meters):
+                melody_chroma = convert_txtchord2chroma(melody_chord, bpm, meter, self.duration).permute(1,0) # [C=24, T]
+                melody_chromas.append(melody_chroma)
+            melody_chromas = torch.stack(melody_chromas, dim=0)
+            assert melody_chromas.dim() == 3
+            melody_chords = list(melody_chromas)
+        else:
+            for melody in melody_chords:
+                if melody is not None:
+                    assert melody.dim() == 2, "One melody in the list has the wrong number of dims."
+        
+        fs = self.sample_rate / 640
+        beats = []
+        for bpm, meter in zip(bpms, meters):
+            beat = np.zeros(int(fs * self.duration))
+            beat_gap = int(60 / bpm * fs)
+            beat[::beat_gap] = 1
+            bar = np.zeros(int(fs * self.duration))
+            bar[::beat_gap * meter] = 1
+            kernel = np.array([0.05, 0.1, 0.3, 0.9, 0.3, 0.1, 0.05])
+            beat = np.convolve(beat , kernel, 'same')
+            beat = beat + bar
+            beats.append(torch.tensor(beat).unsqueeze(0)) # [C, T]
+        beats = list(torch.stack(beats, dim=0)) # [B, C, T]
+
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions=descriptions, prompt=None,
+                                                                        melody_chords=melody_chords, beats=beats, bpms=bpms)
+        assert prompt_tokens is None
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+
+    def generate_for_eval(self, descriptions: tp.List[str], melody_chords: tp.List[torch.Tensor], beats: tp.List[torch.Tensor],
+                             bpms: tp.List[float], progress: bool = False, return_tokens: bool = False) -> tp.Union[torch.Tensor,
+                                                                      tp.Tuple[torch.Tensor, torch.Tensor]]:
+
+        # assert melody_chords.dim() == 3
+        # assert beats.dim() == 3
+
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions=descriptions, prompt=None,
+                                                                        melody_chords=melody_chords, beats=beats, bpms=bpms)
+        assert prompt_tokens is None
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+
+
+    def generate_continuation(self, prompt: torch.Tensor, prompt_sample_rate: int,
+                              descriptions: tp.Optional[tp.List[tp.Optional[str]]] = None, audio_channels=1,
+                              progress: bool = False, return_tokens: bool = False) \
+            -> tp.Union[torch.Tensor, tp.Tuple[torch.Tensor, torch.Tensor]]:
+        """Generate samples conditioned on audio prompts.
+
+        Args:
+            prompt (torch.Tensor): A batch of waveforms used for continuation.
+                Prompt should be [B, C, T], or [C, T] if only one sample is generated.
+            prompt_sample_rate (int): Sampling rate of the given audio waveforms.
+            descriptions (list of str, optional): A list of strings used as text conditioning. Defaults to None.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        """
+        if prompt.dim() == 2:
+            prompt = prompt[None]
+        if prompt.dim() != 3:
+            raise ValueError("prompt should have 3 dimensions: [B, C, T] (C = 1).")
+        prompt = convert_audio(prompt, prompt_sample_rate, self.sample_rate, audio_channels)
+        if descriptions is None:
+            descriptions = [None] * len(prompt)
+        attributes, prompt_tokens = self._prepare_tokens_and_attributes(descriptions, prompt)
+        assert prompt_tokens is not None
+        tokens = self._generate_tokens(attributes, prompt_tokens, progress)
+        if return_tokens:
+            return self.generate_audio(tokens), tokens
+        return self.generate_audio(tokens)
+
+    @torch.no_grad()
+    def _prepare_tokens_and_attributes(
+            self,
+            descriptions: tp.Sequence[tp.Optional[str]],
+            prompt: tp.Optional[torch.Tensor],
+            melody_wavs: tp.Optional[MelodyList] = None,
+            melody_chords: tp.Optional[MelodyList] = None,
+            beats : tp.Optional[MelodyList] = None,
+            bpms : tp.Optional[list] = None,
+    ) -> tp.Tuple[tp.List[ConditioningAttributes], tp.Optional[torch.Tensor]]:
+        """Prepare model inputs.
+
+        Args:
+            descriptions (list of str): A list of strings used as text conditioning.
+            prompt (torch.Tensor): A batch of waveforms used for continuation.
+            melody_wavs (torch.Tensor, optional): A batch of waveforms
+                used as melody conditioning. Defaults to None.
+        """
+        attributes = [
+            ConditioningAttributes(text={'description': description})
+            for description in descriptions]
+
+        if melody_wavs is None:
+            for attr in attributes:
+                attr.wav['self_wav'] = WavCondition(
+                    torch.zeros((1, 1, 1), device=self.device),
+                    torch.tensor([0], device=self.device),
+                    sample_rate=[self.sample_rate],
+                    path=[None])
+        else:
+            if 'self_wav' not in self.lm.condition_provider.conditioners:
+                raise RuntimeError("This model doesn't support melody conditioning. "
+                                   "Use the `melody` model.")
+            assert len(melody_wavs) == len(descriptions), \
+                f"number of melody wavs must match number of descriptions! " \
+                f"got melody len={len(melody_wavs)}, and descriptions len={len(descriptions)}"
+            for attr, melody in zip(attributes, melody_wavs):
+                if melody is None:
+                    attr.wav['self_wav'] = WavCondition(
+                        torch.zeros((1, 1, 1), device=self.device),
+                        torch.tensor([0], device=self.device),
+                        sample_rate=[self.sample_rate],
+                        path=[None])
+                else:
+                    attr.wav['self_wav'] = WavCondition(
+                        melody[None].to(device=self.device),
+                        torch.tensor([melody.shape[-1]], device=self.device),
+                        sample_rate=[self.sample_rate],
+                        path=[None],
+                    )
+
+        if melody_chords is None:
+            for attr in attributes:
+                attr.chord['chord'] = ChordCondition(
+                    torch.zeros((1, 12, 1), device=self.device),
+                    torch.tensor([0], device=self.device),
+                    bpm=[None],
+                    path=[None])
+        else:
+            # if 'chord' not in self.lm.condition_provider.conditioners:
+            #     raise RuntimeError("This model doesn't support chord conditioning. "
+            #                        "Use the `chord` model.")
+            assert len(melody_chords) == len(descriptions), \
+                f"number of melody_chords must match number of descriptions! " \
+                f"got melody len={len(melody_chords)}, and descriptions len={len(descriptions)}"
+            for attr, chord, bpm in zip(attributes, melody_chords, bpms):
+                if chord is None:
+                    attr.chord['chord'] = ChordCondition(
+                        torch.zeros((1, 1, 1), device=self.device),
+                        torch.tensor([0], device=self.device),
+                        bpm=[None],
+                        path=[None])
+                else:
+                    attr.chord['chord'] = ChordCondition(
+                        chord[None].to(device=self.device),
+                        torch.tensor([chord.shape[-1]], device=self.device),
+                        bpm=[bpm],
+                        path=[None],
+                    )
+        
+        if beats is None:
+            for attr in attributes:
+                attr.beat['beat'] = BeatCondition(
+                    torch.zeros((1, 1, 1), device=self.device),
+                    torch.tensor([0], device=self.device),
+                    bpm=[None],
+                    path=[None])
+        else:
+            # if 'beat' not in self.lm.condition_provider.conditioners:
+            #     raise RuntimeError("This model doesn't support beat conditioning. "
+            #                        "Use the `beat` model.")
+            assert len(beats) == len(descriptions), \
+                f"number of beats must match number of descriptions! " \
+                f"got melody len={len(beats)}, and descriptions len={len(descriptions)}"
+            for attr, beat, bpm in zip(attributes, beats, bpms):
+                if beat is None:
+                    attr.beat['beat'] = BeatCondition(
+                        torch.zeros((1, 1, 1), device=self.device),
+                        torch.tensor([0], device=self.device),
+                        bpm=[None],
+                        path=[None])
+                else:
+                    attr.beat['beat'] = BeatCondition(
+                        beat[None].to(device=self.device),
+                        torch.tensor([beat.shape[-1]], device=self.device),
+                        bpm=[bpm],
+                        path=[None],
+                    )
+
+        if prompt is not None:
+            if descriptions is not None:
+                assert len(descriptions) == len(prompt), "Prompt and nb. descriptions doesn't match"
+            prompt = prompt.to(self.device)
+            prompt_tokens, scale = self.compression_model.encode(prompt)
+            assert scale is None
+        else:
+            prompt_tokens = None
+        return attributes, prompt_tokens
+
+    def _generate_tokens(self, attributes: tp.List[ConditioningAttributes],
+                         prompt_tokens: tp.Optional[torch.Tensor], progress: bool = False) -> torch.Tensor:
+        """Generate discrete audio tokens given audio prompt and/or conditions.
+
+        Args:
+            attributes (list of ConditioningAttributes): Conditions used for generation (text/melody).
+            prompt_tokens (torch.Tensor, optional): Audio prompt used for continuation.
+            progress (bool, optional): Flag to display progress of the generation process. Defaults to False.
+        Returns:
+            torch.Tensor: Generated audio, of shape [B, C, T], T is defined by the generation params.
+        """
+        total_gen_len = int(self.duration * self.frame_rate)
+        max_prompt_len = int(min(self.duration, self.max_duration) * self.frame_rate)
+        current_gen_offset: int = 0
+
+        def _progress_callback(generated_tokens: int, tokens_to_generate: int):
+            generated_tokens += current_gen_offset
+            if self._progress_callback is not None:
+                # Note that total_gen_len might be quite wrong depending on the
+                # codebook pattern used, but with delay it is almost accurate.
+                self._progress_callback(generated_tokens, total_gen_len)
+            else:
+                print(f'{generated_tokens: 6d} / {total_gen_len: 6d}', end='\r')
+
+        if prompt_tokens is not None:
+            assert max_prompt_len >= prompt_tokens.shape[-1], \
+                "Prompt is longer than audio to generate"
+
+        callback = None
+        if progress:
+            callback = _progress_callback
+
+        if self.duration <= self.max_duration:
+            # generate by sampling from LM, simple case.
+            with self.autocast:
+                gen_tokens = self.lm.generate(
+                    prompt_tokens, attributes,
+                    callback=callback, max_gen_len=total_gen_len, **self.generation_params)
+
+        else:
+            # now this gets a bit messier, we need to handle prompts,
+            # melody conditioning etc.
+            ref_wavs = [attr.wav['self_wav'] for attr in attributes]
+            all_tokens = []
+            if prompt_tokens is None:
+                prompt_length = 0
+            else:
+                all_tokens.append(prompt_tokens)
+                prompt_length = prompt_tokens.shape[-1]
+
+            stride_tokens = int(self.frame_rate * self.extend_stride)
+
+            while current_gen_offset + prompt_length < total_gen_len:
+                time_offset = current_gen_offset / self.frame_rate
+                chunk_duration = min(self.duration - time_offset, self.max_duration)
+                max_gen_len = int(chunk_duration * self.frame_rate)
+                for attr, ref_wav in zip(attributes, ref_wavs):
+                    wav_length = ref_wav.length.item()
+                    if wav_length == 0:
+                        continue
+                    # We will extend the wav periodically if it not long enough.
+                    # we have to do it here rather than in conditioners.py as otherwise
+                    # we wouldn't have the full wav.
+                    initial_position = int(time_offset * self.sample_rate)
+                    wav_target_length = int(self.max_duration * self.sample_rate)
+                    positions = torch.arange(initial_position,
+                                             initial_position + wav_target_length, device=self.device)
+                    attr.wav['self_wav'] = WavCondition(
+                        ref_wav[0][..., positions % wav_length],
+                        torch.full_like(ref_wav[1], wav_target_length),
+                        [self.sample_rate] * ref_wav[0].size(0),
+                        [None], [0.])
+                with self.autocast:
+                    gen_tokens = self.lm.generate(
+                        prompt_tokens, attributes,
+                        callback=callback, max_gen_len=max_gen_len, **self.generation_params)
+                if prompt_tokens is None:
+                    all_tokens.append(gen_tokens)
+                else:
+                    all_tokens.append(gen_tokens[:, :, prompt_tokens.shape[-1]:])
+                prompt_tokens = gen_tokens[:, :, stride_tokens:]
+                prompt_length = prompt_tokens.shape[-1]
+                current_gen_offset += stride_tokens
+
+            gen_tokens = torch.cat(all_tokens, dim=-1)
+        return gen_tokens
+
+    def generate_audio(self, gen_tokens: torch.Tensor):
+        """Generate Audio from tokens"""
+        assert gen_tokens.dim() == 3
+        with torch.no_grad():
+            n_channel = gen_tokens.shape[1]
+            gen_audio = self.compression_model.decode(gen_tokens, None)
+        return gen_audio
diff --git a/audiocraft/audiocraft/models/unet.py b/audiocraft/audiocraft/models/unet.py
new file mode 100644
index 0000000000000000000000000000000000000000..db4a6df8e309c21fede37abdbe3c862932027641
--- /dev/null
+++ b/audiocraft/audiocraft/models/unet.py
@@ -0,0 +1,214 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Pytorch Unet Module used for diffusion.
+"""
+
+from dataclasses import dataclass
+import typing as tp
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from audiocraft.modules.transformer import StreamingTransformer, create_sin_embedding
+
+
+@dataclass
+class Output:
+    sample: torch.Tensor
+
+
+def get_model(cfg, channels: int, side: int, num_steps: int):
+    if cfg.model == 'unet':
+        return DiffusionUnet(
+            chin=channels, num_steps=num_steps, **cfg.diffusion_unet)
+    else:
+        raise RuntimeError('Not Implemented')
+
+
+class ResBlock(nn.Module):
+    def __init__(self, channels: int, kernel: int = 3, norm_groups: int = 4,
+                 dilation: int = 1, activation: tp.Type[nn.Module] = nn.ReLU,
+                 dropout: float = 0.):
+        super().__init__()
+        stride = 1
+        padding = dilation * (kernel - stride) // 2
+        Conv = nn.Conv1d
+        Drop = nn.Dropout1d
+        self.norm1 = nn.GroupNorm(norm_groups, channels)
+        self.conv1 = Conv(channels, channels, kernel, 1, padding, dilation=dilation)
+        self.activation1 = activation()
+        self.dropout1 = Drop(dropout)
+
+        self.norm2 = nn.GroupNorm(norm_groups, channels)
+        self.conv2 = Conv(channels, channels, kernel, 1, padding, dilation=dilation)
+        self.activation2 = activation()
+        self.dropout2 = Drop(dropout)
+
+    def forward(self, x):
+        h = self.dropout1(self.conv1(self.activation1(self.norm1(x))))
+        h = self.dropout2(self.conv2(self.activation2(self.norm2(h))))
+        return x + h
+
+
+class DecoderLayer(nn.Module):
+    def __init__(self, chin: int, chout: int, kernel: int = 4, stride: int = 2,
+                 norm_groups: int = 4, res_blocks: int = 1, activation: tp.Type[nn.Module] = nn.ReLU,
+                 dropout: float = 0.):
+        super().__init__()
+        padding = (kernel - stride) // 2
+        self.res_blocks = nn.Sequential(
+            *[ResBlock(chin, norm_groups=norm_groups, dilation=2**idx, dropout=dropout)
+              for idx in range(res_blocks)])
+        self.norm = nn.GroupNorm(norm_groups, chin)
+        ConvTr = nn.ConvTranspose1d
+        self.convtr = ConvTr(chin, chout, kernel, stride, padding, bias=False)
+        self.activation = activation()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.res_blocks(x)
+        x = self.norm(x)
+        x = self.activation(x)
+        x = self.convtr(x)
+        return x
+
+
+class EncoderLayer(nn.Module):
+    def __init__(self, chin: int, chout: int, kernel: int = 4, stride: int = 2,
+                 norm_groups: int = 4, res_blocks: int = 1, activation: tp.Type[nn.Module] = nn.ReLU,
+                 dropout: float = 0.):
+        super().__init__()
+        padding = (kernel - stride) // 2
+        Conv = nn.Conv1d
+        self.conv = Conv(chin, chout, kernel, stride, padding, bias=False)
+        self.norm = nn.GroupNorm(norm_groups, chout)
+        self.activation = activation()
+        self.res_blocks = nn.Sequential(
+            *[ResBlock(chout, norm_groups=norm_groups, dilation=2**idx, dropout=dropout)
+              for idx in range(res_blocks)])
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, C, T = x.shape
+        stride, = self.conv.stride
+        pad = (stride - (T % stride)) % stride
+        x = F.pad(x, (0, pad))
+
+        x = self.conv(x)
+        x = self.norm(x)
+        x = self.activation(x)
+        x = self.res_blocks(x)
+        return x
+
+
+class BLSTM(nn.Module):
+    """BiLSTM with same hidden units as input dim.
+    """
+    def __init__(self, dim, layers=2):
+        super().__init__()
+        self.lstm = nn.LSTM(bidirectional=True, num_layers=layers, hidden_size=dim, input_size=dim)
+        self.linear = nn.Linear(2 * dim, dim)
+
+    def forward(self, x):
+        x = x.permute(2, 0, 1)
+        x = self.lstm(x)[0]
+        x = self.linear(x)
+        x = x.permute(1, 2, 0)
+        return x
+
+
+class DiffusionUnet(nn.Module):
+    def __init__(self, chin: int = 3, hidden: int = 24, depth: int = 3, growth: float = 2.,
+                 max_channels: int = 10_000, num_steps: int = 1000, emb_all_layers=False, cross_attention: bool = False,
+                 bilstm: bool = False, transformer: bool = False,
+                 codec_dim: tp.Optional[int] = None, **kwargs):
+        super().__init__()
+        self.encoders = nn.ModuleList()
+        self.decoders = nn.ModuleList()
+        self.embeddings: tp.Optional[nn.ModuleList] = None
+        self.embedding = nn.Embedding(num_steps, hidden)
+        if emb_all_layers:
+            self.embeddings = nn.ModuleList()
+        self.condition_embedding: tp.Optional[nn.Module] = None
+        for d in range(depth):
+            encoder = EncoderLayer(chin, hidden, **kwargs)
+            decoder = DecoderLayer(hidden, chin, **kwargs)
+            self.encoders.append(encoder)
+            self.decoders.insert(0, decoder)
+            if emb_all_layers and d > 0:
+                assert self.embeddings is not None
+                self.embeddings.append(nn.Embedding(num_steps, hidden))
+            chin = hidden
+            hidden = min(int(chin * growth), max_channels)
+        self.bilstm: tp.Optional[nn.Module]
+        if bilstm:
+            self.bilstm = BLSTM(chin)
+        else:
+            self.bilstm = None
+        self.use_transformer = transformer
+        self.cross_attention = False
+        if transformer:
+            self.cross_attention = cross_attention
+            self.transformer = StreamingTransformer(chin, 8, 6, bias_ff=False, bias_attn=False,
+                                                    cross_attention=cross_attention)
+
+        self.use_codec = False
+        if codec_dim is not None:
+            self.conv_codec = nn.Conv1d(codec_dim, chin, 1)
+            self.use_codec = True
+
+    def forward(self, x: torch.Tensor, step: tp.Union[int, torch.Tensor], condition: tp.Optional[torch.Tensor] = None):
+        skips = []
+        bs = x.size(0)
+        z = x
+        view_args = [1]
+        if type(step) is torch.Tensor:
+            step_tensor = step
+        else:
+            step_tensor = torch.tensor([step], device=x.device, dtype=torch.long).expand(bs)
+
+        for idx, encoder in enumerate(self.encoders):
+            z = encoder(z)
+            if idx == 0:
+                z = z + self.embedding(step_tensor).view(bs, -1, *view_args).expand_as(z)
+            elif self.embeddings is not None:
+                z = z + self.embeddings[idx - 1](step_tensor).view(bs, -1, *view_args).expand_as(z)
+
+            skips.append(z)
+
+        if self.use_codec:  # insert condition in the bottleneck
+            assert condition is not None, "Model defined for conditionnal generation"
+            condition_emb = self.conv_codec(condition)  # reshape to the bottleneck dim
+            assert condition_emb.size(-1) <= 2 * z.size(-1), \
+                f"You are downsampling the conditionning with factor >=2 : {condition_emb.size(-1)=} and {z.size(-1)=}"
+            if not self.cross_attention:
+
+                condition_emb = torch.nn.functional.interpolate(condition_emb, z.size(-1))
+                assert z.size() == condition_emb.size()
+                z += condition_emb
+                cross_attention_src = None
+            else:
+                cross_attention_src = condition_emb.permute(0, 2, 1)  # B, T, C
+                B, T, C = cross_attention_src.shape
+                positions = torch.arange(T, device=x.device).view(1, -1, 1)
+                pos_emb = create_sin_embedding(positions, C, max_period=10_000, dtype=cross_attention_src.dtype)
+                cross_attention_src = cross_attention_src + pos_emb
+        if self.use_transformer:
+            z = self.transformer(z.permute(0, 2, 1), cross_attention_src=cross_attention_src).permute(0, 2, 1)
+        else:
+            if self.bilstm is None:
+                z = torch.zeros_like(z)
+            else:
+                z = self.bilstm(z)
+
+        for decoder in self.decoders:
+            s = skips.pop(-1)
+            z = z[:, :, :s.shape[2]]
+            z = z + s
+            z = decoder(z)
+
+        z = z[:, :, :x.shape[2]]
+        return Output(z)
diff --git a/audiocraft/audiocraft/modules/__init__.py b/audiocraft/audiocraft/modules/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..61418616ef18f0ecca56a007c43af4a731d98b9b
--- /dev/null
+++ b/audiocraft/audiocraft/modules/__init__.py
@@ -0,0 +1,22 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Modules used for building the models."""
+
+# flake8: noqa
+from .conv import (
+    NormConv1d,
+    NormConv2d,
+    NormConvTranspose1d,
+    NormConvTranspose2d,
+    StreamableConv1d,
+    StreamableConvTranspose1d,
+    pad_for_conv1d,
+    pad1d,
+    unpad1d,
+)
+from .lstm import StreamableLSTM
+from .seanet import SEANetEncoder, SEANetDecoder
+from .transformer import StreamingTransformer
\ No newline at end of file
diff --git a/audiocraft/audiocraft/modules/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..62712228f5a9fe15ea967b1a9c293231e2e3d057
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/activations.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/activations.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..06b1745b1b13769b4b05528223029cf3ada9324a
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/activations.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/chroma.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/chroma.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..6da8ea24700237cd7d5dd5f0c80e836749b01202
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/chroma.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/codebooks_patterns.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/codebooks_patterns.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7ff99fa78f252b1766c85e2bc8f41e630b5c3183
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/codebooks_patterns.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/conditioners.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/conditioners.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d913486a7d5fc0f4c01ee66fad77e707e3aed0c1
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/conditioners.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/conv.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/conv.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e7ee0d9c7ad787a81d18ce7a0ce7f259f6280e4e
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/conv.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/diffusion_schedule.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/diffusion_schedule.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8afe3f7fc4cd9a7b52d94a15695bc987a810a0ce
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/diffusion_schedule.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/lstm.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/lstm.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..4c7c5865c61429551ca24a3011a37a9dc72de1a9
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/lstm.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/rope.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/rope.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dff1b5155c978387270255eec10d6635362e69c1
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/rope.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/seanet.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/seanet.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b725c6e7cfbf4070315262f6f54e11a16e0a7c4e
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/seanet.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/streaming.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/streaming.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b737da74c537c940197ee703224e0f04d60f8853
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/streaming.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/__pycache__/transformer.cpython-311.pyc b/audiocraft/audiocraft/modules/__pycache__/transformer.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..209eb91453bc7c634746352dfb78521e8b5574fe
Binary files /dev/null and b/audiocraft/audiocraft/modules/__pycache__/transformer.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/modules/activations.py b/audiocraft/audiocraft/modules/activations.py
new file mode 100644
index 0000000000000000000000000000000000000000..2d83d7c4c2dc84c64b724eadbe06157507d4f20d
--- /dev/null
+++ b/audiocraft/audiocraft/modules/activations.py
@@ -0,0 +1,96 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+import torch.nn as nn
+from torch import Tensor
+from typing import Union, Callable
+
+
+class CustomGLU(nn.Module):
+    """Custom Gated Linear Unit activation.
+    Applies a modified gated linear unit :math:`a * f(b)` where :math:`a` is the first half
+    of the input matrices, :math:`b` is the second half, and :math:`f` is a provided activation
+    function (i.e. sigmoid, swish, etc.).
+
+    Args:
+        activation (nn.Module): The custom activation to apply in the Gated Linear Unit
+        dim (int): the dimension on which to split the input. Default: -1
+
+    Shape:
+        - Input: :math:`(\ast_1, N, \ast_2)` where `*` means, any number of additional
+          dimensions
+        - Output: :math:`(\ast_1, M, \ast_2)` where :math:`M=N/2`
+
+    Examples::
+        >>> m = CustomGLU(nn.Sigmoid())
+        >>> input = torch.randn(4, 2)
+        >>> output = m(input)
+    """
+    def __init__(self, activation: nn.Module, dim: int = -1):
+        super(CustomGLU, self).__init__()
+        self.dim = dim
+        self.activation = activation
+
+    def forward(self, x: Tensor):
+        assert x.shape[self.dim] % 2 == 0  # M = N / 2
+        a, b = torch.chunk(x, 2, dim=self.dim)
+        return a * self.activation(b)
+
+
+class SwiGLU(CustomGLU):
+    """SiLU Gated Linear Unit activation.
+    Applies SiLU Gated Linear Unit :math:`a * SiLU(b)` where :math:`a` is
+    the first half of the input matrices, :math:`b` is the second half.
+
+    Args:
+        dim (int): the dimension on which to split the input. Default: -1
+    """
+    def __init__(self, dim: int = -1):
+        super(SwiGLU, self).__init__(nn.SiLU(), dim)
+
+
+class GeGLU(CustomGLU):
+    """GeLU Gated Linear Unit activation.
+    Applies GeLU Gated Linear Unit :math:`a * GELU(b)` where :math:`a` is
+    the first half of the input matrices, :math:`b` is the second half.
+
+    Args:
+        dim (int): the dimension on which to split the input. Default: -1
+    """
+    def __init__(self, dim: int = -1):
+        super(GeGLU, self).__init__(nn.GELU(), dim)
+
+
+class ReGLU(CustomGLU):
+    """ReLU Gated Linear Unit activation.
+    Applies ReLU Gated Linear Unit :math:`a * ReLU(b)` where :math:`a` is
+    the first half of the input matrices, :math:`b` is the second half.
+
+    Args:
+        dim (int): the dimension on which to split the input. Default: -1
+    """
+    def __init__(self, dim: int = -1):
+        super(ReGLU, self).__init__(nn.ReLU(), dim)
+
+
+def get_activation_fn(
+    activation: Union[str, Callable[[Tensor], Tensor]]
+) -> Union[str, Callable[[Tensor], Tensor]]:
+    """Helper function to map an activation string to the activation class.
+    If the supplied activation is not a string that is recognized, the activation is passed back.
+
+    Args:
+        activation (str, or Callable[[Tensor], Tensor]): Activation to check
+    """
+    if isinstance(activation, str):
+        if activation == "reglu":
+            return ReGLU()
+        elif activation == "geglu":
+            return GeGLU()
+        elif activation == "swiglu":
+            return SwiGLU()
+    return activation
diff --git a/audiocraft/audiocraft/modules/chroma.py b/audiocraft/audiocraft/modules/chroma.py
new file mode 100644
index 0000000000000000000000000000000000000000..e84fb66b4a4aaefb0b3ccac8a9a44c3b20e48f61
--- /dev/null
+++ b/audiocraft/audiocraft/modules/chroma.py
@@ -0,0 +1,66 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+import typing as tp
+
+from einops import rearrange
+from librosa import filters
+import torch
+from torch import nn
+import torch.nn.functional as F
+import torchaudio
+
+
+class ChromaExtractor(nn.Module):
+    """Chroma extraction and quantization.
+
+    Args:
+        sample_rate (int): Sample rate for the chroma extraction.
+        n_chroma (int): Number of chroma bins for the chroma extraction.
+        radix2_exp (int): Size of stft window for the chroma extraction (power of 2, e.g. 12 -> 2^12).
+        nfft (int, optional): Number of FFT.
+        winlen (int, optional): Window length.
+        winhop (int, optional): Window hop size.
+        argmax (bool, optional): Whether to use argmax. Defaults to False.
+        norm (float, optional): Norm for chroma normalization. Defaults to inf.
+    """
+    def __init__(self, sample_rate: int, n_chroma: int = 12, radix2_exp: int = 12, nfft: tp.Optional[int] = None,
+                 winlen: tp.Optional[int] = None, winhop: tp.Optional[int] = None, argmax: bool = False,
+                 norm: float = torch.inf):
+        super().__init__()
+        self.winlen = winlen or 2 ** radix2_exp
+        self.nfft = nfft or self.winlen
+        self.winhop = winhop or (self.winlen // 4)
+        self.sample_rate = sample_rate
+        self.n_chroma = n_chroma
+        self.norm = norm
+        self.argmax = argmax
+        self.register_buffer('fbanks', torch.from_numpy(filters.chroma(sr=sample_rate, n_fft=self.nfft, tuning=0,
+                                                                       n_chroma=self.n_chroma)), persistent=False)
+        self.spec = torchaudio.transforms.Spectrogram(n_fft=self.nfft, win_length=self.winlen,
+                                                      hop_length=self.winhop, power=2, center=True,
+                                                      pad=0, normalized=True)
+
+    def forward(self, wav: torch.Tensor) -> torch.Tensor:
+        T = wav.shape[-1]
+        # in case we are getting a wav that was dropped out (nullified)
+        # from the conditioner, make sure wav length is no less that nfft
+        if T < self.nfft:
+            pad = self.nfft - T
+            r = 0 if pad % 2 == 0 else 1
+            wav = F.pad(wav, (pad // 2, pad // 2 + r), 'constant', 0)
+            assert wav.shape[-1] == self.nfft, f"expected len {self.nfft} but got {wav.shape[-1]}"
+
+        spec = self.spec(wav).squeeze(1)
+        raw_chroma = torch.einsum('cf,...ft->...ct', self.fbanks, spec)
+        norm_chroma = torch.nn.functional.normalize(raw_chroma, p=self.norm, dim=-2, eps=1e-6)
+        norm_chroma = rearrange(norm_chroma, 'b d t -> b t d')
+
+        if self.argmax:
+            idx = norm_chroma.argmax(-1, keepdim=True)
+            norm_chroma[:] = 0
+            norm_chroma.scatter_(dim=-1, index=idx, value=1)
+
+        return norm_chroma
diff --git a/audiocraft/audiocraft/modules/codebooks_patterns.py b/audiocraft/audiocraft/modules/codebooks_patterns.py
new file mode 100644
index 0000000000000000000000000000000000000000..1bfc767dce8d804dd1058a92924713af599be808
--- /dev/null
+++ b/audiocraft/audiocraft/modules/codebooks_patterns.py
@@ -0,0 +1,542 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from collections import namedtuple
+from dataclasses import dataclass
+from functools import lru_cache
+import logging
+import typing as tp
+
+from abc import ABC, abstractmethod
+import torch
+
+LayoutCoord = namedtuple('LayoutCoord', ['t', 'q'])  # (timestep, codebook index)
+PatternLayout = tp.List[tp.List[LayoutCoord]]  # Sequence of coordinates
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class Pattern:
+    """Base implementation of a pattern over a sequence with multiple codebooks.
+
+    The codebook pattern consists in a layout, defining for each sequence step
+    the list of coordinates of each codebook timestep in the resulting interleaved sequence.
+    The first item of the pattern is always an empty list in order to properly insert a special token
+    to start with. For convenience, we also keep track of ``n_q`` the number of codebooks used for the pattern
+    and ``timesteps`` the number of timesteps corresponding to the original sequence.
+
+    The pattern provides convenient methods to build and revert interleaved sequences from it:
+    ``build_pattern_sequence`` maps a given a dense input tensor of multi-codebook sequence from [B, K, T]
+        to the interleaved sequence of shape [B, K, S] applying the pattern, with S being the batch size,
+        K being the number of codebooks, T the number of original timesteps and S the number of sequence steps
+        for the output sequence. The unfilled positions are replaced with a special token and the built sequence
+        is returned along with a mask indicating valid tokens.
+    ``revert_pattern_sequence`` maps back an interleaved sequence of shape [B, K, S] to the original alignment
+        of codebooks across timesteps to an output tensor of shape [B, K, T], using again a special token and a mask
+        to fill and specify invalid positions if needed.
+    See the dedicated methods for more details.
+    """
+    # Pattern layout, for each sequence step, we have a list of coordinates
+    # corresponding to the original codebook timestep and position.
+    # The first list is always an empty list in order to properly insert
+    # a special token to start with.
+    layout: PatternLayout
+    timesteps: int
+    n_q: int
+
+    def __post_init__(self):
+        assert len(self.layout) > 0
+        assert self.layout[0] == []
+        self._validate_layout()
+        self._build_reverted_sequence_scatter_indexes = lru_cache(100)(self._build_reverted_sequence_scatter_indexes)
+        self._build_pattern_sequence_scatter_indexes = lru_cache(100)(self._build_pattern_sequence_scatter_indexes)
+        logger.info("New pattern, time steps: %d, sequence steps: %d", self.timesteps, len(self.layout))
+
+    def _validate_layout(self):
+        """Runs checks on the layout to ensure a valid pattern is defined.
+        A pattern is considered invalid if:
+            - Multiple timesteps for a same codebook are defined in the same sequence step
+            - The timesteps for a given codebook are not in ascending order as we advance in the sequence
+              (this would mean that we have future timesteps before past timesteps).
+        """
+        q_timesteps = {q: 0 for q in range(self.n_q)}
+        for s, seq_coords in enumerate(self.layout):
+            if len(seq_coords) > 0:
+                qs = set()
+                for coord in seq_coords:
+                    qs.add(coord.q)
+                    last_q_timestep = q_timesteps[coord.q]
+                    assert coord.t >= last_q_timestep, \
+                        f"Past timesteps are found in the sequence for codebook = {coord.q} at step {s}"
+                    q_timesteps[coord.q] = coord.t
+                # each sequence step contains at max 1 coordinate per codebook
+                assert len(qs) == len(seq_coords), \
+                    f"Multiple entries for a same codebook are found at step {s}"
+
+    @property
+    def num_sequence_steps(self):
+        return len(self.layout) - 1
+
+    @property
+    def max_delay(self):
+        max_t_in_seq_coords = 0
+        for seq_coords in self.layout[1:]:
+            for coords in seq_coords:
+                max_t_in_seq_coords = max(max_t_in_seq_coords, coords.t + 1)
+        return max_t_in_seq_coords - self.timesteps
+
+    @property
+    def valid_layout(self):
+        valid_step = len(self.layout) - self.max_delay
+        return self.layout[:valid_step]
+
+    def get_sequence_coords_with_timestep(self, t: int, q: tp.Optional[int] = None):
+        """Get codebook coordinates in the layout that corresponds to the specified timestep t
+        and optionally to the codebook q. Coordinates are returned as a tuple with the sequence step
+        and the actual codebook coordinates.
+        """
+        assert t <= self.timesteps, "provided timesteps is greater than the pattern's number of timesteps"
+        if q is not None:
+            assert q <= self.n_q, "provided number of codebooks is greater than the pattern's number of codebooks"
+        coords = []
+        for s, seq_codes in enumerate(self.layout):
+            for code in seq_codes:
+                if code.t == t and (q is None or code.q == q):
+                    coords.append((s, code))
+        return coords
+
+    def get_steps_with_timestep(self, t: int, q: tp.Optional[int] = None) -> tp.List[int]:
+        return [step for step, coords in self.get_sequence_coords_with_timestep(t, q)]
+
+    def get_first_step_with_timesteps(self, t: int, q: tp.Optional[int] = None) -> tp.Optional[int]:
+        steps_with_timesteps = self.get_steps_with_timestep(t, q)
+        return steps_with_timesteps[0] if len(steps_with_timesteps) > 0 else None
+
+    def _build_pattern_sequence_scatter_indexes(self, timesteps: int, n_q: int, keep_only_valid_steps: bool,
+                                                device: tp.Union[torch.device, str] = 'cpu'):
+        """Build scatter indexes corresponding to the pattern, up to the provided sequence_steps.
+
+        Args:
+            timesteps (int): Maximum number of timesteps steps to consider.
+            keep_only_valid_steps (bool): Restrict the pattern layout to match only valid steps.
+            device (torch.device or str): Device for created tensors.
+        Returns:
+            indexes (torch.Tensor): Indexes corresponding to the sequence, of shape [K, S].
+            mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes, of shape [K, S].
+        """
+        # assert n_q == self.n_q, f"invalid number of codebooks for the sequence and the pattern: {n_q} != {self.n_q}"
+        assert timesteps <= self.timesteps, "invalid number of timesteps used to build the sequence from the pattern"
+        # use the proper layout based on whether we limit ourselves to valid steps only or not,
+        # note that using the valid_layout will result in a truncated sequence up to the valid steps
+        ref_layout = self.valid_layout if keep_only_valid_steps else self.layout
+        # single item indexing being super slow with pytorch vs. numpy, so we use numpy here
+        indexes = torch.zeros(n_q, len(ref_layout), dtype=torch.long).numpy()
+        mask = torch.zeros(n_q, len(ref_layout), dtype=torch.bool).numpy()
+        # fill indexes with last sequence step value that will correspond to our special token
+        # the last value is n_q * timesteps as we have flattened z and append special token as the last token
+        # which will correspond to the index: n_q * timesteps
+        indexes[:] = n_q * timesteps
+        # iterate over the pattern and fill scattered indexes and mask
+        for s, sequence_coords in enumerate(ref_layout):
+            for coords in sequence_coords:
+                if coords.t < timesteps:
+                    indexes[coords.q, s] = coords.t + coords.q * timesteps
+                    mask[coords.q, s] = 1
+        indexes = torch.from_numpy(indexes).to(device)
+        mask = torch.from_numpy(mask).to(device)
+        return indexes, mask
+
+    def build_pattern_sequence(self, z: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):
+        """Build sequence corresponding to the pattern from the input tensor z.
+        The sequence is built using up to sequence_steps if specified, and non-pattern
+        coordinates are filled with the special token.
+
+        Args:
+            z (torch.Tensor): Input tensor of multi-codebooks sequence, of shape [B, K, T].
+            special_token (int): Special token used to fill non-pattern coordinates in the new sequence.
+            keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.
+                Steps that are beyond valid steps will be replaced by the special_token in that case.
+        Returns:
+            values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, S] with S
+                corresponding either to the sequence_steps if provided, otherwise to the length of the pattern.
+            indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, S].
+            mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, S].
+        """
+        B, K, T = z.shape
+        indexes, mask = self._build_pattern_sequence_scatter_indexes(
+            T, K, keep_only_valid_steps=keep_only_valid_steps, device=str(z.device)
+        )
+        z = z.view(B, -1)
+        # we append the special token as the last index of our flattened z tensor
+        z = torch.cat([z, torch.zeros_like(z[:, :1]) + special_token], dim=1)
+        values = z[:, indexes.view(-1)]
+        values = values.view(B, K, indexes.shape[-1])
+        return values, indexes, mask
+
+    def _build_reverted_sequence_scatter_indexes(self, sequence_steps: int, n_q: int,
+                                                 keep_only_valid_steps: bool = False,
+                                                 is_model_output: bool = False,
+                                                 device: tp.Union[torch.device, str] = 'cpu'):
+        """Builds scatter indexes required to retrieve the original multi-codebook sequence
+        from interleaving pattern.
+
+        Args:
+            sequence_steps (int): Sequence steps.
+            n_q (int): Number of codebooks.
+            keep_only_valid_steps (bool): Build a sequence from the pattern up to valid (= fully defined) steps.
+                Steps that are beyond valid steps will be replaced by the special_token in that case.
+            is_model_output (bool): Whether to keep the sequence item corresponding to initial special token or not.
+            device (torch.device or str): Device for created tensors.
+        Returns:
+            indexes (torch.Tensor): Indexes for reconstructing the output, of shape [K, T].
+            mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].
+        """
+        ref_layout = self.valid_layout if keep_only_valid_steps else self.layout
+        # TODO(jade): Do we want to further truncate to only valid timesteps here as well?
+        timesteps = self.timesteps
+        #assert n_q == self.n_q, f"invalid number of codebooks for the sequence and the pattern: {n_q} != {self.n_q}"
+        assert sequence_steps <= len(ref_layout), \
+            f"sequence to revert is longer than the defined pattern: {sequence_steps} > {len(ref_layout)}"
+
+        # ensure we take the appropriate indexes to keep the model output from the first special token as well
+        if is_model_output:
+            ref_layout = ref_layout[1:]
+
+        # single item indexing being super slow with pytorch vs. numpy, so we use numpy here
+        indexes = torch.zeros(n_q, timesteps, dtype=torch.long).numpy()
+        mask = torch.zeros(n_q, timesteps, dtype=torch.bool).numpy()
+        # fill indexes with last sequence step value that will correspond to our special token
+        indexes[:] = n_q * sequence_steps
+        for s, sequence_codes in enumerate(ref_layout):
+            if s < sequence_steps:
+                for code in sequence_codes:
+                    if code.t < timesteps:
+                        indexes[code.q, code.t] = s + code.q * sequence_steps
+                        mask[code.q, code.t] = 1
+        indexes = torch.from_numpy(indexes).to(device)
+        mask = torch.from_numpy(mask).to(device)
+        return indexes, mask
+
+    def revert_pattern_sequence(self, s: torch.Tensor, special_token: int, keep_only_valid_steps: bool = False):
+        """Revert a sequence built from the pattern back to the original multi-codebook sequence without interleaving.
+        The sequence is reverted using up to timesteps if specified, and non-pattern coordinates
+        are filled with the special token.
+
+        Args:
+            s (torch.Tensor): Interleaved sequence tensor obtained from the pattern, of shape [B, K, S].
+            special_token (int or float): Special token used to fill non-pattern coordinates in the new sequence.
+        Returns:
+            values (torch.Tensor): Interleaved sequence matching the pattern, of shape [B, K, T] with T
+                corresponding either to the timesteps if provided, or the total timesteps in pattern otherwise.
+            indexes (torch.Tensor): Indexes corresponding to the interleaved sequence, of shape [K, T].
+            mask (torch.Tensor): Mask corresponding to indexes that matches valid indexes of shape [K, T].
+        """
+        B, K, S = s.shape
+        indexes, mask = self._build_reverted_sequence_scatter_indexes(
+            S, K, keep_only_valid_steps, is_model_output=False, device=str(s.device)
+        )
+        s = s.view(B, -1)
+        # we append the special token as the last index of our flattened z tensor
+        s = torch.cat([s, torch.zeros_like(s[:, :1]) + special_token], dim=1)
+        values = s[:, indexes.view(-1)]
+        values = values.view(B, K, indexes.shape[-1])
+        return values, indexes, mask
+
+    def revert_pattern_logits(self, logits: torch.Tensor, special_token: float, keep_only_valid_steps: bool = False):
+        """Revert model logits obtained on a sequence built from the pattern
+        back to a tensor matching the original sequence.
+
+        This method is similar to ``revert_pattern_sequence`` with the following specificities:
+        1. It is designed to work with the extra cardinality dimension
+        2. We return the logits for the first sequence item that matches the special_token and
+        which matching target in the original sequence is the first item of the sequence,
+        while we skip the last logits as there is no matching target
+        """
+        B, card, K, S = logits.shape
+        indexes, mask = self._build_reverted_sequence_scatter_indexes(
+            S, K, keep_only_valid_steps, is_model_output=True, device=logits.device
+        )
+        logits = logits.reshape(B, card, -1)
+        # we append the special token as the last index of our flattened z tensor
+        logits = torch.cat([logits, torch.zeros_like(logits[:, :, :1]) + special_token], dim=-1)  # [B, card, K x S]
+        values = logits[:, :, indexes.view(-1)]
+        values = values.view(B, card, K, indexes.shape[-1])
+        return values, indexes, mask
+
+
+class CodebooksPatternProvider(ABC):
+    """Abstraction around providing pattern for interleaving codebooks.
+
+    The CodebooksPatternProvider abstraction allows to implement various strategies to
+    define interleaving pattern of sequences composed of multiple codebooks. For a given
+    number of codebooks `n_q`, the pattern provider can generate a specified pattern
+    corresponding to a sequence of `T` timesteps with `n_q` parallel codebooks. This pattern
+    can be used to construct a new sequence from the original codes respecting the specified
+    pattern. The pattern is defined as a list of list of code coordinates, code coordinate
+    being a tuple with the original timestep and codebook to build the new sequence.
+    Note that all patterns must start with an empty list that is then used to insert a first
+    sequence step of special tokens in the newly generated sequence.
+
+    Args:
+        n_q (int): number of codebooks.
+        cached (bool): if True, patterns for a given length are cached. In general
+            that should be true for efficiency reason to avoid synchronization points.
+    """
+    def __init__(self, n_q: int, cached: bool = True, stereo: bool = False):
+        assert n_q > 0
+        if stereo:
+            self.n_q = n_q // 2
+        else:
+            self.n_q = n_q
+        self.get_pattern = lru_cache(100)(self.get_pattern)  # type: ignore
+
+    @abstractmethod
+    def get_pattern(self, timesteps: int) -> Pattern:
+        """Builds pattern with specific interleaving between codebooks.
+
+        Args:
+            timesteps (int): Total number of timesteps.
+        """
+        raise NotImplementedError()
+
+
+class DelayedPatternProvider(CodebooksPatternProvider):
+    """Provider for delayed pattern across delayed codebooks.
+    Codebooks are delayed in the sequence and sequence steps will contain codebooks
+    from different timesteps.
+
+    Example:
+        Taking timesteps=4 and n_q=3, delays=None, the multi-codebook sequence:
+        [[1, 2, 3, 4],
+        [1, 2, 3, 4],
+        [1, 2, 3, 4]]
+        The resulting sequence obtained from the returned pattern is:
+        [[S, 1, 2, 3, 4],
+        [S, S, 1, 2, 3],
+        [S, S, S, 1, 2]]
+        (with S being a special token)
+
+    Args:
+        n_q (int): Number of codebooks.
+        delays (list of int, optional): Delay for each of the codebooks.
+            If delays not defined, each codebook is delayed by 1 compared to the previous one.
+        flatten_first (int): Flatten the first N timesteps.
+        empty_initial (int): Prepend with N empty list of coordinates.
+    """
+    def __init__(self, n_q: int, delays: tp.Optional[tp.List[int]] = None,
+                 flatten_first: int = 0, empty_initial: int = 0):
+        super().__init__(n_q)
+        if delays is None:
+            delays = list(range(n_q))
+        self.delays = delays
+        self.flatten_first = flatten_first
+        self.empty_initial = empty_initial
+        # assert len(self.delays) == self.n_q
+        assert sorted(self.delays) == self.delays
+
+    def get_pattern(self, timesteps: int) -> Pattern:
+        out: PatternLayout = [[]]
+        max_delay = max(self.delays)
+        if self.empty_initial:
+            out += [[] for _ in range(self.empty_initial)]
+        if self.flatten_first:
+            for t in range(min(timesteps, self.flatten_first)):
+                for q in range(self.n_q):
+                    out.append([LayoutCoord(t, q)])
+        for t in range(self.flatten_first, timesteps + max_delay):
+            v = []
+            for q, delay in enumerate(self.delays):
+                t_for_q = t - delay
+                if t_for_q >= self.flatten_first:
+                    v.append(LayoutCoord(t_for_q, q))
+            out.append(v)
+        return Pattern(out, n_q=self.n_q, timesteps=timesteps)
+
+
+class ParallelPatternProvider(DelayedPatternProvider):
+    """Provider for parallel pattern across codebooks.
+    This pattern provider is a special case of the delayed pattern with actually no delay,
+    hence delays=repeat(0, n_q).
+
+    Args:
+        n_q (int): Number of codebooks.
+    """
+    def __init__(self, n_q: int):
+        super().__init__(n_q, [0] * n_q)
+
+
+class UnrolledPatternProvider(CodebooksPatternProvider):
+    """Provider for unrolling codebooks pattern.
+    This pattern provider enables to represent the codebook flattened completely or only to some extend
+    while also specifying a given delay between the flattened codebooks representation, allowing to
+    unroll the codebooks in the sequence.
+
+    Example:
+        1. Flattening of the codebooks.
+        By default, the pattern provider will fully flatten the codebooks such as flattening=range(n_q),
+        taking n_q = 3 and timesteps = 4:
+        [[1, 2, 3, 4],
+         [1, 2, 3, 4],
+         [1, 2, 3, 4]]
+        will result into:
+        [[S, S, 1, S, S, 2, S, S, 3, S, S, 4],
+         [S, 1, S, S, 2, S, S, 3, S, S, 4, S],
+         [1, S, S, 2, S, S, 3, S, S, 4, S, S]]
+        2. Partial flattening of the codebooks. The ``flattening`` parameter allows to specify the inner step
+        for each of the codebook, allowing to define which codebook to flatten (or keep in parallel), for example
+        taking n_q = 3, timesteps = 4 and flattening = [0, 1, 1]:
+        [[1, 2, 3, 4],
+         [1, 2, 3, 4],
+         [1, 2, 3, 4]]
+        will result into:
+        [[S, 1, S, S, 2, S, S, 3, S, S, 4, S],
+         [S, 1, S, S, 2, S, S, 3, S, S, 4, S],
+         [1, S, S, 2, S, S, 3, S, S, 4, S, S]]
+        3. Flattening with delay. The ``delay`` parameter allows to further unroll the sequence of codebooks
+        allowing to specify the delay per codebook. Note that the delay between codebooks flattened to the
+        same inner timestep should be coherent. For example, taking n_q = 3, timesteps = 4, flattening = [0, 1, 1]
+        and delays = [0, 3, 3]:
+        [[1, 2, 3, 4],
+         [1, 2, 3, 4],
+         [1, 2, 3, 4]]
+        will result into:
+        [[S, S, S, 1, S, 2, S, 3, S, 4],
+         [S, S, S, 1, S, 2, S, 3, S, 4],
+         [1, 2, 3, S, 4, S, 5, S, 6, S]]
+
+    Args:
+        n_q (int): Number of codebooks.
+        flattening (list of int, optional): Flattening schema over the codebooks. If not defined,
+            the codebooks will be flattened to 1 codebook per step, meaning that the sequence will
+            have n_q extra steps for each timestep.
+        delays (list of int, optional): Delay for each of the codebooks. If not defined,
+            no delay is added and therefore will default to [0] * ``n_q``.
+            Note that two codebooks that will be flattened to the same inner step
+            should have the same delay, otherwise the pattern is considered as invalid.
+    """
+    FlattenedCodebook = namedtuple('FlattenedCodebook', ['codebooks', 'delay'])
+
+    def __init__(self, n_q: int, flattening: tp.Optional[tp.List[int]] = None,
+                 delays: tp.Optional[tp.List[int]] = None):
+        super().__init__(n_q)
+        if flattening is None:
+            flattening = list(range(n_q))
+        if delays is None:
+            delays = [0] * n_q
+        assert len(flattening) == n_q
+        assert len(delays) == n_q
+        assert sorted(flattening) == flattening
+        assert sorted(delays) == delays
+        self._flattened_codebooks = self._build_flattened_codebooks(delays, flattening)
+        self.max_delay = max(delays)
+
+    def _build_flattened_codebooks(self, delays: tp.List[int], flattening: tp.List[int]):
+        """Build a flattened codebooks representation as a dictionary of inner step
+        and the actual codebook indices corresponding to the flattened codebook. For convenience, we
+        also store the delay associated to the flattened codebook to avoid maintaining an extra mapping.
+        """
+        flattened_codebooks: dict = {}
+        for q, (inner_step, delay) in enumerate(zip(flattening, delays)):
+            if inner_step not in flattened_codebooks:
+                flat_codebook = UnrolledPatternProvider.FlattenedCodebook(codebooks=[q], delay=delay)
+            else:
+                flat_codebook = flattened_codebooks[inner_step]
+                assert flat_codebook.delay == delay, (
+                    "Delay and flattening between codebooks is inconsistent: ",
+                    "two codebooks flattened to the same position should have the same delay."
+                )
+                flat_codebook.codebooks.append(q)
+            flattened_codebooks[inner_step] = flat_codebook
+        return flattened_codebooks
+
+    @property
+    def _num_inner_steps(self):
+        """Number of inner steps to unroll between timesteps in order to flatten the codebooks.
+        """
+        return max([inner_step for inner_step in self._flattened_codebooks.keys()]) + 1
+
+    def num_virtual_steps(self, timesteps: int) -> int:
+        return timesteps * self._num_inner_steps + 1
+
+    def get_pattern(self, timesteps: int) -> Pattern:
+        """Builds pattern for delay across codebooks.
+
+        Args:
+            timesteps (int): Total number of timesteps.
+        """
+        # the PatternLayout is built as a tuple of sequence position and list of coordinates
+        # so that it can be reordered properly given the required delay between codebooks of given timesteps
+        indexed_out: list = [(-1, [])]
+        max_timesteps = timesteps + self.max_delay
+        for t in range(max_timesteps):
+            # for each timestep, we unroll the flattened codebooks,
+            # emitting the sequence step with the corresponding delay
+            for step in range(self._num_inner_steps):
+                if step in self._flattened_codebooks:
+                    # we have codebooks at this virtual step to emit
+                    step_codebooks = self._flattened_codebooks[step]
+                    t_for_q = t + step_codebooks.delay
+                    coords = [LayoutCoord(t, q) for q in step_codebooks.codebooks]
+                    if t_for_q < max_timesteps and t < max_timesteps:
+                        indexed_out.append((t_for_q, coords))
+                else:
+                    # there is no codebook in this virtual step so we emit an empty list
+                    indexed_out.append((t, []))
+        out = [coords for _, coords in sorted(indexed_out)]
+        return Pattern(out, n_q=self.n_q, timesteps=timesteps)
+
+
+class VALLEPattern(CodebooksPatternProvider):
+    """Almost VALL-E style pattern.
+    We further allow some delays for the codebooks other than the first one.
+
+    Args:
+        n_q (int): Number of codebooks.
+        delays (list of int, optional): Delay for each of the codebooks.
+            If delays not defined, each codebook is delayed by 1 compared to the previous one.
+    """
+    def __init__(self, n_q: int, delays: tp.Optional[tp.List[int]] = None):
+        super().__init__(n_q)
+        if delays is None:
+            delays = [0] * (n_q - 1)
+        self.delays = delays
+        assert len(self.delays) == self.n_q - 1
+        assert sorted(self.delays) == self.delays
+
+    def get_pattern(self, timesteps: int) -> Pattern:
+        out: PatternLayout = [[]]
+        for t in range(timesteps):
+            out.append([LayoutCoord(t, 0)])
+        max_delay = max(self.delays)
+        for t in range(timesteps + max_delay):
+            v = []
+            for q, delay in enumerate(self.delays):
+                t_for_q = t - delay
+                if t_for_q >= 0:
+                    v.append(LayoutCoord(t_for_q, q + 1))
+            out.append(v)
+        return Pattern(out, n_q=self.n_q, timesteps=timesteps)
+
+
+class MusicLMPattern(CodebooksPatternProvider):
+    """Almost MusicLM style pattern. This is equivalent to full flattening
+    but in a different order.
+
+    Args:
+        n_q (int): Number of codebooks.
+        group_by (int): Number of codebooks to group together.
+    """
+    def __init__(self, n_q: int, group_by: int = 2):
+        super().__init__(n_q)
+        self.group_by = group_by
+
+    def get_pattern(self, timesteps: int) -> Pattern:
+        out: PatternLayout = [[]]
+        for offset in range(0, self.n_q, self.group_by):
+            for t in range(timesteps):
+                for q in range(offset, offset + self.group_by):
+                    out.append([LayoutCoord(t, q)])
+        return Pattern(out, n_q=self.n_q, timesteps=timesteps)
diff --git a/audiocraft/audiocraft/modules/conditioners.py b/audiocraft/audiocraft/modules/conditioners.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d657979c401d806209f7f1af6df1062b7321277
--- /dev/null
+++ b/audiocraft/audiocraft/modules/conditioners.py
@@ -0,0 +1,1678 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+import pretty_midi
+from collections import defaultdict
+from copy import deepcopy
+from dataclasses import dataclass, field
+from itertools import chain
+import logging
+import math
+from pathlib import Path
+import random
+import re
+import typing as tp
+import warnings
+
+import einops
+from num2words import num2words
+import spacy
+from transformers import RobertaTokenizer, T5EncoderModel, T5Tokenizer  # type: ignore
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.nn.utils.rnn import pad_sequence
+
+from .chroma import ChromaExtractor
+from .streaming import StreamingModule
+from .transformer import create_sin_embedding
+from ..data.audio import audio_read
+from ..data.audio_dataset import SegmentInfo
+from ..data.audio_utils import convert_audio
+from ..environment import AudioCraftEnvironment
+from ..quantization import ResidualVectorQuantizer
+from ..utils.autocast import TorchAutocast
+from ..utils.cache import EmbeddingCache
+from ..utils.utils import collate, hash_trick, length_to_mask, load_clap_state_dict, warn_once
+
+
+logger = logging.getLogger(__name__)
+TextCondition = tp.Optional[str]  # a text condition can be a string or None (if doesn't exist)
+ConditionType = tp.Tuple[torch.Tensor, torch.Tensor]  # condition, mask
+
+
+class WavCondition(tp.NamedTuple):
+    wav: torch.Tensor
+    length: torch.Tensor
+    sample_rate: tp.List[int]
+    path: tp.List[tp.Optional[str]] = []
+    seek_time: tp.List[tp.Optional[float]] = []
+
+
+class ChordCondition(tp.NamedTuple):
+    chord: torch.Tensor
+    length: torch.Tensor
+    bpm: tp.List[tp.Optional[float]] = []
+    path: tp.List[tp.Optional[str]] = []
+    seek_frame: tp.List[tp.Optional[float]] = []
+
+
+class BeatCondition(tp.NamedTuple):
+    beat: torch.Tensor
+    length: torch.Tensor
+    bpm: tp.List[tp.Optional[float]] = []
+    path: tp.List[tp.Optional[str]] = []
+    seek_frame: tp.List[tp.Optional[float]] = []
+
+
+class JointEmbedCondition(tp.NamedTuple):
+    wav: torch.Tensor
+    text: tp.List[tp.Optional[str]]
+    length: torch.Tensor
+    sample_rate: tp.List[int]
+    path: tp.List[tp.Optional[str]] = []
+    seek_time: tp.List[tp.Optional[float]] = []
+
+
+@dataclass
+class ConditioningAttributes:
+    text: tp.Dict[str, tp.Optional[str]] = field(default_factory=dict)
+    wav: tp.Dict[str, WavCondition] = field(default_factory=dict)
+    beat: tp.Dict[str, BeatCondition] = field(default_factory=dict)
+    chord: tp.Dict[str, ChordCondition] = field(default_factory=dict)
+    joint_embed: tp.Dict[str, JointEmbedCondition] = field(default_factory=dict)
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    @property
+    def text_attributes(self):
+        return self.text.keys()
+
+    @property
+    def wav_attributes(self):
+        return self.wav.keys()
+
+    @property
+    def beat_attributes(self):
+        return self.beat.keys()
+    
+    @property
+    def chord_attributes(self):
+        return self.chord.keys()
+
+    @property
+    def joint_embed_attributes(self):
+        return self.joint_embed.keys()
+
+    @property
+    def attributes(self):
+        return {
+            "text": self.text_attributes,
+            "wav": self.wav_attributes,
+            "beat" : self.beat_attributes,
+            "chord": self.chord_attributes,
+            "joint_embed": self.joint_embed_attributes,
+        }
+
+    def to_flat_dict(self):
+        return {
+            **{f"text.{k}": v for k, v in self.text.items()},
+            **{f"wav.{k}": v for k, v in self.wav.items()},
+            **{f"beat.{k}": v for k, v in self.beat.items()},
+            **{f"chord.{k}": v for k, v in self.chord.items()},
+            **{f"joint_embed.{k}": v for k, v in self.joint_embed.items()}
+        }
+
+    @classmethod
+    def from_flat_dict(cls, x):
+        out = cls()
+        for k, v in x.items():
+            kind, att = k.split(".")
+            out[kind][att] = v
+        return out
+
+
+class SegmentWithAttributes(SegmentInfo):
+    """Base class for all dataclasses that are used for conditioning.
+    All child classes should implement `to_condition_attributes` that converts
+    the existing attributes to a dataclass of type ConditioningAttributes.
+    """
+    def to_condition_attributes(self) -> ConditioningAttributes:
+        raise NotImplementedError()
+
+
+def nullify_condition(condition: ConditionType, dim: int = 1):
+    """Transform an input condition to a null condition.
+    The way it is done by converting it to a single zero vector similarly
+    to how it is done inside WhiteSpaceTokenizer and NoopTokenizer.
+
+    Args:
+        condition (ConditionType): A tuple of condition and mask (tuple[torch.Tensor, torch.Tensor])
+        dim (int): The dimension that will be truncated (should be the time dimension)
+        WARNING!: dim should not be the batch dimension!
+    Returns:
+        ConditionType: A tuple of null condition and mask
+    """
+    assert dim != 0, "dim cannot be the batch dimension!"
+    assert isinstance(condition, tuple) and \
+        isinstance(condition[0], torch.Tensor) and \
+        isinstance(condition[1], torch.Tensor), "'nullify_condition' got an unexpected input type!"
+    cond, mask = condition
+    B = cond.shape[0]
+    last_dim = cond.dim() - 1
+    out = cond.transpose(dim, last_dim)
+    out = 0. * out[..., :1]
+    out = out.transpose(dim, last_dim)
+    mask = torch.zeros((B, 1), device=out.device).int()
+    assert cond.dim() == out.dim()
+    return out, mask
+
+
+def nullify_wav(cond: WavCondition) -> WavCondition:
+    """Transform a WavCondition to a nullified WavCondition.
+    It replaces the wav by a null tensor, forces its length to 0, and replaces metadata by dummy attributes.
+
+    Args:
+        cond (WavCondition): Wav condition with wav, tensor of shape [B, T].
+    Returns:
+        WavCondition: Nullified wav condition.
+    """
+    null_wav, _ = nullify_condition((cond.wav, torch.zeros_like(cond.wav)), dim=cond.wav.dim() - 1)
+    return WavCondition(
+        wav=null_wav,
+        length=torch.tensor([0] * cond.wav.shape[0], device=cond.wav.device),
+        sample_rate=cond.sample_rate,
+        path=[None] * cond.wav.shape[0],
+        seek_time=[None] * cond.wav.shape[0],
+    )
+
+def nullify_chord(cond: ChordCondition) -> ChordCondition:
+    """Transform a ChordCondition to a nullified ChordCondition.
+    It replaces the wav by a null tensor, forces its length to 0, and replaces metadata by dummy attributes.
+
+    Args:
+        cond (ChordCondition): Chord condition with chord, tensor of shape [B, C, T].
+    Returns:
+        ChordCondition: Nullified chord condition.
+    """
+    null_chord, _ = nullify_condition((cond.chord, torch.zeros_like(cond.chord)), dim=cond.chord.dim() - 1)
+    return ChordCondition(
+        chord=null_chord,
+        length=torch.tensor([0] * cond.chord.shape[0], device=cond.chord.device),
+        bpm=[None] * cond.chord.shape[0],
+        path=[None] * cond.chord.shape[0],
+        seek_frame=[None] * cond.chord.shape[0],
+    )
+
+
+def nullify_beat(cond: BeatCondition) -> BeatCondition:
+    """
+    Args:
+        cond (ChordCondition): Chord condition with chord, tensor of shape [B, C, T].
+    Returns:
+        ChordCondition: Nullified chord condition.
+    """
+    null_beat, _ = nullify_condition((cond.beat, torch.zeros_like(cond.beat)), dim=cond.beat.dim() - 1)
+    return BeatCondition(
+        beat=null_beat,
+        length=torch.tensor([0] * cond.beat.shape[0], device=cond.beat.device),
+        bpm=[None] * cond.beat.shape[0],
+        path=[None] * cond.beat.shape[0],
+        seek_frame=[None] * cond.beat.shape[0],
+    )
+
+
+def nullify_joint_embed(embed: JointEmbedCondition) -> JointEmbedCondition:
+    """Nullify the joint embedding condition by replacing it by a null tensor, forcing its length to 0,
+    and replacing metadata by dummy attributes.
+
+    Args:
+        cond (JointEmbedCondition): Joint embedding condition with wav and text, wav tensor of shape [B, C, T].
+    """
+    null_wav, _ = nullify_condition((embed.wav, torch.zeros_like(embed.wav)), dim=embed.wav.dim() - 1)
+    return JointEmbedCondition(
+        wav=null_wav, text=[None] * len(embed.text),
+        length=torch.LongTensor([0]).to(embed.wav.device),
+        sample_rate=embed.sample_rate,
+        path=[None] * embed.wav.shape[0],
+        seek_time=[0] * embed.wav.shape[0],
+    )
+
+
+class Tokenizer:
+    """Base tokenizer implementation
+    (in case we want to introduce more advances tokenizers in the future).
+    """
+    def __call__(self, texts: tp.List[tp.Optional[str]]) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        raise NotImplementedError()
+
+
+class WhiteSpaceTokenizer(Tokenizer):
+    """This tokenizer should be used for natural language descriptions.
+    For example:
+    ["he didn't, know he's going home.", 'shorter sentence'] =>
+    [[78, 62, 31,  4, 78, 25, 19, 34],
+    [59, 77,  0,  0,  0,  0,  0,  0]]
+    """
+    PUNCTUATION = "?:!.,;"
+
+    def __init__(self, n_bins: int, pad_idx: int = 0, language: str = "en_core_web_sm",
+                 lemma: bool = True, stopwords: bool = True) -> None:
+        self.n_bins = n_bins
+        self.pad_idx = pad_idx
+        self.lemma = lemma
+        self.stopwords = stopwords
+        try:
+            self.nlp = spacy.load(language)
+        except IOError:
+            spacy.cli.download(language)  # type: ignore
+            self.nlp = spacy.load(language)
+
+    @tp.no_type_check
+    def __call__(self, texts: tp.List[tp.Optional[str]],
+                 return_text: bool = False) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        """Take a list of strings and convert them to a tensor of indices.
+
+        Args:
+            texts (list[str]): List of strings.
+            return_text (bool, optional): Whether to return text as additional tuple item. Defaults to False.
+        Returns:
+            tuple[torch.Tensor, torch.Tensor]:
+                - Indices of words in the LUT.
+                - And a mask indicating where the padding tokens are
+        """
+        output, lengths = [], []
+        texts = deepcopy(texts)
+        for i, text in enumerate(texts):
+            # if current sample doesn't have a certain attribute, replace with pad token
+            if text is None:
+                output.append(torch.Tensor([self.pad_idx]))
+                lengths.append(0)
+                continue
+
+            # convert numbers to words
+            text = re.sub(r"(\d+)", lambda x: num2words(int(x.group(0))), text)  # type: ignore
+            # normalize text
+            text = self.nlp(text)  # type: ignore
+            # remove stopwords
+            if self.stopwords:
+                text = [w for w in text if not w.is_stop]  # type: ignore
+            # remove punctuation
+            text = [w for w in text if w.text not in self.PUNCTUATION]  # type: ignore
+            # lemmatize if needed
+            text = [getattr(t, "lemma_" if self.lemma else "text") for t in text]  # type: ignore
+
+            texts[i] = " ".join(text)
+            lengths.append(len(text))
+            # convert to tensor
+            tokens = torch.Tensor([hash_trick(w, self.n_bins) for w in text])
+            output.append(tokens)
+
+        mask = length_to_mask(torch.IntTensor(lengths)).int()
+        padded_output = pad_sequence(output, padding_value=self.pad_idx).int().t()
+        if return_text:
+            return padded_output, mask, texts  # type: ignore
+        return padded_output, mask
+
+
+class NoopTokenizer(Tokenizer):
+    """This tokenizer should be used for global conditioners such as: artist, genre, key, etc.
+    The difference between this and WhiteSpaceTokenizer is that NoopTokenizer does not split
+    strings, so "Jeff Buckley" will get it's own index. Whereas WhiteSpaceTokenizer will
+    split it to ["Jeff", "Buckley"] and return an index per word.
+
+    For example:
+    ["Queen", "ABBA", "Jeff Buckley"] => [43, 55, 101]
+    ["Metal", "Rock", "Classical"] => [0, 223, 51]
+    """
+    def __init__(self, n_bins: int, pad_idx: int = 0):
+        self.n_bins = n_bins
+        self.pad_idx = pad_idx
+
+    def __call__(self, texts: tp.List[tp.Optional[str]]) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        output, lengths = [], []
+        for text in texts:
+            # if current sample doesn't have a certain attribute, replace with pad token
+            if text is None:
+                output.append(self.pad_idx)
+                lengths.append(0)
+            else:
+                output.append(hash_trick(text, self.n_bins))
+                lengths.append(1)
+
+        tokens = torch.LongTensor(output).unsqueeze(1)
+        mask = length_to_mask(torch.IntTensor(lengths)).int()
+        return tokens, mask
+
+
+class BaseConditioner(nn.Module):
+    """Base model for all conditioner modules.
+    We allow the output dim to be different than the hidden dim for two reasons:
+    1) keep our LUTs small when the vocab is large;
+    2) make all condition dims consistent.
+
+    Args:
+        dim (int): Hidden dim of the model.
+        output_dim (int): Output dim of the conditioner.
+    """
+    def __init__(self, dim: int, output_dim: int):
+        super().__init__()
+        self.dim = dim
+        self.output_dim = output_dim
+        self.output_proj = nn.Linear(dim, output_dim)
+
+    def tokenize(self, *args, **kwargs) -> tp.Any:
+        """Should be any part of the processing that will lead to a synchronization
+        point, e.g. BPE tokenization with transfer to the GPU.
+
+        The returned value will be saved and return later when calling forward().
+        """
+        raise NotImplementedError()
+
+    def forward(self, inputs: tp.Any) -> ConditionType:
+        """Gets input that should be used as conditioning (e.g, genre, description or a waveform).
+        Outputs a ConditionType, after the input data was embedded as a dense vector.
+
+        Returns:
+            ConditionType:
+                - A tensor of size [B, T, D] where B is the batch size, T is the length of the
+                  output embedding and D is the dimension of the embedding.
+                - And a mask indicating where the padding tokens.
+        """
+        raise NotImplementedError()
+
+
+class TextConditioner(BaseConditioner):
+    ...
+
+
+class LUTConditioner(TextConditioner):
+    """Lookup table TextConditioner.
+
+    Args:
+        n_bins (int): Number of bins.
+        dim (int): Hidden dim of the model (text-encoder/LUT).
+        output_dim (int): Output dim of the conditioner.
+        tokenizer (str): Name of the tokenizer.
+        pad_idx (int, optional): Index for padding token. Defaults to 0.
+    """
+    def __init__(self, n_bins: int, dim: int, output_dim: int, tokenizer: str, pad_idx: int = 0):
+        super().__init__(dim, output_dim)
+        self.embed = nn.Embedding(n_bins, dim)
+        self.tokenizer: Tokenizer
+        if tokenizer == 'whitespace':
+            self.tokenizer = WhiteSpaceTokenizer(n_bins, pad_idx=pad_idx)
+        elif tokenizer == 'noop':
+            self.tokenizer = NoopTokenizer(n_bins, pad_idx=pad_idx)
+        else:
+            raise ValueError(f"unrecognized tokenizer `{tokenizer}`.")
+
+    def tokenize(self, x: tp.List[tp.Optional[str]]) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        device = self.embed.weight.device
+        tokens, mask = self.tokenizer(x)
+        tokens, mask = tokens.to(device), mask.to(device)
+        return tokens, mask
+
+    def forward(self, inputs: tp.Tuple[torch.Tensor, torch.Tensor]) -> ConditionType:
+        tokens, mask = inputs
+        embeds = self.embed(tokens)
+        embeds = self.output_proj(embeds)
+        embeds = (embeds * mask.unsqueeze(-1))
+        return embeds, mask
+
+
+class T5Conditioner(TextConditioner):
+    """T5-based TextConditioner.
+
+    Args:
+        name (str): Name of the T5 model.
+        output_dim (int): Output dim of the conditioner.
+        finetune (bool): Whether to fine-tune T5 at train time.
+        device (str): Device for T5 Conditioner.
+        autocast_dtype (tp.Optional[str], optional): Autocast dtype.
+        word_dropout (float, optional): Word dropout probability.
+        normalize_text (bool, optional): Whether to apply text normalization.
+    """
+    MODELS = ["t5-small", "t5-base", "t5-large", "t5-3b", "t5-11b",
+              "google/flan-t5-small", "google/flan-t5-base", "google/flan-t5-large",
+              "google/flan-t5-xl", "google/flan-t5-xxl"]
+    MODELS_DIMS = {
+        "t5-small": 512,
+        "t5-base": 768,
+        "t5-large": 1024,
+        "t5-3b": 1024,
+        "t5-11b": 1024,
+        "google/flan-t5-small": 512,
+        "google/flan-t5-base": 768,
+        "google/flan-t5-large": 1024,
+        "google/flan-t5-3b": 1024,
+        "google/flan-t5-11b": 1024,
+    }
+
+    def __init__(self, name: str, output_dim: int, finetune: bool, device: str,
+                 autocast_dtype: tp.Optional[str] = 'float32', word_dropout: float = 0.,
+                 normalize_text: bool = False):
+        assert name in self.MODELS, f"Unrecognized t5 model name (should in {self.MODELS})"
+        super().__init__(self.MODELS_DIMS[name], output_dim)
+        self.device = device
+        self.name = name
+        self.finetune = finetune
+        self.word_dropout = word_dropout
+        if autocast_dtype is None or self.device == 'cpu':
+            self.autocast = TorchAutocast(enabled=False)
+            if self.device != 'cpu':
+                logger.warning("T5 has no autocast, this might lead to NaN")
+        else:
+            dtype = getattr(torch, autocast_dtype)
+            assert isinstance(dtype, torch.dtype)
+            logger.info(f"T5 will be evaluated with autocast as {autocast_dtype}")
+            self.autocast = TorchAutocast(enabled=True, device_type=self.device, dtype=dtype)
+        # Let's disable logging temporarily because T5 will vomit some errors otherwise.
+        # thanks https://gist.github.com/simon-weber/7853144
+        previous_level = logging.root.manager.disable
+        logging.disable(logging.ERROR)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")
+            try:
+                self.t5_tokenizer = T5Tokenizer.from_pretrained(name)
+                t5 = T5EncoderModel.from_pretrained(name).train(mode=finetune)
+            finally:
+                logging.disable(previous_level)
+        if finetune:
+            self.t5 = t5
+        else:
+            # this makes sure that the t5 models is not part
+            # of the saved checkpoint
+            self.__dict__['t5'] = t5.to(device)
+
+        self.normalize_text = normalize_text
+        if normalize_text:
+            self.text_normalizer = WhiteSpaceTokenizer(1, lemma=True, stopwords=True)
+
+    def tokenize(self, x: tp.List[tp.Optional[str]]) -> tp.Dict[str, torch.Tensor]:
+        # if current sample doesn't have a certain attribute, replace with empty string
+        entries: tp.List[str] = [xi if xi is not None else "" for xi in x]
+        if self.normalize_text:
+            _, _, entries = self.text_normalizer(entries, return_text=True)
+        if self.word_dropout > 0. and self.training:
+            new_entries = []
+            for entry in entries:
+                words = [word for word in entry.split(" ") if random.random() >= self.word_dropout]
+                new_entries.append(" ".join(words))
+            entries = new_entries
+
+        empty_idx = torch.LongTensor([i for i, xi in enumerate(entries) if xi == ""])
+
+        inputs = self.t5_tokenizer(entries, return_tensors='pt', padding=True).to(self.device)
+        mask = inputs['attention_mask']
+        mask[empty_idx, :] = 0  # zero-out index where the input is non-existant
+        return inputs
+
+    def forward(self, inputs: tp.Dict[str, torch.Tensor]) -> ConditionType:
+        mask = inputs['attention_mask']
+        with torch.set_grad_enabled(self.finetune), self.autocast:
+            embeds = self.t5(**inputs).last_hidden_state
+        embeds = self.output_proj(embeds.to(self.output_proj.weight))
+        embeds = (embeds * mask.unsqueeze(-1))
+        return embeds, mask
+
+
+class WaveformConditioner(BaseConditioner):
+    """Base class for all conditioners that take a waveform as input.
+    Classes that inherit must implement `_get_wav_embedding` that outputs
+    a continuous tensor, and `_downsampling_factor` that returns the down-sampling
+    factor of the embedding model.
+
+    Args:
+        dim (int): The internal representation dimension.
+        output_dim (int): Output dimension.
+        device (tp.Union[torch.device, str]): Device.
+    """
+    def __init__(self, dim: int, output_dim: int, device: tp.Union[torch.device, str]):
+        super().__init__(dim, output_dim)
+        self.device = device
+
+    def tokenize(self, x: WavCondition) -> WavCondition:
+        wav, length, sample_rate, path, seek_time = x
+        assert length is not None
+        return WavCondition(wav.to(self.device), length.to(self.device), sample_rate, path, seek_time)
+
+    def _get_wav_embedding(self, x: WavCondition) -> torch.Tensor:
+        """Gets as input a WavCondition and returns a dense embedding."""
+        raise NotImplementedError()
+
+    def _downsampling_factor(self):
+        """Returns the downsampling factor of the embedding model."""
+        raise NotImplementedError()
+
+    def forward(self, x: WavCondition) -> ConditionType:
+        """Extract condition embedding and mask from a waveform and its metadata.
+        Args:
+            x (WavCondition): Waveform condition containing raw waveform and metadata.
+        Returns:
+            ConditionType: a dense vector representing the conditioning along with its mask
+        """
+        wav, lengths, *_ = x
+        with torch.no_grad():
+            embeds = self._get_wav_embedding(x)
+        embeds = embeds.to(self.output_proj.weight)
+        embeds = self.output_proj(embeds)
+
+        if lengths is not None:
+            lengths = lengths / self._downsampling_factor()
+            mask = length_to_mask(lengths, max_len=embeds.shape[1]).int()  # type: ignore
+        else:
+            mask = torch.ones_like(embeds)
+        embeds = (embeds * mask.unsqueeze(2).to(self.device))
+
+        return embeds, mask
+
+
+class ChromaStemConditioner(WaveformConditioner):
+    """Chroma conditioner based on stems.
+    The ChromaStemConditioner uses DEMUCS to first filter out drums and bass, as
+    the drums and bass often dominate the chroma leading to the chroma features
+    not containing information about the melody.
+
+    Args:
+        output_dim (int): Output dimension for the conditioner.
+        sample_rate (int): Sample rate for the chroma extractor.
+        n_chroma (int): Number of chroma bins for the chroma extractor.
+        radix2_exp (int): Size of stft window for the chroma extractor (power of 2, e.g. 12 -> 2^12).
+        duration (int): duration used during training. This is later used for correct padding
+            in case we are using chroma as prefix.
+        match_len_on_eval (bool, optional): if True then all chromas are padded to the training
+            duration. Defaults to False.
+        eval_wavs (str, optional): path to a dataset manifest with waveform, this waveforms are used as
+            conditions during eval (for cases where we don't want to leak test conditions like MusicCaps).
+            Defaults to None.
+        n_eval_wavs (int, optional): limits the number of waveforms used for conditioning. Defaults to 0.
+        device (tp.Union[torch.device, str], optional): Device for the conditioner.
+        **kwargs: Additional parameters for the chroma extractor.
+    """
+    def __init__(self, output_dim: int, sample_rate: int, n_chroma: int, radix2_exp: int,
+                 duration: float, match_len_on_eval: bool = True, eval_wavs: tp.Optional[str] = None,
+                 n_eval_wavs: int = 0, cache_path: tp.Optional[tp.Union[str, Path]] = None,
+                 device: tp.Union[torch.device, str] = 'cpu', **kwargs):
+        from demucs import pretrained
+        super().__init__(dim=n_chroma, output_dim=output_dim, device=device)
+        self.autocast = TorchAutocast(enabled=device != 'cpu', device_type=self.device, dtype=torch.float32)
+        self.sample_rate = sample_rate
+        self.match_len_on_eval = match_len_on_eval
+        self.duration = duration
+        self.__dict__['demucs'] = pretrained.get_model('htdemucs').to(device)
+        stem_sources: list = self.demucs.sources  # type: ignore
+        self.stem_indices = torch.LongTensor([stem_sources.index('vocals'), stem_sources.index('other')]).to(device)
+        self.chroma = ChromaExtractor(sample_rate=sample_rate, n_chroma=n_chroma,
+                                      radix2_exp=radix2_exp, **kwargs).to(device)
+        self.chroma_len = self._get_chroma_len()
+        self.eval_wavs: tp.Optional[torch.Tensor] = self._load_eval_wavs(eval_wavs, n_eval_wavs)
+        self.cache = None
+        if cache_path is not None:
+            self.cache = EmbeddingCache(Path(cache_path) / 'wav', self.device,
+                                        compute_embed_fn=self._get_full_chroma_for_cache,
+                                        extract_embed_fn=self._extract_chroma_chunk)
+
+    def _downsampling_factor(self) -> int:
+        return self.chroma.winhop
+
+    def _load_eval_wavs(self, path: tp.Optional[str], num_samples: int) -> tp.Optional[torch.Tensor]:
+        """Load pre-defined waveforms from a json.
+        These waveforms will be used for chroma extraction during evaluation.
+        This is done to make the evaluation on MusicCaps fair (we shouldn't see the chromas of MusicCaps).
+        """
+        if path is None:
+            return None
+
+        logger.info(f"Loading evaluation wavs from {path}")
+        from audiocraft.data.audio_dataset import AudioDataset
+        dataset: AudioDataset = AudioDataset.from_meta(
+            path, segment_duration=self.duration, min_audio_duration=self.duration,
+            sample_rate=self.sample_rate, channels=1)
+
+        if len(dataset) > 0:
+            eval_wavs = dataset.collater([dataset[i] for i in range(num_samples)]).to(self.device)
+            logger.info(f"Using {len(eval_wavs)} evaluation wavs for chroma-stem conditioner")
+            return eval_wavs
+        else:
+            raise ValueError("Could not find evaluation wavs, check lengths of wavs")
+
+    def reset_eval_wavs(self, eval_wavs: tp.Optional[torch.Tensor]) -> None:
+        self.eval_wavs = eval_wavs
+
+    def has_eval_wavs(self) -> bool:
+        return self.eval_wavs is not None
+
+    def _sample_eval_wavs(self, num_samples: int) -> torch.Tensor:
+        """Sample wavs from a predefined list."""
+        assert self.eval_wavs is not None, "Cannot sample eval wavs as no eval wavs provided."
+        total_eval_wavs = len(self.eval_wavs)
+        out = self.eval_wavs
+        if num_samples > total_eval_wavs:
+            out = self.eval_wavs.repeat(num_samples // total_eval_wavs + 1, 1, 1)
+        return out[torch.randperm(len(out))][:num_samples]
+
+    def _get_chroma_len(self) -> int:
+        """Get length of chroma during training."""
+        dummy_wav = torch.zeros((1, int(self.sample_rate * self.duration)), device=self.device)
+        dummy_chr = self.chroma(dummy_wav)
+        return dummy_chr.shape[1]
+
+    @torch.no_grad()
+    def _get_stemmed_wav(self, wav: torch.Tensor, sample_rate: int) -> torch.Tensor:
+        """Get parts of the wav that holds the melody, extracting the main stems from the wav."""
+        from demucs.apply import apply_model
+        from demucs.audio import convert_audio
+        with self.autocast:
+            wav = convert_audio(
+                wav, sample_rate, self.demucs.samplerate, self.demucs.audio_channels)  # type: ignore
+            stems = apply_model(self.demucs, wav, device=self.device)
+            stems = stems[:, self.stem_indices]  # extract relevant stems for melody conditioning
+            mix_wav = stems.sum(1)  # merge extracted stems to single waveform
+            mix_wav = convert_audio(mix_wav, self.demucs.samplerate, self.sample_rate, 1)  # type: ignore
+            return mix_wav
+
+    @torch.no_grad()
+    def _extract_chroma(self, wav: torch.Tensor) -> torch.Tensor:
+        """Extract chroma features from the waveform."""
+        with self.autocast:
+            return self.chroma(wav)
+
+    @torch.no_grad()
+    def _compute_wav_embedding(self, wav: torch.Tensor, sample_rate: int) -> torch.Tensor:
+        """Compute wav embedding, applying stem and chroma extraction."""
+        # avoid 0-size tensors when we are working with null conds
+        if wav.shape[-1] == 1:
+            return self._extract_chroma(wav)
+        stems = self._get_stemmed_wav(wav, sample_rate)
+        chroma = self._extract_chroma(stems)
+        return chroma
+
+    @torch.no_grad()
+    def _get_full_chroma_for_cache(self, path: tp.Union[str, Path], x: WavCondition, idx: int) -> torch.Tensor:
+        """Extract chroma from the whole audio waveform at the given path."""
+        wav, sr = audio_read(path)
+        wav = wav[None].to(self.device)
+        wav = convert_audio(wav, sr, self.sample_rate, to_channels=1)
+        chroma = self._compute_wav_embedding(wav, self.sample_rate)[0]
+        return chroma
+
+    def _extract_chroma_chunk(self, full_chroma: torch.Tensor, x: WavCondition, idx: int) -> torch.Tensor:
+        """Extract a chunk of chroma from the full chroma derived from the full waveform."""
+        wav_length = x.wav.shape[-1]
+        seek_time = x.seek_time[idx]
+        assert seek_time is not None, (
+            "WavCondition seek_time is required "
+            "when extracting chroma chunks from pre-computed chroma.")
+        full_chroma = full_chroma.float()
+        frame_rate = self.sample_rate / self._downsampling_factor()
+        target_length = int(frame_rate * wav_length / self.sample_rate)
+        index = int(frame_rate * seek_time)
+        out = full_chroma[index: index + target_length]
+        out = F.pad(out[None], (0, 0, 0, target_length - out.shape[0]))[0]
+        return out.to(self.device)
+
+    @torch.no_grad()
+    def _get_wav_embedding(self, x: WavCondition) -> torch.Tensor:
+        """Get the wav embedding from the WavCondition.
+        The conditioner will either extract the embedding on-the-fly computing it from the condition wav directly
+        or will rely on the embedding cache to load the pre-computed embedding if relevant.
+        """
+        sampled_wav: tp.Optional[torch.Tensor] = None
+        if not self.training and self.eval_wavs is not None:
+            warn_once(logger, "Using precomputed evaluation wavs!")
+            sampled_wav = self._sample_eval_wavs(len(x.wav))
+
+        no_undefined_paths = all(p is not None for p in x.path)
+        no_nullified_cond = x.wav.shape[-1] > 1
+        if sampled_wav is not None:
+            chroma = self._compute_wav_embedding(sampled_wav, self.sample_rate)
+        elif self.cache is not None and no_undefined_paths and no_nullified_cond:
+            paths = [Path(p) for p in x.path if p is not None]
+            chroma = self.cache.get_embed_from_cache(paths, x)
+        else:
+            assert all(sr == x.sample_rate[0] for sr in x.sample_rate), "All sample rates in batch should be equal."
+            chroma = self._compute_wav_embedding(x.wav, x.sample_rate[0])
+
+        if self.match_len_on_eval:
+            B, T, C = chroma.shape
+            if T > self.chroma_len:
+                chroma = chroma[:, :self.chroma_len]
+                logger.debug(f"Chroma was truncated to match length! ({T} -> {chroma.shape[1]})")
+            elif T < self.chroma_len:
+                n_repeat = int(math.ceil(self.chroma_len / T))
+                chroma = chroma.repeat(1, n_repeat, 1)
+                chroma = chroma[:, :self.chroma_len]
+                logger.debug(f"Chroma was repeated to match length! ({T} -> {chroma.shape[1]})")
+
+        return chroma
+
+    def tokenize(self, x: WavCondition) -> WavCondition:
+        """Apply WavConditioner tokenization and populate cache if needed."""
+        x = super().tokenize(x)
+        no_undefined_paths = all(p is not None for p in x.path)
+        if self.cache is not None and no_undefined_paths:
+            paths = [Path(p) for p in x.path if p is not None]
+            self.cache.populate_embed_cache(paths, x)
+        return x
+
+class ChordProgressionConditioner(BaseConditioner):
+    """Chord progression conditioning supporting chord progression conditioning.
+
+    Args:
+        dim (int): Dimension.
+        output_dim (int): Output dimension.
+        device (str): Device.
+        attribute (str): Attribute used by the conditioner.
+        autocast_dtype (str): Autocast for the conditioner.
+    """
+
+    def __init__(self, output_dim: int, device: str, name: str):
+        n_chroma = 12
+        # n_chroma = 24
+        super().__init__(dim=n_chroma, output_dim=output_dim)
+        self.device = device
+    
+    def forward(self, x: ChordCondition) -> ConditionType:            
+        chord, lengths, *_ = x
+        embeds = chord.to(self.output_proj.weight) # chrod is already a tensor, [N, C]
+        embeds = self.output_proj(embeds)
+
+        if lengths is not None:
+            mask = length_to_mask(lengths, max_len=embeds.shape[1]).int()  # type: ignore
+        else:
+            mask = torch.ones_like(embeds)
+        embeds = (embeds * mask.unsqueeze(2).to(self.device))
+
+        return embeds, mask
+    
+    def tokenize(self, x: ChordCondition) -> ChordCondition:
+        """Apply ChordConditioner tokenization and populate cache if needed."""
+        chord, length, bpm, path, seek_frame = x
+        chord = F.pad(chord, (0, length[0] - chord.shape[-1])) # [B, C, t] -> [B, C, T]
+        chord = chord.permute(0, 2, 1) # [B, T, C]
+        x = ChordCondition(chord.to(self.device), length.to(self.device), bpm, path, seek_frame)
+        return x
+
+class BeatConditioner(BaseConditioner):
+    """Beat conditioning supporting beat conditioning.
+
+    Args:
+        dim (int): Dimension.
+        output_dim (int): Output dimension.
+        device (str): Device.
+        attribute (str): Attribute used by the conditioner.
+        autocast_dtype (str): Autocast for the conditioner.
+    """
+
+    def __init__(self, output_dim: int, device: str, name: str):
+        beat_channel = 1
+        super().__init__(dim=beat_channel, output_dim=output_dim)
+        self.device = device
+    
+    def forward(self, x: BeatCondition) -> ConditionType:            
+        beat, lengths, *_ = x
+        embeds = beat.to(self.output_proj.weight) # chrod is already a tensor, [N, C]
+        embeds = self.output_proj(embeds)
+
+        if lengths is not None:
+            mask = length_to_mask(lengths, max_len=embeds.shape[1]).int()  # type: ignore
+        else:
+            mask = torch.ones_like(embeds)
+        embeds = (embeds * mask.unsqueeze(2).to(self.device))
+
+        return embeds, mask
+    
+    def tokenize(self, x: BeatCondition) -> BeatCondition:
+        """Apply ChordConditioner tokenization and populate cache if needed."""
+        beat, length, bpm, path, seek_frame = x
+        beat = F.pad(beat, (0, length[0] - beat.shape[-1])) # [B, C, t] -> [B, C, T]
+        beat = beat.permute(0, 2, 1) # [B, T, C]
+        x = BeatCondition(beat.to(self.device), length.to(self.device), bpm, path, seek_frame)
+        return x
+
+
+class JointEmbeddingConditioner(BaseConditioner):
+    """Joint embedding conditioning supporting both audio or text conditioning.
+
+    Args:
+        dim (int): Dimension.
+        output_dim (int): Output dimension.
+        device (str): Device.
+        attribute (str): Attribute used by the conditioner.
+        autocast_dtype (str): Autocast for the conditioner.
+        quantize (bool): Whether to quantize the CLAP embedding.
+        n_q (int): Number of residual quantizers (used if quantize is true).
+        bins (int): Quantizers' codebooks size (used if quantize is true).
+        kwargs: Additional parameters for residual vector quantizer.
+    """
+    def __init__(self, dim: int, output_dim: int, device: str, attribute: str,
+                 autocast_dtype: tp.Optional[str] = 'float32', quantize: bool = True,
+                 n_q: int = 12, bins: int = 1024, **kwargs):
+        super().__init__(dim=dim, output_dim=output_dim)
+        self.device = device
+        self.attribute = attribute
+        if autocast_dtype is None or device == 'cpu':
+            self.autocast = TorchAutocast(enabled=False)
+            logger.warning("JointEmbeddingConditioner has no autocast, this might lead to NaN.")
+        else:
+            dtype = getattr(torch, autocast_dtype)
+            assert isinstance(dtype, torch.dtype)
+            logger.info(f"JointEmbeddingConditioner will be evaluated with autocast as {autocast_dtype}.")
+            self.autocast = TorchAutocast(enabled=True, device_type=self.device, dtype=dtype)
+        # residual vector quantizer to discretize the conditioned embedding
+        self.quantizer: tp.Optional[ResidualVectorQuantizer] = None
+        if quantize:
+            self.quantizer = ResidualVectorQuantizer(dim, n_q=n_q, bins=bins, **kwargs)
+
+    def _get_embed(self, x: JointEmbedCondition) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        """Get joint embedding in latent space from the inputs.
+
+        Returns:
+            tuple[torch.Tensor, torch.Tensor]: Tensor for the latent embedding
+                and corresponding empty indexes.
+        """
+        raise NotImplementedError()
+
+    def forward(self, x: JointEmbedCondition) -> ConditionType:
+        with self.autocast:
+            embed, empty_idx = self._get_embed(x)
+            if self.quantizer is not None:
+                embed = embed.view(-1, self.dim, 1)
+                q_res = self.quantizer(embed, frame_rate=1)
+                out_embed = q_res.x.view(-1, self.dim)
+            else:
+                out_embed = embed
+            out_embed = self.output_proj(out_embed).view(-1, 1, self.output_dim)
+            mask = torch.ones(*out_embed.shape[:2], device=out_embed.device)
+            mask[empty_idx, :] = 0  # zero-out index where the input is non-existant
+            out_embed = (out_embed * mask.unsqueeze(-1))
+            return out_embed, mask
+
+    def tokenize(self, x: JointEmbedCondition) -> JointEmbedCondition:
+        return x
+
+
+class CLAPEmbeddingConditioner(JointEmbeddingConditioner):
+    """Joint Embedding conditioner based on pre-trained CLAP model.
+
+    This CLAP-based conditioner supports a caching mechanism
+    over the computed embeddings for faster training.
+
+    Args:
+        dim (int): Dimension.
+        output_dim (int): Output dimension.
+        device (str): Device.
+        attribute (str): Attribute used by the conditioner.
+        quantize (bool): Whether to quantize the CLAP embedding.
+        n_q (int): Number of residual quantizers (used if quantize is true).
+        bins (int): Quantizers' codebooks size (used if quantize is true).
+        checkpoint (str): Path to CLAP checkpoint.
+        model_arch (str): CLAP model architecture.
+        enable_fusion (bool): Enable fusion for CLAP model.
+        sample_rate (int): Sample rate used by CLAP model.
+        max_audio_length (float): Maximum audio length for CLAP model.
+        audio_stride (float): Stride to use for getting a CLAP embedding on the full sequence.
+        normalize (bool): Whether to normalize the CLAP embedding.
+        text_p (float): Probability of using text representation instead of audio at train time.
+        batch_size (Optional[int]): Batch size for CLAP embedding computation.
+        autocast_dtype (str): Autocast for the conditioner.
+        cache_path (Optional[str]): Path for pre-computed embeddings caching.
+        kwargs: Additional parameters for residual vector quantizer.
+    """
+    def __init__(self, dim: int, output_dim: int, device: str, attribute: str,
+                 quantize: bool, n_q: int, bins: int, checkpoint: tp.Union[str, Path], model_arch: str,
+                 enable_fusion: bool, sample_rate: int, max_audio_length: int, audio_stride: int,
+                 normalize: bool, text_p: bool, batch_size: tp.Optional[int] = None,
+                 autocast_dtype: tp.Optional[str] = 'float32', cache_path: tp.Optional[str] = None, **kwargs):
+        try:
+            import laion_clap  # type: ignore
+        except ImportError:
+            raise ImportError("Please install CLAP to use the CLAPEmbeddingConditioner: 'pip install laion_clap'")
+        checkpoint = AudioCraftEnvironment.resolve_reference_path(checkpoint)
+        clap_tokenize = RobertaTokenizer.from_pretrained('roberta-base')
+        clap_model = laion_clap.CLAP_Module(enable_fusion=enable_fusion, amodel=model_arch)
+        load_clap_state_dict(clap_model, checkpoint)
+        clap_model.eval()
+        clap_model.to(device)
+        super().__init__(dim=dim, output_dim=output_dim, device=device, attribute=attribute,
+                         autocast_dtype=autocast_dtype, quantize=quantize, n_q=n_q, bins=bins,
+                         **kwargs)
+        self.checkpoint = checkpoint
+        self.enable_fusion = enable_fusion
+        self.model_arch = model_arch
+        self.clap: laion_clap.CLAP_Module
+        self.clap_tokenize: RobertaTokenizer
+        self.clap_sample_rate = sample_rate
+        self.clap_max_frames = int(self.clap_sample_rate * max_audio_length)
+        self.clap_stride = int(self.clap_sample_rate * audio_stride)
+        self.batch_size = batch_size or 1
+        self.normalize = normalize
+        self.text_p = text_p
+        self.__dict__['clap_tokenize'] = clap_tokenize
+        self.__dict__['clap'] = clap_model
+        self.wav_cache, self.text_cache = None, None
+        if cache_path is not None:
+            self.wav_cache = EmbeddingCache(Path(cache_path) / 'wav', self.device,
+                                            compute_embed_fn=self._get_wav_embedding_for_cache,
+                                            extract_embed_fn=self._extract_wav_embedding_chunk)
+            self.text_cache = EmbeddingCache(Path(cache_path) / 'text', self.device,
+                                             compute_embed_fn=self._get_text_embedding_for_cache)
+
+    def _tokenizer(self, texts: tp.Union[str, tp.List[str]]) -> dict:
+        # we use the default params from CLAP module here as well
+        return self.clap_tokenize(texts, padding="max_length", truncation=True, max_length=77, return_tensors="pt")
+
+    def _compute_text_embedding(self, text: tp.List[str]) -> torch.Tensor:
+        """Compute text embedding from CLAP model on a given a batch of text.
+
+        Args:
+            text (list[str]): List of text for the batch, with B items.
+        Returns:
+            torch.Tensor: CLAP embedding derived from text, of shape [B, 1, D], with D the CLAP embedding dimension.
+        """
+        with torch.no_grad():
+            embed = self.clap.get_text_embedding(text, tokenizer=self._tokenizer, use_tensor=True)
+            return embed.view(embed.size(0), 1, embed.size(-1))
+
+    def _get_text_embedding_for_cache(self, path: tp.Union[Path, str],
+                                      x: JointEmbedCondition, idx: int) -> torch.Tensor:
+        """Get text embedding function for the cache."""
+        text = x.text[idx]
+        text = text if text is not None else ""
+        return self._compute_text_embedding([text])[0]
+
+    def _preprocess_wav(self, wav: torch.Tensor, length: torch.Tensor, sample_rates: tp.List[int]) -> torch.Tensor:
+        """Preprocess wav to expected format by CLAP model.
+
+        Args:
+            wav (torch.Tensor): Audio wav, of shape [B, C, T].
+            length (torch.Tensor): Actual length of the audio for each item in the batch, of shape [B].
+            sample_rates (list[int]): Sample rates for each sample in the batch
+        Returns:
+            torch.Tensor: Audio wav of shape [B, T].
+        """
+        assert wav.dim() == 3, "Expecting wav to be [B, C, T]"
+        if sample_rates is not None:
+            _wav = []
+            for i, audio in enumerate(wav):
+                sr = sample_rates[i]
+                audio = convert_audio(audio, from_rate=sr, to_rate=self.clap_sample_rate, to_channels=1)
+                _wav.append(audio)
+            wav = torch.stack(_wav, dim=0)
+        wav = wav.mean(dim=1)
+        return wav
+
+    def _compute_wav_embedding(self, wav: torch.Tensor, length: torch.Tensor,
+                               sample_rates: tp.List[int], reduce_mean: bool = False) -> torch.Tensor:
+        """Compute audio wave embedding from CLAP model.
+
+        Since CLAP operates on a fixed sequence length audio inputs and we need to process longer audio sequences,
+        we calculate the wav embeddings on `clap_max_frames` windows with `clap_stride`-second stride and
+        average the resulting embeddings.
+
+        Args:
+            wav (torch.Tensor): Audio wav, of shape [B, C, T].
+            length (torch.Tensor): Actual length of the audio for each item in the batch, of shape [B].
+            sample_rates (list[int]): Sample rates for each sample in the batch.
+            reduce_mean (bool): Whether to get the average tensor.
+        Returns:
+            torch.Tensor: Audio embedding of shape [B, F, D], F being the number of chunks, D the dimension.
+        """
+        with torch.no_grad():
+            wav = self._preprocess_wav(wav, length, sample_rates)
+            B, T = wav.shape
+            if T >= self.clap_max_frames:
+                wav = wav.unfold(-1, self.clap_max_frames, self.clap_stride)  # [B, F, T]
+            else:
+                wav = wav.view(-1, 1, T)  # [B, F, T] with F=1
+            wav = einops.rearrange(wav, 'b f t -> (b f) t')
+            embed_list = []
+            for i in range(0, wav.size(0), self.batch_size):
+                _wav = wav[i:i+self.batch_size, ...]
+                _embed = self.clap.get_audio_embedding_from_data(_wav, use_tensor=True)
+                embed_list.append(_embed)
+            embed = torch.cat(embed_list, dim=0)
+            embed = einops.rearrange(embed, '(b f) d -> b f d', b=B)
+            if reduce_mean:
+                embed = embed.mean(dim=1, keepdim=True)
+            return embed  # [B, F, D] with F=1 if reduce_mean is True
+
+    def _get_wav_embedding_for_cache(self, path: tp.Union[str, Path],
+                                     x: JointEmbedCondition, idx: int) -> torch.Tensor:
+        """Compute audio wave embedding for the cache.
+        The embedding is computed on a given audio read from file.
+
+        Args:
+            path (str or Path): Path to the full audio file.
+        Returns:
+            torch.Tensor: Single-item tensor of shape [F, D], F being the number of chunks, D the dimension.
+        """
+        wav, sr = audio_read(path)  # [C, T]
+        wav = wav.unsqueeze(0).to(self.device)  # [1, C, T]
+        wav_len = torch.LongTensor([wav.shape[-1]]).to(self.device)
+        embed = self._compute_wav_embedding(wav, wav_len, [sr], reduce_mean=False)  # [B, F, D]
+        return embed.squeeze(0)  # [F, D]
+
+    def _extract_wav_embedding_chunk(self, full_embed: torch.Tensor, x: JointEmbedCondition, idx: int) -> torch.Tensor:
+        """Extract the chunk of embedding matching the seek_time and length from the full CLAP audio embedding.
+
+        Args:
+            full_embed (torch.Tensor): CLAP embedding computed on the full wave, of shape [F, D].
+            x (JointEmbedCondition): Joint embedding condition for the full batch.
+            idx (int): Index considered for the given embedding to extract.
+        Returns:
+            torch.Tensor: Wav embedding averaged on sliding window, of shape [1, D].
+        """
+        sample_rate = x.sample_rate[idx]
+        seek_time = x.seek_time[idx]
+        seek_time = 0. if seek_time is None else seek_time
+        clap_stride = int(self.clap_stride / self.clap_sample_rate) * sample_rate
+        end_seek_time = seek_time + self.clap_max_frames / self.clap_sample_rate
+        start_offset = int(seek_time * sample_rate // clap_stride)
+        end_offset = int(end_seek_time * sample_rate // clap_stride)
+        wav_embed = full_embed[start_offset:end_offset, ...]
+        wav_embed = wav_embed.mean(dim=0, keepdim=True)
+        return wav_embed.to(self.device)  # [F, D]
+
+    def _get_text_embedding(self, x: JointEmbedCondition) -> torch.Tensor:
+        """Get CLAP embedding from a batch of text descriptions."""
+        no_nullified_cond = x.wav.shape[-1] > 1  # we don't want to read from cache when condition dropout
+        if self.text_cache is not None and no_nullified_cond:
+            assert all(p is not None for p in x.path), "Cache requires all JointEmbedCondition paths to be provided"
+            paths = [Path(p) for p in x.path if p is not None]
+            embed = self.text_cache.get_embed_from_cache(paths, x)
+        else:
+            text = [xi if xi is not None else "" for xi in x.text]
+            embed = self._compute_text_embedding(text)
+        if self.normalize:
+            embed = torch.nn.functional.normalize(embed, p=2.0, dim=-1)
+        return embed
+
+    def _get_wav_embedding(self, x: JointEmbedCondition) -> torch.Tensor:
+        """Get CLAP embedding from a batch of audio tensors (and corresponding sample rates)."""
+        no_undefined_paths = all(p is not None for p in x.path)
+        no_nullified_cond = x.wav.shape[-1] > 1  # we don't want to read from cache when condition dropout
+        if self.wav_cache is not None and no_undefined_paths and no_nullified_cond:
+            paths = [Path(p) for p in x.path if p is not None]
+            embed = self.wav_cache.get_embed_from_cache(paths, x)
+        else:
+            embed = self._compute_wav_embedding(x.wav, x.length, x.sample_rate, reduce_mean=True)
+        if self.normalize:
+            embed = torch.nn.functional.normalize(embed, p=2.0, dim=-1)
+        return embed
+
+    def tokenize(self, x: JointEmbedCondition) -> JointEmbedCondition:
+        # Trying to limit as much as possible sync points when the cache is warm.
+        no_undefined_paths = all(p is not None for p in x.path)
+        if self.wav_cache is not None and no_undefined_paths:
+            assert all([p is not None for p in x.path]), "Cache requires all JointEmbedCondition paths to be provided"
+            paths = [Path(p) for p in x.path if p is not None]
+            self.wav_cache.populate_embed_cache(paths, x)
+        if self.text_cache is not None and no_undefined_paths:
+            assert all([p is not None for p in x.path]), "Cache requires all JointEmbedCondition paths to be provided"
+            paths = [Path(p) for p in x.path if p is not None]
+            self.text_cache.populate_embed_cache(paths, x)
+        return x
+
+    def _get_embed(self, x: JointEmbedCondition) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+        """Extract shared latent representation from either the wav or the text using CLAP."""
+        # decide whether to use text embedding at train time or not
+        use_text_embed = random.random() < self.text_p
+        if self.training and not use_text_embed:
+            embed = self._get_wav_embedding(x)
+            empty_idx = torch.LongTensor([])  # we assume we always have the audio wav
+        else:
+            embed = self._get_text_embedding(x)
+            empty_idx = torch.LongTensor([i for i, xi in enumerate(x.text) if xi is None or xi == ""])
+        return embed, empty_idx
+
+
+def dropout_condition(sample: ConditioningAttributes, condition_type: str, condition: str) -> ConditioningAttributes:
+    """Utility function for nullifying an attribute inside an ConditioningAttributes object.
+    If the condition is of type "wav", then nullify it using `nullify_condition` function.
+    If the condition is of any other type, set its value to None.
+    Works in-place.
+    """
+    if condition_type not in ['text', 'wav', 'beat', 'chord', 'joint_embed']:
+        raise ValueError(
+            "dropout_condition got an unexpected condition type!"
+            f" expected 'text', 'wav' or 'joint_embed' but got '{condition_type}'"
+        )
+
+    if condition not in getattr(sample, condition_type):
+        raise ValueError(
+            "dropout_condition received an unexpected condition!"
+            f" expected wav={sample.wav.keys()} and text={sample.text.keys()}"
+            f" but got '{condition}' of type '{condition_type}'!"
+        )
+
+    if condition_type == 'wav':
+        wav_cond = sample.wav[condition]
+        sample.wav[condition] = nullify_wav(wav_cond)
+    elif condition_type == 'beat':
+        beat_cond = sample.beat[condition]
+        sample.beat[condition] = nullify_beat(beat_cond)
+    elif condition_type == 'chord':
+        chord_cond = sample.chord[condition]
+        sample.chord[condition] = nullify_chord(chord_cond)
+    elif condition_type == 'joint_embed':
+        embed = sample.joint_embed[condition]
+        sample.joint_embed[condition] = nullify_joint_embed(embed)
+    else:
+        sample.text[condition] = None
+
+    return sample
+
+
+class DropoutModule(nn.Module):
+    """Base module for all dropout modules."""
+    def __init__(self, seed: int = 1234):
+        super().__init__()
+        self.rng = torch.Generator()
+        self.rng.manual_seed(seed)
+
+
+class AttributeDropout(DropoutModule):
+    """Dropout with a given probability per attribute.
+    This is different from the behavior of ClassifierFreeGuidanceDropout as this allows for attributes
+    to be dropped out separately. For example, "artist" can be dropped while "genre" remains.
+    This is in contrast to ClassifierFreeGuidanceDropout where if "artist" is dropped "genre"
+    must also be dropped.
+
+    Args:
+        p (tp.Dict[str, float]): A dict mapping between attributes and dropout probability. For example:
+            ...
+            "genre": 0.1,
+            "artist": 0.5,
+            "wav": 0.25,
+            ...
+        active_on_eval (bool, optional): Whether the dropout is active at eval. Default to False.
+        seed (int, optional): Random seed.
+    """
+    def __init__(self, p: tp.Dict[str, tp.Dict[str, float]], active_on_eval: bool = False, seed: int = 1234):
+        super().__init__(seed=seed)
+        self.active_on_eval = active_on_eval
+        # construct dict that return the values from p otherwise 0
+        self.p = {}
+        for condition_type, probs in p.items():
+            self.p[condition_type] = defaultdict(lambda: 0, probs)
+
+    def forward(self, samples: tp.List[ConditioningAttributes]) -> tp.List[ConditioningAttributes]:
+        """
+        Args:
+            samples (list[ConditioningAttributes]): List of conditions.
+        Returns:
+            list[ConditioningAttributes]: List of conditions after certain attributes were set to None.
+        """
+        if not self.training and not self.active_on_eval:
+            return samples
+
+        samples = deepcopy(samples)
+        for condition_type, ps in self.p.items():  # for condition types [text, wav]
+            for condition, p in ps.items():  # for attributes of each type (e.g., [artist, genre])
+                if torch.rand(1, generator=self.rng).item() < p:
+                    for sample in samples:
+                        dropout_condition(sample, condition_type, condition)
+        return samples
+
+    def __repr__(self):
+        return f"AttributeDropout({dict(self.p)})"
+
+
+class ClassifierFreeGuidanceDropout(DropoutModule):
+    """Classifier Free Guidance dropout.
+    All attributes are dropped with the same probability.
+
+    Args:
+        p (float): Probability to apply condition dropout during training.
+        seed (int): Random seed.
+    """
+    def __init__(self, p: float, seed: int = 1234):
+        super().__init__(seed=seed)
+        self.p = p
+
+    def forward(self, samples: tp.List[ConditioningAttributes]) -> tp.List[ConditioningAttributes]:
+        """
+        Args:
+            samples (list[ConditioningAttributes]): List of conditions.
+        Returns:
+            list[ConditioningAttributes]: List of conditions after all attributes were set to None.
+        """
+        if not self.training:
+            return samples
+
+        # decide on which attributes to drop in a batched fashion
+        drop = torch.rand(1, generator=self.rng).item() < self.p
+        if not drop:
+            return samples
+
+        # nullify conditions of all attributes
+        samples = deepcopy(samples)
+        for condition_type in ["wav", "text", "beat", "chord"]:
+            for sample in samples:
+                for condition in sample.attributes[condition_type]:
+                    dropout_condition(sample, condition_type, condition)
+        return samples
+
+    def __repr__(self):
+        return f"ClassifierFreeGuidanceDropout(p={self.p})"
+
+
+class ConditioningProvider(nn.Module):
+    """Prepare and provide conditions given all the supported conditioners.
+
+    Args:
+        conditioners (dict): Dictionary of conditioners.
+        device (torch.device or str, optional): Device for conditioners and output condition types.
+    """
+    def __init__(self, conditioners: tp.Dict[str, BaseConditioner], device: tp.Union[torch.device, str] = "cpu"):
+        super().__init__()
+        self.device = device
+        self.conditioners = nn.ModuleDict(conditioners)
+
+    @property
+    def joint_embed_conditions(self):
+        return [m.attribute for m in self.conditioners.values() if isinstance(m, JointEmbeddingConditioner)]
+
+    @property
+    def has_joint_embed_conditions(self):
+        return len(self.joint_embed_conditions) > 0
+
+    @property
+    def text_conditions(self):
+        return [k for k, v in self.conditioners.items() if isinstance(v, TextConditioner)]
+
+    @property
+    def wav_conditions(self):
+        return [k for k, v in self.conditioners.items() if isinstance(v, WaveformConditioner)]
+
+    @property
+    def beat_conditions(self):
+        return [k for k, v in self.conditioners.items() if isinstance(v, BeatConditioner)]
+
+    @property
+    def chord_conditions(self):
+        return [k for k, v in self.conditioners.items() if isinstance(v, ChordProgressionConditioner)]
+    
+    @property
+    def has_wav_condition(self):
+        return len(self.wav_conditions) > 0
+
+    def tokenize(self, inputs: tp.List[ConditioningAttributes]) -> tp.Dict[str, tp.Any]:
+        """Match attributes/wavs with existing conditioners in self, and compute tokenize them accordingly.
+        This should be called before starting any real GPU work to avoid synchronization points.
+        This will return a dict matching conditioner names to their arbitrary tokenized representations.
+
+        Args:
+            inputs (list[ConditioningAttributes]): List of ConditioningAttributes objects containing
+                text and wav conditions.
+        """
+        assert all([isinstance(x, ConditioningAttributes) for x in inputs]), (
+            "Got unexpected types input for conditioner! should be tp.List[ConditioningAttributes]",
+            f" but types were {set([type(x) for x in inputs])}"
+        )
+
+        output = {}
+        text = self._collate_text(inputs)
+        beats = self._collate_beats(inputs)
+        chords = self._collate_chords(inputs)
+        wavs = self._collate_wavs(inputs)
+        joint_embeds = self._collate_joint_embeds(inputs)
+
+        assert set(text.keys() | wavs.keys() | chords.keys() | beats.keys() | joint_embeds.keys()).issubset(set(self.conditioners.keys())), (
+            f"Got an unexpected attribute! Expected {self.conditioners.keys()}, ",
+            f"got {text.keys(), wavs.keys(), chords.keys(), beats.keys(), joint_embeds.keys()}"
+        )
+
+        for attribute, batch in chain(text.items(), wavs.items(), chords.items(), beats.items(), joint_embeds.items()):
+            output[attribute] = self.conditioners[attribute].tokenize(batch)
+        return output
+
+    def forward(self, tokenized: tp.Dict[str, tp.Any]) -> tp.Dict[str, ConditionType]:
+        """Compute pairs of `(embedding, mask)` using the configured conditioners and the tokenized representations.
+        The output is for example:
+        {
+            "genre": (torch.Tensor([B, 1, D_genre]), torch.Tensor([B, 1])),
+            "description": (torch.Tensor([B, T_desc, D_desc]), torch.Tensor([B, T_desc])),
+            ...
+        }
+
+        Args:
+            tokenized (dict): Dict of tokenized representations as returned by `tokenize()`.
+        """
+        output = {}
+        for attribute, inputs in tokenized.items():
+            condition, mask = self.conditioners[attribute](inputs)
+            output[attribute] = (condition, mask)
+        return output
+
+    def _collate_text(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, tp.List[tp.Optional[str]]]:
+        """Given a list of ConditioningAttributes objects, compile a dictionary where the keys
+        are the attributes and the values are the aggregated input per attribute.
+        For example:
+        Input:
+        [
+            ConditioningAttributes(text={"genre": "Rock", "description": "A rock song with a guitar solo"}, wav=...),
+            ConditioningAttributes(text={"genre": "Hip-hop", "description": "A hip-hop verse"}, wav=...),
+        ]
+        Output:
+        {
+            "genre": ["Rock", "Hip-hop"],
+            "description": ["A rock song with a guitar solo", "A hip-hop verse"]
+        }
+
+        Args:
+            samples (list of ConditioningAttributes): List of ConditioningAttributes samples.
+        Returns:
+            dict[str, list[str, optional]]: A dictionary mapping an attribute name to text batch.
+        """
+        out: tp.Dict[str, tp.List[tp.Optional[str]]] = defaultdict(list)
+        texts = [x.text for x in samples]
+        for text in texts:
+            for condition in self.text_conditions:
+                out[condition].append(text[condition])
+        return out
+
+    def _collate_wavs(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, WavCondition]:
+        """Generate a dict where the keys are attributes by which we fetch similar wavs,
+        and the values are Tensors of wavs according to said attributes.
+
+        *Note*: by the time the samples reach this function, each sample should have some waveform
+        inside the "wav" attribute. It should be either:
+        1. A real waveform
+        2. A null waveform due to the sample having no similar waveforms (nullified by the dataset)
+        3. A null waveform due to it being dropped in a dropout module (nullified by dropout)
+
+        Args:
+            samples (list of ConditioningAttributes): List of ConditioningAttributes samples.
+        Returns:
+            dict[str, WavCondition]: A dictionary mapping an attribute name to wavs.
+        """
+        wavs = defaultdict(list)
+        lengths = defaultdict(list)
+        sample_rates = defaultdict(list)
+        paths = defaultdict(list)
+        seek_times = defaultdict(list)
+        out: tp.Dict[str, WavCondition] = {}
+
+        for sample in samples:
+            for attribute in self.wav_conditions:
+                wav, length, sample_rate, path, seek_time = sample.wav[attribute]
+                assert wav.dim() == 3, f"Got wav with dim={wav.dim()}, but expected 3 [1, C, T]"
+                assert wav.size(0) == 1, f"Got wav [B, C, T] with shape={wav.shape}, but expected B == 1"
+                # mono-channel conditioning
+                wav = wav.mean(1, keepdim=True)  # [1, 1, T]
+                wavs[attribute].append(wav.flatten())  # [T]
+                lengths[attribute].append(length)
+                sample_rates[attribute].extend(sample_rate)
+                paths[attribute].extend(path)
+                seek_times[attribute].extend(seek_time)
+
+        # stack all wavs to a single tensor
+        for attribute in self.wav_conditions:
+            stacked_wav, _ = collate(wavs[attribute], dim=0)
+            out[attribute] = WavCondition(
+                stacked_wav.unsqueeze(1), torch.cat(lengths[attribute]), sample_rates[attribute],
+                paths[attribute], seek_times[attribute])
+
+        return out
+    
+    def _collate_chords(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, ChordCondition]:
+        """Generate a dict where the keys are attributes by which we fetch similar wavs,
+        and the values are Tensors of wavs according to said attributes.
+
+        *Note*: by the time the samples reach this function, each sample should have some waveform
+        inside the "wav" attribute. It should be either:
+        1. A real waveform
+        2. A null waveform due to the sample having no similar waveforms (nullified by the dataset)
+        3. A null waveform due to it being dropped in a dropout module (nullified by dropout)
+
+        Args:
+            samples (list of ConditioningAttributes): List of ConditioningAttributes samples.
+        Returns:
+            dict[str, WavCondition]: A dictionary mapping an attribute name to wavs.
+        """
+        chords = defaultdict(list)
+        lengths = defaultdict(list)
+        bpms = defaultdict(list)
+        paths = defaultdict(list)
+        seek_frames = defaultdict(list)
+        out: tp.Dict[str, ChordCondition] = {}
+
+        for sample in samples: # sample = ConditioningAttributes(text={"genre": "Rock", "description": "A rock song with a guitar solo"}, wav=...)
+            for attribute in self.chord_conditions: # self.chord_conditions = ['chord']
+                chord, length, bpm, path, seek_frame = sample.chord[attribute]
+                assert chord.dim() == 3, f"Got chord with dim={chord.dim()}, but expected 3 [1, C, T]"
+                assert chord.size(0) == 1, f"Got chord [B, C, T] with shape={chord.shape}, but expected B == 1"
+                chords[attribute].append(chord.squeeze(0))  # [1, C, T] -> [N * [C, T]]
+                lengths[attribute].append(length) # [N, 1]
+                bpms[attribute].extend(bpm) # [N]
+                paths[attribute].extend(path) # [N]
+                seek_frames[attribute].extend(seek_frame) # [N]
+
+        # stack all chords to a single tensor
+        for attribute in self.chord_conditions:
+            stacked_chord, _ = collate(chords[attribute], dim=1) # tensor padded here
+            out[attribute] = ChordCondition(
+                stacked_chord, torch.cat(lengths[attribute]), bpms[attribute],
+                paths[attribute], seek_frames[attribute])
+        # print(f"chords shape: {chords[attribute][0].shape}")
+        # print(f"stack chords shape: {stacked_chord.shape}")
+        return out
+
+    def _collate_beats(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, ChordCondition]:
+        """Generate a dict where the keys are attributes by which we fetch similar wavs,
+        and the values are Tensors of wavs according to said attributes.
+
+        Args:
+            samples (list of ConditioningAttributes): List of ConditioningAttributes samples.
+        Returns:
+            dict[str, WavCondition]: A dictionary mapping an attribute name to wavs.
+        """
+        beats = defaultdict(list)
+        lengths = defaultdict(list)
+        bpms = defaultdict(list)
+        paths = defaultdict(list)
+        seek_frames = defaultdict(list)
+        out: tp.Dict[str, ChordCondition] = {}
+
+        for sample in samples: # sample = ConditioningAttributes(text={"genre": "Rock", "description": "A rock song with a guitar solo"}, wav=...)
+            for attribute in self.beat_conditions: # self.chord_conditions = ['chord']
+                beat, length, bpm, path, seek_frame = sample.beat[attribute]
+                assert beat.dim() == 3, f"Got chord with dim={beat.dim()}, but expected 3 [1, C, T]"
+                assert beat.size(0) == 1, f"Got chord [B, C, T] with shape={beat.shape}, but expected B == 1"
+                beats[attribute].append(beat.squeeze(0))  # [1, C, T] -> [N * [C, T]]
+                lengths[attribute].append(length) # [N, 1]
+                bpms[attribute].extend(bpm) # [N]
+                paths[attribute].extend(path) # [N]
+                seek_frames[attribute].extend(seek_frame) # [N]
+
+        # stack all chords to a single tensor
+        for attribute in self.beat_conditions:
+            stacked_beat, _ = collate(beats[attribute], dim=1) # tensor padded here
+            out[attribute] = BeatCondition(
+                stacked_beat, torch.cat(lengths[attribute]), bpms[attribute],
+                paths[attribute], seek_frames[attribute])
+        # print(f"chords shape: {chords[attribute][0].shape}")
+        # print(f"stack chords shape: {stacked_chord.shape}")
+        return out
+
+    def _collate_joint_embeds(self, samples: tp.List[ConditioningAttributes]) -> tp.Dict[str, JointEmbedCondition]:
+        """Generate a dict where the keys are attributes by which we compute joint embeddings,
+        and the values are Tensors of pre-computed embeddings and the corresponding text attributes.
+
+        Args:
+            samples (list[ConditioningAttributes]): List of ConditioningAttributes samples.
+        Returns:
+            A dictionary mapping an attribute name to joint embeddings.
+        """
+        texts = defaultdict(list)
+        wavs = defaultdict(list)
+        lengths = defaultdict(list)
+        sample_rates = defaultdict(list)
+        paths = defaultdict(list)
+        seek_times = defaultdict(list)
+        channels: int = 0
+
+        out = {}
+        for sample in samples:
+            for attribute in self.joint_embed_conditions:
+                wav, text, length, sample_rate, path, seek_time = sample.joint_embed[attribute]
+                assert wav.dim() == 3
+                if channels == 0:
+                    channels = wav.size(1)
+                else:
+                    assert channels == wav.size(1), "not all audio has same number of channels in batch"
+                assert wav.size(0) == 1, "Expecting single-wav batch in the collate method"
+                wav = einops.rearrange(wav, "b c t -> (b c t)")  # [1, C, T] => [C * T]
+                wavs[attribute].append(wav)
+                texts[attribute].extend(text)
+                lengths[attribute].append(length)
+                sample_rates[attribute].extend(sample_rate)
+                paths[attribute].extend(path)
+                seek_times[attribute].extend(seek_time)
+
+        for attribute in self.joint_embed_conditions:
+            stacked_texts = texts[attribute]
+            stacked_paths = paths[attribute]
+            stacked_seek_times = seek_times[attribute]
+            stacked_wavs = pad_sequence(wavs[attribute]).to(self.device)
+            stacked_wavs = einops.rearrange(stacked_wavs, "(c t) b -> b c t", c=channels)
+            stacked_sample_rates = sample_rates[attribute]
+            stacked_lengths = torch.cat(lengths[attribute]).to(self.device)
+            assert stacked_lengths.size(0) == stacked_wavs.size(0)
+            assert len(stacked_sample_rates) == stacked_wavs.size(0)
+            assert len(stacked_texts) == stacked_wavs.size(0)
+            out[attribute] = JointEmbedCondition(
+                text=stacked_texts, wav=stacked_wavs,
+                length=stacked_lengths, sample_rate=stacked_sample_rates,
+                path=stacked_paths, seek_time=stacked_seek_times)
+
+        return out
+
+
+class ConditionFuser(StreamingModule):
+    """Condition fuser handles the logic to combine the different conditions
+    to the actual model input.
+
+    Args:
+        fuse2cond (tp.Dict[str, str]): A dictionary that says how to fuse
+            each condition. For example:
+            {
+                "prepend": ["description"],
+                "sum": ["genre", "bpm"],
+                "cross": ["description"],
+            }
+        cross_attention_pos_emb (bool, optional): Use positional embeddings in cross attention.
+        cross_attention_pos_emb_scale (int): Scale for positional embeddings in cross attention if used.
+    """
+    FUSING_METHODS = ["sum", "prepend", "cross", "input_interpolate", "concat"]
+
+    def __init__(self, fuse2cond: tp.Dict[str, tp.List[str]], cross_attention_pos_emb: bool = False,
+                 cross_attention_pos_emb_scale: float = 1.0, in_attn: bool = False):
+        super().__init__()
+        assert all(
+            [k in self.FUSING_METHODS for k in fuse2cond.keys()]
+        ), f"Got invalid fuse method, allowed methods: {self.FUSING_METHODS}"
+        self.cross_attention_pos_emb = cross_attention_pos_emb
+        self.cross_attention_pos_emb_scale = cross_attention_pos_emb_scale
+        self.fuse2cond: tp.Dict[str, tp.List[str]] = fuse2cond
+        self.cond2fuse: tp.Dict[str, str] = {}
+        self.in_attn = in_attn
+
+        for fuse_method, conditions in fuse2cond.items():
+            for condition in conditions:
+                if not condition in self.cond2fuse.keys():
+                    self.cond2fuse[condition] = [fuse_method]
+                else:
+                    self.cond2fuse[condition].append(fuse_method)
+
+
+    def forward(
+        self,
+        input: torch.Tensor,
+        conditions: tp.Dict[str, ConditionType]
+    ) -> tp.Tuple[torch.Tensor, tp.Optional[torch.Tensor]]:
+        """Fuse the conditions to the provided model input.
+
+        Args:
+            input (torch.Tensor): Transformer input.
+            conditions (dict[str, ConditionType]): Dict of conditions.
+        Returns:
+            tuple[torch.Tensor, torch.Tensor]: The first tensor is the transformer input
+                after the conditions have been fused. The second output tensor is the tensor
+                used for cross-attention or None if no cross attention inputs exist.
+        """
+
+        B, T, _ = input.shape # [B, T, C]
+        if self.in_attn:
+            in_attn_cond = torch.zeros_like(input)
+        else:
+            in_attn_cond = None
+
+        if 'offsets' in self._streaming_state:
+            first_step = False
+            offsets = self._streaming_state['offsets']
+        else:
+            first_step = True
+            offsets = torch.zeros(B, dtype=torch.long, device=input.device)
+
+        assert set(conditions.keys()).issubset(set(self.cond2fuse.keys())), \
+            f"given conditions contain unknown attributes for fuser, " \
+            f"expected {self.cond2fuse.keys()}, got {conditions.keys()}"
+        cross_attention_output = None
+
+        for cond_type, (cond, cond_mask) in conditions.items():
+            fuse_methods = self.cond2fuse[cond_type]
+            for op in fuse_methods:
+                if op == 'sum':
+                    cond_sum = cond[:, offsets[0]:offsets[0]+T]
+                    if cond_sum.shape[1] != 0:
+                        if cond_sum.shape[1] < T:
+                            cond_sum = F.pad(cond_sum, (0, 0, 0, T-cond_sum.shape[1]), "constant", 0) # pad last special token dim
+                        input[:, -cond_sum.shape[1]:, :] = input[:, -cond_sum.shape[1]:, :] + cond_sum
+                        if self.in_attn:
+                            in_attn_cond += cond_sum
+                
+                elif op == 'input_interpolate':
+                    cond = einops.rearrange(cond, "b t d -> b d t")
+                    cond = F.interpolate(cond, size=input.shape[1])
+                    input += einops.rearrange(cond, "b d t -> b t d")
+                
+                elif op == 'prepend':
+                    if cond_type == 'chord':
+                        cond_prepend = torch.zeros(cond.shape[0], 235, cond.shape[2], device=cond.device) # original musicgen melody has 235 length chroma
+                        if cond.shape[1] == 1500: # if condition not dropout
+                            for i in range(235):
+                                cond_prepend[:, i, :] = cond[:, round(i * (1500/235)), :] # n_frame of chord = 30*50 into 235 time steps
+                    else:
+                        cond_prepend = cond
+                    
+                    if first_step:
+                        input = torch.cat([cond_prepend, input], dim=1)
+                    
+                elif op == 'cross':
+                    if cross_attention_output is not None:
+                        cross_attention_output = torch.cat([cross_attention_output, cond], dim=1)
+                    else:
+                        cross_attention_output = cond
+                else:
+                    raise ValueError(f"unknown op ({op})")
+
+
+        if self.cross_attention_pos_emb and cross_attention_output is not None:
+            positions = torch.arange(
+                cross_attention_output.shape[1],
+                device=cross_attention_output.device
+            ).view(1, -1, 1)
+            pos_emb = create_sin_embedding(positions, cross_attention_output.shape[-1])
+            cross_attention_output = cross_attention_output + self.cross_attention_pos_emb_scale * pos_emb
+
+        if self._is_streaming:
+            self._streaming_state['offsets'] = offsets + T
+        
+        return input, in_attn_cond, cross_attention_output
\ No newline at end of file
diff --git a/audiocraft/audiocraft/modules/conv.py b/audiocraft/audiocraft/modules/conv.py
new file mode 100644
index 0000000000000000000000000000000000000000..d115cbf8729b642ed78608bd00a4d0fd5afae6fd
--- /dev/null
+++ b/audiocraft/audiocraft/modules/conv.py
@@ -0,0 +1,243 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+import typing as tp
+import warnings
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.nn.utils import spectral_norm, weight_norm
+
+
+CONV_NORMALIZATIONS = frozenset(['none', 'weight_norm', 'spectral_norm',
+                                 'time_group_norm'])
+
+
+def apply_parametrization_norm(module: nn.Module, norm: str = 'none'):
+    assert norm in CONV_NORMALIZATIONS
+    if norm == 'weight_norm':
+        return weight_norm(module)
+    elif norm == 'spectral_norm':
+        return spectral_norm(module)
+    else:
+        # We already check was in CONV_NORMALIZATION, so any other choice
+        # doesn't need reparametrization.
+        return module
+
+
+def get_norm_module(module: nn.Module, causal: bool = False, norm: str = 'none', **norm_kwargs):
+    """Return the proper normalization module. If causal is True, this will ensure the returned
+    module is causal, or return an error if the normalization doesn't support causal evaluation.
+    """
+    assert norm in CONV_NORMALIZATIONS
+    if norm == 'time_group_norm':
+        if causal:
+            raise ValueError("GroupNorm doesn't support causal evaluation.")
+        assert isinstance(module, nn.modules.conv._ConvNd)
+        return nn.GroupNorm(1, module.out_channels, **norm_kwargs)
+    else:
+        return nn.Identity()
+
+
+def get_extra_padding_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int,
+                                 padding_total: int = 0) -> int:
+    """See `pad_for_conv1d`."""
+    length = x.shape[-1]
+    n_frames = (length - kernel_size + padding_total) / stride + 1
+    ideal_length = (math.ceil(n_frames) - 1) * stride + (kernel_size - padding_total)
+    return ideal_length - length
+
+
+def pad_for_conv1d(x: torch.Tensor, kernel_size: int, stride: int, padding_total: int = 0):
+    """Pad for a convolution to make sure that the last window is full.
+    Extra padding is added at the end. This is required to ensure that we can rebuild
+    an output of the same length, as otherwise, even with padding, some time steps
+    might get removed.
+    For instance, with total padding = 4, kernel size = 4, stride = 2:
+        0 0 1 2 3 4 5 0 0   # (0s are padding)
+        1   2   3           # (output frames of a convolution, last 0 is never used)
+        0 0 1 2 3 4 5 0     # (output of tr. conv., but pos. 5 is going to get removed as padding)
+            1 2 3 4         # once you removed padding, we are missing one time step !
+    """
+    extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
+    return F.pad(x, (0, extra_padding))
+
+
+def pad1d(x: torch.Tensor, paddings: tp.Tuple[int, int], mode: str = 'constant', value: float = 0.):
+    """Tiny wrapper around F.pad, just to allow for reflect padding on small input.
+    If this is the case, we insert extra 0 padding to the right before the reflection happen.
+    """
+    length = x.shape[-1]
+    padding_left, padding_right = paddings
+    assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
+    if mode == 'reflect':
+        max_pad = max(padding_left, padding_right)
+        extra_pad = 0
+        if length <= max_pad:
+            extra_pad = max_pad - length + 1
+            x = F.pad(x, (0, extra_pad))
+        padded = F.pad(x, paddings, mode, value)
+        end = padded.shape[-1] - extra_pad
+        return padded[..., :end]
+    else:
+        return F.pad(x, paddings, mode, value)
+
+
+def unpad1d(x: torch.Tensor, paddings: tp.Tuple[int, int]):
+    """Remove padding from x, handling properly zero padding. Only for 1d!"""
+    padding_left, padding_right = paddings
+    assert padding_left >= 0 and padding_right >= 0, (padding_left, padding_right)
+    assert (padding_left + padding_right) <= x.shape[-1]
+    end = x.shape[-1] - padding_right
+    return x[..., padding_left: end]
+
+
+class NormConv1d(nn.Module):
+    """Wrapper around Conv1d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, causal: bool = False, norm: str = 'none',
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.conv = apply_parametrization_norm(nn.Conv1d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.conv, causal, norm, **norm_kwargs)
+        self.norm_type = norm
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.norm(x)
+        return x
+
+
+class NormConv2d(nn.Module):
+    """Wrapper around Conv2d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.conv = apply_parametrization_norm(nn.Conv2d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.conv, causal=False, norm=norm, **norm_kwargs)
+        self.norm_type = norm
+
+    def forward(self, x):
+        x = self.conv(x)
+        x = self.norm(x)
+        return x
+
+
+class NormConvTranspose1d(nn.Module):
+    """Wrapper around ConvTranspose1d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, causal: bool = False, norm: str = 'none',
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.convtr = apply_parametrization_norm(nn.ConvTranspose1d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.convtr, causal, norm, **norm_kwargs)
+        self.norm_type = norm
+
+    def forward(self, x):
+        x = self.convtr(x)
+        x = self.norm(x)
+        return x
+
+
+class NormConvTranspose2d(nn.Module):
+    """Wrapper around ConvTranspose2d and normalization applied to this conv
+    to provide a uniform interface across normalization approaches.
+    """
+    def __init__(self, *args, norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {}, **kwargs):
+        super().__init__()
+        self.convtr = apply_parametrization_norm(nn.ConvTranspose2d(*args, **kwargs), norm)
+        self.norm = get_norm_module(self.convtr, causal=False, norm=norm, **norm_kwargs)
+
+    def forward(self, x):
+        x = self.convtr(x)
+        x = self.norm(x)
+        return x
+
+
+class StreamableConv1d(nn.Module):
+    """Conv1d with some builtin handling of asymmetric or causal padding
+    and normalization.
+    """
+    def __init__(self, in_channels: int, out_channels: int,
+                 kernel_size: int, stride: int = 1, dilation: int = 1,
+                 groups: int = 1, bias: bool = True, causal: bool = False,
+                 norm: str = 'none', norm_kwargs: tp.Dict[str, tp.Any] = {},
+                 pad_mode: str = 'reflect'):
+        super().__init__()
+        # warn user on unusual setup between dilation and stride
+        if stride > 1 and dilation > 1:
+            warnings.warn("StreamableConv1d has been initialized with stride > 1 and dilation > 1"
+                          f" (kernel_size={kernel_size} stride={stride}, dilation={dilation}).")
+        self.conv = NormConv1d(in_channels, out_channels, kernel_size, stride,
+                               dilation=dilation, groups=groups, bias=bias, causal=causal,
+                               norm=norm, norm_kwargs=norm_kwargs)
+        self.causal = causal
+        self.pad_mode = pad_mode
+
+    def forward(self, x):
+        B, C, T = x.shape
+        kernel_size = self.conv.conv.kernel_size[0]
+        stride = self.conv.conv.stride[0]
+        dilation = self.conv.conv.dilation[0]
+        kernel_size = (kernel_size - 1) * dilation + 1  # effective kernel size with dilations
+        padding_total = kernel_size - stride
+        extra_padding = get_extra_padding_for_conv1d(x, kernel_size, stride, padding_total)
+        if self.causal:
+            # Left padding for causal
+            x = pad1d(x, (padding_total, extra_padding), mode=self.pad_mode)
+        else:
+            # Asymmetric padding required for odd strides
+            padding_right = padding_total // 2
+            padding_left = padding_total - padding_right
+            x = pad1d(x, (padding_left, padding_right + extra_padding), mode=self.pad_mode)
+        return self.conv(x)
+
+
+class StreamableConvTranspose1d(nn.Module):
+    """ConvTranspose1d with some builtin handling of asymmetric or causal padding
+    and normalization.
+    """
+    def __init__(self, in_channels: int, out_channels: int,
+                 kernel_size: int, stride: int = 1, causal: bool = False,
+                 norm: str = 'none', trim_right_ratio: float = 1.,
+                 norm_kwargs: tp.Dict[str, tp.Any] = {}):
+        super().__init__()
+        self.convtr = NormConvTranspose1d(in_channels, out_channels, kernel_size, stride,
+                                          causal=causal, norm=norm, norm_kwargs=norm_kwargs)
+        self.causal = causal
+        self.trim_right_ratio = trim_right_ratio
+        assert self.causal or self.trim_right_ratio == 1., \
+            "`trim_right_ratio` != 1.0 only makes sense for causal convolutions"
+        assert self.trim_right_ratio >= 0. and self.trim_right_ratio <= 1.
+
+    def forward(self, x):
+        kernel_size = self.convtr.convtr.kernel_size[0]
+        stride = self.convtr.convtr.stride[0]
+        padding_total = kernel_size - stride
+
+        y = self.convtr(x)
+
+        # We will only trim fixed padding. Extra padding from `pad_for_conv1d` would be
+        # removed at the very end, when keeping only the right length for the output,
+        # as removing it here would require also passing the length at the matching layer
+        # in the encoder.
+        if self.causal:
+            # Trim the padding on the right according to the specified ratio
+            # if trim_right_ratio = 1.0, trim everything from right
+            padding_right = math.ceil(padding_total * self.trim_right_ratio)
+            padding_left = padding_total - padding_right
+            y = unpad1d(y, (padding_left, padding_right))
+        else:
+            # Asymmetric padding required for odd strides
+            padding_right = padding_total // 2
+            padding_left = padding_total - padding_right
+            y = unpad1d(y, (padding_left, padding_right))
+        return y
diff --git a/audiocraft/audiocraft/modules/diffusion_schedule.py b/audiocraft/audiocraft/modules/diffusion_schedule.py
new file mode 100644
index 0000000000000000000000000000000000000000..74ca6e3f2e7c4ff904d96dade315b0b46856778d
--- /dev/null
+++ b/audiocraft/audiocraft/modules/diffusion_schedule.py
@@ -0,0 +1,272 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Functions for Noise Schedule, defines diffusion process, reverse process and data processor.
+"""
+
+from collections import namedtuple
+import random
+import typing as tp
+import julius
+import torch
+
+TrainingItem = namedtuple("TrainingItem", "noisy noise step")
+
+
+def betas_from_alpha_bar(alpha_bar):
+    alphas = torch.cat([torch.Tensor([alpha_bar[0]]), alpha_bar[1:]/alpha_bar[:-1]])
+    return 1 - alphas
+
+
+class SampleProcessor(torch.nn.Module):
+    def project_sample(self, x: torch.Tensor):
+        """Project the original sample to the 'space' where the diffusion will happen."""
+        return x
+
+    def return_sample(self, z: torch.Tensor):
+        """Project back from diffusion space to the actual sample space."""
+        return z
+
+
+class MultiBandProcessor(SampleProcessor):
+    """
+    MultiBand sample processor. The input audio is splitted across
+    frequency bands evenly distributed in mel-scale.
+
+    Each band will be rescaled to match the power distribution
+    of Gaussian noise in that band, using online metrics
+    computed on the first few samples.
+
+    Args:
+        n_bands (int): Number of mel-bands to split the signal over.
+        sample_rate (int): Sample rate of the audio.
+        num_samples (int): Number of samples to use to fit the rescaling
+            for each band. The processor won't be stable
+            until it has seen that many samples.
+        power_std (float or list/tensor): The rescaling factor computed to match the
+            power of Gaussian noise in each band is taken to
+            that power, i.e. `1.` means full correction of the energy
+            in each band, and values less than `1` means only partial
+            correction. Can be used to balance the relative importance
+            of low vs. high freq in typical audio signals.
+    """
+    def __init__(self, n_bands: int = 8, sample_rate: float = 24_000,
+                 num_samples: int = 10_000, power_std: tp.Union[float, tp.List[float], torch.Tensor] = 1.):
+        super().__init__()
+        self.n_bands = n_bands
+        self.split_bands = julius.SplitBands(sample_rate, n_bands=n_bands)
+        self.num_samples = num_samples
+        self.power_std = power_std
+        if isinstance(power_std, list):
+            assert len(power_std) == n_bands
+            power_std = torch.tensor(power_std)
+        self.register_buffer('counts', torch.zeros(1))
+        self.register_buffer('sum_x', torch.zeros(n_bands))
+        self.register_buffer('sum_x2', torch.zeros(n_bands))
+        self.register_buffer('sum_target_x2', torch.zeros(n_bands))
+        self.counts: torch.Tensor
+        self.sum_x: torch.Tensor
+        self.sum_x2: torch.Tensor
+        self.sum_target_x2: torch.Tensor
+
+    @property
+    def mean(self):
+        mean = self.sum_x / self.counts
+        return mean
+
+    @property
+    def std(self):
+        std = (self.sum_x2 / self.counts - self.mean**2).clamp(min=0).sqrt()
+        return std
+
+    @property
+    def target_std(self):
+        target_std = self.sum_target_x2 / self.counts
+        return target_std
+
+    def project_sample(self, x: torch.Tensor):
+        assert x.dim() == 3
+        bands = self.split_bands(x)
+        if self.counts.item() < self.num_samples:
+            ref_bands = self.split_bands(torch.randn_like(x))
+            self.counts += len(x)
+            self.sum_x += bands.mean(dim=(2, 3)).sum(dim=1)
+            self.sum_x2 += bands.pow(2).mean(dim=(2, 3)).sum(dim=1)
+            self.sum_target_x2 += ref_bands.pow(2).mean(dim=(2, 3)).sum(dim=1)
+        rescale = (self.target_std / self.std.clamp(min=1e-12)) ** self.power_std  # same output size
+        bands = (bands - self.mean.view(-1, 1, 1, 1)) * rescale.view(-1, 1, 1, 1)
+        return bands.sum(dim=0)
+
+    def return_sample(self, x: torch.Tensor):
+        assert x.dim() == 3
+        bands = self.split_bands(x)
+        rescale = (self.std / self.target_std) ** self.power_std
+        bands = bands * rescale.view(-1, 1, 1, 1) + self.mean.view(-1, 1, 1, 1)
+        return bands.sum(dim=0)
+
+
+class NoiseSchedule:
+    """Noise schedule for diffusion.
+
+    Args:
+        beta_t0 (float): Variance of the first diffusion step.
+        beta_t1 (float): Variance of the last diffusion step.
+        beta_exp (float): Power schedule exponent
+        num_steps (int): Number of diffusion step.
+        variance (str): choice of the sigma value for the denoising eq. Choices: "beta" or "beta_tilde"
+        clip (float): clipping value for the denoising steps
+        rescale (float): rescaling value to avoid vanishing signals unused by default (i.e 1)
+        repartition (str): shape of the schedule only power schedule is supported
+        sample_processor (SampleProcessor): Module that normalize data to match better the gaussian distribution
+        noise_scale (float): Scaling factor for the noise
+    """
+    def __init__(self, beta_t0: float = 1e-4, beta_t1: float = 0.02, num_steps: int = 1000, variance: str = 'beta',
+                 clip: float = 5., rescale: float = 1., device='cuda', beta_exp: float = 1,
+                 repartition: str = "power", alpha_sigmoid: dict = {}, n_bands: tp.Optional[int] = None,
+                 sample_processor: SampleProcessor = SampleProcessor(), noise_scale: float = 1.0, **kwargs):
+
+        self.beta_t0 = beta_t0
+        self.beta_t1 = beta_t1
+        self.variance = variance
+        self.num_steps = num_steps
+        self.clip = clip
+        self.sample_processor = sample_processor
+        self.rescale = rescale
+        self.n_bands = n_bands
+        self.noise_scale = noise_scale
+        assert n_bands is None
+        if repartition == "power":
+            self.betas = torch.linspace(beta_t0 ** (1 / beta_exp), beta_t1 ** (1 / beta_exp), num_steps,
+                                        device=device, dtype=torch.float) ** beta_exp
+        else:
+            raise RuntimeError('Not implemented')
+        self.rng = random.Random(1234)
+
+    def get_beta(self, step: tp.Union[int, torch.Tensor]):
+        if self.n_bands is None:
+            return self.betas[step]
+        else:
+            return self.betas[:, step]  # [n_bands, len(step)]
+
+    def get_initial_noise(self, x: torch.Tensor):
+        if self.n_bands is None:
+            return torch.randn_like(x)
+        return torch.randn((x.size(0), self.n_bands, x.size(2)))
+
+    def get_alpha_bar(self, step: tp.Optional[tp.Union[int, torch.Tensor]] = None) -> torch.Tensor:
+        """Return 'alpha_bar', either for a given step, or as a tensor with its value for each step."""
+        if step is None:
+            return (1 - self.betas).cumprod(dim=-1)  # works for simgle and multi bands
+        if type(step) is int:
+            return (1 - self.betas[:step + 1]).prod()
+        else:
+            return (1 - self.betas).cumprod(dim=0)[step].view(-1, 1, 1)
+
+    def get_training_item(self, x: torch.Tensor, tensor_step: bool = False) -> TrainingItem:
+        """Create a noisy data item for diffusion model training:
+
+        Args:
+            x (torch.Tensor): clean audio data torch.tensor(bs, 1, T)
+            tensor_step (bool): If tensor_step = false, only one step t is sample,
+                the whole batch is diffused to the same step and t is int.
+                If tensor_step = true, t is a tensor of size (x.size(0),)
+                every element of the batch is diffused to a independently sampled.
+        """
+        step: tp.Union[int, torch.Tensor]
+        if tensor_step:
+            bs = x.size(0)
+            step = torch.randint(0, self.num_steps, size=(bs,), device=x.device)
+        else:
+            step = self.rng.randrange(self.num_steps)
+        alpha_bar = self.get_alpha_bar(step)  # [batch_size, n_bands, 1]
+
+        x = self.sample_processor.project_sample(x)
+        noise = torch.randn_like(x)
+        noisy = (alpha_bar.sqrt() / self.rescale) * x + (1 - alpha_bar).sqrt() * noise * self.noise_scale
+        return TrainingItem(noisy, noise, step)
+
+    def generate(self, model: torch.nn.Module, initial: tp.Optional[torch.Tensor] = None,
+                 condition: tp.Optional[torch.Tensor] = None, return_list: bool = False):
+        """Full ddpm reverse process.
+
+        Args:
+            model (nn.Module): Diffusion model.
+            initial (tensor): Initial Noise.
+            condition (tensor): Input conditionning Tensor (e.g. encodec compressed representation).
+            return_list (bool): Whether to return the whole process or only the sampled point.
+        """
+        alpha_bar = self.get_alpha_bar(step=self.num_steps - 1)
+        current = initial
+        iterates = [initial]
+        for step in range(self.num_steps)[::-1]:
+            with torch.no_grad():
+                estimate = model(current, step, condition=condition).sample
+            alpha = 1 - self.betas[step]
+            previous = (current - (1 - alpha) / (1 - alpha_bar).sqrt() * estimate) / alpha.sqrt()
+            previous_alpha_bar = self.get_alpha_bar(step=step - 1)
+            if step == 0:
+                sigma2 = 0
+            elif self.variance == 'beta':
+                sigma2 = 1 - alpha
+            elif self.variance == 'beta_tilde':
+                sigma2 = (1 - previous_alpha_bar) / (1 - alpha_bar) * (1 - alpha)
+            elif self.variance == 'none':
+                sigma2 = 0
+            else:
+                raise ValueError(f'Invalid variance type {self.variance}')
+
+            if sigma2 > 0:
+                previous += sigma2**0.5 * torch.randn_like(previous) * self.noise_scale
+            if self.clip:
+                previous = previous.clamp(-self.clip, self.clip)
+            current = previous
+            alpha_bar = previous_alpha_bar
+            if step == 0:
+                previous *= self.rescale
+            if return_list:
+                iterates.append(previous.cpu())
+
+        if return_list:
+            return iterates
+        else:
+            return self.sample_processor.return_sample(previous)
+
+    def generate_subsampled(self, model: torch.nn.Module, initial: torch.Tensor, step_list: tp.Optional[list] = None,
+                            condition: tp.Optional[torch.Tensor] = None, return_list: bool = False):
+        """Reverse process that only goes through Markov chain states in step_list."""
+        if step_list is None:
+            step_list = list(range(1000))[::-50] + [0]
+        alpha_bar = self.get_alpha_bar(step=self.num_steps - 1)
+        alpha_bars_subsampled = (1 - self.betas).cumprod(dim=0)[list(reversed(step_list))].cpu()
+        betas_subsampled = betas_from_alpha_bar(alpha_bars_subsampled)
+        current = initial * self.noise_scale
+        iterates = [current]
+        for idx, step in enumerate(step_list[:-1]):
+            with torch.no_grad():
+                estimate = model(current, step, condition=condition).sample * self.noise_scale
+            alpha = 1 - betas_subsampled[-1 - idx]
+            previous = (current - (1 - alpha) / (1 - alpha_bar).sqrt() * estimate) / alpha.sqrt()
+            previous_alpha_bar = self.get_alpha_bar(step_list[idx + 1])
+            if step == step_list[-2]:
+                sigma2 = 0
+                previous_alpha_bar = torch.tensor(1.0)
+            else:
+                sigma2 = (1 - previous_alpha_bar) / (1 - alpha_bar) * (1 - alpha)
+            if sigma2 > 0:
+                previous += sigma2**0.5 * torch.randn_like(previous) * self.noise_scale
+            if self.clip:
+                previous = previous.clamp(-self.clip, self.clip)
+            current = previous
+            alpha_bar = previous_alpha_bar
+            if step == 0:
+                previous *= self.rescale
+            if return_list:
+                iterates.append(previous.cpu())
+        if return_list:
+            return iterates
+        else:
+            return self.sample_processor.return_sample(previous)
diff --git a/audiocraft/audiocraft/modules/lstm.py b/audiocraft/audiocraft/modules/lstm.py
new file mode 100644
index 0000000000000000000000000000000000000000..c0866175950c1ca4f6cca98649525e6481853bba
--- /dev/null
+++ b/audiocraft/audiocraft/modules/lstm.py
@@ -0,0 +1,25 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from torch import nn
+
+
+class StreamableLSTM(nn.Module):
+    """LSTM without worrying about the hidden state, nor the layout of the data.
+    Expects input as convolutional layout.
+    """
+    def __init__(self, dimension: int, num_layers: int = 2, skip: bool = True):
+        super().__init__()
+        self.skip = skip
+        self.lstm = nn.LSTM(dimension, dimension, num_layers)
+
+    def forward(self, x):
+        x = x.permute(2, 0, 1)
+        y, _ = self.lstm(x)
+        if self.skip:
+            y = y + x
+        y = y.permute(1, 2, 0)
+        return y
diff --git a/audiocraft/audiocraft/modules/rope.py b/audiocraft/audiocraft/modules/rope.py
new file mode 100644
index 0000000000000000000000000000000000000000..503e6748df2bb72b3c864c20b37cba5498ffdd21
--- /dev/null
+++ b/audiocraft/audiocraft/modules/rope.py
@@ -0,0 +1,121 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+from torch import nn
+import torch
+
+
+class XPos(nn.Module):
+    """Length-extrapolatable positional embedding (xPos) from [Sun et al 2022](https://arxiv.org/abs/2212.10554v1).
+    This applies an exponential decay to the RoPE rotation matrix.
+
+    Args:
+        dim (int): Embedding dimension.
+        smoothing (float): Smoothing factor applied to the decay rates.
+        base_scale (int): Base decay rate, given in terms of scaling time.
+        device (torch.device, optional): Device on which to initialize the module.
+        dtype (torch.dtype): dtype to use to generate the embedding.
+    """
+    def __init__(self, dim: int, smoothing: float = 0.4, base_scale: int = 512,
+                 device=None, dtype: torch.dtype = torch.float32):
+        super().__init__()
+        assert dim % 2 == 0
+        assert dtype in [torch.float64, torch.float32]
+        self.dtype = dtype
+        self.base_scale = base_scale
+
+        half_dim = dim // 2
+        adim = torch.arange(half_dim, device=device, dtype=dtype)
+        decay_rates = (adim / half_dim + smoothing) / (1.0 + smoothing)
+        self.register_buffer("decay_rates", decay_rates)
+        self.decay: tp.Optional[torch.Tensor] = None
+
+    def get_decay(self, start: int, end: int):
+        """Create complex decay tensor, cache values for fast computation."""
+        if self.decay is None or end > self.decay.shape[0]:
+            assert isinstance(self.decay_rates, torch.Tensor)  # Satisfy type checker.
+            idx = torch.arange(end, device=self.decay_rates.device, dtype=self.dtype)
+            power = idx / self.base_scale
+            scale = self.decay_rates ** power.unsqueeze(-1)
+            self.decay = torch.polar(scale, torch.zeros_like(scale))
+        return self.decay[start:end]  # [T, C/2]
+
+
+class RotaryEmbedding(nn.Module):
+    """Rotary positional embedding (RoPE) from [Su et al 2022](https://arxiv.org/abs/2104.09864).
+
+    Args:
+        dim (int): Embedding dimension (twice the number of frequencies).
+        max_period (float): Maximum period of the rotation frequencies.
+        xpos (bool): Use xPos, applies an exponential decay to rotation matrix.
+        scale (float): Scale of positional embedding, set to 0 to deactivate.
+        device (torch.device, optional): Device on which to initialize the module.
+        dtype (torch.dtype): dtype to use to generate the embedding.
+    """
+    def __init__(self, dim: int, max_period: float = 10000.0, xpos: bool = False,
+                 scale: float = 1.0, device=None, dtype: torch.dtype = torch.float32):
+        super().__init__()
+        assert dim % 2 == 0
+        self.scale = scale
+        assert dtype in [torch.float64, torch.float32]
+        self.dtype = dtype
+
+        adim = torch.arange(0, dim, 2, device=device, dtype=dtype)[: (dim // 2)]
+        frequencies = 1.0 / (max_period ** (adim / dim))
+        self.register_buffer("frequencies", frequencies)
+        self.rotation: tp.Optional[torch.Tensor] = None
+
+        self.xpos = XPos(dim, device=device, dtype=dtype) if xpos else None
+
+    def get_rotation(self, start: int, end: int):
+        """Create complex rotation tensor, cache values for fast computation."""
+        if self.rotation is None or end > self.rotation.shape[0]:
+            assert isinstance(self.frequencies, torch.Tensor)  # Satisfy type checker.
+            idx = torch.arange(end, device=self.frequencies.device, dtype=self.dtype)
+            angles = torch.outer(idx, self.frequencies)
+            self.rotation = torch.polar(torch.ones_like(angles), angles)
+        return self.rotation[start:end]
+
+    def rotate(self, x: torch.Tensor, start: int = 0, invert_decay: bool = False):
+        """Apply rope rotation to query or key tensor."""
+        T = x.shape[1]
+        rotation = self.get_rotation(start, start + T).unsqueeze(0).unsqueeze(2)
+
+        if self.xpos:
+            decay = self.xpos.get_decay(start, start + T).unsqueeze(0).unsqueeze(2)
+        else:
+            decay = 1.0
+
+        if invert_decay:
+            decay = decay ** -1
+
+        x_complex = torch.view_as_complex(x.to(self.dtype).reshape(*x.shape[:-1], -1, 2))
+        scaled_rotation = (rotation * decay) * self.scale + (1.0 - self.scale)
+        x_out = torch.view_as_real(x_complex * scaled_rotation).flatten(-2)
+
+        return x_out.type_as(x)
+
+    def rotate_qk(self, query: torch.Tensor, key: torch.Tensor, start: int = 0):
+        """ Apply rope rotation to both query and key tensors.
+        Supports streaming mode, in which query and key are not expected to have the same shape.
+        In streaming mode, key will be of length [P + C] with P the cached past timesteps, but
+        query will be [C] (typically C == 1).
+
+        Args:
+            query (torch.Tensor): Query to rotate.
+            key (torch.Tensor): Key to rotate.
+            start (int): Start index of the sequence for time offset.
+        """
+        query_timesteps = query.shape[1]
+        key_timesteps = key.shape[1]
+        streaming_offset = key_timesteps - query_timesteps
+
+        query_out = self.rotate(query, start + streaming_offset)
+        key_out = self.rotate(key, start, invert_decay=True)
+
+        return query_out, key_out
diff --git a/audiocraft/audiocraft/modules/seanet.py b/audiocraft/audiocraft/modules/seanet.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e5998e9153afb6e68ea410d565e00ea835db248
--- /dev/null
+++ b/audiocraft/audiocraft/modules/seanet.py
@@ -0,0 +1,258 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import numpy as np
+import torch.nn as nn
+
+from .conv import StreamableConv1d, StreamableConvTranspose1d
+from .lstm import StreamableLSTM
+
+
+class SEANetResnetBlock(nn.Module):
+    """Residual block from SEANet model.
+
+    Args:
+        dim (int): Dimension of the input/output.
+        kernel_sizes (list): List of kernel sizes for the convolutions.
+        dilations (list): List of dilations for the convolutions.
+        activation (str): Activation function.
+        activation_params (dict): Parameters to provide to the activation function.
+        norm (str): Normalization method.
+        norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
+        causal (bool): Whether to use fully causal convolution.
+        pad_mode (str): Padding mode for the convolutions.
+        compress (int): Reduced dimensionality in residual branches (from Demucs v3).
+        true_skip (bool): Whether to use true skip connection or a simple
+            (streamable) convolution as the skip connection.
+    """
+    def __init__(self, dim: int, kernel_sizes: tp.List[int] = [3, 1], dilations: tp.List[int] = [1, 1],
+                 activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
+                 norm: str = 'none', norm_params: tp.Dict[str, tp.Any] = {}, causal: bool = False,
+                 pad_mode: str = 'reflect', compress: int = 2, true_skip: bool = True):
+        super().__init__()
+        assert len(kernel_sizes) == len(dilations), 'Number of kernel sizes should match number of dilations'
+        act = getattr(nn, activation)
+        hidden = dim // compress
+        block = []
+        for i, (kernel_size, dilation) in enumerate(zip(kernel_sizes, dilations)):
+            in_chs = dim if i == 0 else hidden
+            out_chs = dim if i == len(kernel_sizes) - 1 else hidden
+            block += [
+                act(**activation_params),
+                StreamableConv1d(in_chs, out_chs, kernel_size=kernel_size, dilation=dilation,
+                                 norm=norm, norm_kwargs=norm_params,
+                                 causal=causal, pad_mode=pad_mode),
+            ]
+        self.block = nn.Sequential(*block)
+        self.shortcut: nn.Module
+        if true_skip:
+            self.shortcut = nn.Identity()
+        else:
+            self.shortcut = StreamableConv1d(dim, dim, kernel_size=1, norm=norm, norm_kwargs=norm_params,
+                                             causal=causal, pad_mode=pad_mode)
+
+    def forward(self, x):
+        return self.shortcut(x) + self.block(x)
+
+
+class SEANetEncoder(nn.Module):
+    """SEANet encoder.
+
+    Args:
+        channels (int): Audio channels.
+        dimension (int): Intermediate representation dimension.
+        n_filters (int): Base width for the model.
+        n_residual_layers (int): nb of residual layers.
+        ratios (Sequence[int]): kernel size and stride ratios. The encoder uses downsampling ratios instead of
+            upsampling ratios, hence it will use the ratios in the reverse order to the ones specified here
+            that must match the decoder order. We use the decoder order as some models may only employ the decoder.
+        activation (str): Activation function.
+        activation_params (dict): Parameters to provide to the activation function.
+        norm (str): Normalization method.
+        norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
+        kernel_size (int): Kernel size for the initial convolution.
+        last_kernel_size (int): Kernel size for the initial convolution.
+        residual_kernel_size (int): Kernel size for the residual layers.
+        dilation_base (int): How much to increase the dilation with each layer.
+        causal (bool): Whether to use fully causal convolution.
+        pad_mode (str): Padding mode for the convolutions.
+        true_skip (bool): Whether to use true skip connection or a simple
+            (streamable) convolution as the skip connection in the residual network blocks.
+        compress (int): Reduced dimensionality in residual branches (from Demucs v3).
+        lstm (int): Number of LSTM layers at the end of the encoder.
+        disable_norm_outer_blocks (int): Number of blocks for which we don't apply norm.
+            For the encoder, it corresponds to the N first blocks.
+    """
+    def __init__(self, channels: int = 1, dimension: int = 128, n_filters: int = 32, n_residual_layers: int = 3,
+                 ratios: tp.List[int] = [8, 5, 4, 2], activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
+                 norm: str = 'none', norm_params: tp.Dict[str, tp.Any] = {}, kernel_size: int = 7,
+                 last_kernel_size: int = 7, residual_kernel_size: int = 3, dilation_base: int = 2, causal: bool = False,
+                 pad_mode: str = 'reflect', true_skip: bool = True, compress: int = 2, lstm: int = 0,
+                 disable_norm_outer_blocks: int = 0):
+        super().__init__()
+        self.channels = channels
+        self.dimension = dimension
+        self.n_filters = n_filters
+        self.ratios = list(reversed(ratios))
+        del ratios
+        self.n_residual_layers = n_residual_layers
+        self.hop_length = np.prod(self.ratios)
+        self.n_blocks = len(self.ratios) + 2  # first and last conv + residual blocks
+        self.disable_norm_outer_blocks = disable_norm_outer_blocks
+        assert self.disable_norm_outer_blocks >= 0 and self.disable_norm_outer_blocks <= self.n_blocks, \
+            "Number of blocks for which to disable norm is invalid." \
+            "It should be lower or equal to the actual number of blocks in the network and greater or equal to 0."
+
+        act = getattr(nn, activation)
+        mult = 1
+        model: tp.List[nn.Module] = [
+            StreamableConv1d(channels, mult * n_filters, kernel_size,
+                             norm='none' if self.disable_norm_outer_blocks >= 1 else norm,
+                             norm_kwargs=norm_params, causal=causal, pad_mode=pad_mode)
+        ]
+        # Downsample to raw audio scale
+        for i, ratio in enumerate(self.ratios):
+            block_norm = 'none' if self.disable_norm_outer_blocks >= i + 2 else norm
+            # Add residual layers
+            for j in range(n_residual_layers):
+                model += [
+                    SEANetResnetBlock(mult * n_filters, kernel_sizes=[residual_kernel_size, 1],
+                                      dilations=[dilation_base ** j, 1],
+                                      norm=block_norm, norm_params=norm_params,
+                                      activation=activation, activation_params=activation_params,
+                                      causal=causal, pad_mode=pad_mode, compress=compress, true_skip=true_skip)]
+
+            # Add downsampling layers
+            model += [
+                act(**activation_params),
+                StreamableConv1d(mult * n_filters, mult * n_filters * 2,
+                                 kernel_size=ratio * 2, stride=ratio,
+                                 norm=block_norm, norm_kwargs=norm_params,
+                                 causal=causal, pad_mode=pad_mode),
+            ]
+            mult *= 2
+
+        if lstm:
+            model += [StreamableLSTM(mult * n_filters, num_layers=lstm)]
+
+        model += [
+            act(**activation_params),
+            StreamableConv1d(mult * n_filters, dimension, last_kernel_size,
+                             norm='none' if self.disable_norm_outer_blocks == self.n_blocks else norm,
+                             norm_kwargs=norm_params, causal=causal, pad_mode=pad_mode)
+        ]
+
+        self.model = nn.Sequential(*model)
+
+    def forward(self, x):
+        return self.model(x)
+
+
+class SEANetDecoder(nn.Module):
+    """SEANet decoder.
+
+    Args:
+        channels (int): Audio channels.
+        dimension (int): Intermediate representation dimension.
+        n_filters (int): Base width for the model.
+        n_residual_layers (int): nb of residual layers.
+        ratios (Sequence[int]): kernel size and stride ratios.
+        activation (str): Activation function.
+        activation_params (dict): Parameters to provide to the activation function.
+        final_activation (str): Final activation function after all convolutions.
+        final_activation_params (dict): Parameters to provide to the activation function.
+        norm (str): Normalization method.
+        norm_params (dict): Parameters to provide to the underlying normalization used along with the convolution.
+        kernel_size (int): Kernel size for the initial convolution.
+        last_kernel_size (int): Kernel size for the initial convolution.
+        residual_kernel_size (int): Kernel size for the residual layers.
+        dilation_base (int): How much to increase the dilation with each layer.
+        causal (bool): Whether to use fully causal convolution.
+        pad_mode (str): Padding mode for the convolutions.
+        true_skip (bool): Whether to use true skip connection or a simple.
+            (streamable) convolution as the skip connection in the residual network blocks.
+        compress (int): Reduced dimensionality in residual branches (from Demucs v3).
+        lstm (int): Number of LSTM layers at the end of the encoder.
+        disable_norm_outer_blocks (int): Number of blocks for which we don't apply norm.
+            For the decoder, it corresponds to the N last blocks.
+        trim_right_ratio (float): Ratio for trimming at the right of the transposed convolution under the causal setup.
+            If equal to 1.0, it means that all the trimming is done at the right.
+    """
+    def __init__(self, channels: int = 1, dimension: int = 128, n_filters: int = 32, n_residual_layers: int = 3,
+                 ratios: tp.List[int] = [8, 5, 4, 2], activation: str = 'ELU', activation_params: dict = {'alpha': 1.0},
+                 final_activation: tp.Optional[str] = None, final_activation_params: tp.Optional[dict] = None,
+                 norm: str = 'none', norm_params: tp.Dict[str, tp.Any] = {}, kernel_size: int = 7,
+                 last_kernel_size: int = 7, residual_kernel_size: int = 3, dilation_base: int = 2, causal: bool = False,
+                 pad_mode: str = 'reflect', true_skip: bool = True, compress: int = 2, lstm: int = 0,
+                 disable_norm_outer_blocks: int = 0, trim_right_ratio: float = 1.0):
+        super().__init__()
+        self.dimension = dimension
+        self.channels = channels
+        self.n_filters = n_filters
+        self.ratios = ratios
+        del ratios
+        self.n_residual_layers = n_residual_layers
+        self.hop_length = np.prod(self.ratios)
+        self.n_blocks = len(self.ratios) + 2  # first and last conv + residual blocks
+        self.disable_norm_outer_blocks = disable_norm_outer_blocks
+        assert self.disable_norm_outer_blocks >= 0 and self.disable_norm_outer_blocks <= self.n_blocks, \
+            "Number of blocks for which to disable norm is invalid." \
+            "It should be lower or equal to the actual number of blocks in the network and greater or equal to 0."
+
+        act = getattr(nn, activation)
+        mult = int(2 ** len(self.ratios))
+        model: tp.List[nn.Module] = [
+            StreamableConv1d(dimension, mult * n_filters, kernel_size,
+                             norm='none' if self.disable_norm_outer_blocks == self.n_blocks else norm,
+                             norm_kwargs=norm_params, causal=causal, pad_mode=pad_mode)
+        ]
+
+        if lstm:
+            model += [StreamableLSTM(mult * n_filters, num_layers=lstm)]
+
+        # Upsample to raw audio scale
+        for i, ratio in enumerate(self.ratios):
+            block_norm = 'none' if self.disable_norm_outer_blocks >= self.n_blocks - (i + 1) else norm
+            # Add upsampling layers
+            model += [
+                act(**activation_params),
+                StreamableConvTranspose1d(mult * n_filters, mult * n_filters // 2,
+                                          kernel_size=ratio * 2, stride=ratio,
+                                          norm=block_norm, norm_kwargs=norm_params,
+                                          causal=causal, trim_right_ratio=trim_right_ratio),
+            ]
+            # Add residual layers
+            for j in range(n_residual_layers):
+                model += [
+                    SEANetResnetBlock(mult * n_filters // 2, kernel_sizes=[residual_kernel_size, 1],
+                                      dilations=[dilation_base ** j, 1],
+                                      activation=activation, activation_params=activation_params,
+                                      norm=block_norm, norm_params=norm_params, causal=causal,
+                                      pad_mode=pad_mode, compress=compress, true_skip=true_skip)]
+
+            mult //= 2
+
+        # Add final layers
+        model += [
+            act(**activation_params),
+            StreamableConv1d(n_filters, channels, last_kernel_size,
+                             norm='none' if self.disable_norm_outer_blocks >= 1 else norm,
+                             norm_kwargs=norm_params, causal=causal, pad_mode=pad_mode)
+        ]
+        # Add optional final activation to decoder (eg. tanh)
+        if final_activation is not None:
+            final_act = getattr(nn, final_activation)
+            final_activation_params = final_activation_params or {}
+            model += [
+                final_act(**final_activation_params)
+            ]
+        self.model = nn.Sequential(*model)
+
+    def forward(self, z):
+        y = self.model(z)
+        return y
diff --git a/audiocraft/audiocraft/modules/streaming.py b/audiocraft/audiocraft/modules/streaming.py
new file mode 100644
index 0000000000000000000000000000000000000000..fba06936294ca15d72acd2d44f9dbda39a638107
--- /dev/null
+++ b/audiocraft/audiocraft/modules/streaming.py
@@ -0,0 +1,131 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Streaming module API that should be implemented by all Streaming components,
+"""
+
+from contextlib import contextmanager
+import typing as tp
+from torch import nn
+import torch
+
+
+State = tp.Dict[str, torch.Tensor]
+
+
+class StreamingModule(nn.Module):
+    """Common API for streaming components.
+
+    Each streaming component has a streaming state, which is just a dict[str, Tensor].
+    By convention, the first dim of each tensor must be the batch size.
+    Don't use dots in the key names, as this would clash with submodules
+    (like in state_dict).
+
+    If `self._is_streaming` is True, the component should use and remember
+    the proper state inside `self._streaming_state`.
+
+    To set a streaming component in streaming state, use
+
+        with module.streaming():
+            ...
+
+    This will automatically reset the streaming state when exiting the context manager.
+    This also automatically propagates to all streaming children module.
+
+    Some module might also implement the `StreamingModule.flush` method, although
+    this one is trickier, as all parents module must be StreamingModule and implement
+    it as well for it to work properly. See `StreamingSequential` after.
+    """
+    def __init__(self) -> None:
+        super().__init__()
+        self._streaming_state: State = {}
+        self._is_streaming = False
+
+    def _apply_named_streaming(self, fn: tp.Any):
+        for name, module in self.named_modules():
+            if isinstance(module, StreamingModule):
+                fn(name, module)
+
+    def _set_streaming(self, streaming: bool):
+        def _set_streaming(name, module):
+            module._is_streaming = streaming
+        self._apply_named_streaming(_set_streaming)
+
+    @contextmanager
+    def streaming(self):
+        """Context manager to enter streaming mode. Reset streaming state on exit."""
+        self._set_streaming(True)
+        try:
+            yield
+        finally:
+            self._set_streaming(False)
+            self.reset_streaming()
+
+    def reset_streaming(self):
+        """Reset the streaming state."""
+        def _reset(name: str, module: StreamingModule):
+            module._streaming_state.clear()
+
+        self._apply_named_streaming(_reset)
+
+    def get_streaming_state(self) -> State:
+        """Return the streaming state, including that of sub-modules."""
+        state: State = {}
+
+        def _add(name: str, module: StreamingModule):
+            if name:
+                name += "."
+            for key, value in module._streaming_state.items():
+                state[name + key] = value
+
+        self._apply_named_streaming(_add)
+        return state
+
+    def set_streaming_state(self, state: State):
+        """Set the streaming state, including that of sub-modules."""
+        state = dict(state)
+
+        def _set(name: str, module: StreamingModule):
+            if name:
+                name += "."
+            module._streaming_state.clear()
+            for key, value in list(state.items()):
+                # complexity is not ideal here, but probably fine.
+                if key.startswith(name):
+                    local_key = key[len(name):]
+                    if '.' not in local_key:
+                        module._streaming_state[local_key] = value
+                        del state[key]
+
+        self._apply_named_streaming(_set)
+        assert len(state) == 0, list(state.keys())
+
+    def flush(self, x: tp.Optional[torch.Tensor] = None):
+        """Flush any remaining outputs that were waiting for completion.
+        Typically, for convolutions, this will add the final padding
+        and process the last buffer.
+
+        This should take an optional argument `x`, which will be provided
+        if a module before this one in the streaming pipeline has already
+        spitted out a flushed out buffer.
+        """
+        if x is None:
+            return None
+        else:
+            return self(x)
+
+
+class StreamingSequential(StreamingModule, nn.Sequential):
+    """A streaming compatible alternative of `nn.Sequential`.
+    """
+    def flush(self, x: tp.Optional[torch.Tensor] = None):
+        for module in self:
+            if isinstance(module, StreamingModule):
+                x = module.flush(x)
+            elif x is not None:
+                x = module(x)
+        return x
diff --git a/audiocraft/audiocraft/modules/transformer.py b/audiocraft/audiocraft/modules/transformer.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdc45cf87ad44e2bed3c7f5499429c87d81797c0
--- /dev/null
+++ b/audiocraft/audiocraft/modules/transformer.py
@@ -0,0 +1,752 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Transformer model, with streaming support, xformer attention support
+and easy causal attention with a potentially finite receptive field.
+
+See `StreamingTransformer` for more information.
+
+Unlike regular PyTorch Transformer, we make the hard choice that batches are first.
+"""
+
+import typing as tp
+
+from einops import rearrange
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+from torch.utils.checkpoint import checkpoint as torch_checkpoint
+from xformers import ops
+
+from .rope import RotaryEmbedding
+from .streaming import StreamingModule
+
+_efficient_attention_backend: str = 'torch'
+
+
+def set_efficient_attention_backend(backend: str = 'torch'):
+    # Using torch by default, it seems a bit faster on older P100 GPUs (~20% faster).
+    global _efficient_attention_backend
+    assert _efficient_attention_backend in ['xformers', 'torch']
+    _efficient_attention_backend = backend
+
+
+def _get_attention_time_dimension() -> int:
+    if _efficient_attention_backend == 'torch':
+        return 2
+    else:
+        return 1
+
+
+def _is_profiled() -> bool:
+    # Return true if we are currently running with a xformers profiler activated.
+    try:
+        from xformers.profiler import profiler
+    except ImportError:
+        return False
+    return profiler._Profiler._CURRENT_PROFILER is not None
+
+
+def create_norm_fn(norm_type: str, dim: int, **kwargs) -> nn.Module:
+    """Create normalization module for transformer encoder layer.
+
+    Args:
+        norm_type (str): Normalization method.
+        dim (int): Dimension of the normalized layer.
+        **kwargs (dict): Additional parameters for normalization layer.
+    Returns:
+        nn.Module: Normalization module.
+    """
+    if norm_type == 'layer_norm':
+        return nn.LayerNorm(dim, eps=1e-5, **kwargs)
+    else:
+        raise ValueError(f"Unknown norm type: {norm_type}")
+
+
+def create_sin_embedding(positions: torch.Tensor, dim: int, max_period: float = 10000,
+                         dtype: torch.dtype = torch.float32) -> torch.Tensor:
+    """Create sinusoidal positional embedding, with shape `[B, T, C]`.
+
+    Args:
+        positions (torch.Tensor): LongTensor of positions.
+        dim (int): Dimension of the embedding.
+        max_period (float): Maximum period of the cosine/sine functions.
+        dtype (torch.dtype or str): dtype to use to generate the embedding.
+    Returns:
+        torch.Tensor: Sinusoidal positional embedding.
+    """
+    # We aim for BTC format
+    assert dim % 2 == 0
+    half_dim = dim // 2
+    positions = positions.to(dtype)
+    adim = torch.arange(half_dim, device=positions.device, dtype=dtype).view(1, 1, -1)
+    max_period_tensor = torch.full([], max_period, device=positions.device, dtype=dtype)  # avoid sync point
+    phase = positions / (max_period_tensor ** (adim / (half_dim - 1)))
+    return torch.cat([torch.cos(phase), torch.sin(phase)], dim=-1)
+
+
+def expand_repeated_kv(x: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """torch.repeat_interleave(x, dim=2, repeats=n_rep) from xlformers."""
+    if n_rep == 1:
+        return x
+    if _efficient_attention_backend == 'torch':
+        bs, n_kv_heads, slen, head_dim = x.shape
+        return (
+            x[:, :, None, :, :]
+            .expand(bs, n_kv_heads, n_rep, slen, head_dim)
+            .reshape(bs, n_kv_heads * n_rep, slen, head_dim)
+        )
+    else:
+        bs, slen, n_kv_heads, head_dim = x.shape
+        return (
+            x[:, :, :, None, :]
+            .expand(bs, slen, n_kv_heads, n_rep, head_dim)
+            .reshape(bs, slen, n_kv_heads * n_rep, head_dim)
+        )
+
+
+class LayerScale(nn.Module):
+    """Layer scale from [Touvron et al 2021] (https://arxiv.org/pdf/2103.17239.pdf).
+    This rescales diagonally the residual outputs close to 0, with a learnt scale.
+
+    Args:
+        channels (int): Number of channels.
+        init (float): Initial scale.
+        channel_last (bool): If True, expect `[*, C]` shaped tensors, otherwise, `[*, C, T]`.
+        device (torch.device or str, optional): Device on which to initialize the module.
+        dtype (torch.dtype, optional): dtype to use to initialize the module.
+    """
+    def __init__(self, channels: int, init: float = 1e-4, channel_last: bool = True,
+                 device=None, dtype=None):
+        super().__init__()
+        self.channel_last = channel_last
+        self.scale = nn.Parameter(
+            torch.full((channels,), init,
+                       requires_grad=True, device=device, dtype=dtype))
+
+    def forward(self, x: torch.Tensor):
+        if self.channel_last:
+            return self.scale * x
+        else:
+            return self.scale[:, None] * x
+
+
+class StreamingMultiheadAttention(StreamingModule):
+    """Similar to `nn.MultiheadAttention` but with support for streaming, causal evaluation.
+
+    Args:
+        embed_dim (int): Dimension to project to.
+        num_heads (int): Number of heads.
+        dropout (float): Dropout level.
+        bias (bool): Use bias in projections.
+        causal (bool): Causal mask applied automatically.
+        past_context (int, optional): Receptive field for the causal mask, infinite if None.
+        custom (bool): Use custom MHA implementation, for testing / benchmarking.
+        memory_efficient (bool): Use xformers based memory efficient attention.
+        attention_as_float32 (bool): Perform the attention as float32
+            (especially important with memory_efficient as autocast won't do this automatically).
+        rope (`RotaryEmbedding`, optional): Rope embedding to use.
+        cross_attention: Should be true when used as a cross attention.
+            All keys and values must be available at once, streaming is only for the queries.
+            Cannot be used with `causal` or `rope` (as it wouldn't make sens to
+            interpret the time steps in the keys relative to those in the queries).
+        safe_streaming (bool): Bug fix, will go away with xformers update.
+        qk_layer_norm (bool): Layer normalization applied to queries and keys before dot product.
+        kv_repeat (int): If > 1, will repeat keys and queries multiple times (need to divide num_heads).
+            This will lead to faster decoding time on A100 or other GPUs with tensorcore.
+        device (torch.device, optional): Device on which to initialize.
+        dtype (torch.dtype, optional): dtype to use.
+    """
+    def __init__(self, embed_dim: int, num_heads: int, dropout: float = 0.0, bias: bool = True,
+                 causal: bool = False, past_context: tp.Optional[int] = None, custom: bool = False,
+                 memory_efficient: bool = False, attention_as_float32: bool = False,
+                 rope: tp.Optional[RotaryEmbedding] = None, cross_attention: bool = False,
+                 safe_streaming: bool = True, qk_layer_norm: bool = False, kv_repeat: int = 1,
+                 device=None, dtype=None):
+        super().__init__()
+        factory_kwargs = {'device': device, 'dtype': dtype}
+        if past_context is not None:
+            assert causal
+
+        self.embed_dim = embed_dim
+        self.causal = causal
+        self.past_context = past_context
+        self.memory_efficient = memory_efficient
+        self.attention_as_float32 = attention_as_float32
+        self.rope = rope
+        self.cross_attention = cross_attention
+        self.safe_streaming = safe_streaming
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.kv_repeat = kv_repeat
+        if cross_attention:
+            assert not causal, "Causal cannot work with cross attention."
+            assert rope is None, "Rope cannot work with cross attention."
+
+        if memory_efficient:
+            _verify_xformers_memory_efficient_compat()
+
+        self.custom = _is_custom(custom, memory_efficient)
+        if self.custom:
+            out_dim = embed_dim
+            assert num_heads % kv_repeat == 0
+            assert not cross_attention or kv_repeat == 1
+            num_kv = num_heads // kv_repeat
+            kv_dim = (embed_dim // num_heads) * num_kv
+            out_dim += 2 * kv_dim
+            in_proj = nn.Linear(embed_dim, out_dim, bias=bias, **factory_kwargs)
+            # We try to follow the default PyTorch MHA convention, to easily compare results.
+            self.in_proj_weight = in_proj.weight
+            self.in_proj_bias = in_proj.bias
+            if bias:
+                self.in_proj_bias.data.zero_()  # Following Pytorch convention
+            self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias, **factory_kwargs)
+            if bias:
+                self.out_proj.bias.data.zero_()
+        else:
+            assert not qk_layer_norm
+            assert kv_repeat == 1
+            self.mha = nn.MultiheadAttention(
+                embed_dim, num_heads, dropout=dropout, bias=bias, batch_first=True,
+                **factory_kwargs)
+        self.qk_layer_norm = qk_layer_norm
+        if qk_layer_norm:
+            assert self.custom
+            assert kv_repeat == 1
+            ln_dim = embed_dim
+            self.q_layer_norm = nn.LayerNorm(ln_dim)
+            self.k_layer_norm = nn.LayerNorm(ln_dim)
+
+    def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
+        if not self.custom:
+            # Support compat with regular MHA
+            keys = [n for n, _ in self.mha.named_parameters()]
+            for key in keys:
+                if prefix + key in state_dict:
+                    state_dict[prefix + "mha." + key] = state_dict.pop(prefix + key)
+        super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
+
+    def _get_mask(self, current_steps: int, device: torch.device, dtype: torch.dtype):
+        # Return a causal mask, accounting for potentially stored past keys/values
+        # We actually return a bias for the attention score, as this has the same
+        # convention both in the builtin MHA in Pytorch, and Xformers functions.
+        time_dim = _get_attention_time_dimension()
+        if self.memory_efficient:
+            from xformers.ops import LowerTriangularMask
+            if current_steps == 1:
+                # If we only have one step, then we do not need a mask.
+                return None
+            elif 'past_keys' in self._streaming_state:
+                raise RuntimeError("Not supported at the moment")
+            else:
+                # Then we can safely use a lower triangular mask
+                return LowerTriangularMask()
+        if self._streaming_state:
+            past_keys = self._streaming_state['past_keys']
+            past_steps = past_keys.shape[time_dim]
+        else:
+            past_steps = 0
+
+        queries_pos = torch.arange(
+            past_steps, current_steps + past_steps, device=device).view(-1, 1)
+        keys_pos = torch.arange(past_steps + current_steps, device=device).view(1, -1)
+        delta = queries_pos - keys_pos
+        valid = delta >= 0
+        if self.past_context is not None:
+            valid &= (delta <= self.past_context)
+        return torch.where(
+            valid,
+            torch.zeros([], device=device, dtype=dtype),
+            torch.full([], float('-inf'), device=device, dtype=dtype))
+
+    def _complete_kv(self, k, v):
+        time_dim = _get_attention_time_dimension()
+        if self.cross_attention:
+            # With cross attention we assume all keys and values
+            # are already available, and streaming is with respect
+            # to the queries only.
+            return k, v
+        # Complete the key/value pair using the streaming state.
+        if self._streaming_state:
+            pk = self._streaming_state['past_keys']
+            nk = torch.cat([pk, k], dim=time_dim)
+            if v is k:
+                nv = nk
+            else:
+                pv = self._streaming_state['past_values']
+                nv = torch.cat([pv, v], dim=time_dim)
+        else:
+            nk = k
+            nv = v
+
+        assert nk.shape[time_dim] == nv.shape[time_dim]
+        offset = 0
+        if self.past_context is not None:
+            offset = max(0, nk.shape[time_dim] - self.past_context)
+        if self._is_streaming:
+            self._streaming_state['past_keys'] = nk[:, offset:]
+            if v is not k:
+                self._streaming_state['past_values'] = nv[:, offset:]
+            if 'offset' in self._streaming_state:
+                self._streaming_state['offset'] += offset
+            else:
+                self._streaming_state['offset'] = torch.tensor(0)
+        return nk, nv
+
+    def _apply_rope(self, query: torch.Tensor, key: torch.Tensor):
+        # TODO: fix and verify layout.
+        assert _efficient_attention_backend == 'xformers', "Rope not supported with torch attn."
+        # Apply rope embeddings to query and key tensors.
+        assert self.rope is not None
+        if 'past_keys' in self._streaming_state:
+            past_keys_offset = self._streaming_state['past_keys'].shape[1]
+        else:
+            past_keys_offset = 0
+        if 'offset' in self._streaming_state:
+            past_context_offset = int(self._streaming_state['offset'].item())
+        else:
+            past_context_offset = 0
+        streaming_offset = past_context_offset + past_keys_offset
+        return self.rope.rotate_qk(query, key, start=streaming_offset)
+
+    def forward(self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor,
+                key_padding_mask=None, need_weights=False, attn_mask=None,
+                average_attn_weights=True, is_causal=False):
+        assert attn_mask is None
+        assert not is_causal, ("New param added in torch 2.0.1 not supported, "
+                               "use the causal args in the constructor.")
+
+        time_dim = _get_attention_time_dimension()
+        if time_dim == 2:
+            layout = "b h t d"
+        else:
+            layout = "b t h d"
+        dtype = query.dtype
+        if self._is_streaming:
+            assert self.causal or self.cross_attention, \
+                "Streaming only available for causal or cross attention"
+
+        if self.causal:
+            # At the moment we specialize only for the self-attention case.
+            assert query.shape[1] == key.shape[1], "Causal only for same length query / key / value"
+            assert value.shape[1] == key.shape[1], "Causal only for same length query / key / value"
+            attn_mask = self._get_mask(query.shape[1], query.device, query.dtype)
+
+        if self.custom:
+            # custom implementation
+            assert need_weights is False
+            assert key_padding_mask is None
+            if self.cross_attention:
+                # Different queries, keys, values, we have to spit manually the weights
+                # before applying the linear.
+                dim = self.in_proj_weight.shape[0] // 3
+                if self.in_proj_bias is None:
+                    bias_q, bias_k, bias_v = None, None, None
+                else:
+                    bias_q = self.in_proj_bias[:dim]
+                    bias_k = self.in_proj_bias[dim: 2 * dim]
+                    bias_v = self.in_proj_bias[2 * dim:]
+                q = nn.functional.linear(query, self.in_proj_weight[:dim], bias_q)
+                # todo: when streaming, we could actually save k, v and check the shape actually match.
+                k = nn.functional.linear(key, self.in_proj_weight[dim: 2 * dim], bias_k)
+                v = nn.functional.linear(value, self.in_proj_weight[2 * dim:], bias_v)
+                if self.qk_layer_norm is True:
+                    q = self.q_layer_norm(q)
+                    k = self.k_layer_norm(k)
+                q, k, v = [rearrange(x, f"b t (h d) -> {layout}", h=self.num_heads) for x in [q, k, v]]
+            else:
+                if not _is_profiled():
+                    # profiling breaks that propertysomehow.
+                    assert query is key, "specialized implementation"
+                    assert value is key, "specialized implementation"
+                projected = nn.functional.linear(query, self.in_proj_weight, self.in_proj_bias)
+                if self.kv_repeat == 1:
+                    if time_dim == 2:
+                        bound_layout = "b h p t d"
+                    else:
+                        bound_layout = "b t p h d"
+                    packed = rearrange(projected, f"b t (p h d) -> {bound_layout}", p=3, h=self.num_heads)
+                    q, k, v = ops.unbind(packed, dim=2)
+                else:
+                    embed_dim = self.embed_dim
+                    per_head_dim = (embed_dim // self.num_heads)
+                    kv_heads = self.num_heads // self.kv_repeat
+                    q = projected[:, :, :embed_dim]
+                    start = embed_dim
+                    end = start + per_head_dim * kv_heads
+                    k = projected[:, :, start: end]
+                    v = projected[:, :, end:]
+                    q = rearrange(q, f"b t (h d) -> {layout}", h=self.num_heads)
+                    k = rearrange(k, f"b t (h d) -> {layout}", h=kv_heads)
+                    v = rearrange(v, f"b t (h d) -> {layout}", h=kv_heads)
+
+                if self.qk_layer_norm is True:
+                    assert self.kv_repeat == 1
+                    q, k = [rearrange(x, f"{layout} -> b t (h d)") for x in [q, k]]
+                    q = self.q_layer_norm(q)
+                    k = self.k_layer_norm(k)
+                    q, k = [rearrange(x, f"b t (h d) -> {layout}", h=self.num_heads) for x in [q, k]]
+                if self.rope:
+                    q, k = self._apply_rope(q, k)
+                k, v = self._complete_kv(k, v)
+                if self.kv_repeat > 1:
+                    k = expand_repeated_kv(k, self.kv_repeat)
+                    v = expand_repeated_kv(v, self.kv_repeat)
+            if self.attention_as_float32:
+                q, k, v = [x.float() for x in [q, k, v]]
+            if self.memory_efficient:
+                p = self.dropout if self.training else 0
+                if _efficient_attention_backend == 'torch':
+                    x = torch.nn.functional.scaled_dot_product_attention(
+                        q, k, v, is_causal=attn_mask is not None, dropout_p=p)
+                else:
+                    x = ops.memory_efficient_attention(q, k, v, attn_mask, p=p)
+            else:
+                # We include the dot product as float32, for consistency
+                # with the other implementations that include that step
+                # as part of the attention. Note that when using `autocast`,
+                # the einsums would be done as bfloat16, but the softmax
+                # would be done as bfloat16, so `attention_as_float32` will
+                # extend a bit the range of operations done in float32,
+                # although this should make no difference.
+                q = q / q.shape[-1] ** 0.5
+                key_layout = layout.replace('t', 'k')
+                query_layout = layout
+                if self._is_streaming and self.safe_streaming and q.device.type == 'cuda':
+                    with torch.autocast(device_type=q.device.type, dtype=torch.float32):
+                        pre_w = torch.einsum(f"{query_layout},{key_layout}-> b h t k", q, k)
+                else:
+                    pre_w = torch.einsum(f"{query_layout},{key_layout}-> b h t k", q, k)
+                if attn_mask is not None:
+                    pre_w = pre_w + attn_mask
+                w = torch.softmax(pre_w, dim=-1)
+                w = F.dropout(w, self.dropout, training=self.training).to(v)
+                # Key and value have the same format.
+                x = torch.einsum(f"b h t k, {key_layout} -> {layout}", w, v)
+            x = x.to(dtype)
+            x = rearrange(x, f"{layout} -> b t (h d)", h=self.num_heads)
+            x = self.out_proj(x)
+        else:
+            key, value = self._complete_kv(key, value)
+            if self.attention_as_float32:
+                query, key, value = [x.float() for x in [query, key, value]]
+            x, _ = self.mha(
+                query, key, value, key_padding_mask,
+                need_weights, attn_mask, average_attn_weights)
+            x = x.to(dtype)
+
+        return x, None
+
+
+class StreamingTransformerLayer(nn.TransformerEncoderLayer):
+    """TransformerLayer with Streaming / Causal support.
+    This also integrates cross_attention, when passing `cross_attention=True`,
+    rather than having two separate classes like in PyTorch.
+
+    Args:
+        d_model (int): Dimension of the data.
+        num_heads (int): Number of heads.
+        dim_feedforward (int): Intermediate dimension of FF module.
+        dropout (float): Dropout both for MHA and FF.
+        bias_ff (bool): Use bias for FF.
+        bias_attn (bool): Use bias for MHA.
+        causal (bool): Causal mask applied automatically.
+        past_context (int, optional): Receptive field for the causal mask, infinite if None.
+        custom (bool): Use custom MHA implementation, for testing / benchmarking.
+        memory_efficient (bool): Use xformers based memory efficient attention.
+        attention_as_float32 (bool): Perform the attention as float32
+            (especially important with memory_efficient as autocast won't do this automatically).
+        qk_layer_norm (bool): Layer normalization applied to queries and keys before dot product in attention.
+        qk_layer_norm_cross (bool): Same for the cross attention.
+        cross_attention (bool): If True, expect to get secondary input for cross-attention.
+            Cross attention will use the default MHA, as it typically won't require
+            special treatment.
+        layer_scale (float, optional): If not None, LayerScale will be used with
+            the given value as initial scale.
+        rope (`RotaryEmbedding`, optional): Rope embedding to use.
+        attention_dropout (float, optional): If not None, separate the value of the dimension dropout
+            in FFN and of the attention dropout.
+        kv_repeat (int): If > 1, will repeat keys and queries multiple times (need to divide num_heads).
+            This will lead to faster decoding time on A100 or other GPUs with tensorcore.
+        device (torch.device, optional): Device on which to initialize.
+        dtype (torch.dtype, optional): dtype to use.
+        **kwargs: See `nn.TransformerEncoderLayer`.
+    """
+    def __init__(self, d_model: int, num_heads: int, dim_feedforward: int = 2048, dropout: float = 0.1,
+                 bias_ff: bool = True, bias_attn: bool = True, causal: bool = False,
+                 past_context: tp.Optional[int] = None, custom: bool = False,
+                 memory_efficient: bool = False, attention_as_float32: bool = False,
+                 qk_layer_norm: bool = False, qk_layer_norm_cross: bool = False,
+                 cross_attention: bool = False, layer_scale: tp.Optional[float] = None,
+                 rope: tp.Optional[RotaryEmbedding] = None, attention_dropout: tp.Optional[float] = None,
+                 kv_repeat: int = 1, norm: str = 'layer_norm', device=None, dtype=None, **kwargs):
+        super().__init__(d_model, num_heads, dim_feedforward, dropout,
+                         device=device, dtype=dtype, batch_first=True, **kwargs)
+        factory_kwargs = {'device': device, 'dtype': dtype}
+        # Redefine self_attn to our streaming multi-head attention
+        attn_kwargs: tp.Dict[str, tp.Any] = {
+            'embed_dim': d_model,
+            'num_heads': num_heads,
+            'dropout': dropout if attention_dropout is None else attention_dropout,
+            'bias': bias_attn,
+            'custom': custom,
+            'memory_efficient': memory_efficient,
+            'attention_as_float32': attention_as_float32,
+        }
+        self.self_attn: StreamingMultiheadAttention = StreamingMultiheadAttention(
+            causal=causal, past_context=past_context, rope=rope, qk_layer_norm=qk_layer_norm,
+            kv_repeat=kv_repeat, **attn_kwargs, **factory_kwargs)  # type: ignore
+        # Redefine feedforward layers to expose bias parameter
+        self.linear1 = nn.Linear(d_model, dim_feedforward, bias=bias_ff, **factory_kwargs)
+        self.linear2 = nn.Linear(dim_feedforward, d_model, bias=bias_ff, **factory_kwargs)
+
+        self.layer_scale_1: nn.Module
+        self.layer_scale_2: nn.Module
+        if layer_scale is None:
+            self.layer_scale_1 = nn.Identity()
+            self.layer_scale_2 = nn.Identity()
+        else:
+            self.layer_scale_1 = LayerScale(d_model, layer_scale, **factory_kwargs)
+            self.layer_scale_2 = LayerScale(d_model, layer_scale, **factory_kwargs)
+
+        self.cross_attention: tp.Optional[nn.Module] = None
+        if cross_attention:
+            self.cross_attention = StreamingMultiheadAttention(
+                cross_attention=True, qk_layer_norm=qk_layer_norm_cross,
+                **attn_kwargs, **factory_kwargs)
+            # Norm and dropout
+            self.dropout_cross = nn.Dropout(dropout)
+            # eps value matching that used in PyTorch reference implementation.
+            self.norm_cross = nn.LayerNorm(d_model, eps=1e-5, **factory_kwargs)
+            self.layer_scale_cross: nn.Module
+            if layer_scale is None:
+                self.layer_scale_cross = nn.Identity()
+            else:
+                self.layer_scale_cross = LayerScale(d_model, layer_scale, **factory_kwargs)
+        self.norm1 = create_norm_fn(norm, d_model, **factory_kwargs)  # type: ignore
+        self.norm2 = create_norm_fn(norm, d_model, **factory_kwargs)  # type: ignore
+
+    def _cross_attention_block(self, src: torch.Tensor,
+                               cross_attention_src: torch.Tensor) -> torch.Tensor:
+        assert self.cross_attention is not None
+        # queries are from src, keys and values from cross_attention_src.
+        x = self.cross_attention(
+            src, cross_attention_src, cross_attention_src, need_weights=False)[0]
+        return self.dropout_cross(x)  # type: ignore
+
+    def forward(self, src: torch.Tensor, src_mask: tp.Optional[torch.Tensor] = None,  # type: ignore
+                src_key_padding_mask: tp.Optional[torch.Tensor] = None,
+                cross_attention_src: tp.Optional[torch.Tensor] = None):
+        if self.cross_attention is None:
+            assert cross_attention_src is None
+        else:
+            assert cross_attention_src is not None
+        x = src
+        if self.norm_first:
+            x = x + self.layer_scale_1(
+                self._sa_block(self.norm1(x), src_mask, src_key_padding_mask))
+            if cross_attention_src is not None:
+                x = x + self.layer_scale_cross(
+                    self._cross_attention_block(
+                        self.norm_cross(x), cross_attention_src))
+            x = x + self.layer_scale_2(self._ff_block(self.norm2(x)))
+        else:
+            x = self.norm1(x + self.layer_scale_1(
+                self._sa_block(x, src_mask, src_key_padding_mask)))
+            if cross_attention_src is not None:
+                x = self.norm_cross(
+                    x + self.layer_scale_cross(
+                        self._cross_attention_block(src, cross_attention_src)))
+            x = self.norm2(x + self.layer_scale_2(self._ff_block(x)))
+        return x
+
+
+class StreamingTransformer(StreamingModule):
+    """Transformer with Streaming / Causal support.
+
+    Args:
+        d_model (int): Dimension of the data.
+        num_heads (int): Number of heads.
+        dim_feedforward (int): Intermediate dimension of FF module.
+        dropout (float): Dropout both for MHA and FF.
+        bias_ff (bool): Use bias for FF.
+        bias_attn (bool): Use bias for MHA.
+        causal (bool): Causal mask applied automatically.
+        past_context (int, optional): Receptive field for the causal mask, infinite if None.
+        custom (bool): Use custom MHA implementation, for testing / benchmarking.
+        memory_efficient (bool): Use xformers based memory efficient attention.
+        attention_as_float32 (bool): Perform the attention as float32
+            (especially important with memory_efficient as autocast won't do this automatically).
+        cross_attention (bool): If True, expect to get secondary input for cross-attention.
+        layer_scale (float, optional): If not None, LayerScale will be used
+            with the given value as initial scale.
+        positional_embedding (str): Positional embedding strategy (sin, rope, or sin_rope).
+        max_period (float): Maximum period of the time embedding.
+        positional_scale (float): Scale of positional embedding, set to 0 to deactivate.
+        xpos (bool): Apply xpos exponential decay to positional embedding (rope only).
+        lr (float, optional): learning rate override through the `make_optim_group` API.
+        weight_decay (float, optional): Weight_decay override through the `make_optim_group` API.
+        layer_class: (subclass of `StreamingTransformerLayer): class to use
+            to initialize the layers, allowing further customization outside of AudioCraft.
+        checkpointing (str): Checkpointing strategy to reduce memory usage.
+            No checkpointing if set to 'none'. Per layer checkpointing using PyTorch
+            if set to 'torch' (entire layer checkpointed, i.e. linears are evaluated twice,
+            minimal memory usage, but maximal runtime). Finally, `xformers_default` provide
+            a policy for opting-out some operations of the checkpointing like
+            linear layers and attention, providing a middle ground between speed and memory.
+        device (torch.device, optional): Device on which to initialize.
+        dtype (torch.dtype, optional): dtype to use.
+        **kwargs: See `nn.TransformerEncoderLayer`.
+    """
+    def __init__(self, d_model: int, num_heads: int, num_layers: int, dim_feedforward: int = 2048,
+                 dropout: float = 0.1, bias_ff: bool = True, bias_attn: bool = True,
+                 causal: bool = False, past_context: tp.Optional[int] = None,
+                 custom: bool = False, memory_efficient: bool = False, attention_as_float32: bool = False,
+                 cross_attention: bool = False, layer_scale: tp.Optional[float] = None,
+                 positional_embedding: str = 'sin', max_period: float = 10_000, positional_scale: float = 1.,
+                 xpos: bool = False, lr: tp.Optional[float] = None, weight_decay: tp.Optional[float] = None,
+                 layer_class: tp.Type[StreamingTransformerLayer] = StreamingTransformerLayer,
+                 checkpointing: str = 'none', device=None, dtype=None, **kwargs):
+        super().__init__()
+        assert d_model % num_heads == 0
+
+        self.positional_embedding = positional_embedding
+        self.max_period = max_period
+        self.positional_scale = positional_scale
+        self.weight_decay = weight_decay
+        self.lr = lr
+
+        assert positional_embedding in ['sin', 'rope', 'sin_rope']
+        self.rope: tp.Optional[RotaryEmbedding] = None
+        if self.positional_embedding in ['rope', 'sin_rope']:
+            assert _is_custom(custom, memory_efficient)
+            self.rope = RotaryEmbedding(d_model // num_heads, max_period=max_period,
+                                        xpos=xpos, scale=positional_scale, device=device)
+
+        self.checkpointing = checkpointing
+
+        assert checkpointing in ['none', 'torch', 'xformers_default', 'xformers_mm']
+        if self.checkpointing.startswith('xformers'):
+            _verify_xformers_internal_compat()
+
+        self.layers = nn.ModuleList()
+        for idx in range(num_layers):
+            self.layers.append(
+                layer_class(
+                    d_model=d_model, num_heads=num_heads, dim_feedforward=dim_feedforward,
+                    dropout=dropout, bias_ff=bias_ff, bias_attn=bias_attn,
+                    causal=causal, past_context=past_context, custom=custom,
+                    memory_efficient=memory_efficient, attention_as_float32=attention_as_float32,
+                    cross_attention=cross_attention, layer_scale=layer_scale, rope=self.rope,
+                    device=device, dtype=dtype, **kwargs))
+
+        if self.checkpointing != 'none':
+            for layer in self.layers:
+                # see audiocraft/optim/fsdp.py, magic signal to indicate this requires fixing the
+                # backward hook inside of FSDP...
+                layer._magma_checkpointed = True  # type: ignore
+                assert layer.layer_drop == 0., "Need further checking"  # type: ignore
+
+    def _apply_layer(self, layer, *args, **kwargs):
+        method = self.checkpointing
+        if method == 'none':
+            return layer(*args, **kwargs)
+        elif method == 'torch':
+            return torch_checkpoint(layer, *args, use_reentrant=False, **kwargs)
+        elif method.startswith('xformers'):
+            from xformers.checkpoint_fairinternal import checkpoint, _get_default_policy
+            if method == 'xformers_default':
+                # those operations will be saved, and not recomputed.
+                # According to Francisco we can get smarter policies but this is a good start.
+                allow_list = [
+                    "xformers.efficient_attention_forward_cutlass.default",
+                    "xformers_flash.flash_fwd.default",
+                    "aten.addmm.default",
+                    "aten.mm.default",
+                ]
+            elif method == 'xformers_mm':
+                # those operations will be saved, and not recomputed.
+                # According to Francisco we can get smarter policies but this is a good start.
+                allow_list = [
+                    "aten.addmm.default",
+                    "aten.mm.default",
+                ]
+            else:
+                raise ValueError(f"xformers checkpointing xformers policy {method} is not known.")
+            policy_fn = _get_default_policy(allow_list)
+            return checkpoint(layer, *args, policy_fn=policy_fn, **kwargs)
+        else:
+            raise ValueError(f"Checkpointing method {method} is unknown.")
+
+    def forward(self, x: torch.Tensor, in_attn_src: torch.Tensor, *args, **kwargs):
+        B, T, C = x.shape
+        if in_attn_src is not None:
+            _, in_attn_t, _ = in_attn_src.shape
+
+        if 'offsets' in self._streaming_state:
+            offsets = self._streaming_state['offsets']
+        else:
+            offsets = torch.zeros(B, dtype=torch.long, device=x.device)
+
+        if self.positional_embedding in ['sin', 'sin_rope']:
+            positions = torch.arange(T, device=x.device).view(1, -1, 1)
+            positions = positions + offsets.view(-1, 1, 1)
+            pos_emb = create_sin_embedding(positions, C, max_period=self.max_period, dtype=x.dtype)
+            x = x + self.positional_scale * pos_emb
+
+        for idx, layer in enumerate(self.layers):
+            if (idx % 4 == 0) and (idx < 36) and (idx != 0):
+                if in_attn_src is not None:
+                    x[:, -in_attn_t:, :] += in_attn_src
+            x = self._apply_layer(layer, x, *args, **kwargs)
+
+        if self._is_streaming:
+            self._streaming_state['offsets'] = offsets + T
+
+        return x
+
+    def make_optim_group(self):
+        group = {"params": list(self.parameters())}
+        if self.lr is not None:
+            group["lr"] = self.lr
+        if self.weight_decay is not None:
+            group["weight_decay"] = self.weight_decay
+        return group
+
+
+# special attention related function
+
+def _verify_xformers_memory_efficient_compat():
+    try:
+        from xformers.ops import memory_efficient_attention, LowerTriangularMask  # noqa
+    except ImportError:
+        raise ImportError(
+            "xformers is not installed. Please install it and try again.\n"
+            "To install on AWS and Azure, run \n"
+            "FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST='8.0'\\\n"
+            "pip install -U git+https://git@github.com/fairinternal/xformers.git#egg=xformers\n"
+            "To install on FAIR Cluster, run \n"
+            "FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST='6.0;7.0'\\\n"
+            "pip install -U git+https://git@github.com/fairinternal/xformers.git#egg=xformers\n")
+
+
+def _verify_xformers_internal_compat():
+    try:
+        from xformers.checkpoint_fairinternal import checkpoint, _get_default_policy  # noqa
+    except ImportError:
+        raise ImportError(
+            "Francisco's fairinternal xformers is not installed. Please install it and try again.\n"
+            "To install on AWS and Azure, run \n"
+            "FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST='8.0'\\\n"
+            "pip install -U git+https://git@github.com/fairinternal/xformers.git#egg=xformers\n"
+            "To install on FAIR Cluster, run \n"
+            "FORCE_CUDA=1 TORCH_CUDA_ARCH_LIST='6.0;7.0'\\\n"
+            "pip install -U git+https://git@github.com/fairinternal/xformers.git#egg=xformers\n")
+
+
+def _is_custom(custom: bool, memory_efficient: bool):
+    return custom or memory_efficient
diff --git a/audiocraft/audiocraft/optim/__init__.py b/audiocraft/audiocraft/optim/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..f48c17dfafa9a2be46a91ed1fb64f54c5572a730
--- /dev/null
+++ b/audiocraft/audiocraft/optim/__init__.py
@@ -0,0 +1,16 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Optimization stuff. In particular, optimizers (DAdaptAdam), schedulers
+and Exponential Moving Average.
+"""
+
+# flake8: noqa
+from .cosine_lr_scheduler import CosineLRScheduler
+from .dadam import DAdaptAdam
+from .inverse_sqrt_lr_scheduler import InverseSquareRootLRScheduler
+from .linear_warmup_lr_scheduler import LinearWarmupLRScheduler
+from .polynomial_decay_lr_scheduler import PolynomialDecayLRScheduler
+from .ema import ModuleDictEMA
diff --git a/audiocraft/audiocraft/optim/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ea7df3e581582945cdb2f4c3f929e32dadf8a213
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/cosine_lr_scheduler.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/cosine_lr_scheduler.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..d5097cf28e02745c4a6d4b847e74e74cd544af4b
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/cosine_lr_scheduler.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/dadam.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/dadam.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7f74a9afebdecda78edf2983f0c37e2dd8398b65
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/dadam.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/ema.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/ema.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2eaca706c5f31558985e0f3c095393c7a746df4a
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/ema.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/fsdp.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/fsdp.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..27f6b9f5076074bcb30c8296fc22cff8c6c09f99
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/fsdp.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/inverse_sqrt_lr_scheduler.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/inverse_sqrt_lr_scheduler.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..646ae383db386d84dd5370e0eb98633c00d7e63a
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/inverse_sqrt_lr_scheduler.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/linear_warmup_lr_scheduler.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/linear_warmup_lr_scheduler.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..76489594d3db29881ec02557a42b99a66f19c6c1
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/linear_warmup_lr_scheduler.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/__pycache__/polynomial_decay_lr_scheduler.cpython-311.pyc b/audiocraft/audiocraft/optim/__pycache__/polynomial_decay_lr_scheduler.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..95055259f9bf9cd6bb2274ec11276d004afd422c
Binary files /dev/null and b/audiocraft/audiocraft/optim/__pycache__/polynomial_decay_lr_scheduler.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/optim/cosine_lr_scheduler.py b/audiocraft/audiocraft/optim/cosine_lr_scheduler.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e4f0bbf28f1ad893a301f1bfac1da8e97370337
--- /dev/null
+++ b/audiocraft/audiocraft/optim/cosine_lr_scheduler.py
@@ -0,0 +1,48 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import _LRScheduler
+
+
+class CosineLRScheduler(_LRScheduler):
+    """Cosine LR scheduler.
+
+    Args:
+        optimizer (Optimizer): Torch optimizer.
+        warmup_steps (int): Number of warmup steps.
+        total_steps (int): Total number of steps.
+        lr_min_ratio (float): Minimum learning rate.
+        cycle_length (float): Cycle length.
+    """
+    def __init__(self, optimizer: Optimizer, total_steps: int, warmup_steps: int,
+                 lr_min_ratio: float = 0.0, cycle_length: float = 1.0):
+        self.warmup_steps = warmup_steps
+        assert self.warmup_steps >= 0
+        self.total_steps = total_steps
+        assert self.total_steps >= 0
+        self.lr_min_ratio = lr_min_ratio
+        self.cycle_length = cycle_length
+        super().__init__(optimizer)
+
+    def _get_sched_lr(self, lr: float, step: int):
+        if step < self.warmup_steps:
+            lr_ratio = step / self.warmup_steps
+            lr = lr_ratio * lr
+        elif step <= self.total_steps:
+            s = (step - self.warmup_steps) / (self.total_steps - self.warmup_steps)
+            lr_ratio = self.lr_min_ratio + 0.5 * (1 - self.lr_min_ratio) * \
+                (1. + math.cos(math.pi * s / self.cycle_length))
+            lr = lr_ratio * lr
+        else:
+            lr_ratio = self.lr_min_ratio
+            lr = lr_ratio * lr
+        return lr
+
+    def get_lr(self):
+        return [self._get_sched_lr(lr, self.last_epoch) for lr in self.base_lrs]
diff --git a/audiocraft/audiocraft/optim/dadam.py b/audiocraft/audiocraft/optim/dadam.py
new file mode 100644
index 0000000000000000000000000000000000000000..a84402f744867610180b9576b2ee3302501fd035
--- /dev/null
+++ b/audiocraft/audiocraft/optim/dadam.py
@@ -0,0 +1,252 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+import torch
+import torch.optim
+import torch.distributed as dist
+
+if TYPE_CHECKING:
+    from torch.optim.optimizer import _params_t
+else:
+    _params_t = Any
+
+
+logger = logging.getLogger(__name__)
+
+
+def to_real(x):
+    if torch.is_complex(x):
+        return x.real
+    else:
+        return x
+
+
+class DAdaptAdam(torch.optim.Optimizer):
+    """Adam with D-Adaptation automatic step-sizes.
+    Leave LR set to 1 unless you encounter instability.
+
+    Args:
+        params (iterable):
+            Iterable of parameters to optimize or dicts defining parameter groups.
+        lr (float):
+            Learning rate adjustment parameter. Increases or decreases the D-adapted learning rate.
+        betas (tuple[float, float], optional): coefficients used for computing
+            running averages of gradient and its square (default: (0.9, 0.999))
+        momentum (float):
+            Momentum value in  the range [0,1) (default: 0.9).
+        eps (float):
+            Term added to the denominator outside of the root operation to improve numerical stability. (default: 1e-8).
+        weight_decay (float):
+            Weight decay, i.e. a L2 penalty (default: 0).
+        log_every (int):
+            Log using print every k steps, default 0 (no logging).
+        decouple (boolean):
+            Use AdamW style decoupled weight decay
+        d0 (float):
+            Initial D estimate for D-adaptation (default 1e-6). Rarely needs changing.
+        growth_rate (float):
+            prevent the D estimate from growing faster than this multiplicative rate.
+            Default is inf, for unrestricted. Values like 1.02 give a kind of learning
+            rate warmup effect.
+        fsdp_in_use (bool):
+            If you're using sharded parameters, this should be set to True. The optimizer
+            will attempt to auto-detect this, but if you're using an implementation other
+            than PyTorch's builtin version, the auto-detection won't work.
+    """
+    def __init__(self, params, lr=1.0,
+                 betas=(0.9, 0.999),
+                 eps=1e-8,
+                 weight_decay=0,
+                 log_every=0,
+                 decouple=True,
+                 d0=1e-6,
+                 growth_rate=float('inf')):
+        if not 0.0 < d0:
+            raise ValueError("Invalid d0 value: {}".format(d0))
+        if not 0.0 < lr:
+            raise ValueError("Invalid learning rate: {}".format(lr))
+        if not 0.0 < eps:
+            raise ValueError("Invalid epsilon value: {}".format(eps))
+        if not 0.0 <= betas[0] < 1.0:
+            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
+        if not 0.0 <= betas[1] < 1.0:
+            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
+
+        if decouple:
+            logger.info("Using decoupled weight decay")
+
+        from .fsdp import is_fsdp_used
+        fsdp_in_use = is_fsdp_used()
+        defaults = dict(lr=lr, betas=betas, eps=eps,
+                        weight_decay=weight_decay,
+                        d=d0,
+                        k=0,
+                        gsq_weighted=0.0,
+                        log_every=log_every,
+                        decouple=decouple,
+                        growth_rate=growth_rate,
+                        fsdp_in_use=fsdp_in_use)
+
+        super().__init__(params, defaults)
+
+    @property
+    def supports_memory_efficient_fp16(self):
+        return False
+
+    @property
+    def supports_flat_params(self):
+        return True
+
+    def step(self, closure=None):
+        """Performs a single optimization step.
+
+        Args:
+            closure (callable, optional): A closure that reevaluates the model
+                and returns the loss.
+        """
+        loss = None
+        if closure is not None:
+            loss = closure()
+
+        g_sq = 0.0
+        sksq_weighted = 0.0
+        sk_l1 = 0.0
+
+        lr = max(group['lr'] for group in self.param_groups)
+
+        group = self.param_groups[0]
+        gsq_weighted = group['gsq_weighted']
+        d = group['d']
+        dlr = d*lr
+
+        growth_rate = group['growth_rate']
+        decouple = group['decouple']
+        fsdp_in_use = group['fsdp_in_use']
+        log_every = group['log_every']
+
+        beta1, beta2 = group['betas']
+
+        for group in self.param_groups:
+            group_lr = group['lr']
+            decay = group['weight_decay']
+            k = group['k']
+            eps = group['eps']
+
+            if group_lr not in [lr, 0.0]:
+                raise RuntimeError("Setting different lr values in different parameter "
+                                   "groups is only supported for values of 0")
+
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                if hasattr(p, "_fsdp_flattened"):
+                    fsdp_in_use = True
+                grad = p.grad.data
+
+                # Apply weight decay (coupled variant)
+                if decay != 0 and not decouple:
+                    grad.add_(p.data, alpha=decay)
+
+                state = self.state[p]
+
+                # State initialization
+                if 'step' not in state:
+                    state['step'] = 0
+                    state['s'] = torch.zeros_like(p.data, memory_format=torch.preserve_format).detach()
+                    # Exponential moving average of gradient values
+                    state['exp_avg'] = torch.zeros_like(p.data, memory_format=torch.preserve_format).detach()
+                    # Exponential moving average of squared gradient values
+                    state['exp_avg_sq'] = torch.zeros_like(
+                        to_real(p.data), memory_format=torch.preserve_format).detach()
+
+                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
+
+                grad_grad = to_real(grad * grad.conj())
+
+                # Adam EMA updates
+                if group_lr > 0:
+                    exp_avg.mul_(beta1).add_(grad, alpha=dlr*(1-beta1))
+                    exp_avg_sq.mul_(beta2).add_(grad_grad, alpha=1-beta2)
+
+                    denom = exp_avg_sq.sqrt().add_(eps)
+
+                    g_sq += grad_grad.div_(denom).sum().item()
+
+                    s = state['s']
+                    s.mul_(beta2).add_(grad, alpha=dlr*(1-beta2))
+                    sksq_weighted += to_real(s * s.conj()).div_(denom).sum().item()
+                    sk_l1 += s.abs().sum().item()
+
+            ######
+
+        gsq_weighted = beta2*gsq_weighted + g_sq*(dlr**2)*(1-beta2)
+        d_hat = d
+
+        # if we have not done any progres, return
+        # if we have any gradients available, will have sk_l1 > 0 (unless \|g\|=0)
+        if sk_l1 == 0:
+            return loss
+
+        if lr > 0.0:
+            if fsdp_in_use:
+                dist_tensor = torch.zeros(3, device='cuda')
+                dist_tensor[0] = sksq_weighted
+                dist_tensor[1] = gsq_weighted
+                dist_tensor[2] = sk_l1
+                dist.all_reduce(dist_tensor, op=dist.ReduceOp.SUM)
+                global_sksq_weighted = dist_tensor[0]
+                global_gsq_weighted = dist_tensor[1]
+                global_sk_l1 = dist_tensor[2]
+            else:
+                global_sksq_weighted = sksq_weighted
+                global_gsq_weighted = gsq_weighted
+                global_sk_l1 = sk_l1
+
+            d_hat = (global_sksq_weighted/(1-beta2) - global_gsq_weighted)/global_sk_l1
+            d = max(d, min(d_hat, d*growth_rate))
+
+        if log_every > 0 and k % log_every == 0:
+            logger.info(
+                f"(k={k}) dlr: {dlr:1.1e} d_hat: {d_hat:1.1e}, d: {d:1.8}. "
+                f"sksq_weighted={global_sksq_weighted:1.1e} gsq_weighted={global_gsq_weighted:1.1e} "
+                f"sk_l1={global_sk_l1:1.1e}{' (FSDP)' if fsdp_in_use else ''}")
+
+        for group in self.param_groups:
+            group['gsq_weighted'] = gsq_weighted
+            group['d'] = d
+
+            group_lr = group['lr']
+            decay = group['weight_decay']
+            k = group['k']
+            eps = group['eps']
+
+            for p in group['params']:
+                if p.grad is None:
+                    continue
+                grad = p.grad.data
+
+                state = self.state[p]
+
+                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
+
+                state['step'] += 1
+
+                denom = exp_avg_sq.sqrt().add_(eps)
+                denom = denom.type(p.type())
+
+                # Apply weight decay (decoupled variant)
+                if decay != 0 and decouple and group_lr > 0:
+                    p.data.add_(p.data, alpha=-decay * dlr)
+
+                # Take step
+                p.data.addcdiv_(exp_avg, denom, value=-1)
+
+            group['k'] = k + 1
+
+        return loss
diff --git a/audiocraft/audiocraft/optim/ema.py b/audiocraft/audiocraft/optim/ema.py
new file mode 100644
index 0000000000000000000000000000000000000000..4337eaff066a8ca124dca3e3e63ee36e417c055c
--- /dev/null
+++ b/audiocraft/audiocraft/optim/ema.py
@@ -0,0 +1,85 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# ModelEMA implementation is taken from
+# https://github.com/facebookresearch/demucs
+
+from collections import defaultdict
+import typing as tp
+
+import torch
+import torch.nn as nn
+
+
+def _get_all_non_persistent_buffers_set(module: nn.Module, root: str = "") -> set:
+    names: set = set()
+    for (name, sub_module) in module.named_modules():
+        if name == '':
+            buffer_names = module._non_persistent_buffers_set
+            buffer_names = {f"{root}.{buff_name}" if len(root) > 0 else buff_name
+                            for buff_name in buffer_names}
+            names.update(buffer_names)
+        else:
+            sub_name = f"{root}.{name}" if len(root) > 0 else name
+            sub_buffer_names = _get_all_non_persistent_buffers_set(sub_module, sub_name)
+            names.update(sub_buffer_names)
+    return names
+
+
+def _get_named_tensors(module: nn.Module):
+    non_persistent_buffers_set = _get_all_non_persistent_buffers_set(module)
+    named_buffers = [(name, buffer) for (name, buffer) in module.named_buffers()
+                     if name not in non_persistent_buffers_set]
+    named_parameters = list(module.named_parameters())
+    return named_parameters + named_buffers
+
+
+class ModuleDictEMA:
+    """Exponential Moving Average over a nn.ModuleDict.
+
+    You can switch to the EMA weights temporarily.
+    """
+    def __init__(self, module_dict: nn.ModuleDict, decay: float = 0.999,
+                 unbias: bool = True, device: tp.Union[torch.device, str] = 'cpu'):
+        self.decay = decay
+        self.module_dict = module_dict
+        self.state: dict = defaultdict(dict)
+        self.count = 0
+        self.device = device
+        self.unbias = unbias
+        self._init()
+
+    def _init(self):
+        for module_name, module in self.module_dict.items():
+            for key, val in _get_named_tensors(module):
+                if not val.is_floating_point():
+                    continue
+                device = self.device or val.device
+                if key not in self.state[module_name]:
+                    self.state[module_name][key] = val.detach().to(device, copy=True)
+
+    def step(self):
+        if self.unbias:
+            self.count = self.count * self.decay + 1
+            w = 1 / self.count
+        else:
+            w = 1 - self.decay
+        for module_name, module in self.module_dict.items():
+            for key, val in _get_named_tensors(module):
+                if not val.is_floating_point():
+                    continue
+                device = self.device or val.device
+                self.state[module_name][key].mul_(1 - w)
+                self.state[module_name][key].add_(val.detach().to(device), alpha=w)
+
+    def state_dict(self):
+        return {'state': self.state, 'count': self.count}
+
+    def load_state_dict(self, state):
+        self.count = state['count']
+        for module_name, module in state['state'].items():
+            for key, val in module.items():
+                self.state[module_name][key].copy_(val)
diff --git a/audiocraft/audiocraft/optim/fsdp.py b/audiocraft/audiocraft/optim/fsdp.py
new file mode 100644
index 0000000000000000000000000000000000000000..b3c1a55b6bf1a33092a021c5cefbbb2ae848918a
--- /dev/null
+++ b/audiocraft/audiocraft/optim/fsdp.py
@@ -0,0 +1,195 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Wrapper around FSDP for more convenient use in the training loops.
+"""
+
+from contextlib import contextmanager
+import typing as tp
+import dora
+import torch
+
+from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
+from torch.distributed.fsdp import (
+    MixedPrecision, ShardingStrategy, FullStateDictConfig, StateDictType)
+from torch.distributed._shard.sharded_tensor.api import ShardedTensor
+
+
+def is_fsdp_used() -> bool:
+    """Return whether we are using FSDP."""
+    # A bit of a hack but should work from anywhere.
+    if dora.is_xp():
+        cfg = dora.get_xp().cfg
+        if hasattr(cfg, 'fsdp'):
+            return cfg.fsdp.use
+    return False
+
+
+def is_sharded_tensor(x: tp.Any) -> bool:
+    return isinstance(x, ShardedTensor)
+
+
+@contextmanager
+def switch_to_full_state_dict(models: tp.List[FSDP]):
+    # Another bug in FSDP makes it that we cannot use the `state_dict_type` API,
+    # so let's do thing manually.
+    for model in models:
+        FSDP.set_state_dict_type(  # type: ignore
+            model, StateDictType.FULL_STATE_DICT,
+            FullStateDictConfig(offload_to_cpu=True, rank0_only=True))
+    try:
+        yield
+    finally:
+        for model in models:
+            FSDP.set_state_dict_type(model, StateDictType.LOCAL_STATE_DICT)  # type: ignore
+
+
+def wrap_with_fsdp(cfg, model: torch.nn.Module,
+                   block_classes: tp.Optional[tp.Set[tp.Type]] = None) -> FSDP:
+    """Wraps a model with FSDP."""
+    # Some of the typing is disabled until this gets integrated
+    # into the stable version of PyTorch.
+    from torch.distributed.fsdp.wrap import ModuleWrapPolicy  # type: ignore
+
+    # we import this here to prevent circular import.
+    from ..modules.transformer import StreamingTransformerLayer
+    from ..modules.conditioners import ConditioningProvider
+
+    _fix_post_backward_hook()
+
+    assert cfg.use
+    sharding_strategy_dict = {
+        "no_shard": ShardingStrategy.NO_SHARD,
+        "shard_grad_op": ShardingStrategy.SHARD_GRAD_OP,
+        "full_shard": ShardingStrategy.FULL_SHARD,
+    }
+
+    dtype_dict = {
+        "float32": torch.float32,
+        "float16": torch.float16,
+        "bfloat16": torch.bfloat16,
+    }
+
+    mixed_precision_config = MixedPrecision(
+        param_dtype=dtype_dict[cfg.param_dtype],
+        reduce_dtype=dtype_dict[cfg.reduce_dtype],
+        buffer_dtype=dtype_dict[cfg.buffer_dtype],
+    )
+
+    sharding_strategy_config = sharding_strategy_dict[cfg.sharding_strategy]
+    # The following is going to require being a bit smart
+    # when doing LM, because this would flush the weights for every time step
+    # during generation. One possiblity is to use hybrid sharding:
+    # See: https://pytorch.org/docs/master/fsdp.html#torch.distributed.fsdp.ShardingStrategy
+    assert sharding_strategy_config != ShardingStrategy.FULL_SHARD, \
+        "Not supported at the moment, requires a bit more work."
+
+    local_rank = dora.distrib.get_distrib_spec().local_rank
+    assert local_rank < torch.cuda.device_count(), "Please upgrade Dora!"
+
+    auto_wrap_policy = None
+    if block_classes is None:
+        block_classes = {StreamingTransformerLayer, ConditioningProvider}
+    if cfg.per_block:
+        auto_wrap_policy = ModuleWrapPolicy(block_classes)
+    wrapped = _FSDPFixStateDict(
+        model,
+        sharding_strategy=sharding_strategy_config,
+        mixed_precision=mixed_precision_config,
+        device_id=local_rank,
+        sync_module_states=True,
+        use_orig_params=True,
+        auto_wrap_policy=auto_wrap_policy,
+    )  # type: ignore
+    FSDP.set_state_dict_type(wrapped, StateDictType.LOCAL_STATE_DICT)  # type: ignore
+
+    # Let the wrapped model know about the wrapping!
+    # We use __dict__ to avoid it going into the state dict.
+    # This is a bit dirty, but needed during generation, as otherwise
+    # the wrapped model would call itself and bypass FSDP.
+    for module in FSDP.fsdp_modules(wrapped):
+        original = module._fsdp_wrapped_module
+        original.__dict__['_fsdp'] = module
+    return wrapped
+
+
+def purge_fsdp(model: FSDP):
+    """Purge the FSDP cached shard inside the model. This should
+    allow setting the best state or switching to the EMA.
+    """
+    from torch.distributed.fsdp._runtime_utils import _reshard  # type: ignore
+    for module in FSDP.fsdp_modules(model):
+        handles = module._handles
+        if not handles:
+            continue
+        handle = handles[0]
+        unsharded_flat_param = handle._get_padded_unsharded_flat_param()
+        storage_size: int = unsharded_flat_param._typed_storage()._size()  # type: ignore
+        if storage_size == 0:
+            continue
+        true_list = [True for h in handles]
+        _reshard(module, handles, true_list)
+
+
+class _FSDPFixStateDict(FSDP):
+    @staticmethod
+    def _name_without_fsdp_prefix(name: str) -> str:
+        from torch.distributed.fsdp._common_utils import FSDP_WRAPPED_MODULE  # type: ignore
+        parts = name.split('.')
+        new_parts = [part for part in parts if part != FSDP_WRAPPED_MODULE]
+        return '.'.join(new_parts)
+
+    def state_dict(self) -> tp.Dict[str, tp.Any]:  # type: ignore
+        state = dict(super().state_dict())
+        for key, value in list(state.items()):
+            if is_sharded_tensor(value):
+                del state[key]
+        return state
+
+    def load_state_dict(self, state: tp.Dict[str, tp.Any]):  # type: ignore
+        if self._state_dict_type is StateDictType.FULL_STATE_DICT:
+            super().load_state_dict(state)
+            purge_fsdp(self)
+            return
+        # Fix FSDP load state dict in all situation.
+        # Use this only with LOCAL_STATE_DICT !!!
+        current_state = dict(super().state_dict())
+        for key, value in state.items():
+            key = _FSDPFixStateDict._name_without_fsdp_prefix(key)
+            if key not in current_state:
+                # Emulate strict loading manually.
+                raise RuntimeError(f"Unknown state key {key}")
+            current_state[key].copy_(value)
+
+        # Purging cached weights from previous forward.
+        purge_fsdp(self)
+
+
+_hook_fixed = False
+
+
+def _fix_post_backward_hook():
+    global _hook_fixed
+    if _hook_fixed:
+        return
+    _hook_fixed = True
+
+    from torch.distributed.fsdp import _runtime_utils
+    from torch.distributed.fsdp._common_utils import TrainingState, HandleTrainingState
+    old_hook = _runtime_utils._post_backward_hook
+
+    def _post_backward_hook(state, handle, *args, **kwargs):
+        checkpointed = getattr(state._fsdp_wrapped_module, '_audiocraft_checkpointed', False)
+        if checkpointed:
+            # there will be one more forward in the backward with checkpointing and that will
+            # massively confuse FSDP, so we have to make it think everything
+            # is going according to the plan.
+            state.training_state = TrainingState.FORWARD_BACKWARD
+            handle._training_state = HandleTrainingState.BACKWARD_PRE
+        old_hook(state, handle, *args, **kwargs)
+
+    _runtime_utils._post_backward_hook = _post_backward_hook
diff --git a/audiocraft/audiocraft/optim/inverse_sqrt_lr_scheduler.py b/audiocraft/audiocraft/optim/inverse_sqrt_lr_scheduler.py
new file mode 100644
index 0000000000000000000000000000000000000000..920192e8842c5635bf6f7f76618fa4a6f4b0114a
--- /dev/null
+++ b/audiocraft/audiocraft/optim/inverse_sqrt_lr_scheduler.py
@@ -0,0 +1,38 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import _LRScheduler
+
+
+class InverseSquareRootLRScheduler(_LRScheduler):
+    """Inverse square root LR scheduler.
+
+    Args:
+        optimizer (Optimizer): Torch optimizer.
+        warmup_steps (int): Number of warmup steps.
+        warmup_init_lr (tp.Optional[float]): Initial learning rate
+            during warmup phase. When not set, use the provided learning rate.
+    """
+    def __init__(self, optimizer: Optimizer, warmup_steps: int, warmup_init_lr: tp.Optional[float] = 0):
+        self.warmup_steps = warmup_steps
+        self.warmup_init_lr = warmup_init_lr
+        super().__init__(optimizer)
+
+    def _get_sched_lr(self, lr: float, step: int):
+        if step < self.warmup_steps:
+            warmup_init_lr = self.warmup_init_lr or 0
+            lr_step = (lr - warmup_init_lr) / self.warmup_steps
+            lr = warmup_init_lr + step * lr_step
+        else:
+            decay_factor = lr * self.warmup_steps**0.5
+            lr = decay_factor * step**-0.5
+        return lr
+
+    def get_lr(self):
+        return [self._get_sched_lr(base_lr, self._step_count) for base_lr in self.base_lrs]
diff --git a/audiocraft/audiocraft/optim/linear_warmup_lr_scheduler.py b/audiocraft/audiocraft/optim/linear_warmup_lr_scheduler.py
new file mode 100644
index 0000000000000000000000000000000000000000..03274a1ae52b6f20473973b77619f34b2bddd6a1
--- /dev/null
+++ b/audiocraft/audiocraft/optim/linear_warmup_lr_scheduler.py
@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import _LRScheduler
+
+
+class LinearWarmupLRScheduler(_LRScheduler):
+    """Inverse square root LR scheduler.
+
+    Args:
+        optimizer (Optimizer): Torch optimizer.
+        warmup_steps (int): Number of warmup steps.
+        warmup_init_lr (tp.Optional[float]): Initial learning rate
+            during warmup phase. When not set, use the provided learning rate.
+    """
+    def __init__(self, optimizer: Optimizer, warmup_steps: int, warmup_init_lr: tp.Optional[float] = 0):
+        self.warmup_steps = warmup_steps
+        self.warmup_init_lr = warmup_init_lr
+        super().__init__(optimizer)
+
+    def _get_sched_lr(self, lr: float, step: int):
+        if step < self.warmup_steps:
+            warmup_init_lr = self.warmup_init_lr or 0
+            lr_step = (lr - warmup_init_lr) / self.warmup_steps
+            lr = warmup_init_lr + step * lr_step
+        return lr
+
+    def get_lr(self):
+        return [self._get_sched_lr(base_lr, self.last_epoch) for base_lr in self.base_lrs]
diff --git a/audiocraft/audiocraft/optim/polynomial_decay_lr_scheduler.py b/audiocraft/audiocraft/optim/polynomial_decay_lr_scheduler.py
new file mode 100644
index 0000000000000000000000000000000000000000..c5ea30b094538269dbb0055ab3163f84d1cf6e90
--- /dev/null
+++ b/audiocraft/audiocraft/optim/polynomial_decay_lr_scheduler.py
@@ -0,0 +1,47 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from torch.optim import Optimizer
+from torch.optim.lr_scheduler import _LRScheduler
+
+
+class PolynomialDecayLRScheduler(_LRScheduler):
+    """Polynomial decay LR scheduler.
+
+    Args:
+        optimizer (Optimizer): Torch optimizer.
+        warmup_steps (int): Number of warmup steps.
+        total_steps (int): Total number of steps.
+        end_lr (float): Final learning rate to achieve over total number of steps.
+        zero_lr_warmup_steps (int): Number of steps with a learning rate of value 0.
+        power (float): Decay exponent.
+    """
+    def __init__(self, optimizer: Optimizer, warmup_steps: int, total_steps: int,
+                 end_lr: float = 0., zero_lr_warmup_steps: int = 0, power: float = 1.):
+        self.warmup_steps = warmup_steps
+        self.total_steps = total_steps
+        self.end_lr = end_lr
+        self.zero_lr_warmup_steps = zero_lr_warmup_steps
+        self.power = power
+        super().__init__(optimizer)
+
+    def _get_sched_lr(self, lr: float, step: int):
+        if self.zero_lr_warmup_steps > 0 and step <= self.zero_lr_warmup_steps:
+            lr = 0
+        elif self.warmup_steps > 0 and step <= self.warmup_steps + self.zero_lr_warmup_steps:
+            lr_ratio = (step - self.zero_lr_warmup_steps) / float(self.warmup_steps)
+            lr = lr_ratio * lr
+        elif step >= self.total_steps:
+            lr = self.end_lr
+        else:
+            total_warmup_steps = self.warmup_steps + self.zero_lr_warmup_steps
+            lr_range = lr - self.end_lr
+            pct_remaining = 1 - (step - total_warmup_steps) / (self.total_steps - total_warmup_steps)
+            lr = lr_range * pct_remaining ** self.power + self.end_lr
+        return lr
+
+    def get_lr(self):
+        return [self._get_sched_lr(base_lr, self.last_epoch) for base_lr in self.base_lrs]
diff --git a/audiocraft/audiocraft/py.typed b/audiocraft/audiocraft/py.typed
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/audiocraft/audiocraft/quantization/__init__.py b/audiocraft/audiocraft/quantization/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e0c7e429ab96d67be667e23bf7a0ffa389c036b
--- /dev/null
+++ b/audiocraft/audiocraft/quantization/__init__.py
@@ -0,0 +1,9 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""RVQ."""
+# flake8: noqa
+from .vq import ResidualVectorQuantizer
+from .base import BaseQuantizer, DummyQuantizer, QuantizedResult
diff --git a/audiocraft/audiocraft/quantization/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/quantization/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3b500a65692c06df8dff32b951dbb73a634c296e
Binary files /dev/null and b/audiocraft/audiocraft/quantization/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/quantization/__pycache__/base.cpython-311.pyc b/audiocraft/audiocraft/quantization/__pycache__/base.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2874f6d0c656d2eacf7d4d2115ae612114bfc167
Binary files /dev/null and b/audiocraft/audiocraft/quantization/__pycache__/base.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/quantization/__pycache__/core_vq.cpython-311.pyc b/audiocraft/audiocraft/quantization/__pycache__/core_vq.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..086aaecbbebbd0d2ab05046e77f7e933023f5b76
Binary files /dev/null and b/audiocraft/audiocraft/quantization/__pycache__/core_vq.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/quantization/__pycache__/vq.cpython-311.pyc b/audiocraft/audiocraft/quantization/__pycache__/vq.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dc8eeeb7fc1b12b54d614b49bb309b85e7de640e
Binary files /dev/null and b/audiocraft/audiocraft/quantization/__pycache__/vq.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/quantization/base.py b/audiocraft/audiocraft/quantization/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..a77fefb98e62a5bbc6385910261ffdde2ffa5a25
--- /dev/null
+++ b/audiocraft/audiocraft/quantization/base.py
@@ -0,0 +1,99 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Base class for all quantizers.
+"""
+
+from dataclasses import dataclass, field
+import typing as tp
+
+import torch
+from torch import nn
+
+
+@dataclass
+class QuantizedResult:
+    x: torch.Tensor
+    codes: torch.Tensor
+    bandwidth: torch.Tensor  # bandwidth in kb/s used, per batch item.
+    penalty: tp.Optional[torch.Tensor] = None
+    metrics: dict = field(default_factory=dict)
+
+
+class BaseQuantizer(nn.Module):
+    """Base class for quantizers.
+    """
+
+    def forward(self, x: torch.Tensor, frame_rate: int) -> QuantizedResult:
+        """
+        Given input tensor x, returns first the quantized (or approximately quantized)
+        representation along with quantized codes, bandwidth, and any penalty term for the loss.
+        Finally, this returns a dict of metrics to update logging etc.
+        Frame rate must be passed so that the bandwidth is properly computed.
+        """
+        raise NotImplementedError()
+
+    def encode(self, x: torch.Tensor) -> torch.Tensor:
+        """Encode a given input tensor with the specified sample rate at the given bandwidth."""
+        raise NotImplementedError()
+
+    def decode(self, codes: torch.Tensor) -> torch.Tensor:
+        """Decode the given codes to the quantized representation."""
+        raise NotImplementedError()
+
+    @property
+    def total_codebooks(self):
+        """Total number of codebooks."""
+        raise NotImplementedError()
+
+    @property
+    def num_codebooks(self):
+        """Number of active codebooks."""
+        raise NotImplementedError()
+
+    def set_num_codebooks(self, n: int):
+        """Set the number of active codebooks."""
+        raise NotImplementedError()
+
+
+class DummyQuantizer(BaseQuantizer):
+    """Fake quantizer that actually does not perform any quantization.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor, frame_rate: int):
+        q = x.unsqueeze(1)
+        return QuantizedResult(x, q, torch.tensor(q.numel() * 32 * frame_rate / 1000 / len(x)).to(x))
+
+    def encode(self, x: torch.Tensor) -> torch.Tensor:
+        """Encode a given input tensor with the specified sample rate at the given bandwidth.
+        In the case of the DummyQuantizer, the codes are actually identical
+        to the input and resulting quantized representation as no quantization is done.
+        """
+        return x.unsqueeze(1)
+
+    def decode(self, codes: torch.Tensor) -> torch.Tensor:
+        """Decode the given codes to the quantized representation.
+        In the case of the DummyQuantizer, the codes are actually identical
+        to the input and resulting quantized representation as no quantization is done.
+        """
+        return codes.squeeze(1)
+
+    @property
+    def total_codebooks(self):
+        """Total number of codebooks."""
+        return 1
+
+    @property
+    def num_codebooks(self):
+        """Total number of codebooks."""
+        return self.total_codebooks
+
+    def set_num_codebooks(self, n: int):
+        """Set the number of active codebooks."""
+        raise AttributeError("Cannot override the number of codebooks for the dummy quantizer")
diff --git a/audiocraft/audiocraft/quantization/core_vq.py b/audiocraft/audiocraft/quantization/core_vq.py
new file mode 100644
index 0000000000000000000000000000000000000000..da02a6ce3a7de15353f0fba9e826052beb67c436
--- /dev/null
+++ b/audiocraft/audiocraft/quantization/core_vq.py
@@ -0,0 +1,400 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+from einops import rearrange, repeat
+import flashy
+import torch
+from torch import nn, einsum
+import torch.nn.functional as F
+
+
+def exists(val: tp.Optional[tp.Any]) -> bool:
+    return val is not None
+
+
+def default(val: tp.Any, d: tp.Any) -> tp.Any:
+    return val if exists(val) else d
+
+
+def l2norm(t):
+    return F.normalize(t, p=2, dim=-1)
+
+
+def ema_inplace(moving_avg, new, decay: float):
+    moving_avg.data.mul_(decay).add_(new, alpha=(1 - decay))
+
+
+def laplace_smoothing(x, n_categories: int, epsilon: float = 1e-5):
+    return (x + epsilon) / (x.sum() + n_categories * epsilon)
+
+
+def uniform_init(*shape: int):
+    t = torch.empty(shape)
+    nn.init.kaiming_uniform_(t)
+    return t
+
+
+def sample_vectors(samples, num: int):
+    num_samples, device = samples.shape[0], samples.device
+
+    if num_samples >= num:
+        indices = torch.randperm(num_samples, device=device)[:num]
+    else:
+        indices = torch.randint(0, num_samples, (num,), device=device)
+
+    return samples[indices]
+
+
+def kmeans(samples, num_clusters: int, num_iters: int = 10):
+    dim, dtype = samples.shape[-1], samples.dtype
+
+    means = sample_vectors(samples, num_clusters)
+
+    for _ in range(num_iters):
+        diffs = rearrange(samples, "n d -> n () d") - rearrange(
+            means, "c d -> () c d"
+        )
+        dists = -(diffs ** 2).sum(dim=-1)
+
+        buckets = dists.max(dim=-1).indices
+        bins = torch.bincount(buckets, minlength=num_clusters)
+        zero_mask = bins == 0
+        bins_min_clamped = bins.masked_fill(zero_mask, 1)
+
+        new_means = buckets.new_zeros(num_clusters, dim, dtype=dtype)
+        new_means.scatter_add_(0, repeat(buckets, "n -> n d", d=dim), samples)
+        new_means = new_means / bins_min_clamped[..., None]
+
+        means = torch.where(zero_mask[..., None], means, new_means)
+
+    return means, bins
+
+
+def orthogonal_loss_fn(t):
+    # eq (2) from https://arxiv.org/abs/2112.00384
+    n = t.shape[0]
+    normed_codes = l2norm(t)
+    identity = torch.eye(n, device=t.device)
+    cosine_sim = einsum("i d, j d -> i j", normed_codes, normed_codes)
+    return ((cosine_sim - identity) ** 2).sum() / (n ** 2)
+
+
+class EuclideanCodebook(nn.Module):
+    """Codebook with Euclidean distance.
+
+    Args:
+        dim (int): Dimension.
+        codebook_size (int): Codebook size.
+        kmeans_init (bool): Whether to use k-means to initialize the codebooks.
+            If set to true, run the k-means algorithm on the first training batch and use
+            the learned centroids as initialization.
+        kmeans_iters (int): Number of iterations used for k-means algorithm at initialization.
+        decay (float): Decay for exponential moving average over the codebooks.
+        epsilon (float): Epsilon value for numerical stability.
+        threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
+            that have an exponential moving average cluster size less than the specified threshold with
+            randomly selected vector from the current batch.
+    """
+    def __init__(
+        self,
+        dim: int,
+        codebook_size: int,
+        kmeans_init: int = False,
+        kmeans_iters: int = 10,
+        decay: float = 0.8,
+        epsilon: float = 1e-5,
+        threshold_ema_dead_code: int = 2,
+    ):
+        super().__init__()
+        self.decay = decay
+        init_fn: tp.Union[tp.Callable[..., torch.Tensor], tp.Any] = uniform_init if not kmeans_init else torch.zeros
+        embed = init_fn(codebook_size, dim)
+
+        self.codebook_size = codebook_size
+
+        self.kmeans_iters = kmeans_iters
+        self.epsilon = epsilon
+        self.threshold_ema_dead_code = threshold_ema_dead_code
+
+        self.register_buffer("inited", torch.Tensor([not kmeans_init]))
+        self.register_buffer("cluster_size", torch.zeros(codebook_size))
+        self.register_buffer("embed", embed)
+        self.register_buffer("embed_avg", embed.clone())
+
+    @torch.jit.ignore
+    def init_embed_(self, data):
+        if self.inited:
+            return
+
+        embed, cluster_size = kmeans(data, self.codebook_size, self.kmeans_iters)
+        self.embed.data.copy_(embed)
+        self.embed_avg.data.copy_(embed.clone())
+        self.cluster_size.data.copy_(cluster_size)
+        self.inited.data.copy_(torch.Tensor([True]))
+        # Make sure all buffers across workers are in sync after initialization
+        flashy.distrib.broadcast_tensors(self.buffers())
+
+    def replace_(self, samples, mask):
+        modified_codebook = torch.where(
+            mask[..., None], sample_vectors(samples, self.codebook_size), self.embed
+        )
+        self.embed.data.copy_(modified_codebook)
+
+    def expire_codes_(self, batch_samples):
+        if self.threshold_ema_dead_code == 0:
+            return
+
+        expired_codes = self.cluster_size < self.threshold_ema_dead_code
+        if not torch.any(expired_codes):
+            return
+
+        batch_samples = rearrange(batch_samples, "... d -> (...) d")
+        self.replace_(batch_samples, mask=expired_codes)
+        flashy.distrib.broadcast_tensors(self.buffers())
+
+    def preprocess(self, x):
+        x = rearrange(x, "... d -> (...) d")
+        return x
+
+    def quantize(self, x):
+        embed = self.embed.t()
+        dist = -(
+            x.pow(2).sum(1, keepdim=True)
+            - 2 * x @ embed
+            + embed.pow(2).sum(0, keepdim=True)
+        )
+        embed_ind = dist.max(dim=-1).indices
+        return embed_ind
+
+    def postprocess_emb(self, embed_ind, shape):
+        return embed_ind.view(*shape[:-1])
+
+    def dequantize(self, embed_ind):
+        quantize = F.embedding(embed_ind, self.embed)
+        return quantize
+
+    def encode(self, x):
+        shape = x.shape
+        # pre-process
+        x = self.preprocess(x)
+        # quantize
+        embed_ind = self.quantize(x)
+        # post-process
+        embed_ind = self.postprocess_emb(embed_ind, shape)
+        return embed_ind
+
+    def decode(self, embed_ind):
+        quantize = self.dequantize(embed_ind)
+        return quantize
+
+    def forward(self, x):
+        shape, dtype = x.shape, x.dtype
+        x = self.preprocess(x)
+        self.init_embed_(x)
+
+        embed_ind = self.quantize(x)
+        embed_onehot = F.one_hot(embed_ind, self.codebook_size).type(dtype)
+        embed_ind = self.postprocess_emb(embed_ind, shape)
+        quantize = self.dequantize(embed_ind)
+
+        if self.training:
+            # We do the expiry of code at that point as buffers are in sync
+            # and all the workers will take the same decision.
+            self.expire_codes_(x)
+            ema_inplace(self.cluster_size, embed_onehot.sum(0), self.decay)
+            embed_sum = x.t() @ embed_onehot
+            ema_inplace(self.embed_avg, embed_sum.t(), self.decay)
+            cluster_size = (
+                laplace_smoothing(self.cluster_size, self.codebook_size, self.epsilon)
+                * self.cluster_size.sum()
+            )
+            embed_normalized = self.embed_avg / cluster_size.unsqueeze(1)
+            self.embed.data.copy_(embed_normalized)
+
+        return quantize, embed_ind
+
+
+class VectorQuantization(nn.Module):
+    """Vector quantization implementation.
+    Currently supports only euclidean distance.
+
+    Args:
+        dim (int): Dimension
+        codebook_size (int): Codebook size
+        codebook_dim (int): Codebook dimension. If not defined, uses the specified dimension in dim.
+        decay (float): Decay for exponential moving average over the codebooks.
+        epsilon (float): Epsilon value for numerical stability.
+        kmeans_init (bool): Whether to use kmeans to initialize the codebooks.
+        kmeans_iters (int): Number of iterations used for kmeans initialization.
+        threshold_ema_dead_code (int):
+        channels_last (bool): Channels are the last dimension in the input tensors.
+        commitment_weight (float): Weight for commitment loss.
+        orthogonal_reg_weight (float): Orthogonal regularization weights.
+        orthogonal_reg_active_codes_only (bool): Apply orthogonal regularization only on active codes.
+        orthogonal_reg_max_codes (optional int): Maximum number of codes to consider
+            for orthogonal regularization.
+        threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
+            that have an exponential moving average cluster size less than the specified threshold with
+            randomly selected vector from the current batch.
+    """
+    def __init__(
+        self,
+        dim: int,
+        codebook_size: int,
+        codebook_dim: tp.Optional[int] = None,
+        decay: float = 0.8,
+        epsilon: float = 1e-5,
+        kmeans_init: bool = False,
+        kmeans_iters: int = 10,
+        threshold_ema_dead_code: int = 2,
+        channels_last: bool = False,
+        commitment_weight: float = 1.,
+        orthogonal_reg_weight: float = 0.0,
+        orthogonal_reg_active_codes_only: bool = False,
+        orthogonal_reg_max_codes: tp.Optional[int] = None,
+    ):
+        super().__init__()
+        _codebook_dim: int = default(codebook_dim, dim)
+
+        requires_projection = _codebook_dim != dim
+        self.project_in = (nn.Linear(dim, _codebook_dim) if requires_projection else nn.Identity())
+        self.project_out = (nn.Linear(_codebook_dim, dim) if requires_projection else nn.Identity())
+
+        self.epsilon = epsilon
+        self.commitment_weight = commitment_weight
+
+        self.orthogonal_reg_weight = orthogonal_reg_weight
+        self.orthogonal_reg_active_codes_only = orthogonal_reg_active_codes_only
+        self.orthogonal_reg_max_codes = orthogonal_reg_max_codes
+
+        self._codebook = EuclideanCodebook(dim=_codebook_dim, codebook_size=codebook_size,
+                                           kmeans_init=kmeans_init, kmeans_iters=kmeans_iters,
+                                           decay=decay, epsilon=epsilon,
+                                           threshold_ema_dead_code=threshold_ema_dead_code)
+        self.codebook_size = codebook_size
+
+        self.channels_last = channels_last
+
+    @property
+    def codebook(self):
+        return self._codebook.embed
+
+    @property
+    def inited(self):
+        return self._codebook.inited
+
+    def _preprocess(self, x):
+        if not self.channels_last:
+            x = rearrange(x, "b d n -> b n d")
+        return x
+
+    def _postprocess(self, quantize):
+        if not self.channels_last:
+            quantize = rearrange(quantize, "b n d -> b d n")
+        return quantize
+
+    def encode(self, x):
+        x = self._preprocess(x)
+        x = self.project_in(x)
+        embed_in = self._codebook.encode(x)
+        return embed_in
+
+    def decode(self, embed_ind):
+        quantize = self._codebook.decode(embed_ind)
+        quantize = self.project_out(quantize)
+        quantize = self._postprocess(quantize)
+        return quantize
+
+    def forward(self, x):
+        device = x.device
+        x = self._preprocess(x)
+
+        x = self.project_in(x)
+        quantize, embed_ind = self._codebook(x)
+
+        if self.training:
+            quantize = x + (quantize - x).detach()
+
+        loss = torch.tensor([0.0], device=device, requires_grad=self.training)
+
+        if self.training:
+            if self.commitment_weight > 0:
+                commit_loss = F.mse_loss(quantize.detach(), x)
+                loss = loss + commit_loss * self.commitment_weight
+
+            if self.orthogonal_reg_weight > 0:
+                codebook = self.codebook
+
+                if self.orthogonal_reg_active_codes_only:
+                    # only calculate orthogonal loss for the activated codes for this batch
+                    unique_code_ids = torch.unique(embed_ind)
+                    codebook = codebook[unique_code_ids]
+
+                num_codes = codebook.shape[0]
+                if exists(self.orthogonal_reg_max_codes) and num_codes > self.orthogonal_reg_max_codes:
+                    rand_ids = torch.randperm(num_codes, device=device)[:self.orthogonal_reg_max_codes]
+                    codebook = codebook[rand_ids]
+
+                orthogonal_reg_loss = orthogonal_loss_fn(codebook)
+                loss = loss + orthogonal_reg_loss * self.orthogonal_reg_weight
+
+        quantize = self.project_out(quantize)
+        quantize = self._postprocess(quantize)
+
+        return quantize, embed_ind, loss
+
+
+class ResidualVectorQuantization(nn.Module):
+    """Residual vector quantization implementation.
+
+    Follows Algorithm 1. in https://arxiv.org/pdf/2107.03312.pdf
+    """
+    def __init__(self, *, num_quantizers, **kwargs):
+        super().__init__()
+        self.layers = nn.ModuleList(
+            [VectorQuantization(**kwargs) for _ in range(num_quantizers)]
+        )
+
+    def forward(self, x, n_q: tp.Optional[int] = None):
+        quantized_out = 0.0
+        residual = x
+
+        all_losses = []
+        all_indices = []
+
+        n_q = n_q or len(self.layers)
+
+        for i, layer in enumerate(self.layers[:n_q]):
+            quantized, indices, loss = layer(residual)
+            residual = residual - quantized
+            quantized_out = quantized_out + quantized
+            all_indices.append(indices)
+            all_losses.append(loss)
+
+        out_losses, out_indices = map(torch.stack, (all_losses, all_indices))
+        return quantized_out, out_indices, out_losses
+
+    def encode(self, x: torch.Tensor, n_q: tp.Optional[int] = None) -> torch.Tensor:
+        residual = x
+        all_indices = []
+        n_q = n_q or len(self.layers)
+        for layer in self.layers[:n_q]:
+            indices = layer.encode(residual)
+            quantized = layer.decode(indices)
+            residual = residual - quantized
+            all_indices.append(indices)
+        out_indices = torch.stack(all_indices)
+        return out_indices
+
+    def decode(self, q_indices: torch.Tensor) -> torch.Tensor:
+        quantized_out = torch.tensor(0.0, device=q_indices.device)
+        for i, indices in enumerate(q_indices):
+            layer = self.layers[i]
+            quantized = layer.decode(indices)
+            quantized_out = quantized_out + quantized
+        return quantized_out
diff --git a/audiocraft/audiocraft/quantization/vq.py b/audiocraft/audiocraft/quantization/vq.py
new file mode 100644
index 0000000000000000000000000000000000000000..aa57bea59db95ddae35e0657f723ca3a29ee943b
--- /dev/null
+++ b/audiocraft/audiocraft/quantization/vq.py
@@ -0,0 +1,115 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import math
+import typing as tp
+
+import torch
+
+from .base import BaseQuantizer, QuantizedResult
+from .core_vq import ResidualVectorQuantization
+
+
+class ResidualVectorQuantizer(BaseQuantizer):
+    """Residual Vector Quantizer.
+
+    Args:
+        dimension (int): Dimension of the codebooks.
+        n_q (int): Number of residual vector quantizers used.
+        q_dropout (bool): Random quantizer drop out at train time.
+        bins (int): Codebook size.
+        decay (float): Decay for exponential moving average over the codebooks.
+        kmeans_init (bool): Whether to use kmeans to initialize the codebooks.
+        kmeans_iters (int): Number of iterations used for kmeans initialization.
+        threshold_ema_dead_code (int): Threshold for dead code expiration. Replace any codes
+            that have an exponential moving average cluster size less than the specified threshold with
+            randomly selected vector from the current batch.
+        orthogonal_reg_weight (float): Orthogonal regularization weights.
+        orthogonal_reg_active_codes_only (bool): Apply orthogonal regularization only on active codes.
+        orthogonal_reg_max_codes (optional int): Maximum number of codes to consider.
+            for orthogonal regularization.
+    """
+    def __init__(
+        self,
+        dimension: int = 256,
+        n_q: int = 8,
+        q_dropout: bool = False,
+        bins: int = 1024,
+        decay: float = 0.99,
+        kmeans_init: bool = True,
+        kmeans_iters: int = 10,
+        threshold_ema_dead_code: int = 2,
+        orthogonal_reg_weight: float = 0.0,
+        orthogonal_reg_active_codes_only: bool = False,
+        orthogonal_reg_max_codes: tp.Optional[int] = None,
+    ):
+        super().__init__()
+        self.max_n_q = n_q
+        self.n_q = n_q
+        self.q_dropout = q_dropout
+        self.dimension = dimension
+        self.bins = bins
+        self.decay = decay
+        self.kmeans_init = kmeans_init
+        self.kmeans_iters = kmeans_iters
+        self.threshold_ema_dead_code = threshold_ema_dead_code
+        self.orthogonal_reg_weight = orthogonal_reg_weight
+        self.orthogonal_reg_active_codes_only = orthogonal_reg_active_codes_only
+        self.orthogonal_reg_max_codes = orthogonal_reg_max_codes
+        self.vq = ResidualVectorQuantization(
+            dim=self.dimension,
+            codebook_size=self.bins,
+            num_quantizers=self.n_q,
+            decay=self.decay,
+            kmeans_init=self.kmeans_init,
+            kmeans_iters=self.kmeans_iters,
+            threshold_ema_dead_code=self.threshold_ema_dead_code,
+            orthogonal_reg_weight=self.orthogonal_reg_weight,
+            orthogonal_reg_active_codes_only=self.orthogonal_reg_active_codes_only,
+            orthogonal_reg_max_codes=self.orthogonal_reg_max_codes,
+            channels_last=False
+        )
+
+    def forward(self, x: torch.Tensor, frame_rate: int):
+        n_q = self.n_q
+        if self.training and self.q_dropout:
+            n_q = int(torch.randint(1, self.n_q + 1, (1,)).item())
+        bw_per_q = math.log2(self.bins) * frame_rate / 1000
+        quantized, codes, commit_loss = self.vq(x, n_q=n_q)
+        codes = codes.transpose(0, 1)
+        # codes is [B, K, T], with T frames, K nb of codebooks.
+        bw = torch.tensor(n_q * bw_per_q).to(x)
+        return QuantizedResult(quantized, codes, bw, penalty=torch.mean(commit_loss))
+
+    def encode(self, x: torch.Tensor) -> torch.Tensor:
+        """Encode a given input tensor with the specified frame rate at the given bandwidth.
+        The RVQ encode method sets the appropriate number of quantizer to use
+        and returns indices for each quantizer.
+        """
+        n_q = self.n_q
+        codes = self.vq.encode(x, n_q=n_q)
+        codes = codes.transpose(0, 1)
+        # codes is [B, K, T], with T frames, K nb of codebooks.
+        return codes
+
+    def decode(self, codes: torch.Tensor) -> torch.Tensor:
+        """Decode the given codes to the quantized representation."""
+        # codes is [B, K, T], with T frames, K nb of codebooks, vq.decode expects [K, B, T].
+        codes = codes.transpose(0, 1)
+        quantized = self.vq.decode(codes)
+        return quantized
+
+    @property
+    def total_codebooks(self):
+        return self.max_n_q
+
+    @property
+    def num_codebooks(self):
+        return self.n_q
+
+    def set_num_codebooks(self, n: int):
+        assert n > 0 and n <= self.max_n_q
+        self.n_q = n
diff --git a/audiocraft/audiocraft/solvers/__init__.py b/audiocraft/audiocraft/solvers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae19f3a8c51abf469697d6affa91449d668716ba
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""
+Solvers. A Solver is a training recipe, combining the dataloaders, models,
+optimizer, losses etc into a single convenient object.
+"""
+
+# flake8: noqa
+from .audiogen import AudioGenSolver
+from .builders import get_solver
+from .base import StandardSolver
+from .compression import CompressionSolver
+from .musicgen import MusicGenSolver
+from .diffusion import DiffusionSolver
diff --git a/audiocraft/audiocraft/solvers/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..7a729df3a624d38e1f88090914cbfc5b298f0828
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/__pycache__/audiogen.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/audiogen.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..000c01bc8d7cd1a46692aca0e49a926e93f79fed
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/audiogen.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/__pycache__/base.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/base.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e9da2965e67be0fb9b878aa29b5a940147201bb7
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/base.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/__pycache__/builders.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/builders.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2da29792bcd01b33ccac5755c30b392040948904
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/builders.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/__pycache__/compression.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/compression.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..b8fed1c7b2d243a2d3edefa92f6d8071509b8c28
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/compression.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/__pycache__/diffusion.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/diffusion.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..f5ced6280c572c4fa197db693d472e709244aae8
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/diffusion.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/__pycache__/musicgen.cpython-311.pyc b/audiocraft/audiocraft/solvers/__pycache__/musicgen.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e0bf6167640481c3554a3f4740690147d8ad2bc6
Binary files /dev/null and b/audiocraft/audiocraft/solvers/__pycache__/musicgen.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/solvers/audiogen.py b/audiocraft/audiocraft/solvers/audiogen.py
new file mode 100644
index 0000000000000000000000000000000000000000..1568f97fe7b84b90c7ef760ef5606fe0a475545a
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/audiogen.py
@@ -0,0 +1,19 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from . import builders, musicgen
+
+
+class AudioGenSolver(musicgen.MusicGenSolver):
+    """Solver for AudioGen re-implementation training task.
+
+    Note that this implementation does not strictly follows
+    the method proposed in https://arxiv.org/abs/2209.15352
+    but is derived from MusicGen's training pipeline.
+
+    More information can be found in the AudioGen model card.
+    """
+    DATASET_TYPE: builders.DatasetType = builders.DatasetType.SOUND
diff --git a/audiocraft/audiocraft/solvers/base.py b/audiocraft/audiocraft/solvers/base.py
new file mode 100644
index 0000000000000000000000000000000000000000..0432e44a36838c5731711f9d54f81822b21f20bd
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/base.py
@@ -0,0 +1,631 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from pathlib import Path
+import typing as tp
+
+import flashy
+import omegaconf
+import torch
+from torch import nn
+
+from .. import optim
+from ..optim import fsdp
+from ..utils import checkpoint
+from ..utils.autocast import TorchAutocast
+from ..utils.best_state import BestStateDictManager
+from ..utils.deadlock import DeadlockDetect
+from ..utils.profiler import Profiler
+from ..utils.utils import copy_state, dict_from_config, model_hash, with_rank_rng
+
+
+class StandardSolver(ABC, flashy.BaseSolver):
+    """Standard solver for AudioCraft.
+
+    The standard solver implements a base training loop with the following stages:
+    train, valid, evaluate and generate that are expected to be all defined for
+    solvers in AudioCraft. It also provides a nice default management of Dora history replay,
+    checkpoint management across epoch, and logging configuration.
+
+    AudioCraft solvers must inherit from the StandardSolver and define the methods
+    associated to each stage as well as the show, build_model and build_dataloaders methods.
+    """
+    def __init__(self, cfg: omegaconf.DictConfig):
+        super().__init__()
+        self.logger.info(f"Instantiating solver {self.__class__.__name__} for XP {self.xp.sig}")
+        self.logger.info(f"All XP logs are stored in {self.xp.folder}")
+        self.cfg = cfg
+        self.device = cfg.device
+        self.model: nn.Module
+        self._continue_best_source_keys = ['best_state', 'fsdp_best_state']
+        self._fsdp_modules: tp.List[fsdp.FSDP] = []
+        self._ema_sources: nn.ModuleDict = nn.ModuleDict()
+        self.ema: tp.Optional[optim.ModuleDictEMA] = None
+        self.dataloaders: tp.Dict[str, torch.utils.data.DataLoader] = dict()
+        self._log_updates = self.cfg.logging.get('log_updates', 10)
+        if self.cfg.logging.log_tensorboard:
+            self.init_tensorboard(**self.cfg.get('tensorboard'))
+        if self.cfg.logging.log_wandb and self:
+            self.init_wandb(**self.cfg.get('wandb'))
+        # keep a copy of the best performing state for stateful objects
+        # used for evaluation and generation stages
+        dtype_best: tp.Optional[torch.dtype] = None
+        if self.cfg.fsdp.use:
+            dtype_best = getattr(torch, self.cfg.fsdp.param_dtype)  # type: ignore
+            assert isinstance(dtype_best, torch.dtype)
+        elif self.cfg.autocast:
+            dtype_best = getattr(torch, self.cfg.autocast_dtype)  # type: ignore
+            assert isinstance(dtype_best, torch.dtype)
+        self.best_state: BestStateDictManager = BestStateDictManager(dtype=dtype_best)
+        # Hacky support for keeping a copy of the full best state in rank0.
+        self.fsdp_best_state: tp.Dict[str, tp.Any] = {}
+        self.register_stateful('best_state', 'fsdp_best_state')  # register best_state object to keep it in state_dict
+        self._new_best_state: bool = False  # should save a new checkpoint
+        # instantiate datasets and appropriate number of updates per epoch
+        self.build_dataloaders()
+        if self.cfg.execute_only is None:
+            assert 'train' in self.dataloaders, "The train dataset split must be provided."
+            assert 'valid' in self.dataloaders, "The valid dataset split must be provided."
+        self.train_updates_per_epoch = len(self.dataloaders['train']) if 'train' in self.dataloaders else 0
+        if self.cfg.optim.updates_per_epoch:
+            self.train_updates_per_epoch = self.cfg.optim.updates_per_epoch
+        self.total_updates = self.train_updates_per_epoch * self.cfg.optim.epochs
+        # instantiate model & exponential moving average on the model
+        self.build_model()
+        self.logger.info("Model hash: %s", model_hash(self.model))
+        assert 'model' in self.stateful.sources, \
+            "Please register the model to stateful with self.register_stateful('model') in build_model."
+        self.profiler = Profiler(self.model, **self.cfg.profiler)
+        self.initialize_ema()
+        self.register_stateful('ema')
+        assert self.ema is None or 'ema' in self.stateful.sources, \
+            "Please register the ema to stateful with self.register_stateful('ema') in build_model."
+        self.deadlock_detect = DeadlockDetect(**self.cfg.deadlock)
+        # basic statistics on the trained model
+        model_size = sum(p.numel() for p in self.model.parameters() if p.requires_grad) / 1e6
+        # one copy of grad, one copy of momentum, one copy of denominator and model weights.
+        # and 4 bytes for each float!
+        mem_usage = model_size * 4 * 4 / 1000
+        self.logger.info("Model size: %.2f M params", model_size)
+        self.logger.info("Base memory usage, with model, grad and optim: %.2f GB", mem_usage)
+
+    @property
+    def autocast(self):
+        """Convenient autocast (or not) using the solver configuration."""
+        return TorchAutocast(enabled=self.cfg.autocast, device_type=self.device, dtype=self.autocast_dtype)
+
+    def _get_state_source(self, name) -> flashy.state.StateDictSource:
+        # Internal utility to get a state source from the solver
+        return self.stateful.sources[name]
+
+    @property
+    def best_metric_name(self) -> tp.Optional[str]:
+        """Metric name used to identify the best state. This metric should be stored in the metrics
+        used on the stage for best state identification (most likely, `valid`). If None, then
+        no best state is saved.
+        """
+        return None
+
+    def register_best_state(self, *args: str):
+        """Register state sources in `BestStateDictManager` to keep their best states along with their
+        latest states. The best state will be used at evaluation stages instead of the latest states.
+
+        Shortcut around `BestStateDictManager.register` method. You can pass any number of
+        attribute, included nested attributes and those will be included into the checkpoints
+        and automatically restored when `BaseSolver.restore` is called.
+        """
+        for name in args:
+            state_source = self._get_state_source(name)
+            assert name in self.stateful.sources, "Registered states in best should be registered in stateful first!"
+            self.best_state.register(name, state_source)
+
+    def register_ema(self, *args: str):
+        """Register state sources for exponential moving average.
+
+        The registered sources are used to instantiate a ModuleDictEMA instance.
+        The ModuleDictEMA keeps a `nn.ModuleDict` module that is updated when self.ema.step() is called
+        and swapped with the original state sources with self.swap_ema_state() method.
+
+        Usage:
+            self.register_ema('model')
+        """
+        assert self.ema is None, "Cannot register state source to already instantiated EMA."
+        for name in args:
+            self._ema_sources[name] = getattr(self, name)
+
+    def wrap_with_fsdp(self, model: torch.nn.Module, *args, **kwargs):
+        model = fsdp.wrap_with_fsdp(self.cfg.fsdp, model, *args, **kwargs)
+        if isinstance(model, fsdp.FSDP):
+            self._fsdp_modules.append(model)
+        return model
+
+    def update_best_state_from_stage(self, stage_name: str = 'valid'):
+        """Update latest best state based on pending metrics of a given stage. This method relies
+        on the `BestStateDictManager.update` method to update the best state_dict with latest weights
+        if the registered states happen to match to the best performing setup.
+        """
+        if self.best_metric_name is None:
+            # when no best metric is defined, the last state is always the best
+            self._new_best_state = True
+            self.logger.info("Updating best state with current state.")
+        else:
+            assert stage_name in self._pending_metrics, f"Metrics for stage {stage_name} not found."
+            assert self.best_metric_name in self._pending_metrics[stage_name], \
+                f"Best metric not found in {stage_name} metrics. Cannot register best state"
+            current_score = self._pending_metrics[stage_name][self.best_metric_name]
+            all_best_metric_scores = [
+                past_metrics[stage_name][self.best_metric_name]
+                for past_metrics in self.history
+            ]
+            all_best_metric_scores.append(current_score)
+            best_score = min(all_best_metric_scores)
+            self._new_best_state = current_score == best_score
+            if self._new_best_state:
+                old_best = min(all_best_metric_scores[:-1] + [float('inf')])
+                self.logger.info(
+                    f"New best state with {self.best_metric_name}={current_score:.3f} (was {old_best:.3f})")
+
+        if self._new_best_state:
+            if self.cfg.fsdp.use:
+                # this will give an empty state dict on all ranks but the rank 0
+                # which will have a copy in memory of the full model.
+                with fsdp.switch_to_full_state_dict(self._fsdp_modules):
+                    for name in self.best_state.states.keys():
+                        state_source = self._get_state_source(name)
+                        self.best_state.update(name, state_source)
+                    # we save to a different dict.
+                    self.fsdp_best_state.update(self.best_state.state_dict())
+                # We cannot efficiently load fsdp_best_state when using FSDP,
+                # so we have do do a second pass, with the local shards.
+            for name in self.best_state.states.keys():
+                state_source = self._get_state_source(name)
+                self.best_state.update(name, state_source)
+
+    def _load_new_state_dict(self, state_dict: dict) -> dict:
+        old_states = {}
+        for name, new_state in state_dict.items():
+            state_source = self._get_state_source(name)
+            old_states[name] = copy_state(state_source.state_dict())
+            state_source.load_state_dict(new_state)
+        return old_states
+
+    @contextmanager
+    def swap_best_state(self):
+        self.logger.debug(f"Swapping to best state for: {', '.join(self.best_state.state_dict().keys())}")
+        old_states = self._load_new_state_dict(self.best_state.state_dict())
+        try:
+            yield
+        finally:
+            self.logger.debug("Swapping back from best to original state")
+            for name, old_state in old_states.items():
+                state_source = self._get_state_source(name)
+                state_source.load_state_dict(old_state)
+
+    @contextmanager
+    def swap_ema_state(self):
+        if self.ema is None:
+            yield
+        else:
+            ema_state_dict = self.ema.state_dict()['state']
+            self.logger.debug(f"Swapping to EMA state for: {', '.join(ema_state_dict.keys())}")
+            old_states = self._load_new_state_dict(ema_state_dict)
+            try:
+                yield
+            finally:
+                self.logger.debug("Swapping back from EMA state to original state")
+                for name, old_state in old_states.items():
+                    state_source = self._get_state_source(name)
+                    state_source.load_state_dict(old_state)
+
+    @property
+    def is_training(self):
+        return self.current_stage == 'train'
+
+    def log_model_summary(self, model: nn.Module):
+        """Log model summary, architecture and size of the model."""
+        self.logger.info(model)
+        mb = sum(p.numel() for p in model.parameters()) * 4 / 2 ** 20
+        self.logger.info("Size: %.1f MB", mb)
+
+    @abstractmethod
+    def build_model(self):
+        """Method to implement to initialize model."""
+        ...
+
+    def initialize_ema(self):
+        """Initialize exponential moving average with the registered sources.
+        EMA object is created if the optim.ema.model.decay value is non-null.
+        """
+        from .builders import get_ema
+        self.ema = get_ema(self._ema_sources, self.cfg.optim.ema)
+        if self.ema is None:
+            self.logger.info('No EMA on the model.')
+        else:
+            assert self.cfg.optim.ema.updates > 0
+            self.logger.info(
+                f'Initializing EMA on the model with decay = {self.ema.decay}'
+                f' every {self.cfg.optim.ema.updates} updates'
+            )
+
+    @abstractmethod
+    def build_dataloaders(self):
+        """Method to implement to initialize dataloaders."""
+        ...
+
+    @abstractmethod
+    def show(self):
+        """Method to log any information without running the job."""
+        ...
+
+    @property
+    def log_updates(self):
+        # convenient access to log updates
+        return self._log_updates
+
+    def checkpoint_path(self, **kwargs):
+        kwargs.setdefault('use_fsdp', self.cfg.fsdp.use)
+        return self.folder / checkpoint.checkpoint_name(**kwargs)
+
+    def epoch_checkpoint_path(self, epoch: int, **kwargs):
+        kwargs.setdefault('use_fsdp', self.cfg.fsdp.use)
+        return self.folder / checkpoint.checkpoint_name(str(epoch), **kwargs)
+
+    def checkpoint_path_with_name(self, name: str, **kwargs):
+        kwargs.setdefault('use_fsdp', self.cfg.fsdp.use)
+        return self.folder / checkpoint.checkpoint_name(name=name, **kwargs)
+
+    def save_checkpoints(self):
+        """Save checkpoint, optionally keeping a copy for a given epoch."""
+        is_sharded = self.cfg.fsdp.use
+        if not flashy.distrib.is_rank_zero() and not is_sharded:
+            return
+        self.logger.info("Model hash: %s", model_hash(self.model))
+        state = self.state_dict()
+        epoch = self.epoch - 1  # pushing metrics will increase the epoch in Flashy, so we do -1 here
+
+        # save minimal state_dict as new checkpoint every X epoch
+        if self.cfg.checkpoint.save_every:
+            if epoch % self.cfg.checkpoint.save_every == 0:
+                minimal_state = state
+                if self.cfg.checkpoint.keep_every_states is not None and len(self.cfg.checkpoint.keep_every_states) > 0:
+                    minimal_state = {
+                        name: source for name, source in state.items()
+                        if name in self.cfg.checkpoint.keep_every_states
+                    }
+                epoch_checkpoint_path = self.epoch_checkpoint_path(epoch)
+                checkpoint.save_checkpoint(minimal_state, epoch_checkpoint_path, is_sharded)
+
+        # save checkpoint as latest checkpoint
+        if self.cfg.checkpoint.save_last:
+            last_checkpoint_path = self.checkpoint_path()
+            checkpoint.save_checkpoint(state, last_checkpoint_path, is_sharded)
+
+        # flush any stale checkpoint to reduce disk footprint
+        checkpoint.flush_stale_checkpoints(self.checkpoint_path())
+
+    def load_from_pretrained(self, name: str) -> dict:
+        raise NotImplementedError("Solver does not provide a way to load pretrained models.")
+
+    def load_checkpoints(self, load_best: bool = False, ignore_state_keys: tp.List[str] = []) -> tp.Optional[dict]:
+        """Load last checkpoint or the one specified in continue_from.
+
+        Args:
+            load_best (bool): Whether to load from best state dict or not.
+                Best state dict is always used when not loading the current xp.
+            ignore_state_keys (list of str): List of sources to ignore when loading the state, e.g. `optimizer`.
+        Returns:
+            state (dict, optional): The loaded state dictionary.
+        """
+        # load checkpoints from xp folder or cfg.continue_from
+        is_sharded = self.cfg.fsdp.use
+        load_from_path: tp.Optional[Path] = None
+        checkpoint_source: tp.Optional[checkpoint.CheckpointSource] = None
+
+        if load_best:
+            self.logger.info("Trying to load state_dict from best state.")
+
+        state: tp.Optional[dict] = None
+        rank0_checkpoint_path = self.checkpoint_path(use_fsdp=False)
+        current_checkpoint_path = self.checkpoint_path()
+        _pretrained_prefix = '//pretrained/'
+        continue_pretrained = (self.cfg.continue_from or '').startswith(_pretrained_prefix)
+        if rank0_checkpoint_path.exists():
+            self.logger.info(f"Loading existing checkpoint: {current_checkpoint_path}")
+            load_from_path = current_checkpoint_path
+            checkpoint.check_sharded_checkpoint(current_checkpoint_path, rank0_checkpoint_path)
+            checkpoint_source = checkpoint.CheckpointSource.CURRENT_XP
+        elif self.cfg.continue_from and not continue_pretrained:
+            self.logger.info(f"Continuing from provided checkpoint: {self.cfg.continue_from}")
+            # we're always continuing from consolidated checkpoints: self.cfg.use_fsdp and not continue_best
+            load_from_path = checkpoint.resolve_checkpoint_path(self.cfg.continue_from, use_fsdp=False)
+            if load_from_path is None:
+                self.logger.error('Could not resolve the continue_from checkpoint %s', self.cfg.continue_from)
+                raise RuntimeError(f'Could not resolve continue_from checkpoint {self.cfg.continue_from}')
+            checkpoint_source = checkpoint.CheckpointSource.OTHER
+
+        if load_from_path is not None:
+            state = checkpoint.load_checkpoint(load_from_path, is_sharded)
+        elif continue_pretrained:
+            self.logger.info("Loading a pretrained model. Ignoring 'load_best' and 'ignore_state_keys' params.")
+            state = self.load_from_pretrained(self.cfg.continue_from[len(_pretrained_prefix):])
+            checkpoint_source = checkpoint.CheckpointSource.PRETRAINED
+            load_best = True
+
+        # checkpoints are not from the current xp, we only retrieve the best state
+        if checkpoint_source is not None and checkpoint_source != checkpoint.CheckpointSource.CURRENT_XP:
+            assert state is not None
+            self.logger.info("Checkpoint source is not the current xp: Load state_dict from best state.")
+            load_best = True
+            state = {key: state[key] for key in self._continue_best_source_keys if key in state}
+            # loaded checkpoints are FSDP checkpoints: we're reading the best state
+            # from FSDP and we drop the regular best_state
+            if 'fsdp_best_state' in state and state['fsdp_best_state']:
+                state.pop('best_state', None)
+                self.logger.info("... Loaded checkpoint has FSDP best state")
+            # FSDP is enabled in the solver, if the loaded checkpoints do not have FSDP support
+            # then we're initializing FSDP best state with the regular best state
+            elif self.cfg.fsdp.use:
+                if 'fsdp_best_state' not in state or not state['fsdp_best_state']:
+                    # we swap non-FSDP checkpoints best_state to FSDP-compatible best state
+                    state['fsdp_best_state'] = state.pop('best_state')
+                    self.logger.info("... Loaded checkpoint does not have FSDP best state. Use regular best state")
+
+        if state is not None:
+            if load_best:
+                self.logger.info("Ignoring keys when loading best %r", ignore_state_keys)
+                for key in set(ignore_state_keys):
+                    if key in state:
+                        state.pop(key)
+                has_best_state = 'best_state' in state or 'fsdp_best_state' in state
+                assert has_best_state, ("Trying to load best state but neither 'best_state'",
+                                        " or 'fsdp_best_state' found in checkpoints.")
+            self.load_state_dict(state)
+
+        # for FSDP, let's make extra sure nothing bad happened with out of sync
+        # checkpoints across workers.
+        epoch = float(self.epoch)
+        avg_epoch = flashy.distrib.average_metrics({'epoch': epoch})['epoch']
+        if avg_epoch != epoch:
+            raise RuntimeError(
+                f"Inconsistent loading of checkpoints happened, our epoch is {epoch} "
+                f"but average of epochs is {avg_epoch}, at least one gpu must have a "
+                "different epoch number.")
+
+        # on load_best, properly reinitialize state_dict, best states and ema
+        # otherwise we load from the current xp and don't alter anything
+        if load_best:
+            self.logger.info("Loading state_dict from best state.")
+            if not self.cfg.fsdp.use and self.fsdp_best_state:
+                # loading from an FSDP checkpoint but with FSDP deactivated
+                self.logger.info("... Loading from FSDP best state dict.")
+                self.best_state.load_state_dict(self.fsdp_best_state)
+
+            # if load_best, we permanently override the regular state_dict with the best state
+            if self.cfg.fsdp.use:
+                self.logger.info("FSDP is used, loading from FSDP best state.")
+                with fsdp.switch_to_full_state_dict(self._fsdp_modules):
+                    # this might be really fragile but okay for now.
+                    self.load_state_dict(self.fsdp_best_state)
+            else:
+                # we permanently swap the stateful objects to their best state
+                self._load_new_state_dict(self.best_state.state_dict())
+
+            # the EMA modules should also be instantiated with best state.
+            # the easiest way to do so is to reinitialize a new EMA with best state loaded.
+            if self.ema is not None:
+                self.logger.info("Re-initializing EMA from best state")
+                self.initialize_ema()
+
+            if self.cfg.fsdp.use:
+                self.logger.info("Re-initializing best state after using FSDP best state.")
+                for name in self.best_state.states.keys():
+                    state_source = self._get_state_source(name)
+                    self.best_state.update(name, state_source)
+
+        return state
+
+    def restore(self, load_best: bool = False, replay_metrics: bool = False,
+                ignore_state_keys: tp.List[str] = []) -> bool:
+        """Restore the status of a solver for a given xp.
+
+        Args:
+            load_best (bool): if `True`, load the best state from the checkpoint.
+            replay_metrics (bool): if `True`, logs all the metrics from past epochs.
+            ignore_state_keys (list of str): list of sources to ignore when loading the state, e.g. `optimizer`.
+        """
+        self.logger.info("Restoring weights and history.")
+        restored_checkpoints = self.load_checkpoints(load_best, ignore_state_keys)
+
+        self.logger.info("Model hash: %s", model_hash(self.model))
+
+        if replay_metrics and len(self.history) > 0:
+            self.logger.info("Replaying past metrics...")
+            for epoch, stages in enumerate(self.history):
+                for stage_name, metrics in stages.items():
+                    # We manually log the metrics summary to the result logger
+                    # as we don't want to add them to the pending metrics
+                    self.result_logger._log_summary(stage_name, metrics, step=epoch + 1, step_name='epoch',
+                                                    formatter=self.get_formatter(stage_name))
+        return restored_checkpoints is not None
+
+    def commit(self, save_checkpoints: bool = True):
+        """Commit metrics to dora and save checkpoints at the end of an epoch."""
+        # we override commit to introduce more complex checkpoint saving behaviors
+        self.history.append(self._pending_metrics)  # This will increase self.epoch
+        if save_checkpoints:
+            self.save_checkpoints()
+        self._start_epoch()
+        if flashy.distrib.is_rank_zero():
+            self.xp.link.update_history(self.history)
+
+    def run_epoch(self):
+        """Run a single epoch with all stages.
+
+        Metrics for a given stage are stored in _pending_metrics and committed by the solver afterwards.
+        Children solvers can extend this method with custom behavior, e.g.:
+
+            def run_epoch(self):
+                ... # custom code
+                super().run_epoch()
+                ... # custom code
+        """
+        self.run_stage('train', self.train)
+        with torch.no_grad():
+            with self.swap_ema_state():
+                self.run_stage('valid', self.valid)
+                # the best state is updated with EMA states if available
+                self.update_best_state_from_stage('valid')
+            with self.swap_best_state():
+                if self.should_run_stage('evaluate'):
+                    self.run_stage('evaluate', self.evaluate)
+                if self.should_run_stage('generate'):
+                    self.run_stage('generate', with_rank_rng()(self.generate))
+
+    def run(self):
+        """Training loop."""
+        assert len(self.state_dict()) > 0
+        self.restore(replay_metrics=True)  # load checkpoint and replay history
+        self.log_hyperparams(dict_from_config(self.cfg))
+        for epoch in range(self.epoch, self.cfg.optim.epochs + 1):
+            if self.should_stop_training():
+                return
+            self.run_epoch()
+            # Commit will send the metrics to Dora and save checkpoints by default.
+            self.commit()
+
+    def should_stop_training(self) -> bool:
+        """Check whether we should stop training or not."""
+        return self.epoch > self.cfg.optim.epochs
+
+    def should_run_stage(self, stage_name) -> bool:
+        """Check whether we want to run the specified stages."""
+        stage_every = self.cfg[stage_name].get('every', None)
+        is_last_epoch = self.epoch == self.cfg.optim.epochs
+        is_epoch_every = (stage_every and self.epoch % stage_every == 0)
+        return is_last_epoch or is_epoch_every
+
+    @abstractmethod
+    def run_step(self, idx: int, batch: tp.Any, metrics: dict):
+        """Perform one training or valid step on a given batch."""
+        ...
+
+    def common_train_valid(self, dataset_split: str, **kwargs: tp.Any):
+        """Common logic for train and valid stages."""
+        self.model.train(self.is_training)
+
+        loader = self.dataloaders[dataset_split]
+        # get a different order for distributed training, otherwise this will get ignored
+        if flashy.distrib.world_size() > 1 \
+           and isinstance(loader.sampler, torch.utils.data.distributed.DistributedSampler):
+            loader.sampler.set_epoch(self.epoch)
+        updates_per_epoch = self.train_updates_per_epoch if self.is_training else len(loader)
+        if self.cfg.benchmark_no_load:
+            self.logger.warning("Fake loading for benchmarking: re-using first batch")
+            batch = next(iter(loader))
+            loader = [batch] * updates_per_epoch  # type: ignore
+        lp = self.log_progress(self.current_stage, loader, total=updates_per_epoch, updates=self.log_updates)
+        average = flashy.averager()  # epoch wise average
+        instant_average = flashy.averager()  # average between two logging
+        metrics: dict = {}
+
+        with self.profiler, self.deadlock_detect:  # profiler will only run for the first 20 updates.
+            for idx, batch in enumerate(lp):
+                self.deadlock_detect.update('batch')
+                if idx >= updates_per_epoch:
+                    break
+                metrics = {}
+                metrics = self.run_step(idx, batch, metrics)
+                self.deadlock_detect.update('step')
+                # run EMA step
+                if self.ema is not None and self.is_training and (idx + 1) % self.cfg.optim.ema.updates == 0:
+                    self.logger.debug("EMA model step")
+                    self.ema.step()
+                self.deadlock_detect.update('ema')
+                self.profiler.step()
+                instant_metrics = instant_average(metrics)
+                if lp.update(**instant_metrics):
+                    instant_average = flashy.averager()  # reset averager between two logging
+                metrics = average(metrics)  # epoch wise average
+                self.deadlock_detect.update('end_batch')
+
+        metrics = flashy.distrib.average_metrics(metrics, updates_per_epoch)
+        return metrics
+
+    def train(self):
+        """Train stage."""
+        return self.common_train_valid('train')
+
+    def valid(self):
+        """Valid stage."""
+        return self.common_train_valid('valid')
+
+    @abstractmethod
+    def evaluate(self):
+        """Evaluate stage."""
+        ...
+
+    @abstractmethod
+    def generate(self):
+        """Generate stage."""
+        ...
+
+    def run_one_stage(self, stage_name: str):
+        """Run only the specified stage.
+        This method is useful to only generate samples from a trained experiment
+        or rerun the validation or evaluation stages.
+        """
+        fn = {
+            'generate': with_rank_rng()(self.generate),
+            'evaluate': self.evaluate,
+            'valid': self.valid,
+        }
+        if stage_name not in fn:
+            raise ValueError(f'Trying to run stage {stage_name} is not supported.')
+        assert len(self.state_dict()) > 0
+        self._start_epoch()
+        with torch.no_grad(), self.swap_best_state():
+            self.run_stage(stage_name, fn[stage_name])
+        if not self.cfg.execute_inplace:
+            self.commit(save_checkpoints=False)
+
+    @staticmethod
+    def get_eval_solver_from_sig(sig: str, dtype: tp.Optional[str] = None,
+                                 device: tp.Optional[str] = None, autocast: bool = True,
+                                 batch_size: tp.Optional[int] = None,
+                                 override_cfg: tp.Optional[tp.Union[dict, omegaconf.DictConfig]] = None,
+                                 **kwargs):
+        """Mostly a convenience function around audiocraft.train.get_solver_from_sig,
+        populating all the proper param, deactivating EMA, FSDP, loading the best state,
+        basically all you need to get a solver ready to "play" with in single GPU mode
+        and with minimal memory overhead.
+
+        Args:
+            sig (str): signature to load.
+            dtype (str or None): potential dtype, as a string, i.e. 'float16'.
+            device (str or None): potential device, as a string, i.e. 'cuda'.
+            override_cfg (dict or omegaconf.DictConfig or None): potential device, as a string, i.e. 'cuda'.
+        """
+        from audiocraft import train
+        our_override_cfg: tp.Dict[str, tp.Any] = {'optim': {'ema': {'use': False}}}
+        our_override_cfg['autocast'] = autocast
+        if dtype is not None:
+            our_override_cfg['dtype'] = dtype
+        if device is not None:
+            our_override_cfg['device'] = device
+        if batch_size is not None:
+            our_override_cfg['dataset'] = {'batch_size': batch_size}
+        if override_cfg is None:
+            override_cfg = {}
+        override_cfg = omegaconf.OmegaConf.merge(
+            omegaconf.DictConfig(override_cfg), omegaconf.DictConfig(our_override_cfg))  # type: ignore
+        solver = train.get_solver_from_sig(
+            sig, override_cfg=override_cfg,
+            load_best=True, disable_fsdp=True,
+            ignore_state_keys=['optimizer', 'ema'], **kwargs)
+        solver.model.eval()
+        return solver
diff --git a/audiocraft/audiocraft/solvers/builders.py b/audiocraft/audiocraft/solvers/builders.py
new file mode 100644
index 0000000000000000000000000000000000000000..304d8f08d33a70e8be9388c855b2ae43bdf2683b
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/builders.py
@@ -0,0 +1,363 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+All the functions to build the relevant solvers and used objects
+from the Hydra config.
+"""
+
+from enum import Enum
+import logging
+import typing as tp
+
+import dora
+import flashy
+import omegaconf
+import torch
+from torch import nn
+from torch.optim import Optimizer
+# LRScheduler was renamed in some torch versions
+try:
+    from torch.optim.lr_scheduler import LRScheduler  # type: ignore
+except ImportError:
+    from torch.optim.lr_scheduler import _LRScheduler as LRScheduler
+
+from .base import StandardSolver
+from .. import adversarial, data, losses, metrics, optim
+from ..utils.utils import dict_from_config, get_loader
+
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetType(Enum):
+    AUDIO = "audio"
+    MUSIC = "music"
+    SOUND = "sound"
+
+
+def get_solver(cfg: omegaconf.DictConfig) -> StandardSolver:
+    """Instantiate solver from config."""
+    from .audiogen import AudioGenSolver
+    from .compression import CompressionSolver
+    from .musicgen import MusicGenSolver
+    from .diffusion import DiffusionSolver
+    klass = {
+        'compression': CompressionSolver,
+        'musicgen': MusicGenSolver,
+        'audiogen': AudioGenSolver,
+        'lm': MusicGenSolver,  # backward compatibility
+        'diffusion': DiffusionSolver,
+        'sound_lm': AudioGenSolver,  # backward compatibility
+    }[cfg.solver]
+    return klass(cfg)  # type: ignore
+
+
+def get_optim_parameter_groups(model: nn.Module):
+    """Create parameter groups for the model using the appropriate method
+    if defined for each modules, to create the different groups.
+
+    Args:
+        model (nn.Module): torch model
+    Returns:
+        List of parameter groups
+    """
+    seen_params: tp.Set[nn.parameter.Parameter] = set()
+    other_params = []
+    groups = []
+    for name, module in model.named_modules():
+        if hasattr(module, 'make_optim_group'):
+            group = module.make_optim_group()
+            params = set(group['params'])
+            assert params.isdisjoint(seen_params)
+            seen_params |= set(params)
+            groups.append(group)
+    for param in model.parameters():
+        if param not in seen_params:
+            other_params.append(param)
+    groups.insert(0, {'params': other_params})
+    parameters = groups
+    return parameters
+
+
+def get_optimizer(params: tp.Union[nn.Module, tp.Iterable[torch.Tensor]], cfg: omegaconf.DictConfig) -> Optimizer:
+    """Build torch optimizer from config and set of parameters.
+    Supported optimizers: Adam, AdamW
+
+    Args:
+        params (nn.Module or iterable of torch.Tensor): Parameters to optimize.
+        cfg (DictConfig): Optimization-related configuration.
+    Returns:
+        torch.optim.Optimizer.
+    """
+    if 'optimizer' not in cfg:
+        if getattr(cfg, 'optim', None) is not None:
+            raise KeyError("Optimizer not found in config. Try instantiating optimizer from cfg.optim?")
+        else:
+            raise KeyError("Optimizer not found in config.")
+
+    parameters = get_optim_parameter_groups(params) if isinstance(params, nn.Module) else params
+    optimizer: torch.optim.Optimizer
+    if cfg.optimizer == 'adam':
+        optimizer = torch.optim.Adam(parameters, lr=cfg.lr, **cfg.adam)
+    elif cfg.optimizer == 'adamw':
+        optimizer = torch.optim.AdamW(parameters, lr=cfg.lr, **cfg.adam)
+    elif cfg.optimizer == 'dadam':
+        optimizer = optim.DAdaptAdam(parameters, lr=cfg.lr, **cfg.adam)
+    else:
+        raise ValueError(f"Unsupported LR Scheduler: {cfg.lr_scheduler}")
+    return optimizer
+
+
+def get_lr_scheduler(optimizer: torch.optim.Optimizer,
+                     cfg: omegaconf.DictConfig,
+                     total_updates: int) -> tp.Optional[LRScheduler]:
+    """Build torch learning rate scheduler from config and associated optimizer.
+    Supported learning rate schedulers: ExponentialLRScheduler, PlateauLRScheduler
+
+    Args:
+        optimizer (torch.optim.Optimizer): Optimizer.
+        cfg (DictConfig): Schedule-related configuration.
+        total_updates (int): Total number of updates.
+    Returns:
+        torch.optim.Optimizer.
+    """
+    if 'lr_scheduler' not in cfg:
+        raise KeyError("LR Scheduler not found in config")
+
+    lr_sched: tp.Optional[LRScheduler] = None
+    if cfg.lr_scheduler == 'step':
+        lr_sched = torch.optim.lr_scheduler.StepLR(optimizer, **cfg.step)
+    elif cfg.lr_scheduler == 'exponential':
+        lr_sched = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=cfg.exponential)
+    elif cfg.lr_scheduler == 'cosine':
+        kwargs = dict_from_config(cfg.cosine)
+        warmup_steps = kwargs.pop('warmup')
+        lr_sched = optim.CosineLRScheduler(
+            optimizer, warmup_steps=warmup_steps, total_steps=total_updates, **kwargs)
+    elif cfg.lr_scheduler == 'polynomial_decay':
+        kwargs = dict_from_config(cfg.polynomial_decay)
+        warmup_steps = kwargs.pop('warmup')
+        lr_sched = optim.PolynomialDecayLRScheduler(
+            optimizer, warmup_steps=warmup_steps, total_steps=total_updates, **kwargs)
+    elif cfg.lr_scheduler == 'inverse_sqrt':
+        kwargs = dict_from_config(cfg.inverse_sqrt)
+        warmup_steps = kwargs.pop('warmup')
+        lr_sched = optim.InverseSquareRootLRScheduler(optimizer, warmup_steps=warmup_steps, **kwargs)
+    elif cfg.lr_scheduler == 'linear_warmup':
+        kwargs = dict_from_config(cfg.linear_warmup)
+        warmup_steps = kwargs.pop('warmup')
+        lr_sched = optim.LinearWarmupLRScheduler(optimizer, warmup_steps=warmup_steps, **kwargs)
+    elif cfg.lr_scheduler is not None:
+        raise ValueError(f"Unsupported LR Scheduler: {cfg.lr_scheduler}")
+    return lr_sched
+
+
+def get_ema(module_dict: nn.ModuleDict, cfg: omegaconf.DictConfig) -> tp.Optional[optim.ModuleDictEMA]:
+    """Initialize Exponential Moving Average.
+
+    Args:
+        module_dict (nn.ModuleDict): ModuleDict for which to compute the EMA.
+        cfg (omegaconf.DictConfig): Optim EMA configuration.
+    Returns:
+        optim.ModuleDictEMA: EMA version of the ModuleDict.
+    """
+    kw: tp.Dict[str, tp.Any] = dict(cfg)
+    use = kw.pop('use', False)
+    decay = kw.pop('decay', None)
+    device = kw.pop('device', None)
+    if not use:
+        return None
+    if len(module_dict) == 0:
+        raise ValueError("Trying to build EMA but an empty module_dict source is provided!")
+    ema_module = optim.ModuleDictEMA(module_dict, decay=decay, device=device)
+    return ema_module
+
+
+def get_loss(loss_name: str, cfg: omegaconf.DictConfig):
+    """Instantiate loss from configuration."""
+    klass = {
+        'l1': torch.nn.L1Loss,
+        'l2': torch.nn.MSELoss,
+        'mel': losses.MelSpectrogramL1Loss,
+        'mrstft': losses.MRSTFTLoss,
+        'msspec': losses.MultiScaleMelSpectrogramLoss,
+        'sisnr': losses.SISNR,
+    }[loss_name]
+    kwargs = dict(getattr(cfg, loss_name))
+    return klass(**kwargs)
+
+
+def get_balancer(loss_weights: tp.Dict[str, float], cfg: omegaconf.DictConfig) -> losses.Balancer:
+    """Instantiate loss balancer from configuration for the provided weights."""
+    kwargs: tp.Dict[str, tp.Any] = dict_from_config(cfg)
+    return losses.Balancer(loss_weights, **kwargs)
+
+
+def get_adversary(name: str, cfg: omegaconf.DictConfig) -> nn.Module:
+    """Initialize adversary from config."""
+    klass = {
+        'msd': adversarial.MultiScaleDiscriminator,
+        'mpd': adversarial.MultiPeriodDiscriminator,
+        'msstftd': adversarial.MultiScaleSTFTDiscriminator,
+    }[name]
+    adv_cfg: tp.Dict[str, tp.Any] = dict(getattr(cfg, name))
+    return klass(**adv_cfg)
+
+
+def get_adversarial_losses(cfg) -> nn.ModuleDict:
+    """Initialize dict of adversarial losses from config."""
+    device = cfg.device
+    adv_cfg = getattr(cfg, 'adversarial')
+    adversaries = adv_cfg.get('adversaries', [])
+    adv_loss_name = adv_cfg['adv_loss']
+    feat_loss_name = adv_cfg.get('feat_loss')
+    normalize = adv_cfg.get('normalize', True)
+    feat_loss: tp.Optional[adversarial.FeatureMatchingLoss] = None
+    if feat_loss_name:
+        assert feat_loss_name in ['l1', 'l2'], f"Feature loss only support L1 or L2 but {feat_loss_name} found."
+        loss = get_loss(feat_loss_name, cfg)
+        feat_loss = adversarial.FeatureMatchingLoss(loss, normalize)
+    loss = adversarial.get_adv_criterion(adv_loss_name)
+    loss_real = adversarial.get_real_criterion(adv_loss_name)
+    loss_fake = adversarial.get_fake_criterion(adv_loss_name)
+    adv_losses = nn.ModuleDict()
+    for adv_name in adversaries:
+        adversary = get_adversary(adv_name, cfg).to(device)
+        optimizer = get_optimizer(adversary.parameters(), cfg.optim)
+        adv_loss = adversarial.AdversarialLoss(
+            adversary,
+            optimizer,
+            loss=loss,
+            loss_real=loss_real,
+            loss_fake=loss_fake,
+            loss_feat=feat_loss,
+            normalize=normalize
+        )
+        adv_losses[adv_name] = adv_loss
+    return adv_losses
+
+
+def get_visqol(cfg: omegaconf.DictConfig) -> metrics.ViSQOL:
+    """Instantiate ViSQOL metric from config."""
+    kwargs = dict_from_config(cfg)
+    return metrics.ViSQOL(**kwargs)
+
+
+def get_fad(cfg: omegaconf.DictConfig) -> metrics.FrechetAudioDistanceMetric:
+    """Instantiate Frechet Audio Distance metric from config."""
+    kwargs = dict_from_config(cfg.tf)
+    xp = dora.get_xp()
+    kwargs['log_folder'] = xp.folder
+    return metrics.FrechetAudioDistanceMetric(**kwargs)
+
+
+def get_kldiv(cfg: omegaconf.DictConfig) -> metrics.KLDivergenceMetric:
+    """Instantiate KL-Divergence metric from config."""
+    kld_metrics = {
+        'passt': metrics.PasstKLDivergenceMetric,
+    }
+    klass = kld_metrics[cfg.model]
+    kwargs = dict_from_config(cfg.get(cfg.model))
+    return klass(**kwargs)
+
+
+def get_text_consistency(cfg: omegaconf.DictConfig) -> metrics.TextConsistencyMetric:
+    """Instantiate Text Consistency metric from config."""
+    text_consistency_metrics = {
+        'clap': metrics.CLAPTextConsistencyMetric
+    }
+    klass = text_consistency_metrics[cfg.model]
+    kwargs = dict_from_config(cfg.get(cfg.model))
+    return klass(**kwargs)
+
+
+def get_chroma_cosine_similarity(cfg: omegaconf.DictConfig) -> metrics.ChromaCosineSimilarityMetric:
+    """Instantiate Chroma Cosine Similarity metric from config."""
+    assert cfg.model == 'chroma_base', "Only support 'chroma_base' method for chroma cosine similarity metric"
+    kwargs = dict_from_config(cfg.get(cfg.model))
+    return metrics.ChromaCosineSimilarityMetric(**kwargs)
+
+
+def get_audio_datasets(cfg: omegaconf.DictConfig,
+                       dataset_type: DatasetType = DatasetType.AUDIO) -> tp.Dict[str, torch.utils.data.DataLoader]:
+    """Build AudioDataset from configuration.
+
+    Args:
+        cfg (omegaconf.DictConfig): Configuration.
+        dataset_type: The type of dataset to create.
+    Returns:
+        dict[str, torch.utils.data.DataLoader]: Map of dataloader for each data split.
+    """
+    dataloaders: dict = {}
+
+    sample_rate = cfg.sample_rate
+    channels = cfg.channels
+    seed = cfg.seed
+    max_sample_rate = cfg.datasource.max_sample_rate
+    max_channels = cfg.datasource.max_channels
+
+    assert cfg.dataset is not None, "Could not find dataset definition in config"
+
+    dataset_cfg = dict_from_config(cfg.dataset)
+    splits_cfg: dict = {}
+    splits_cfg['train'] = dataset_cfg.pop('train')
+    splits_cfg['valid'] = dataset_cfg.pop('valid')
+    splits_cfg['evaluate'] = dataset_cfg.pop('evaluate')
+    splits_cfg['generate'] = dataset_cfg.pop('generate')
+    execute_only_stage = cfg.get('execute_only', None)
+
+    for split, path in cfg.datasource.items():
+        if not isinstance(path, str):
+            continue  # skipping this as not a path
+        if execute_only_stage is not None and split != execute_only_stage:
+            continue
+        logger.info(f"Loading audio data split {split}: {str(path)}")
+        assert (
+            cfg.sample_rate <= max_sample_rate
+        ), f"Expecting a max sample rate of {max_sample_rate} for datasource but {sample_rate} found."
+        assert (
+            cfg.channels <= max_channels
+        ), f"Expecting a max number of channels of {max_channels} for datasource but {channels} found."
+
+        split_cfg = splits_cfg[split]
+        split_kwargs = {k: v for k, v in split_cfg.items()}
+        kwargs = {**dataset_cfg, **split_kwargs}  # split kwargs overrides default dataset_cfg
+        kwargs['sample_rate'] = sample_rate
+        kwargs['channels'] = channels
+
+        if kwargs.get('permutation_on_files') and cfg.optim.updates_per_epoch:
+            kwargs['num_samples'] = (
+                flashy.distrib.world_size() * cfg.dataset.batch_size * cfg.optim.updates_per_epoch)
+
+        num_samples = kwargs['num_samples']
+        shuffle = kwargs['shuffle']
+
+        return_info = kwargs.pop('return_info')
+        batch_size = kwargs.pop('batch_size', None)
+        num_workers = kwargs.pop('num_workers')
+
+        if dataset_type == DatasetType.MUSIC:
+            dataset = data.music_dataset.MusicDataset.from_meta(path, **kwargs)
+        elif dataset_type == DatasetType.SOUND:
+            dataset = data.sound_dataset.SoundDataset.from_meta(path, **kwargs)
+        elif dataset_type == DatasetType.AUDIO:
+            dataset = data.info_audio_dataset.InfoAudioDataset.from_meta(path, return_info=return_info, **kwargs)
+        else:
+            raise ValueError(f"Dataset type is unsupported: {dataset_type}")
+
+        loader = get_loader(
+            dataset,
+            num_samples,
+            batch_size=batch_size,
+            num_workers=num_workers,
+            seed=seed,
+            collate_fn=dataset.collater if return_info else None,
+            shuffle=shuffle,
+        )
+        dataloaders[split] = loader
+
+    return dataloaders
diff --git a/audiocraft/audiocraft/solvers/compression.py b/audiocraft/audiocraft/solvers/compression.py
new file mode 100644
index 0000000000000000000000000000000000000000..b757503472a3bfbf90e1636999e64913848a7474
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/compression.py
@@ -0,0 +1,328 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import multiprocessing
+from pathlib import Path
+import typing as tp
+
+import flashy
+import omegaconf
+import torch
+from torch import nn
+
+from . import base, builders
+from .. import models, quantization
+from ..utils import checkpoint
+from ..utils.samples.manager import SampleManager
+from ..utils.utils import get_pool_executor
+
+
+logger = logging.getLogger(__name__)
+
+
+class CompressionSolver(base.StandardSolver):
+    """Solver for compression task.
+
+    The compression task combines a set of perceptual and objective losses
+    to train an EncodecModel (composed of an encoder-decoder and a quantizer)
+    to perform high fidelity audio reconstruction.
+    """
+    def __init__(self, cfg: omegaconf.DictConfig):
+        super().__init__(cfg)
+        self.rng: torch.Generator  # set at each epoch
+        self.adv_losses = builders.get_adversarial_losses(self.cfg)
+        self.aux_losses = nn.ModuleDict()
+        self.info_losses = nn.ModuleDict()
+        assert not cfg.fsdp.use, "FSDP not supported by CompressionSolver."
+        loss_weights = dict()
+        for loss_name, weight in self.cfg.losses.items():
+            if loss_name in ['adv', 'feat']:
+                for adv_name, _ in self.adv_losses.items():
+                    loss_weights[f'{loss_name}_{adv_name}'] = weight
+            elif weight > 0:
+                self.aux_losses[loss_name] = builders.get_loss(loss_name, self.cfg)
+                loss_weights[loss_name] = weight
+            else:
+                self.info_losses[loss_name] = builders.get_loss(loss_name, self.cfg)
+        self.balancer = builders.get_balancer(loss_weights, self.cfg.balancer)
+        self.register_stateful('adv_losses')
+
+    @property
+    def best_metric_name(self) -> tp.Optional[str]:
+        # best model is the last for the compression model
+        return None
+
+    def build_model(self):
+        """Instantiate model and optimizer."""
+        # Model and optimizer
+        self.model = models.builders.get_compression_model(self.cfg).to(self.device)
+        self.optimizer = builders.get_optimizer(self.model.parameters(), self.cfg.optim)
+        self.register_stateful('model', 'optimizer')
+        self.register_best_state('model')
+        self.register_ema('model')
+
+    def build_dataloaders(self):
+        """Instantiate audio dataloaders for each stage."""
+        self.dataloaders = builders.get_audio_datasets(self.cfg)
+
+    def show(self):
+        """Show the compression model and employed adversarial loss."""
+        self.logger.info(f"Compression model with {self.model.quantizer.total_codebooks} codebooks:")
+        self.log_model_summary(self.model)
+        self.logger.info("Adversarial loss:")
+        self.log_model_summary(self.adv_losses)
+        self.logger.info("Auxiliary losses:")
+        self.logger.info(self.aux_losses)
+        self.logger.info("Info losses:")
+        self.logger.info(self.info_losses)
+
+    def run_step(self, idx: int, batch: torch.Tensor, metrics: dict):
+        """Perform one training or valid step on a given batch."""
+        x = batch.to(self.device)
+        y = x.clone()
+
+        qres = self.model(x)
+        assert isinstance(qres, quantization.QuantizedResult)
+        y_pred = qres.x
+        # Log bandwidth in kb/s
+        metrics['bandwidth'] = qres.bandwidth.mean()
+
+        if self.is_training:
+            d_losses: dict = {}
+            if len(self.adv_losses) > 0 and torch.rand(1, generator=self.rng).item() <= 1 / self.cfg.adversarial.every:
+                for adv_name, adversary in self.adv_losses.items():
+                    disc_loss = adversary.train_adv(y_pred, y)
+                    d_losses[f'd_{adv_name}'] = disc_loss
+                metrics['d_loss'] = torch.sum(torch.stack(list(d_losses.values())))
+            metrics.update(d_losses)
+
+        balanced_losses: dict = {}
+        other_losses: dict = {}
+
+        # penalty from quantization
+        if qres.penalty is not None and qres.penalty.requires_grad:
+            other_losses['penalty'] = qres.penalty  # penalty term from the quantizer
+
+        # adversarial losses
+        for adv_name, adversary in self.adv_losses.items():
+            adv_loss, feat_loss = adversary(y_pred, y)
+            balanced_losses[f'adv_{adv_name}'] = adv_loss
+            balanced_losses[f'feat_{adv_name}'] = feat_loss
+
+        # auxiliary losses
+        for loss_name, criterion in self.aux_losses.items():
+            loss = criterion(y_pred, y)
+            balanced_losses[loss_name] = loss
+
+        # weighted losses
+        metrics.update(balanced_losses)
+        metrics.update(other_losses)
+        metrics.update(qres.metrics)
+
+        if self.is_training:
+            # backprop losses that are not handled by balancer
+            other_loss = torch.tensor(0., device=self.device)
+            if 'penalty' in other_losses:
+                other_loss += other_losses['penalty']
+            if other_loss.requires_grad:
+                other_loss.backward(retain_graph=True)
+                ratio1 = sum(p.grad.data.norm(p=2).pow(2)
+                             for p in self.model.parameters() if p.grad is not None)
+                assert isinstance(ratio1, torch.Tensor)
+                metrics['ratio1'] = ratio1.sqrt()
+
+            # balancer losses backward, returns effective training loss
+            # with effective weights at the current batch.
+            metrics['g_loss'] = self.balancer.backward(balanced_losses, y_pred)
+            # add metrics corresponding to weight ratios
+            metrics.update(self.balancer.metrics)
+            ratio2 = sum(p.grad.data.norm(p=2).pow(2)
+                         for p in self.model.parameters() if p.grad is not None)
+            assert isinstance(ratio2, torch.Tensor)
+            metrics['ratio2'] = ratio2.sqrt()
+
+            # optim
+            flashy.distrib.sync_model(self.model)
+            if self.cfg.optim.max_norm:
+                torch.nn.utils.clip_grad_norm_(
+                    self.model.parameters(), self.cfg.optim.max_norm
+                )
+            self.optimizer.step()
+            self.optimizer.zero_grad()
+
+        # informative losses only
+        info_losses: dict = {}
+        with torch.no_grad():
+            for loss_name, criterion in self.info_losses.items():
+                loss = criterion(y_pred, y)
+                info_losses[loss_name] = loss
+
+        metrics.update(info_losses)
+
+        # aggregated GAN losses: this is useful to report adv and feat across different adversarial loss setups
+        adv_losses = [loss for loss_name, loss in metrics.items() if loss_name.startswith('adv')]
+        if len(adv_losses) > 0:
+            metrics['adv'] = torch.sum(torch.stack(adv_losses))
+        feat_losses = [loss for loss_name, loss in metrics.items() if loss_name.startswith('feat')]
+        if len(feat_losses) > 0:
+            metrics['feat'] = torch.sum(torch.stack(feat_losses))
+
+        return metrics
+
+    def run_epoch(self):
+        # reset random seed at the beginning of the epoch
+        self.rng = torch.Generator()
+        self.rng.manual_seed(1234 + self.epoch)
+        # run epoch
+        super().run_epoch()
+
+    def evaluate(self):
+        """Evaluate stage. Runs audio reconstruction evaluation."""
+        self.model.eval()
+        evaluate_stage_name = str(self.current_stage)
+
+        loader = self.dataloaders['evaluate']
+        updates = len(loader)
+        lp = self.log_progress(f'{evaluate_stage_name} inference', loader, total=updates, updates=self.log_updates)
+        average = flashy.averager()
+
+        pendings = []
+        ctx = multiprocessing.get_context('spawn')
+        with get_pool_executor(self.cfg.evaluate.num_workers, mp_context=ctx) as pool:
+            for idx, batch in enumerate(lp):
+                x = batch.to(self.device)
+                with torch.no_grad():
+                    qres = self.model(x)
+
+                y_pred = qres.x.cpu()
+                y = batch.cpu()  # should already be on CPU but just in case
+                pendings.append(pool.submit(evaluate_audio_reconstruction, y_pred, y, self.cfg))
+
+            metrics_lp = self.log_progress(f'{evaluate_stage_name} metrics', pendings, updates=self.log_updates)
+            for pending in metrics_lp:
+                metrics = pending.result()
+                metrics = average(metrics)
+
+        metrics = flashy.distrib.average_metrics(metrics, len(loader))
+        return metrics
+
+    def generate(self):
+        """Generate stage."""
+        self.model.eval()
+        sample_manager = SampleManager(self.xp, map_reference_to_sample_id=True)
+        generate_stage_name = str(self.current_stage)
+
+        loader = self.dataloaders['generate']
+        updates = len(loader)
+        lp = self.log_progress(generate_stage_name, loader, total=updates, updates=self.log_updates)
+
+        for batch in lp:
+            reference, _ = batch
+            reference = reference.to(self.device)
+            with torch.no_grad():
+                qres = self.model(reference)
+            assert isinstance(qres, quantization.QuantizedResult)
+
+            reference = reference.cpu()
+            estimate = qres.x.cpu()
+            sample_manager.add_samples(estimate, self.epoch, ground_truth_wavs=reference)
+
+        flashy.distrib.barrier()
+
+    def load_from_pretrained(self, name: str) -> dict:
+        model = models.CompressionModel.get_pretrained(name)
+        if isinstance(model, models.DAC):
+            raise RuntimeError("Cannot fine tune a DAC model.")
+        elif isinstance(model, models.HFEncodecCompressionModel):
+            self.logger.warning('Trying to automatically convert a HuggingFace model '
+                                'to AudioCraft, this might fail!')
+            state = model.model.state_dict()
+            new_state = {}
+            for k, v in state.items():
+                if k.startswith('decoder.layers') and '.conv.' in k and '.block.' not in k:
+                    # We need to determine if this a convtr or a regular conv.
+                    layer = int(k.split('.')[2])
+                    if isinstance(model.model.decoder.layers[layer].conv, torch.nn.ConvTranspose1d):
+
+                        k = k.replace('.conv.', '.convtr.')
+                k = k.replace('encoder.layers.', 'encoder.model.')
+                k = k.replace('decoder.layers.', 'decoder.model.')
+                k = k.replace('conv.', 'conv.conv.')
+                k = k.replace('convtr.', 'convtr.convtr.')
+                k = k.replace('quantizer.layers.', 'quantizer.vq.layers.')
+                k = k.replace('.codebook.', '._codebook.')
+                new_state[k] = v
+            state = new_state
+        elif isinstance(model, models.EncodecModel):
+            state = model.state_dict()
+        else:
+            raise RuntimeError(f"Cannot fine tune model type {type(model)}.")
+        return {
+            'best_state': {'model': state}
+        }
+
+    @staticmethod
+    def model_from_checkpoint(checkpoint_path: tp.Union[Path, str],
+                              device: tp.Union[torch.device, str] = 'cpu') -> models.CompressionModel:
+        """Instantiate a CompressionModel from a given checkpoint path or dora sig.
+        This method is a convenient endpoint to load a CompressionModel to use in other solvers.
+
+        Args:
+            checkpoint_path (Path or str): Path to checkpoint or dora sig from where the checkpoint is resolved.
+                This also supports pre-trained models by using a path of the form //pretrained/NAME.
+                See `model_from_pretrained` for a list of supported pretrained models.
+            use_ema (bool): Use EMA variant of the model instead of the actual model.
+            device (torch.device or str): Device on which the model is loaded.
+        """
+        checkpoint_path = str(checkpoint_path)
+        if checkpoint_path.startswith('//pretrained/'):
+            name = checkpoint_path.split('/', 3)[-1]
+            return models.CompressionModel.get_pretrained(name, device)
+        logger = logging.getLogger(__name__)
+        logger.info(f"Loading compression model from checkpoint: {checkpoint_path}")
+        _checkpoint_path = checkpoint.resolve_checkpoint_path(checkpoint_path, use_fsdp=False)
+        assert _checkpoint_path is not None, f"Could not resolve compression model checkpoint path: {checkpoint_path}"
+        state = checkpoint.load_checkpoint(_checkpoint_path)
+        assert state is not None and 'xp.cfg' in state, f"Could not load compression model from ckpt: {checkpoint_path}"
+        cfg = state['xp.cfg']
+        cfg.device = device
+        compression_model = models.builders.get_compression_model(cfg).to(device)
+        assert compression_model.sample_rate == cfg.sample_rate, "Compression model sample rate should match"
+
+        assert 'best_state' in state and state['best_state'] != {}
+        assert 'exported' not in state, "When loading an exported checkpoint, use the //pretrained/ prefix."
+        compression_model.load_state_dict(state['best_state']['model'])
+        compression_model.eval()
+        logger.info("Compression model loaded!")
+        return compression_model
+
+    @staticmethod
+    def wrapped_model_from_checkpoint(cfg: omegaconf.DictConfig,
+                                      checkpoint_path: tp.Union[Path, str],
+                                      device: tp.Union[torch.device, str] = 'cpu') -> models.CompressionModel:
+        """Instantiate a wrapped CompressionModel from a given checkpoint path or dora sig.
+
+        Args:
+            cfg (omegaconf.DictConfig): Configuration to read from for wrapped mode.
+            checkpoint_path (Path or str): Path to checkpoint or dora sig from where the checkpoint is resolved.
+            use_ema (bool): Use EMA variant of the model instead of the actual model.
+            device (torch.device or str): Device on which the model is loaded.
+        """
+        compression_model = CompressionSolver.model_from_checkpoint(checkpoint_path, device)
+        compression_model = models.builders.get_wrapped_compression_model(compression_model, cfg)
+        return compression_model
+
+
+def evaluate_audio_reconstruction(y_pred: torch.Tensor, y: torch.Tensor, cfg: omegaconf.DictConfig) -> dict:
+    """Audio reconstruction evaluation method that can be conveniently pickled."""
+    metrics = {}
+    if cfg.evaluate.metrics.visqol:
+        visqol = builders.get_visqol(cfg.metrics.visqol)
+        metrics['visqol'] = visqol(y_pred, y, cfg.sample_rate)
+    sisnr = builders.get_loss('sisnr', cfg)
+    metrics['sisnr'] = sisnr(y_pred, y)
+    return metrics
diff --git a/audiocraft/audiocraft/solvers/diffusion.py b/audiocraft/audiocraft/solvers/diffusion.py
new file mode 100644
index 0000000000000000000000000000000000000000..93dea2520836f458ab1b8514dca952b51d113ec2
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/diffusion.py
@@ -0,0 +1,279 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import typing as tp
+
+import flashy
+import julius
+import omegaconf
+import torch
+import torch.nn.functional as F
+
+from . import builders
+from . import base
+from .. import models
+from ..modules.diffusion_schedule import NoiseSchedule
+from ..metrics import RelativeVolumeMel
+from ..models.builders import get_processor
+from ..utils.samples.manager import SampleManager
+from ..solvers.compression import CompressionSolver
+
+
+class PerStageMetrics:
+    """Handle prompting the metrics per stage.
+    It outputs the metrics per range of diffusion states.
+    e.g. avg loss when t in [250, 500]
+    """
+    def __init__(self, num_steps: int, num_stages: int = 4):
+        self.num_steps = num_steps
+        self.num_stages = num_stages
+
+    def __call__(self, losses: dict, step: tp.Union[int, torch.Tensor]):
+        if type(step) is int:
+            stage = int((step / self.num_steps) * self.num_stages)
+            return {f"{name}_{stage}": loss for name, loss in losses.items()}
+        elif type(step) is torch.Tensor:
+            stage_tensor = ((step / self.num_steps) * self.num_stages).long()
+            out: tp.Dict[str, float] = {}
+            for stage_idx in range(self.num_stages):
+                mask = (stage_tensor == stage_idx)
+                N = mask.sum()
+                stage_out = {}
+                if N > 0:  # pass if no elements in the stage
+                    for name, loss in losses.items():
+                        stage_loss = (mask * loss).sum() / N
+                        stage_out[f"{name}_{stage_idx}"] = stage_loss
+                out = {**out, **stage_out}
+            return out
+
+
+class DataProcess:
+    """Apply filtering or resampling.
+
+    Args:
+        initial_sr (int): Initial sample rate.
+        target_sr (int): Target sample rate.
+        use_resampling: Whether to use resampling or not.
+        use_filter (bool):
+        n_bands (int): Number of bands to consider.
+        idx_band (int):
+        device (torch.device or str):
+        cutoffs ():
+        boost (bool):
+    """
+    def __init__(self, initial_sr: int = 24000, target_sr: int = 16000, use_resampling: bool = False,
+                 use_filter: bool = False, n_bands: int = 4,
+                 idx_band: int = 0, device: torch.device = torch.device('cpu'), cutoffs=None, boost=False):
+        """Apply filtering or resampling
+        Args:
+            initial_sr (int): sample rate of the dataset
+            target_sr (int): sample rate after resampling
+            use_resampling (bool): whether or not performs resampling
+            use_filter (bool): when True filter the data to keep only one frequency band
+            n_bands (int): Number of bands used
+            cuts (none or list): The cutoff frequencies of the band filtering
+                                if None then we use mel scale bands.
+            idx_band (int): index of the frequency band. 0 are lows ... (n_bands - 1) highs
+            boost (bool): make the data scale match our music dataset.
+        """
+        assert idx_band < n_bands
+        self.idx_band = idx_band
+        if use_filter:
+            if cutoffs is not None:
+                self.filter = julius.SplitBands(sample_rate=initial_sr, cutoffs=cutoffs).to(device)
+            else:
+                self.filter = julius.SplitBands(sample_rate=initial_sr, n_bands=n_bands).to(device)
+        self.use_filter = use_filter
+        self.use_resampling = use_resampling
+        self.target_sr = target_sr
+        self.initial_sr = initial_sr
+        self.boost = boost
+
+    def process_data(self, x, metric=False):
+        if x is None:
+            return None
+        if self.boost:
+            x /= torch.clamp(x.std(dim=(1, 2), keepdim=True), min=1e-4)
+            x * 0.22
+        if self.use_filter and not metric:
+            x = self.filter(x)[self.idx_band]
+        if self.use_resampling:
+            x = julius.resample_frac(x, old_sr=self.initial_sr, new_sr=self.target_sr)
+        return x
+
+    def inverse_process(self, x):
+        """Upsampling only."""
+        if self.use_resampling:
+            x = julius.resample_frac(x, old_sr=self.target_sr, new_sr=self.target_sr)
+        return x
+
+
+class DiffusionSolver(base.StandardSolver):
+    """Solver for compression task.
+
+    The diffusion task allows for MultiBand diffusion model training.
+
+    Args:
+        cfg (DictConfig): Configuration.
+    """
+    def __init__(self, cfg: omegaconf.DictConfig):
+        super().__init__(cfg)
+        self.cfg = cfg
+        self.device = cfg.device
+        self.sample_rate: int = self.cfg.sample_rate
+        self.codec_model = CompressionSolver.model_from_checkpoint(
+            cfg.compression_model_checkpoint, device=self.device)
+
+        self.codec_model.set_num_codebooks(cfg.n_q)
+        assert self.codec_model.sample_rate == self.cfg.sample_rate, (
+            f"Codec model sample rate is {self.codec_model.sample_rate} but "
+            f"Solver sample rate is {self.cfg.sample_rate}."
+            )
+        assert self.codec_model.sample_rate == self.sample_rate, \
+            f"Sample rate of solver {self.sample_rate} and codec {self.codec_model.sample_rate} " \
+            "don't match."
+
+        self.sample_processor = get_processor(cfg.processor, sample_rate=self.sample_rate)
+        self.register_stateful('sample_processor')
+        self.sample_processor.to(self.device)
+
+        self.schedule = NoiseSchedule(
+            **cfg.schedule, device=self.device, sample_processor=self.sample_processor)
+
+        self.eval_metric: tp.Optional[torch.nn.Module] = None
+
+        self.rvm = RelativeVolumeMel()
+        self.data_processor = DataProcess(initial_sr=self.sample_rate, target_sr=cfg.resampling.target_sr,
+                                          use_resampling=cfg.resampling.use, cutoffs=cfg.filter.cutoffs,
+                                          use_filter=cfg.filter.use, n_bands=cfg.filter.n_bands,
+                                          idx_band=cfg.filter.idx_band, device=self.device)
+
+    @property
+    def best_metric_name(self) -> tp.Optional[str]:
+        if self._current_stage == "evaluate":
+            return 'rvm'
+        else:
+            return 'loss'
+
+    @torch.no_grad()
+    def get_condition(self, wav: torch.Tensor) -> torch.Tensor:
+        codes, scale = self.codec_model.encode(wav)
+        assert scale is None, "Scaled compression models not supported."
+        emb = self.codec_model.decode_latent(codes)
+        return emb
+
+    def build_model(self):
+        """Build model and optimizer as well as optional Exponential Moving Average of the model.
+        """
+        # Model and optimizer
+        self.model = models.builders.get_diffusion_model(self.cfg).to(self.device)
+        self.optimizer = builders.get_optimizer(self.model.parameters(), self.cfg.optim)
+        self.register_stateful('model', 'optimizer')
+        self.register_best_state('model')
+        self.register_ema('model')
+
+    def build_dataloaders(self):
+        """Build audio dataloaders for each stage."""
+        self.dataloaders = builders.get_audio_datasets(self.cfg)
+
+    def show(self):
+        # TODO
+        raise NotImplementedError()
+
+    def run_step(self, idx: int, batch: torch.Tensor, metrics: dict):
+        """Perform one training or valid step on a given batch."""
+        x = batch.to(self.device)
+        loss_fun = F.mse_loss if self.cfg.loss.kind == 'mse' else F.l1_loss
+
+        condition = self.get_condition(x)  # [bs, 128, T/hop, n_emb]
+        sample = self.data_processor.process_data(x)
+
+        input_, target, step = self.schedule.get_training_item(sample,
+                                                               tensor_step=self.cfg.schedule.variable_step_batch)
+        out = self.model(input_, step, condition=condition).sample
+
+        base_loss = loss_fun(out, target, reduction='none').mean(dim=(1, 2))
+        reference_loss = loss_fun(input_, target, reduction='none').mean(dim=(1, 2))
+        loss = base_loss / reference_loss ** self.cfg.loss.norm_power
+
+        if self.is_training:
+            loss.mean().backward()
+            flashy.distrib.sync_model(self.model)
+            self.optimizer.step()
+            self.optimizer.zero_grad()
+        metrics = {
+            'loss': loss.mean(), 'normed_loss': (base_loss / reference_loss).mean(),
+            }
+        metrics.update(self.per_stage({'loss': loss, 'normed_loss': base_loss / reference_loss}, step))
+        metrics.update({
+            'std_in': input_.std(), 'std_out': out.std()})
+        return metrics
+
+    def run_epoch(self):
+        # reset random seed at the beginning of the epoch
+        self.rng = torch.Generator()
+        self.rng.manual_seed(1234 + self.epoch)
+        self.per_stage = PerStageMetrics(self.schedule.num_steps, self.cfg.metrics.num_stage)
+        # run epoch
+        super().run_epoch()
+
+    def evaluate(self):
+        """Evaluate stage.
+        Runs audio reconstruction evaluation.
+        """
+        self.model.eval()
+        evaluate_stage_name = f'{self.current_stage}'
+        loader = self.dataloaders['evaluate']
+        updates = len(loader)
+        lp = self.log_progress(f'{evaluate_stage_name} estimate', loader, total=updates, updates=self.log_updates)
+
+        metrics = {}
+        n = 1
+        for idx, batch in enumerate(lp):
+            x = batch.to(self.device)
+            with torch.no_grad():
+                y_pred = self.regenerate(x)
+
+            y_pred = y_pred.cpu()
+            y = batch.cpu()  # should already be on CPU but just in case
+            rvm = self.rvm(y_pred, y)
+            lp.update(**rvm)
+            if len(metrics) == 0:
+                metrics = rvm
+            else:
+                for key in rvm.keys():
+                    metrics[key] = (metrics[key] * n + rvm[key]) / (n + 1)
+        metrics = flashy.distrib.average_metrics(metrics)
+        return metrics
+
+    @torch.no_grad()
+    def regenerate(self, wav: torch.Tensor, step_list: tp.Optional[list] = None):
+        """Regenerate the given waveform."""
+        condition = self.get_condition(wav)
+        initial = self.schedule.get_initial_noise(self.data_processor.process_data(wav))  # sampling rate changes.
+        result = self.schedule.generate_subsampled(self.model, initial=initial, condition=condition,
+                                                   step_list=step_list)
+        result = self.data_processor.inverse_process(result)
+        return result
+
+    def generate(self):
+        """Generate stage."""
+        sample_manager = SampleManager(self.xp)
+        self.model.eval()
+        generate_stage_name = f'{self.current_stage}'
+
+        loader = self.dataloaders['generate']
+        updates = len(loader)
+        lp = self.log_progress(generate_stage_name, loader, total=updates, updates=self.log_updates)
+
+        for batch in lp:
+            reference, _ = batch
+            reference = reference.to(self.device)
+            estimate = self.regenerate(reference)
+            reference = reference.cpu()
+            estimate = estimate.cpu()
+            sample_manager.add_samples(estimate, self.epoch, ground_truth_wavs=reference)
+        flashy.distrib.barrier()
diff --git a/audiocraft/audiocraft/solvers/musicgen.py b/audiocraft/audiocraft/solvers/musicgen.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab2167b7958023274b04deedecc1b0b694dc83c7
--- /dev/null
+++ b/audiocraft/audiocraft/solvers/musicgen.py
@@ -0,0 +1,721 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from pathlib import Path
+import time
+import typing as tp
+
+import flashy
+import math
+import omegaconf
+import torch
+from torch.nn import functional as F
+
+from . import base, builders
+from .compression import CompressionSolver
+from .. import metrics as eval_metrics
+from .. import models
+from ..data.audio_dataset import AudioDataset
+from ..data.music_dataset import MusicDataset, MusicInfo, AudioInfo
+from ..data.audio_utils import normalize_audio
+from ..modules.conditioners import JointEmbedCondition, SegmentWithAttributes, WavCondition
+from ..utils.cache import CachedBatchWriter, CachedBatchLoader
+from ..utils.samples.manager import SampleManager
+from ..utils.utils import get_dataset_from_loader, is_jsonable, warn_once
+
+
+class MusicGenSolver(base.StandardSolver):
+    """Solver for MusicGen training task.
+
+    Used in: https://arxiv.org/abs/2306.05284
+    """
+    DATASET_TYPE: builders.DatasetType = builders.DatasetType.MUSIC
+
+    def __init__(self, cfg: omegaconf.DictConfig):
+        super().__init__(cfg)
+        # easier access to sampling parameters
+        self.generation_params = {
+            'use_sampling': self.cfg.generate.lm.use_sampling,
+            'temp': self.cfg.generate.lm.temp,
+            'top_k': self.cfg.generate.lm.top_k,
+            'top_p': self.cfg.generate.lm.top_p,
+        }
+        self._best_metric_name: tp.Optional[str] = 'ce'
+
+        self._cached_batch_writer = None
+        self._cached_batch_loader = None
+        if cfg.cache.path:
+            if cfg.cache.write:
+                self._cached_batch_writer = CachedBatchWriter(Path(cfg.cache.path))
+                if self.cfg.cache.write_num_shards:
+                    self.logger.warning("Multiple shard cache, best_metric_name will be set to None.")
+                    self._best_metric_name = None
+            else:
+                self._cached_batch_loader = CachedBatchLoader(
+                    Path(cfg.cache.path), cfg.dataset.batch_size, cfg.dataset.num_workers,
+                    min_length=self.cfg.optim.updates_per_epoch or 1)
+                self.dataloaders['original_train'] = self.dataloaders['train']
+                self.dataloaders['train'] = self._cached_batch_loader  # type: ignore
+
+    @staticmethod
+    def get_eval_solver_from_sig(sig: str, dtype: tp.Optional[str] = None,
+                                 device: tp.Optional[str] = None, autocast: bool = True,
+                                 batch_size: tp.Optional[int] = None,
+                                 override_cfg: tp.Optional[tp.Union[dict, omegaconf.DictConfig]] = None,
+                                 **kwargs):
+        """Mostly a convenience function around magma.train.get_solver_from_sig,
+        populating all the proper param, deactivating EMA, FSDP, loading the best state,
+        basically all you need to get a solver ready to "play" with in single GPU mode
+        and with minimal memory overhead.
+
+        Args:
+            sig (str): signature to load.
+            dtype (str or None): potential dtype, as a string, i.e. 'float16'.
+            device (str or None): potential device, as a string, i.e. 'cuda'.
+            override_cfg (dict or omegaconf.DictConfig or None): potential device, as a string, i.e. 'cuda'.
+        """
+        from audiocraft import train
+        our_override_cfg: tp.Dict[str, tp.Any] = {'optim': {'ema': {'use': False}}}
+        our_override_cfg['autocast'] = autocast
+        if dtype is not None:
+            our_override_cfg['dtype'] = dtype
+        if device is not None:
+            our_override_cfg['device'] = device
+        if batch_size is not None:
+            our_override_cfg['dataset'] = {'batch_size': batch_size}
+        if override_cfg is None:
+            override_cfg = {}
+        override_cfg = omegaconf.OmegaConf.merge(
+            omegaconf.DictConfig(override_cfg), omegaconf.DictConfig(our_override_cfg))  # type: ignore
+        solver = train.get_solver_from_sig(
+            sig, override_cfg=override_cfg,
+            load_best=True, disable_fsdp=True,
+            ignore_state_keys=['optimizer', 'ema'], **kwargs)
+        solver.model.eval()
+        return solver
+
+    def get_formatter(self, stage_name: str) -> flashy.Formatter:
+        return flashy.Formatter({
+            'lr': '.2E',
+            'ce': '.3f',
+            'ppl': '.3f',
+            'grad_norm': '.3E',
+        }, exclude_keys=['ce_q*', 'ppl_q*'])
+
+    @property
+    def best_metric_name(self) -> tp.Optional[str]:
+        return self._best_metric_name
+
+    def build_model(self) -> None:
+        """Instantiate models and optimizer."""
+        # we can potentially not use all quantizers with which the EnCodec model was trained
+        # (e.g. we trained the model with quantizers dropout)
+        self.compression_model = CompressionSolver.wrapped_model_from_checkpoint(
+            self.cfg, self.cfg.compression_model_checkpoint, device=self.device)
+        assert self.compression_model.sample_rate == self.cfg.sample_rate, (
+            f"Compression model sample rate is {self.compression_model.sample_rate} but "
+            f"Solver sample rate is {self.cfg.sample_rate}."
+            )
+        # ensure we have matching configuration between LM and compression model
+        assert self.cfg.transformer_lm.card == self.compression_model.cardinality, (
+            "Cardinalities of the LM and compression model don't match: ",
+            f"LM cardinality is {self.cfg.transformer_lm.card} vs ",
+            f"compression model cardinality is {self.compression_model.cardinality}"
+        )
+        #assert self.cfg.transformer_lm.n_q == self.compression_model.num_codebooks, (
+        #    "Numbers of codebooks of the LM and compression models don't match: ",
+        #    f"LM number of codebooks is {self.cfg.transformer_lm.n_q} vs ",
+        #    f"compression model numer of codebooks is {self.compression_model.num_codebooks}"
+        #)
+        self.logger.info("Compression model has %d codebooks with %d cardinality, and a framerate of %d",
+                         self.compression_model.num_codebooks, self.compression_model.cardinality,
+                         self.compression_model.frame_rate)
+        # instantiate LM model
+        self.model: models.LMModel = models.builders.get_lm_model(self.cfg).to(self.device)
+        if self.cfg.fsdp.use:
+            assert not self.cfg.autocast, "Cannot use autocast with fsdp"
+            self.model = self.wrap_with_fsdp(self.model)
+        
+        # freeze some weight
+        for name, param in self.model.named_parameters():
+            param.requires_grad = False
+        
+        layer_idxs = [idx for idx in range(0, 48, 4)] # jump freeze
+        for name, param in self.model.named_parameters():
+            for idx in layer_idxs:
+                if name.startswith(f"transformer.layers.{idx}."):
+                    param.requires_grad = True
+            if name.startswith("out_norm") or name.startswith("linears"):
+                    param.requires_grad = True
+            if name.startswith("condition_provider.conditioners.chord") or name.startswith("condition_provider.conditioners.beat"):
+                    param.requires_grad = True
+            # if name.startswith("condition_provider.conditioners.beat"):
+            #         param.requires_grad = True
+            # if name.startswith("emb"):
+            #         param.requires_grad = True
+        
+        self.register_ema('model')
+        # initialize optimization
+        self.optimizer = builders.get_optimizer(builders.get_optim_parameter_groups(self.model), self.cfg.optim)
+        self.lr_scheduler = builders.get_lr_scheduler(self.optimizer, self.cfg.schedule, self.total_updates)
+        self.register_stateful('compression_model', 'model', 'optimizer', 'lr_scheduler')
+        self.register_best_state('model')
+        self.autocast_dtype = {
+            'float16': torch.float16, 'bfloat16': torch.bfloat16
+        }[self.cfg.autocast_dtype]
+        self.scaler: tp.Optional[torch.cuda.amp.GradScaler] = None
+        if self.cfg.fsdp.use:
+            need_scaler = self.cfg.fsdp.param_dtype == 'float16'
+        else:
+            need_scaler = self.cfg.autocast and self.autocast_dtype is torch.float16
+        if need_scaler:
+            if self.cfg.fsdp.use:
+                from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler
+                self.scaler = ShardedGradScaler()  # type: ignore
+            else:
+                self.scaler = torch.cuda.amp.GradScaler()
+            self.register_stateful('scaler')
+
+    def build_dataloaders(self) -> None:
+        """Instantiate audio dataloaders for each stage."""
+        self.dataloaders = builders.get_audio_datasets(self.cfg, dataset_type=self.DATASET_TYPE)
+
+    def show(self) -> None:
+        """Show the compression model and LM model."""
+        self.logger.info("Compression model:")
+        self.log_model_summary(self.compression_model)
+        self.logger.info("LM model:")
+        self.log_model_summary(self.model)
+
+    def load_state_dict(self, state: dict) -> None:
+        if 'condition_provider' in state:
+            model_state = state['model']
+            condition_provider_state = state.pop('condition_provider')
+            prefix = 'condition_provider.'
+            for key, value in condition_provider_state.items():
+                key = prefix + key
+                assert key not in model_state
+                model_state[key] = value
+        super().load_state_dict(state)
+
+    def load_from_pretrained(self, name: str):
+        # TODO: support native HF versions of MusicGen.
+        lm_pkg = models.loaders.load_lm_model_ckpt(name)
+        state: dict = {
+            'best_state': {
+                'model': lm_pkg['best_state'],
+            },
+        }
+        return state
+
+    def _compute_cross_entropy(
+        self, logits: torch.Tensor, targets: torch.Tensor, mask: torch.Tensor
+    ) -> tp.Tuple[torch.Tensor, tp.List[torch.Tensor]]:
+        """Compute cross entropy between multi-codebook targets and model's logits.
+        The cross entropy is computed per codebook to provide codebook-level cross entropy.
+        Valid timesteps for each of the codebook are pulled from the mask, where invalid
+        timesteps are set to 0.
+
+        Args:
+            logits (torch.Tensor): Model's logits of shape [B, K, T, card].
+            targets (torch.Tensor): Target codes, of shape [B, K, T].
+            mask (torch.Tensor): Mask for valid target codes, of shape [B, K, T].
+        Returns:
+            ce (torch.Tensor): Cross entropy averaged over the codebooks
+            ce_per_codebook (list of torch.Tensor): Cross entropy per codebook (detached).
+        """
+        B, K, T = targets.shape
+        assert logits.shape[:-1] == targets.shape
+        assert mask.shape == targets.shape
+        ce = torch.zeros([], device=targets.device)
+        ce_per_codebook: tp.List[torch.Tensor] = []
+        for k in range(K):
+            logits_k = logits[:, k, ...].contiguous().view(-1, logits.size(-1))  # [B x T, card]
+            targets_k = targets[:, k, ...].contiguous().view(-1)  # [B x T]
+            mask_k = mask[:, k, ...].contiguous().view(-1)  # [B x T]
+            ce_targets = targets_k[mask_k]
+            ce_logits = logits_k[mask_k]
+            q_ce = F.cross_entropy(ce_logits, ce_targets)
+            ce += q_ce
+            ce_per_codebook.append(q_ce.detach())
+        # average cross entropy across codebooks
+        ce = ce / K
+        return ce, ce_per_codebook
+
+    @torch.no_grad()
+    def _prepare_tokens_and_attributes(
+        self, batch: tp.Tuple[torch.Tensor, tp.List[SegmentWithAttributes]],
+        check_synchronization_points: bool = False
+    ) -> tp.Tuple[dict, torch.Tensor, torch.Tensor]:
+        """Prepare input batchs for language model training.
+
+        Args:
+            batch (tuple[torch.Tensor, list[SegmentWithAttributes]]): Input batch with audio tensor of shape [B, C, T]
+                and corresponding metadata as SegmentWithAttributes (with B items).
+            check_synchronization_points (bool): Whether to check for synchronization points slowing down training.
+        Returns:
+            Condition tensors (dict[str, any]): Preprocessed condition attributes.
+            Tokens (torch.Tensor): Audio tokens from compression model, of shape [B, K, T_s],
+                with B the batch size, K the number of codebooks, T_s the token timesteps.
+            Padding mask (torch.Tensor): Mask with valid positions in the tokens tensor, of shape [B, K, T_s].
+        """
+        if self._cached_batch_loader is None or self.current_stage != "train":
+            audio, infos = batch
+            audio = audio.to(self.device)
+            audio_tokens = None
+            assert audio.size(0) == len(infos), (
+                f"Mismatch between number of items in audio batch ({audio.size(0)})",
+                f" and in metadata ({len(infos)})"
+            )
+        else:
+            audio = None
+            # In that case the batch will be a tuple coming from the _cached_batch_writer bit below.
+            infos, = batch  # type: ignore
+            assert all([isinstance(info, AudioInfo) for info in infos])
+            assert all([info.audio_tokens is not None for info in infos])  # type: ignore
+            audio_tokens = torch.stack([info.audio_tokens for info in infos]).to(self.device)  # type: ignore
+            audio_tokens = audio_tokens.long()
+            for info in infos:
+                if isinstance(info, MusicInfo):
+                    # Careful here, if you want to use this condition_wav (e.b. chroma conditioning),
+                    # then you must be using the chroma cache! otherwise the code will try
+                    # to use this segment and fail (by that I mean you will see NaN everywhere).
+                    info.self_wav = WavCondition(
+                        torch.full([1, info.channels, info.total_frames], float('NaN')),
+                        length=torch.tensor([info.n_frames]),
+                        sample_rate=[info.sample_rate],
+                        path=[info.meta.path],
+                        seek_time=[info.seek_time])
+                    dataset = get_dataset_from_loader(self.dataloaders['original_train'])
+                    assert isinstance(dataset, MusicDataset), type(dataset)
+                    if dataset.paraphraser is not None and info.description is not None:
+                        # Hackingly reapplying paraphraser when using cache.
+                        info.description = dataset.paraphraser.sample_paraphrase(
+                            info.meta.path, info.description)
+        # prepare attributes
+        attributes = [info.to_condition_attributes() for info in infos]
+        attributes = self.model.cfg_dropout(attributes)
+        attributes = self.model.att_dropout(attributes)
+        tokenized = self.model.condition_provider.tokenize(attributes)
+
+        # Now we should be synchronization free.
+        if self.device == "cuda" and check_synchronization_points:
+            torch.cuda.set_sync_debug_mode("warn")
+
+        if audio_tokens is None:
+            with torch.no_grad():
+                audio_tokens, scale = self.compression_model.encode(audio)
+                assert scale is None, "Scaled compression model not supported with LM."
+
+        with self.autocast:
+            condition_tensors = self.model.condition_provider(tokenized)
+
+        # create a padding mask to hold valid vs invalid positions
+        padding_mask = torch.ones_like(audio_tokens, dtype=torch.bool, device=audio_tokens.device)
+        # replace encodec tokens from padded audio with special_token_id
+        if self.cfg.tokens.padding_with_special_token:
+            audio_tokens = audio_tokens.clone()
+            padding_mask = padding_mask.clone()
+            token_sample_rate = self.compression_model.frame_rate
+            B, K, T_s = audio_tokens.shape
+            for i in range(B):
+                n_samples = infos[i].n_frames
+                audio_sample_rate = infos[i].sample_rate
+                # take the last token generated from actual audio frames (non-padded audio)
+                valid_tokens = math.floor(float(n_samples) / audio_sample_rate * token_sample_rate)
+                audio_tokens[i, :, valid_tokens:] = self.model.special_token_id
+                padding_mask[i, :, valid_tokens:] = 0
+
+        if self.device == "cuda" and check_synchronization_points:
+            torch.cuda.set_sync_debug_mode("default")
+
+        if self._cached_batch_writer is not None and self.current_stage == 'train':
+            assert self._cached_batch_loader is None
+            assert audio_tokens is not None
+            for info, one_audio_tokens in zip(infos, audio_tokens):
+                assert isinstance(info, AudioInfo)
+                if isinstance(info, MusicInfo):
+                    assert not info.joint_embed, "joint_embed and cache not supported yet."
+                    info.self_wav = None
+                assert one_audio_tokens.max() < 2**15, one_audio_tokens.max().item()
+                info.audio_tokens = one_audio_tokens.short().cpu()
+            self._cached_batch_writer.save(infos)
+
+        return condition_tensors, audio_tokens, padding_mask
+
+    def run_step(self, idx: int, batch: tp.Tuple[torch.Tensor, tp.List[SegmentWithAttributes]], metrics: dict) -> dict:
+        """Perform one training or valid step on a given batch."""
+        check_synchronization_points = idx == 1 and self.device == 'cuda'
+
+        condition_tensors, audio_tokens, padding_mask = self._prepare_tokens_and_attributes(
+            batch, check_synchronization_points)
+
+        self.deadlock_detect.update('tokens_and_conditions')
+
+        if check_synchronization_points:
+            torch.cuda.set_sync_debug_mode('warn')
+
+        with self.autocast:
+            model_output = self.model.compute_predictions(audio_tokens, [], condition_tensors)  # type: ignore
+            logits = model_output.logits
+            mask = padding_mask & model_output.mask
+            ce, ce_per_codebook = self._compute_cross_entropy(logits, audio_tokens, mask)
+            loss = ce
+        self.deadlock_detect.update('loss')
+
+        if check_synchronization_points:
+            torch.cuda.set_sync_debug_mode('default')
+
+        if self.is_training:
+            metrics['lr'] = self.optimizer.param_groups[0]['lr']
+            if self.scaler is not None:
+                loss = self.scaler.scale(loss)
+            self.deadlock_detect.update('scale')
+            # apply grad accum
+            loss = loss / self.cfg.optim.grad_accum_steps
+            if self.cfg.fsdp.use:
+                loss.backward()
+                flashy.distrib.average_tensors(self.model.buffers())
+            elif self.cfg.optim.eager_sync:
+                with flashy.distrib.eager_sync_model(self.model):
+                    loss.backward()
+            else:
+                # this should always be slower but can be useful
+                # for weird use cases like multiple backwards.
+                loss.backward()
+                flashy.distrib.sync_model(self.model)
+            self.deadlock_detect.update('backward')
+
+            if idx % self.cfg.optim.grad_accum_steps == 0:
+                if self.scaler is not None:
+                    self.scaler.unscale_(self.optimizer)
+                if self.cfg.optim.max_norm:
+                    if self.cfg.fsdp.use:
+                        metrics['grad_norm'] = self.model.clip_grad_norm_(self.cfg.optim.max_norm)  # type: ignore
+                    else:
+                        metrics['grad_norm'] = torch.nn.utils.clip_grad_norm_(
+                            self.model.parameters(), self.cfg.optim.max_norm
+                        )
+                if self.scaler is None:
+                    self.optimizer.step()
+                else:
+                    self.scaler.step(self.optimizer)
+                    self.scaler.update()
+                if self.lr_scheduler:
+                    self.lr_scheduler.step()
+                self.optimizer.zero_grad()
+                self.deadlock_detect.update('optim')
+                if self.scaler is not None:
+                    scale = self.scaler.get_scale()
+                    metrics['grad_scale'] = scale
+                if not loss.isfinite().all():
+                    raise RuntimeError("Model probably diverged.")
+
+        metrics['ce'] = ce
+        metrics['ppl'] = torch.exp(ce)
+        for k, ce_q in enumerate(ce_per_codebook):
+            metrics[f'ce_q{k + 1}'] = ce_q
+            metrics[f'ppl_q{k + 1}'] = torch.exp(ce_q)
+
+        return metrics
+
+    @torch.no_grad()
+    def run_generate_step(self, batch: tp.Tuple[torch.Tensor, tp.List[SegmentWithAttributes]],
+                          gen_duration: float, prompt_duration: tp.Optional[float] = None,
+                          remove_prompt: bool = False,
+                          **generation_params) -> dict:
+        """Run generate step on a batch of optional audio tensor and corresponding attributes.
+
+        Args:
+            batch (tuple[torch.Tensor, list[SegmentWithAttributes]]):
+            use_prompt (bool): Whether to do audio continuation generation with prompt from audio batch.
+            gen_duration (float): Target audio duration for the generation.
+            prompt_duration (float, optional): Duration for the audio prompt to use for continuation.
+            remove_prompt (bool, optional): Whether to remove the prompt from the generated audio.
+            generation_params: Additional generation parameters.
+        Returns:
+            gen_outputs (dict): Generation outputs, consisting in audio, audio tokens from both the generation
+                and the prompt along with additional information.
+        """
+        bench_start = time.time()
+        audio, meta = batch
+        assert audio.size(0) == len(meta), (
+            f"Mismatch between number of items in audio batch ({audio.size(0)})",
+            f" and in metadata ({len(meta)})"
+        )
+        # prepare attributes
+        attributes = [x.to_condition_attributes() for x in meta]
+        # TODO: Add dropout for chroma?
+
+        # prepare audio prompt
+        if prompt_duration is None:
+            prompt_audio = None
+        else:
+            assert prompt_duration < gen_duration, "Prompt duration must be lower than target generation duration"
+            prompt_audio_frames = int(prompt_duration * self.compression_model.sample_rate)
+            prompt_audio = audio[..., :prompt_audio_frames]
+
+        # get audio tokens from compression model
+        if prompt_audio is None or prompt_audio.nelement() == 0:
+            num_samples = len(attributes)
+            prompt_tokens = None
+        else:
+            num_samples = None
+            prompt_audio = prompt_audio.to(self.device)
+            prompt_tokens, scale = self.compression_model.encode(prompt_audio)
+            assert scale is None, "Compression model in MusicGen should not require rescaling."
+
+        # generate by sampling from the LM
+        with self.autocast:
+            total_gen_len = math.ceil(gen_duration * self.compression_model.frame_rate)
+            gen_tokens = self.model.generate(
+                prompt_tokens, attributes, max_gen_len=total_gen_len,
+                num_samples=num_samples, **self.generation_params)
+
+        # generate audio from tokens
+        assert gen_tokens.dim() == 3
+        gen_audio = self.compression_model.decode(gen_tokens, None)
+
+        bench_end = time.time()
+        gen_outputs = {
+            'rtf': (bench_end - bench_start) / gen_duration,
+            'ref_audio': audio,
+            'gen_audio': gen_audio,
+            'gen_tokens': gen_tokens,
+            'prompt_audio': prompt_audio,
+            'prompt_tokens': prompt_tokens,
+        }
+        return gen_outputs
+
+    def generate_audio(self) -> dict:
+        """Audio generation stage."""
+        generate_stage_name = f'{self.current_stage}'
+        sample_manager = SampleManager(self.xp)
+        self.logger.info(f"Generating samples in {sample_manager.base_folder}")
+        loader = self.dataloaders['generate']
+        updates = len(loader)
+        lp = self.log_progress(generate_stage_name, loader, total=updates, updates=self.log_updates)
+
+        dataset = get_dataset_from_loader(loader)
+        dataset_duration = dataset.segment_duration
+        assert dataset_duration is not None
+        assert isinstance(dataset, AudioDataset)
+        target_duration = self.cfg.generate.lm.gen_duration
+        prompt_duration = self.cfg.generate.lm.prompt_duration
+        if target_duration is None:
+            target_duration = dataset_duration
+        if prompt_duration is None:
+            prompt_duration = dataset_duration / 4
+        assert prompt_duration < dataset_duration, (
+            f"Specified prompt duration ({prompt_duration}s) is longer",
+            f" than reference audio duration ({dataset_duration}s)"
+        )
+
+        def get_hydrated_conditions(meta: tp.List[SegmentWithAttributes]):
+            hydrated_conditions = []
+            for sample in [x.to_condition_attributes() for x in meta]:
+                cond_dict = {}
+                for cond_type in sample.__annotations__.keys():
+                    for cond_key, cond_val in getattr(sample, cond_type).items():
+                        if cond_key not in self.model.condition_provider.conditioners.keys():
+                            continue
+                        if is_jsonable(cond_val):
+                            cond_dict[cond_key] = cond_val
+                        elif isinstance(cond_val, WavCondition):
+                            cond_dict[cond_key] = cond_val.path
+                        elif isinstance(cond_val, JointEmbedCondition):
+                            cond_dict[cond_key] = cond_val.text  # only support text at inference for now
+                        else:
+                            # if we reached this point, it is not clear how to log the condition
+                            # so we just log the type.
+                            cond_dict[cond_key] = str(type(cond_val))
+                            continue
+                hydrated_conditions.append(cond_dict)
+            return hydrated_conditions
+
+        metrics: dict = {}
+        average = flashy.averager()
+        for batch in lp:
+            audio, meta = batch
+            # metadata for sample manager
+            hydrated_conditions = get_hydrated_conditions(meta)
+            sample_generation_params = {
+                **{f'classifier_free_guidance_{k}': v for k, v in self.cfg.classifier_free_guidance.items()},
+                **self.generation_params
+            }
+            if self.cfg.generate.lm.unprompted_samples:
+                if self.cfg.generate.lm.gen_gt_samples:
+                    # get the ground truth instead of generation
+                    self.logger.warn(
+                        "Use ground truth instead of audio generation as generate.lm.gen_gt_samples=true")
+                    gen_unprompted_audio = audio
+                    rtf = 1.
+                else:
+                    gen_unprompted_outputs = self.run_generate_step(
+                        batch, gen_duration=target_duration, prompt_duration=prompt_duration,
+                        **self.generation_params)
+                    gen_unprompted_audio = gen_unprompted_outputs['gen_audio'].cpu()
+                    rtf = gen_unprompted_outputs['rtf']
+                sample_manager.add_samples(
+                    gen_unprompted_audio, self.epoch, hydrated_conditions,
+                    ground_truth_wavs=audio, generation_args=sample_generation_params)
+
+            if self.cfg.generate.lm.prompted_samples:
+                gen_outputs = self.run_generate_step(
+                    batch, gen_duration=target_duration, prompt_duration=prompt_duration,
+                    **self.generation_params)
+                gen_audio = gen_outputs['gen_audio'].cpu()
+                prompt_audio = gen_outputs['prompt_audio'].cpu()
+                sample_manager.add_samples(
+                    gen_audio, self.epoch, hydrated_conditions,
+                    prompt_wavs=prompt_audio, ground_truth_wavs=audio,
+                    generation_args=sample_generation_params)
+
+            metrics['rtf'] = rtf
+            metrics = average(metrics)
+
+        flashy.distrib.barrier()
+        return metrics
+
+    def generate(self) -> dict:
+        """Generate stage."""
+        self.model.eval()
+        with torch.no_grad():
+            return self.generate_audio()
+
+    def run_epoch(self):
+        if self.cfg.cache.write:
+            if ((self.epoch - 1) % self.cfg.cache.write_num_shards) != self.cfg.cache.write_shard:
+                return
+        super().run_epoch()
+
+    def train(self):
+        """Train stage.
+        """
+        if self._cached_batch_writer is not None:
+            self._cached_batch_writer.start_epoch(self.epoch)
+        if self._cached_batch_loader is None:
+            dataset = get_dataset_from_loader(self.dataloaders['train'])
+            assert isinstance(dataset, AudioDataset)
+            dataset.current_epoch = self.epoch
+        else:
+            self._cached_batch_loader.start_epoch(self.epoch)
+        return super().train()
+
+    def evaluate_audio_generation(self) -> dict:
+        """Evaluate audio generation with off-the-shelf metrics."""
+        evaluate_stage_name = f'{self.current_stage}_generation'
+        # instantiate evaluation metrics, if at least one metric is defined, run audio generation evaluation
+        fad: tp.Optional[eval_metrics.FrechetAudioDistanceMetric] = None
+        kldiv: tp.Optional[eval_metrics.KLDivergenceMetric] = None
+        text_consistency: tp.Optional[eval_metrics.TextConsistencyMetric] = None
+        chroma_cosine: tp.Optional[eval_metrics.ChromaCosineSimilarityMetric] = None
+        should_run_eval = False
+        eval_chroma_wavs: tp.Optional[torch.Tensor] = None
+        if self.cfg.evaluate.metrics.fad:
+            fad = builders.get_fad(self.cfg.metrics.fad).to(self.device)
+            should_run_eval = True
+        if self.cfg.evaluate.metrics.kld:
+            kldiv = builders.get_kldiv(self.cfg.metrics.kld).to(self.device)
+            should_run_eval = True
+        if self.cfg.evaluate.metrics.text_consistency:
+            text_consistency = builders.get_text_consistency(self.cfg.metrics.text_consistency).to(self.device)
+            should_run_eval = True
+        if self.cfg.evaluate.metrics.chroma_cosine:
+            chroma_cosine = builders.get_chroma_cosine_similarity(self.cfg.metrics.chroma_cosine).to(self.device)
+            # if we have predefind wavs for chroma we should purge them for computing the cosine metric
+            has_predefined_eval_chromas = 'self_wav' in self.model.condition_provider.conditioners and \
+                                          self.model.condition_provider.conditioners['self_wav'].has_eval_wavs()
+            if has_predefined_eval_chromas:
+                warn_once(self.logger, "Attempting to run cosine eval for config with pre-defined eval chromas! "
+                                       'Resetting eval chromas to None for evaluation.')
+                eval_chroma_wavs = self.model.condition_provider.conditioners.self_wav.eval_wavs  # type: ignore
+                self.model.condition_provider.conditioners.self_wav.reset_eval_wavs(None)  # type: ignore
+            should_run_eval = True
+
+        def get_compressed_audio(audio: torch.Tensor) -> torch.Tensor:
+            audio_tokens, scale = self.compression_model.encode(audio.to(self.device))
+            compressed_audio = self.compression_model.decode(audio_tokens, scale)
+            return compressed_audio[..., :audio.shape[-1]]
+
+        metrics: dict = {}
+        if should_run_eval:
+            loader = self.dataloaders['evaluate']
+            updates = len(loader)
+            lp = self.log_progress(f'{evaluate_stage_name} inference', loader, total=updates, updates=self.log_updates)
+            average = flashy.averager()
+            dataset = get_dataset_from_loader(loader)
+            assert isinstance(dataset, AudioDataset)
+            self.logger.info(f"Computing evaluation metrics on {len(dataset)} samples")
+
+            for idx, batch in enumerate(lp):
+                audio, meta = batch
+                assert all([self.cfg.sample_rate == m.sample_rate for m in meta])
+
+                target_duration = audio.shape[-1] / self.cfg.sample_rate
+                if self.cfg.evaluate.fixed_generation_duration:
+                    target_duration = self.cfg.evaluate.fixed_generation_duration
+
+                gen_outputs = self.run_generate_step(
+                    batch, gen_duration=target_duration,
+                    **self.generation_params
+                )
+                y_pred = gen_outputs['gen_audio'].detach()
+                y_pred = y_pred[..., :audio.shape[-1]]
+
+                normalize_kwargs = dict(self.cfg.generate.audio)
+                normalize_kwargs.pop('format', None)
+                y_pred = torch.stack([normalize_audio(w, **normalize_kwargs) for w in y_pred], dim=0).cpu()
+                y = audio.cpu()  # should already be on CPU but just in case
+                sizes = torch.tensor([m.n_frames for m in meta])  # actual sizes without padding
+                sample_rates = torch.tensor([m.sample_rate for m in meta])  # sample rates for audio samples
+                audio_stems = [Path(m.meta.path).stem + f"_{m.seek_time}" for m in meta]
+
+                if fad is not None:
+                    if self.cfg.metrics.fad.use_gt:
+                        y_pred = get_compressed_audio(y).cpu()
+                    fad.update(y_pred, y, sizes, sample_rates, audio_stems)
+                if kldiv is not None:
+                    if self.cfg.metrics.kld.use_gt:
+                        y_pred = get_compressed_audio(y).cpu()
+                    kldiv.update(y_pred, y, sizes, sample_rates)
+                if text_consistency is not None:
+                    texts = [m.description for m in meta]
+                    if self.cfg.metrics.text_consistency.use_gt:
+                        y_pred = y
+                    text_consistency.update(y_pred, texts, sizes, sample_rates)
+                if chroma_cosine is not None:
+                    if self.cfg.metrics.chroma_cosine.use_gt:
+                        y_pred = get_compressed_audio(y).cpu()
+                    chroma_cosine.update(y_pred, y, sizes, sample_rates)
+                    # restore chroma conditioner's eval chroma wavs
+                    if eval_chroma_wavs is not None:
+                        self.model.condition_provider.conditioners['self_wav'].reset_eval_wavs(eval_chroma_wavs)
+
+            flashy.distrib.barrier()
+            if fad is not None:
+                metrics['fad'] = fad.compute()
+            if kldiv is not None:
+                kld_metrics = kldiv.compute()
+                metrics.update(kld_metrics)
+            if text_consistency is not None:
+                metrics['text_consistency'] = text_consistency.compute()
+            if chroma_cosine is not None:
+                metrics['chroma_cosine'] = chroma_cosine.compute()
+            metrics = average(metrics)
+            metrics = flashy.distrib.average_metrics(metrics, len(loader))
+
+        return metrics
+
+    def evaluate(self) -> dict:
+        """Evaluate stage."""
+        self.model.eval()
+        with torch.no_grad():
+            metrics: dict = {}
+            if self.cfg.evaluate.metrics.base:
+                metrics.update(self.common_train_valid('evaluate'))
+            gen_metrics = self.evaluate_audio_generation()
+            return {**metrics, **gen_metrics}
diff --git a/audiocraft/audiocraft/train.py b/audiocraft/audiocraft/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..22dd117830bb403829d0a60b1b95e120d1e6978b
--- /dev/null
+++ b/audiocraft/audiocraft/train.py
@@ -0,0 +1,157 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Entry point for dora to launch solvers for running training loops.
+See more info on how to use dora: https://github.com/facebookresearch/dora
+"""
+
+import logging
+import multiprocessing
+import os
+import sys
+import typing as tp
+
+from dora import git_save, hydra_main, XP
+import flashy
+import hydra
+import omegaconf
+
+from .environment import AudioCraftEnvironment
+from .utils.cluster import get_slurm_parameters
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_config_dset_paths(cfg):
+    """Enable Dora to load manifest from git clone repository."""
+    # manifest files for the different splits
+    for key, value in cfg.datasource.items():
+        if isinstance(value, str):
+            cfg.datasource[key] = git_save.to_absolute_path(value)
+
+
+def get_solver(cfg):
+    from . import solvers
+    # Convert batch size to batch size for each GPU
+    assert cfg.dataset.batch_size % flashy.distrib.world_size() == 0
+    cfg.dataset.batch_size //= flashy.distrib.world_size()
+    for split in ['train', 'valid', 'evaluate', 'generate']:
+        if hasattr(cfg.dataset, split) and hasattr(cfg.dataset[split], 'batch_size'):
+            assert cfg.dataset[split].batch_size % flashy.distrib.world_size() == 0
+            cfg.dataset[split].batch_size //= flashy.distrib.world_size()
+    resolve_config_dset_paths(cfg)
+    solver = solvers.get_solver(cfg)
+    return solver
+
+
+def get_solver_from_xp(xp: XP, override_cfg: tp.Optional[tp.Union[dict, omegaconf.DictConfig]] = None,
+                       restore: bool = True, load_best: bool = True,
+                       ignore_state_keys: tp.List[str] = [], disable_fsdp: bool = True):
+    """Given a XP, return the Solver object.
+
+    Args:
+        xp (XP): Dora experiment for which to retrieve the solver.
+        override_cfg (dict or None): If not None, should be a dict used to
+            override some values in the config of `xp`. This will not impact
+            the XP signature or folder. The format is different
+            than the one used in Dora grids, nested keys should actually be nested dicts,
+            not flattened, e.g. `{'optim': {'batch_size': 32}}`.
+        restore (bool): If `True` (the default), restore state from the last checkpoint.
+        load_best (bool): If `True` (the default), load the best state from the checkpoint.
+        ignore_state_keys (list[str]): List of sources to ignore when loading the state, e.g. `optimizer`.
+        disable_fsdp (bool): if True, disables FSDP entirely. This will
+            also automatically skip loading the EMA. For solver specific
+            state sources, like the optimizer, you might want to
+            use along `ignore_state_keys=['optimizer']`. Must be used with `load_best=True`.
+    """
+    logger.info(f"Loading solver from XP {xp.sig}. "
+                f"Overrides used: {xp.argv}")
+    cfg = xp.cfg
+    if override_cfg is not None:
+        cfg = omegaconf.OmegaConf.merge(cfg, omegaconf.DictConfig(override_cfg))
+    if disable_fsdp and cfg.fsdp.use:
+        cfg.fsdp.use = False
+        assert load_best is True
+        # ignoring some keys that were FSDP sharded like model, ema, and best_state.
+        # fsdp_best_state will be used in that case. When using a specific solver,
+        # one is responsible for adding the relevant keys, e.g. 'optimizer'.
+        # We could make something to automatically register those inside the solver, but that
+        # seem overkill at this point.
+        ignore_state_keys = ignore_state_keys + ['model', 'ema', 'best_state']
+
+    try:
+        with xp.enter():
+            solver = get_solver(cfg)
+            if restore:
+                solver.restore(load_best=load_best, ignore_state_keys=ignore_state_keys)
+        return solver
+    finally:
+        hydra.core.global_hydra.GlobalHydra.instance().clear()
+
+
+def get_solver_from_sig(sig: str, *args, **kwargs):
+    """Return Solver object from Dora signature, i.e. to play with it from a notebook.
+    See `get_solver_from_xp` for more information.
+    """
+    xp = main.get_xp_from_sig(sig)
+    return get_solver_from_xp(xp, *args, **kwargs)
+
+
+def init_seed_and_system(cfg):
+    import numpy as np
+    import torch
+    import random
+    from audiocraft.modules.transformer import set_efficient_attention_backend
+
+    multiprocessing.set_start_method(cfg.mp_start_method)
+    logger.debug('Setting mp start method to %s', cfg.mp_start_method)
+    random.seed(cfg.seed)
+    np.random.seed(cfg.seed)
+    # torch also initialize cuda seed if available
+    torch.manual_seed(cfg.seed)
+    torch.set_num_threads(cfg.num_threads)
+    os.environ['MKL_NUM_THREADS'] = str(cfg.num_threads)
+    os.environ['OMP_NUM_THREADS'] = str(cfg.num_threads)
+    logger.debug('Setting num threads to %d', cfg.num_threads)
+    set_efficient_attention_backend(cfg.efficient_attention_backend)
+    logger.debug('Setting efficient attention backend to %s', cfg.efficient_attention_backend)
+
+
+@hydra_main(config_path='../config', config_name='config', version_base='1.1')
+def main(cfg):
+    init_seed_and_system(cfg)
+
+    # Setup logging both to XP specific folder, and to stderr.
+    log_name = '%s.log.{rank}' % cfg.execute_only if cfg.execute_only else 'solver.log.{rank}'
+    flashy.setup_logging(level=str(cfg.logging.level).upper(), log_name=log_name)
+    # Initialize distributed training, no need to specify anything when using Dora.
+    flashy.distrib.init()
+    solver = get_solver(cfg)
+    if cfg.show:
+        solver.show()
+        return
+
+    if cfg.execute_only:
+        assert cfg.execute_inplace or cfg.continue_from is not None, \
+            "Please explicitly specify the checkpoint to continue from with continue_from=<sig_or_path> " + \
+            "when running with execute_only or set execute_inplace to True."
+        solver.restore(replay_metrics=False)  # load checkpoint
+        solver.run_one_stage(cfg.execute_only)
+        return
+
+    return solver.run()
+
+
+main.dora.dir = AudioCraftEnvironment.get_dora_dir()
+main._base_cfg.slurm = get_slurm_parameters(main._base_cfg.slurm)
+
+if main.dora.shared is not None and not os.access(main.dora.shared, os.R_OK):
+    print("No read permission on dora.shared folder, ignoring it.", file=sys.stderr)
+    main.dora.shared = None
+
+if __name__ == '__main__':
+    main()
diff --git a/audiocraft/audiocraft/utils/__init__.py b/audiocraft/audiocraft/utils/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..75e25a0212f98e4a18d97c86c6cda225636a3215
--- /dev/null
+++ b/audiocraft/audiocraft/utils/__init__.py
@@ -0,0 +1,6 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+"""Utilities."""
diff --git a/audiocraft/audiocraft/utils/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..093b5eb8070af48aea57c7c726a9f5d5f8262e50
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/autocast.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/autocast.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..605b9b3ab2738226b1464ce76bb3bd80a9abc568
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/autocast.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/best_state.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/best_state.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..1c4626d6833196e4447cc68dea2a8b1c0b2efe20
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/best_state.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/cache.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/cache.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..fc2a3836af340d27cc29eae79176b9283a78b5d4
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/cache.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/checkpoint.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/checkpoint.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..10d9f0186035f7ecc47864cc57b527c821c50be9
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/checkpoint.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/cluster.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/cluster.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3b5fe8e616f43de5539822172f5478582eb8c5e3
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/cluster.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/deadlock.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/deadlock.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..dbc86c87ba5d39c11c0d081c5e65d11df09dde78
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/deadlock.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/export.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/export.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..e5d39b027b8c230a6280515eab89f67cd63b1b85
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/export.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/profiler.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/profiler.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..0829d206ed84c57aef536602d3e8550b56125c0d
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/profiler.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/__pycache__/utils.cpython-311.pyc b/audiocraft/audiocraft/utils/__pycache__/utils.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..8f782fb1fd7b3fa4801e2bceb48ea863c160eb85
Binary files /dev/null and b/audiocraft/audiocraft/utils/__pycache__/utils.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/autocast.py b/audiocraft/audiocraft/utils/autocast.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed644843bb37cf8a92a20fbd51d6cebaa43b9a08
--- /dev/null
+++ b/audiocraft/audiocraft/utils/autocast.py
@@ -0,0 +1,40 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import torch
+
+
+class TorchAutocast:
+    """TorchAutocast utility class.
+    Allows you to enable and disable autocast. This is specially useful
+    when dealing with different architectures and clusters with different
+    levels of support.
+
+    Args:
+        enabled (bool): Whether to enable torch.autocast or not.
+        args: Additional args for torch.autocast.
+        kwargs: Additional kwargs for torch.autocast
+    """
+    def __init__(self, enabled: bool, *args, **kwargs):
+        self.autocast = torch.autocast(*args, **kwargs) if enabled else None
+
+    def __enter__(self):
+        if self.autocast is None:
+            return
+        try:
+            self.autocast.__enter__()
+        except RuntimeError:
+            device = self.autocast.device
+            dtype = self.autocast.fast_dtype
+            raise RuntimeError(
+                f"There was an error autocasting with dtype={dtype} device={device}\n"
+                "If you are on the FAIR Cluster, you might need to use autocast_dtype=float16"
+            )
+
+    def __exit__(self, *args, **kwargs):
+        if self.autocast is None:
+            return
+        self.autocast.__exit__(*args, **kwargs)
diff --git a/audiocraft/audiocraft/utils/best_state.py b/audiocraft/audiocraft/utils/best_state.py
new file mode 100644
index 0000000000000000000000000000000000000000..f5ad551432ad5cb0f83278b5d2100f9aa287958b
--- /dev/null
+++ b/audiocraft/audiocraft/utils/best_state.py
@@ -0,0 +1,81 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from collections import defaultdict
+import logging
+import typing as tp
+
+import flashy
+import torch
+
+from ..optim import ModuleDictEMA
+from .utils import copy_state
+
+
+logger = logging.getLogger(__name__)
+
+
+class BestStateDictManager(flashy.state.StateDictSource):
+    """BestStateDictManager maintains a copy of best state_dict() for registered sources.
+
+    BestStateDictManager has two main attributes:
+        states (dict): State dict of the registered StateDictSource.
+        param_ids (dict): Dict of parameter ids for registered states from ModuleDictEMA and other sources.
+
+    When registering new sources, the BestStateDictManager will ensure two conflicting sources between
+    ModuleDictEMA and original modules are not both registered as it would otherwise create ambiguity about
+    what to consider for best state.
+
+    Args:
+        device (torch.device or str): Device on which we keep the copy.
+        dtype (torch.dtype): Data type for the state parameters.
+    """
+    def __init__(self, device: tp.Union[torch.device, str] = 'cpu',
+                 dtype: tp.Optional[torch.dtype] = None):
+        self.device = device
+        self.states: dict = {}
+        self.param_ids: dict = defaultdict(dict)
+        self.dtype = dtype
+
+    def _get_parameter_ids(self, state_dict):
+        return {id(p): name for name, p in state_dict.items() if isinstance(p, torch.Tensor)}
+
+    def _validate_no_parameter_ids_overlap(self, name: str, param_ids: dict):
+        for registered_name, registered_param_ids in self.param_ids.items():
+            if registered_name != name:
+                overlap = set.intersection(registered_param_ids.keys(), param_ids.keys())
+                assert len(overlap) == 0, f"Found {len(overlap)} / {len(param_ids.keys())} overlapping parameters"
+                f" in {name} and already registered {registered_name}: {' '.join(overlap)}"
+
+    def update(self, name: str, source: flashy.state.StateDictSource):
+        if name not in self.states:
+            raise ValueError(f"{name} missing from registered states.")
+        self.states[name] = copy_state(source.state_dict(), device=self.device, dtype=self.dtype)
+
+    def register(self, name: str, source: flashy.state.StateDictSource):
+        if name in self.states:
+            raise ValueError(f"{name} already present in states.")
+        # Registering parameter ids for EMA and non-EMA states allows us to check that
+        # there is no overlap that would create ambiguity about how to handle the best state
+        param_ids = self._get_parameter_ids(source.state_dict())
+        if isinstance(source, ModuleDictEMA):
+            logger.debug(f"Registering to best state: ModuleDictEMA '{name}' with {len(param_ids)} params")
+            self._validate_no_parameter_ids_overlap(name, param_ids)
+            self.param_ids[name] = param_ids
+        else:
+            logger.debug(f"Registering to best state: StateDictSource '{name}' with {len(param_ids)} params")
+            self._validate_no_parameter_ids_overlap('base', param_ids)
+            self.param_ids['base'].update(param_ids)
+        # Register state
+        self.states[name] = copy_state(source.state_dict(), device=self.device, dtype=self.dtype)
+
+    def state_dict(self) -> flashy.state.StateDict:
+        return self.states
+
+    def load_state_dict(self, state: flashy.state.StateDict):
+        for name, sub_state in state.items():
+            for k, v in sub_state.items():
+                self.states[name][k].copy_(v)
diff --git a/audiocraft/audiocraft/utils/cache.py b/audiocraft/audiocraft/utils/cache.py
new file mode 100644
index 0000000000000000000000000000000000000000..f7f82064e8f43b86af1071cab4d967cca9b5bd86
--- /dev/null
+++ b/audiocraft/audiocraft/utils/cache.py
@@ -0,0 +1,323 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from concurrent.futures import ThreadPoolExecutor
+from collections import deque
+from functools import partial
+from hashlib import sha1
+import logging
+from pathlib import Path
+import sys
+import typing as tp
+import zipfile
+
+import flashy
+import torch
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_full_embed(full_embed: torch.Tensor, x: tp.Any, idx: int, device: tp.Union[str, torch.device]) -> torch.Tensor:
+    """Utility function for the EmbeddingCache, returning the full embedding without any chunking.
+    This method can be used in case there is no need in extracting a chunk of the full embedding
+    read from the cache.
+
+    Args:
+        full_embed (torch.Tensor): The full embedding.
+        x (any): Batch object from which the full embedding is derived.
+        idx (torch.Tensor): Index of object to consider in the batch object.
+    Returns:
+        full_embed (torch.Tensor): The full embedding
+    """
+    return full_embed.to(device)
+
+
+class EmbeddingCache:
+    """Cache around embeddings computation for faster execution.
+    The EmbeddingCache is storing pre-computed embeddings on disk and provides a simple API
+    to retrieve the pre-computed embeddings on full inputs and extract only a given chunk
+    using a user-provided function. When the cache is warm (all embeddings are pre-computed),
+    the EmbeddingCache allows for faster training as it removes the need of computing the embeddings.
+    Additionally, it provides in-memory cache around the loaded embeddings to limit IO footprint
+    and synchronization points in the forward calls.
+
+    Args:
+        cache_path (Path): Path to folder where all pre-computed embeddings are saved on disk.
+        device (str or torch.device): Device on which the embedding is returned.
+        compute_embed_fn (callable[[Path, any, int], torch.Tensor], optional): Function to compute
+            the embedding from a given object and path. This user provided function can compute the
+            embedding from the provided object or using the provided path as entry point. The last parameter
+            specify the index corresponding to the current embedding in the object that can represent batch metadata.
+        extract_embed_fn (callable[[torch.Tensor, any, int], torch.Tensor], optional): Function to extract
+            the desired embedding chunk from the full embedding loaded from the cache. The last parameter
+            specify the index corresponding to the current embedding in the object that can represent batch metadata.
+            If not specified, will return the full embedding unmodified.
+    """
+    def __init__(self, cache_path: tp.Union[str, Path], device: tp.Union[str, torch.device],
+                 compute_embed_fn: tp.Callable[[Path, tp.Any, int], torch.Tensor],
+                 extract_embed_fn: tp.Optional[tp.Callable[[torch.Tensor, tp.Any, int], torch.Tensor]] = None):
+        self.cache_path = Path(cache_path)
+        self.device = device
+        self._compute_embed_fn = compute_embed_fn
+        self._extract_embed_fn: tp.Callable[[torch.Tensor, tp.Any, int], torch.Tensor]
+        if extract_embed_fn is not None:
+            self._extract_embed_fn = extract_embed_fn
+        else:
+            self._extract_embed_fn = partial(get_full_embed, device=device)
+        if self.cache_path is not None:
+            self.cache_path.mkdir(exist_ok=True, parents=True)
+            logger.info(f"Cache instantiated at: {self.cache_path}")
+            self.pool = ThreadPoolExecutor(8)
+            self.pool.__enter__()
+        self._current_batch_cache: dict = {}
+        self._memory_cache: dict = {}
+
+    def _get_cache_path(self, path: tp.Union[Path, str]):
+        """Get cache path for the given file path."""
+        sig = sha1(str(path).encode()).hexdigest()
+        return self.cache_path / sig
+
+    @staticmethod
+    def _get_full_embed_from_cache(cache: Path):
+        """Loads full pre-computed embedding from the cache."""
+        try:
+            embed = torch.load(cache, 'cpu')
+        except Exception as exc:
+            logger.error("Error loading %s: %r", cache, exc)
+            embed = None
+        return embed
+
+    def get_embed_from_cache(self, paths: tp.List[Path], x: tp.Any) -> torch.Tensor:
+        """Get embedding from cache, computing and storing it to cache if not already cached.
+        The EmbeddingCache first tries to load the embedding from the in-memory cache
+        containing the pre-computed chunks populated through `populate_embed_cache`.
+        If not found, the full embedding is computed and stored on disk to be later accessed
+        to populate the in-memory cache, and the desired embedding chunk is extracted and returned.
+
+        Args:
+            paths (list[Path or str]): List of paths from where the embeddings can be loaded.
+            x (any): Object from which the embedding is extracted.
+        """
+        embeds = []
+        for idx, path in enumerate(paths):
+            cache = self._get_cache_path(path)
+            if cache in self._current_batch_cache:
+                embed = self._current_batch_cache[cache]
+            else:
+                full_embed = self._compute_embed_fn(path, x, idx)
+                try:
+                    with flashy.utils.write_and_rename(cache, pid=True) as f:
+                        torch.save(full_embed.cpu(), f)
+                except Exception as exc:
+                    logger.error('Error saving embed %s (%s): %r', cache, full_embed.shape, exc)
+                else:
+                    logger.info('New embed cache saved: %s (%s)', cache, full_embed.shape)
+                    embed = self._extract_embed_fn(full_embed, x, idx)
+            embeds.append(embed)
+        embed = torch.stack(embeds, dim=0)
+        return embed
+
+    def populate_embed_cache(self, paths: tp.List[Path], x: tp.Any) -> None:
+        """Populate in-memory caches for embeddings reading from the embeddings stored on disk.
+        The in-memory caches consist in a cache for the full embedding and another cache for the
+        final embedding chunk. Such caches are used to limit the IO access when computing the actual embeddings
+        and reduce the IO footprint and synchronization points during forward passes.
+
+        Args:
+            paths (list[Path]): List of paths from where the embeddings can be loaded.
+            x (any): Object from which the embedding is extracted.
+        """
+        self._current_batch_cache.clear()
+        if self.cache_path is not None:
+            futures: list = []
+            for path in paths:
+                assert path is not None, "Path is required for computation from cache"
+                cache = self._get_cache_path(path)
+                if cache in self._memory_cache or not cache.exists():
+                    futures.append(None)
+                else:
+                    futures.append(self.pool.submit(EmbeddingCache._get_full_embed_from_cache, cache))
+            for idx, (path, future) in enumerate(zip(paths, futures)):
+                assert path is not None
+                cache = self._get_cache_path(path)
+                full_embed = None
+                if future is None:
+                    if cache in self._memory_cache:
+                        full_embed = self._memory_cache[cache]
+                else:
+                    full_embed = future.result()
+                    if full_embed is not None:
+                        self._memory_cache[cache] = full_embed
+                        full_embed = full_embed.to(self.device)
+                if full_embed is not None:
+                    embed = self._extract_embed_fn(full_embed, x, idx)
+                    self._current_batch_cache[cache] = embed
+
+
+class CachedBatchWriter:
+    """Write pre computed caches for mini batches. This can
+    make loading a lot more efficient depending on your filesystem.
+
+    Args:
+        cache_folder (Path): folder in which the cached minibatches
+            will be stored.
+
+    Inside cache folder, the structure is the following:
+    `epoch_number / update_number.zip`
+    And the zip file contains one entry per batch item.
+
+    It is possible to use the cache with a batch size smaller than
+    created with but obviously not larger. Make sure to call the
+    `start_epoch(epoch)` method for indicating changes of epochs.
+
+    See the grid `audiocraft/grids/musicgen/musicgen_warmup_cache.py`
+    for an example of how to warmup the cache.
+    """
+    def __init__(self, cache_folder: Path):
+        self.cache_folder = cache_folder
+        self._current_epoch: tp.Optional[int] = None
+        self._current_index = 0
+
+    def start_epoch(self, epoch: int):
+        """Call at the beginning of each epoch.
+        """
+        self._current_epoch = epoch
+        self._current_index = 0
+        self._zip_path.parent.mkdir(exist_ok=True, parents=True)
+
+    @staticmethod
+    def _get_zip_path(cache_folder: Path, epoch: int, index: int):
+        return cache_folder / f"{epoch:05d}" / f"{index:06d}.zip"
+
+    @property
+    def _zip_path(self):
+        assert self._current_epoch is not None
+        return CachedBatchWriter._get_zip_path(self.cache_folder, self._current_epoch, self._current_index)
+
+    def save(self, *content):
+        """Save one mini batch. This function is distributed-aware
+        and will automatically merge all the items from the different
+        workers.
+        """
+        all_contents = []
+        for rank in range(flashy.distrib.world_size()):
+            their_content = flashy.distrib.broadcast_object(content, src=rank)
+            all_contents.append(their_content)
+
+        if flashy.distrib.is_rank_zero():
+            idx = 0
+            with flashy.utils.write_and_rename(self._zip_path) as tmp:
+                with zipfile.ZipFile(tmp, 'w') as zf:
+                    for content in all_contents:
+                        for vals in zip(*content):
+                            with zf.open(f'{idx}', 'w') as f:  # type: ignore
+                                torch.save(vals, f)
+                            idx += 1
+        flashy.distrib.barrier()
+        self._current_index += 1
+
+
+class CachedBatchLoader:
+    """Loader for cached mini-batches dumped with `CachedBatchWriter`.
+
+    Args:
+        cache_folder (Path): folder in which the cached minibatches are stored.
+        batch_size (int): batch size (per GPU) expected.
+        num_workers (int): number of workers to use for loading.
+        min_length (int): minimum expected length for each epoch. If some
+            mini-batches are missing, and error is raised.
+
+    This is iterable just like a regular DataLoader.
+    """
+
+    def __init__(self, cache_folder: Path, batch_size: int,
+                 num_workers: int = 10, min_length: int = 1):
+        self.cache_folder = cache_folder
+        self.batch_size = batch_size
+        self.num_workers = num_workers
+        self.min_length = min_length
+        self._current_epoch: tp.Optional[int] = None
+        self.sampler = None  # for compatibility with the regular DataLoader
+
+    def __len__(self):
+        path = CachedBatchWriter._get_zip_path(self.cache_folder, self._current_epoch or 0, 0).parent
+        return len([p for p in path.iterdir() if p.suffix == ".zip"])
+
+    def start_epoch(self, epoch: int):
+        """Call at the beginning of each epoch.
+        """
+        self._current_epoch = epoch
+
+    def _zip_path(self, index: int):
+        assert self._current_epoch is not None
+        return CachedBatchWriter._get_zip_path(self.cache_folder, self._current_epoch, index)
+
+    def _load_one(self, index: int):
+        zip_path = self._zip_path(index)
+        if not zip_path.exists():
+            if index < self.min_length:
+                raise RuntimeError(f"Cache should have at least {self.min_length} batches, but {index} doesn't exist")
+
+            return None
+        mode = "rb" if sys.version_info >= (3, 9) else "r"
+        try:
+            with zipfile.ZipFile(zip_path, 'r') as zf:
+                rank = flashy.distrib.rank()
+                world_size = flashy.distrib.world_size()
+                root = zipfile.Path(zf)
+                items = list(root.iterdir())
+                total_batch_size = self.batch_size * world_size
+                if len(items) < total_batch_size:
+                    raise RuntimeError(
+                        f"The cache can handle a max batch size of {len(items)}, "
+                        f"but {total_batch_size} is needed.")
+                start = rank * self.batch_size
+                items = items[start: start + self.batch_size]
+                assert len(items) == self.batch_size
+                entries = []
+                entries = [torch.load(item.open(mode), 'cpu') for item in items]  # type: ignore
+                transposed = zip(*entries)
+                out = []
+                for part in transposed:
+                    assert len(part) > 0
+                    if isinstance(part[0], torch.Tensor):
+                        out.append(torch.stack(part))
+                    else:
+                        out.append(part)
+                return out
+        except Exception:
+            logger.error("Error when reading zip path %s", zip_path)
+            raise
+
+    def __iter__(self):
+        """This will yields tuples, exactly as provided to the
+        `CachedBatchWriter.save` method.
+        """
+        pool = ThreadPoolExecutor(self.num_workers)
+        next_index = 0
+        queue = deque()
+
+        def _get_next():
+            nonlocal next_index
+            r = queue.popleft().result()
+            if r is None:
+                return None
+            else:
+                queue.append(pool.submit(self._load_one, next_index))
+                next_index += 1
+            return r
+
+        with pool:
+            # fill the buffer of fetching jobs.
+            for _ in range(2 * self.num_workers):
+                queue.append(pool.submit(self._load_one, next_index))
+                next_index += 1
+            while True:
+                batch = _get_next()
+                if batch is None:
+                    return
+                yield batch
diff --git a/audiocraft/audiocraft/utils/checkpoint.py b/audiocraft/audiocraft/utils/checkpoint.py
new file mode 100644
index 0000000000000000000000000000000000000000..f6f871837e09c5cc7832b85b0d80b84f59e87ca0
--- /dev/null
+++ b/audiocraft/audiocraft/utils/checkpoint.py
@@ -0,0 +1,161 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from enum import Enum
+import logging
+from pathlib import Path
+import re
+import typing as tp
+
+import flashy
+import torch
+
+from ..environment import AudioCraftEnvironment
+
+
+logger = logging.getLogger(__name__)
+
+
+class CheckpointSource(Enum):
+    CURRENT_XP = "current_xp"
+    PRETRAINED = "pretrained"
+    OTHER = "other"
+
+
+def checkpoint_name(name: tp.Optional[str] = None, rank: tp.Optional[int] = None, use_fsdp: bool = False) -> str:
+    """Checkpoint name formatted for all use in AudioCraft codebase and has the following format:
+    `checkpoint_<name>.th(.<rank>)`. By convention, name is expected to be empty for last checkpoint,
+    'best' for the best checkpoint or the epoch number.
+
+    Args:
+        name (str, optional): Name suffix for the checkpoint file stem.
+        rank (optional, int): Rank for distributed processing, retrieved with flashy if not provided.
+        use_fsdp (bool): Whether the calling solver relies on FSDP.
+    Returns:
+        str: The checkpoint name.
+    """
+    suffix = ''
+    if rank is None:
+        rank = flashy.distrib.rank()
+    if rank > 0 and use_fsdp:
+        suffix = '.' + str(rank)
+    name_part = ''
+    if name is not None:
+        name_part = f'_{name}'
+    return f'checkpoint{name_part}.th{suffix}'
+
+
+def is_sharded_checkpoint(path: Path) -> bool:
+    """Whether the checkpoint at the given path corresponds to a sharded checkpoint across rank."""
+    return re.search(r'\.th\.\d+$', path.name) is not None
+
+
+def resolve_checkpoint_path(sig_or_path: tp.Union[Path, str], name: tp.Optional[str] = None,
+                            use_fsdp: bool = False) -> tp.Optional[Path]:
+    """Resolve a given checkpoint path for a provided dora sig or path.
+
+    Args:
+        sig_or_path (Path or str): Checkpoint path or dora signature.
+        name (str, optional): Name suffix for the checkpoint file stem.
+        rank (optional, int): Rank for distributed processing, retrieved with flashy if not provided.
+        use_fsdp (bool): Whether the calling solver relies on FSDP.
+    Returns:
+        Path, optional: Resolved checkpoint path, if it exists.
+    """
+    from audiocraft import train
+    xps_root = train.main.dora.dir / 'xps'
+    sig_or_path = str(sig_or_path)
+    if sig_or_path.startswith('//sig/'):
+        sig = sig_or_path[len('//sig/'):]
+        path = xps_root / sig
+    else:
+        path = Path(sig_or_path)
+        path = AudioCraftEnvironment.resolve_reference_path(path)
+
+    if path.is_dir():
+        path = path / checkpoint_name(name, use_fsdp=use_fsdp)
+
+    if path.exists():
+        return path
+    else:
+        return None
+
+
+def load_checkpoint(checkpoint_path: Path, is_sharded: bool = False) -> tp.Any:
+    """Load state from checkpoints at the specified checkpoint path."""
+    if is_sharded:
+        rank0_checkpoint_path = checkpoint_path.parent / checkpoint_name(use_fsdp=False)
+        if rank0_checkpoint_path.exists():
+            check_sharded_checkpoint(checkpoint_path, rank0_checkpoint_path)
+    state = torch.load(checkpoint_path, 'cpu')
+    logger.info("Checkpoint loaded from %s", checkpoint_path)
+    return state
+
+
+def save_checkpoint(state: tp.Any, checkpoint_path: Path, is_sharded: bool = False) -> None:
+    """Save state to disk to the specified checkpoint_path."""
+    _safe_save_checkpoint(state, checkpoint_path, is_sharded)
+    logger.info("Checkpoint saved to %s", checkpoint_path)
+
+
+def flush_stale_checkpoints(checkpoint_path: Path, keep_last: tp.Optional[int] = None) -> None:
+    """Flush checkpoints to only keep last N checkpoints."""
+    if keep_last is None or keep_last <= 0:
+        return
+    checkpoint_dir = checkpoint_path.parent
+    suffix = ''
+    if flashy.distrib.rank() > 0:
+        suffix = f'.{flashy.distrib.rank()}'
+    checkpoint_files_with_epoch = []
+    for path in Path(checkpoint_dir).glob(f'checkpoint_*.th{suffix}'):
+        epoch_part = path.name.split('.', 1)[0].split('_', 1)[1]
+        if epoch_part.isdigit():
+            checkpoint_files_with_epoch.append((path, int(epoch_part)))
+    checkpoint_files = [path for path, _ in list(sorted(checkpoint_files_with_epoch, key=lambda t: t[1]))]
+    total_to_flush = max(0, len(checkpoint_files) - keep_last)
+    files_to_flush = checkpoint_files[:total_to_flush]
+    for path in files_to_flush:
+        logger.debug("Removing checkpoint: %s", str(path))
+        path.unlink(missing_ok=True)
+
+
+def check_sharded_checkpoint(checkpoint_path: Path, rank0_checkpoint_path: Path) -> None:
+    """Check sharded checkpoint state, ensuring the checkpoints are not corrupted."""
+    # Finish the work of a previous run that got interrupted while dumping.
+    old_path = Path(str(checkpoint_path) + '.old')
+    if old_path.exists():
+        raise RuntimeError(
+            f"Old checkpoint {old_path} from previous version of this code exist, cannot safely proceed.")
+    token = Path(str(rank0_checkpoint_path) + '.tmp.done')
+    tmp_path = Path(str(checkpoint_path) + '.tmp')
+    if token.exists():
+        if tmp_path.exists():
+            tmp_path.rename(checkpoint_path)
+    flashy.distrib.barrier()
+    if flashy.distrib.is_rank_zero() and token.exists():
+        token.unlink()
+
+
+def _safe_save_checkpoint(state: tp.Any, checkpoint_path: Path, is_sharded: bool = False) -> None:
+    """Save checkpoints in a safe manner even with when sharded checkpoints across nodes."""
+    def _barrier_if_sharded():
+        if is_sharded:
+            flashy.distrib.barrier()
+
+    if flashy.distrib.is_rank_zero():
+        token = Path(str(checkpoint_path) + '.tmp.done')
+        if token.exists():
+            token.unlink()
+    _barrier_if_sharded()
+    with flashy.utils.write_and_rename(checkpoint_path) as f:
+        torch.save(state, f)
+        _barrier_if_sharded()
+        if flashy.distrib.is_rank_zero():
+            token.touch()
+        _barrier_if_sharded()
+    _barrier_if_sharded()
+    if flashy.distrib.rank() == 0:
+        token.unlink()
diff --git a/audiocraft/audiocraft/utils/cluster.py b/audiocraft/audiocraft/utils/cluster.py
new file mode 100644
index 0000000000000000000000000000000000000000..3380d031739d473fb859c76b9c25350f47fa77e8
--- /dev/null
+++ b/audiocraft/audiocraft/utils/cluster.py
@@ -0,0 +1,75 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Utility functions for SLURM configuration and cluster settings.
+"""
+
+from enum import Enum
+import os
+import socket
+import typing as tp
+
+import omegaconf
+
+
+class ClusterType(Enum):
+    AWS = "aws"
+    FAIR = "fair"
+    RSC = "rsc"
+    LOCAL_DARWIN = "darwin"
+    DEFAULT = "default"  # used for any other cluster.
+
+
+def _guess_cluster_type() -> ClusterType:
+    uname = os.uname()
+    fqdn = socket.getfqdn()
+    if uname.sysname == "Linux" and (uname.release.endswith("-aws") or ".ec2" in fqdn):
+        return ClusterType.AWS
+
+    if fqdn.endswith(".fair"):
+        return ClusterType.FAIR
+
+    if fqdn.endswith(".facebook.com"):
+        return ClusterType.RSC
+
+    if uname.sysname == "Darwin":
+        return ClusterType.LOCAL_DARWIN
+
+    return ClusterType.DEFAULT
+
+
+def get_cluster_type(
+    cluster_type: tp.Optional[ClusterType] = None,
+) -> tp.Optional[ClusterType]:
+    if cluster_type is None:
+        return _guess_cluster_type()
+
+    return cluster_type
+
+
+def get_slurm_parameters(
+    cfg: omegaconf.DictConfig, cluster_type: tp.Optional[ClusterType] = None
+) -> omegaconf.DictConfig:
+    """Update SLURM parameters in configuration based on cluster type.
+    If the cluster type is not specify, it infers it automatically.
+    """
+    from ..environment import AudioCraftEnvironment
+    cluster_type = get_cluster_type(cluster_type)
+    # apply cluster-specific adjustments
+    if cluster_type == ClusterType.AWS:
+        cfg["mem_per_gpu"] = None
+        cfg["constraint"] = None
+        cfg["setup"] = []
+    elif cluster_type == ClusterType.RSC:
+        cfg["mem_per_gpu"] = None
+        cfg["setup"] = []
+        cfg["constraint"] = None
+        cfg["partition"] = "learn"
+    slurm_exclude = AudioCraftEnvironment.get_slurm_exclude()
+    if slurm_exclude is not None:
+        cfg["exclude"] = slurm_exclude
+    return cfg
diff --git a/audiocraft/audiocraft/utils/deadlock.py b/audiocraft/audiocraft/utils/deadlock.py
new file mode 100644
index 0000000000000000000000000000000000000000..8abd1bbeea5909e664cf816c020bd7c37effdb66
--- /dev/null
+++ b/audiocraft/audiocraft/utils/deadlock.py
@@ -0,0 +1,58 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import os
+from queue import Queue, Empty
+import signal
+import sys
+import threading
+import traceback
+
+logger = logging.getLogger(__name__)
+
+
+class DeadlockDetect:
+    def __init__(self, use: bool = False, timeout: float = 120.):
+        self.use = use
+        self.timeout = timeout
+        self._queue: Queue = Queue()
+
+    def update(self, stage: str):
+        if self.use:
+            self._queue.put(stage)
+
+    def __enter__(self):
+        if self.use:
+            self._thread = threading.Thread(target=self._detector_thread)
+            self._thread.start()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.use:
+            self._queue.put(None)
+            self._thread.join()
+
+    def _detector_thread(self):
+        logger.debug("Deadlock detector started")
+        last_stage = "init"
+        while True:
+            try:
+                stage = self._queue.get(timeout=self.timeout)
+            except Empty:
+                break
+            if stage is None:
+                logger.debug("Exiting deadlock detector thread")
+                return
+            else:
+                last_stage = stage
+        logger.error("Deadlock detector timed out, last stage was %s", last_stage)
+        for th in threading.enumerate():
+            print(th, file=sys.stderr)
+            traceback.print_stack(sys._current_frames()[th.ident])
+            print(file=sys.stderr)
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os.kill(os.getpid(), signal.SIGKILL)
diff --git a/audiocraft/audiocraft/utils/export.py b/audiocraft/audiocraft/utils/export.py
new file mode 100644
index 0000000000000000000000000000000000000000..28b214017d9ac23934b67e8254a96131cefa6501
--- /dev/null
+++ b/audiocraft/audiocraft/utils/export.py
@@ -0,0 +1,79 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Utility to export a training checkpoint to a lightweight release checkpoint.
+"""
+
+from pathlib import Path
+import typing as tp
+
+from omegaconf import OmegaConf
+import torch
+
+from audiocraft import __version__
+
+
+def export_encodec(checkpoint_path: tp.Union[Path, str], out_file: tp.Union[Path, str]):
+    """Export only the best state from the given EnCodec checkpoint. This
+    should be used if you trained your own EnCodec model.
+    """
+    pkg = torch.load(checkpoint_path, 'cpu')
+    new_pkg = {
+        'best_state': pkg['best_state']['model'],
+        'xp.cfg': OmegaConf.to_yaml(pkg['xp.cfg']),
+        'version': __version__,
+        'exported': True,
+    }
+    Path(out_file).parent.mkdir(exist_ok=True, parents=True)
+    torch.save(new_pkg, out_file)
+    return out_file
+
+
+def export_pretrained_compression_model(pretrained_encodec: str, out_file: tp.Union[Path, str]):
+    """Export a compression model (potentially EnCodec) from a pretrained model.
+    This is required for packaging the audio tokenizer along a MusicGen or AudioGen model.
+    Do not include the //pretrained/ prefix. For instance if you trained a model
+    with `facebook/encodec_32khz`, just put that as a name. Same for `dac_44khz`.
+
+    In that case, this will not actually include a copy of the model, simply the reference
+    to the model used.
+    """
+    if Path(pretrained_encodec).exists():
+        pkg = torch.load(pretrained_encodec)
+        assert 'best_state' in pkg
+        assert 'xp.cfg' in pkg
+        assert 'version' in pkg
+        assert 'exported' in pkg
+    else:
+        pkg = {
+            'pretrained': pretrained_encodec,
+            'exported': True,
+            'version': __version__,
+        }
+    Path(out_file).parent.mkdir(exist_ok=True, parents=True)
+    torch.save(pkg, out_file)
+
+
+def export_lm(checkpoint_path: tp.Union[Path, str], out_file: tp.Union[Path, str]):
+    """Export only the best state from the given MusicGen or AudioGen checkpoint.
+    """
+    pkg = torch.load(checkpoint_path, 'cpu')
+    if pkg['fsdp_best_state']:
+        best_state = pkg['fsdp_best_state']['model']
+    else:
+        assert pkg['best_state']
+        best_state = pkg['best_state']['model']
+    new_pkg = {
+        'best_state': best_state,
+        'xp.cfg': OmegaConf.to_yaml(pkg['xp.cfg']),
+        'version': __version__,
+        'exported': True,
+    }
+
+    Path(out_file).parent.mkdir(exist_ok=True, parents=True)
+    torch.save(new_pkg, out_file)
+    return out_file
diff --git a/audiocraft/audiocraft/utils/export_legacy.py b/audiocraft/audiocraft/utils/export_legacy.py
new file mode 100644
index 0000000000000000000000000000000000000000..52f145f3148c3e9fdba436273bc45480fbae6481
--- /dev/null
+++ b/audiocraft/audiocraft/utils/export_legacy.py
@@ -0,0 +1,56 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Legacy functions used at the time of the first release, kept for referencd.
+"""
+
+from pathlib import Path
+import typing as tp
+
+from omegaconf import OmegaConf, DictConfig
+import torch
+
+
+def _clean_lm_cfg(cfg: DictConfig):
+    OmegaConf.set_struct(cfg, False)
+    # This used to be set automatically in the LM solver, need a more robust solution
+    # for the future.
+    cfg['transformer_lm']['card'] = 2048
+    cfg['transformer_lm']['n_q'] = 4
+    # Experimental params no longer supported.
+    bad_params = ['spectral_norm_attn_iters', 'spectral_norm_ff_iters',
+                  'residual_balancer_attn', 'residual_balancer_ff', 'layer_drop']
+    for name in bad_params:
+        del cfg['transformer_lm'][name]
+    OmegaConf.set_struct(cfg, True)
+    return cfg
+
+
+def export_encodec(checkpoint_path: tp.Union[Path, str], out_folder: tp.Union[Path, str]):
+    sig = Path(checkpoint_path).parent.name
+    assert len(sig) == 8, "Not a valid Dora signature"
+    pkg = torch.load(checkpoint_path, 'cpu')
+    new_pkg = {
+        'best_state': pkg['ema']['state']['model'],
+        'xp.cfg': OmegaConf.to_yaml(pkg['xp.cfg']),
+    }
+    out_file = Path(out_folder) / f'{sig}.th'
+    torch.save(new_pkg, out_file)
+    return out_file
+
+
+def export_lm(checkpoint_path: tp.Union[Path, str], out_folder: tp.Union[Path, str]):
+    sig = Path(checkpoint_path).parent.name
+    assert len(sig) == 8, "Not a valid Dora signature"
+    pkg = torch.load(checkpoint_path, 'cpu')
+    new_pkg = {
+        'best_state': pkg['fsdp_best_state']['model'],
+        'xp.cfg': OmegaConf.to_yaml(_clean_lm_cfg(pkg['xp.cfg']))
+    }
+    out_file = Path(out_folder) / f'{sig}.th'
+    torch.save(new_pkg, out_file)
+    return out_file
diff --git a/audiocraft/audiocraft/utils/notebook.py b/audiocraft/audiocraft/utils/notebook.py
new file mode 100644
index 0000000000000000000000000000000000000000..019b9d19e5bef976bedddf428fd25da42a8a9726
--- /dev/null
+++ b/audiocraft/audiocraft/utils/notebook.py
@@ -0,0 +1,32 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+try:
+    import IPython.display as ipd  # type: ignore
+except ImportError:
+    # Note in a notebook...
+    pass
+
+
+import torch
+
+
+def display_audio(samples: torch.Tensor, sample_rate: int):
+    """Renders an audio player for the given audio samples.
+
+    Args:
+        samples (torch.Tensor): a Tensor of decoded audio samples
+            with shapes [B, C, T] or [C, T]
+        sample_rate (int): sample rate audio should be displayed with.
+    """
+    assert samples.dim() == 2 or samples.dim() == 3
+
+    samples = samples.detach().cpu()
+    if samples.dim() == 2:
+        samples = samples[None, ...]
+
+    for audio in samples:
+        ipd.display(ipd.Audio(audio, rate=sample_rate))
diff --git a/audiocraft/audiocraft/utils/profiler.py b/audiocraft/audiocraft/utils/profiler.py
new file mode 100644
index 0000000000000000000000000000000000000000..b45b6d15910b50305c7b212c089ffad3c25b324d
--- /dev/null
+++ b/audiocraft/audiocraft/utils/profiler.py
@@ -0,0 +1,38 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import logging
+import typing as tp
+
+import dora
+import torch
+
+
+logger = logging.getLogger(__name__)
+
+
+class Profiler:
+    """Context manager wrapper for xformers profiler.
+    """
+    def __init__(self, module: torch.nn.Module, enabled: bool = False):
+        self.profiler: tp.Optional[tp.Any] = None
+        if enabled:
+            from xformers.profiler import profile
+            output_dir = dora.get_xp().folder / 'profiler_data'
+            logger.info("Profiling activated, results with be saved to %s", output_dir)
+            self.profiler = profile(output_dir=output_dir, module=module)
+
+    def step(self):
+        if self.profiler is not None:
+            self.profiler.step()  # type: ignore
+
+    def __enter__(self):
+        if self.profiler is not None:
+            return self.profiler.__enter__()  # type: ignore
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        if self.profiler is not None:
+            return self.profiler.__exit__(exc_type, exc_value, exc_tb)  # type: ignore
diff --git a/audiocraft/audiocraft/utils/samples/__init__.py b/audiocraft/audiocraft/utils/samples/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..0952fcc3f57e34b3747962e9ebd6fc57aeea63fa
--- /dev/null
+++ b/audiocraft/audiocraft/utils/samples/__init__.py
@@ -0,0 +1,5 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
diff --git a/audiocraft/audiocraft/utils/samples/__pycache__/__init__.cpython-311.pyc b/audiocraft/audiocraft/utils/samples/__pycache__/__init__.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..19d854c4561dbe27b2b24cdd4768e5bcb3f59685
Binary files /dev/null and b/audiocraft/audiocraft/utils/samples/__pycache__/__init__.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/samples/__pycache__/manager.cpython-311.pyc b/audiocraft/audiocraft/utils/samples/__pycache__/manager.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..ddc98a809dfb0203679bb4440fdec7e8b12541af
Binary files /dev/null and b/audiocraft/audiocraft/utils/samples/__pycache__/manager.cpython-311.pyc differ
diff --git a/audiocraft/audiocraft/utils/samples/manager.py b/audiocraft/audiocraft/utils/samples/manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..bf0fb21b2d2867c03f7cce6f27d9524fdb89b51d
--- /dev/null
+++ b/audiocraft/audiocraft/utils/samples/manager.py
@@ -0,0 +1,386 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+API that can manage the storage and retrieval of generated samples produced by experiments.
+
+It offers the following benefits:
+* Samples are stored in a consistent way across epoch
+* Metadata about the samples can be stored and retrieved
+* Can retrieve audio
+* Identifiers are reliable and deterministic for prompted and conditioned samples
+* Can request the samples for multiple XPs, grouped by sample identifier
+* For no-input samples (not prompt and no conditions), samples across XPs are matched
+  by sorting their identifiers
+"""
+
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import asdict, dataclass
+from functools import lru_cache
+import hashlib
+import json
+import logging
+from pathlib import Path
+import re
+import typing as tp
+import unicodedata
+import uuid
+
+import dora
+import torch
+
+from ...data.audio import audio_read, audio_write
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ReferenceSample:
+    id: str
+    path: str
+    duration: float
+
+
+@dataclass
+class Sample:
+    id: str
+    path: str
+    epoch: int
+    duration: float
+    conditioning: tp.Optional[tp.Dict[str, tp.Any]]
+    prompt: tp.Optional[ReferenceSample]
+    reference: tp.Optional[ReferenceSample]
+    generation_args: tp.Optional[tp.Dict[str, tp.Any]]
+
+    def __hash__(self):
+        return hash(self.id)
+
+    def audio(self) -> tp.Tuple[torch.Tensor, int]:
+        return audio_read(self.path)
+
+    def audio_prompt(self) -> tp.Optional[tp.Tuple[torch.Tensor, int]]:
+        return audio_read(self.prompt.path) if self.prompt is not None else None
+
+    def audio_reference(self) -> tp.Optional[tp.Tuple[torch.Tensor, int]]:
+        return audio_read(self.reference.path) if self.reference is not None else None
+
+
+class SampleManager:
+    """Audio samples IO handling within a given dora xp.
+
+    The sample manager handles the dumping and loading logic for generated and
+    references samples across epochs for a given xp, providing a simple API to
+    store, retrieve and compare audio samples.
+
+    Args:
+        xp (dora.XP): Dora experiment object. The XP contains information on the XP folder
+            where all outputs are stored and the configuration of the experiment,
+            which is useful to retrieve audio-related parameters.
+        map_reference_to_sample_id (bool): Whether to use the sample_id for all reference samples
+            instead of generating a dedicated hash id. This is useful to allow easier comparison
+            with ground truth sample from the files directly without having to read the JSON metadata
+            to do the mapping (at the cost of potentially dumping duplicate prompts/references
+            depending on the task).
+    """
+    def __init__(self, xp: dora.XP, map_reference_to_sample_id: bool = False):
+        self.xp = xp
+        self.base_folder: Path = xp.folder / xp.cfg.generate.path
+        self.reference_folder = self.base_folder / 'reference'
+        self.map_reference_to_sample_id = map_reference_to_sample_id
+        self.samples: tp.List[Sample] = []
+        self._load_samples()
+
+    @property
+    def latest_epoch(self):
+        """Latest epoch across all samples."""
+        return max(self.samples, key=lambda x: x.epoch).epoch if self.samples else 0
+
+    def _load_samples(self):
+        """Scan the sample folder and load existing samples."""
+        jsons = self.base_folder.glob('**/*.json')
+        with ThreadPoolExecutor(6) as pool:
+            self.samples = list(pool.map(self._load_sample, jsons))
+
+    @staticmethod
+    @lru_cache(2**26)
+    def _load_sample(json_file: Path) -> Sample:
+        with open(json_file, 'r') as f:
+            data: tp.Dict[str, tp.Any] = json.load(f)
+        # fetch prompt data
+        prompt_data = data.get('prompt')
+        prompt = ReferenceSample(id=prompt_data['id'], path=prompt_data['path'],
+                                 duration=prompt_data['duration']) if prompt_data else None
+        # fetch reference data
+        reference_data = data.get('reference')
+        reference = ReferenceSample(id=reference_data['id'], path=reference_data['path'],
+                                    duration=reference_data['duration']) if reference_data else None
+        # build sample object
+        return Sample(id=data['id'], path=data['path'], epoch=data['epoch'], duration=data['duration'],
+                      prompt=prompt, conditioning=data.get('conditioning'), reference=reference,
+                      generation_args=data.get('generation_args'))
+
+    def _init_hash(self):
+        return hashlib.sha1()
+
+    def _get_tensor_id(self, tensor: torch.Tensor) -> str:
+        hash_id = self._init_hash()
+        hash_id.update(tensor.numpy().data)
+        return hash_id.hexdigest()
+
+    def _get_sample_id(self, index: int, prompt_wav: tp.Optional[torch.Tensor],
+                       conditions: tp.Optional[tp.Dict[str, str]]) -> str:
+        """Computes an id for a sample given its input data.
+        This id is deterministic if prompt and/or conditions are provided by using a sha1 hash on the input.
+        Otherwise, a random id of the form "noinput_{uuid4().hex}" is returned.
+
+        Args:
+            index (int): Batch index, Helpful to differentiate samples from the same batch.
+            prompt_wav (torch.Tensor): Prompt used during generation.
+            conditions (dict[str, str]): Conditioning used during generation.
+        """
+        # For totally unconditioned generations we will just use a random UUID.
+        # The function get_samples_for_xps will do a simple ordered match with a custom key.
+        if prompt_wav is None and not conditions:
+            return f"noinput_{uuid.uuid4().hex}"
+
+        # Human readable portion
+        hr_label = ""
+        # Create a deterministic id using hashing
+        hash_id = self._init_hash()
+        hash_id.update(f"{index}".encode())
+        if prompt_wav is not None:
+            hash_id.update(prompt_wav.numpy().data)
+            hr_label += "_prompted"
+        else:
+            hr_label += "_unprompted"
+        if conditions:
+            encoded_json = json.dumps(conditions, sort_keys=True).encode()
+            hash_id.update(encoded_json)
+            cond_str = "-".join([f"{key}={slugify(value)}"
+                                 for key, value in sorted(conditions.items())])
+            cond_str = cond_str[:100]  # some raw text might be too long to be a valid filename
+            cond_str = cond_str if len(cond_str) > 0 else "unconditioned"
+            hr_label += f"_{cond_str}"
+        else:
+            hr_label += "_unconditioned"
+
+        return hash_id.hexdigest() + hr_label
+
+    def _store_audio(self, wav: torch.Tensor, stem_path: Path, overwrite: bool = False) -> Path:
+        """Stores the audio with the given stem path using the XP's configuration.
+
+        Args:
+            wav (torch.Tensor): Audio to store.
+            stem_path (Path): Path in sample output directory with file stem to use.
+            overwrite (bool): When False (default), skips storing an existing audio file.
+        Returns:
+            Path: The path at which the audio is stored.
+        """
+        existing_paths = [
+            path for path in stem_path.parent.glob(stem_path.stem + '.*')
+            if path.suffix != '.json'
+        ]
+        exists = len(existing_paths) > 0
+        if exists and overwrite:
+            logger.warning(f"Overwriting existing audio file with stem path {stem_path}")
+        elif exists:
+            return existing_paths[0]
+
+        audio_path = audio_write(stem_path, wav, **self.xp.cfg.generate.audio)
+        return audio_path
+
+    def add_sample(self, sample_wav: torch.Tensor, epoch: int, index: int = 0,
+                   conditions: tp.Optional[tp.Dict[str, str]] = None, prompt_wav: tp.Optional[torch.Tensor] = None,
+                   ground_truth_wav: tp.Optional[torch.Tensor] = None,
+                   generation_args: tp.Optional[tp.Dict[str, tp.Any]] = None) -> Sample:
+        """Adds a single sample.
+        The sample is stored in the XP's sample output directory, under a corresponding epoch folder.
+        Each sample is assigned an id which is computed using the input data. In addition to the
+        sample itself, a json file containing associated metadata is stored next to it.
+
+        Args:
+            sample_wav (torch.Tensor): sample audio to store. Tensor of shape [channels, shape].
+            epoch (int): current training epoch.
+            index (int): helpful to differentiate samples from the same batch.
+            conditions (dict[str, str], optional): conditioning used during generation.
+            prompt_wav (torch.Tensor, optional): prompt used during generation. Tensor of shape [channels, shape].
+            ground_truth_wav (torch.Tensor, optional): reference audio where prompt was extracted from.
+                Tensor of shape [channels, shape].
+            generation_args (dict[str, any], optional): dictionary of other arguments used during generation.
+        Returns:
+            Sample: The saved sample.
+        """
+        sample_id = self._get_sample_id(index, prompt_wav, conditions)
+        reuse_id = self.map_reference_to_sample_id
+        prompt, ground_truth = None, None
+        if prompt_wav is not None:
+            prompt_id = sample_id if reuse_id else self._get_tensor_id(prompt_wav.sum(0, keepdim=True))
+            prompt_duration = prompt_wav.shape[-1] / self.xp.cfg.sample_rate
+            prompt_path = self._store_audio(prompt_wav, self.base_folder / str(epoch) / 'prompt' / prompt_id)
+            prompt = ReferenceSample(prompt_id, str(prompt_path), prompt_duration)
+        if ground_truth_wav is not None:
+            ground_truth_id = sample_id if reuse_id else self._get_tensor_id(ground_truth_wav.sum(0, keepdim=True))
+            ground_truth_duration = ground_truth_wav.shape[-1] / self.xp.cfg.sample_rate
+            ground_truth_path = self._store_audio(ground_truth_wav, self.base_folder / 'reference' / ground_truth_id)
+            ground_truth = ReferenceSample(ground_truth_id, str(ground_truth_path), ground_truth_duration)
+        sample_path = self._store_audio(sample_wav, self.base_folder / str(epoch) / sample_id, overwrite=True)
+        duration = sample_wav.shape[-1] / self.xp.cfg.sample_rate
+        sample = Sample(sample_id, str(sample_path), epoch, duration, conditions, prompt, ground_truth, generation_args)
+        self.samples.append(sample)
+        with open(sample_path.with_suffix('.json'), 'w') as f:
+            json.dump(asdict(sample), f, indent=2)
+        return sample
+
+    def add_samples(self, samples_wavs: torch.Tensor, epoch: int,
+                    conditioning: tp.Optional[tp.List[tp.Dict[str, tp.Any]]] = None,
+                    prompt_wavs: tp.Optional[torch.Tensor] = None,
+                    ground_truth_wavs: tp.Optional[torch.Tensor] = None,
+                    generation_args: tp.Optional[tp.Dict[str, tp.Any]] = None) -> tp.List[Sample]:
+        """Adds a batch of samples.
+        The samples are stored in the XP's sample output directory, under a corresponding
+        epoch folder. Each sample is assigned an id which is computed using the input data and their batch index.
+        In addition to the sample itself, a json file containing associated metadata is stored next to it.
+
+        Args:
+            sample_wavs (torch.Tensor): Batch of audio wavs to store. Tensor of shape [batch_size, channels, shape].
+            epoch (int): Current training epoch.
+            conditioning (list of dict[str, str], optional): List of conditions used during generation,
+                one per sample in the batch.
+            prompt_wavs (torch.Tensor, optional): Prompts used during generation. Tensor of shape
+                [batch_size, channels, shape].
+            ground_truth_wav (torch.Tensor, optional): Reference audio where prompts were extracted from.
+                Tensor of shape [batch_size, channels, shape].
+            generation_args (dict[str, Any], optional): Dictionary of other arguments used during generation.
+        Returns:
+            samples (list of Sample): The saved audio samples with prompts, ground truth and metadata.
+        """
+        samples = []
+        for idx, wav in enumerate(samples_wavs):
+            prompt_wav = prompt_wavs[idx] if prompt_wavs is not None else None
+            gt_wav = ground_truth_wavs[idx] if ground_truth_wavs is not None else None
+            conditions = conditioning[idx] if conditioning is not None else None
+            samples.append(self.add_sample(wav, epoch, idx, conditions, prompt_wav, gt_wav, generation_args))
+        return samples
+
+    def get_samples(self, epoch: int = -1, max_epoch: int = -1, exclude_prompted: bool = False,
+                    exclude_unprompted: bool = False, exclude_conditioned: bool = False,
+                    exclude_unconditioned: bool = False) -> tp.Set[Sample]:
+        """Returns a set of samples for this XP. Optionally, you can filter which samples to obtain.
+        Please note that existing samples are loaded during the manager's initialization, and added samples through this
+        manager are also tracked. Any other external changes are not tracked automatically, so creating a new manager
+        is the only way detect them.
+
+        Args:
+            epoch (int): If provided, only return samples corresponding to this epoch.
+            max_epoch (int): If provided, only return samples corresponding to the latest epoch that is <= max_epoch.
+            exclude_prompted (bool): If True, does not include samples that used a prompt.
+            exclude_unprompted (bool): If True, does not include samples that did not use a prompt.
+            exclude_conditioned (bool): If True, excludes samples that used conditioning.
+            exclude_unconditioned (bool): If True, excludes samples that did not use conditioning.
+        Returns:
+            Samples (set of Sample): The retrieved samples matching the provided filters.
+        """
+        if max_epoch >= 0:
+            samples_epoch = max(sample.epoch for sample in self.samples if sample.epoch <= max_epoch)
+        else:
+            samples_epoch = self.latest_epoch if epoch < 0 else epoch
+        samples = {
+            sample
+            for sample in self.samples
+            if (
+                (sample.epoch == samples_epoch) and
+                (not exclude_prompted or sample.prompt is None) and
+                (not exclude_unprompted or sample.prompt is not None) and
+                (not exclude_conditioned or not sample.conditioning) and
+                (not exclude_unconditioned or sample.conditioning)
+            )
+        }
+        return samples
+
+
+def slugify(value: tp.Any, allow_unicode: bool = False):
+    """Process string for safer file naming.
+
+    Taken from https://github.com/django/django/blob/master/django/utils/text.py
+
+    Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
+    dashes to single dashes. Remove characters that aren't alphanumerics,
+    underscores, or hyphens. Convert to lowercase. Also strip leading and
+    trailing whitespace, dashes, and underscores.
+    """
+    value = str(value)
+    if allow_unicode:
+        value = unicodedata.normalize("NFKC", value)
+    else:
+        value = (
+            unicodedata.normalize("NFKD", value)
+            .encode("ascii", "ignore")
+            .decode("ascii")
+        )
+    value = re.sub(r"[^\w\s-]", "", value.lower())
+    return re.sub(r"[-\s]+", "-", value).strip("-_")
+
+
+def _match_stable_samples(samples_per_xp: tp.List[tp.Set[Sample]]) -> tp.Dict[str, tp.List[Sample]]:
+    # Create a dictionary of stable id -> sample per XP
+    stable_samples_per_xp = [{
+        sample.id: sample for sample in samples
+        if sample.prompt is not None or sample.conditioning
+    } for samples in samples_per_xp]
+    # Set of all stable ids
+    stable_ids = {id for samples in stable_samples_per_xp for id in samples.keys()}
+    # Dictionary of stable id -> list of samples. If an XP does not have it, assign None
+    stable_samples = {id: [xp.get(id) for xp in stable_samples_per_xp] for id in stable_ids}
+    # Filter out ids that contain None values (we only want matched samples after all)
+    # cast is necessary to avoid mypy linter errors.
+    return {id: tp.cast(tp.List[Sample], samples) for id, samples in stable_samples.items() if None not in samples}
+
+
+def _match_unstable_samples(samples_per_xp: tp.List[tp.Set[Sample]]) -> tp.Dict[str, tp.List[Sample]]:
+    # For unstable ids, we use a sorted list since we'll match them in order
+    unstable_samples_per_xp = [[
+        sample for sample in sorted(samples, key=lambda x: x.id)
+        if sample.prompt is None and not sample.conditioning
+    ] for samples in samples_per_xp]
+    # Trim samples per xp so all samples can have a match
+    min_len = min([len(samples) for samples in unstable_samples_per_xp])
+    unstable_samples_per_xp = [samples[:min_len] for samples in unstable_samples_per_xp]
+    # Dictionary of index -> list of matched samples
+    return {
+        f'noinput_{i}': [samples[i] for samples in unstable_samples_per_xp] for i in range(min_len)
+    }
+
+
+def get_samples_for_xps(xps: tp.List[dora.XP], **kwargs) -> tp.Dict[str, tp.List[Sample]]:
+    """Gets a dictionary of matched samples across the given XPs.
+    Each dictionary entry maps a sample id to a list of samples for that id. The number of samples per id
+    will always match the number of XPs provided and will correspond to each XP in the same order given.
+    In other words, only samples that can be match across all provided XPs will be returned
+    in order to satisfy this rule.
+
+    There are two types of ids that can be returned: stable and unstable.
+    * Stable IDs are deterministic ids that were computed by the SampleManager given a sample's inputs
+      (prompts/conditioning). This is why we can match them across XPs.
+    * Unstable IDs are of the form "noinput_{idx}" and are generated on-the-fly, in order to map samples
+      that used non-deterministic, random ids. This is the case for samples that did not use prompts or
+      conditioning for their generation. This function will sort these samples by their id and match them
+      by their index.
+
+    Args:
+        xps: a list of XPs to match samples from.
+        start_epoch (int): If provided, only return samples corresponding to this epoch or newer.
+        end_epoch (int): If provided, only return samples corresponding to this epoch or older.
+        exclude_prompted (bool): If True, does not include samples that used a prompt.
+        exclude_unprompted (bool): If True, does not include samples that did not use a prompt.
+        exclude_conditioned (bool): If True, excludes samples that used conditioning.
+        exclude_unconditioned (bool): If True, excludes samples that did not use conditioning.
+    """
+    managers = [SampleManager(xp) for xp in xps]
+    samples_per_xp = [manager.get_samples(**kwargs) for manager in managers]
+    stable_samples = _match_stable_samples(samples_per_xp)
+    unstable_samples = _match_unstable_samples(samples_per_xp)
+    return dict(stable_samples, **unstable_samples)
diff --git a/audiocraft/audiocraft/utils/utils.py b/audiocraft/audiocraft/utils/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..3135d70e949a058095ef84dd87b49384546c465c
--- /dev/null
+++ b/audiocraft/audiocraft/utils/utils.py
@@ -0,0 +1,298 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from concurrent.futures import ProcessPoolExecutor
+from contextlib import contextmanager
+from functools import wraps, lru_cache
+import hashlib
+import json
+import logging
+from pathlib import Path
+import typing as tp
+
+import flashy
+import flashy.distrib
+import omegaconf
+import torch
+from torch.nn.utils.rnn import pad_sequence
+
+
+logger = logging.getLogger(__name__)
+
+
+def model_hash(model: torch.nn.Module) -> str:
+    """Return a model hash. This should allow us to track regressions in model init
+    from the logs of past experiments.
+    """
+    hasher = hashlib.sha1()
+    for p in model.parameters():
+        hasher.update(p.data.cpu().numpy().tobytes())
+    return hasher.hexdigest()
+
+
+def dict_from_config(cfg: omegaconf.DictConfig) -> dict:
+    """Convenience function to map an omegaconf configuration to a dictionary.
+
+    Args:
+        cfg (omegaconf.DictConfig): Original configuration to map to dict.
+    Returns:
+        dict: Config as dictionary object.
+    """
+    dct = omegaconf.OmegaConf.to_container(cfg, resolve=True)
+    assert isinstance(dct, dict)
+    return dct
+
+
+def random_subset(dataset, max_samples: int, seed: int = 42) -> torch.utils.data.Subset:
+    if max_samples >= len(dataset):
+        return dataset
+
+    generator = torch.Generator().manual_seed(seed)
+    perm = torch.randperm(len(dataset), generator=generator)
+    return torch.utils.data.Subset(dataset, perm[:max_samples].tolist())
+
+
+def get_loader(dataset, num_samples: tp.Optional[int], batch_size: int,
+               num_workers: int, seed: int, **kwargs) -> torch.utils.data.DataLoader:
+    """Convenience function to load dataset into a dataloader with optional subset sampling.
+
+    Args:
+        dataset: Dataset to load.
+        num_samples (Optional[int]): Number of samples to limit subset size.
+        batch_size (int): Batch size.
+        num_workers (int): Number of workers for data loading.
+        seed (int): Random seed.
+    """
+    if num_samples is not None:
+        dataset = random_subset(dataset, num_samples, seed)
+
+    dataloader = flashy.distrib.loader(
+        dataset,
+        batch_size=batch_size,
+        num_workers=num_workers,
+        **kwargs
+    )
+    return dataloader
+
+
+def get_dataset_from_loader(dataloader):
+    dataset = dataloader.dataset
+    if isinstance(dataset, torch.utils.data.Subset):
+        return dataset.dataset
+    else:
+        return dataset
+
+
+def multinomial(input: torch.Tensor, num_samples: int, replacement=False, *, generator=None):
+    """torch.multinomial with arbitrary number of dimensions, and number of candidates on the last dimension.
+
+    Args:
+        input (torch.Tensor): The input tensor containing probabilities.
+        num_samples (int): Number of samples to draw.
+        replacement (bool): Whether to draw with replacement or not.
+    Keywords args:
+        generator (torch.Generator): A pseudorandom number generator for sampling.
+    Returns:
+        torch.Tensor: Last dimension contains num_samples indices
+            sampled from the multinomial probability distribution
+            located in the last dimension of tensor input.
+    """
+    input_ = input.reshape(-1, input.shape[-1])
+    output_ = torch.multinomial(input_, num_samples=num_samples, replacement=replacement, generator=generator)
+    output = output_.reshape(*list(input.shape[:-1]), -1)
+    return output
+
+
+def sample_top_k(probs: torch.Tensor, k: int) -> torch.Tensor:
+    """Sample next token from top K values along the last dimension of the input probs tensor.
+
+    Args:
+        probs (torch.Tensor): Input probabilities with token candidates on the last dimension.
+        k (int): The k in “top-k”.
+    Returns:
+        torch.Tensor: Sampled tokens.
+    """
+    top_k_value, _ = torch.topk(probs, k, dim=-1)
+    min_value_top_k = top_k_value[..., [-1]]
+    probs *= (probs >= min_value_top_k).float()
+    probs.div_(probs.sum(dim=-1, keepdim=True))
+    next_token = multinomial(probs, num_samples=1)
+    return next_token
+
+
+def sample_top_p(probs: torch.Tensor, p: float) -> torch.Tensor:
+    """Sample next token from top P probabilities along the last dimension of the input probs tensor.
+
+    Args:
+        probs (torch.Tensor): Input probabilities with token candidates on the last dimension.
+        p (int): The p in “top-p”.
+    Returns:
+        torch.Tensor: Sampled tokens.
+    """
+    probs_sort, probs_idx = torch.sort(probs, dim=-1, descending=True)
+    probs_sum = torch.cumsum(probs_sort, dim=-1)
+    mask = probs_sum - probs_sort > p
+    probs_sort *= (~mask).float()
+    probs_sort.div_(probs_sort.sum(dim=-1, keepdim=True))
+    next_token = multinomial(probs_sort, num_samples=1)
+    next_token = torch.gather(probs_idx, -1, next_token)
+    return next_token
+
+
+class DummyPoolExecutor:
+    """Dummy pool executor to use when we actually have only 1 worker.
+    (e.g. instead of ProcessPoolExecutor).
+    """
+    class DummyResult:
+        def __init__(self, func, *args, **kwargs):
+            self.func = func
+            self.args = args
+            self.kwargs = kwargs
+
+        def result(self):
+            return self.func(*self.args, **self.kwargs)
+
+    def __init__(self, workers, mp_context=None):
+        pass
+
+    def submit(self, func, *args, **kwargs):
+        return DummyPoolExecutor.DummyResult(func, *args, **kwargs)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_tb):
+        return
+
+
+def get_pool_executor(num_workers: int, mp_context=None):
+    return ProcessPoolExecutor(num_workers, mp_context) if num_workers > 1 else DummyPoolExecutor(1)
+
+
+def length_to_mask(lengths: torch.Tensor, max_len: tp.Optional[int] = None) -> torch.Tensor:
+    """Utility function to convert a tensor of sequence lengths to a mask (useful when working on padded sequences).
+    For example: [3, 5] => [[1, 1, 1, 0, 0], [1, 1, 1, 1, 1]]
+
+    Args:
+        lengths (torch.Tensor): tensor with lengths
+        max_len (int): can set the max length manually. Defaults to None.
+    Returns:
+        torch.Tensor: mask with 0s where there is pad tokens else 1s
+    """
+    assert len(lengths.shape) == 1, "Length shape should be 1 dimensional."
+    final_length = lengths.max().item() if not max_len else max_len
+    final_length = max(final_length, 1)  # if all seqs are of len zero we don't want a zero-size tensor
+    return torch.arange(final_length)[None, :].to(lengths.device) < lengths[:, None]
+
+
+def hash_trick(word: str, vocab_size: int) -> int:
+    """Hash trick to pair each word with an index
+
+    Args:
+        word (str): word we wish to convert to an index
+        vocab_size (int): size of the vocabulary
+    Returns:
+        int: index of the word in the embedding LUT
+    """
+    hash = int(hashlib.sha256(word.encode("utf-8")).hexdigest(), 16)
+    return hash % vocab_size
+
+
+def with_rank_rng(base_seed: int = 1234):
+    """Decorator for a function so that the function will use a Random Number Generator
+    whose state depend on the GPU rank. The original RNG state is restored upon returning.
+
+    Args:
+        base_seed (int): Random seed.
+    """
+    def _decorator(fun: tp.Callable):
+        @wraps(fun)
+        def _decorated(*args, **kwargs):
+            state = torch.get_rng_state()
+            seed = base_seed ^ flashy.distrib.rank()
+            torch.manual_seed(seed)
+            logger.debug('Rank dependent seed set to %d', seed)
+            try:
+                return fun(*args, **kwargs)
+            finally:
+                torch.set_rng_state(state)
+                logger.debug('RNG state restored.')
+        return _decorated
+    return _decorator
+
+
+def collate(tensors: tp.List[torch.Tensor], dim: int = 0) -> tp.Tuple[torch.Tensor, torch.Tensor]:
+    """Get a list of tensors and collate them to a single tensor. according to the following logic:
+    - `dim` specifies the time dimension which will be stacked and padded.
+    - The output will contain 1 new dimension (dimension index 0) which will be the size of
+    of the original list.
+
+    Args:
+        tensors (tp.List[torch.Tensor]): List of tensors to collate.
+        dim (int): Dimension which will be stacked and padded.
+    Returns:
+        tp.Tuple[torch.Tensor, torch.Tensor]:
+            torch.Tensor: Stacked and padded tensor. The output will contain 1 new dimension
+                (dimension index 0) which will be the size of the original list.
+            torch.Tensor: Tensor containing length of original tensor sizes (without padding).
+    """
+    tensors = [x.transpose(0, dim) for x in tensors]
+    lens = torch.LongTensor([len(x) for x in tensors])
+    padded_tensors = pad_sequence(tensors)
+    padded_tensors = padded_tensors.transpose(0, 1)
+    padded_tensors = padded_tensors.transpose(1, dim + 1)
+    return padded_tensors, lens
+
+
+# TODO: Move to flashy?
+def copy_state(state: tp.Any, device: tp.Union[torch.device, str] = 'cpu',
+               dtype: tp.Optional[torch.dtype] = None) -> tp.Any:
+    if isinstance(state, torch.Tensor):
+        if dtype is None or not state.is_floating_point():
+            dtype = state.dtype
+        return state.detach().to(device=device, dtype=dtype, copy=True)
+    elif isinstance(state, dict):
+        return {k: copy_state(v, device, dtype) for k, v in state.items()}
+    elif isinstance(state, list):
+        return [copy_state(v, device, dtype) for v in state]
+
+
+# TODO: Move to flashy?
+@contextmanager
+def swap_state(model, state, **kwargs):
+    old_state = copy_state(model.state_dict())
+    model.load_state_dict(state, **kwargs)
+    try:
+        yield
+    finally:
+        model.load_state_dict(old_state)
+
+
+@lru_cache(None)
+def warn_once(logger, msg):
+    """Warn about a given message only once."""
+    logger.warning(msg)
+
+
+def is_jsonable(x: tp.Any):
+    """Check if an object can be serialized into a json:"""
+    try:
+        json.dumps(x)
+        return True
+    except (TypeError, OverflowError):
+        return False
+
+
+def load_clap_state_dict(clap_model, path: tp.Union[str, Path]):
+    """Wrapper around state dict loading of CLAP model
+    addressing compatibility issues between CLAP and AudioCraft
+    HuggingFace transformer version.
+    See: https://github.com/LAION-AI/CLAP/issues/118
+    """
+    from clap_module.factory import load_state_dict  # type: ignore
+    pkg = load_state_dict(path)
+    pkg.pop('text_branch.embeddings.position_ids', None)
+    clap_model.model.load_state_dict(pkg)
diff --git a/audiocraft/config/conditioner/chord2music_inattn.yaml b/audiocraft/config/conditioner/chord2music_inattn.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2569e12f5843d78269f610a70a712f7ce636dd8e
--- /dev/null
+++ b/audiocraft/config/conditioner/chord2music_inattn.yaml
@@ -0,0 +1,45 @@
+# @package __global__
+
+classifier_free_guidance:
+  training_dropout: 0.2
+  inference_coef: 3.0
+
+attribute_dropout:
+  args:
+      active_on_eval: false
+  text: {}
+  chord:
+    chord: 0.5
+  beat:
+    beat: 0.5
+
+fuser:
+  cross_attention_pos_emb: false
+  cross_attention_pos_emb_scale: 1
+  in_attn : true
+  sum: [chord, beat]
+  prepend: [chord, description]
+  cross: []
+  input_interpolate: []
+  
+conditioners:
+  description:
+    model: t5
+    t5:
+      name: t5-base
+      finetune: false
+      word_dropout: 0.2
+      normalize_text: false
+  chord:
+    model: chord
+    chord:
+      name: chord
+  beat:
+    model: beat
+    beat:
+      name: beat
+dataset:
+  train:
+    merge_text_p: 0.25
+    drop_desc_p: 0.5
+    drop_other_p: 0.5
diff --git a/audiocraft/config/conditioner/chroma2music.yaml b/audiocraft/config/conditioner/chroma2music.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..91d37e758ef183678cff3f7a880b6bab2e36b03c
--- /dev/null
+++ b/audiocraft/config/conditioner/chroma2music.yaml
@@ -0,0 +1,46 @@
+# @package __global__
+
+classifier_free_guidance:
+  training_dropout: 0.2
+  inference_coef: 3.0
+
+attribute_dropout:
+  args:
+    active_on_eval: false
+  text: {}
+  wav:
+    self_wav: 0.5
+
+fuser:
+  cross_attention_pos_emb: false
+  cross_attention_pos_emb_scale: 1
+  sum: []
+  prepend: [self_wav, description]
+  cross: []
+  input_interpolate: []
+
+conditioners:
+  self_wav:
+    model: chroma_stem
+    chroma_stem:
+      sample_rate: ${sample_rate}
+      n_chroma: 12
+      radix2_exp: 14
+      argmax: true
+      match_len_on_eval: false
+      eval_wavs: null
+      n_eval_wavs: 100
+      cache_path: null
+  description:
+    model: t5
+    t5:
+      name: t5-base
+      finetune: false
+      word_dropout: 0.2
+      normalize_text: false
+
+dataset:
+  train:
+    merge_text_p: 0.25
+    drop_desc_p: 0.5
+    drop_other_p: 0.5
diff --git a/audiocraft/config/conditioner/chroma_text2music.yaml b/audiocraft/config/conditioner/chroma_text2music.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3a2b685ab82c14a8bfa1e603b9d1f69af29fbd0b
--- /dev/null
+++ b/audiocraft/config/conditioner/chroma_text2music.yaml
@@ -0,0 +1,46 @@
+# @package __global__
+
+classifier_free_guidance:
+  training_dropout: 0.2
+  inference_coef: 3.0
+
+attribute_dropout:
+  args:
+    active_on_eval: false
+  text: {}
+  wav:
+    self_wav: 0.5
+
+fuser:
+  cross_attention_pos_emb: false
+  cross_attention_pos_emb_scale: 1
+  sum: []
+  prepend: [self_wav]
+  cross: [description]
+  input_interpolate: []
+
+conditioners:
+  self_wav:
+    model: chroma_stem
+    chroma_stem:
+      sample_rate: ${sample_rate}
+      n_chroma: 12
+      radix2_exp: 14
+      argmax: true
+      match_len_on_eval: false
+      eval_wavs: null
+      n_eval_wavs: 100
+      cache_path: null
+  description:
+    model: t5
+    t5:
+      name: t5-base
+      finetune: false
+      word_dropout: 0.2
+      normalize_text: false
+
+dataset:
+  train:
+    merge_text_p: 0.25
+    drop_desc_p: 0.5
+    drop_other_p: 0.5
diff --git a/audiocraft/config/conditioner/clapemb2music.yaml b/audiocraft/config/conditioner/clapemb2music.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8500a826e7379b4a8baaf67570e233f7bac7e5da
--- /dev/null
+++ b/audiocraft/config/conditioner/clapemb2music.yaml
@@ -0,0 +1,44 @@
+# @package __global__
+
+classifier_free_guidance:
+  training_dropout: 0.3
+  inference_coef: 3.0
+
+attribute_dropout:
+  text: {}
+  wav: {}
+
+fuser:
+  cross_attention_pos_emb: false
+  cross_attention_pos_emb_scale: 1
+  sum: []
+  prepend: []
+  cross: [description]
+  input_interpolate: []
+
+conditioners:
+  description:
+    model: clap
+    clap:
+      checkpoint: //reference/clap/music_audioset_epoch_15_esc_90.14.pt
+      model_arch: 'HTSAT-base'
+      enable_fusion: false
+      sample_rate: 44100
+      max_audio_length: 10
+      audio_stride: 1
+      dim: 512
+      attribute: description
+      normalize: true
+      quantize: true  # use RVQ quantization
+      n_q: 12
+      bins: 1024
+      kmeans_iters: 50
+      text_p: 0.  # probability of using text embed at train time
+      cache_path: null
+
+dataset:
+  joint_embed_attributes: [description]
+  train:
+    merge_text_p: 0.25
+    drop_desc_p: 0.5
+    drop_other_p: 0.5
diff --git a/audiocraft/config/conditioner/none.yaml b/audiocraft/config/conditioner/none.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c8e33156281e2af7616307da5c05b8094ee012e0
--- /dev/null
+++ b/audiocraft/config/conditioner/none.yaml
@@ -0,0 +1,20 @@
+# @package __global__
+
+# No conditioning
+
+classifier_free_guidance:
+  training_dropout: 0
+  inference_coef: 1
+
+attribute_dropout:
+  text: {}
+  wav: {}
+
+fuser:
+  sum: []
+  concat: []
+  prepend: []
+  cross: []
+  input_interpolate: []
+
+conditioners: null
diff --git a/audiocraft/config/conditioner/text2music.yaml b/audiocraft/config/conditioner/text2music.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2d0fe6cfa3fb33bcdb4f9fd16bd5ab4034c68b7b
--- /dev/null
+++ b/audiocraft/config/conditioner/text2music.yaml
@@ -0,0 +1,30 @@
+# @package __global__
+
+classifier_free_guidance:
+  training_dropout: 0.3
+  inference_coef: 3.0
+
+attribute_dropout: {}
+
+fuser:
+  cross_attention_pos_emb: false
+  cross_attention_pos_emb_scale: 1
+  sum: []
+  prepend: []
+  cross: [description]
+  input_interpolate: []
+
+conditioners:
+  description:
+    model: t5
+    t5:
+      name: t5-base
+      finetune: false
+      word_dropout: 0.3
+      normalize_text: false
+
+dataset:
+  train:
+    merge_text_p: 0.25
+    drop_desc_p: 0.5
+    drop_other_p: 0.5
diff --git a/audiocraft/config/conditioner/text2sound.yaml b/audiocraft/config/conditioner/text2sound.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..555d4b7c3cecf0ec06c8cb25440b2f426c098ad2
--- /dev/null
+++ b/audiocraft/config/conditioner/text2sound.yaml
@@ -0,0 +1,24 @@
+# @package __global__
+
+classifier_free_guidance:
+  training_dropout: 0.1
+  inference_coef: 3.0
+
+attribute_dropout: {}
+
+fuser:
+  cross_attention_pos_emb: false
+  cross_attention_pos_emb_scale: 1
+  sum: []
+  prepend: []
+  cross: [description]
+  input_interpolate: []
+
+conditioners:
+  description:
+    model: t5
+    t5:
+      name: t5-large
+      finetune: false
+      word_dropout: 0.
+      normalize_text: false
diff --git a/audiocraft/config/config.yaml b/audiocraft/config/config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..6b0b7866eafac173fe7b056ad5920be1df57a947
--- /dev/null
+++ b/audiocraft/config/config.yaml
@@ -0,0 +1,75 @@
+# WARNING: This is the base configuration file shared across ALL solvers in AudioCraft
+# Please don't update this file directly. Instead use distinct configuration files
+# to override the below configuration.
+defaults:
+  - _self_
+  - dset: default
+  - solver: default
+
+device: cuda
+dtype: float32
+autocast: false
+autocast_dtype: bfloat16
+seed: 2036
+show: false  # just show the model and its size and exit
+continue_from:  # continue from a given sig or path
+execute_only:  # can be set to generate/evaluate/valid to run that stage
+execute_inplace: false # don't enforce continue_from to be set
+                       # to enable inplace execution of the stage. This assume
+                       # that you know what you are doing and execute stage
+                       # preserving the original xp sig.
+benchmark_no_load: false  # if set to true, will repeat the same batch instead of loading them
+
+efficient_attention_backend: torch  # can be torch or xformers.
+num_threads: 1                      # called with torch.set_num_thread.
+mp_start_method: forkserver               # multiprocessing method (spawn, fork or fork_server).
+
+
+label:  # use this if you want twice the same exp, with a name.
+
+# logging parameters
+logging:
+  level: INFO
+  log_updates: 10
+  log_tensorboard: false
+  log_wandb: false
+tensorboard:
+  with_media_logging: false
+  name:  # optional name for the experiment
+  sub_dir:  # optional sub directory to store tensorboard data
+wandb:
+  with_media_logging: true
+  project:  # project name
+  name:  # optional name for the experiment
+  group:  # optional group
+
+# SLURM launcher configuration.
+slurm:
+  gpus: 4  # convenience parameter, number of GPUs to use.
+  mem_per_gpu: 40  # in GB, total mem is automatically scaled with `gpus`.
+  time: 3600
+  constraint:
+  partition:
+  comment:
+  setup: []
+  exclude: ''
+
+# dora parameters
+dora:
+  # Output folder for all artifacts of an experiment.
+  dir: /checkpoint/${oc.env:USER}/experiments/audiocraft/outputs
+  # The following entries will be ignored by dora when computing the unique XP signature.
+  # Note that slurm.* and dora.* are automatically ignored.
+  exclude: [
+    'device', 'wandb.*', 'tensorboard.*', 'logging.*',
+    'dataset.num_workers', 'eval.num_workers', 'special.*',
+    'metrics.visqol.bin', 'metrics.fad.bin',
+    'execute_only', 'execute_best', 'generate.every',
+    'optim.eager_sync', 'profiler.*', 'deadlock.*',
+    'efficient_attention_backend', 'num_threads', 'mp_start_method',
+  ]
+  use_rendezvous: false
+  # for grids, always run from a clean repo, allowing reliable runs and storing
+  # the exact commit. Your repo must be absolutely pristine clean.
+  # Local `dora run` are not impacted for easier debugging.
+  git_save: true
diff --git a/audiocraft/config/dset/audio/audiocaps_16khz.yaml b/audiocraft/config/dset/audio/audiocaps_16khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..14f5d6a4fcbf4426b7987d4427ca2d98d17d6c5b
--- /dev/null
+++ b/audiocraft/config/dset/audio/audiocaps_16khz.yaml
@@ -0,0 +1,11 @@
+# @package __global__
+
+# AudioCaps dataset
+datasource:
+  max_sample_rate: 16000
+  max_channels: 1
+
+  train: null  # only evaluation set
+  valid: null  # only evaluation set
+  evaluate: egs/audiocaps/audiocaps_16khz
+  generate: egs/audiocaps/audiocaps_16khz # identical to evaluate
diff --git a/audiocraft/config/dset/audio/default.yaml b/audiocraft/config/dset/audio/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..80be23e999c6366cc89ebcf55af6b958c0e45158
--- /dev/null
+++ b/audiocraft/config/dset/audio/default.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+datasource:
+  max_sample_rate: ???
+  max_channels: ???
+
+  train: ???
+  valid: ???
+  evaluate: ???
+  generate: null
diff --git a/audiocraft/config/dset/audio/example.yaml b/audiocraft/config/dset/audio/example.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d559d6d79a1cc05a82bb09f267c446258ef9ca55
--- /dev/null
+++ b/audiocraft/config/dset/audio/example.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+datasource:
+  max_sample_rate: 44100
+  max_channels: 2
+
+  train: egs/example
+  valid: egs/example
+  evaluate: egs/example
+  generate: egs/example
diff --git a/audiocraft/config/dset/audio/train.yaml b/audiocraft/config/dset/audio/train.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..df915cd6ee51ae2af4f413e68e6570a7a73ef770
--- /dev/null
+++ b/audiocraft/config/dset/audio/train.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+datasource:
+  max_sample_rate: 44100
+  max_channels: 2
+
+  train: egs/YT_backing_tracks_0615
+  valid: egs/YT_backing_tracks_0615
+  evaluate: egs/YT_backing_tracks_0615
+  generate: egs/YT_backing_tracks_0615
\ No newline at end of file
diff --git a/audiocraft/config/dset/audio/train_backing.yaml b/audiocraft/config/dset/audio/train_backing.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8da9fa930eba5a27c9955a33dd27e88a4a8f76e6
--- /dev/null
+++ b/audiocraft/config/dset/audio/train_backing.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+datasource:
+  max_sample_rate: 48000
+  max_channels: 2
+
+  train: egs/5_genre_backing
+  valid: egs/musdb_valid
+  evaluate: egs/musdb_valid
+  generate: egs/musdb_valid
\ No newline at end of file
diff --git a/audiocraft/config/dset/default.yaml b/audiocraft/config/dset/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b5d730130e090b38a42984a8a87e1eea01cbf031
--- /dev/null
+++ b/audiocraft/config/dset/default.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+# WARNING: This is a base configuration file shared across ALL solvers in AudioCraft
+# Please don't update this file directly. Instead use distinct configuration files
+# to override the below configuration.
+datasource:
+  train: ???
+  valid: ???
+  evaluate: ???
+  generate: ???
diff --git a/audiocraft/config/dset/internal/music_10k_32khz.yaml b/audiocraft/config/dset/internal/music_10k_32khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..036628abfeaa89279790547bbb5b3ee9dd69cea3
--- /dev/null
+++ b/audiocraft/config/dset/internal/music_10k_32khz.yaml
@@ -0,0 +1,11 @@
+# @package __global__
+
+# high quality music dataset with no artist overlap between splits
+datasource:
+  max_sample_rate: 32000
+  max_channels: 1
+
+  train: egs/music/music_10k_32khz/train
+  valid: egs/music/music_10k_32khz/valid
+  evaluate: egs/music/music_10k_32khz/test
+  generate: egs/music/music_10k_32khz/test # identical to evaluate
diff --git a/audiocraft/config/dset/internal/music_400k_32khz.yaml b/audiocraft/config/dset/internal/music_400k_32khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..7786880ab9c0464a0423d906c18d62bdf7194463
--- /dev/null
+++ b/audiocraft/config/dset/internal/music_400k_32khz.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+datasource:
+  max_sample_rate: 32000
+  max_channels: 1
+
+  train: egs/music/music_400k_32khz/train
+  valid: egs/music/music_400k_32khz/valid
+  evaluate: egs/music/music_400k_32khz/test
+  generate: egs/music/music_400k_32khz/test # identical to evaluate
diff --git a/audiocraft/config/dset/internal/sounds_16khz.yaml b/audiocraft/config/dset/internal/sounds_16khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4f3401a1b44ce300e22f3f64ef9c54d5c013c153
--- /dev/null
+++ b/audiocraft/config/dset/internal/sounds_16khz.yaml
@@ -0,0 +1,12 @@
+# @package __global__
+
+# environmental sounds dataset compiling all datasets
+# with applied filters on tags
+datasource:
+  max_sample_rate: 16000
+  max_channels: 1
+
+  train: egs/sound/sounds_16khz/train
+  valid: egs/sound/sounds_16khz/valid
+  evaluate: egs/sound/sounds_16khz/test
+  generate: egs/sound/sounds_16khz/test # identical to evaluate
diff --git a/audiocraft/config/model/encodec/default.yaml b/audiocraft/config/model/encodec/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ec62c6c8ef9a686890bdca8b8f27a2f1c232205d
--- /dev/null
+++ b/audiocraft/config/model/encodec/default.yaml
@@ -0,0 +1,54 @@
+# @package __global__
+
+compression_model: encodec
+
+encodec:
+  autoencoder: seanet
+  quantizer: rvq
+  sample_rate: ${sample_rate}
+  channels: ${channels}
+  causal: false
+  renormalize: false
+
+seanet:
+  dimension: 128
+  channels: ${channels}
+  causal: ${encodec.causal}
+  n_filters: 32
+  n_residual_layers: 1
+  ratios: [8, 5, 4, 2]
+  activation: ELU
+  activation_params: {"alpha": 1.}
+  norm: weight_norm
+  norm_params: {}
+  kernel_size: 7
+  residual_kernel_size: 3
+  last_kernel_size: 7
+  dilation_base: 2
+  pad_mode: constant
+  true_skip: true
+  compress: 2
+  lstm: 2
+  disable_norm_outer_blocks: 0
+  # Specific encoder or decoder params.
+  # You can also override any param for the encoder or decoder only
+  # by using Hydra `+param=` syntax, i.e.`
+  # `+seanet.decoder.n_filters=64`.
+  decoder:
+    trim_right_ratio: 1.0
+    final_activation: null
+    final_activation_params: null
+  encoder: {}
+
+rvq:
+  n_q: 8
+  q_dropout: false
+  bins: 1024
+  decay: 0.99
+  kmeans_init: true
+  kmeans_iters: 50
+  threshold_ema_dead_code: 2
+  orthogonal_reg_weight: 0.0
+  orthogonal_reg_active_codes_only: false
+
+no_quant: {}
diff --git a/audiocraft/config/model/encodec/encodec_base_causal.yaml b/audiocraft/config/model/encodec/encodec_base_causal.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3ca555bcdc69433f172915400bb71c3b63e68681
--- /dev/null
+++ b/audiocraft/config/model/encodec/encodec_base_causal.yaml
@@ -0,0 +1,11 @@
+# @package __global__
+
+defaults:
+  - encodec/default
+
+encodec:
+  causal: true
+
+rvq:
+  n_q: 32
+  q_dropout: true
diff --git a/audiocraft/config/model/encodec/encodec_large_nq4_s320.yaml b/audiocraft/config/model/encodec/encodec_large_nq4_s320.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5f2d77590afd8a81185358c705a6e42853e257c3
--- /dev/null
+++ b/audiocraft/config/model/encodec/encodec_large_nq4_s320.yaml
@@ -0,0 +1,13 @@
+# @package __global__
+
+defaults:
+  - encodec/default
+
+seanet:
+  # default ratios are [8, 5, 4, 2]
+  n_filters: 64
+
+rvq:
+  bins: 2048
+  n_q: 4
+  q_dropout: false
diff --git a/audiocraft/config/model/encodec/encodec_large_nq4_s640.yaml b/audiocraft/config/model/encodec/encodec_large_nq4_s640.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3fcb7e87f4f700554164b0a58e9927b2f96a2c5a
--- /dev/null
+++ b/audiocraft/config/model/encodec/encodec_large_nq4_s640.yaml
@@ -0,0 +1,13 @@
+# @package __global__
+
+defaults:
+  - encodec/default
+
+seanet:
+  ratios: [8, 5, 4, 4]
+  n_filters: 64
+
+rvq:
+  bins: 2048
+  n_q: 4
+  q_dropout: false
diff --git a/audiocraft/config/model/lm/audiogen_lm.yaml b/audiocraft/config/model/lm/audiogen_lm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..696f74620af193c12208ce66fdb93a37f8ea9d80
--- /dev/null
+++ b/audiocraft/config/model/lm/audiogen_lm.yaml
@@ -0,0 +1,36 @@
+# @package __global__
+
+defaults:
+  - lm/default
+  - override /conditioner: text2sound
+  - override /model/lm/model_scale: small # prefer this group to set model scale instead of transformer_lm keys directly
+
+lm_model: transformer_lm
+
+codebooks_pattern:
+  modeling: delay
+  delay:
+    delays: [0, 1, 2, 3]
+    flatten_first: 0
+    empty_initial: 0
+  unroll:
+    flattening: [0, 1, 2, 3]
+    delays: [0, 0, 0, 0]
+  music_lm:
+    group_by: 2
+  valle:
+    delays: [0, 0, 0]
+
+transformer_lm:
+  n_q: 4
+  card: 2048
+  memory_efficient: true
+  bias_proj: false
+  bias_ff: false
+  bias_attn: false
+  norm_first: true
+  layer_scale: null
+  weight_init: gaussian
+  depthwise_init: current
+  zero_bias_init: true
+  attention_as_float32: false
diff --git a/audiocraft/config/model/lm/default.yaml b/audiocraft/config/model/lm/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2d256ad14ef69d25d62c19b73599937c8546e79b
--- /dev/null
+++ b/audiocraft/config/model/lm/default.yaml
@@ -0,0 +1,47 @@
+# @package __global__
+defaults:
+  - _self_
+  - /model/lm/model_scale: base # prefer this group to set model scale instead of transformer_lm keys directly
+
+lm_model: transformer_lm
+
+codebooks_pattern:
+  modeling: parallel
+
+transformer_lm:
+  dim: 512
+  num_heads: 8
+  num_layers: 8
+  hidden_scale: 4
+  n_q: 8                   # number of streams to model
+  card: 1024
+  dropout: 0.
+  emb_lr: null
+  activation: gelu
+  norm_first: false        # use pre-norm instead of post-norm
+  bias_ff: true            # use bias for the feedforward
+  bias_attn: true          # use bias for the attention
+  bias_proj: true          # use bias for the output projections
+  past_context: null
+  causal: true
+  custom: false                 # use custom MHA implementation
+  memory_efficient: false       # use flash attention
+  attention_as_float32: false   # use float32 for the attention part,
+                                # recommended at the moment when memory_efficient is True.
+  layer_scale: null
+  positional_embedding: sin     # positional embedding strategy (sin, rope, or sin_rope).
+  xpos: false                   # apply xpos decay (rope only).
+  checkpointing: none      # layer checkpointing method, can be none, torch, xformers_default.
+                           # torch is the slowest but uses the least memory,
+                           # xformers_default is somewhere in between.
+  weight_init: null     # weight initialization (null, gaussian or uniform)
+  depthwise_init: null  # perform depthwise initialization (null, current, global)
+  zero_bias_init: false # initialize bias to zero if bias in linears and
+                        # if a weight_init method is used.
+  norm: layer_norm             # normalization method to use in transformer.
+  cross_attention: false
+  qk_layer_norm: false
+  qk_layer_norm_cross: false
+  attention_dropout: null
+  kv_repeat: 1
+  two_step_cfg: false          # whether to do true 2 steps CFG, potentially resolving some padding issues or not...
diff --git a/audiocraft/config/model/lm/model_scale/base.yaml b/audiocraft/config/model/lm/model_scale/base.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3da88d2305e4c380435de1a3eecfe311ecfc82f9
--- /dev/null
+++ b/audiocraft/config/model/lm/model_scale/base.yaml
@@ -0,0 +1,3 @@
+# @package __global__
+
+# overrides nothing because default is already transformer base (~ 60M params)
diff --git a/audiocraft/config/model/lm/model_scale/large.yaml b/audiocraft/config/model/lm/model_scale/large.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..d355bfb93618003ac8994bc093eb7bc96ac60114
--- /dev/null
+++ b/audiocraft/config/model/lm/model_scale/large.yaml
@@ -0,0 +1,7 @@
+# @package _global_
+
+# gpt2 inspired, even bigger (~3.3B params)
+transformer_lm:
+  dim: 2048
+  num_heads: 32
+  num_layers: 48
diff --git a/audiocraft/config/model/lm/model_scale/medium.yaml b/audiocraft/config/model/lm/model_scale/medium.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..c825d1ff6c3b8cc9ae4959a898e14b40409d95e8
--- /dev/null
+++ b/audiocraft/config/model/lm/model_scale/medium.yaml
@@ -0,0 +1,7 @@
+# @package _global_
+
+# gpt2 like (~1.5B params)
+transformer_lm:
+  dim: 1536
+  num_heads: 24
+  num_layers: 48
diff --git a/audiocraft/config/model/lm/model_scale/medium_small.yaml b/audiocraft/config/model/lm/model_scale/medium_small.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..8debdc58182e340dc19ec6fc1c345d15de9d0e46
--- /dev/null
+++ b/audiocraft/config/model/lm/model_scale/medium_small.yaml
@@ -0,0 +1,8 @@
+# @package _global_
+
+# ???M
+
+transformer_lm:
+  dim: 1280
+  num_heads: 20
+  num_layers: 36
diff --git a/audiocraft/config/model/lm/model_scale/small.yaml b/audiocraft/config/model/lm/model_scale/small.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..88d89cb5ac1b183fb3a9092834cea83aa16c70a8
--- /dev/null
+++ b/audiocraft/config/model/lm/model_scale/small.yaml
@@ -0,0 +1,8 @@
+# @package _global_
+
+# 300M Param.
+
+transformer_lm:
+  dim: 1024
+  num_heads: 16
+  num_layers: 24
diff --git a/audiocraft/config/model/lm/model_scale/xsmall.yaml b/audiocraft/config/model/lm/model_scale/xsmall.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..e98d4370d4fe7497f12aeb58f092a88797d1afa1
--- /dev/null
+++ b/audiocraft/config/model/lm/model_scale/xsmall.yaml
@@ -0,0 +1,8 @@
+# @package _global_
+# just used for debugging or when we just want to populate the cache
+# and do not care about training.
+
+transformer_lm:
+  dim: 64
+  num_heads: 2
+  num_layers: 2
diff --git a/audiocraft/config/model/lm/musicgen_lm.yaml b/audiocraft/config/model/lm/musicgen_lm.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5bc87a628789a34e381e2aa8ba5ef6ed780669d7
--- /dev/null
+++ b/audiocraft/config/model/lm/musicgen_lm.yaml
@@ -0,0 +1,36 @@
+# @package __global__
+
+defaults:
+  - lm/default
+  - override /conditioner: text2music
+  - override /model/lm/model_scale: small # prefer this group to set model scale instead of transformer_lm keys directly
+
+lm_model: transformer_lm
+
+codebooks_pattern:
+  modeling: delay
+  delay:
+    delays: [0, 1, 2, 3]
+    flatten_first: 0
+    empty_initial: 0
+  unroll:
+    flattening: [0, 1, 2, 3]
+    delays: [0, 0, 0, 0]
+  music_lm:
+    group_by: 2
+  valle:
+    delays: [0, 0, 0]
+
+transformer_lm:
+  n_q: 4
+  card: 2048
+  memory_efficient: true
+  bias_proj: false
+  bias_ff: false
+  bias_attn: false
+  norm_first: true
+  layer_scale: null
+  weight_init: gaussian
+  depthwise_init: current
+  zero_bias_init: true
+  attention_as_float32: false
diff --git a/audiocraft/config/model/none.yaml b/audiocraft/config/model/none.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1d4169f468d462c794ee6ed25017c3d78ae45d06
--- /dev/null
+++ b/audiocraft/config/model/none.yaml
@@ -0,0 +1,4 @@
+# @package __global__
+
+# This file exist so that model is recognized as a config group
+# by Hydra, and Dora. A bit weird we might need a better fix someday.
diff --git a/audiocraft/config/model/score/basic.yaml b/audiocraft/config/model/score/basic.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..75fbc3783942602beaddaa38d0aca977aeee2dda
--- /dev/null
+++ b/audiocraft/config/model/score/basic.yaml
@@ -0,0 +1,17 @@
+# @package _global_
+
+diffusion_unet:
+  hidden: 48
+  depth: 4
+  res_blocks: 1
+  norm_groups: 4
+  kernel: 8
+  stride: 4
+  growth: 4
+  max_channels: 10_000
+  dropout: 0.
+  emb_all_layers: true
+  bilstm: false
+  codec_dim: null
+  transformer: false
+  cross_attention: false
\ No newline at end of file
diff --git a/audiocraft/config/solver/audiogen/audiogen_base_16khz.yaml b/audiocraft/config/solver/audiogen/audiogen_base_16khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..dd6aee785c74db19ce9d6f488e68e6eeb471c026
--- /dev/null
+++ b/audiocraft/config/solver/audiogen/audiogen_base_16khz.yaml
@@ -0,0 +1,70 @@
+# @package __global__
+
+# This is the training loop solver
+# for the base AudioGen model (text-to-sound)
+# on monophonic audio sampled at 16 kHz
+# using a similar EnCodec+LM setup to MusicGen
+defaults:
+  - audiogen/default
+  - /model: lm/audiogen_lm
+  - override /dset: audio/default
+  - _self_
+
+autocast: true
+autocast_dtype: float16
+
+# EnCodec large trained on mono-channel music audio sampled at 16khz
+# with a total stride of 320 leading to 50 frames/s.
+# rvq.n_q=4, rvq.bins=2048, no quantization dropout
+# (transformer_lm card and n_q must be compatible)
+compression_model_checkpoint: //reference/bd44a852/checkpoint.th
+
+channels: 1
+sample_rate: 16000
+
+deadlock:
+  use: true  # deadlock detection
+
+dataset:
+  batch_size: 128  # matching AudioGen paper setup (256 * mix_p=0.5 = 128)
+  num_workers: 10
+  segment_duration: 10
+  min_segment_ratio: 1.0
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+  external_metadata_source: null
+  # sample mixing augmentation at train time
+  train:
+    batch_size: 256  # matching AudioGen paper setup
+    aug_p: 0.5  # perform audio mixing 50% of the time
+    mix_p: 0.5  # proportion of batch items mixed together
+                # important: note that this will reduce the
+                # actual batch size used at train time
+                # which will be equal to mix_p * batch_size
+    mix_snr_low: -5
+    mix_snr_high: 5
+    mix_min_overlap: 0.5
+
+generate:
+  lm:
+    use_sampling: true
+    top_k: 250
+    top_p: 0.0
+
+optim:
+  epochs: 100
+  optimizer: adamw
+  lr: 5e-4
+  ema:
+    use: true
+    updates: 10
+    device: cuda
+
+logging:
+  log_tensorboard: true
+
+schedule:
+  lr_scheduler: inverse_sqrt
+  inverse_sqrt:
+    warmup: 3000
+    warmup_init_lr: 0.0
diff --git a/audiocraft/config/solver/audiogen/debug.yaml b/audiocraft/config/solver/audiogen/debug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fbda8281c6d552d9445e04fee498641a26549aa5
--- /dev/null
+++ b/audiocraft/config/solver/audiogen/debug.yaml
@@ -0,0 +1,52 @@
+# @package __global__
+
+# This is a minimal debugging configuration
+# for MusicGen training solver
+defaults:
+  - audiogen/default
+  - /model: lm/audiogen_lm
+  - override /model/lm/model_scale: xsmall
+  - override /dset: audio/example
+  - _self_
+
+autocast: false
+compression_model_checkpoint: null
+
+codebooks_pattern:
+  modeling: parallel
+
+channels: 1
+sample_rate: 16000
+
+deadlock:
+  use: false  # deadlock detection
+
+dataset:
+  batch_size: 4
+  segment_duration: 5
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+
+generate:
+  audio:
+    strategy: peak
+  lm:
+    use_sampling: false
+    top_k: 0
+    top_p: 0.0
+
+checkpoint:
+  save_every: 0
+  keep_last: 0
+
+optim:
+  epochs: 2
+  updates_per_epoch: 10
+  optimizer: adamw
+  lr: 1e-4
+
+logging:
+  log_tensorboard: true
+
+schedule:
+  lr_scheduler: null
diff --git a/audiocraft/config/solver/audiogen/default.yaml b/audiocraft/config/solver/audiogen/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..afee63c65e0dd7350e3e89d2133bbca221d17631
--- /dev/null
+++ b/audiocraft/config/solver/audiogen/default.yaml
@@ -0,0 +1,40 @@
+# @package __global__
+
+defaults:
+  - /solver/musicgen/default
+  - _self_
+  - /solver/audiogen/evaluation: none
+  - override /dset: audio/default
+
+# See config/solver/musicgen/default.yaml for a list of possible values.
+# We only keep the most important here.
+
+autocast: true
+autocast_dtype: float16
+
+solver: audiogen
+sample_rate: ???
+channels: ???
+compression_model_checkpoint: ???
+
+tokens:
+  padding_with_special_token: false
+
+dataset:
+  batch_size: 128
+  segment_duration: 10
+  min_segment_ratio: 1.0  # lower values such as 0.5 result in generations with a lot of silence.
+
+optim:
+  epochs: 100
+  updates_per_epoch: 2000
+  lr: 1e-4
+  optimizer: adamw
+  max_norm: 1.0
+  adam:
+    betas: [0.9, 0.95]
+    weight_decay: 0.1
+    eps: 1e-8
+
+schedule:
+  lr_scheduler: null
diff --git a/audiocraft/config/solver/audiogen/evaluation/none.yaml b/audiocraft/config/solver/audiogen/evaluation/none.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1e739995ed6488700527529862a7a24f1afdcc7a
--- /dev/null
+++ b/audiocraft/config/solver/audiogen/evaluation/none.yaml
@@ -0,0 +1,5 @@
+# @package __global__
+
+dataset:
+  evaluate:
+    num_samples: 10000
diff --git a/audiocraft/config/solver/audiogen/evaluation/objective_eval.yaml b/audiocraft/config/solver/audiogen/evaluation/objective_eval.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..32fcc10033f3c3ff317216fe2876c65c6834e59b
--- /dev/null
+++ b/audiocraft/config/solver/audiogen/evaluation/objective_eval.yaml
@@ -0,0 +1,29 @@
+# @package __global__
+
+# Setup for execute only on audiocaps for audio generation
+# evaluation with objective metrics
+# execute_only=evaluate
+
+dataset:
+  max_audio_duration: null
+  # ensure the proper values are broadcasted here for evaluate
+  evaluate:
+    min_audio_duration: 1.  # some metrics requires a minimum audio length
+    max_audio_duration: null  # all samples from audiocaps should be ~10s
+    num_samples: null
+    segment_duration: null
+  generate:
+    min_audio_duration: 1.
+    max_audio_duration: null
+    num_samples: 500
+
+evaluate:
+  metrics:
+    fad: true
+    kld: true
+    text_consistency: true
+
+metrics:
+  kld:
+    passt:
+      pretrained_length: 10  # similarly to reported results in AudioGen paper
diff --git a/audiocraft/config/solver/compression/debug.yaml b/audiocraft/config/solver/compression/debug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..54dac175278d4ff509b0e44905d6b6195441f2c6
--- /dev/null
+++ b/audiocraft/config/solver/compression/debug.yaml
@@ -0,0 +1,55 @@
+# @package __global__
+
+defaults:
+  - compression/default
+  - /model: encodec/encodec_base_causal
+  - override /dset: audio/example
+  - _self_
+
+channels: 1
+sample_rate: 16000
+
+# debug config uses just L1
+losses:
+  adv: 0.
+  feat: 0.
+  l1: 1.
+  mel: 0.
+  msspec: 0.
+# no balancer
+balancer:
+  balance_grads: false
+  ema_decay: 1.
+  total_norm: 1.
+  per_batch_item: false
+# no adversaries
+adversarial:
+  adversaries: []
+  adv_loss: hinge
+  feat_loss: l1
+
+# faster model for local dev
+seanet:
+  dimension: 16
+  n_filters: 4
+
+# very small dataset
+dataset:
+  batch_size: 8
+  num_workers: 10
+  num_samples: 100
+  segment_duration: 1
+  evaluate:
+    batch_size: 32
+  generate:
+    batch_size: 1
+    num_samples: 5
+    segment_duration: 10
+
+# limited training
+evaluate:
+  every: 5
+generate:
+  every: 5
+optim:
+  epochs: 50
diff --git a/audiocraft/config/solver/compression/default.yaml b/audiocraft/config/solver/compression/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..41c812ba9ff8afe7ee10302ad5b9f05b745877d9
--- /dev/null
+++ b/audiocraft/config/solver/compression/default.yaml
@@ -0,0 +1,160 @@
+# @package __global__
+
+defaults:
+  - ../default
+  - override /dset: audio/default
+  - _self_
+
+solver: compression
+sample_rate: ???
+channels: ???
+
+# loss balancing
+losses:
+  adv: 4.
+  feat: 4.
+  l1: 0.1
+  mel: 0.
+  msspec: 2.
+  sisnr: 0.
+balancer:
+  balance_grads: true
+  ema_decay: 0.999
+  per_batch_item: true
+  total_norm: 1.
+
+adversarial:
+  every: 1
+  adversaries: [msstftd]
+  adv_loss: hinge
+  feat_loss: l1
+
+# losses hyperparameters
+l1: {}
+l2: {}
+mrstft:
+  factor_sc: .5
+  factor_mag: .5
+  normalized: false
+mel:
+  sample_rate: ${sample_rate}
+  n_fft: 1024
+  hop_length: 256
+  win_length: 1024
+  n_mels: 64
+  f_min: 64
+  f_max: null
+  normalized: false
+  floor_level: 1e-5
+sisnr:
+  sample_rate: ${sample_rate}
+  segment: 5.
+msspec:
+  sample_rate: ${sample_rate}
+  range_start: 6
+  range_end: 11
+  n_mels: 64
+  f_min: 64
+  f_max: null
+  normalized: true
+  alphas: false
+  floor_level: 1e-5
+
+# metrics
+metrics:
+  visqol:
+    mode: audio
+    bin: null  # path to visqol install
+    model: tcdaudio14_aacvopus_coresv_svrnsim_n.68_g.01_c1.model # visqol v3
+
+# adversaries hyperparameters
+msstftd:
+  in_channels: 1
+  out_channels: 1
+  filters: 32
+  norm: weight_norm
+  n_ffts: [1024, 2048, 512, 256, 128]
+  hop_lengths: [256, 512, 128, 64, 32]
+  win_lengths: [1024, 2048, 512, 256, 128]
+  activation: LeakyReLU
+  activation_params: {negative_slope: 0.3}
+msd:
+  in_channels: 1
+  out_channels: 1
+  scale_norms: [spectral_norm, weight_norm, weight_norm]
+  kernel_sizes: [5, 3]
+  filters: 16
+  max_filters: 1024
+  downsample_scales: [4, 4, 4, 4]
+  inner_kernel_sizes: null
+  groups: [4, 4, 4, 4]
+  strides: null
+  paddings: null
+  activation: LeakyReLU
+  activation_params: {negative_slope: 0.3}
+mpd:
+  in_channels: 1
+  out_channels: 1
+  periods: [2, 3, 5, 7, 11]
+  n_layers: 5
+  kernel_size: 5
+  stride: 3
+  filters: 8
+  filter_scales: 4
+  max_filters: 1024
+  activation: LeakyReLU
+  activation_params: {negative_slope: 0.3}
+  norm: weight_norm
+
+# data hyperparameters
+dataset:
+  batch_size: 64
+  num_workers: 10
+  segment_duration: 1
+  train:
+    num_samples: 500000
+  valid:
+    num_samples: 10000
+  evaluate:
+    batch_size: 32
+    num_samples: 10000
+  generate:
+    batch_size: 32
+    num_samples: 50
+    segment_duration: 10
+
+# solver hyperparameters
+evaluate:
+  every: 25
+  num_workers: 5
+  metrics:
+    visqol: false
+    sisnr: true
+generate:
+  every: 25
+  num_workers: 5
+  audio:
+    sample_rate: ${sample_rate}
+
+# checkpointing schedule
+checkpoint:
+  save_last: true
+  save_every: 25
+  keep_last: 10
+  keep_every_states: null
+
+# optimization hyperparameters
+optim:
+  epochs: 200
+  updates_per_epoch: 2000
+  lr: 3e-4
+  max_norm: 0.
+  optimizer: adam
+  adam:
+    betas: [0.5, 0.9]
+    weight_decay: 0.
+  ema:
+    use: true         # whether to use EMA or not
+    updates: 1        # update at every step
+    device: ${device} # device for EMA, can be put on GPU if more frequent updates
+    decay: 0.99       # EMA decay value, if null, no EMA is used
diff --git a/audiocraft/config/solver/compression/encodec_audiogen_16khz.yaml b/audiocraft/config/solver/compression/encodec_audiogen_16khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..654deaa01ba9cace3f7144cc91921791c081b32a
--- /dev/null
+++ b/audiocraft/config/solver/compression/encodec_audiogen_16khz.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+defaults:
+  - compression/default
+  - /model: encodec/encodec_large_nq4_s320
+  - override /dset: audio/default
+  - _self_
+
+channels: 1
+sample_rate: 16000
diff --git a/audiocraft/config/solver/compression/encodec_base_24khz.yaml b/audiocraft/config/solver/compression/encodec_base_24khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..018ad1cd61af84b616ad3088f055e8eaa36729eb
--- /dev/null
+++ b/audiocraft/config/solver/compression/encodec_base_24khz.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+defaults:
+  - compression/default
+  - /model: encodec/encodec_base_causal
+  - override /dset: audio/default
+  - _self_
+
+channels: 1
+sample_rate: 24000
diff --git a/audiocraft/config/solver/compression/encodec_musicgen_32khz.yaml b/audiocraft/config/solver/compression/encodec_musicgen_32khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..eca4b90fb221372dace164fe59bb15822207a980
--- /dev/null
+++ b/audiocraft/config/solver/compression/encodec_musicgen_32khz.yaml
@@ -0,0 +1,10 @@
+# @package __global__
+
+defaults:
+  - compression/default
+  - /model: encodec/encodec_large_nq4_s640
+  - override /dset: audio/default
+  - _self_
+
+channels: 1
+sample_rate: 32000
diff --git a/audiocraft/config/solver/default.yaml b/audiocraft/config/solver/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..2981c54c7c56e234c27f1bbeeb6ebdf23c64e0ff
--- /dev/null
+++ b/audiocraft/config/solver/default.yaml
@@ -0,0 +1,109 @@
+# @package __global__
+
+# WARNING: This is a base configuration file shared across ALL solvers in AudioCraft
+# Please don't update this file directly. Instead use distinct configuration files
+# to override the below configuration.
+solver: ???
+
+fsdp:
+  use: false  # should we use FSDP.
+  param_dtype: float16  # equivalent to autocast_dtype for FSDP.
+  reduce_dtype: float32  # gradient averaging dtype, float32 will give max stability.
+  buffer_dtype: float32  # dtype used for buffers, we don't have much buffers, so let's leave it.
+  sharding_strategy: shard_grad_op  # can be shard_grad_op or full_shard.
+                                    # full_shard will use less memory but slower ??
+  per_block: true  # If True, uses nested FSDP.
+
+profiler:
+  enabled: false
+
+deadlock:
+  use: false
+  timeout: 600
+
+dataset:
+  batch_size: ???
+  num_workers: 10
+  segment_duration: null
+  num_samples: null
+  return_info: false
+  shuffle: false
+  sample_on_duration: true
+  sample_on_weight: true
+  min_segment_ratio: 0.5
+  train:
+    num_samples: null
+    shuffle: true
+    shuffle_seed: 0  # if you want to sample the data differently.
+    permutation_on_files: false
+  valid:
+    num_samples: null
+  evaluate:
+    num_samples: null
+  generate:
+    num_samples: null
+    return_info: true
+
+checkpoint:
+  save_last: true
+  save_every: null
+  keep_last: null
+  keep_every_states: null
+
+generate:
+  every: null
+  path: 'samples'
+  audio:
+    format: 'mp3'
+    strategy: 'clip'
+    sample_rate: null
+  lm:
+    use_sampling: false
+    temp: 1.0
+    top_k: 0
+    top_p: 0.0
+evaluate:
+  every: null
+  num_workers: 5
+  truncate_audio: null
+  fixed_generation_duration: null  # in secs
+  metrics:
+    base: true  # run default evaluation (e.g. like train/valid stage)
+
+optim:
+  epochs: ???
+  updates_per_epoch: null
+  lr: ???
+  optimizer: ???
+  adam:
+    betas: [0.9, 0.999]
+    weight_decay: 0.
+  ema:
+    use: false  # whether to use EMA or not
+    updates: ${optim.updates_per_epoch}  # frequency of updates of the EMA
+    device: cpu  # device for EMA, can be put on GPU if more frequent updates
+    decay: 0.99  # EMA decay value, if null, no EMA is used
+  grad_accum_steps: 1
+
+schedule:
+  lr_scheduler: null
+  step:
+    step_size: null
+    gamma: null
+  exponential:
+    lr_decay: null
+  cosine:
+    warmup: null
+    lr_min_ratio: 0.0
+    cycle_length: 1.0
+  polynomial_decay:
+    warmup: null
+    zero_lr_warmup_steps: 0
+    end_lr: 0.0
+    power: 1
+  inverse_sqrt:
+    warmup: null
+    warmup_init_lr: 0.0
+  linear_warmup:
+    warmup: null
+    warmup_init_lr: 0.0
diff --git a/audiocraft/config/solver/musicgen/debug.yaml b/audiocraft/config/solver/musicgen/debug.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..ec658f9d2fb0262cc8eab19d0cf333963c646a98
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/debug.yaml
@@ -0,0 +1,55 @@
+# @package __global__
+
+# This is a minimal debugging configuration
+# for MusicGen training solver
+defaults:
+  - musicgen/default
+  - /model: lm/musicgen_lm
+  - override /model/lm/model_scale: xsmall
+  - override /dset: audio/example
+  - _self_
+
+autocast: false
+compression_model_checkpoint: //pretrained/debug_compression_model
+transformer_lm:
+  n_q: 4
+  card: 400
+
+codebooks_pattern:
+  modeling: parallel
+
+channels: 1
+sample_rate: 32000
+
+deadlock:
+  use: false  # deadlock detection
+
+dataset:
+  batch_size: 4
+  segment_duration: 5
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+
+generate:
+  audio:
+    strategy: peak
+  lm:
+    use_sampling: false
+    top_k: 0
+    top_p: 0.0
+
+checkpoint:
+  save_every: 0
+  keep_last: 0
+
+optim:
+  epochs: 2
+  updates_per_epoch: 10
+  optimizer: adamw
+  lr: 1e-4
+
+logging:
+  log_tensorboard: true
+
+schedule:
+  lr_scheduler: null
diff --git a/audiocraft/config/solver/musicgen/default.yaml b/audiocraft/config/solver/musicgen/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..16dc85d1a8b64b03eb4d4dcad1ae71e39f23455f
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/default.yaml
@@ -0,0 +1,120 @@
+# @package __global__
+
+defaults:
+  - /solver/default
+  - /conditioner: none
+  - _self_
+  - /solver/musicgen/evaluation: none
+  - override /dset: audio/default
+
+autocast: true
+autocast_dtype: float16
+
+solver: musicgen
+sample_rate: ???
+channels: ???
+compression_model_checkpoint: ???
+
+tokens:
+  padding_with_special_token: false
+
+cache:
+  path:
+  write: false
+  write_shard: 0
+  write_num_shards: 1
+
+
+dataset:
+  batch_size: 128
+  num_workers: 10
+  segment_duration: 30
+  min_segment_ratio: 0.8  # lower values such as 0.5 result in generations with a lot of silence.
+  return_info: true
+  train:
+    num_samples: 1000000 # need a randomly large number here for AudioDataset
+  valid:
+    num_samples: 10000
+  generate:
+    num_samples: 5
+
+metrics:
+  fad:
+    use_gt: false
+    model: tf
+    tf:
+      bin: null  # path to local frechet_audio_distance code
+      model_path: //reference/fad/vggish_model.ckpt
+  kld:
+    use_gt: false
+    model: passt
+    passt:
+      pretrained_length: 20
+  text_consistency:
+    use_gt: false
+    model: clap
+    clap:
+      model_path: //reference/clap/music_audioset_epoch_15_esc_90.14.pt
+      model_arch: 'HTSAT-base'
+      enable_fusion: false
+  chroma_cosine:
+    use_gt: false
+    model: chroma_base
+    chroma_base:
+      sample_rate: ${sample_rate}
+      n_chroma: 12
+      radix2_exp: 14
+      argmax: true
+
+generate:
+  every: 25
+  num_workers: 4
+  path: samples
+  audio:
+    format: wav
+    strategy: loudness
+    sample_rate: ${sample_rate}
+    loudness_headroom_db: 14
+  lm:
+    prompted_samples: true
+    unprompted_samples: true
+    gen_gt_samples: false
+    prompt_duration: null   # if not set, will use dataset.generate.segment_duration / 4
+    gen_duration: null      # if not set, will use dataset.generate.segment_duration
+    remove_prompts: false
+    # generation params
+    use_sampling: false
+    temp: 1.0
+    top_k: 0
+    top_p: 0.0
+evaluate:
+  every: 25
+  num_workers: 4
+  metrics:
+    base: false
+    fad: false
+    kld: false
+    text_consistency: false
+    chroma_cosine: false
+
+checkpoint:
+  save_last: true
+  save_every: 25
+  keep_last: 10
+  keep_every_states: null
+
+optim:
+  epochs: 200
+  updates_per_epoch: 2000
+  lr: 1e-4
+  optimizer: adamw
+  max_norm: 1.0
+  eager_sync: true
+  adam:
+    betas: [0.9, 0.95]
+    weight_decay: 0.1
+    eps: 1e-8
+  grad_accum_steps: 1
+
+schedule:
+  lr_scheduler: null
diff --git a/audiocraft/config/solver/musicgen/dummy_train.yaml b/audiocraft/config/solver/musicgen/dummy_train.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..40aa99997fb49ca606e88049ddc93882bd599ea0
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/dummy_train.yaml
@@ -0,0 +1,65 @@
+# @package __global__
+
+# This is the training loop solver
+# for the base MusicGen model (text-to-music)
+defaults:
+  - musicgen/default
+  - /model: lm/musicgen_lm
+  - override /dset: audio/train_backing
+  - _self_
+
+autocast: true
+autocast_dtype: float16
+
+# EnCodec large trained on mono-channel music audio sampled at 32khz
+# with a total stride of 640 leading to 50 frames/s.
+# rvq.n_q=4, rvq.bins=2048, no quantization dropout
+# (transformer_lm card and n_q must be compatible)
+compression_model_checkpoint: //pretrained/facebook/encodec_32khz
+
+channels: 1
+sample_rate: 32000
+
+deadlock:
+  use: true  # deadlock detection
+
+dataset:
+  batch_size: 8  # 1 GPU(A100)
+  num_workers: 8
+  segment_duration: 30
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+  valid:
+    num_samples: 4
+
+generate:
+  lm:
+    use_sampling: true
+    top_k: 250
+    top_p: 0.0
+
+checkpoint:
+  save_last: true
+  save_every: 25
+  keep_every_states: null
+
+optim:
+  epochs: 1
+  updates_per_epoch: 1
+  optimizer: dadam
+  lr: 1e-32
+  max_norm: 1.0
+  ema:
+    use: false
+    updates: 10
+    device: cuda
+
+logging:
+  log_tensorboard: false
+
+schedule:
+  lr_scheduler: cosine
+  cosine:
+    warmup: 0
+    lr_min_ratio: 0.0
+    cycle_length: 1.0
\ No newline at end of file
diff --git a/audiocraft/config/solver/musicgen/evaluation/none.yaml b/audiocraft/config/solver/musicgen/evaluation/none.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1e739995ed6488700527529862a7a24f1afdcc7a
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/evaluation/none.yaml
@@ -0,0 +1,5 @@
+# @package __global__
+
+dataset:
+  evaluate:
+    num_samples: 10000
diff --git a/audiocraft/config/solver/musicgen/evaluation/objective_eval.yaml b/audiocraft/config/solver/musicgen/evaluation/objective_eval.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..4881e9d86cddf36b306a75fb498253e1e12ec5be
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/evaluation/objective_eval.yaml
@@ -0,0 +1,24 @@
+# @package __global__
+
+# Setup for execute only on musiccaps for audio generation
+# evaluation with objective metrics
+# execute_only=evaluate
+
+dataset:
+  max_audio_duration: null
+  # ensure the proper values are broadcasted here for evaluate
+  evaluate:
+    min_audio_duration: 1.  # some metrics requires a minimum audio length
+    max_audio_duration: null  # all samples from musiccaps should be < 20s
+    num_samples: null
+    segment_duration: null
+  generate:
+    min_audio_duration: 1.
+    max_audio_duration: null
+    num_samples: 500
+
+evaluate:
+  metrics:
+    fad: true
+    kld: true
+    text_consistency: true
diff --git a/audiocraft/config/solver/musicgen/multigpu_finetune.yaml b/audiocraft/config/solver/musicgen/multigpu_finetune.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..fa1ee8a373cffb9290879275d9a7d29beb6a7cd1
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/multigpu_finetune.yaml
@@ -0,0 +1,63 @@
+# @package __global__
+
+# This is the training loop solver
+# for the base MusicGen model (text-to-music)
+defaults:
+  - musicgen/default
+  - /model: lm/musicgen_lm
+  - _self_
+
+autocast: true
+autocast_dtype: float16
+
+# EnCodec large trained on mono-channel music audio sampled at 32khz
+# with a total stride of 640 leading to 50 frames/s.
+# rvq.n_q=4, rvq.bins=2048, no quantization dropout
+# (transformer_lm card and n_q must be compatible)
+compression_model_checkpoint: //pretrained/facebook/encodec_32khz
+
+channels: 1
+sample_rate: 32000
+
+deadlock:
+  use: true  # deadlock detection
+
+dataset:
+  batch_size: 8  # 4 GPUs(3090)
+  num_workers: 8
+  segment_duration: 30
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+  valid:
+    num_samples: 4
+
+generate:
+  lm:
+    use_sampling: true
+    top_k: 250
+    top_p: 0.0
+
+checkpoint:
+  save_last: true
+  save_every: 25
+  keep_every_states: null
+
+optim:
+  epochs: 100
+  optimizer: dadam
+  lr: 1.0
+  max_norm: 1.0
+  ema:
+    use: false
+    updates: 10
+    device: cuda
+
+logging:
+  log_tensorboard: true
+
+schedule:
+  lr_scheduler: cosine
+  cosine:
+    warmup: 5
+    lr_min_ratio: 0.0
+    cycle_length: 1.0
\ No newline at end of file
diff --git a/audiocraft/config/solver/musicgen/musicgen_base_32khz.yaml b/audiocraft/config/solver/musicgen/musicgen_base_32khz.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..b32c9c898a70718f91af862caa79f5553a5107e1
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/musicgen_base_32khz.yaml
@@ -0,0 +1,55 @@
+# @package __global__
+
+# This is the training loop solver
+# for the base MusicGen model (text-to-music)
+# on monophonic audio sampled at 32 kHz
+defaults:
+  - musicgen/default
+  - /model: lm/musicgen_lm
+  - override /dset: audio/default
+  - _self_
+
+autocast: true
+autocast_dtype: float16
+
+# EnCodec large trained on mono-channel music audio sampled at 32khz
+# with a total stride of 640 leading to 50 frames/s.
+# rvq.n_q=4, rvq.bins=2048, no quantization dropout
+# (transformer_lm card and n_q must be compatible)
+compression_model_checkpoint: //pretrained/facebook/encodec_32khz
+
+channels: 1
+sample_rate: 32000
+
+deadlock:
+  use: true  # deadlock detection
+
+dataset:
+  batch_size: 192  # 32 GPUs
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+
+generate:
+  lm:
+    use_sampling: true
+    top_k: 250
+    top_p: 0.0
+
+optim:
+  epochs: 500
+  optimizer: dadam
+  lr: 1
+  ema:
+    use: true
+    updates: 10
+    device: cuda
+
+logging:
+  log_tensorboard: true
+
+schedule:
+  lr_scheduler: cosine
+  cosine:
+    warmup: 4000
+    lr_min_ratio: 0.0
+    cycle_length: 1.0
diff --git a/audiocraft/config/solver/musicgen/single_finetune.yaml b/audiocraft/config/solver/musicgen/single_finetune.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..902dee3ebddb34d7ee5b6cc9b60caff4b3b9b0c6
--- /dev/null
+++ b/audiocraft/config/solver/musicgen/single_finetune.yaml
@@ -0,0 +1,63 @@
+# @package __global__
+
+# This is the training loop solver
+# for the base MusicGen model (text-to-music)
+defaults:
+  - musicgen/default
+  - /model: lm/musicgen_lm
+  - _self_
+
+autocast: true
+autocast_dtype: float16
+
+# EnCodec large trained on mono-channel music audio sampled at 32khz
+# with a total stride of 640 leading to 50 frames/s.
+# rvq.n_q=4, rvq.bins=2048, no quantization dropout
+# (transformer_lm card and n_q must be compatible)
+compression_model_checkpoint: //pretrained/facebook/encodec_32khz
+
+channels: 1
+sample_rate: 32000
+
+deadlock:
+  use: true  # deadlock detection
+
+dataset:
+  batch_size: 2  # 1 GPU(3090)
+  num_workers: 2
+  segment_duration: 30
+  sample_on_weight: false  # Uniform sampling all the way
+  sample_on_duration: false  # Uniform sampling all the way
+  valid:
+    num_samples: 4
+
+generate:
+  lm:
+    use_sampling: true
+    top_k: 250
+    top_p: 0.0
+
+checkpoint:
+  save_last: true
+  save_every: 25
+  keep_every_states: null
+
+optim:
+  epochs: 100
+  optimizer: dadam
+  lr: 1.0
+  max_norm: 1.0
+  ema:
+    use: false
+    updates: 10
+    device: cuda
+
+logging:
+  log_tensorboard: true
+
+schedule:
+  lr_scheduler: cosine
+  cosine:
+    warmup: 5
+    lr_min_ratio: 0.0
+    cycle_length: 1.0
\ No newline at end of file
diff --git a/audiocraft/config/teams/default.yaml b/audiocraft/config/teams/default.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..3e684c27a0bf23876323e64d766eb74913f685b8
--- /dev/null
+++ b/audiocraft/config/teams/default.yaml
@@ -0,0 +1,12 @@
+default:
+  dora_dir: ./training_weights
+  partitions:
+    global: debug
+    team: debug
+  reference_dir: ./
+darwin:  # if we detect we are on a Mac, then most likely we are doing unit testing etc.
+  dora_dir: ./training_weights
+  partitions:
+    global: debug
+    team: debug
+  reference_dir: ./
diff --git a/audiocraft/config/teams/labs.yaml b/audiocraft/config/teams/labs.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..da350a94bc5758531ced5d9e4332624fe86f3d57
--- /dev/null
+++ b/audiocraft/config/teams/labs.yaml
@@ -0,0 +1,28 @@
+aws:
+  dora_dir: /fsx-audio-craft-llm/${oc.env:USER}/experiments/audiocraft/outputs
+  partitions:
+    global: learnlab
+    team: learnlab
+  reference_dir: /fsx-audio-craft-llm/shared/audiocraft/reference
+  dataset_mappers:
+    "^/checkpoint/[a-z]+": "/fsx-audio-craft-llm"
+fair:
+  dora_dir: /checkpoint/${oc.env:USER}/experiments/audiocraft/outputs
+  partitions:
+    global: learnlab
+    team: learnlab
+  reference_dir: /large_experiments/audiocraft/reference
+  dataset_mappers:
+    "^/datasets01/datasets01": "/datasets01"
+darwin:
+  dora_dir: /tmp/audiocraft_${oc.env:USER}
+  partitions:
+    global: debug
+    team: debug
+  reference_dir: /tmp
+rsc:
+  dora_dir: /checkpoint/audiocraft/${oc.env:USER}/experiments/audiocraft/outputs
+  partitions:
+    global: learn
+    team: learn
+  reference_dir: /checkpoint/audiocraft/shared/reference
diff --git a/audiocraft/dataset/example/clip/sample_1/beats.npy b/audiocraft/dataset/example/clip/sample_1/beats.npy
new file mode 100644
index 0000000000000000000000000000000000000000..0194428ecdf0fed5be17e112e6e4c4f9ac7a7cd7
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_1/beats.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:774a64a3e1bc8f704bebb961ab9ef43cdf20e07a6470149230a80691f0d6b1eb
+size 784
diff --git a/audiocraft/dataset/example/clip/sample_1/chord.lab b/audiocraft/dataset/example/clip/sample_1/chord.lab
new file mode 100644
index 0000000000000000000000000000000000000000..390e4f55a1a9ff3b582901b0c9fefed27155d06f
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_1/chord.lab
@@ -0,0 +1,22 @@
+0.000 1.389 G
+1.389 2.963 E:min7
+2.963 4.352 C
+4.352 5.833 D
+5.833 7.315 G
+7.315 8.796 E:min7
+8.796 10.185 C
+10.185 11.574 D
+11.574 13.056 G
+13.056 14.630 E:min7
+14.630 16.111 C
+16.111 17.315 D
+17.315 18.981 G
+18.981 20.463 E:min7
+20.463 21.852 C
+21.852 22.870 D
+22.870 24.815 G
+24.815 26.204 E:min7
+26.204 26.296 E:min
+26.296 27.778 C
+27.778 29.167 D
+29.167 30.000 G
diff --git a/audiocraft/dataset/example/clip/sample_1/no_vocal.wav b/audiocraft/dataset/example/clip/sample_1/no_vocal.wav
new file mode 100644
index 0000000000000000000000000000000000000000..9e738015b2202fbf01283b003509a4fcf51c30d5
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_1/no_vocal.wav
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64d92035567b0a88cedcdfaf828b7c4f38b11d9900e8acbe4f8f236f2edfc27f
+size 5292044
diff --git a/audiocraft/dataset/example/clip/sample_1/tags.json b/audiocraft/dataset/example/clip/sample_1/tags.json
new file mode 100644
index 0000000000000000000000000000000000000000..c55ad53a4afa32fbccb396f762c782b195dd2252
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_1/tags.json
@@ -0,0 +1 @@
+{"key": "", "artist": "", "sample_rate": 44100, "file_extension": "wav", "description": "chill song with guitar and drum", "keywords": "", "duration": 30.0, "bpm": "", "genre": "", "title": "", "name": "", "instrument": "Mix", "moods": [], "path": "dataset/example/sample_1/no_vocal.wav"}
\ No newline at end of file
diff --git a/audiocraft/dataset/example/clip/sample_2/beats.npy b/audiocraft/dataset/example/clip/sample_2/beats.npy
new file mode 100644
index 0000000000000000000000000000000000000000..8d21b2c8af07deb00ffe4c282a3ffb96fd38b10f
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_2/beats.npy
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:26673d903483bfef082b9b84035b5db0b96b5b60887cae04841bec177ece54f5
+size 1136
diff --git a/audiocraft/dataset/example/clip/sample_2/chord.lab b/audiocraft/dataset/example/clip/sample_2/chord.lab
new file mode 100644
index 0000000000000000000000000000000000000000..ab82b72148475c8cd51e459126d9706626598ffb
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_2/chord.lab
@@ -0,0 +1,49 @@
+0.000 0.648 E:min
+0.648 0.741 E
+0.741 1.204 F#:min
+1.204 1.296 D
+1.296 1.389 E:min
+1.389 1.759 G
+1.759 2.685 E:min
+2.685 3.611 D
+3.611 4.722 E:min
+4.722 4.907 B:min
+4.907 5.185 E:min
+5.185 5.556 G
+5.556 7.130 E:min
+7.130 7.407 G
+7.407 8.426 E:min
+8.426 8.796 F#:min7
+8.796 8.981 E:min
+8.981 9.352 G
+9.352 10.185 E:min
+10.185 10.833 F#:min7
+10.833 11.111 E:min
+11.111 11.296 G
+11.296 12.130 E:min
+12.130 12.778 F#:min7
+12.778 13.056 E:min
+13.056 13.148 G
+13.148 14.167 E:min
+14.167 14.537 F#:min7
+14.537 16.204 E:min
+16.204 16.389 F#:min7
+16.389 19.074 E:min
+19.074 19.259 A
+19.259 20.000 A:min
+20.000 20.370 N
+20.370 21.111 G
+21.111 21.852 E:min
+21.852 22.315 F#:min7
+22.315 22.407 D
+22.407 22.963 G
+22.963 24.907 D
+24.907 25.741 E:min
+25.741 26.204 F#:min7
+26.204 26.296 E:min
+26.296 26.759 G
+26.759 27.593 E:min
+27.593 28.148 F#:min7
+28.148 28.611 G
+28.611 29.537 E:min
+29.537 30.000 F#:min7
diff --git a/audiocraft/dataset/example/clip/sample_2/no_vocal.wav b/audiocraft/dataset/example/clip/sample_2/no_vocal.wav
new file mode 100644
index 0000000000000000000000000000000000000000..1352673b88c7544ecf413edc3d9bc659747e821c
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_2/no_vocal.wav
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:180a41036fe7245cb34eb6a8de5cf630b93367d4c18d55d1b98b8e76fd2d81a9
+size 5292044
diff --git a/audiocraft/dataset/example/clip/sample_2/tags.json b/audiocraft/dataset/example/clip/sample_2/tags.json
new file mode 100644
index 0000000000000000000000000000000000000000..ca1d22127ae18971bc15c4a555b4e5ed7fa204aa
--- /dev/null
+++ b/audiocraft/dataset/example/clip/sample_2/tags.json
@@ -0,0 +1 @@
+{"key": "", "artist": "", "sample_rate": 44100, "file_extension": "wav", "description": "cool song from BKS", "keywords": "", "duration": 30.0, "bpm": "", "genre": "", "title": "", "name": "", "instrument": "Mix", "moods": [], "path": "dataset/example/sample_2/no_vocal.wav"}
\ No newline at end of file
diff --git a/audiocraft/egs/.DS_Store b/audiocraft/egs/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..57a55533b24c0913b16270b1e0331e8066b90fde
Binary files /dev/null and b/audiocraft/egs/.DS_Store differ
diff --git a/audiocraft/egs/example/data.jsonl b/audiocraft/egs/example/data.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b00f36b76ff0e9d8281513d85a278489b14cb08e
--- /dev/null
+++ b/audiocraft/egs/example/data.jsonl
@@ -0,0 +1,2 @@
+{"path": "dataset/example/clip/sample_1/no_vocal.wav", "duration": 30.0, "sample_rate": 44100, "bpm": "", "amplitude": null, "weight": null, "info_path": null}
+{"path": "dataset/example/clip/sample_2/no_vocal.wav", "duration": 30.0, "sample_rate": 44100, "bpm": "", "amplitude": null, "weight": null, "info_path": null}
diff --git a/audiocraft/export_weight.py b/audiocraft/export_weight.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f89e113e90946758e8c4f5975e64c6ad400e5a9
--- /dev/null
+++ b/audiocraft/export_weight.py
@@ -0,0 +1,12 @@
+from audiocraft.utils import export
+from audiocraft import train
+import os
+from pathlib import Path
+
+sig = "your_training_signature"
+output_dir = "./ckpt/output_weight_dir"
+
+
+folder = f"./audiocraft_default/xps/{sig}"
+export.export_lm(Path(folder) / 'checkpoint.th', os.path.join(output_dir, 'state_dict.bin'))
+export.export_pretrained_compression_model('facebook/encodec_32khz', os.path.join(output_dir, 'compression_state_dict.bin'))
\ No newline at end of file
diff --git a/audiocraft/generate_chord_beat.py b/audiocraft/generate_chord_beat.py
new file mode 100644
index 0000000000000000000000000000000000000000..e34c879a1589ff394196e96f8e96bb049979add3
--- /dev/null
+++ b/audiocraft/generate_chord_beat.py
@@ -0,0 +1,49 @@
+from audiocraft.data.audio import audio_write
+import audiocraft.models
+import numpy as np
+import pandas as pd
+import os
+import torch
+
+# set hparams
+output_dir = 'example_1' ### change this output directory
+
+
+duration = 30
+num_samples = 5
+bs = 1
+
+
+# load your model
+musicgen = audiocraft.models.MusicGen.get_pretrained('./ckpt/musicongen') ### change this path
+musicgen.set_generation_params(duration=duration, extend_stride=duration//2, top_k = 250)
+
+
+chords = ['C G A:min F',
+          'A:min F C G',
+          'C F G F',
+          'C A:min F G',
+          'D:min G C A:min',
+          ]
+
+descriptions = ["A laid-back blues shuffle with a relaxed tempo, warm guitar tones, and a comfortable groove, perfect for a slow dance or a night in. Instruments: electric guitar, bass, drums."] * num_samples
+
+bpms = [120] * num_samples
+
+meters = [4] * num_samples
+
+wav = []
+for i in range(num_samples//bs):
+  print(f"starting {i} batch...")
+  temp = musicgen.generate_with_chords_and_beats(descriptions[i*bs:(i+1)*bs], 
+                                                  chords[i*bs:(i+1)*bs],
+                                                  bpms[i*bs:(i+1)*bs], 
+                                                  meters[i*bs:(i+1)*bs]
+                                                  )
+  wav.extend(temp.cpu())
+
+# save and display generated audio
+for idx, one_wav in enumerate(wav):
+  
+  sav_path = os.path.join('./output_samples', output_dir, chords[idx] + "|" + descriptions[idx]).replace(" ", "_")
+  audio_write(sav_path, one_wav.cpu(), musicgen.sample_rate, strategy='loudness', loudness_compressor=True)
\ No newline at end of file
diff --git a/preproc/0_demix/README.md b/preproc/0_demix/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..33b0e0089d9398b58829048a9e0d0b20e2f4f993
--- /dev/null
+++ b/preproc/0_demix/README.md
@@ -0,0 +1,16 @@
+Two-stem demixing.
+
+## Installation
+```bash
+source install.sh
+```
+
+## running
+```bash
+python main.py
+```
+
+## Monitoring
+* for each full-length song
+    * GPU: ~1G
+    * Time: ~25 seconds (on 3090)
\ No newline at end of file
diff --git a/preproc/0_demix/install.sh b/preproc/0_demix/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..dc6614fda5d150cbb75ce628b421eeb7d11adef1
--- /dev/null
+++ b/preproc/0_demix/install.sh
@@ -0,0 +1,7 @@
+cd demucs
+apt-get update
+apt-get install tmux vim -y
+conda env update -f environment-cuda.yml
+conda activate demucs
+pip install -e .
+conda update ffmpeg
\ No newline at end of file
diff --git a/preproc/0_demix/main.py b/preproc/0_demix/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..7947972cbd06bcb1dcc5fc1fbf2e2989916ac690
--- /dev/null
+++ b/preproc/0_demix/main.py
@@ -0,0 +1,122 @@
+'''
+two-stems separation
+'''
+import os
+from pathlib import Path
+import uuid
+import subprocess
+
+import time
+import shutil
+import datetime
+
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+if __name__ == '__main__':
+    path_rootdir = '../audiocraft/dataset/example/full'
+
+    st_idx, ed_idx = 0, None
+    ext_src = 'mp3'
+    ext_dst = 'wav'
+
+    # list files
+    filelist = traverse_dir(
+        path_rootdir,
+        extension='mp3',
+        str_include='',
+        is_sort=True)
+    num_file = len(filelist)
+    if ed_idx is None:
+        ed_idx = num_file
+    print(' [i] num files:', num_file)
+
+    # make tmpdir for demucs
+    tmp_dir = os.path.join('tmp', str(uuid.uuid4()).split('-')[0])
+    print('tmp_dir:', tmp_dir)
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    os.makedirs(tmp_dir)
+
+    # start running
+    for i in range(st_idx, ed_idx):
+        print(f'==={i}/{num_file}  [{st_idx} - {ed_idx}]====================')
+        start_time = time.time()
+        
+        # path
+        srcfile = filelist[i]
+        print(srcfile)
+        srcfile_dir = os.path.dirname(srcfile)
+        source_folder = os.path.join(tmp_dir, 'htdemucs', 'full')
+        path_src_vocals = os.path.join(source_folder, f'vocals.{ext_dst}')
+        path_src_no_vocals = os.path.join(source_folder, f'no_vocals.{ext_dst}')
+        path_dst_vocals = os.path.join(srcfile_dir, f'vocals.{ext_dst}')
+        path_dst_no_vocals =  os.path.join(srcfile_dir, f'no_vocal.{ext_dst}')
+
+        if os.path.exists(path_dst_no_vocals):
+            print('[o] existed')
+            continue
+
+        # source separation
+        cmd_list = [
+            'demucs', 
+            '--two-stems=vocals', 
+            f'{srcfile}',
+            '-o',  
+            f'{tmp_dir}'
+          ]
+        
+        if ext_dst == 'mp3':
+            print('[i] save in mp3 format')
+            cmd_list.append('--mp3')
+        subprocess.run(cmd_list)
+        
+        # copy from tmp to dst
+        shutil.copy2(path_src_vocals, path_dst_vocals)
+        shutil.copy2(path_src_no_vocals, path_dst_no_vocals)
+
+        # end
+        end_time = time.time()
+        runtime = end_time - start_time
+        print('testing time:', str(datetime.timedelta(seconds=runtime))+'\n')
+
+    shutil.rmtree(tmp_dir)
\ No newline at end of file
diff --git a/preproc/1_beats-crop/1_mm.wav b/preproc/1_beats-crop/1_mm.wav
new file mode 100644
index 0000000000000000000000000000000000000000..70a4049ec2e289436c6e0d2ee428179686dfe37d
--- /dev/null
+++ b/preproc/1_beats-crop/1_mm.wav
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6299586f1bc5f213bb607bf80457bfab9675c22aa0343a3c8b805d9f49d42c5
+size 23984564
diff --git a/preproc/1_beats-crop/1_nn.wav b/preproc/1_beats-crop/1_nn.wav
new file mode 100644
index 0000000000000000000000000000000000000000..444a5b59df9d6c5b97451e58c97806ef97e3fe9e
--- /dev/null
+++ b/preproc/1_beats-crop/1_nn.wav
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65376ab9e4446f31127dfc3d79d4664e95f3558222c3f1e2448b3b00075e11ce
+size 23984564
diff --git a/preproc/1_beats-crop/README.md b/preproc/1_beats-crop/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..47c7b14ddaecb115a9e0000b778403f06d7051db
--- /dev/null
+++ b/preproc/1_beats-crop/README.md
@@ -0,0 +1,29 @@
+Two-stem demixing.
+
+## Installation
+```bash
+source install.sh
+```
+
+## running
+```bash
+python main_beat_nn.py # much fater than main_beat.py (madmom)
+python main_crop.py
+python main_filter.py
+```
+
+## Monitoring
+* for each full-length song
+    * GPU: 
+        * cpu: no
+        * gpu: ~1G
+    * Time: ~90 seconds
+
+## BeatNet v.s. Madmom
+* w5HP2Xcy_eQ: 4m30s 
+    * beatnet (cpu): 8 sec
+    * beatnet (gpu): 15 sec
+    * madmom: 60 sec 
+* 7_bo2zs50bg: 10m34s
+    * beatnet (cpu): 13 sec
+    * beatnet (gpu): 13 sec
\ No newline at end of file
diff --git a/preproc/1_beats-crop/install.sh b/preproc/1_beats-crop/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..3e0b4722c7e3b0b3148fae1dfdb8f1372f37b778
--- /dev/null
+++ b/preproc/1_beats-crop/install.sh
@@ -0,0 +1,4 @@
+pip install --upgrade --no-deps --force-reinstall 'git+https://github.com/CPJKU/madmom.git'
+pip install librosa
+apt-get update
+apt-get install ffmpeg -y
\ No newline at end of file
diff --git a/preproc/1_beats-crop/install_nn.sh b/preproc/1_beats-crop/install_nn.sh
new file mode 100644
index 0000000000000000000000000000000000000000..117abf9176cc498915acdce0dd11456e645a5b03
--- /dev/null
+++ b/preproc/1_beats-crop/install_nn.sh
@@ -0,0 +1,14 @@
+apt-get update
+apt-get install tmux vim git gcc -y
+conda create -n beat python=3.9 -y
+conda activate beat
+# # pip install --upgrade --no-deps --force-reinstall 'git+https://github.com/CPJKU/madmom.git'
+# pip install pyproject-toml
+# # pip install git+https://github.com/CPJKU/madmom
+pip install -e git+https://github.com/CPJKU/madmom#egg=madmom
+pip install BeatNet
+pip install torch==2.0.1
+apt-get install portaudio19-dev -y
+pip install pyaudio
+conda install ffmpeg -y
+pip install tqdm
\ No newline at end of file
diff --git a/preproc/1_beats-crop/main_beat.py b/preproc/1_beats-crop/main_beat.py
new file mode 100644
index 0000000000000000000000000000000000000000..aabb6b29c47f9e8c4244ac2b411eed9b08773e2e
--- /dev/null
+++ b/preproc/1_beats-crop/main_beat.py
@@ -0,0 +1,158 @@
+import os
+import uuid
+import librosa
+import soundfile as sf
+import numpy as np
+
+import time
+import datetime
+from tqdm import tqdm
+
+import multiprocessing
+
+from madmom.features.downbeats import DBNDownBeatTrackingProcessor
+from madmom.features.downbeats import RNNDownBeatProcessor
+
+
+def export_audio_with_click(proc_res, path_audio, path_output, sr=44100):
+    # extract time
+    times_beat = proc_res[np.where(proc_res[:, 1]!=1)][:, 0]
+    times_downbeat = proc_res[np.where(proc_res[:, 1]==1)][:, 0]
+
+    # load
+    y, _ = librosa.core.load(path_audio, sr=sr) 
+
+    # click audio
+    y_beat = librosa.clicks(times=times_beat, sr=sr, click_freq=1200, click_duration=0.5) * 0.6
+    y_downbeat = librosa.clicks(times=times_downbeat, sr=sr, click_freq=600, click_duration=0.5)
+
+    # merge
+    max_len = max(len(y), len(y_beat), len(y_downbeat))
+    y_integrate = np.zeros(max_len)
+    y_integrate[:len(y_beat)] += y_beat
+    y_integrate[:len(y_downbeat)] += y_downbeat
+    y_integrate[:len(y)] += y
+
+    # librosa.output.write_wav(path_output, y_integrate, sr)
+    sf.write(path_output, y_integrate, sr)
+
+
+def estimate_beat(path_audio):
+    # print('[*] estimating beats...')
+    proc = DBNDownBeatTrackingProcessor(beats_per_bar=[3, 4], fps=100)
+    act = RNNDownBeatProcessor()(path_audio)
+    proc_res = proc(act) 
+    return proc_res
+
+def process(path_inpfile, path_outfile):
+    pid = os.getpid()
+    # start_time = time.time()
+    # print(f'[PID: {pid}] > inp:', path_inpfile)
+    # print(f'[PID: {pid}] > out:', path_outfile)
+
+    if os.path.exists(path_outfile):
+        print('[o] existed')
+        return True
+    
+    # estimate beats
+    beats = estimate_beat(path_inpfile)
+    os.makedirs(os.path.dirname(path_outfile), exist_ok=True)
+    np.save(path_outfile, beats)
+
+    # export_audio_with_click(beats, path_inpfile, 'tmp.wav') # option
+    # end
+    # end_time = time.time()
+    # runtime = end_time - start_time
+    # print(f'[PID: {pid}] testing time:', str(datetime.timedelta(seconds=runtime))+'\n')
+    return True
+
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+def gen_pairs(path_inpdir, audio_basename, ext):
+    pairs = []
+    filelist = traverse_dir(
+            path_inpdir,
+            extension=ext,
+            str_include=audio_basename,
+            is_sort=True)
+    num_files = len(filelist)
+    print(' > num of files (total):', num_files)
+
+    for fidx in range(num_files): # p0
+        path_inpfile = filelist[fidx]
+        # path_outfile = os.path.join(
+        #     os.path.dirname(path_inpfile), 'beats.npy')
+        path_outfile = path_inpfile.replace('.wav', '.npy')
+
+        if os.path.exists(path_outfile):
+            print('[o] existed')
+            continue
+
+        pairs.append((path_inpfile, path_outfile))
+    num_files = len(pairs)
+    print(' > num of files (unprocessed):', num_files)
+    return pairs, num_files
+
+if __name__ == '__main__':
+    path_rootdir = '../audiocraft/dataset/example/full'
+    audio_basename = 'no_vocals'
+    ext = 'wav'
+
+    # list files
+    pairs, num_files = gen_pairs(path_rootdir, audio_basename, ext)
+    
+    # count cpu
+    cpu_count = 4
+    print(' > cpu count:', cpu_count)
+
+    start_time_all = time.time()
+    with multiprocessing.Pool(processes=cpu_count) as pool, tqdm(total=num_files) as progress_bar:
+        results = []
+        for result in pool.starmap(process, pairs):
+            results.append(result)
+            progress_bar.update(1)
+
+    # finish
+    end_time_all = time.time()
+    runtime = end_time_all - start_time_all
+    print(f'testing time:', str(datetime.timedelta(seconds=runtime))+'\n')
\ No newline at end of file
diff --git a/preproc/1_beats-crop/main_beat_nn.py b/preproc/1_beats-crop/main_beat_nn.py
new file mode 100644
index 0000000000000000000000000000000000000000..1400cd60e05f10b47463ec6e77860fc3a801975e
--- /dev/null
+++ b/preproc/1_beats-crop/main_beat_nn.py
@@ -0,0 +1,148 @@
+import os
+from BeatNet.BeatNet import BeatNet
+
+import time
+import datetime
+from tqdm import tqdm
+
+import soundfile as sf
+import librosa
+import numpy as np
+
+
+device = 'cuda' # 'cpu' or 'cuda', I found there is no difference
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+def estimate_beat_beatnet(path_audio):
+    estimator = BeatNet(
+        1, 
+        mode='offline', 
+        inference_model='DBN',
+        plot=[], 
+        thread=False, 
+        device=device)
+    
+    beats = estimator.process(path_audio)
+    return beats
+
+
+def estimate_beat_madmom(path_audio):
+    from madmom.features.downbeats import DBNDownBeatTrackingProcessor
+    from madmom.features.downbeats import RNNDownBeatProcessor
+    # print('[*] estimating beats...')
+    proc = DBNDownBeatTrackingProcessor(beats_per_bar=[3, 4], fps=100)
+    act = RNNDownBeatProcessor()(path_audio)
+    proc_res = proc(act) 
+    return proc_res
+
+def export_audio_with_click(proc_res, path_audio, path_output, sr=44100):
+    # extract time
+    times_beat = proc_res[np.where(proc_res[:, 1]!=1)][:, 0]
+    times_downbeat = proc_res[np.where(proc_res[:, 1]==1)][:, 0]
+
+    # load
+    y, _ = librosa.core.load(path_audio, sr=sr) 
+
+    # click audio
+    y_beat = librosa.clicks(times=times_beat, sr=sr, click_freq=1200, click_duration=0.5) * 0.6
+    y_downbeat = librosa.clicks(times=times_downbeat, sr=sr, click_freq=600, click_duration=0.5)
+
+    # merge
+    max_len = max(len(y), len(y_beat), len(y_downbeat))
+    y_integrate = np.zeros(max_len)
+    y_integrate[:len(y_beat)] += y_beat
+    y_integrate[:len(y_downbeat)] += y_downbeat
+    y_integrate[:len(y)] += y
+
+    # librosa.output.write_wav(path_output, y_integrate, sr)
+    sf.write(path_output, y_integrate, sr)
+
+
+if __name__ == '__main__':
+    path_rootdir = '../audiocraft/dataset/example/full'
+    audio_base = 'no_vocals'
+    ext = 'wav'
+    st, ed = 0, None
+
+
+    filelist = traverse_dir(
+        path_rootdir,
+        extension=ext,
+        str_include=audio_base,
+        is_sort=True)
+    num_files = len(filelist)
+    print(' > num files:', num_files)
+    if ed is None:
+        ed = num_files
+
+    # run
+    start_time_all = time.time()
+
+    for i in range(num_files-1,-1,-1):
+        start_time_one = time.time()
+        print("==={}/{}======[{} - {}]========".format(
+            i, num_files, st, ed))
+        path_audio = filelist[i]
+        path_outfile = path_audio.replace('no_vocals.wav', 'beats.npy')
+            
+
+        print(' inp >', path_audio)
+        print(' out >', path_outfile)
+        if os.path.exists(path_outfile):
+            print('[o] existed')
+            continue
+    
+        beats = estimate_beat_beatnet(path_audio)
+
+        # save
+        np.save(path_outfile, beats)
+
+        end_time_one = time.time()
+        runtime = end_time_one - start_time_one
+        print(f' > runtime:', str(datetime.timedelta(seconds=runtime))+'\n')
+
+    end_time_all = time.time()
+    runtime = end_time_all - start_time_all
+    print(f'testing time:', str(datetime.timedelta(seconds=runtime))+'\n')
+
+   
\ No newline at end of file
diff --git a/preproc/1_beats-crop/main_crop.py b/preproc/1_beats-crop/main_crop.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2741191529f1713bc23842c66f2c78d6cb61df5
--- /dev/null
+++ b/preproc/1_beats-crop/main_crop.py
@@ -0,0 +1,172 @@
+import os
+
+import numpy as np
+import soundfile as sf
+import librosa
+
+import time
+import datetime
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+CROP_LEN_SEC = 30
+
+BAR_FIRST = 8
+
+if __name__ == '__main__':
+    start_time_all = time.time()
+
+    path_dset = '../audiocraft/dataset/example'
+    path_inpdir = os.path.join(path_dset, 'full')
+    path_outdir = os.path.join(path_dset, 'clip')
+    st, ed = 0, None
+
+    filelist = traverse_dir(
+        path_inpdir,
+        extension='wav',
+        str_include='no_vocal',
+        is_pure=True,
+        is_sort=True)
+    num_files = len(filelist)
+    ed = num_files if ed is None else ed
+    print(' > num files:', num_files)
+
+    for fidx in range(num_files-1, -1, -1):
+        start_time_iter = time.time()
+        print(f'==={fidx}/{num_files}====={st}-{ed}============')
+        fn = filelist[fidx]
+        dn = os.path.dirname(fn)
+        path_audio = os.path.join(path_inpdir, fn)
+        path_beats = os.path.join(path_inpdir, dn, 'beats.npy')
+
+        print(fn)
+        if not os.path.exists(path_audio):
+            raise FileNotFoundError(path_beats)
+        path_out_sndir = os.path.join(path_outdir, dn)
+
+        if os.path.exists(path_out_sndir):
+            print('[o] existed')
+            continue
+
+        # ==========
+        try: 
+            beats = np.load(path_beats)
+            wav, sr = sf.read(path_audio, always_2d=True)
+            duration =  len(wav) / sr
+            print(' > wav:', wav.shape)
+            print(' > sr: ', sr)
+            print(' > ch: ', wav.shape[1])
+            print(' > duration:', len(wav) / sr)
+
+            bar_idx = np.where(beats[:, 1] == 1)[0]
+            num_bars = len(bar_idx)
+            print(' > number of bars:', num_bars)
+
+            BAR_HOP = int(30 / (duration  / num_bars))
+            print(' > bar hop:', BAR_HOP)
+
+            bar_starts = [bar_idx[i] for i in range(3, len(bar_idx), BAR_HOP)]
+
+            clip_starts = []
+            for bs in bar_starts:
+                item = (
+                    beats[bs, 0], # seconds
+                    bs # index
+                )
+                clip_starts.append(item)
+
+            max_sample = wav.shape[0] - 10*sr
+            CLIP_LEN_SAMPLE = CROP_LEN_SEC*sr
+
+            # crop
+            count_clips = 0
+            for uid, (clip_st_sec, bidx) in enumerate(clip_starts):
+                # boundaries
+                clip_ed_sec = clip_st_sec + CROP_LEN_SEC
+                clip_st_sample = int(clip_st_sec*sr)
+                clip_ed_sample = clip_st_sample + CLIP_LEN_SAMPLE
+                if clip_ed_sample > max_sample:
+                    break
+                
+                # crop
+                clip_wav = wav[clip_st_sample:clip_ed_sample]
+                clip_beats = []
+                
+                for bi in range(bidx, len(beats)):
+                    if beats[bi, 0] < clip_ed_sec:
+                        clip_beats.append(
+                            [beats[bi, 0]-clip_st_sec, beats[bi, 1]]
+                        )
+                
+                # save
+                path_out_audio_clip = os.path.join(
+                    path_out_sndir, str(bidx),'no_vocal.wav')
+
+                if os.path.exists(path_out_audio_clip):
+                    print('[o] existed')
+                    continue
+                
+                path_out_beats_clip = os.path.join(
+                    path_out_sndir, str(bidx), 'beats.npy')
+                os.makedirs(
+                    os.path.dirname(path_out_audio_clip), exist_ok=True)
+                sf.write(path_out_audio_clip, clip_wav, sr)
+                np.save(path_out_beats_clip, clip_beats)
+
+                count_clips += 1
+            print(' > count:',  count_clips)
+        except:
+            print('[x] aborted')
+            continue 
+
+        # finish
+        end_time_iter = time.time()
+        runtime = end_time_iter - start_time_iter
+        print(f'testing time:', str(datetime.timedelta(seconds=runtime))+'\n')
+            
+
+    # finish
+    print('\n\n\n-------------------------------')
+    print(' [o] Done')
+    end_time_all = time.time()
+    runtime = end_time_all - start_time_all
+    print(f'Total time:', str(datetime.timedelta(seconds=runtime))+'\n')
+
diff --git a/preproc/1_beats-crop/main_filter.py b/preproc/1_beats-crop/main_filter.py
new file mode 100644
index 0000000000000000000000000000000000000000..09e4840c94bd217f9d49e13f3c45a861ae975de4
--- /dev/null
+++ b/preproc/1_beats-crop/main_filter.py
@@ -0,0 +1,131 @@
+import os
+import numpy as np
+import math
+import soundfile as sf
+import json
+import shutil
+import uuid
+
+import time
+import datetime
+PIVOT_RATIO = 0.8
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+def convert_to_decibel(arr, min_db=-120):
+    ref = 1
+    if arr!=0:
+        return 20 * np.log10(abs(arr) / ref)
+    else:
+        return min_db
+
+
+def compute_framewise_dbfs(
+        signal, 
+        win_len=1024, 
+        hop_len=512):
+    
+    db_list = []
+    for ed in range(win_len, signal.shape[0], hop_len):
+        st = ed - win_len
+        win_amplitude = np.mean(signal[st:ed, :])
+        db_list.append(convert_to_decibel(win_amplitude))
+    db_list = np.array(db_list)
+    a = db_list < -80
+    ratio = a.sum() / a.shape[0]
+    return ratio
+
+
+if __name__ == '__main__':
+
+    start_time_all = time.time()
+    root_dir = '../audiocraft/dataset/example/clip'
+    files = traverse_dir(
+        root_dir,
+        str_include='no_vocal',
+        extension='wav',
+        is_sort=True)
+    num_files = len(files)
+    print(' > num of files:', num_files)
+
+    #  save
+    res = []
+    ld_report = 'loudness_report_{}.txt'.format(str(uuid.uuid1()).split('-')[0])
+    with open(ld_report, 'w') as f:
+        for fidx in range(num_files):
+            print('---({}/{})-------------'.format(fidx, num_files))
+            file = files[fidx]
+            signal, _ = sf.read(file, always_2d=True)
+            ratio = compute_framewise_dbfs(signal) 
+            print(file)
+            print(ratio)
+            res.append((file, ratio))
+
+            f.write("{}-----:{}\n".format(file, ratio))
+
+    
+    with open(ld_report, 'r') as f:
+        data = [line.strip().split('-----:') for line in f]
+
+    # sort
+    data = sorted(data, key=lambda x: float(x[1]))
+    pivot = int(len(data) * PIVOT_RATIO)
+    print('\n\n\n============================')
+    print('pivot:', pivot)
+    n_samples = len(data) - pivot
+    not_ok_samples = data[-n_samples:]
+    print('not ok samples:', n_samples)
+
+    for i in range(n_samples):
+        path_fn, ratio = not_ok_samples[i]
+        print(path_fn, ratio)
+        try:
+            shutil.rmtree(os.path.dirname(path_fn))
+        except:
+            continue 
+
+    # finish
+    print('\n\n\n-------------------------------')
+    print(' [o] Done')
+    end_time_all = time.time()
+    runtime = end_time_all - start_time_all
+    print(f'Total time:', str(datetime.timedelta(seconds=runtime))+'\n')
\ No newline at end of file
diff --git a/preproc/2_chord/README.md b/preproc/2_chord/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..549290084aa739d6b77a3dece931b04878aa2d3b
--- /dev/null
+++ b/preproc/2_chord/README.md
@@ -0,0 +1,17 @@
+## Installation
+```bash
+pip install -r requirements.txt 
+```
+
+## running
+```bash
+cd BTC-ISMIR19
+python main.py
+```
+
+## Monitoring
+* for each 30s clip
+    * ~0.5s
+* for each full-length song
+    * GPU: no
+    * Time: ~90 seconds
diff --git a/preproc/2_chord/install.sh b/preproc/2_chord/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..39c0147b6f658d56ceb7bc0f50de7e678a12fd9c
--- /dev/null
+++ b/preproc/2_chord/install.sh
@@ -0,0 +1,6 @@
+apt-get update
+conda create -n chord python=3.8 -y
+conda activate chord
+apt-get install vim tmux ffmpeg git rsync -y
+cd BTC-ISMIR19
+pip install -r requirements.txt
\ No newline at end of file
diff --git a/preproc/3_1_ytjsons2tags/main.py b/preproc/3_1_ytjsons2tags/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8581f749c970046533325769a53bb0b2c793cdd
--- /dev/null
+++ b/preproc/3_1_ytjsons2tags/main.py
@@ -0,0 +1,123 @@
+import os
+import json
+import soundfile as sf
+import numpy as np
+
+from tqdm import tqdm
+import time
+import librosa
+import sys
+
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+
+def yt2json(path_audio, path_json, output_path):
+
+    # load
+    wav, sr = sf.read(path_audio, always_2d=True)
+    duration = len(wav) / sr
+    with open(path_json ,'r') as f:
+        json_str = f.read()
+    yt_json = json.loads(json_str)
+
+    mg_json = {"key": "", "artist": "", "sample_rate": sr, 
+               "file_extension": "wav", "description": "", 
+               "keywords": "", "duration": duration, "bpm": "", 
+               "genre": "", "title": "", "name": "", "instrument": "Mix", "moods": []}
+    
+    mg_json["artist"] = yt_json["uploader"]
+    mg_json["description"] = yt_json["title"]
+    mg_json["keywords"] = ", ".join(yt_json["tags"])
+    mg_json["name"] = yt_json["id"]
+    mg_json["path"] = str(path_audio)
+    
+    with open(output_path, 'w') as js_file:
+        json.dump(mg_json, js_file)
+    
+
+if __name__ == '__main__':
+
+    root_dir = '../audiocraft/dataset/example/clip'
+    base_audio = 'no_vocal'
+    base_ext = 'wav'
+    st, ed = 0, None
+
+    audio_paths = traverse_dir(
+            root_dir,
+            str_include=base_audio,
+            extension=base_ext,
+            is_sort=True)
+
+    num_files = len(audio_paths)
+    print(' > num of files:', num_files)
+    if ed is None:
+        ed = num_files
+
+    # run
+    err_files = [] 
+    for i in range(st, ed): 
+        print("==={}/{}======[{} - {}]========".format(
+            i, num_files, st, ed))
+
+        # path
+        path_audio = audio_paths[i]
+        dn = os.path.dirname(path_audio)
+        json_dn = '/'.join(dn.split('/')[:-1]).replace('clip', 'full')
+        path_json = os.path.join(json_dn, 'crawl_info.json') # replace the name of crawled json for each yt song here
+        print(path_audio)
+        print(path_json)
+        output_path = path_audio.replace('no_vocal.wav', 'tags.json')
+
+        # export abs midi
+        try:
+            yt2json(path_audio, path_json, output_path)
+           
+        except:
+            print('[x] error')
+            err_files.append(path_audio)
+            sys.exit(1)
+            continue
+    print('\n\n\n==================')
+    print('Error Files:')
+    for idx, err_f in enumerate(err_files):
+        print(idx, '-', err_f)
\ No newline at end of file
diff --git a/preproc/3_tags/README.md b/preproc/3_tags/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..2c9dd14b912619354a3d550815746051e729212b
--- /dev/null
+++ b/preproc/3_tags/README.md
@@ -0,0 +1,6 @@
+* Installation
+    * run `install.sh`
+* run `main.py`
+* Can only run on 3090
+    * for each 30s clip, it takes about 2~3 seconds
+    * 12GB VRAM required, 2 process per GPU
\ No newline at end of file
diff --git a/preproc/3_tags/essentia/__pycache__/metadata.cpython-311.pyc b/preproc/3_tags/essentia/__pycache__/metadata.cpython-311.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..34ebd65d9efa4cc709e59740964adc7c2aabece2
Binary files /dev/null and b/preproc/3_tags/essentia/__pycache__/metadata.cpython-311.pyc differ
diff --git a/preproc/3_tags/essentia/__pycache__/metadata.cpython-38.pyc b/preproc/3_tags/essentia/__pycache__/metadata.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..58cfe239e2b19165cde1d74229e146a320a81e5e
Binary files /dev/null and b/preproc/3_tags/essentia/__pycache__/metadata.cpython-38.pyc differ
diff --git a/preproc/3_tags/essentia/__pycache__/metadata.cpython-39.pyc b/preproc/3_tags/essentia/__pycache__/metadata.cpython-39.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..2f61871fc2fb06aefd9e50e7f449e022d623df70
Binary files /dev/null and b/preproc/3_tags/essentia/__pycache__/metadata.cpython-39.pyc differ
diff --git a/preproc/3_tags/essentia/discogs-effnet-bs64-1.pb b/preproc/3_tags/essentia/discogs-effnet-bs64-1.pb
new file mode 100644
index 0000000000000000000000000000000000000000..dbe8f4f319adc5467be7eb812a1e6835e5830c40
--- /dev/null
+++ b/preproc/3_tags/essentia/discogs-effnet-bs64-1.pb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ed9af50d5367c0b9c795b294b00e7599e4943244f4cbd376869f3bfc87721b1
+size 18366619
diff --git a/preproc/3_tags/essentia/genre_discogs400-discogs-effnet-1.pb b/preproc/3_tags/essentia/genre_discogs400-discogs-effnet-1.pb
new file mode 100644
index 0000000000000000000000000000000000000000..4617e86d9f8ffe8098c07a08d3d3d3979c6f562d
--- /dev/null
+++ b/preproc/3_tags/essentia/genre_discogs400-discogs-effnet-1.pb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3885ba078a35249af94b8e5e4247689afac40deca4401a4bc888daf5a579c01c
+size 2057977
diff --git a/preproc/3_tags/essentia/install.sh b/preproc/3_tags/essentia/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..ab8a68081466095fe9a14296cf6f4a9322197bdc
--- /dev/null
+++ b/preproc/3_tags/essentia/install.sh
@@ -0,0 +1,5 @@
+apt-get update
+apt-get install tmux vim -y
+apt-get install ffmpeg
+pip install essentia-tensorflow
+pip install tensorflow
\ No newline at end of file
diff --git a/preproc/3_tags/essentia/main.py b/preproc/3_tags/essentia/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..47ba0069b4d3db7d46e9ab3c34faf1ab36945a70
--- /dev/null
+++ b/preproc/3_tags/essentia/main.py
@@ -0,0 +1,255 @@
+import os
+import json
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' 
+import subprocess as sp
+import librosa
+
+from metadata import genre_labels, mood_theme_classes, instrument_classes
+import numpy as np
+
+import sys
+import time
+import datetime
+
+
+os.environ['CUDA_VISIBLE_DEVICES'] = '1'
+
+# sp.call(["curl", "https://essentia.upf.edu/models/classification-heads/genre_discogs400/genre_discogs400-discogs-effnet-1.pb", "--output", "genre_discogs400-discogs-effnet-1.pb"])
+# sp.call(["curl", "https://essentia.upf.edu/models/feature-extractors/discogs-effnet/discogs-effnet-bs64-1.pb", "--output", "discogs-effnet-bs64-1.pb"])
+# sp.call(["curl", "https://essentia.upf.edu/models/classification-heads/mtg_jamendo_moodtheme/mtg_jamendo_moodtheme-discogs-effnet-1.pb", "--output", "mtg_jamendo_moodtheme-discogs-effnet-1.pb"])
+# sp.call(["curl", "https://essentia.upf.edu/models/classification-heads/mtg_jamendo_instrument/mtg_jamendo_instrument-discogs-effnet-1.pb", "--output", "mtg_jamendo_instrument-discogs-effnet-1.pb"])
+
+import sys
+
+# os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
+from essentia.standard import (
+    MonoLoader,
+    TensorflowPredictEffnetDiscogs,
+    TensorflowPredict2D,
+)
+os.environ['CUDA_VISIBLE_DEVICES'] = '1'
+
+def filter_predictions(predictions, class_list, threshold=0.1):
+    predictions_mean = np.mean(predictions, axis=0)
+    sorted_indices = np.argsort(predictions_mean)[::-1]
+    filtered_indices = [i for i in sorted_indices if predictions_mean[i] > threshold]
+    filtered_labels = [class_list[i] for i in filtered_indices]
+    filtered_values = [predictions_mean[i] for i in filtered_indices]
+    return filtered_labels, filtered_values
+
+def make_comma_separated_unique(tags):
+    seen_tags = set()
+    result = []
+    for tag in ', '.join(tags).split(', '):
+        if tag not in seen_tags:
+            result.append(tag)
+            seen_tags.add(tag)
+    return ', '.join(result)
+
+# embedding_model = TensorflowPredictEffnetDiscogs(graphFilename="discogs-effnet-bs64-1.pb", output="PartitionedCall:1")
+# genre_model = TensorflowPredict2D(graphFilename="genre_discogs400-discogs-effnet-1.pb", input="serving_default_model_Placeholder", output="PartitionedCall:0")
+# mood_model = TensorflowPredict2D(graphFilename="mtg_jamendo_moodtheme-discogs-effnet-1.pb")
+# instrument_model = TensorflowPredict2D(graphFilename="mtg_jamendo_instrument-discogs-effnet-1.pb")
+
+def get_audio_features(audio_filename):
+    audio = MonoLoader(filename=audio_filename, sampleRate=16000, resampleQuality=4)()
+    embedding_model = TensorflowPredictEffnetDiscogs(graphFilename="discogs-effnet-bs64-1.pb", output="PartitionedCall:1")
+    embeddings = embedding_model(audio)
+
+    result_dict = {}
+
+    # Predicting genres
+    genre_model = TensorflowPredict2D(graphFilename="genre_discogs400-discogs-effnet-1.pb", input="serving_default_model_Placeholder", output="PartitionedCall:0")
+    predictions = genre_model(embeddings)
+    filtered_labels, _ = filter_predictions(predictions, genre_labels)
+    filtered_labels = ', '.join(filtered_labels).replace("---", ", ").split(', ')
+    result_dict['genres'] = make_comma_separated_unique(filtered_labels)
+
+    # Predicting mood/theme
+    mood_model = TensorflowPredict2D(graphFilename="mtg_jamendo_moodtheme-discogs-effnet-1.pb")
+    predictions = mood_model(embeddings)
+    filtered_labels, _ = filter_predictions(predictions, mood_theme_classes, threshold=0.05)
+    result_dict['moods'] = make_comma_separated_unique(filtered_labels)
+
+    # Predicting instruments
+    instrument_model = TensorflowPredict2D(graphFilename="mtg_jamendo_instrument-discogs-effnet-1.pb")
+    predictions = instrument_model(embeddings)
+    filtered_labels, _ = filter_predictions(predictions, instrument_classes)
+    result_dict['instruments'] = filtered_labels
+
+    return result_dict
+
+
+def test():
+    filename = 'Mr_Blue_Sky_Pomplamoose.mp3'
+
+    # extract features
+    result = get_audio_features(str(filename))
+
+    # load audio
+    sr = librosa.get_samplerate(str(filename))
+    y, sr_load = librosa.load(str(filename), sr=sr)
+    length = librosa.get_duration(y=y, sr=sr)
+    assert sr == sr_load
+
+    # tempo
+    tempo, _ = librosa.beat.beat_track(y=y, sr=sr)
+    tempo = round(tempo)
+
+    # get key
+    chroma = librosa.feature.chroma_stft(y=y, sr=sr)
+    key = np.argmax(np.sum(chroma, axis=1))
+    key = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'][key]
+
+    # entry
+    entry = {
+        "key": f"{key}",
+        "artist": "",
+        "sample_rate": sr,
+        "file_extension": "wav",
+        "description": "",
+        "keywords": "",
+        "duration": length,
+        "bpm": tempo,
+        "genre": result.get('genres', ""),
+        "title": "",
+        "name": "",
+        "instrument": result.get('instruments', ""),
+        "moods": result.get('moods', []),
+        "path": str(filename),
+    }
+
+    # save
+    with open(str(filename).rsplit('.', 1)[0] + '.json', "w") as file:
+        json.dump(entry, file)
+
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+def process_one(filename):
+    dn = os.path.dirname(str(filename))
+    path_outfile = os.path.join(dn, 'tags.json')
+    if os.path.exists(path_outfile):
+        print('[o] exsited')
+        return
+
+    # extract features
+    result = get_audio_features(str(filename))
+
+    # load audio
+    sr = librosa.get_samplerate(str(filename))
+    y, sr_load = librosa.load(str(filename), sr=sr)
+    assert sr==sr_load
+
+    # tempo
+    tempo, _ = librosa.beat.beat_track(y=y, sr=sr)
+    tempo = round(tempo)
+
+    # get key
+    chroma = librosa.feature.chroma_stft(y=y, sr=sr)
+    key = np.argmax(np.sum(chroma, axis=1))
+    key = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'][key]
+
+    # get duration
+    length = librosa.get_duration(y=y, sr=sr)
+
+    genre = result.get('genres', "")
+    instr = result.get('instruments', "")
+    description = f"{genre} style music with instrument: {', '.join(instr)}"
+
+    # entry
+    entry = {
+        "key": f"{key}",
+        "artist": "",
+        "sample_rate": sr,
+        "file_extension": "wav",
+        "description": description,
+        "keywords": "",
+        "duration": length,
+        "bpm": tempo,
+        "genre": genre,
+        "title": "",
+        "name": "",
+        "instrument": instr,
+        "moods": result.get('moods', []),
+        "path": str(filename),
+    }
+
+    # save
+    print('[o] save to', path_outfile)
+    with open(path_outfile, "w") as file:
+        json.dump(entry, file)
+
+if __name__ == '__main__':
+    root_dir = '../audiocraft/dataset/example/clip'
+
+    base_audio = 'no_vocal'
+    base_ext = 'wav'
+    st, ed = 0, None
+
+    audio_paths = traverse_dir(
+            root_dir,
+            str_include=base_audio,
+            extension=base_ext,
+            is_sort=True)
+    num_files = len(audio_paths)
+    print(' > num of files:', num_files)
+    if ed is None:
+        ed = num_files
+
+    # Chord recognition and save lab file
+    for i in range(st, ed): 
+        print("==={}/{}======[{} - {}]========".format(
+            i, num_files, st, ed))
+        
+        filename = audio_paths[i]
+        print(filename)
+        
+        start_time = time.time()
+        try:
+            process_one(filename)
+        except:
+            print('[x] aborted')
+            continue
+        end_time = time.time()
+        runtime = end_time - start_time
+        print('testing time:', str(datetime.timedelta(seconds=runtime))+'\n')
\ No newline at end of file
diff --git a/preproc/3_tags/essentia/metadata.py b/preproc/3_tags/essentia/metadata.py
new file mode 100644
index 0000000000000000000000000000000000000000..c45a5aeebd49858e83aca76d647cb544d6bfa206
--- /dev/null
+++ b/preproc/3_tags/essentia/metadata.py
@@ -0,0 +1,504 @@
+# @title metadata (labels) for essentia
+
+genre_labels = [
+    "Blues---Boogie Woogie",
+    "Blues---Chicago Blues",
+    "Blues---Country Blues",
+    "Blues---Delta Blues",
+    "Blues---Electric Blues",
+    "Blues---Harmonica Blues",
+    "Blues---Jump Blues",
+    "Blues---Louisiana Blues",
+    "Blues---Modern Electric Blues",
+    "Blues---Piano Blues",
+    "Blues---Rhythm & Blues",
+    "Blues---Texas Blues",
+    "Brass & Military---Brass Band",
+    "Brass & Military---Marches",
+    "Brass & Military---Military",
+    "Children's---Educational",
+    "Children's---Nursery Rhymes",
+    "Children's---Story",
+    "Classical---Baroque",
+    "Classical---Choral",
+    "Classical---Classical",
+    "Classical---Contemporary",
+    "Classical---Impressionist",
+    "Classical---Medieval",
+    "Classical---Modern",
+    "Classical---Neo-Classical",
+    "Classical---Neo-Romantic",
+    "Classical---Opera",
+    "Classical---Post-Modern",
+    "Classical---Renaissance",
+    "Classical---Romantic",
+    "Electronic---Abstract",
+    "Electronic---Acid",
+    "Electronic---Acid House",
+    "Electronic---Acid Jazz",
+    "Electronic---Ambient",
+    "Electronic---Bassline",
+    "Electronic---Beatdown",
+    "Electronic---Berlin-School",
+    "Electronic---Big Beat",
+    "Electronic---Bleep",
+    "Electronic---Breakbeat",
+    "Electronic---Breakcore",
+    "Electronic---Breaks",
+    "Electronic---Broken Beat",
+    "Electronic---Chillwave",
+    "Electronic---Chiptune",
+    "Electronic---Dance-pop",
+    "Electronic---Dark Ambient",
+    "Electronic---Darkwave",
+    "Electronic---Deep House",
+    "Electronic---Deep Techno",
+    "Electronic---Disco",
+    "Electronic---Disco Polo",
+    "Electronic---Donk",
+    "Electronic---Downtempo",
+    "Electronic---Drone",
+    "Electronic---Drum n Bass",
+    "Electronic---Dub",
+    "Electronic---Dub Techno",
+    "Electronic---Dubstep",
+    "Electronic---Dungeon Synth",
+    "Electronic---EBM",
+    "Electronic---Electro",
+    "Electronic---Electro House",
+    "Electronic---Electroclash",
+    "Electronic---Euro House",
+    "Electronic---Euro-Disco",
+    "Electronic---Eurobeat",
+    "Electronic---Eurodance",
+    "Electronic---Experimental",
+    "Electronic---Freestyle",
+    "Electronic---Future Jazz",
+    "Electronic---Gabber",
+    "Electronic---Garage House",
+    "Electronic---Ghetto",
+    "Electronic---Ghetto House",
+    "Electronic---Glitch",
+    "Electronic---Goa Trance",
+    "Electronic---Grime",
+    "Electronic---Halftime",
+    "Electronic---Hands Up",
+    "Electronic---Happy Hardcore",
+    "Electronic---Hard House",
+    "Electronic---Hard Techno",
+    "Electronic---Hard Trance",
+    "Electronic---Hardcore",
+    "Electronic---Hardstyle",
+    "Electronic---Hi NRG",
+    "Electronic---Hip Hop",
+    "Electronic---Hip-House",
+    "Electronic---House",
+    "Electronic---IDM",
+    "Electronic---Illbient",
+    "Electronic---Industrial",
+    "Electronic---Italo House",
+    "Electronic---Italo-Disco",
+    "Electronic---Italodance",
+    "Electronic---Jazzdance",
+    "Electronic---Juke",
+    "Electronic---Jumpstyle",
+    "Electronic---Jungle",
+    "Electronic---Latin",
+    "Electronic---Leftfield",
+    "Electronic---Makina",
+    "Electronic---Minimal",
+    "Electronic---Minimal Techno",
+    "Electronic---Modern Classical",
+    "Electronic---Musique Concrète",
+    "Electronic---Neofolk",
+    "Electronic---New Age",
+    "Electronic---New Beat",
+    "Electronic---New Wave",
+    "Electronic---Noise",
+    "Electronic---Nu-Disco",
+    "Electronic---Power Electronics",
+    "Electronic---Progressive Breaks",
+    "Electronic---Progressive House",
+    "Electronic---Progressive Trance",
+    "Electronic---Psy-Trance",
+    "Electronic---Rhythmic Noise",
+    "Electronic---Schranz",
+    "Electronic---Sound Collage",
+    "Electronic---Speed Garage",
+    "Electronic---Speedcore",
+    "Electronic---Synth-pop",
+    "Electronic---Synthwave",
+    "Electronic---Tech House",
+    "Electronic---Tech Trance",
+    "Electronic---Techno",
+    "Electronic---Trance",
+    "Electronic---Tribal",
+    "Electronic---Tribal House",
+    "Electronic---Trip Hop",
+    "Electronic---Tropical House",
+    "Electronic---UK Garage",
+    "Electronic---Vaporwave",
+    "Folk, World, & Country---African",
+    "Folk, World, & Country---Bluegrass",
+    "Folk, World, & Country---Cajun",
+    "Folk, World, & Country---Canzone Napoletana",
+    "Folk, World, & Country---Catalan Music",
+    "Folk, World, & Country---Celtic",
+    "Folk, World, & Country---Country",
+    "Folk, World, & Country---Fado",
+    "Folk, World, & Country---Flamenco",
+    "Folk, World, & Country---Folk",
+    "Folk, World, & Country---Gospel",
+    "Folk, World, & Country---Highlife",
+    "Folk, World, & Country---Hillbilly",
+    "Folk, World, & Country---Hindustani",
+    "Folk, World, & Country---Honky Tonk",
+    "Folk, World, & Country---Indian Classical",
+    "Folk, World, & Country---Laïkó",
+    "Folk, World, & Country---Nordic",
+    "Folk, World, & Country---Pacific",
+    "Folk, World, & Country---Polka",
+    "Folk, World, & Country---Raï",
+    "Folk, World, & Country---Romani",
+    "Folk, World, & Country---Soukous",
+    "Folk, World, & Country---Séga",
+    "Folk, World, & Country---Volksmusik",
+    "Folk, World, & Country---Zouk",
+    "Folk, World, & Country---Éntekhno",
+    "Funk / Soul---Afrobeat",
+    "Funk / Soul---Boogie",
+    "Funk / Soul---Contemporary R&B",
+    "Funk / Soul---Disco",
+    "Funk / Soul---Free Funk",
+    "Funk / Soul---Funk",
+    "Funk / Soul---Gospel",
+    "Funk / Soul---Neo Soul",
+    "Funk / Soul---New Jack Swing",
+    "Funk / Soul---P.Funk",
+    "Funk / Soul---Psychedelic",
+    "Funk / Soul---Rhythm & Blues",
+    "Funk / Soul---Soul",
+    "Funk / Soul---Swingbeat",
+    "Funk / Soul---UK Street Soul",
+    "Hip Hop---Bass Music",
+    "Hip Hop---Boom Bap",
+    "Hip Hop---Bounce",
+    "Hip Hop---Britcore",
+    "Hip Hop---Cloud Rap",
+    "Hip Hop---Conscious",
+    "Hip Hop---Crunk",
+    "Hip Hop---Cut-up/DJ",
+    "Hip Hop---DJ Battle Tool",
+    "Hip Hop---Electro",
+    "Hip Hop---G-Funk",
+    "Hip Hop---Gangsta",
+    "Hip Hop---Grime",
+    "Hip Hop---Hardcore Hip-Hop",
+    "Hip Hop---Horrorcore",
+    "Hip Hop---Instrumental",
+    "Hip Hop---Jazzy Hip-Hop",
+    "Hip Hop---Miami Bass",
+    "Hip Hop---Pop Rap",
+    "Hip Hop---Ragga HipHop",
+    "Hip Hop---RnB/Swing",
+    "Hip Hop---Screw",
+    "Hip Hop---Thug Rap",
+    "Hip Hop---Trap",
+    "Hip Hop---Trip Hop",
+    "Hip Hop---Turntablism",
+    "Jazz---Afro-Cuban Jazz",
+    "Jazz---Afrobeat",
+    "Jazz---Avant-garde Jazz",
+    "Jazz---Big Band",
+    "Jazz---Bop",
+    "Jazz---Bossa Nova",
+    "Jazz---Contemporary Jazz",
+    "Jazz---Cool Jazz",
+    "Jazz---Dixieland",
+    "Jazz---Easy Listening",
+    "Jazz---Free Improvisation",
+    "Jazz---Free Jazz",
+    "Jazz---Fusion",
+    "Jazz---Gypsy Jazz",
+    "Jazz---Hard Bop",
+    "Jazz---Jazz-Funk",
+    "Jazz---Jazz-Rock",
+    "Jazz---Latin Jazz",
+    "Jazz---Modal",
+    "Jazz---Post Bop",
+    "Jazz---Ragtime",
+    "Jazz---Smooth Jazz",
+    "Jazz---Soul-Jazz",
+    "Jazz---Space-Age",
+    "Jazz---Swing",
+    "Latin---Afro-Cuban",
+    "Latin---Baião",
+    "Latin---Batucada",
+    "Latin---Beguine",
+    "Latin---Bolero",
+    "Latin---Boogaloo",
+    "Latin---Bossanova",
+    "Latin---Cha-Cha",
+    "Latin---Charanga",
+    "Latin---Compas",
+    "Latin---Cubano",
+    "Latin---Cumbia",
+    "Latin---Descarga",
+    "Latin---Forró",
+    "Latin---Guaguancó",
+    "Latin---Guajira",
+    "Latin---Guaracha",
+    "Latin---MPB",
+    "Latin---Mambo",
+    "Latin---Mariachi",
+    "Latin---Merengue",
+    "Latin---Norteño",
+    "Latin---Nueva Cancion",
+    "Latin---Pachanga",
+    "Latin---Porro",
+    "Latin---Ranchera",
+    "Latin---Reggaeton",
+    "Latin---Rumba",
+    "Latin---Salsa",
+    "Latin---Samba",
+    "Latin---Son",
+    "Latin---Son Montuno",
+    "Latin---Tango",
+    "Latin---Tejano",
+    "Latin---Vallenato",
+    "Non-Music---Audiobook",
+    "Non-Music---Comedy",
+    "Non-Music---Dialogue",
+    "Non-Music---Education",
+    "Non-Music---Field Recording",
+    "Non-Music---Interview",
+    "Non-Music---Monolog",
+    "Non-Music---Poetry",
+    "Non-Music---Political",
+    "Non-Music---Promotional",
+    "Non-Music---Radioplay",
+    "Non-Music---Religious",
+    "Non-Music---Spoken Word",
+    "Pop---Ballad",
+    "Pop---Bollywood",
+    "Pop---Bubblegum",
+    "Pop---Chanson",
+    "Pop---City Pop",
+    "Pop---Europop",
+    "Pop---Indie Pop",
+    "Pop---J-pop",
+    "Pop---K-pop",
+    "Pop---Kayōkyoku",
+    "Pop---Light Music",
+    "Pop---Music Hall",
+    "Pop---Novelty",
+    "Pop---Parody",
+    "Pop---Schlager",
+    "Pop---Vocal",
+    "Reggae---Calypso",
+    "Reggae---Dancehall",
+    "Reggae---Dub",
+    "Reggae---Lovers Rock",
+    "Reggae---Ragga",
+    "Reggae---Reggae",
+    "Reggae---Reggae-Pop",
+    "Reggae---Rocksteady",
+    "Reggae---Roots Reggae",
+    "Reggae---Ska",
+    "Reggae---Soca",
+    "Rock---AOR",
+    "Rock---Acid Rock",
+    "Rock---Acoustic",
+    "Rock---Alternative Rock",
+    "Rock---Arena Rock",
+    "Rock---Art Rock",
+    "Rock---Atmospheric Black Metal",
+    "Rock---Avantgarde",
+    "Rock---Beat",
+    "Rock---Black Metal",
+    "Rock---Blues Rock",
+    "Rock---Brit Pop",
+    "Rock---Classic Rock",
+    "Rock---Coldwave",
+    "Rock---Country Rock",
+    "Rock---Crust",
+    "Rock---Death Metal",
+    "Rock---Deathcore",
+    "Rock---Deathrock",
+    "Rock---Depressive Black Metal",
+    "Rock---Doo Wop",
+    "Rock---Doom Metal",
+    "Rock---Dream Pop",
+    "Rock---Emo",
+    "Rock---Ethereal",
+    "Rock---Experimental",
+    "Rock---Folk Metal",
+    "Rock---Folk Rock",
+    "Rock---Funeral Doom Metal",
+    "Rock---Funk Metal",
+    "Rock---Garage Rock",
+    "Rock---Glam",
+    "Rock---Goregrind",
+    "Rock---Goth Rock",
+    "Rock---Gothic Metal",
+    "Rock---Grindcore",
+    "Rock---Grunge",
+    "Rock---Hard Rock",
+    "Rock---Hardcore",
+    "Rock---Heavy Metal",
+    "Rock---Indie Rock",
+    "Rock---Industrial",
+    "Rock---Krautrock",
+    "Rock---Lo-Fi",
+    "Rock---Lounge",
+    "Rock---Math Rock",
+    "Rock---Melodic Death Metal",
+    "Rock---Melodic Hardcore",
+    "Rock---Metalcore",
+    "Rock---Mod",
+    "Rock---Neofolk",
+    "Rock---New Wave",
+    "Rock---No Wave",
+    "Rock---Noise",
+    "Rock---Noisecore",
+    "Rock---Nu Metal",
+    "Rock---Oi",
+    "Rock---Parody",
+    "Rock---Pop Punk",
+    "Rock---Pop Rock",
+    "Rock---Pornogrind",
+    "Rock---Post Rock",
+    "Rock---Post-Hardcore",
+    "Rock---Post-Metal",
+    "Rock---Post-Punk",
+    "Rock---Power Metal",
+    "Rock---Power Pop",
+    "Rock---Power Violence",
+    "Rock---Prog Rock",
+    "Rock---Progressive Metal",
+    "Rock---Psychedelic Rock",
+    "Rock---Psychobilly",
+    "Rock---Pub Rock",
+    "Rock---Punk",
+    "Rock---Rock & Roll",
+    "Rock---Rockabilly",
+    "Rock---Shoegaze",
+    "Rock---Ska",
+    "Rock---Sludge Metal",
+    "Rock---Soft Rock",
+    "Rock---Southern Rock",
+    "Rock---Space Rock",
+    "Rock---Speed Metal",
+    "Rock---Stoner Rock",
+    "Rock---Surf",
+    "Rock---Symphonic Rock",
+    "Rock---Technical Death Metal",
+    "Rock---Thrash",
+    "Rock---Twist",
+    "Rock---Viking Metal",
+    "Rock---Yé-Yé",
+    "Stage & Screen---Musical",
+    "Stage & Screen---Score",
+    "Stage & Screen---Soundtrack",
+    "Stage & Screen---Theme",
+]
+mood_theme_classes = [
+    "action",
+    "adventure",
+    "advertising",
+    "background",
+    "ballad",
+    "calm",
+    "children",
+    "christmas",
+    "commercial",
+    "cool",
+    "corporate",
+    "dark",
+    "deep",
+    "documentary",
+    "drama",
+    "dramatic",
+    "dream",
+    "emotional",
+    "energetic",
+    "epic",
+    "fast",
+    "film",
+    "fun",
+    "funny",
+    "game",
+    "groovy",
+    "happy",
+    "heavy",
+    "holiday",
+    "hopeful",
+    "inspiring",
+    "love",
+    "meditative",
+    "melancholic",
+    "melodic",
+    "motivational",
+    "movie",
+    "nature",
+    "party",
+    "positive",
+    "powerful",
+    "relaxing",
+    "retro",
+    "romantic",
+    "sad",
+    "sexy",
+    "slow",
+    "soft",
+    "soundscape",
+    "space",
+    "sport",
+    "summer",
+    "trailer",
+    "travel",
+    "upbeat",
+    "uplifting"
+]
+instrument_classes = [
+    "accordion",
+    "acousticbassguitar",
+    "acousticguitar",
+    "bass",
+    "beat",
+    "bell",
+    "bongo",
+    "brass",
+    "cello",
+    "clarinet",
+    "classicalguitar",
+    "computer",
+    "doublebass",
+    "drummachine",
+    "drums",
+    "electricguitar",
+    "electricpiano",
+    "flute",
+    "guitar",
+    "harmonica",
+    "harp",
+    "horn",
+    "keyboard",
+    "oboe",
+    "orchestra",
+    "organ",
+    "pad",
+    "percussion",
+    "piano",
+    "pipeorgan",
+    "rhodes",
+    "sampler",
+    "saxophone",
+    "strings",
+    "synthesizer",
+    "trombone",
+    "trumpet",
+    "viola",
+    "violin",
+    "voice"
+]
\ No newline at end of file
diff --git a/preproc/3_tags/essentia/mtg_jamendo_instrument-discogs-effnet-1.pb b/preproc/3_tags/essentia/mtg_jamendo_instrument-discogs-effnet-1.pb
new file mode 100644
index 0000000000000000000000000000000000000000..0143cc9cb96b19eff7f22b11d8ec24e4cbec987e
--- /dev/null
+++ b/preproc/3_tags/essentia/mtg_jamendo_instrument-discogs-effnet-1.pb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e8c3003c722e098da371b6a1f7ad0ce62fac0dcfc09c7c7997d430941196c2a
+size 2706836
diff --git a/preproc/3_tags/essentia/mtg_jamendo_moodtheme-discogs-effnet-1.pb b/preproc/3_tags/essentia/mtg_jamendo_moodtheme-discogs-effnet-1.pb
new file mode 100644
index 0000000000000000000000000000000000000000..72fecec9a7699d9e7955ad0610ef9db05d00fa0b
--- /dev/null
+++ b/preproc/3_tags/essentia/mtg_jamendo_moodtheme-discogs-effnet-1.pb
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03f2b047020aee4ab39f8880da7bdae2a36d06a1508d656c6d424ad4d6de07a9
+size 2739668
diff --git a/preproc/3_tags/essentia/tags_convert.py b/preproc/3_tags/essentia/tags_convert.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e9e2c41af3c89a93bb908aeeb4792eba00666ec
--- /dev/null
+++ b/preproc/3_tags/essentia/tags_convert.py
@@ -0,0 +1,137 @@
+import os
+import json
+import soundfile as sf
+import numpy as np
+
+from tqdm import tqdm
+import time
+import librosa
+import sys
+
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+
+def yt2json(path_audio, path_json, output_path):
+
+    # load
+    wav, sr = sf.read(path_audio, always_2d=True)
+    duration = len(wav) / sr
+    with open(path_json ,'r') as f:
+        json_str = f.read()
+    ess_json = json.loads(json_str)
+
+    # get tempo
+    # tempo, _ = librosa.beat.beat_track(y=wav, sr=sr)
+    # tempo = round(tempo)
+
+    # get key (takes long time)
+    # chroma = librosa.feature.chroma_stft(y=wav, sr=sr)
+    # key = np.argmax(np.sum(chroma, axis=1))
+    # key = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'][key]
+
+    mg_json = {"key": "", "artist": "", "sample_rate": 0, 
+               "file_extension": "wav", "description": "", 
+               "keywords": "", "duration": 0, "bpm": "", 
+               "genre": "", "title": "", "name": "", "instrument": "", "moods": [], "path":""}
+    
+    mg_json["key"] = ess_json["key"]
+    mg_json["sample_rate"] = ess_json["sample_rate"]
+    mg_json["duration"] = ess_json["duration"]
+    mg_json["bpm"] = ess_json["bpm"]
+    mg_json["genre"] = ess_json["genre"]
+    mg_json["instrument"] = ess_json["instrument"]
+    mg_json["moods"] = ess_json["moods"]
+    mg_json["path"] = ess_json["path"]
+
+    mg_json["description"] = f"{ess_json['genre']} style music with instrument: {', '.join(ess_json['instrument'])}"
+    
+    
+    with open(output_path, 'w') as js_file:
+        json.dump(mg_json, js_file)
+    
+
+if __name__ == '__main__':
+
+    root_dir = '../audiocraft/dataset/example/clip'
+    base_audio = 'no_vocal'
+    base_ext = 'wav'
+    st, ed = 0, None
+
+    audio_paths = traverse_dir(
+            root_dir,
+            str_include=base_audio,
+            extension=base_ext,
+            is_sort=True)
+
+    num_files = len(audio_paths)
+    print(' > num of files:', num_files)
+    if ed is None:
+        ed = num_files
+
+    # run
+    err_files = [] 
+    for i in range(st, ed): 
+        print("==={}/{}======[{} - {}]========".format(
+            i, num_files, st, ed))
+
+        # path
+        path_audio = audio_paths[i]
+        dn = os.path.dirname(path_audio)
+        
+        path_json = path_audio.replace('no_vocal.wav', 'extracted_tags.json')
+        print(path_audio)
+        print(path_json)
+        output_path = path_audio.replace('no_vocal.wav', 'tags.json')
+
+        # export abs midi
+        try:
+            yt2json(path_audio, path_json, output_path)
+           
+        except:
+            print('[x] error')
+            err_files.append(path_audio)
+            continue
+    print('\n\n\n==================')
+    print('Error Files:')
+    for idx, err_f in enumerate(err_files):
+        print(idx, '-', err_f)
\ No newline at end of file
diff --git a/preproc/3_tags/install.sh b/preproc/3_tags/install.sh
new file mode 100644
index 0000000000000000000000000000000000000000..4e59c61454196187967376b17a5c0b001ef30d6e
--- /dev/null
+++ b/preproc/3_tags/install.sh
@@ -0,0 +1,8 @@
+conda create -n tags python=3.9 -y
+conda activate tags
+apt-get update
+apt-get install tmux vim git gcc -y
+apt-get install ffmpeg -y
+pip install essentia-tensorflow
+pip install tensorflow
+pip install librosa
\ No newline at end of file
diff --git a/preproc/dump_jsonl.py b/preproc/dump_jsonl.py
new file mode 100644
index 0000000000000000000000000000000000000000..fc3e42944d0426bd5f73129dacf5c3c5298a81d2
--- /dev/null
+++ b/preproc/dump_jsonl.py
@@ -0,0 +1,89 @@
+import os
+import json
+
+import sys
+import librosa
+
+def traverse_dir(
+        root_dir,
+        extension,
+        amount=None,
+        str_include=None,
+        str_exclude=None,
+        is_pure=False,
+        is_sort=False,
+        is_ext=True):
+
+    file_list = []
+    cnt = 0
+    for root, _, files in os.walk(root_dir):
+        for file in files:
+            if file.endswith(extension):
+                # path
+                mix_path = os.path.join(root, file)
+                pure_path = mix_path[len(root_dir)+1:] if is_pure else mix_path
+
+                # amount
+                if (amount is not None) and (cnt == amount):
+                    if is_sort:
+                        file_list.sort()
+                    return file_list
+                
+                # check string
+                if (str_include is not None) and (str_include not in pure_path):
+                    continue
+                if (str_exclude is not None) and (str_exclude in pure_path):
+                    continue
+                
+                if not is_ext:
+                    ext = pure_path.split('.')[-1]
+                    pure_path = pure_path[:-(len(ext)+1)]
+                file_list.append(pure_path)
+                cnt += 1
+    if is_sort:
+        file_list.sort()
+    return file_list
+
+
+if __name__ == '__main__':
+    root_dir = '../audiocraft/dataset/example/clip'
+    path_jsonl = '../audiocraft/egs/example/data.jsonl'
+
+    filelist = traverse_dir(
+        root_dir,
+        extension='wav',
+        str_include='no_vocal',
+        is_sort=True)
+    num_files = len(filelist)
+
+    with open(path_jsonl, "w") as train_file:
+    
+        for fidx in range(num_files):
+            print(f'==={fidx}/{num_files}================')
+            path_wave = filelist[fidx]
+            path_json = os.path.join(
+                os.path.dirname(path_wave), 'tags.json')
+
+            sr = librosa.get_samplerate(path_wave)
+            
+            print('path_wave:', path_wave)
+            print('path_json:', path_json)
+
+            with open(path_json, 'r') as f:
+                data = json.load(f)
+            assert sr == data['sample_rate']
+
+            final = {
+                'path': data['path'],
+                'duration': data['duration'],
+                "sample_rate": data['sample_rate'],
+                "bpm": data['bpm'],
+                "amplitude": None, 
+                "weight": None, 
+                "info_path": None
+            }
+            train_file.write(json.dumps(final) + '\n')
+    print('\n\n\n==================')
+    print('num files:', num_files)
+
+   
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d365b94c466879231749998cf6b1f29dfec54c1a
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,25 @@
+av==11.0.0
+einops
+flashy==0.0.1
+hydra-core==1.1
+hydra_colorlog
+julius
+num2words
+numpy==1.24.4
+sentencepiece
+spacy==3.6.1
+torch==2.0.0
+torchaudio==2.0.0
+tqdm
+transformers==4.31.0  # need Encodec there.
+xformers==0.0.22
+demucs
+librosa
+soundfile
+torchmetrics
+encodec
+protobuf
+torchvision==0.16.0
+torchtext==0.16.0
+pesq
+pystoi
\ No newline at end of file