add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"pad_token": "[PAD]"}

tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":{"max_length":2048,"strategy":"LongestFirst","stride":0},"padding":null,"added_tokens":[{"id":0,"special":true,"content":"[UNK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":1,"special":true,"content":"[CLS]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":2,"special":true,"content":"[SEP]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":3,"special":true,"content":"[PAD]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false},{"id":4,"special":true,"content":"[MASK]","single_word":false,"lstrip":false,"rstrip":false,"normalized":false}],"normalizer":null,"pre_tokenizer":{"type":"WhitespaceSplit"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"[CLS]":1,"[SEP]":2,"[PAD]":3,"[MASK]":4,"NOTE_ON=67":5,"NOTE_OFF=60":6,"INST=125":7,"PIECE_START":8,"NOTE_ON=12":9,"NOTE_OFF=28":10,"NOTE_ON=80":11,"INST=115":12,"INST=88":13,"INST=76":14,"NOTE_OFF=18":15,"NOTE_ON=4":16,"INST=71":17,"NOTE_OFF=85":18,"NOTE_ON=28":19,"INST=13":20,"INST=73":21,"NOTE_OFF=24":22,"NOTE_OFF=94":23,"NOTE_OFF=102":24,"DENSITY=9":25,"NOTE_ON=103":26,"NOTE_ON=7":27,"INST=85":28,"NOTE_OFF=15":29,"INST=4":30,"INST=56":31,"DENSITY=8":32,"NOTE_OFF=16":33,"NOTE_ON=1":34,"NOTE_OFF=101":35,"NOTE_ON=53":36,"DENSITY=3":37,"NOTE_ON=93":38,"NOTE_OFF=107":39,"NOTE_ON=118":40,"NOTE_ON=38":41,"INST=5":42,"INST=54":43,"NOTE_ON=89":44,"INST=47":45,"NOTE_ON=23":46,"INST=79":47,"INST=52":48,"NOTE_ON=54":49,"NOTE_ON=24":50,"INST=15":51,"NOTE_OFF=109":52,"NOTE_ON=15":53,"INST=31":54,"INST=90":55,"NOTE_OFF=74":56,"DENSITY=4":57,"NOTE_ON=25":58,"NOTE_OFF=23":59,"INST=127":60,"NOTE_ON=22":61,"INST=126":62,"NOTE_ON=108":63,"NOTE_ON=82":64,"INST=34":65,"BAR_END":66,"NOTE_ON=107":67,"NOTE_OFF=121":68,"NOTE_ON=0":69,"NOTE_ON=10":70,"INST=63":71,"NOTE_OFF=57":72,"INST=7":73,"INST=75":74,"NOTE_ON=74":75,"NOTE_OFF=29":76,"NOTE_ON=83":77,"NOTE_OFF=45":78,"NOTE_OFF=81":79,"NOTE_OFF=20":80,"NOTE_ON=81":81,"INST=123":82,"INST=14":83,"NOTE_OFF=96":84,"NOTE_ON=76":85,"INST=95":86,"NOTE_OFF=36":87,"INST=61":88,"INST=12":89,"NOTE_OFF=59":90,"NOTE_OFF=66":91,"NOTE_OFF=78":92,"INST=64":93,"TIME_DELTA=2":94,"NOTE_ON=64":95,"NOTE_OFF=118":96,"NOTE_OFF=47":97,"NOTE_OFF=52":98,"INST=22":99,"NOTE_OFF=62":100,"INST=3":101,"NOTE_OFF=84":102,"NOTE_ON=19":103,"INST=92":104,"NOTE_ON=102":105,"INST=108":106,"NOTE_OFF=83":107,"INST=58":108,"INST=119":109,"INST=55":110,"NOTE_ON=84":111,"INST=19":112,"INST=53":113,"TRACK_START":114,"NOTE_OFF=70":115,"NOTE_OFF=125":116,"INST=33":117,"INST=1":118,"NOTE_ON=40":119,"TRACK_END":120,"INST=86":121,"NOTE_OFF=5":122,"NOTE_OFF=49":123,"NOTE_OFF=26":124,"BAR_START":125,"NOTE_ON=59":126,"INST=114":127,"NOTE_OFF=37":128,"INST=102":129,"INST=84":130,"NOTE_OFF=65":131,"NOTE_OFF=72":132,"NOTE_OFF=56":133,"NOTE_OFF=111":134,"NOTE_OFF=126":135,"INST=65":136,"NOTE_ON=16":137,"NOTE_ON=13":138,"NOTE_ON=47":139,"NOTE_ON=106":140,"NOTE_ON=125":141,"NOTE_ON=122":142,"NOTE_ON=113":143,"INST=69":144,"TIME_DELTA=16":145,"NOTE_ON=72":146,"NOTE_ON=104":147,"NOTE_ON=60":148,"NOTE_OFF=112":149,"NOTE_OFF=120":150,"NOTE_OFF=113":151,"NOTE_OFF=86":152,"INST=50":153,"NOTE_OFF=108":154,"INST=51":155,"TIME_DELTA=14":156,"NOTE_ON=32":157,"NOTE_OFF=50":158,"TIME_DELTA=11":159,"NOTE_OFF=92":160,"NOTE_ON=27":161,"NOTE_OFF=6":162,"INST=74":163,"INST=59":164,"INST=26":165,"INST=124":166,"NOTE_ON=2":167,"NOTE_OFF=32":168,"NOTE_ON=11":169,"NOTE_OFF=68":170,"INST=42":171,"NOTE_ON=63":172,"NOTE_OFF=19":173,"NOTE_ON=94":174,"NOTE_ON=126":175,"INST=117":176,"DENSITY=2":177,"INST=109":178,"INST=83":179,"NOTE_ON=121":180,"INST=120":181,"INST=29":182,"NOTE_ON=101":183,"INST=17":184,"NOTE_OFF=25":185,"NOTE_ON=61":186,"INST=81":187,"INST=80":188,"NOTE_ON=123":189,"NOTE_ON=56":190,"INST=78":191,"INST=35":192,"NOTE_OFF=69":193,"INST=36":194,"INST=11":195,"INST=43":196,"INST=112":197,"NOTE_OFF=27":198,"NOTE_ON=71":199,"INST=94":200,"NOTE_OFF=8":201,"NOTE_OFF=22":202,"TIME_DELTA=4":203,"NOTE_OFF=100":204,"NOTE_ON=55":205,"NOTE_ON=48":206,"NOTE_OFF=14":207,"INST=44":208,"NOTE_OFF=123":209,"NOTE_ON=90":210,"INST=60":211,"NOTE_OFF=39":212,"NOTE_OFF=115":213,"PIECE_END":214,"DENSITY=1":215,"DENSITY=5":216,"INST=46":217,"INST=70":218,"INST=18":219,"NOTE_OFF=55":220,"INST=38":221,"NOTE_ON=43":222,"NOTE_ON=69":223,"INST=41":224,"DENSITY=6":225,"INST=122":226,"INST=49":227,"NOTE_OFF=77":228,"TIME_DELTA=1":229,"NOTE_OFF=11":230,"NOTE_ON=78":231,"NOTE_ON=112":232,"NOTE_ON=52":233,"INST=111":234,"INST=67":235,"NOTE_ON=114":236,"INST=28":237,"NOTE_OFF=41":238,"NOTE_ON=96":239,"NOTE_ON=44":240,"INST=25":241,"NOTE_ON=49":242,"INST=113":243,"NOTE_ON=62":244,"NOTE_OFF=97":245,"NOTE_OFF=76":246,"NOTE_OFF=13":247,"NOTE_ON=116":248,"INST=99":249,"DENSITY=7":250,"INST=16":251,"INST=21":252,"INST=91":253,"TIME_DELTA=6":254,"NOTE_OFF=119":255,"NOTE_OFF=43":256,"NOTE_OFF=105":257,"NOTE_OFF=61":258,"NOTE_ON=29":259,"NOTE_ON=30":260,"NOTE_ON=68":261,"NOTE_OFF=0":262,"INST=32":263,"INST=57":264,"TIME_DELTA=10":265,"NOTE_ON=39":266,"NOTE_OFF=21":267,"INST=68":268,"NOTE_ON=66":269,"NOTE_OFF=104":270,"NOTE_ON=120":271,"NOTE_OFF=1":272,"NOTE_OFF=79":273,"NOTE_OFF=98":274,"TIME_DELTA=9":275,"INST=101":276,"NOTE_ON=18":277,"NOTE_ON=75":278,"NOTE_ON=99":279,"NOTE_OFF=10":280,"INST=8":281,"NOTE_OFF=73":282,"NOTE_OFF=90":283,"INST=104":284,"NOTE_ON=87":285,"INST=24":286,"NOTE_OFF=89":287,"NOTE_OFF=48":288,"INST=103":289,"TIME_DELTA=12":290,"NOTE_OFF=38":291,"NOTE_ON=41":292,"NOTE_OFF=88":293,"INST=118":294,"INST=82":295,"NOTE_ON=115":296,"TIME_DELTA=13":297,"INST=2":298,"NOTE_ON=58":299,"NOTE_ON=92":300,"NOTE_ON=21":301,"NOTE_ON=57":302,"NOTE_OFF=42":303,"NOTE_ON=34":304,"INST=110":305,"NOTE_OFF=53":306,"NOTE_OFF=93":307,"NOTE_ON=42":308,"NOTE_ON=79":309,"DENSITY=0":310,"TIME_DELTA=8":311,"INST=62":312,"INST=100":313,"NOTE_OFF=106":314,"NOTE_ON=36":315,"INST=72":316,"NOTE_OFF=82":317,"INST=45":318,"NOTE_OFF=2":319,"INST=87":320,"NOTE_OFF=95":321,"INST=9":322,"NOTE_OFF=114":323,"INST=93":324,"INST=40":325,"NOTE_OFF=64":326,"NOTE_OFF=91":327,"NOTE_ON=73":328,"TIME_DELTA=15":329,"INST=105":330,"NOTE_ON=3":331,"NOTE_ON=77":332,"INST=66":333,"INST=77":334,"NOTE_ON=127":335,"NOTE_ON=9":336,"NOTE_OFF=17":337,"NOTE_ON=100":338,"NOTE_ON=33":339,"INST=97":340,"TIME_DELTA=3":341,"NOTE_OFF=12":342,"NOTE_ON=50":343,"NOTE_OFF=80":344,"NOTE_ON=8":345,"NOTE_OFF=54":346,"NOTE_ON=91":347,"NOTE_ON=124":348,"INST=89":349,"INST=98":350,"NOTE_OFF=46":351,"NOTE_OFF=117":352,"INST=39":353,"NOTE_ON=20":354,"TIME_DELTA=7":355,"NOTE_OFF=124":356,"NOTE_OFF=71":357,"NOTE_OFF=67":358,"NOTE_ON=117":359,"NOTE_ON=110":360,"NOTE_ON=111":361,"NOTE_ON=14":362,"INST=30":363,"NOTE_OFF=9":364,"NOTE_ON=86":365,"INST=27":366,"NOTE_OFF=127":367,"NOTE_ON=51":368,"INST=121":369,"NOTE_ON=17":370,"NOTE_OFF=51":371,"NOTE_ON=98":372,"NOTE_OFF=30":373,"INST=20":374,"INST=96":375,"NOTE_ON=31":376,"INST=6":377,"NOTE_ON=35":378,"INST=23":379,"NOTE_OFF=99":380,"NOTE_OFF=34":381,"NOTE_OFF=116":382,"INST=0":383,"NOTE_ON=6":384,"NOTE_OFF=31":385,"NOTE_ON=46":386,"INST=37":387,"NOTE_ON=105":388,"INST=10":389,"NOTE_ON=70":390,"NOTE_ON=26":391,"NOTE_OFF=40":392,"NOTE_OFF=103":393,"INST=106":394,"NOTE_OFF=4":395,"INST=DRUMS":396,"NOTE_OFF=75":397,"NOTE_ON=45":398,"NOTE_OFF=3":399,"NOTE_OFF=122":400,"NOTE_OFF=35":401,"NOTE_OFF=58":402,"NOTE_OFF=7":403,"NOTE_ON=5":404,"NOTE_ON=119":405,"NOTE_ON=109":406,"TIME_DELTA=5":407,"NOTE_OFF=87":408,"NOTE_ON=37":409,"NOTE_ON=88":410,"NOTE_ON=97":411,"NOTE_OFF=33":412,"INST=48":413,"INST=107":414,"NOTE_OFF=110":415,"NOTE_ON=85":416,"NOTE_ON=65":417,"NOTE_OFF=63":418,"INST=116":419,"NOTE_OFF=44":420,"NOTE_ON=95":421},"unk_token":"[UNK]"}}

tokenizer_config.json

@@ -0,0 +1 @@
+{"tokenizer_class": "PreTrainedTokenizerFast"}