OpenSLU / save /stack /tokenizer.json
LightChen2333's picture
Upload 34 files
37b9e99
{"name": "word_tokenizer", "token_map": {"[PAD]": 0, "[UNK]": 1, "i": 2, "want": 3, "to": 4, "fly": 5, "from": 6, "baltimore": 7, "dallas": 8, "round": 9, "trip": 10, "fares": 11, "philadelphia": 12, "less": 13, "than": 14, "1000": 15, "dollars": 16, "denver": 17, "pittsburgh": 18, "show": 19, "me": 20, "the": 21, "flights": 22, "arriving": 23, "on": 24, "june": 25, "fourteenth": 26, "what": 27, "are": 28, "which": 29, "depart": 30, "san": 31, "francisco": 32, "washington": 33, "via": 34, "indianapolis": 35, "and": 36, "arrive": 37, "by": 38, "9": 39, "pm": 40, "airlines": 41, "boston": 42, "dc": 43, "other": 44, "cities": 45, "i'm": 46, "looking": 47, "for": 48, "a": 49, "flight": 50, "charlotte": 51, "las": 52, "vegas": 53, "that": 54, "stops": 55, "in": 56, "st.": 57, "louis": 58, "hopefully": 59, "dinner": 60, "how": 61, "can": 62, "find": 63, "out": 64, "okay": 65, "then": 66, "i'd": 67, "like": 68, "travel": 69, "atlanta": 70, "september": 71, "fourth": 72, "all": 73, "cincinnati": 74, "us": 75, "air": 76, "diego": 77, "afternoon": 78, "what's": 79, "available": 80, "tuesday": 81, "leave": 82, "phoenix": 83, "paul": 84, "minnesota": 85, "after": 86, "noon": 87, "american": 88, "chicago": 89, "los": 90, "angeles": 91, "morning": 92, "types": 93, "of": 94, "ground": 95, "transportation": 96, "there": 97, "airport": 98, "next": 99, "two": 100, "days": 101, "nashville": 102, "jose": 103, "or": 104, "tacoma": 105, "does": 106, "continental": 107, "milwaukee": 108, "many": 109, "twa": 110, "have": 111, "business": 112, "class": 113, "first": 114, "least": 115, "expensive": 116, "one": 117, "way": 118, "fare": 119, "booking": 120, "classes": 121, "wednesday": 122, "nineteenth": 123, "july": 124, "fifth": 125, "7": 126, "please": 127, "list": 128, "departing": 129, "general": 130, "mitchell": 131, "international": 132, "time": 133, "zone": 134, "is": 135, "serves": 136, "meal": 137, "seattle": 138, "salt": 139, "lake": 140, "city": 141, "you": 142, "with": 143, "economy": 144, "leaving": 145, "miami": 146, "cleveland": 147, "give": 148, "between": 149, "their": 150, "cost": 151, "code": 152, "y": 153, "mean": 154, "could": 155, "tell": 156, "leaves": 157, "united": 158, "airline": 159, "over": 160, "departures": 161, "arrivals": 162, "earliest": 163, "latest": 164, "return": 165, "within": 166, "same": 167, "day": 168, "orlando": 169, "either": 170, "evening": 171, "thursday": 172, "originating": 173, "going": 174, "order": 175, "snack": 176, "do": 177, "at": 178, "645": 179, "am": 180, "into": 181, "atlanta's": 182, "friday": 183, "qx": 184, "would": 185, "information": 186, "but": 187, "stopover": 188, "some": 189, "oakland": 190, "monday": 191, "know": 192, "type": 193, "aircraft": 194, "used": 195, "detroit": 196, "twenty": 197, "eighth": 198, "petersburg": 199, "take": 200, "begins": 201, "lands": 202, "fort": 203, "worth": 204, "stop": 205, "tomorrow": 206, "noontime": 207, "around": 208, "need": 209, "northwest": 210, "toronto": 211, "memphis": 212, "thirtieth": 213, "nonstop": 214, "houston": 215, "august": 216, "twentieth": 217, "ewr": 218, "seventh": 219, "newark": 220, "11": 221, "lowest": 222, "delta": 223, "has": 224, "go": 225, "any": 226, "jet": 227, "mco": 228, "new": 229, "jersey": 230, "ontario": 231, "saturday": 232, "york": 233, "long": 234, "it": 235, "get": 236, "prices": 237, "an": 238, "inexpensive": 239, "breakfast": 240, "direct": 241, "what're": 242, "sunday": 243, "colorado": 244, "see": 245, "serving": 246, "before": 247, "o'clock": 248, "january": 249, "1992": 250, "10": 251, "2": 252, "445": 253, "515": 254, "6": 255, "8": 256, "coach": 257, "only": 258, "weekdays": 259, "la": 260, "3": 261, "my": 262, "choices": 263, "early": 264, "minneapolis": 265, "cheapest": 266, "flying": 267, "possible": 268, "daily": 269, "beach": 270, "stopping": 271, "kansas": 272, "night": 273, "serve": 274, "meals": 275, "heading": 276, "kind": 277, "once": 278, "mornings": 279, "ff": 280, "arrangements": 281, "served": 282, "canadian": 283, "california": 284, "about": 285, "530": 286, "kinds": 287, "requesting": 288, "landing": 289, "distance": 290, "downtown": 291, "love": 292, "field": 293, "this": 294, "tampa": 295, "florida": 296, "5": 297, "must": 298, "be": 299, "advertises": 300, "having": 301, "more": 302, "november": 303, "eleventh": 304, "services": 305, "qo": 306, "american's": 307, "last": 308, "using": 309, "dl": 310, "217": 311, "montreal": 312, "service": 313, "ticket": 314, "should": 315, "near": 316, "also": 317, "missouri": 318, "utah": 319, "interested": 320, "those": 321, "when": 322, "north": 323, "carolina": 324, "415": 325, "200": 326, "explain": 327, "codes": 328, "sd": 329, "d": 330, "trying": 331, "plane": 332, "flies": 333, "weekday": 334, "columbus": 335, "1991": 336, "carries": 337, "smallest": 338, "number": 339, "passengers": 340, "takeoffs": 341, "landings": 342, "book": 343, "shortest": 344, "both": 345, "nationair": 346, "823": 347, "guardia": 348, "as": 349, "well": 350, "sixteenth": 351, "rental": 352, "car": 353, "rates": 354, "through": 355, "boeing": 356, "757": 357, "limousine": 358, "listing": 359, "canada": 360, "much": 361, "71": 362, "airfare": 363, "12": 364, "third": 365, "seating": 366, "capacity": 367, "arrives": 368, "bwi": 369, "ninth": 370, "late": 371, "nevada": 372, "4": 373, "price": 374, "fifteenth": 375, "eighteenth": 376, "returning": 377, "following": 378, "capacities": 379, "planes": 380, "1145": 381, "use": 382, "burbank": 383, "may": 384, "america": 385, "west": 386, "now": 387, "eastern": 388, "825": 389, "555": 390, "area": 391, "schedule": 392, "dfw": 393, "these": 394, "connecting": 395, "make": 396, "connection": 397, "lunch": 398, "f": 399, "belong": 400, "most": 401, "tickets": 402, "logan": 403, "vicinity": 404, "210": 405, "wednesdays": 406, "thursdays": 407, "yes": 408, "will": 409, "continuing": 410, "1039": 411, "southwest": 412, "times": 413, "400": 414, "week": 415, "if": 416, "813": 417, "enroute": 418, "another": 419, "twelfth": 420, "turboprop": 421, "420": 422, "today": 423, "1": 424, "we're": 425, "westchester": 426, "county": 427, "various": 428, "airplanes": 429, "uses": 430, "yn": 431, "852": 432, "transport": 433, "display": 434, "under": 435, "500": 436, "airfares": 437, "back": 438, "hours": 439, "fn": 440, "options": 441, "december": 442, "second": 443, "april": 444, "ohio": 445, "departs": 446, "2153": 447, "schedules": 448, "who": 449, "restriction": 450, "ap": 451, "57": 452, "layover": 453, "abbreviation": 454, "stands": 455, "1291": 456, "324": 457, "again": 458, "offer": 459, "dc10": 460, "currently": 461, "represented": 462, "database": 463, "arizona": 464, "1505": 465, "sixth": 466, "3724": 467, "three": 468, "including": 469, "connections": 470, "numbers": 471, "six": 472, "1100": 473, "destination": 474, "838": 475, "no": 476, "h": 477, "traveling": 478, "ap57": 479, "far": 480, "lufthansa": 481, "abbreviations": 482, "such": 483, "aa": 484, "459": 485, "where": 486, "ua": 487, "281": 488, "your": 489, "texas": 490, "1500": 491, "bound": 492, "includes": 493, "right": 494, "airports": 495, "eight": 496, "sixteen": 497, "trips": 498, "seventeenth": 499, "thrift": 500, "delta's": 501, "departure": 502, "listed": 503, "1055": 504, "405": 505, "midnight": 506, "hi": 507, "630": 508, "question": 509, "live": 510, "stand": 511, "ten": 512, "people": 513, "during": 514, "2100": 515, "gets": 516, "just": 517, "philly": 518, "21": 519, "airplane": 520, "1765": 521, "iah": 522, "737": 523, "midwest": 524, "express": 525, "s": 526, "designate": 527, "747": 528, "650": 529, "goes": 530, "reaches": 531, "seventeen": 532, "sorry": 533, "anywhere": 534, "provided": 535, "d10": 536, "toward": 537, "preferably": 538, "rate": 539, "difference": 540, "q": 541, "b": 542, "ac": 543, "tower": 544, "tenth": 545, "hp": 546, "4400": 547, "georgia": 548, "offers": 549, "fine": 550, "201": 551, "343": 552, "october": 553, "ea": 554, "jfk": 555, "name": 556, "arrange": 557, "largest": 558, "connect": 559, "operating": 560, "sundays": 561, "720": 562, "land": 563, "final": 564, "don't": 565, "stopovers": 566, "total": 567, "friday's": 568, "755": 569, "cheap": 570, "sfo": 571, "thirty": 572, "across": 573, "continent": 574, "makes": 575, "1220": 576, "co": 577, "1209": 578, "wanted": 579, "1850": 580, "without": 581, "listings": 582, "local": 583, "wish": 584, "bring": 585, "up": 586, "home": 587, "417": 588, "approximately": 589, "actually": 590, "1200": 591, "230": 592, "819": 593, "serviced": 594, "928": 595, "reservation": 596, "limousines": 597, "taxi": 598, "fit": 599, "72s": 600, "352": 601, "1133": 602, "43": 603, "define": 604, "directly": 605, "m80": 606, "close": 607, "restrictions": 608, "430": 609, "718": 610, "hou": 611, "costs": 612, "466": 613, "march": 614, "1026": 615, "1024": 616, "different": 617, "rentals": 618, "each": 619, "arrival": 620, "say": 621, "mealtime": 622, "932": 623, "1115": 624, "1245": 625, "include": 626, "whether": 627, "offered": 628, "130": 629, "alaska": 630, "296": 631, "they": 632, "106": 633, "york's": 634, "497766": 635, "itinerary": 636, "coming": 637, "month": 638, "bur": 639, "travels": 640, "pennsylvania": 641, "usa": 642, "1288": 643, "c": 644, "names": 645, "sure": 646, "meaning": 647, "ap80": 648, "269": 649, "reservations": 650, "d9s": 651, "sunday's": 652, "f28": 653, "934": 654, "earlier": 655, "1017": 656, "date": 657, "thank": 658, "oak": 659, "atl": 660, "cp": 661, "3357": 662, "1045": 663, "limo": 664, "845": 665, "sometime": 666, "1222": 667, "i'll": 668, "tennessee": 669, "0900": 670, "hello": 671, "let": 672, "repeat": 673, "provide": 674, "still": 675, "along": 676, "operation": 677, "year": 678, "one's": 679, "great": 680, "too": 681, "nighttime": 682, "1300": 683, "saturdays": 684, "416": 685, "four": 686, "257": 687, "minimum": 688, "intercontinental": 689, "february": 690, "spend": 691, "lastest": 692, "thing": 693, "originate": 694, "describe": 695, "concerning": 696, "sa": 697, "help": 698, "1700": 699, "225": 700, "1158": 701, "equipment": 702, "let's": 703, "wednesday's": 704, "quebec": 705, "highest": 706, "starting": 707, "taking": 708, "311": 709, "1230": 710, "able": 711, "put": 712, "later": 713, "takes": 714, "amount": 715, "qw": 716, "seven": 717, "maximum": 718, "yyz": 719, "it's": 720, "80": 721, "place": 722, "equal": 723, "while": 724, "train": 725, "look": 726, "815": 727, "takeoff": 728, "plan": 729, "2134": 730, "297": 731, "323": 732, "229": 733, "329": 734, "runs": 735, "730": 736, "closest": 737, "dulles": 738, "73s": 739, "so": 740, "economic": 741, "single": 742, "supper": 743, "110": 744, "calling": 745, "1205": 746, "55": 747, "michigan": 748, "proper": 749, "regarding": 750, "seats": 751, "19": 752, "m": 753, "midway": 754, "besides": 755, "reverse": 756, "1993": 757, "402": 758, "level": 759, "reaching": 760, "771": 761, "straight": 762, "located": 763, "305": 764, "repeating": 765, "indiana": 766, "connects": 767, "beginning": 768, "staying": 769, "town": 770, "cars": 771, "nonstops": 772, "300": 773, "345": 774, "dinnertime": 775, "sort": 776, "route": 777, "j31": 778, "tuesdays": 779, "212": 780, "705": 781, "red": 782, "eye": 783, "laying": 784, "friends": 785, "visit": 786, "here": 787, "them": 788, "lives": 789, "rent": 790, "279": 791, "137338": 792, "transcontinental": 793, "trans": 794, "world": 795, "1030": 796, "1130": 797, "come": 798, "727": 799, "1020": 800, "505": 801, "that's": 802, "163": 803, "ls": 804, "greatest": 805, "i've": 806, "got": 807, "somebody": 808, "else": 809, "wants": 810, "charges": 811, "734": 812, "carried": 813, "thirteenth": 814, "making": 815, "733": 816, "everywhere": 817, "prefer": 818, "run": 819, "non": 820, "315": 821, "746": 822, "companies": 823, "buy": 824, "very": 825, "270": 826, "locate": 827, "hartfield": 828, "start": 829, "98": 830, "inform": 831, "oh": 832, "82": 833, "139": 834, "1600": 835, "eleven": 836, "ord": 837, "mia": 838, "qualify": 839, "doesn't": 840, "mondays": 841, "catch": 842, "priced": 843, "bna": 844, "being": 845, "working": 846, "scenario": 847, "767": 848, "1940": 849, "150": 850, "100": 851, "afternoons": 852, "provides": 853, "723": 854, "1110": 855, "symbols": 856, "grounds": 857, "nw": 858, "539": 859, "soon": 860, "thereafter": 861, "scheduled": 862, "instead": 863, "810": 864, "lester": 865, "pearson": 866, "stapleton": 867, "615": 868, "twelve": 869, "bay": 870, "sounds": 871, "o'hare": 872, "ap68": 873, "fridays": 874, "try": 875, "fifteen": 876, "nights": 877, "determine": 878, "hold": 879, "lax": 880, "seat": 881, "k": 882, "planning": 883, "discount": 884, "summer": 885, "cover": 886, "271": 887, "tonight": 888, "off": 889, "124": 890, "thanks": 891, "longest": 892, "kindly": 893, "afterwards": 894, "overnight": 895, "1083": 896, "428": 897, "anything": 898, "1059": 899}}