{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "A100", "machine_shape": "hm" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "ee9c321418ce4322a0d6b28a3f2ca6a1": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e3e7f4f191ab43b581e38d2b79dc3e89", "IPY_MODEL_d4d0156eae3c41fe9f1f7ffd1cdd8c35", "IPY_MODEL_8cf941ff295e449589a2d9a454cfefba" ], "layout": "IPY_MODEL_a592ff42cd3346bca504eedba0f3955b" } }, "e3e7f4f191ab43b581e38d2b79dc3e89": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_417708e3b88b4764a97ad0d6ef78e38b", "placeholder": "​", "style": "IPY_MODEL_ef7b5cca37be4c0285aa863783edd83d", "value": "README.md: 100%" } }, "d4d0156eae3c41fe9f1f7ffd1cdd8c35": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_66dc16b4997f46439bed98ad8d0e8732", "max": 2872, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_cbd8f4782051447085d2c4c84b8185fb", "value": 2872 } }, "8cf941ff295e449589a2d9a454cfefba": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f20f1c3e78e84e46b9f7022c791775e4", "placeholder": "​", "style": "IPY_MODEL_d87fdc84c0eb4243804b5e6c86f7eb78", "value": " 2.87k/2.87k [00:00<00:00, 172kB/s]" } }, "a592ff42cd3346bca504eedba0f3955b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "417708e3b88b4764a97ad0d6ef78e38b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ef7b5cca37be4c0285aa863783edd83d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "66dc16b4997f46439bed98ad8d0e8732": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "cbd8f4782051447085d2c4c84b8185fb": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "f20f1c3e78e84e46b9f7022c791775e4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d87fdc84c0eb4243804b5e6c86f7eb78": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5ccea58adc994a8082d77a0fd3dd5175": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_2bfa1a551787407397a7d14faffaa471", "IPY_MODEL_f3bec46279cd4f99a7e7693aca5a463a", "IPY_MODEL_f275a7c407fa4b699039ed7bd9a7cdec" ], "layout": "IPY_MODEL_7e25a8d5567f474cb450095ed1b409fa" } }, "2bfa1a551787407397a7d14faffaa471": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f4c4efcc5f10400caac3695865596340", "placeholder": "​", "style": "IPY_MODEL_65c6ebc06606451bb347f7291f4af0cb", "value": "train-00000-of-00001.parquet: 100%" } }, "f3bec46279cd4f99a7e7693aca5a463a": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_751e56f58fef4cfa8dbc032790667c4f", "max": 13578023, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_ea9b7f2ae3cc4d819d1eedd09b498f4d", "value": 13578023 } }, "f275a7c407fa4b699039ed7bd9a7cdec": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e631c93fc21348528300a7a1013d39f5", "placeholder": "​", "style": "IPY_MODEL_e3caeb64cd5e4cb08265ebcb7040b340", "value": " 13.6M/13.6M [00:00<00:00, 35.5MB/s]" } }, "7e25a8d5567f474cb450095ed1b409fa": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f4c4efcc5f10400caac3695865596340": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "65c6ebc06606451bb347f7291f4af0cb": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "751e56f58fef4cfa8dbc032790667c4f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ea9b7f2ae3cc4d819d1eedd09b498f4d": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "e631c93fc21348528300a7a1013d39f5": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e3caeb64cd5e4cb08265ebcb7040b340": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "029202e5e41d404385ac4a3a36989700": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_9069aafbe84f47a6860ba99ff9de8fcd", "IPY_MODEL_41e04d59393245c898bae1bd80c824fe", "IPY_MODEL_acc44cac406a4f51ad13b82ad141f9ab" ], "layout": "IPY_MODEL_254f3a26f42e4ce193c7ea8283eb77af" } }, "9069aafbe84f47a6860ba99ff9de8fcd": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_50566b0a37d64188beea5ea55edf60bc", "placeholder": "​", "style": "IPY_MODEL_bbbd5baf90614ccaae15010e1982bd96", "value": "Generating train split: 100%" } }, "41e04d59393245c898bae1bd80c824fe": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6ddb7eaee2b7405e988582e5f6edc4d0", "max": 99545, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_5a1097b3c10846e29c9b838e9b3d41e7", "value": 99545 } }, "acc44cac406a4f51ad13b82ad141f9ab": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_8b7e0c70f2e349fbae1146ee73014ebc", "placeholder": "​", "style": "IPY_MODEL_ed9ff4cd49b646288e6d3d74efcec647", "value": " 99545/99545 [00:00<00:00, 354750.16 examples/s]" } }, "254f3a26f42e4ce193c7ea8283eb77af": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "50566b0a37d64188beea5ea55edf60bc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "bbbd5baf90614ccaae15010e1982bd96": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6ddb7eaee2b7405e988582e5f6edc4d0": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "5a1097b3c10846e29c9b838e9b3d41e7": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "8b7e0c70f2e349fbae1146ee73014ebc": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ed9ff4cd49b646288e6d3d74efcec647": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "880e3efa6bcc49cd98207c04c476e918": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_76ebb3ae1d354a139207ba45b0161206", "IPY_MODEL_8415ed49d34846efbb7d84341fe931c5", "IPY_MODEL_749f0a339075455f97b18ae42163457f" ], "layout": "IPY_MODEL_4f65674d3ae44bd7aa89873233d9c421" } }, "76ebb3ae1d354a139207ba45b0161206": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_c454e4d7594543ee971fd29f32c02e0b", "placeholder": "​", "style": "IPY_MODEL_f5a82328e6e842eeb772c171ccdb57ab", "value": "Map: 100%" } }, "8415ed49d34846efbb7d84341fe931c5": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f7cfa914ad6841559b1eb9b203474000", "max": 99545, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_6c09c2b4125d4eadad1a446095209d7f", "value": 99545 } }, "749f0a339075455f97b18ae42163457f": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b5d8b7c444214f2ab616b12eec614716", "placeholder": "​", "style": "IPY_MODEL_ea8e7d69c63f491ca615cddb77c0dde2", "value": " 99545/99545 [00:16<00:00, 7233.81 examples/s]" } }, "4f65674d3ae44bd7aa89873233d9c421": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c454e4d7594543ee971fd29f32c02e0b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "f5a82328e6e842eeb772c171ccdb57ab": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f7cfa914ad6841559b1eb9b203474000": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6c09c2b4125d4eadad1a446095209d7f": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "b5d8b7c444214f2ab616b12eec614716": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ea8e7d69c63f491ca615cddb77c0dde2": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "696d4ebbf97c44e9a6cdff707b87e953": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_b7abe11e0fb041d49137296e1d4bf43e", "IPY_MODEL_83d1e890914f4206b53e7c8fd3eafdec", "IPY_MODEL_97258ff7b0874640bcb47e1d1beb73ad" ], "layout": "IPY_MODEL_ba56149214a7449ab4023150af34380e" } }, "b7abe11e0fb041d49137296e1d4bf43e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e7b683ab266740d2b97981ffd12e0f47", "placeholder": "​", "style": "IPY_MODEL_34884b30a6e24ef2b7992e39fbd487df", "value": "tokenizer_config.json: 100%" } }, "83d1e890914f4206b53e7c8fd3eafdec": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6d8d6f3c10e54a99926a77759aed7980", "max": 373, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_2aaa0e920af74c91b1497d4e2f0d7c7f", "value": 373 } }, "97258ff7b0874640bcb47e1d1beb73ad": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_33b46ebefd7a4e19b20c9113a6306e33", "placeholder": "​", "style": "IPY_MODEL_46ffdf824b8e496bb4c739375d6a4255", "value": " 373/373 [00:00<00:00, 29.3kB/s]" } }, "ba56149214a7449ab4023150af34380e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "e7b683ab266740d2b97981ffd12e0f47": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "34884b30a6e24ef2b7992e39fbd487df": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6d8d6f3c10e54a99926a77759aed7980": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2aaa0e920af74c91b1497d4e2f0d7c7f": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "33b46ebefd7a4e19b20c9113a6306e33": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "46ffdf824b8e496bb4c739375d6a4255": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "bddefea747c1479f94c77a3dd33d9e24": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e1c7251d48a24f68a7990588c43bb431", "IPY_MODEL_4abaea6640c0425986e4ef171c6657cc", "IPY_MODEL_24c89f9d90be49ce8ebd052ab3377f6e" ], "layout": "IPY_MODEL_b933dfc3cb8e442d97be2ed145189923" } }, "e1c7251d48a24f68a7990588c43bb431": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3d17626e1daf485e913ba3d8e9904763", "placeholder": "​", "style": "IPY_MODEL_072e21b6c92e4c7fb79e3deb3ea26002", "value": "vocab.txt: 100%" } }, "4abaea6640c0425986e4ef171c6657cc": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f09e1e08239740299aa68312c3c41d22", "max": 251003, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_b1387e3381de4968938d371692b8e464", "value": 251003 } }, "24c89f9d90be49ce8ebd052ab3377f6e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_90ebf3029fd447ce9ca0dc82a120c7d5", "placeholder": "​", "style": "IPY_MODEL_ce0ccd2032d24fb8b94e4b3c047569d4", "value": " 251k/251k [00:00<00:00, 7.94MB/s]" } }, "b933dfc3cb8e442d97be2ed145189923": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3d17626e1daf485e913ba3d8e9904763": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "072e21b6c92e4c7fb79e3deb3ea26002": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f09e1e08239740299aa68312c3c41d22": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b1387e3381de4968938d371692b8e464": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "90ebf3029fd447ce9ca0dc82a120c7d5": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ce0ccd2032d24fb8b94e4b3c047569d4": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "1b2bd87619f045d5b0a7e346181f2e1e": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5c9ae0652af44b92be325b553edba0d7", "IPY_MODEL_629332724bc04a28b67848977545ebc9", "IPY_MODEL_9761b542588f48b6bee3f472534af6c4" ], "layout": "IPY_MODEL_5ef9e949110d45e984cb5e079c434743" } }, "5c9ae0652af44b92be325b553edba0d7": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0a84cbe234c44ee0a5ee0841760ab58c", "placeholder": "​", "style": "IPY_MODEL_6b378450c5444334847af9ea1710b5ec", "value": "tokenizer.json: 100%" } }, "629332724bc04a28b67848977545ebc9": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_549ca07ce27a41f1ba0565a7035e1295", "max": 497438, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_2e6cba86589a49449d63482bde3aef7d", "value": 497438 } }, "9761b542588f48b6bee3f472534af6c4": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4ad95f20bbad48a7b9386f83946cdd70", "placeholder": "​", "style": "IPY_MODEL_9e1ffe13e96b424abea09ab9b52b7f8a", "value": " 497k/497k [00:00<00:00, 16.5MB/s]" } }, "5ef9e949110d45e984cb5e079c434743": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0a84cbe234c44ee0a5ee0841760ab58c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "6b378450c5444334847af9ea1710b5ec": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "549ca07ce27a41f1ba0565a7035e1295": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "2e6cba86589a49449d63482bde3aef7d": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "4ad95f20bbad48a7b9386f83946cdd70": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9e1ffe13e96b424abea09ab9b52b7f8a": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6707a1f508b14ec38bbe5a79f8577806": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e9401712d1854627bb1ae2d7b3d9ecd2", "IPY_MODEL_9b1ebf0e78ad4b52b0401245d611a8bc", "IPY_MODEL_a3593c2e810943ffa55e63596d749733" ], "layout": "IPY_MODEL_ac343eb8c4c14a4c85c2b4f14ecfce7a" } }, "e9401712d1854627bb1ae2d7b3d9ecd2": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_543afb09b2384517888421a011a88586", "placeholder": "​", "style": "IPY_MODEL_ef6775804f044e5d8c654d42cf74bd08", "value": "special_tokens_map.json: 100%" } }, "9b1ebf0e78ad4b52b0401245d611a8bc": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7b43d651119e436c9d40f290f9eedf68", "max": 112, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_10dd21983d9c42b38374028b115e63e1", "value": 112 } }, "a3593c2e810943ffa55e63596d749733": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_4107503fdb674b1c8c60685fb5fbe1db", "placeholder": "​", "style": "IPY_MODEL_cb2ee95977c94e138024ca77bd05e794", "value": " 112/112 [00:00<00:00, 9.93kB/s]" } }, "ac343eb8c4c14a4c85c2b4f14ecfce7a": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "543afb09b2384517888421a011a88586": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ef6775804f044e5d8c654d42cf74bd08": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7b43d651119e436c9d40f290f9eedf68": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "10dd21983d9c42b38374028b115e63e1": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "4107503fdb674b1c8c60685fb5fbe1db": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "cb2ee95977c94e138024ca77bd05e794": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d0e50ec0c96741d48e76063bb183b8fa": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_8085b5ba4ec34e638ba014fbb37e21d4", "IPY_MODEL_a4a3807c55e8411cbc6c44c15180787b", "IPY_MODEL_9a9197e5815e472eadda980fce33b8cf" ], "layout": "IPY_MODEL_9cdb83209b154af39455ac4009bb5593" } }, "8085b5ba4ec34e638ba014fbb37e21d4": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d76e0ec95e354f8591f4dfc72f4039f9", "placeholder": "​", "style": "IPY_MODEL_7cf4e5265dbb4092bfd7b8b7aedbd25f", "value": "Map: 100%" } }, "a4a3807c55e8411cbc6c44c15180787b": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_bd3c0665a69c4b6aa723fdf30454d337", "max": 99545, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_9149f0d1909f4dac95d3696fb8d61529", "value": 99545 } }, "9a9197e5815e472eadda980fce33b8cf": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e2e9e694e8dc436aba563d05de7e786b", "placeholder": "​", "style": "IPY_MODEL_9e65b1fdbdbd4e91b1c32a6fa52c7561", "value": " 99545/99545 [00:53<00:00, 1716.94 examples/s]" } }, "9cdb83209b154af39455ac4009bb5593": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d76e0ec95e354f8591f4dfc72f4039f9": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7cf4e5265dbb4092bfd7b8b7aedbd25f": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "bd3c0665a69c4b6aa723fdf30454d337": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9149f0d1909f4dac95d3696fb8d61529": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "e2e9e694e8dc436aba563d05de7e786b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "9e65b1fdbdbd4e91b1c32a6fa52c7561": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "code", "source": [ "!pip install transformers datasets seqeval huggingface_hub\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "5v8KnAaD-z9t", "outputId": "ec89bbe3-e698-4e6f-eb27-ea15e3d2a549" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.44.2)\n", "Collecting datasets\n", " Downloading datasets-3.1.0-py3-none-any.whl.metadata (20 kB)\n", "Collecting seqeval\n", " Downloading seqeval-1.2.2.tar.gz (43 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.24.7)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.16.1)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.9.11)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.5)\n", "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.6)\n", "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (17.0.0)\n", "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (2.2.2)\n", "Collecting xxhash (from datasets)\n", " Downloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", "Collecting multiprocess<0.70.17 (from datasets)\n", " Downloading multiprocess-0.70.16-py310-none-any.whl.metadata (7.2 kB)\n", "Collecting fsspec<=2024.9.0,>=2023.1.0 (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets)\n", " Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.10.10)\n", "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.5.2)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.12.2)\n", "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (2.4.3)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (24.2.0)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.5.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.1.0)\n", "Requirement already satisfied: yarl<2.0,>=1.12.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.17.0)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4.0)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.2.3)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.8.30)\n", "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.13.1)\n", "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.4.2)\n", "Requirement already satisfied: threadpoolctl>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.5.0)\n", "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n", "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2024.2)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from yarl<2.0,>=1.12.0->aiohttp->datasets) (0.2.0)\n", "Downloading datasets-3.1.0-py3-none-any.whl (480 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hDownloading xxhash-3.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m16.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hBuilding wheels for collected packages: seqeval\n", " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16161 sha256=c1030f22e743c2a4b8d3cb548cb8e8f138f24b58417a9f47bbc72df908c59d18\n", " Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n", "Successfully built seqeval\n", "Installing collected packages: xxhash, fsspec, dill, multiprocess, seqeval, datasets\n", " Attempting uninstall: fsspec\n", " Found existing installation: fsspec 2024.10.0\n", " Uninstalling fsspec-2024.10.0:\n", " Successfully uninstalled fsspec-2024.10.0\n", "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\u001b[0m\u001b[31m\n", "\u001b[0mSuccessfully installed datasets-3.1.0 dill-0.3.8 fsspec-2024.9.0 multiprocess-0.70.16 seqeval-1.2.2 xxhash-3.5.0\n" ] } ] }, { "cell_type": "code", "source": [ "# Standard library imports\n", "import os # Provides functions for interacting with the operating system\n", "import warnings # Used to handle or suppress warnings\n", "import numpy as np # Essential for numerical operations and array manipulation\n", "import torch # PyTorch library for tensor computations and model handling\n", "import ast # Used for safe evaluation of strings to Python objects (e.g., parsing tokens)\n", "\n", "# Hugging Face and Transformers imports\n", "from datasets import load_dataset # Loads datasets for model training and evaluation\n", "from transformers import (\n", " AutoTokenizer, # Initializes a tokenizer from a pre-trained model\n", " DataCollatorForTokenClassification, # Handles padding and formatting of token classification data\n", " TrainingArguments, # Defines training parameters like batch size and learning rate\n", " Trainer, # High-level API for managing training and evaluation\n", " AutoModelForTokenClassification, # Loads a pre-trained model for token classification tasks\n", " get_linear_schedule_with_warmup, # Learning rate scheduler for gradual warm-up and linear decay\n", " EarlyStoppingCallback # Callback to stop training if validation performance plateaus\n", ")\n", "\n", "# Hugging Face Hub\n", "from huggingface_hub import login # Allows logging in to Hugging Face Hub to upload models\n", "\n", "# seqeval metrics for NER evaluation\n", "from seqeval.metrics import precision_score, recall_score, f1_score, classification_report\n", "# Provides precision, recall, F1-score, and classification report for evaluating NER model performance\n" ], "metadata": { "id": "amREIFSH-z7r" }, "execution_count": 3, "outputs": [] }, { "cell_type": "code", "source": [ "# Log in to Hugging Face Hub\n", "login(token=\"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\")\n" ], "metadata": { "id": "K7adlboI-z4p", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "71221522-e2cb-446f-91d9-d6f5b3a5ef08" }, "execution_count": 4, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\n", "Token is valid (permission: fineGrained).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ] }, { "cell_type": "code", "source": [ "# Disable WandB (Weights & Biases) logging to avoid unwanted log outputs during training\n", "os.environ[\"WANDB_DISABLED\"] = \"true\"\n", "\n", "# Suppress warning messages to keep output clean, especially during training and evaluation\n", "warnings.filterwarnings(\"ignore\")\n" ], "metadata": { "id": "Qccgsjfs-zzA" }, "execution_count": 5, "outputs": [] }, { "cell_type": "code", "source": [ "# Load the Azerbaijani NER dataset from Hugging Face\n", "dataset = load_dataset(\"LocalDoc/azerbaijani-ner-dataset\")\n", "print(dataset) # Display dataset structure (e.g., train/validation splits)\n", "\n", "# Preprocessing function to format tokens and NER tags correctly\n", "def preprocess_example(example):\n", " try:\n", " # Convert string of tokens to a list and parse NER tags to integers\n", " example[\"tokens\"] = ast.literal_eval(example[\"tokens\"])\n", " example[\"ner_tags\"] = list(map(int, ast.literal_eval(example[\"ner_tags\"])))\n", " except (ValueError, SyntaxError) as e:\n", " # Skip and log malformed examples, ensuring error resilience\n", " print(f\"Skipping malformed example: {example['index']} due to error: {e}\")\n", " example[\"tokens\"] = []\n", " example[\"ner_tags\"] = []\n", " return example\n", "\n", "# Apply preprocessing to each dataset entry, ensuring consistent formatting\n", "dataset = dataset.map(preprocess_example)\n" ], "metadata": { "id": "fQ6ttUM8-zwM", "colab": { "base_uri": "https://localhost:8080/", "height": 604, "referenced_widgets": [ "ee9c321418ce4322a0d6b28a3f2ca6a1", "e3e7f4f191ab43b581e38d2b79dc3e89", "d4d0156eae3c41fe9f1f7ffd1cdd8c35", "8cf941ff295e449589a2d9a454cfefba", "a592ff42cd3346bca504eedba0f3955b", "417708e3b88b4764a97ad0d6ef78e38b", "ef7b5cca37be4c0285aa863783edd83d", "66dc16b4997f46439bed98ad8d0e8732", "cbd8f4782051447085d2c4c84b8185fb", "f20f1c3e78e84e46b9f7022c791775e4", "d87fdc84c0eb4243804b5e6c86f7eb78", "5ccea58adc994a8082d77a0fd3dd5175", "2bfa1a551787407397a7d14faffaa471", "f3bec46279cd4f99a7e7693aca5a463a", "f275a7c407fa4b699039ed7bd9a7cdec", "7e25a8d5567f474cb450095ed1b409fa", "f4c4efcc5f10400caac3695865596340", "65c6ebc06606451bb347f7291f4af0cb", "751e56f58fef4cfa8dbc032790667c4f", "ea9b7f2ae3cc4d819d1eedd09b498f4d", "e631c93fc21348528300a7a1013d39f5", "e3caeb64cd5e4cb08265ebcb7040b340", "029202e5e41d404385ac4a3a36989700", "9069aafbe84f47a6860ba99ff9de8fcd", "41e04d59393245c898bae1bd80c824fe", "acc44cac406a4f51ad13b82ad141f9ab", "254f3a26f42e4ce193c7ea8283eb77af", "50566b0a37d64188beea5ea55edf60bc", "bbbd5baf90614ccaae15010e1982bd96", "6ddb7eaee2b7405e988582e5f6edc4d0", "5a1097b3c10846e29c9b838e9b3d41e7", "8b7e0c70f2e349fbae1146ee73014ebc", "ed9ff4cd49b646288e6d3d74efcec647", "880e3efa6bcc49cd98207c04c476e918", "76ebb3ae1d354a139207ba45b0161206", "8415ed49d34846efbb7d84341fe931c5", "749f0a339075455f97b18ae42163457f", "4f65674d3ae44bd7aa89873233d9c421", "c454e4d7594543ee971fd29f32c02e0b", "f5a82328e6e842eeb772c171ccdb57ab", "f7cfa914ad6841559b1eb9b203474000", "6c09c2b4125d4eadad1a446095209d7f", "b5d8b7c444214f2ab616b12eec614716", "ea8e7d69c63f491ca615cddb77c0dde2" ] }, "outputId": "130ce4ea-2cda-44d1-8514-bf5cab1be096" }, "execution_count": 6, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "README.md: 0%| | 0.00/2.87k [00:00" ], "text/html": [ "\n", "
\n", " \n", " \n", " [6300/7000 17:10 < 01:54, 6.11 it/s, Epoch 9/10]\n", "
\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
EpochTraining LossValidation LossPrecisionRecallF1
10.4331000.3067110.7390000.6932820.715412
20.2927000.2757960.7815650.6889370.732334
30.2506000.2751150.7582610.7094250.733031
40.2337000.2730870.7561840.7162770.735689
50.2148000.2784770.7560510.7109960.732832
60.1992000.2861020.7550680.7170120.735548
70.1928000.2971570.7423260.7258020.733971
80.1789000.3045100.7432060.7239300.733442
90.1717000.3138450.7431450.7255350.734234

" ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ " precision recall f1-score support\n", "\n", " ART 0.64 0.11 0.19 1988\n", " DATE 0.46 0.45 0.46 844\n", " EVENT 0.81 0.26 0.40 84\n", " FACILITY 0.68 0.68 0.68 1146\n", " LAW 0.58 0.54 0.56 1103\n", " LOCATION 0.73 0.78 0.75 8806\n", " MONEY 0.55 0.47 0.51 532\n", "ORGANISATION 0.62 0.61 0.62 527\n", " PERCENTAGE 0.78 0.80 0.79 3679\n", " PERSON 0.82 0.83 0.82 6924\n", " PRODUCT 0.80 0.78 0.79 2653\n", " TIME 0.58 0.35 0.44 1634\n", "\n", " micro avg 0.74 0.69 0.72 29920\n", " macro avg 0.67 0.56 0.58 29920\n", "weighted avg 0.73 0.69 0.70 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.55 0.13 0.21 1988\n", " DATE 0.57 0.40 0.47 844\n", " EVENT 0.88 0.33 0.48 84\n", " FACILITY 0.75 0.65 0.70 1146\n", " LAW 0.60 0.56 0.58 1103\n", " LOCATION 0.79 0.76 0.77 8806\n", " MONEY 0.60 0.54 0.57 532\n", "ORGANISATION 0.67 0.64 0.65 527\n", " PERCENTAGE 0.78 0.81 0.80 3679\n", " PERSON 0.87 0.81 0.84 6924\n", " PRODUCT 0.81 0.80 0.81 2653\n", " TIME 0.63 0.36 0.46 1634\n", "\n", " micro avg 0.78 0.69 0.73 29920\n", " macro avg 0.71 0.57 0.61 29920\n", "weighted avg 0.76 0.69 0.71 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.48 0.14 0.22 1988\n", " DATE 0.54 0.44 0.48 844\n", " EVENT 0.88 0.35 0.50 84\n", " FACILITY 0.72 0.68 0.70 1146\n", " LAW 0.60 0.59 0.60 1103\n", " LOCATION 0.75 0.79 0.77 8806\n", " MONEY 0.60 0.54 0.57 532\n", "ORGANISATION 0.63 0.67 0.65 527\n", " PERCENTAGE 0.77 0.83 0.80 3679\n", " PERSON 0.88 0.81 0.84 6924\n", " PRODUCT 0.82 0.81 0.81 2653\n", " TIME 0.57 0.44 0.50 1634\n", "\n", " micro avg 0.76 0.71 0.73 29920\n", " macro avg 0.69 0.59 0.62 29920\n", "weighted avg 0.74 0.71 0.72 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.49 0.14 0.21 1988\n", " DATE 0.49 0.48 0.49 844\n", " EVENT 0.88 0.36 0.51 84\n", " FACILITY 0.72 0.68 0.70 1146\n", " LAW 0.57 0.64 0.60 1103\n", " LOCATION 0.77 0.79 0.78 8806\n", " MONEY 0.62 0.57 0.59 532\n", "ORGANISATION 0.64 0.65 0.64 527\n", " PERCENTAGE 0.77 0.83 0.80 3679\n", " PERSON 0.87 0.81 0.84 6924\n", " PRODUCT 0.82 0.80 0.81 2653\n", " TIME 0.55 0.50 0.52 1634\n", "\n", " micro avg 0.76 0.72 0.74 29920\n", " macro avg 0.68 0.60 0.62 29920\n", "weighted avg 0.74 0.72 0.72 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.35 0.18 0.23 1988\n", " DATE 0.53 0.43 0.48 844\n", " EVENT 0.82 0.39 0.53 84\n", " FACILITY 0.71 0.67 0.69 1146\n", " LAW 0.61 0.59 0.60 1103\n", " LOCATION 0.78 0.78 0.78 8806\n", " MONEY 0.60 0.57 0.58 532\n", "ORGANISATION 0.61 0.67 0.64 527\n", " PERCENTAGE 0.78 0.81 0.80 3679\n", " PERSON 0.86 0.83 0.84 6924\n", " PRODUCT 0.84 0.77 0.81 2653\n", " TIME 0.57 0.48 0.52 1634\n", "\n", " micro avg 0.76 0.71 0.73 29920\n", " macro avg 0.67 0.60 0.63 29920\n", "weighted avg 0.74 0.71 0.72 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.36 0.16 0.22 1988\n", " DATE 0.50 0.48 0.49 844\n", " EVENT 0.82 0.38 0.52 84\n", " FACILITY 0.72 0.68 0.70 1146\n", " LAW 0.63 0.59 0.61 1103\n", " LOCATION 0.77 0.79 0.78 8806\n", " MONEY 0.59 0.60 0.60 532\n", "ORGANISATION 0.67 0.71 0.69 527\n", " PERCENTAGE 0.77 0.84 0.80 3679\n", " PERSON 0.87 0.81 0.84 6924\n", " PRODUCT 0.84 0.80 0.82 2653\n", " TIME 0.57 0.50 0.53 1634\n", "\n", " micro avg 0.76 0.72 0.74 29920\n", " macro avg 0.68 0.61 0.63 29920\n", "weighted avg 0.74 0.72 0.73 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.32 0.18 0.23 1988\n", " DATE 0.50 0.49 0.49 844\n", " EVENT 0.85 0.39 0.54 84\n", " FACILITY 0.73 0.68 0.70 1146\n", " LAW 0.60 0.62 0.61 1103\n", " LOCATION 0.77 0.79 0.78 8806\n", " MONEY 0.58 0.61 0.59 532\n", "ORGANISATION 0.64 0.69 0.66 527\n", " PERCENTAGE 0.78 0.83 0.80 3679\n", " PERSON 0.85 0.83 0.84 6924\n", " PRODUCT 0.81 0.80 0.81 2653\n", " TIME 0.56 0.52 0.54 1634\n", "\n", " micro avg 0.74 0.73 0.73 29920\n", " macro avg 0.66 0.62 0.63 29920\n", "weighted avg 0.73 0.73 0.73 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.29 0.20 0.24 1988\n", " DATE 0.51 0.46 0.49 844\n", " EVENT 0.85 0.40 0.55 84\n", " FACILITY 0.72 0.68 0.70 1146\n", " LAW 0.61 0.62 0.61 1103\n", " LOCATION 0.77 0.79 0.78 8806\n", " MONEY 0.61 0.60 0.60 532\n", "ORGANISATION 0.66 0.70 0.68 527\n", " PERCENTAGE 0.78 0.83 0.80 3679\n", " PERSON 0.85 0.83 0.84 6924\n", " PRODUCT 0.83 0.80 0.81 2653\n", " TIME 0.57 0.51 0.54 1634\n", "\n", " micro avg 0.74 0.72 0.73 29920\n", " macro avg 0.67 0.62 0.64 29920\n", "weighted avg 0.73 0.72 0.73 29920\n", "\n", " precision recall f1-score support\n", "\n", " ART 0.31 0.18 0.23 1988\n", " DATE 0.49 0.47 0.48 844\n", " EVENT 0.83 0.40 0.54 84\n", " FACILITY 0.72 0.68 0.70 1146\n", " LAW 0.61 0.63 0.62 1103\n", " LOCATION 0.77 0.79 0.78 8806\n", " MONEY 0.60 0.60 0.60 532\n", "ORGANISATION 0.66 0.70 0.68 527\n", " PERCENTAGE 0.78 0.82 0.80 3679\n", " PERSON 0.85 0.83 0.84 6924\n", " PRODUCT 0.81 0.81 0.81 2653\n", " TIME 0.55 0.53 0.54 1634\n", "\n", " micro avg 0.74 0.73 0.73 29920\n", " macro avg 0.67 0.62 0.64 29920\n", "weighted avg 0.73 0.73 0.73 29920\n", "\n" ] }, { "output_type": "display_data", "data": { "text/plain": [ "" ], "text/html": [ "\n", "

\n", " \n", " \n", " [78/78 00:05]\n", "
\n", " " ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ " precision recall f1-score support\n", "\n", " ART 0.49 0.14 0.21 1988\n", " DATE 0.49 0.48 0.49 844\n", " EVENT 0.88 0.36 0.51 84\n", " FACILITY 0.72 0.68 0.70 1146\n", " LAW 0.57 0.64 0.60 1103\n", " LOCATION 0.77 0.79 0.78 8806\n", " MONEY 0.62 0.57 0.59 532\n", "ORGANISATION 0.64 0.65 0.64 527\n", " PERCENTAGE 0.77 0.83 0.80 3679\n", " PERSON 0.87 0.81 0.84 6924\n", " PRODUCT 0.82 0.80 0.81 2653\n", " TIME 0.55 0.50 0.52 1634\n", "\n", " micro avg 0.76 0.72 0.74 29920\n", " macro avg 0.68 0.60 0.62 29920\n", "weighted avg 0.74 0.72 0.72 29920\n", "\n", "{'eval_loss': 0.27308720350265503, 'eval_precision': 0.7561836209025793, 'eval_recall': 0.7162767379679145, 'eval_f1': 0.7356893977103037, 'eval_runtime': 10.7521, 'eval_samples_per_second': 925.866, 'eval_steps_per_second': 7.254, 'epoch': 9.0}\n" ] } ] }, { "cell_type": "code", "source": [ "# Define the directory where the trained model and tokenizer will be saved\n", "save_directory = \"./Azeri-Turkish-BERT-NER\"\n", "\n", "# Save the trained model to the specified directory\n", "model.save_pretrained(save_directory)\n", "\n", "# Save the tokenizer to the same directory for compatibility with the model\n", "tokenizer.save_pretrained(save_directory)\n" ], "metadata": { "id": "7yEFe2_n-zPG", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "be0d5e0f-ff43-4be6-984e-fecbc8c0bdf4" }, "execution_count": 18, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "('./Azeri-Turkish-BERT-NER/tokenizer_config.json',\n", " './Azeri-Turkish-BERT-NER/special_tokens_map.json',\n", " './Azeri-Turkish-BERT-NER/vocab.txt',\n", " './Azeri-Turkish-BERT-NER/added_tokens.json',\n", " './Azeri-Turkish-BERT-NER/tokenizer.json')" ] }, "metadata": {}, "execution_count": 18 } ] }, { "cell_type": "code", "source": [ "from transformers import pipeline\n", "\n", "# Load tokenizer and model\n", "tokenizer = AutoTokenizer.from_pretrained(save_directory)\n", "model = AutoModelForTokenClassification.from_pretrained(save_directory)\n", "\n", "# Initialize the NER pipeline\n", "device = 0 if torch.cuda.is_available() else -1\n", "nlp_ner = pipeline(\"ner\", model=model, tokenizer=tokenizer, aggregation_strategy=\"simple\", device=device)\n" ], "metadata": { "id": "zkECg3v9-zNQ" }, "execution_count": 19, "outputs": [] }, { "cell_type": "code", "source": [ "label_mapping = {f\"LABEL_{i}\": label for i, label in enumerate(label_list) if label != \"O\"}\n", "\n", "def evaluate_model(test_texts, true_labels):\n", " predictions = []\n", " for i, text in enumerate(test_texts):\n", " pred_entities = nlp_ner(text)\n", " pred_labels = [label_mapping.get(entity[\"entity_group\"], \"O\") for entity in pred_entities if entity[\"entity_group\"] in label_mapping]\n", " if len(pred_labels) != len(true_labels[i]):\n", " print(f\"Warning: Inconsistent number of entities in sample {i+1}. Adjusting predicted entities.\")\n", " pred_labels = pred_labels[:len(true_labels[i])]\n", " predictions.append(pred_labels)\n", " if all(len(true) == len(pred) for true, pred in zip(true_labels, predictions)):\n", " precision = precision_score(true_labels, predictions)\n", " recall = recall_score(true_labels, predictions)\n", " f1 = f1_score(true_labels, predictions)\n", " print(\"Precision:\", precision)\n", " print(\"Recall:\", recall)\n", " print(\"F1-Score:\", f1)\n", " print(classification_report(true_labels, predictions))\n", " else:\n", " print(\"Error: Could not align all samples correctly for evaluation.\")\n" ], "metadata": { "id": "SOFqXU-M_bxO" }, "execution_count": 20, "outputs": [] }, { "cell_type": "code", "source": [ "test_texts = [\"Shahla Khuduyeva və Pasha Sığorta şirkəti haqqında məlumat.\"]\n", "true_labels = [[\"B-PERSON\", \"B-ORGANISATION\"]]\n", "evaluate_model(test_texts, true_labels)\n" ], "metadata": { "id": "WRCB-_66_buE", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "b80e4507-f1c0-4dc3-f252-83fca43c6a11" }, "execution_count": 21, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Warning: Inconsistent number of entities in sample 1. Adjusting predicted entities.\n", "Precision: 0.5\n", "Recall: 0.5\n", "F1-Score: 0.5\n", " precision recall f1-score support\n", "\n", "ORGANISATION 0.00 0.00 0.00 1\n", " PERSON 0.50 1.00 0.67 1\n", "\n", " micro avg 0.50 0.50 0.50 2\n", " macro avg 0.25 0.50 0.33 2\n", "weighted avg 0.25 0.50 0.33 2\n", "\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "x53zS3Vv_brU" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "5Uoebirj_boo" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "RKounG2l_bl5" }, "execution_count": null, "outputs": [] } ] }