| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "source": [ | |
| "# DAEDRA: Determining Adverse Event Disposition for Regulatory Affairs\n", | |
| "\n", | |
| "DAEDRA is a language model intended to predict the disposition (outcome) of an adverse event based on the text of the event report. Intended to be used to classify reports in passive reporting systems, it is trained on the [VAERS](https://vaers.hhs.gov/) dataset, which contains reports of adverse events following vaccination in the United States." | |
| ], | |
| "metadata": {} | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "%pip install accelerate -U" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": "Requirement already satisfied: accelerate in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.26.1)\nRequirement already satisfied: numpy>=1.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (1.23.5)\nRequirement already satisfied: huggingface-hub in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (0.20.3)\nRequirement already satisfied: pyyaml in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (6.0)\nRequirement already satisfied: psutil in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (5.9.5)\nRequirement already satisfied: torch>=1.10.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (1.12.0)\nRequirement already satisfied: packaging>=20.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (23.1)\nRequirement already satisfied: safetensors>=0.3.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from accelerate) (0.4.2)\nRequirement already satisfied: typing_extensions in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from torch>=1.10.0->accelerate) (4.6.3)\nRequirement already satisfied: requests in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (2.31.0)\nRequirement already satisfied: tqdm>=4.42.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (4.65.0)\nRequirement already satisfied: fsspec>=2023.5.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (2023.10.0)\nRequirement already satisfied: filelock in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from huggingface-hub->accelerate) (3.13.1)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (2023.5.7)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (1.26.16)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (3.4)\nRequirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->huggingface-hub->accelerate) (3.1.0)\nNote: you may need to restart the kernel to use updated packages.\n" | |
| } | |
| ], | |
| "execution_count": 1, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706475754655 | |
| }, | |
| "nteract": { | |
| "transient": { | |
| "deleting": false | |
| } | |
| }, | |
| "tags": [] | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "%pip install transformers datasets shap watermark wandb evaluate codecarbon" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": "Requirement already satisfied: transformers in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (4.37.1)\nRequirement already satisfied: datasets in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.16.1)\nRequirement already satisfied: shap in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.44.1)\nRequirement already satisfied: watermark in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.4.3)\nRequirement already satisfied: wandb in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.16.2)\nRequirement already satisfied: evaluate in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (0.4.1)\nRequirement already satisfied: codecarbon in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (2.3.3)\nRequirement already satisfied: regex!=2019.12.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (2023.12.25)\nRequirement already satisfied: packaging>=20.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (23.1)\nRequirement already satisfied: safetensors>=0.3.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.4.2)\nRequirement already satisfied: pyyaml>=5.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (6.0)\nRequirement already satisfied: numpy>=1.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (1.23.5)\nRequirement already satisfied: filelock in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (3.13.1)\nRequirement already satisfied: requests in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (2.31.0)\nRequirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.20.3)\nRequirement already satisfied: tqdm>=4.27 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (4.65.0)\nRequirement already satisfied: tokenizers<0.19,>=0.14 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from transformers) (0.15.1)\nRequirement already satisfied: pyarrow>=8.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (9.0.0)\nRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2023.10.0)\nRequirement already satisfied: pyarrow-hotfix in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.6)\nRequirement already satisfied: dill<0.3.8,>=0.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.3.7)\nRequirement already satisfied: aiohttp in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.9.1)\nRequirement already satisfied: multiprocess in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (0.70.15)\nRequirement already satisfied: xxhash in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (3.4.1)\nRequirement already satisfied: pandas in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from datasets) (2.0.2)\nRequirement already satisfied: scipy in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (1.10.1)\nRequirement already satisfied: slicer==0.0.7 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (0.0.7)\nRequirement already satisfied: cloudpickle in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (2.2.1)\nRequirement already satisfied: scikit-learn in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (1.2.2)\nRequirement already satisfied: numba in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from shap) (0.58.1)\nRequirement already satisfied: setuptools in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from watermark) (65.6.3)\nRequirement already satisfied: ipython>=6.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from watermark) (8.12.2)\nRequirement already satisfied: importlib-metadata>=1.4 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from watermark) (6.7.0)\nRequirement already satisfied: Click!=8.0.0,>=7.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (8.1.3)\nRequirement already satisfied: typing-extensions in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (4.6.3)\nRequirement already satisfied: protobuf!=4.21.0,<5,>=3.12.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (3.19.6)\nRequirement already satisfied: setproctitle in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (1.3.3)\nRequirement already satisfied: psutil>=5.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (5.9.5)\nRequirement already satisfied: sentry-sdk>=1.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (1.39.2)\nRequirement already satisfied: docker-pycreds>=0.4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (0.4.0)\nRequirement already satisfied: appdirs>=1.4.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (1.4.4)\nRequirement already satisfied: GitPython!=3.1.29,>=1.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from wandb) (3.1.31)\nRequirement already satisfied: responses<0.19 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from evaluate) (0.18.0)\nRequirement already satisfied: prometheus-client in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (0.19.0)\nRequirement already satisfied: py-cpuinfo in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (9.0.0)\nRequirement already satisfied: rapidfuzz in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (3.6.1)\nRequirement already satisfied: pynvml in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (11.5.0)\nRequirement already satisfied: arrow in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from codecarbon) (1.3.0)\nRequirement already satisfied: six>=1.4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from docker-pycreds>=0.4.0->wandb) (1.16.0)\nRequirement already satisfied: yarl<2.0,>=1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.9.4)\nRequirement already satisfied: async-timeout<5.0,>=4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (4.0.3)\nRequirement already satisfied: frozenlist>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.4.1)\nRequirement already satisfied: multidict<7.0,>=4.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (6.0.4)\nRequirement already satisfied: aiosignal>=1.1.2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (1.3.1)\nRequirement already satisfied: attrs>=17.3.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from aiohttp->datasets) (23.1.0)\nRequirement already satisfied: gitdb<5,>=4.0.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from GitPython!=3.1.29,>=1.0.0->wandb) (4.0.10)\nRequirement already satisfied: zipp>=0.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from importlib-metadata>=1.4->watermark) (3.15.0)\nRequirement already satisfied: pickleshare in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.7.5)\nRequirement already satisfied: backcall in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.2.0)\nRequirement already satisfied: stack-data in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.6.2)\nRequirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (3.0.30)\nRequirement already satisfied: matplotlib-inline in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.1.6)\nRequirement already satisfied: decorator in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (5.1.1)\nRequirement already satisfied: traitlets>=5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (5.9.0)\nRequirement already satisfied: jedi>=0.16 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (0.18.2)\nRequirement already satisfied: pygments>=2.4.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (2.15.1)\nRequirement already satisfied: pexpect>4.3 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from ipython>=6.0->watermark) (4.8.0)\nRequirement already satisfied: urllib3<3,>=1.21.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (1.26.16)\nRequirement already satisfied: charset-normalizer<4,>=2 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (3.1.0)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (3.4)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from requests->transformers) (2023.5.7)\nRequirement already satisfied: types-python-dateutil>=2.8.10 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from arrow->codecarbon) (2.8.19.20240106)\nRequirement already satisfied: python-dateutil>=2.7.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from arrow->codecarbon) (2.8.2)\nRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from numba->shap) (0.41.1)\nRequirement already satisfied: pytz>=2020.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2023.3)\nRequirement already satisfied: tzdata>=2022.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pandas->datasets) (2023.3)\nRequirement already satisfied: threadpoolctl>=2.0.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn->shap) (3.1.0)\nRequirement already satisfied: joblib>=1.1.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from scikit-learn->shap) (1.2.0)\nRequirement already satisfied: smmap<6,>=3.0.1 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from gitdb<5,>=4.0.1->GitPython!=3.1.29,>=1.0.0->wandb) (5.0.0)\nRequirement already satisfied: parso<0.9.0,>=0.8.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from jedi>=0.16->ipython>=6.0->watermark) (0.8.3)\nRequirement already satisfied: ptyprocess>=0.5 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from pexpect>4.3->ipython>=6.0->watermark) (0.7.0)\nRequirement already satisfied: wcwidth in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.0->watermark) (0.2.6)\nRequirement already satisfied: asttokens>=2.1.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from stack-data->ipython>=6.0->watermark) (2.2.1)\nRequirement already satisfied: pure-eval in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from stack-data->ipython>=6.0->watermark) (0.2.2)\nRequirement already satisfied: executing>=1.2.0 in /anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages (from stack-data->ipython>=6.0->watermark) (1.2.0)\nNote: you may need to restart the kernel to use updated packages.\n" | |
| } | |
| ], | |
| "execution_count": 2, | |
| "metadata": { | |
| "nteract": { | |
| "transient": { | |
| "deleting": false | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "import pandas as pd\n", | |
| "import numpy as np\n", | |
| "import torch\n", | |
| "import os\n", | |
| "from typing import List, Union\n", | |
| "from transformers import AutoTokenizer, Trainer, AutoModelForSequenceClassification, TrainingArguments, DataCollatorWithPadding, pipeline, AutoModel\n", | |
| "from datasets import load_dataset, Dataset, DatasetDict\n", | |
| "import shap\n", | |
| "import wandb\n", | |
| "import evaluate\n", | |
| "import logging\n", | |
| "\n", | |
| "wandb.finish()\n", | |
| "\n", | |
| "\n", | |
| "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n", | |
| "\n", | |
| "%load_ext watermark" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stderr", | |
| "text": "/anaconda/envs/azureml_py38_PT_TF/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n from .autonotebook import tqdm as notebook_tqdm\n2024-01-29 17:46:15.020290: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA\nTo enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n2024-01-29 17:46:16.031641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n2024-01-29 17:46:16.031779: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n2024-01-29 17:46:16.031793: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n" | |
| } | |
| ], | |
| "execution_count": 3, | |
| "metadata": { | |
| "datalore": { | |
| "hide_input_from_viewers": false, | |
| "hide_output_from_viewers": false, | |
| "node_id": "caZjjFP0OyQNMVgZDiwswE", | |
| "report_properties": { | |
| "rowId": "un8W7ez7ZwoGb5Co6nydEV" | |
| }, | |
| "type": "CODE" | |
| }, | |
| "gather": { | |
| "logged": 1706550378660 | |
| }, | |
| "tags": [] | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "device: str = 'cuda' if torch.cuda.is_available() else 'cpu'\n", | |
| "\n", | |
| "SEED: int = 42\n", | |
| "\n", | |
| "BATCH_SIZE: int = 32\n", | |
| "EPOCHS: int = 5\n", | |
| "model_ckpt: str = \"distilbert-base-uncased\"\n", | |
| "\n", | |
| "# WandB configuration\n", | |
| "os.environ[\"WANDB_PROJECT\"] = \"DAEDRA multiclass model training\" \n", | |
| "os.environ[\"WANDB_LOG_MODEL\"] = \"checkpoint\" # log all model checkpoints\n", | |
| "os.environ[\"WANDB_NOTEBOOK_NAME\"] = \"DAEDRA.ipynb\"" | |
| ], | |
| "outputs": [], | |
| "execution_count": 4, | |
| "metadata": { | |
| "collapsed": false, | |
| "gather": { | |
| "logged": 1706550378812 | |
| }, | |
| "jupyter": { | |
| "outputs_hidden": false | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "%watermark --iversion" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": "shap : 0.44.1\npandas : 2.0.2\nwandb : 0.16.2\nre : 2.2.1\nevaluate: 0.4.1\ntorch : 1.12.0\nnumpy : 1.23.5\nlogging : 0.5.1.2\n\n" | |
| } | |
| ], | |
| "execution_count": 5, | |
| "metadata": { | |
| "collapsed": false, | |
| "jupyter": { | |
| "outputs_hidden": false | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "!nvidia-smi" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": "Mon Jan 29 17:46:18 2024 \r\n+---------------------------------------------------------------------------------------+\r\n| NVIDIA-SMI 535.129.03 Driver Version: 535.129.03 CUDA Version: 12.2 |\r\n|-----------------------------------------+----------------------+----------------------+\r\n| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\r\n| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\r\n| | | MIG M. |\r\n|=========================================+======================+======================|\r\n| 0 Tesla V100-PCIE-16GB Off | 00000001:00:00.0 Off | Off |\r\n| N/A 26C P0 25W / 250W | 4MiB / 16384MiB | 0% Default |\r\n| | | N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n| 1 Tesla V100-PCIE-16GB Off | 00000002:00:00.0 Off | Off |\r\n| N/A 25C P0 23W / 250W | 4MiB / 16384MiB | 0% Default |\r\n| | | N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n| 2 Tesla V100-PCIE-16GB Off | 00000003:00:00.0 Off | Off |\r\n| N/A 25C P0 25W / 250W | 4MiB / 16384MiB | 0% Default |\r\n| | | N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n| 3 Tesla V100-PCIE-16GB Off | 00000004:00:00.0 Off | Off |\r\n| N/A 27C P0 24W / 250W | 4MiB / 16384MiB | 0% Default |\r\n| | | N/A |\r\n+-----------------------------------------+----------------------+----------------------+\r\n \r\n+---------------------------------------------------------------------------------------+\r\n| Processes: |\r\n| GPU GI CI PID Type Process name GPU Memory |\r\n| ID ID Usage |\r\n|=======================================================================================|\r\n| No running processes found |\r\n+---------------------------------------------------------------------------------------+\r\n" | |
| } | |
| ], | |
| "execution_count": 6, | |
| "metadata": { | |
| "datalore": { | |
| "hide_input_from_viewers": true, | |
| "hide_output_from_viewers": true, | |
| "node_id": "UU2oOJhwbIualogG1YyCMd", | |
| "type": "CODE" | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "source": [ | |
| "## Loading the data set" | |
| ], | |
| "metadata": { | |
| "datalore": { | |
| "hide_input_from_viewers": false, | |
| "hide_output_from_viewers": false, | |
| "node_id": "t45KHugmcPVaO0nuk8tGJ9", | |
| "report_properties": { | |
| "rowId": "40nN9Hvgi1clHNV5RAemI5" | |
| }, | |
| "type": "MD" | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "dataset = load_dataset(\"chrisvoncsefalvay/vaers-outcomes\")" | |
| ], | |
| "outputs": [], | |
| "execution_count": 7, | |
| "metadata": { | |
| "collapsed": false, | |
| "gather": { | |
| "logged": 1706550381141 | |
| }, | |
| "jupyter": { | |
| "outputs_hidden": false | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "dataset" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "execute_result", | |
| "execution_count": 8, | |
| "data": { | |
| "text/plain": "DatasetDict({\n train: Dataset({\n features: ['id', 'text', 'label'],\n num_rows: 1270444\n })\n test: Dataset({\n features: ['id', 'text', 'label'],\n num_rows: 272238\n })\n val: Dataset({\n features: ['id', 'text', 'label'],\n num_rows: 272238\n })\n})" | |
| }, | |
| "metadata": {} | |
| } | |
| ], | |
| "execution_count": 8, | |
| "metadata": { | |
| "collapsed": false, | |
| "gather": { | |
| "logged": 1706550381303 | |
| }, | |
| "jupyter": { | |
| "outputs_hidden": false, | |
| "source_hidden": false | |
| }, | |
| "nteract": { | |
| "transient": { | |
| "deleting": false | |
| } | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "SUBSAMPLING = 0.01\n", | |
| "\n", | |
| "if SUBSAMPLING < 1:\n", | |
| " _ = DatasetDict()\n", | |
| " for each in dataset.keys():\n", | |
| " _[each] = dataset[each].shuffle(seed=SEED).select(range(int(len(dataset[each]) * SUBSAMPLING)))\n", | |
| "\n", | |
| " dataset = _" | |
| ], | |
| "outputs": [], | |
| "execution_count": 9, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706550381472 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "source": [ | |
| "## Tokenisation and encoding" | |
| ], | |
| "metadata": {} | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "def encode_ds(ds: Union[Dataset, DatasetDict], tokenizer_model: str = model_ckpt) -> Union[Dataset, DatasetDict]:\n", | |
| " return ds_enc" | |
| ], | |
| "outputs": [], | |
| "execution_count": 10, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706550381637 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "source": [ | |
| "## Evaluation metrics" | |
| ], | |
| "metadata": {} | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "accuracy = evaluate.load(\"accuracy\")\n", | |
| "precision, recall = evaluate.load(\"precision\"), evaluate.load(\"recall\")\n", | |
| "f1 = evaluate.load(\"f1\")" | |
| ], | |
| "outputs": [], | |
| "execution_count": 11, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706550381778 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "def compute_metrics(eval_pred):\n", | |
| " predictions, labels = eval_pred\n", | |
| " predictions = np.argmax(predictions, axis=1)\n", | |
| " return {\n", | |
| " 'accuracy': accuracy.compute(predictions=predictions, references=labels)[\"accuracy\"],\n", | |
| " 'precision_macroaverage': precision.compute(predictions=predictions, references=labels, average='macro')[\"precision\"],\n", | |
| " 'precision_microaverage': precision.compute(predictions=predictions, references=labels, average='micro')[\"precision\"],\n", | |
| " 'recall_macroaverage': recall.compute(predictions=predictions, references=labels, average='macro')[\"recall\"],\n", | |
| " 'recall_microaverage': recall.compute(predictions=predictions, references=labels, average='micro')[\"recall\"],\n", | |
| " 'f1_microaverage': f1.compute(predictions=predictions, references=labels, average='micro')[\"f1\"]\n", | |
| " }" | |
| ], | |
| "outputs": [], | |
| "execution_count": 12, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706550381891 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "source": [ | |
| "## Training" | |
| ], | |
| "metadata": {} | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "source": [ | |
| "We specify a label map – this has to be done manually, even if `Datasets` has a function for it, as `AutoModelForSequenceClassification` requires an object with a length :(" | |
| ], | |
| "metadata": {} | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "label_map = {i: label for i, label in enumerate(dataset[\"test\"].features[\"label\"].names)}" | |
| ], | |
| "outputs": [], | |
| "execution_count": 13, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706550382032 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "def train_from_model(model_ckpt: str, push: bool = False):\n", | |
| " print(f\"Initialising training based on {model_ckpt}...\")\n", | |
| "\n", | |
| " print(\"Tokenising...\")\n", | |
| " tokenizer = AutoTokenizer.from_pretrained(model_ckpt)\n", | |
| "\n", | |
| " cols = dataset[\"train\"].column_names\n", | |
| " cols.remove(\"label\")\n", | |
| " ds_enc = dataset.map(lambda x: tokenizer(x[\"text\"], truncation=True, max_length=512), batched=True, remove_columns=cols)\n", | |
| "\n", | |
| " print(\"Loading model...\")\n", | |
| " try:\n", | |
| " model = AutoModelForSequenceClassification.from_pretrained(model_ckpt, \n", | |
| " num_labels=len(dataset[\"test\"].features[\"label\"].names), \n", | |
| " id2label=label_map, \n", | |
| " label2id={v:k for k,v in label_map.items()})\n", | |
| " except OSError:\n", | |
| " model = AutoModelForSequenceClassification.from_pretrained(model_ckpt, \n", | |
| " num_labels=len(dataset[\"test\"].features[\"label\"].names), \n", | |
| " id2label=label_map, \n", | |
| " label2id={v:k for k,v in label_map.items()},\n", | |
| " from_tf=True)\n", | |
| "\n", | |
| "\n", | |
| " args = TrainingArguments(\n", | |
| " output_dir=\"vaers\",\n", | |
| " evaluation_strategy=\"epoch\",\n", | |
| " save_strategy=\"epoch\",\n", | |
| " learning_rate=2e-5,\n", | |
| " per_device_train_batch_size=BATCH_SIZE,\n", | |
| " per_device_eval_batch_size=BATCH_SIZE,\n", | |
| " num_train_epochs=EPOCHS,\n", | |
| " weight_decay=.01,\n", | |
| " logging_steps=1,\n", | |
| " load_best_model_at_end=True,\n", | |
| " run_name=f\"daedra-training\",\n", | |
| " report_to=[\"wandb\"])\n", | |
| "\n", | |
| " trainer = Trainer(\n", | |
| " model=model,\n", | |
| " args=args,\n", | |
| " train_dataset=ds_enc[\"train\"],\n", | |
| " eval_dataset=ds_enc[\"test\"],\n", | |
| " tokenizer=tokenizer,\n", | |
| " compute_metrics=compute_metrics)\n", | |
| " \n", | |
| " if SUBSAMPLING != 1.0:\n", | |
| " wandb_tag: List[str] = [f\"subsample-{SUBSAMPLING}\"]\n", | |
| " else:\n", | |
| " wandb_tag: List[str] = [f\"full_sample\"]\n", | |
| "\n", | |
| " wandb_tag.append(f\"batch_size-{BATCH_SIZE}\")\n", | |
| " wandb_tag.append(f\"base:{model_ckpt}\")\n", | |
| " \n", | |
| " wandb.init(name=f\"daedra_{SUBSAMPLING}-{model_ckpt}\", tags=wandb_tag, magic=True)\n", | |
| "\n", | |
| " print(\"Starting training...\")\n", | |
| "\n", | |
| " trainer.train()\n", | |
| "\n", | |
| " print(\"Training finished.\")\n", | |
| "\n", | |
| " if push:\n", | |
| " variant = \"full_sample\" if SUBSAMPLING == 1.0 else f\"subsample-{SUBSAMPLING}\"\n", | |
| " tokenizer._tokenizer.save(\"tokenizer.json\")\n", | |
| " tokenizer.push_to_hub(\"chrisvoncsefalvay/daedra\")\n", | |
| " sample = \"full sample\" if SUBSAMPLING == 1.0 else f\"{SUBSAMPLING * 100}% of the full sample\"\n", | |
| "\n", | |
| " model.push_to_hub(\"chrisvoncsefalvay/daedra\", \n", | |
| " variant=variant,\n", | |
| " commit_message=f\"DAEDRA model trained on {sample} of the VAERS dataset (training set size: {dataset['train'].num_rows:,}), based on {model_ckpt}\")" | |
| ], | |
| "outputs": [], | |
| "execution_count": 14, | |
| "metadata": { | |
| "jupyter": { | |
| "outputs_hidden": false, | |
| "source_hidden": false | |
| }, | |
| "nteract": { | |
| "transient": { | |
| "deleting": false | |
| } | |
| }, | |
| "gather": { | |
| "logged": 1706550382160 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "\n", | |
| "base_models = [\n", | |
| " \"bert-base-uncased\",\n", | |
| " \"distilbert-base-uncased\",\n", | |
| "]" | |
| ], | |
| "outputs": [], | |
| "execution_count": 15, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706550382318 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [ | |
| "BATCH_SIZE=1\n", | |
| "\n", | |
| "train_from_model(\"biobert/Bio_ClinicalBERT/\")" | |
| ], | |
| "outputs": [ | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": "Initialising training based on biobert/Bio_ClinicalBERT/...\nTokenising...\nLoading model...\n" | |
| }, | |
| { | |
| "output_type": "stream", | |
| "name": "stderr", | |
| "text": "Map: 100%|██████████| 2722/2722 [00:01<00:00, 2195.12 examples/s]\nAll TF 2.0 model weights were used when initializing BertForSequenceClassification.\n\nAll the weights of BertForSequenceClassification were initialized from the TF 2.0 model.\nIf your task is similar to the task the model of the checkpoint was trained on, you can already use BertForSequenceClassification for predictions without further training.\n" | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "Finishing last run (ID:sg022tqh) before initializing another..." | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": " View run <strong style=\"color:#cdcd00\">daedra_0.01-biobert/Bio_ClinicalBERT/</strong> at: <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/sg022tqh' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/sg022tqh</a><br/> View job at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEzNDcyMTQwMw==/version_details/v6' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjEzNDcyMTQwMw==/version_details/v6</a><br/>Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "Find logs at: <code>./wandb/run-20240129_174816-sg022tqh/logs</code>" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "Successfully finished last run (ID:sg022tqh). Initializing new run:<br/>" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "Tracking run with wandb version 0.16.2" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "Run data is saved locally in <code>/mnt/batch/tasks/shared/LS_root/mounts/clusters/daedra-hptrain-cvc/code/Users/kristof.csefalvay/daedra/notebooks/wandb/run-20240129_174936-kilkkg1j</code>" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "Syncing run <strong><a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kilkkg1j' target=\"_blank\">daedra_0.01-biobert/Bio_ClinicalBERT/</a></strong> to <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": " View project at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training</a>" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": " View run at <a href='https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kilkkg1j' target=\"_blank\">https://wandb.ai/chrisvoncsefalvay/DAEDRA%20multiclass%20model%20training/runs/kilkkg1j</a>" | |
| }, | |
| "metadata": {} | |
| }, | |
| { | |
| "output_type": "stream", | |
| "name": "stdout", | |
| "text": "Starting training...\n" | |
| }, | |
| { | |
| "output_type": "stream", | |
| "name": "stderr", | |
| "text": "Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n" | |
| }, | |
| { | |
| "output_type": "display_data", | |
| "data": { | |
| "text/plain": "<IPython.core.display.HTML object>", | |
| "text/html": "\n <div>\n \n <progress value='1496' max='15880' style='width:300px; height:20px; vertical-align: middle;'></progress>\n [ 1496/15880 07:43 < 1:14:19, 3.23 it/s, Epoch 0.47/5]\n </div>\n <table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: left;\">\n <th>Epoch</th>\n <th>Training Loss</th>\n <th>Validation Loss</th>\n </tr>\n </thead>\n <tbody>\n </tbody>\n</table><p>" | |
| }, | |
| "metadata": {} | |
| } | |
| ], | |
| "execution_count": 21, | |
| "metadata": { | |
| "gather": { | |
| "logged": 1706551053473 | |
| } | |
| } | |
| }, | |
| { | |
| "cell_type": "code", | |
| "source": [], | |
| "outputs": [], | |
| "execution_count": null, | |
| "metadata": { | |
| "jupyter": { | |
| "source_hidden": false, | |
| "outputs_hidden": false | |
| }, | |
| "nteract": { | |
| "transient": { | |
| "deleting": false | |
| } | |
| } | |
| } | |
| } | |
| ], | |
| "metadata": { | |
| "datalore": { | |
| "base_environment": "default", | |
| "computation_mode": "JUPYTER", | |
| "package_manager": "pip", | |
| "packages": [ | |
| { | |
| "name": "datasets", | |
| "source": "PIP", | |
| "version": "2.16.1" | |
| }, | |
| { | |
| "name": "torch", | |
| "source": "PIP", | |
| "version": "2.1.2" | |
| }, | |
| { | |
| "name": "accelerate", | |
| "source": "PIP", | |
| "version": "0.26.1" | |
| } | |
| ], | |
| "report_row_ids": [ | |
| "un8W7ez7ZwoGb5Co6nydEV", | |
| "40nN9Hvgi1clHNV5RAemI5", | |
| "TgRD90H5NSPpKS41OeXI1w", | |
| "ZOm5BfUs3h1EGLaUkBGeEB", | |
| "kOP0CZWNSk6vqE3wkPp7Vc", | |
| "W4PWcOu2O2pRaZyoE2W80h", | |
| "RolbOnQLIftk0vy9mIcz5M", | |
| "8OPhUgbaNJmOdiq5D3a6vK", | |
| "5Qrt3jSvSrpK6Ne1hS6shL", | |
| "hTq7nFUrovN5Ao4u6dIYWZ", | |
| "I8WNZLpJ1DVP2wiCW7YBIB", | |
| "SawhU3I9BewSE1XBPstpNJ", | |
| "80EtLEl2FIE4FqbWnUD3nT" | |
| ], | |
| "version": 3 | |
| }, | |
| "kernel_info": { | |
| "name": "python38-azureml-pt-tf" | |
| }, | |
| "kernelspec": { | |
| "display_name": "azureml_py38_PT_TF", | |
| "language": "python", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "name": "python", | |
| "version": "3.8.5", | |
| "mimetype": "text/x-python", | |
| "codemirror_mode": { | |
| "name": "ipython", | |
| "version": 3 | |
| }, | |
| "pygments_lexer": "ipython3", | |
| "nbconvert_exporter": "python", | |
| "file_extension": ".py" | |
| }, | |
| "microsoft": { | |
| "host": { | |
| "AzureML": { | |
| "notebookHasBeenCompleted": true | |
| } | |
| }, | |
| "ms_spell_check": { | |
| "ms_spell_check_language": "en" | |
| } | |
| }, | |
| "nteract": { | |
| "version": "nteract-front-end@1.0.0" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 4 | |
| } |