Sharif University of Technology
\n", "Natural Language Processing
\n", "Final Project
\n", "Spoiler classification and summary generation
\n", "Authors: Parnian Razavipour, Mobina Salimipanah
\n", "(Equal Contribution)
\n", "Authors: Parnian Razavipour, Mobina Salimipanah
\n", "(Equal Contribution)
\n", "| Epoch | \n", "Training Loss | \n", "Validation Loss | \n", "
|---|---|---|
| 1 | \n", "2.925300 | \n", "2.873685 | \n", "
"
],
"text/plain": [
" "
]
},
"metadata": {}
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"TrainOutput(global_step=3300, training_loss=2.429696747750947, metrics={'train_runtime': 5007.437, 'train_samples_per_second': 0.659, 'train_steps_per_second': 0.659, 'total_flos': 6526373990400000.0, 'train_loss': 2.429696747750947, 'epoch': 3.0})"
]
},
"metadata": {},
"execution_count": 21
}
]
},
{
"cell_type": "markdown",
"source": [
"##**Save the Model (3 epochs trained)**"
],
"metadata": {
"id": "uBp1rPWNI8l9"
}
},
{
"cell_type": "code",
"source": [
"model_save_path = '/content/drive/MyDrive/summary_generation_Led_3'\n",
"trainer.save_model(model_save_path)"
],
"metadata": {
"id": "kFuNP98wGaa2"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"from google.colab import drive\n",
"drive.mount('/content/drive')\n",
"\n",
"# Load the trained model from Google Drive\n",
"model_save_path = '/content/drive/MyDrive/summary_generation_Led_3'\n",
"model = LEDForConditionalGeneration.from_pretrained(model_save_path)\n",
"\n",
"model = model.to(device)\n"
],
"metadata": {
"id": "yopwoSEJgNkT"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"##**Continue training for 1 additional epoch**"
],
"metadata": {
"id": "T8XIu4hFJSQk"
}
},
{
"cell_type": "code",
"source": [
"from transformers import Trainer, TrainingArguments\n",
"\n",
"# Reload the model from the saved checkpoint in Google Drive\n",
"model_save_path = '/content/drive/MyDrive/summary_generation' # Your saved path\n",
"model = LEDForConditionalGeneration.from_pretrained(model_save_path)\n",
"model = model.to(device) # Ensure the model is on the right device (GPU or CPU)\n",
"\n",
"# Define the training arguments for continuing training\n",
"training_args = TrainingArguments(\n",
" output_dir=\"./results\",\n",
" eval_strategy=\"epoch\",\n",
" save_strategy=\"epoch\",\n",
" learning_rate=2e-5,\n",
" per_device_train_batch_size=1, # LED is memory-intensive\n",
" per_device_eval_batch_size=1,\n",
" weight_decay=0.01,\n",
" save_total_limit=3, # Only keep the last 3 models saved\n",
" num_train_epochs=1, # Continue training for 1 additional epoch\n",
" report_to=\"none\",\n",
" logging_dir='./logs', # Directory for storing logs\n",
" logging_steps=500,\n",
" load_best_model_at_end=True,\n",
"\n",
"# Initialize the optimizer (AdamW) again for the new training run\n",
"optimizer = torch.optim.AdamW(model.parameters(), lr=training_args.learning_rate)\n",
"\n",
"trainer = Trainer(\n",
" model=model,\n",
" args=training_args,\n",
" train_dataset=tokenized_datasets[\"train\"],\n",
" eval_dataset=tokenized_datasets[\"val\"],\n",
" tokenizer=tokenizer,\n",
" optimizers=(optimizer, None)\n",
")\n",
"\n",
"trainer.train()\n",
"\n",
"trainer.save_model(\"/content/drive/MyDrive/summary_generation_Led_4\") # Save the new checkpoint\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 144
},
"id": "DnDXbarJm1YG",
"outputId": "90c5c5a8-bf06-4631-e868-17c6e7dc2f4a"
},
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
" "
]
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"There were missing keys in the checkpoint model loaded: ['led.encoder.embed_tokens.weight', 'led.decoder.embed_tokens.weight', 'lm_head.weight'].\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"##**Evaluation on test data**"
],
"metadata": {
"id": "ewfMVjdEJoWF"
}
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "7S7rMmsRzUDK"
},
"outputs": [],
"source": [
"pip install rouge_score"
]
},
{
"cell_type": "code",
"source": [
"pip install nltk\n"
],
"metadata": {
"id": "OFJmH2KUtdod"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import torch, gc\n",
"\n",
"gc.collect()\n",
"torch.cuda.empty_cache()\n"
],
"metadata": {
"id": "SEGn9IQ4to0c"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import nltk\n",
"\n",
"nltk.download('punkt')\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "FliiA5j_-bK6",
"outputId": "a1784ce0-e243-4f06-b2dc-554300ec8726"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"[nltk_data] Downloading package punkt to /root/nltk_data...\n",
"[nltk_data] Unzipping tokenizers/punkt.zip.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"True"
]
},
"metadata": {},
"execution_count": 20
}
]
},
{
"cell_type": "code",
"source": [
"from datasets import load_metric\n",
"import nltk\n",
"\n",
"metric_rouge = load_metric(\"rouge\")\n",
"\n",
"def generate_summary(batch):\n",
" inputs = tokenizer(batch[\"plot_synopsis\"], max_length=3000, truncation=True, padding=\"max_length\", return_tensors=\"pt\")\n",
" inputs = inputs.to(device)\n",
" outputs = model.generate(inputs[\"input_ids\"], max_length=315, min_length=20, length_penalty=2.0, num_beams=4, early_stopping=True)\n",
"\n",
" batch[\"pred_summary\"] = tokenizer.batch_decode(outputs, skip_special_tokens=True)\n",
" return batch\n",
"\n",
"results = dataset[\"test\"].map(generate_summary, batched=True, batch_size=8)\n",
"rouge_score = metric_rouge.compute(predictions=results[\"pred_summary\"], references=results[\"plot_summary\"])\n",
"print(\"ROUGE scores:\")\n",
"print(rouge_score)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 104,
"referenced_widgets": [
"ad570df7408d48d99eb69a50ad8f3201",
"265654d5426b4616b530e7fd1c7cec04",
"ccb396e3506e487284decd31875fa33d",
"2873376fa2264b26ab8dcebe41d46af0",
"13b89ba59e80499f9d6926ee97d9f519",
"edf3f110599943ab822847b15bcfd0d4",
"8d275ca50cce4c48889ddc50769ada21",
"d89b8b1072d9408c96d2f7b2e735c785",
"0e6ac91d7b284b9b82abdedf36e91924",
"e0b2c384ca364c49869ae498683a6ef3",
"a84982a5e3ff40a18ab9b416cd85a111"
]
},
"id": "I1wd38l3sYAy",
"outputId": "4a6e0d20-74b4-4916-edf3-c7b0ef4b5117"
},
"execution_count": null,
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ad570df7408d48d99eb69a50ad8f3201",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Map: 0%| | 0/236 [00:00\n",
" \n",
" \n",
" [3300/3300 1:23:26, Epoch 3/3]\n",
" \n",
" \n",
" \n",
"
\n",
" \n",
" \n",
" \n",
" Epoch \n",
" Training Loss \n",
" Validation Loss \n",
" \n",
" \n",
" 1 \n",
" 2.925300 \n",
" 2.873685 \n",
" \n",
" \n",
" 2 \n",
" 2.409600 \n",
" 2.883492 \n",
" \n",
" \n",
" \n",
"3 \n",
" 2.165300 \n",
" 2.920285 \n",
" \n",
" \n",
" \n",
" [1100/1100 26:32, Epoch 1/1]\n",
" \n",
" \n",
" \n",
"
\n",
" \n",
" \n",
" \n",
" Epoch \n",
" Training Loss \n",
" Validation Loss \n",
" \n",
" \n",
" \n",
"1 \n",
" 2.140500 \n",
" 2.964694 \n",
"