File size: 5,966 Bytes
f71c233 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 |
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import numpy as np
import json
import os
import os.path as osp
# LOAD FINAL RESULTS:
datasets = ["shakespeare_char", "enwik8", "text8"]
folders = os.listdir("./")
final_results = {}
results_info = {}
for folder in folders:
if folder.startswith("run") and osp.isdir(folder):
with open(osp.join(folder, "final_info.json"), "r") as f:
final_results[folder] = json.load(f)
results_dict = np.load(osp.join(folder, "all_results.npy"), allow_pickle=True).item()
run_info = {}
for dataset in datasets:
run_info[dataset] = {}
val_losses = []
train_losses = []
for k in results_dict.keys():
if dataset in k and "val_info" in k:
run_info[dataset]["iters"] = [info["iter"] for info in results_dict[k]]
val_losses.append([info["val/loss"] for info in results_dict[k]])
train_losses.append([info["train/loss"] for info in results_dict[k]])
mean_val_losses = np.mean(val_losses, axis=0)
mean_train_losses = np.mean(train_losses, axis=0)
if len(val_losses) > 0:
sterr_val_losses = np.std(val_losses, axis=0) / np.sqrt(len(val_losses))
stderr_train_losses = np.std(train_losses, axis=0) / np.sqrt(len(train_losses))
else:
sterr_val_losses = np.zeros_like(mean_val_losses)
stderr_train_losses = np.zeros_like(mean_train_losses)
run_info[dataset]["val_loss"] = mean_val_losses
run_info[dataset]["train_loss"] = mean_train_losses
run_info[dataset]["val_loss_sterr"] = sterr_val_losses
run_info[dataset]["train_loss_sterr"] = stderr_train_losses
results_info[folder] = run_info
# CREATE LEGEND -- ADD RUNS HERE THAT WILL BE PLOTTED
labels = {
"run_0": "Baseline",
"run_1": "Multi-Style Adapter",
"run_2": "Fine-tuned Multi-Style Adapter",
"run_3": "Enhanced Style Consistency",
"run_4": "Style Consistency Analysis",
}
# Create a programmatic color palette
def generate_color_palette(n):
cmap = plt.get_cmap('tab20')
return [mcolors.rgb2hex(cmap(i)) for i in np.linspace(0, 1, n)]
# Get the list of runs and generate the color palette
runs = list(labels.keys())
colors = generate_color_palette(len(runs))
# Plot 1: Line plot of training loss for each dataset across the runs with labels
for dataset in datasets:
plt.figure(figsize=(10, 6))
for i, run in enumerate(runs):
iters = results_info[run][dataset]["iters"]
mean = results_info[run][dataset]["train_loss"]
sterr = results_info[run][dataset]["train_loss_sterr"]
plt.plot(iters, mean, label=labels[run], color=colors[i])
plt.fill_between(iters, mean - sterr, mean + sterr, color=colors[i], alpha=0.2)
plt.title(f"Training Loss Across Runs for {dataset} Dataset")
plt.xlabel("Iteration")
plt.ylabel("Training Loss")
plt.legend()
plt.grid(True, which="both", ls="-", alpha=0.2)
plt.tight_layout()
plt.savefig(f"train_loss_{dataset}.png")
plt.close()
# Plot 2: Line plot of validation loss for each dataset across the runs with labels
for dataset in datasets:
plt.figure(figsize=(10, 6))
for i, run in enumerate(runs):
iters = results_info[run][dataset]["iters"]
mean = results_info[run][dataset]["val_loss"]
sterr = results_info[run][dataset]["val_loss_sterr"]
plt.plot(iters, mean, label=labels[run], color=colors[i])
plt.fill_between(iters, mean - sterr, mean + sterr, color=colors[i], alpha=0.2)
plt.title(f"Validation Loss Across Runs for {dataset} Dataset")
plt.xlabel("Iteration")
plt.ylabel("Validation Loss")
plt.legend()
plt.grid(True, which="both", ls="-", alpha=0.2)
plt.tight_layout()
plt.savefig(f"val_loss_{dataset}.png")
plt.close()
# Plot 3: Bar plot of style consistency scores for each dataset across the runs
plt.figure(figsize=(12, 6))
x = np.arange(len(datasets))
width = 0.8 / len(runs)
for i, run in enumerate(runs):
means = []
stds = []
for dataset in datasets:
if 'style_consistency_scores' in final_results[run][dataset].get('means', {}):
means.append(final_results[run][dataset]['means']['style_consistency_scores'].get('mean_consistency', 0))
stds.append(final_results[run][dataset].get('stderrs', {}).get('style_consistency_scores', {}).get('mean_consistency', 0))
else:
means.append(0)
stds.append(0)
plt.bar(x + i*width, means, width, label=labels[run], yerr=stds, capsize=5)
plt.xlabel('Dataset')
plt.ylabel('Style Consistency Score')
plt.title('Style Consistency Scores Across Runs and Datasets')
plt.xticks(x + width*(len(runs)-1)/2, datasets)
plt.legend()
plt.tight_layout()
plt.savefig("style_consistency_scores.png")
plt.close()
# Plot 4: Bar plot of inference speed for each dataset across the runs
plt.figure(figsize=(12, 6))
x = np.arange(len(datasets))
width = 0.8 / len(runs)
for i, run in enumerate(runs):
means = []
stds = []
for dataset in datasets:
if 'avg_inference_tokens_per_second_mean' in final_results[run][dataset]['means']:
means.append(final_results[run][dataset]['means']['avg_inference_tokens_per_second_mean'])
stds.append(final_results[run][dataset]['stderrs'].get('avg_inference_tokens_per_second_mean', 0))
else:
means.append(0)
stds.append(0)
plt.bar(x + i*width, means, width, label=labels[run], yerr=stds, capsize=5)
plt.xlabel('Dataset')
plt.ylabel('Tokens per Second')
plt.title('Inference Speed Across Runs and Datasets')
plt.xticks(x + width*(len(runs)-1)/2, datasets)
plt.legend()
plt.tight_layout()
plt.savefig("inference_speed.png")
plt.close()
|