diff --git "a/contraceptive/tvae/mlu-eval.ipynb" "b/contraceptive/tvae/mlu-eval.ipynb" new file mode 100644--- /dev/null +++ "b/contraceptive/tvae/mlu-eval.ipynb" @@ -0,0 +1,2563 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "982e76f5", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.441740Z", + "iopub.status.busy": "2024-02-29T22:23:56.441354Z", + "iopub.status.idle": "2024-02-29T22:23:56.475164Z", + "shell.execute_reply": "2024-02-29T22:23:56.474284Z" + }, + "papermill": { + "duration": 0.049332, + "end_time": "2024-02-29T22:23:56.477091", + "exception": false, + "start_time": "2024-02-29T22:23:56.427759", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import joblib\n", + "\n", + "#joblib.parallel_backend(\"threading\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "675f0b41", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.502389Z", + "iopub.status.busy": "2024-02-29T22:23:56.502039Z", + "iopub.status.idle": "2024-02-29T22:23:56.508713Z", + "shell.execute_reply": "2024-02-29T22:23:56.507881Z" + }, + "papermill": { + "duration": 0.021493, + "end_time": "2024-02-29T22:23:56.510656", + "exception": false, + "start_time": "2024-02-29T22:23:56.489163", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "'\\n%cd /kaggle/working\\n#!git clone https://github.com/R-N/ml-utility-loss\\n%cd ml-utility-loss\\n!git pull\\n#!pip install .\\n!pip install . --no-deps --force-reinstall --upgrade\\n#'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\"\n", + "%cd /kaggle/working\n", + "#!git clone https://github.com/R-N/ml-utility-loss\n", + "%cd ml-utility-loss\n", + "!git pull\n", + "#!pip install .\n", + "!pip install . --no-deps --force-reinstall --upgrade\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5ae30f5c", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.534297Z", + "iopub.status.busy": "2024-02-29T22:23:56.534007Z", + "iopub.status.idle": "2024-02-29T22:23:56.538072Z", + "shell.execute_reply": "2024-02-29T22:23:56.537225Z" + }, + "papermill": { + "duration": 0.018128, + "end_time": "2024-02-29T22:23:56.539980", + "exception": false, + "start_time": "2024-02-29T22:23:56.521852", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "plt.rcParams['figure.figsize'] = [3,3]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9f42c810", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.563822Z", + "iopub.status.busy": "2024-02-29T22:23:56.563564Z", + "iopub.status.idle": "2024-02-29T22:23:56.567349Z", + "shell.execute_reply": "2024-02-29T22:23:56.566540Z" + }, + "executionInfo": { + "elapsed": 678, + "status": "ok", + "timestamp": 1696841022168, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "ns5hFcVL2yvs", + "papermill": { + "duration": 0.018066, + "end_time": "2024-02-29T22:23:56.569241", + "exception": false, + "start_time": "2024-02-29T22:23:56.551175", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "datasets = [\n", + " \"insurance\",\n", + " \"treatment\",\n", + " \"contraceptive\"\n", + "]\n", + "\n", + "study_dir = \"./\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "85d0c8ce", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.592724Z", + "iopub.status.busy": "2024-02-29T22:23:56.592470Z", + "iopub.status.idle": "2024-02-29T22:23:56.597579Z", + "shell.execute_reply": "2024-02-29T22:23:56.596646Z" + }, + "papermill": { + "duration": 0.019073, + "end_time": "2024-02-29T22:23:56.599378", + "exception": false, + "start_time": "2024-02-29T22:23:56.580305", + "status": "completed" + }, + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "#Parameters\n", + "import os\n", + "\n", + "path_prefix = \"../../../../\"\n", + "\n", + "dataset_dir = os.path.join(path_prefix, \"ml-utility-loss/datasets\")\n", + "dataset_name = \"treatment\"\n", + "model_name=\"ml_utility_2\"\n", + "models = [\"tvae\", \"realtabformer\", \"lct_gan\", \"tab_ddpm_concat\"]\n", + "single_model = \"lct_gan\"\n", + "random_seed = 42\n", + "gp = True\n", + "gp_multiply = True\n", + "folder = \"eval\"\n", + "debug = False\n", + "path = None\n", + "param_index = 0" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7b9b0dd4", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.626607Z", + "iopub.status.busy": "2024-02-29T22:23:56.625833Z", + "iopub.status.idle": "2024-02-29T22:23:56.631327Z", + "shell.execute_reply": "2024-02-29T22:23:56.630534Z" + }, + "papermill": { + "duration": 0.02201, + "end_time": "2024-02-29T22:23:56.633199", + "exception": false, + "start_time": "2024-02-29T22:23:56.611189", + "status": "completed" + }, + "tags": [ + "injected-parameters" + ] + }, + "outputs": [], + "source": [ + "# Parameters\n", + "dataset = \"contraceptive\"\n", + "dataset_name = \"contraceptive\"\n", + "single_model = \"tvae\"\n", + "gp = False\n", + "gp_multiply = False\n", + "random_seed = 2\n", + "debug = False\n", + "folder = \"eval\"\n", + "path_prefix = \"../../../../\"\n", + "path = \"eval/contraceptive/tvae/2\"\n", + "param_index = 2\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bd7c02d6", + "metadata": { + "papermill": { + "duration": 0.011329, + "end_time": "2024-02-29T22:23:56.657143", + "exception": false, + "start_time": "2024-02-29T22:23:56.645814", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5f45b1d0", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.681290Z", + "iopub.status.busy": "2024-02-29T22:23:56.680693Z", + "iopub.status.idle": "2024-02-29T22:23:56.689901Z", + "shell.execute_reply": "2024-02-29T22:23:56.689107Z" + }, + "executionInfo": { + "elapsed": 7, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "UdvXYv3c3LXy", + "papermill": { + "duration": 0.023313, + "end_time": "2024-02-29T22:23:56.691767", + "exception": false, + "start_time": "2024-02-29T22:23:56.668454", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "/kaggle/working\n", + "/kaggle/working/eval/contraceptive/tvae/2\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "import os\n", + "\n", + "%cd /kaggle/working/\n", + "\n", + "if path is None:\n", + " path = os.path.join(folder, dataset_name, single_model, random_seed)\n", + "Path(path).mkdir(parents=True, exist_ok=True)\n", + "\n", + "%cd {path}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f85bf540", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:56.716278Z", + "iopub.status.busy": "2024-02-29T22:23:56.715756Z", + "iopub.status.idle": "2024-02-29T22:23:59.037889Z", + "shell.execute_reply": "2024-02-29T22:23:59.036865Z" + }, + "papermill": { + "duration": 2.337143, + "end_time": "2024-02-29T22:23:59.040406", + "exception": false, + "start_time": "2024-02-29T22:23:56.703263", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Set seed to \n" + ] + } + ], + "source": [ + "from ml_utility_loss.util import seed\n", + "if single_model:\n", + " model_name=f\"{model_name}_{single_model}\"\n", + "if random_seed is not None:\n", + " seed(random_seed)\n", + " print(\"Set seed to\", seed)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "8489feae", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.067269Z", + "iopub.status.busy": "2024-02-29T22:23:59.066864Z", + "iopub.status.idle": "2024-02-29T22:23:59.078105Z", + "shell.execute_reply": "2024-02-29T22:23:59.077378Z" + }, + "papermill": { + "duration": 0.026705, + "end_time": "2024-02-29T22:23:59.080082", + "exception": false, + "start_time": "2024-02-29T22:23:59.053377", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import json\n", + "import os\n", + "\n", + "df = pd.read_csv(os.path.join(dataset_dir, f\"{dataset_name}.csv\"))\n", + "with open(os.path.join(dataset_dir, f\"{dataset_name}.json\")) as f:\n", + " info = json.load(f)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "debcc684", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.105174Z", + "iopub.status.busy": "2024-02-29T22:23:59.104896Z", + "iopub.status.idle": "2024-02-29T22:23:59.112132Z", + "shell.execute_reply": "2024-02-29T22:23:59.111304Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "Vrl2QkoV3o_8", + "papermill": { + "duration": 0.021579, + "end_time": "2024-02-29T22:23:59.114101", + "exception": false, + "start_time": "2024-02-29T22:23:59.092522", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "task = info[\"task\"]\n", + "target = info[\"target\"]\n", + "cat_features = info[\"cat_features\"]\n", + "mixed_features = info[\"mixed_features\"]\n", + "longtail_features = info[\"longtail_features\"]\n", + "integer_features = info[\"integer_features\"]\n", + "\n", + "test = df.sample(frac=0.2, random_state=42)\n", + "train = df[~df.index.isin(test.index)]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "7538184a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.139461Z", + "iopub.status.busy": "2024-02-29T22:23:59.139015Z", + "iopub.status.idle": "2024-02-29T22:23:59.244628Z", + "shell.execute_reply": "2024-02-29T22:23:59.243810Z" + }, + "executionInfo": { + "elapsed": 6, + "status": "ok", + "timestamp": 1696841022169, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "TilUuFk9vqMb", + "papermill": { + "duration": 0.120502, + "end_time": "2024-02-29T22:23:59.247058", + "exception": false, + "start_time": "2024-02-29T22:23:59.126556", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import ml_utility_loss.synthesizers.tab_ddpm.params as TAB_DDPM_PARAMS\n", + "import ml_utility_loss.synthesizers.lct_gan.params as LCT_GAN_PARAMS\n", + "import ml_utility_loss.synthesizers.realtabformer.params as RTF_PARAMS\n", + "from ml_utility_loss.synthesizers.realtabformer.params.default import GPT2_PARAMS, REALTABFORMER_PARAMS\n", + "from ml_utility_loss.util import filter_dict_2, filter_dict\n", + "\n", + "tab_ddpm_params = getattr(TAB_DDPM_PARAMS, dataset_name).BEST\n", + "lct_gan_params = getattr(LCT_GAN_PARAMS, dataset_name).BEST\n", + "lct_ae_params = filter_dict_2(lct_gan_params, LCT_GAN_PARAMS.default.AE_PARAMS)\n", + "rtf_params = getattr(RTF_PARAMS, dataset_name).BEST\n", + "rtf_params = filter_dict(rtf_params, REALTABFORMER_PARAMS)\n", + "\n", + "lct_ae_embedding_size=lct_gan_params[\"embedding_size\"]\n", + "tab_ddpm_normalization=\"quantile\"\n", + "tab_ddpm_cat_encoding=tab_ddpm_params[\"cat_encoding\"]\n", + "#tab_ddpm_cat_encoding=\"one-hot\"\n", + "tab_ddpm_y_policy=\"default\"\n", + "tab_ddpm_is_y_cond=True" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "cca61838", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:23:59.273328Z", + "iopub.status.busy": "2024-02-29T22:23:59.273055Z", + "iopub.status.idle": "2024-02-29T22:24:04.033228Z", + "shell.execute_reply": "2024-02-29T22:24:04.032395Z" + }, + "executionInfo": { + "elapsed": 3113, + "status": "ok", + "timestamp": 1696841025277, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "7Abt8nStvr9Z", + "papermill": { + "duration": 4.775985, + "end_time": "2024-02-29T22:24:04.035818", + "exception": false, + "start_time": "2024-02-29T22:23:59.259833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-29 22:24:01.603445: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-02-29 22:24:01.603521: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-02-29 22:24:01.605533: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_lct_ae\n", + "\n", + "lct_ae = load_lct_ae(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"lct_ae\",\n", + " df_name=\"df\",\n", + ")\n", + "lct_ae = None" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "6f83b7b6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:04.062123Z", + "iopub.status.busy": "2024-02-29T22:24:04.061563Z", + "iopub.status.idle": "2024-02-29T22:24:04.067353Z", + "shell.execute_reply": "2024-02-29T22:24:04.066647Z" + }, + "papermill": { + "duration": 0.020608, + "end_time": "2024-02-29T22:24:04.069312", + "exception": false, + "start_time": "2024-02-29T22:24:04.048704", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.loss_learning.estimator.pipeline import load_rtf_embed\n", + "\n", + "rtf_embed = load_rtf_embed(\n", + " dataset_name=dataset_name,\n", + " model_dir=os.path.join(path_prefix, \"ml-utility-loss/models\"),\n", + " model_name=\"realtabformer\",\n", + " df_name=\"df\",\n", + " ckpt_type=\"best-disc-model\"\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "0026de74", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:04.095589Z", + "iopub.status.busy": "2024-02-29T22:24:04.095238Z", + "iopub.status.idle": "2024-02-29T22:24:12.594652Z", + "shell.execute_reply": "2024-02-29T22:24:12.593579Z" + }, + "executionInfo": { + "elapsed": 20137, + "status": "ok", + "timestamp": 1696841045408, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "tbaguWxAvtPi", + "papermill": { + "duration": 8.515633, + "end_time": "2024-02-29T22:24:12.597235", + "exception": false, + "start_time": "2024-02-29T22:24:04.081602", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:274: ConvergenceWarning: Initialization 1 did not converge. Try different init parameters, or increase max_iter, tol or check for degenerate data.\n", + " warnings.warn(\n", + "/opt/conda/lib/python3.10/site-packages/sklearn/mixture/_base.py:119: ConvergenceWarning: Number of distinct clusters (4) found smaller than n_clusters (10). Possibly due to duplicate points in X.\n", + " .fit(X)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + " 0%| | 0/1 [00:00 torch.Tensor>,\n", + " 'single_model': True,\n", + " 'bias': True,\n", + " 'bias_final': True,\n", + " 'pma_ffn_mode': 'shared',\n", + " 'patience': 10,\n", + " 'inds_init_mode': 'fixnorm',\n", + " 'grad_clip': 0.775,\n", + " 'gradient_penalty_mode': {'gradient_penalty': False,\n", + " 'calc_grad_m': False,\n", + " 'avg_non_role_model_m': False,\n", + " 'inverse_avg_non_role_model_m': False},\n", + " 'synth_data': 2,\n", + " 'dataset_size': 2048,\n", + " 'batch_size': 2,\n", + " 'epochs': 100,\n", + " 'lr_mul': 0.075,\n", + " 'n_warmup_steps': 100,\n", + " 'Optim': functools.partial(, amsgrad=True),\n", + " 'loss_balancer_beta': 0.675,\n", + " 'loss_balancer_r': 0.95,\n", + " 'fixed_role_model': 'tvae',\n", + " 'd_model': 256,\n", + " 'attn_activation': torch.nn.modules.activation.PReLU,\n", + " 'tf_d_inner': 512,\n", + " 'tf_n_layers_enc': 3,\n", + " 'tf_n_head': 32,\n", + " 'tf_activation': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'tf_activation_final': ml_utility_loss.activations.LeakyHardtanh,\n", + " 'ada_d_hid': 1024,\n", + " 'ada_n_layers': 8,\n", + " 'ada_activation': torch.nn.modules.activation.Softsign,\n", + " 'ada_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'head_d_hid': 256,\n", + " 'head_n_layers': 9,\n", + " 'head_n_head': 32,\n", + " 'head_activation': torch.nn.modules.activation.ReLU6,\n", + " 'head_activation_final': ml_utility_loss.activations.LeakyHardsigmoid,\n", + " 'models': ['tvae'],\n", + " 'max_seconds': 3600,\n", + " 'tf_lora': False,\n", + " 'tf_num_inds': 64,\n", + " 'ada_n_seeds': 0,\n", + " 'gradient_penalty_kwargs': {'mag_loss': True,\n", + " 'mse_mag': False,\n", + " 'mag_corr': False,\n", + " 'seq_mag': False,\n", + " 'cos_loss': False,\n", + " 'mag_corr_kwargs': {'only_sign': False},\n", + " 'cos_loss_kwargs': {'only_sign': True, 'cos_matrix': False},\n", + " 'mse_mag_kwargs': {'target': 0.1, 'multiply': False}}}" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import ml_utility_loss.loss_learning.estimator.params2 as PARAMS\n", + "from ml_utility_loss.tuning import map_parameters\n", + "from ml_utility_loss.loss_learning.estimator.params.default import update_param_space, update_param_space_2\n", + "import wandb\n", + "\n", + "#\"\"\"\n", + "param_space = {\n", + " **getattr(PARAMS, dataset_name).PARAM_SPACE,\n", + "}\n", + "params = {\n", + " **getattr(PARAMS, dataset_name).BESTS[param_index],\n", + "}\n", + "if gp:\n", + " params[\"gradient_penalty_mode\"] = \"ALL\"\n", + " params[\"mse_mag\"] = True\n", + " if gp_multiply:\n", + " params[\"mse_mag_multiply\"] = True\n", + " params[\"mse_mag_target\"] = 1.0\n", + " else:\n", + " params[\"mse_mag_multiply\"] = False\n", + " params[\"mse_mag_target\"] = 0.1\n", + "else:\n", + " params[\"gradient_penalty_mode\"] = \"NONE\"\n", + " params[\"mse_mag\"] = False\n", + "params[\"single_model\"] = False\n", + "if models:\n", + " params[\"models\"] = models\n", + "if single_model:\n", + " params[\"fixed_role_model\"] = single_model\n", + " params[\"single_model\"] = True\n", + " params[\"models\"] = [single_model]\n", + "if params[\"fixed_role_model\"] == \"realtabformer\" and dataset_name == \"treatment\":\n", + " params[\"batch_size\"] = 2\n", + "params[\"max_seconds\"] = 3600\n", + "params[\"patience\"] = 10\n", + "params[\"epochs\"] = 100\n", + "if debug:\n", + " params[\"epochs\"] = 2\n", + "with open(\"params.json\", \"w\") as f:\n", + " json.dump(params, f)\n", + "params = map_parameters(params, param_space=param_space)\n", + "params" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "a48bd9e9", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.112927Z", + "iopub.status.busy": "2024-02-29T22:24:13.112643Z", + "iopub.status.idle": "2024-02-29T22:24:13.187354Z", + "shell.execute_reply": "2024-02-29T22:24:13.186375Z" + }, + "papermill": { + "duration": 0.090401, + "end_time": "2024-02-29T22:24:13.189442", + "exception": false, + "start_time": "2024-02-29T22:24:13.099041", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "load_dataset_3_factory 2\n", + "Caching in ../../../../contraceptive/_cache/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_2/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache4/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_4/contraceptive [80, 20]\n", + "Caching in ../../../../contraceptive/_cache5/tvae/all inf False\n", + "Splitting without random!\n", + "Split with reverse index!\n", + "../../../../ml-utility-loss/datasets_5/contraceptive [160, 40]\n", + "[320, 80]\n", + "[320, 80]\n" + ] + } + ], + "source": [ + "train_set, val_set = datasetsn(model=params[\"fixed_role_model\"], synth_data=params[\"synth_data\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "2fcb1418", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.218592Z", + "iopub.status.busy": "2024-02-29T22:24:13.217918Z", + "iopub.status.idle": "2024-02-29T22:24:13.658575Z", + "shell.execute_reply": "2024-02-29T22:24:13.657634Z" + }, + "executionInfo": { + "elapsed": 396850, + "status": "error", + "timestamp": 1696841446059, + "user": { + "displayName": "Rizqi Nur", + "userId": "09644007964068789560" + }, + "user_tz": -420 + }, + "id": "_bt1MQc5kpSk", + "outputId": "01c1d3e5-ac64-461d-835a-b76f4a66e6d6", + "papermill": { + "duration": 0.457774, + "end_time": "2024-02-29T22:24:13.660801", + "exception": false, + "start_time": "2024-02-29T22:24:13.203027", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Creating model of type \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[*] Embedding False True\n", + "['tvae'] 1\n" + ] + } + ], + "source": [ + "from ml_utility_loss.loss_learning.estimator.model.pipeline import remove_non_model_params\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import create_model\n", + "from ml_utility_loss.util import filter_dict, clear_memory\n", + "\n", + "clear_memory()\n", + "\n", + "params2 = remove_non_model_params(params)\n", + "adapters = filter_dict(preprocessor.adapter_sizes, params[\"models\"])\n", + "\n", + "model = create_model(\n", + " adapters=adapters,\n", + " #Body=\"twin_encoder\",\n", + " **params2,\n", + ")\n", + "#cf.apply_weight_standardization(model, n_last_layers_ignore=0)\n", + "print(model.models, len(model.adapters))" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "938f94fc", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.691177Z", + "iopub.status.busy": "2024-02-29T22:24:13.690867Z", + "iopub.status.idle": "2024-02-29T22:24:13.695119Z", + "shell.execute_reply": "2024-02-29T22:24:13.694245Z" + }, + "papermill": { + "duration": 0.022428, + "end_time": "2024-02-29T22:24:13.697307", + "exception": false, + "start_time": "2024-02-29T22:24:13.674879", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "study_name=f\"{model_name}_{dataset_name}\"" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "12fb613e", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.724488Z", + "iopub.status.busy": "2024-02-29T22:24:13.724194Z", + "iopub.status.idle": "2024-02-29T22:24:13.731124Z", + "shell.execute_reply": "2024-02-29T22:24:13.730276Z" + }, + "papermill": { + "duration": 0.022748, + "end_time": "2024-02-29T22:24:13.733118", + "exception": false, + "start_time": "2024-02-29T22:24:13.710370", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "10270216" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def count_parameters(model):\n", + " return sum(p.numel() for p in model.parameters() if p.requires_grad)\n", + "\n", + "count_parameters(model)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "bd386e57", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.760413Z", + "iopub.status.busy": "2024-02-29T22:24:13.760117Z", + "iopub.status.idle": "2024-02-29T22:24:13.843758Z", + "shell.execute_reply": "2024-02-29T22:24:13.842854Z" + }, + "papermill": { + "duration": 0.099517, + "end_time": "2024-02-29T22:24:13.845628", + "exception": false, + "start_time": "2024-02-29T22:24:13.746111", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "========================================================================================================================\n", + "Layer (type:depth-idx) Output Shape Param #\n", + "========================================================================================================================\n", + "MLUtilitySingle [2, 1179, 46] --\n", + "├─Adapter: 1-1 [2, 1179, 46] --\n", + "│ └─Sequential: 2-1 [2, 1179, 256] --\n", + "│ │ └─FeedForward: 3-1 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-1 [2, 1179, 1024] 48,128\n", + "│ │ │ └─Softsign: 4-2 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-2 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-3 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-4 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-3 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-5 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-6 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-4 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-7 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-8 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-5 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-9 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-10 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-6 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-11 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-12 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-7 [2, 1179, 1024] --\n", + "│ │ │ └─Linear: 4-13 [2, 1179, 1024] 1,049,600\n", + "│ │ │ └─Softsign: 4-14 [2, 1179, 1024] --\n", + "│ │ └─FeedForward: 3-8 [2, 1179, 256] --\n", + "│ │ │ └─Linear: 4-15 [2, 1179, 256] 262,400\n", + "│ │ │ └─LeakyHardsigmoid: 4-16 [2, 1179, 256] --\n", + "├─Adapter: 1-2 [2, 294, 46] (recursive)\n", + "│ └─Sequential: 2-2 [2, 294, 256] (recursive)\n", + "│ │ └─FeedForward: 3-9 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-17 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-18 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-10 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-19 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-20 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-11 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-21 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-22 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-12 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-23 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-24 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-13 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-25 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-26 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-14 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-27 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-28 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-15 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Linear: 4-29 [2, 294, 1024] (recursive)\n", + "│ │ │ └─Softsign: 4-30 [2, 294, 1024] --\n", + "│ │ └─FeedForward: 3-16 [2, 294, 256] (recursive)\n", + "│ │ │ └─Linear: 4-31 [2, 294, 256] (recursive)\n", + "│ │ │ └─LeakyHardsigmoid: 4-32 [2, 294, 256] --\n", + "├─TwinEncoder: 1-3 [2, 2048] --\n", + "│ └─Encoder: 2-3 [2, 8, 256] --\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-33 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-1 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-1 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-2 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-1 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-2 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-3 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-4 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-1 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-5 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-6 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-3 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-7 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-8 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-9 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-10 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-2 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-11 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-12 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-2 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-4 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-5 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-6 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-34 [2, 1179, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-3 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-7 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-8 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-13 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-14 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-15 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-16 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-3 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-17 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-18 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-9 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-19 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-20 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-21 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-22 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-4 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-23 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-24 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-4 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-10 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-11 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-12 [2, 1179, 256] 131,328\n", + "│ │ │ └─EncoderLayer: 4-35 [2, 8, 256] --\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-5 [2, 1179, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-13 [2, 64, 256] 16,384\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-14 [2, 64, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-25 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-26 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-27 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-28 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-5 [2, 32, 64, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-29 [2, 64, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-30 [2, 64, 256] 1\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-15 [2, 1179, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-31 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-32 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-33 [2, 64, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-34 [2, 32, 1179, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-6 [2, 32, 1179, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-35 [2, 1179, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-36 [2, 1179, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-6 [2, 1179, 256] --\n", + "│ │ │ │ │ └─Linear: 6-16 [2, 1179, 512] 131,584\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-17 [2, 1179, 512] --\n", + "│ │ │ │ │ └─Linear: 6-18 [2, 1179, 256] 131,328\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-7 [2, 8, 256] --\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-19 [2, 8, 256] 2,048\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-20 [2, 8, 256] --\n", + "│ │ │ │ │ │ └─Linear: 7-37 [2, 8, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-38 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─Linear: 7-39 [2, 1179, 256] 65,536\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-40 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-7 [2, 32, 8, 1179] --\n", + "│ │ │ │ │ │ └─Linear: 7-41 [2, 8, 256] 65,792\n", + "│ │ │ │ │ │ └─PReLU: 7-42 [2, 8, 256] 1\n", + "│ │ │ │ └─DoubleFeedForward: 5-8 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-21 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-22 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-23 [2, 8, 256] (recursive)\n", + "│ └─Encoder: 2-4 [2, 8, 256] (recursive)\n", + "│ │ └─ModuleList: 3-18 -- (recursive)\n", + "│ │ │ └─EncoderLayer: 4-36 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-9 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-24 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-25 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-43 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-44 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-45 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-46 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-8 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-47 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-48 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-26 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-49 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-50 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-51 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-52 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-9 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-53 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-54 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-10 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-27 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-28 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-29 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-37 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-11 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-30 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-31 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-55 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-56 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-57 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-58 [2, 32, 64, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-10 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-59 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-60 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-32 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-61 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-62 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-63 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-64 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-11 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-65 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-66 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-12 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-33 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-34 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-35 [2, 294, 256] (recursive)\n", + "│ │ │ └─EncoderLayer: 4-38 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─SimpleInducedSetAttention: 5-13 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-36 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-37 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-67 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-68 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-69 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-70 [2, 32, 64, 8] --\n", + "│ │ │ │ �� │ │ └─Softmax: 8-12 [2, 32, 64, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-71 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-72 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ └─MultiHeadAttention: 6-38 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-73 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-74 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-75 [2, 64, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-76 [2, 32, 294, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-13 [2, 32, 294, 64] --\n", + "│ │ │ │ │ │ └─Linear: 7-77 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-78 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-14 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-39 [2, 294, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-40 [2, 294, 512] --\n", + "│ │ │ │ │ └─Linear: 6-41 [2, 294, 256] (recursive)\n", + "│ │ │ │ └─PoolingByMultiheadAttention: 5-15 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─TensorInductionPoint: 6-42 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─SimpleMultiHeadAttention: 6-43 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-79 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-80 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─Linear: 7-81 [2, 294, 256] (recursive)\n", + "│ │ │ │ │ │ └─ScaledDotProductAttention: 7-82 [2, 32, 8, 8] --\n", + "│ │ │ │ │ │ │ └─Softmax: 8-14 [2, 32, 8, 294] --\n", + "│ │ │ │ │ │ └─Linear: 7-83 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ │ └─PReLU: 7-84 [2, 8, 256] (recursive)\n", + "│ │ │ │ └─DoubleFeedForward: 5-16 [2, 8, 256] (recursive)\n", + "│ │ │ │ │ └─Linear: 6-44 [2, 8, 512] (recursive)\n", + "│ │ │ │ │ └─LeakyHardtanh: 6-45 [2, 8, 512] --\n", + "│ │ │ │ │ └─Linear: 6-46 [2, 8, 256] (recursive)\n", + "├─Head: 1-4 [2] --\n", + "│ └─Sequential: 2-5 [2, 1] --\n", + "│ │ └─FeedForward: 3-19 [2, 256] --\n", + "│ │ │ └─Linear: 4-39 [2, 256] 524,544\n", + "│ │ │ └─ReLU6: 4-40 [2, 256] --\n", + "│ │ └─FeedForward: 3-20 [2, 256] --\n", + "│ │ │ └─Linear: 4-41 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-42 [2, 256] --\n", + "│ │ └─FeedForward: 3-21 [2, 256] --\n", + "│ │ │ └─Linear: 4-43 [2, 256] 65,792\n", + "│ ��� │ └─ReLU6: 4-44 [2, 256] --\n", + "│ │ └─FeedForward: 3-22 [2, 256] --\n", + "│ │ │ └─Linear: 4-45 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-46 [2, 256] --\n", + "│ │ └─FeedForward: 3-23 [2, 256] --\n", + "│ │ │ └─Linear: 4-47 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-48 [2, 256] --\n", + "│ │ └─FeedForward: 3-24 [2, 256] --\n", + "│ │ │ └─Linear: 4-49 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-50 [2, 256] --\n", + "│ │ └─FeedForward: 3-25 [2, 256] --\n", + "│ │ │ └─Linear: 4-51 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-52 [2, 256] --\n", + "│ │ └─FeedForward: 3-26 [2, 256] --\n", + "│ │ │ └─Linear: 4-53 [2, 256] 65,792\n", + "│ │ │ └─ReLU6: 4-54 [2, 256] --\n", + "│ │ └─FeedForward: 3-27 [2, 1] --\n", + "│ │ │ └─Linear: 4-55 [2, 1] 257\n", + "│ │ │ └─LeakyHardsigmoid: 4-56 [2, 1] --\n", + "========================================================================================================================\n", + "Total params: 10,270,216\n", + "Trainable params: 10,270,216\n", + "Non-trainable params: 0\n", + "Total mult-adds (M): 39.96\n", + "========================================================================================================================\n", + "Input size (MB): 0.54\n", + "Forward/backward pass size (MB): 341.77\n", + "Params size (MB): 41.08\n", + "Estimated Total Size (MB): 383.39\n", + "========================================================================================================================" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from torchinfo import summary\n", + "\n", + "role_model = params[\"fixed_role_model\"]\n", + "s = train_set[0][role_model]\n", + "summary(model[role_model], input_size=((2, *s[0].shape), (2, *s[1].shape)), depth=9) # 8 max" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "0f42c4d1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T22:24:13.877406Z", + "iopub.status.busy": "2024-02-29T22:24:13.877040Z", + "iopub.status.idle": "2024-02-29T23:27:38.382139Z", + "shell.execute_reply": "2024-02-29T23:27:38.381146Z" + }, + "papermill": { + "duration": 3804.523997, + "end_time": "2024-02-29T23:27:38.384713", + "exception": false, + "start_time": "2024-02-29T22:24:13.860716", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.16.3\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: W&B syncing is set to \u001b[1m`offline`\u001b[0m in this directory. \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb online`\u001b[0m or set \u001b[1mWANDB_MODE=online\u001b[0m to enable cloud syncing.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "g_loss_mul 0.1\n", + "Epoch 0\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.02241888580356317, 'avg_role_model_std_loss': 1.4905488636076916, 'avg_role_model_mean_pred_loss': 0.0030782962097319457, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.02241888580356317, 'n_size': 320, 'n_batch': 160, 'duration': 142.862961769104, 'duration_batch': 0.8928935110569001, 'duration_size': 0.44644675552845003, 'avg_pred_std': 0.0961261961127093}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.007483271204910125, 'avg_role_model_std_loss': 7.415935450342414, 'avg_role_model_mean_pred_loss': 9.700103195409149e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.007483271204910125, 'n_size': 80, 'n_batch': 40, 'duration': 32.79719305038452, 'duration_batch': 0.8199298262596131, 'duration_size': 0.40996491312980654, 'avg_pred_std': 0.030276008496821306}\n", + "Epoch 1\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.004170822119840522, 'avg_role_model_std_loss': 2.427984166694133, 'avg_role_model_mean_pred_loss': 5.6218089488430103e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.004170822119840522, 'n_size': 320, 'n_batch': 160, 'duration': 140.515380859375, 'duration_batch': 0.8782211303710937, 'duration_size': 0.43911056518554686, 'avg_pred_std': 0.06761386157022571}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0027781161175880697, 'avg_role_model_std_loss': 5.97971810359972, 'avg_role_model_mean_pred_loss': 8.342038965802879e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0027781161175880697, 'n_size': 80, 'n_batch': 40, 'duration': 32.894118309020996, 'duration_batch': 0.8223529577255249, 'duration_size': 0.41117647886276243, 'avg_pred_std': 0.024398993137219806}\n", + "Epoch 2\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0032142372105653295, 'avg_role_model_std_loss': 3.1775326946756253, 'avg_role_model_mean_pred_loss': 9.600223262965901e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032142372105653295, 'n_size': 320, 'n_batch': 160, 'duration': 144.2568175792694, 'duration_batch': 0.9016051098704339, 'duration_size': 0.4508025549352169, 'avg_pred_std': 0.06473975269825587}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002925431027142622, 'avg_role_model_std_loss': 6.108939102519116, 'avg_role_model_mean_pred_loss': 8.98504544019768e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002925431027142622, 'n_size': 80, 'n_batch': 40, 'duration': 35.61764717102051, 'duration_batch': 0.8904411792755127, 'duration_size': 0.44522058963775635, 'avg_pred_std': 0.028732989538184484}\n", + "Epoch 3\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.003577234379551553, 'avg_role_model_std_loss': 2.9195899648459376, 'avg_role_model_mean_pred_loss': 4.348199776086897e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.003577234379551553, 'n_size': 320, 'n_batch': 160, 'duration': 150.63309359550476, 'duration_batch': 0.9414568349719048, 'duration_size': 0.4707284174859524, 'avg_pred_std': 0.06363038770923594}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0032030581480285035, 'avg_role_model_std_loss': 5.47701075857707, 'avg_role_model_mean_pred_loss': 1.4309088606778708e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0032030581480285035, 'n_size': 80, 'n_batch': 40, 'duration': 32.49244570732117, 'duration_batch': 0.8123111426830292, 'duration_size': 0.4061555713415146, 'avg_pred_std': 0.021739204511686695}\n", + "Epoch 4\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002611448067193578, 'avg_role_model_std_loss': 1.8328958396290929, 'avg_role_model_mean_pred_loss': 8.810737564255572e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002611448067193578, 'n_size': 320, 'n_batch': 160, 'duration': 143.45851230621338, 'duration_batch': 0.8966157019138337, 'duration_size': 0.44830785095691683, 'avg_pred_std': 0.07633217830557441}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0030096135813437288, 'avg_role_model_std_loss': 5.497269465320635, 'avg_role_model_mean_pred_loss': 8.17212617150176e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0030096135813437288, 'n_size': 80, 'n_batch': 40, 'duration': 34.19865131378174, 'duration_batch': 0.8549662828445435, 'duration_size': 0.42748314142227173, 'avg_pred_std': 0.01728544359702937}\n", + "Epoch 5\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.002066187719401569, 'avg_role_model_std_loss': 1.418562725057735, 'avg_role_model_mean_pred_loss': 4.818185051591941e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002066187719401569, 'n_size': 320, 'n_batch': 160, 'duration': 141.1322615146637, 'duration_batch': 0.8820766344666481, 'duration_size': 0.44103831723332404, 'avg_pred_std': 0.07070515162549781}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002357064618456661, 'avg_role_model_std_loss': 3.038762490750969, 'avg_role_model_mean_pred_loss': 4.724545272427605e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002357064618456661, 'n_size': 80, 'n_batch': 40, 'duration': 32.73992657661438, 'duration_batch': 0.8184981644153595, 'duration_size': 0.40924908220767975, 'avg_pred_std': 0.019966062564344612}\n", + "Epoch 6\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0018150892569863686, 'avg_role_model_std_loss': 1.9370867185192977, 'avg_role_model_mean_pred_loss': 4.895915542963466e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0018150892569863686, 'n_size': 320, 'n_batch': 160, 'duration': 142.5343050956726, 'duration_batch': 0.8908394068479538, 'duration_size': 0.4454197034239769, 'avg_pred_std': 0.06771241171363726}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002098434802974225, 'avg_role_model_std_loss': 2.5296149099483842, 'avg_role_model_mean_pred_loss': 6.119135131865683e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002098434802974225, 'n_size': 80, 'n_batch': 40, 'duration': 33.130537033081055, 'duration_batch': 0.8282634258270264, 'duration_size': 0.4141317129135132, 'avg_pred_std': 0.036213114765996576}\n", + "Epoch 7\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0017754018189464205, 'avg_role_model_std_loss': 1.0608564720709155, 'avg_role_model_mean_pred_loss': 4.110462002784865e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0017754018189464205, 'n_size': 320, 'n_batch': 160, 'duration': 150.93800163269043, 'duration_batch': 0.9433625102043152, 'duration_size': 0.4716812551021576, 'avg_pred_std': 0.07812782935689029}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002651070246429299, 'avg_role_model_std_loss': 5.481469099184153, 'avg_role_model_mean_pred_loss': 8.274616622792885e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002651070246429299, 'n_size': 80, 'n_batch': 40, 'duration': 36.68884253501892, 'duration_batch': 0.917221063375473, 'duration_size': 0.4586105316877365, 'avg_pred_std': 0.0201021930330171}\n", + "Epoch 8\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0016320147636861293, 'avg_role_model_std_loss': 1.572569604070008, 'avg_role_model_mean_pred_loss': 3.280935549836465e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0016320147636861293, 'n_size': 320, 'n_batch': 160, 'duration': 152.10332083702087, 'duration_batch': 0.9506457552313805, 'duration_size': 0.47532287761569025, 'avg_pred_std': 0.07706006977591642}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021084082123252303, 'avg_role_model_std_loss': 4.821968620683037, 'avg_role_model_mean_pred_loss': 7.140439676618648e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021084082123252303, 'n_size': 80, 'n_batch': 40, 'duration': 35.41457486152649, 'duration_batch': 0.8853643715381623, 'duration_size': 0.44268218576908114, 'avg_pred_std': 0.032409553838078864}\n", + "Epoch 9\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0014390503529739362, 'avg_role_model_std_loss': 1.1130779689192227, 'avg_role_model_mean_pred_loss': 1.9856122689985296e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0014390503529739362, 'n_size': 320, 'n_batch': 160, 'duration': 148.04402089118958, 'duration_batch': 0.9252751305699348, 'duration_size': 0.4626375652849674, 'avg_pred_std': 0.07865846673303167}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002113264991157848, 'avg_role_model_std_loss': 2.781704166371675, 'avg_role_model_mean_pred_loss': 4.972169583404757e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002113264991157848, 'n_size': 80, 'n_batch': 40, 'duration': 33.858819007873535, 'duration_batch': 0.8464704751968384, 'duration_size': 0.4232352375984192, 'avg_pred_std': 0.02809684935346013}\n", + "Epoch 10\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.001374225791067829, 'avg_role_model_std_loss': 1.163778030275184, 'avg_role_model_mean_pred_loss': 1.7583196497888975e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.001374225791067829, 'n_size': 320, 'n_batch': 160, 'duration': 145.61796760559082, 'duration_batch': 0.9101122975349426, 'duration_size': 0.4550561487674713, 'avg_pred_std': 0.072609595393169}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023332797987222877, 'avg_role_model_std_loss': 2.608034198338737, 'avg_role_model_mean_pred_loss': 1.0012352827615257e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023332797987222877, 'n_size': 80, 'n_batch': 40, 'duration': 33.24501919746399, 'duration_batch': 0.8311254799365997, 'duration_size': 0.41556273996829984, 'avg_pred_std': 0.03622158533107722}\n", + "Epoch 11\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013136249404567478, 'avg_role_model_std_loss': 1.105370874132261, 'avg_role_model_mean_pred_loss': 2.0836581030414523e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013136249404567478, 'n_size': 320, 'n_batch': 160, 'duration': 144.06322360038757, 'duration_batch': 0.9003951475024223, 'duration_size': 0.45019757375121117, 'avg_pred_std': 0.07791482849102067}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0020939761153385915, 'avg_role_model_std_loss': 7.381703513306002, 'avg_role_model_mean_pred_loss': 3.89069975454473e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0020939761153385915, 'n_size': 80, 'n_batch': 40, 'duration': 33.347615242004395, 'duration_batch': 0.8336903810501098, 'duration_size': 0.4168451905250549, 'avg_pred_std': 0.01896329457867978}\n", + "Epoch 12\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0013007374736943688, 'avg_role_model_std_loss': 0.8016777972321465, 'avg_role_model_mean_pred_loss': 1.6661171600203944e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0013007374736943688, 'n_size': 320, 'n_batch': 160, 'duration': 143.7109453678131, 'duration_batch': 0.8981934085488319, 'duration_size': 0.44909670427441595, 'avg_pred_std': 0.07403813572964282}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0021091806715048734, 'avg_role_model_std_loss': 2.195618169948898, 'avg_role_model_mean_pred_loss': 6.716770201746968e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0021091806715048734, 'n_size': 80, 'n_batch': 40, 'duration': 33.3885440826416, 'duration_batch': 0.8347136020660401, 'duration_size': 0.41735680103302003, 'avg_pred_std': 0.03174531738768564}\n", + "Epoch 13\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0011258274745216568, 'avg_role_model_std_loss': 0.9933245053406304, 'avg_role_model_mean_pred_loss': 1.2559000061217402e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0011258274745216568, 'n_size': 320, 'n_batch': 160, 'duration': 143.82260847091675, 'duration_batch': 0.8988913029432297, 'duration_size': 0.44944565147161486, 'avg_pred_std': 0.07367398725546082}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002973305231353152, 'avg_role_model_std_loss': 2.332661612354639, 'avg_role_model_mean_pred_loss': 1.9470425126388857e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002973305231353152, 'n_size': 80, 'n_batch': 40, 'duration': 33.963603019714355, 'duration_batch': 0.8490900754928589, 'duration_size': 0.42454503774642943, 'avg_pred_std': 0.035911593766650186}\n", + "Epoch 14\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010081856245165, 'avg_role_model_std_loss': 1.8124559902420032, 'avg_role_model_mean_pred_loss': 1.1379342504010126e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010081856245165, 'n_size': 320, 'n_batch': 160, 'duration': 144.61974716186523, 'duration_batch': 0.9038734197616577, 'duration_size': 0.45193670988082885, 'avg_pred_std': 0.07099440268893886}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002199153335527626, 'avg_role_model_std_loss': 2.3161073656544886, 'avg_role_model_mean_pred_loss': 7.577638564465472e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002199153335527626, 'n_size': 80, 'n_batch': 40, 'duration': 33.57773303985596, 'duration_batch': 0.8394433259963989, 'duration_size': 0.41972166299819946, 'avg_pred_std': 0.02861409220568021}\n", + "Epoch 15\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0010586415690795547, 'avg_role_model_std_loss': 1.0111776571605908, 'avg_role_model_mean_pred_loss': 1.3155730025755604e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0010586415690795547, 'n_size': 320, 'n_batch': 160, 'duration': 143.99448657035828, 'duration_batch': 0.8999655410647392, 'duration_size': 0.4499827705323696, 'avg_pred_std': 0.07437738951684877}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0024275374956232556, 'avg_role_model_std_loss': 2.65694752669535, 'avg_role_model_mean_pred_loss': 1.2085388890881177e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0024275374956232556, 'n_size': 80, 'n_batch': 40, 'duration': 33.66980719566345, 'duration_batch': 0.8417451798915863, 'duration_size': 0.42087258994579313, 'avg_pred_std': 0.034823847954976374}\n", + "Epoch 16\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008538674877796026, 'avg_role_model_std_loss': 1.5765032050085541, 'avg_role_model_mean_pred_loss': 5.374222879224455e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008538674877796026, 'n_size': 320, 'n_batch': 160, 'duration': 143.34916639328003, 'duration_batch': 0.8959322899580002, 'duration_size': 0.4479661449790001, 'avg_pred_std': 0.08046830528701321}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022838078687641428, 'avg_role_model_std_loss': 2.0230109165978774, 'avg_role_model_mean_pred_loss': 9.47888851559331e-06, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022838078687641428, 'n_size': 80, 'n_batch': 40, 'duration': 33.02995800971985, 'duration_batch': 0.8257489502429962, 'duration_size': 0.4128744751214981, 'avg_pred_std': 0.030621698120376094}\n", + "Epoch 17\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0008248503026644372, 'avg_role_model_std_loss': 0.35676954403032646, 'avg_role_model_mean_pred_loss': 7.142933498651121e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0008248503026644372, 'n_size': 320, 'n_batch': 160, 'duration': 143.77559542655945, 'duration_batch': 0.8985974714159966, 'duration_size': 0.4492987357079983, 'avg_pred_std': 0.08080137882643612}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0023548384517198427, 'avg_role_model_std_loss': 4.805163549015765, 'avg_role_model_mean_pred_loss': 1.1883367263940125e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0023548384517198427, 'n_size': 80, 'n_batch': 40, 'duration': 33.76482057571411, 'duration_batch': 0.8441205143928527, 'duration_size': 0.42206025719642637, 'avg_pred_std': 0.03005029430896684}\n", + "Epoch 18\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007936206464748352, 'avg_role_model_std_loss': 1.0760348675862972, 'avg_role_model_mean_pred_loss': 7.879423535514518e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007936206464748352, 'n_size': 320, 'n_batch': 160, 'duration': 144.43712854385376, 'duration_batch': 0.902732053399086, 'duration_size': 0.451366026699543, 'avg_pred_std': 0.0707601236276787}\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.0022624223027378322, 'avg_role_model_std_loss': 4.505487352633622, 'avg_role_model_mean_pred_loss': 1.0153568444593031e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0022624223027378322, 'n_size': 80, 'n_batch': 40, 'duration': 33.66985249519348, 'duration_batch': 0.841746312379837, 'duration_size': 0.4208731561899185, 'avg_pred_std': 0.03378975939194788}\n", + "Epoch 19\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Train loss {'avg_role_model_loss': 0.0007593693141508595, 'avg_role_model_std_loss': 0.6788086620792548, 'avg_role_model_mean_pred_loss': 5.805468950893806e-07, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.0007593693141508595, 'n_size': 320, 'n_batch': 160, 'duration': 144.87138056755066, 'duration_batch': 0.9054461285471916, 'duration_size': 0.4527230642735958, 'avg_pred_std': 0.08014883432224451}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: - 0.000 MB of 0.000 MB uploaded\r" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Val loss {'avg_role_model_loss': 0.002318795861719991, 'avg_role_model_std_loss': 2.6745841488044872, 'avg_role_model_mean_pred_loss': 1.0226613121978867e-05, 'avg_role_model_g_mag_loss': 0.0, 'avg_role_model_g_cos_loss': 0.0, 'avg_non_role_model_g_mag_loss': 0.0, 'avg_non_role_model_g_cos_loss': 0.0, 'avg_non_role_model_embed_loss': 0.0, 'avg_loss': 0.002318795861719991, 'n_size': 80, 'n_batch': 40, 'duration': 33.691651344299316, 'duration_batch': 0.842291283607483, 'duration_size': 0.4211456418037415, 'avg_pred_std': 0.03418012205511332}\n", + "Time out: 3627.9625329971313/3600\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test █▂▂▂▂▁▁▂▁▁▁▁▁▂▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train █▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test ▆▄▅▃▁▂█▂▇▅█▂▆█▅▇▆▆▇\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train █▂▁▁▄▃▂▄▄▄▃▄▃▃▃▃▅▅▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test █▂▂▂▂▁▁▂▁▁▁▁▁▂▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train █▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test █▁▁▂▁▁▁▁▁▁▁▁▁▂▁▂▁▂▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test █▆▆▅▆▂▂▅▅▂▂█▁▁▁▂▁▅▄\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train ▄▆█▇▅▄▅▃▄▃▃▃▂▃▅▃▄▁▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test ▂▂▆▁▄▁▂█▆▃▂▂▂▃▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train ▂▁▃▇▃▁▂▇█▆▄▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test ▂▂▆▁▄▁▂█▆▃▂▂▂▃▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train ▂▁▃▇▃▁▂▇█▆▄▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test ▂▂▆▁▄▁▂█▆▃▂▂▂▃▃▃▂▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train ▂▁▃▇▃▁▂▇█▆▄▃▃▃▃▃▃▃▃\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train ▁▁▁��▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_test 0.00226\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_loss_train 0.00079\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_embed_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_non_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_test 0.03379\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_pred_std_train 0.07076\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_cos_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_test 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_g_mag_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_test 0.00226\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_loss_train 0.00079\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_test 1e-05\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_mean_pred_loss_train 0.0\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_test 4.50549\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: avg_role_model_std_loss_train 1.07603\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_test 0.84175\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_batch_train 0.90273\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_test 0.42087\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_size_train 0.45137\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_test 33.66985\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: duration_train 144.43713\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_test 40\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_batch_train 160\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_test 80\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: n_size_train 320\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: You can sync this run to the cloud by running:\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[1mwandb sync /kaggle/working/eval/contraceptive/tvae/2/wandb/offline-run-20240229_222415-ekgllb0j\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/offline-run-20240229_222415-ekgllb0j/logs\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Eval loss {'role_model': 'tvae', 'n_size': 399, 'n_batch': 200, 'role_model_metrics': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'pred_duration': 3.313055992126465, 'grad_duration': 2.6909384727478027, 'total_duration': 6.003994464874268, 'pred_std': 0.057808153331279755, 'std_loss': 0.01108523365110159, 'mean_pred_loss': 9.105955314225866e-07, 'pred_rmse': 0.03504936397075653, 'pred_mae': 0.02793470025062561, 'pred_mape': 0.06429529935121536, 'grad_rmse': 0.040074676275253296, 'grad_mae': 0.03164428845047951, 'grad_mape': 0.6164292097091675}, 'non_role_model_metrics': {'avg_loss': 0, 'avg_g_mag_loss': 0, 'avg_g_cos_loss': 0, 'avg_pred_duration': 0, 'avg_grad_duration': 0, 'avg_total_duration': 0, 'avg_pred_std': 0, 'avg_std_loss': 0, 'avg_mean_pred_loss': 0}, 'avg_metrics': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'avg_pred_duration': 3.313055992126465, 'avg_grad_duration': 2.6909384727478027, 'avg_total_duration': 6.003994464874268, 'avg_pred_std': 0.057808153331279755, 'avg_std_loss': 0.01108523365110159, 'avg_mean_pred_loss': 9.105955314225866e-07}, 'min_metrics': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'pred_duration': 3.313055992126465, 'grad_duration': 2.6909384727478027, 'total_duration': 6.003994464874268, 'pred_std': 0.057808153331279755, 'std_loss': 0.01108523365110159, 'mean_pred_loss': 9.105955314225866e-07, 'pred_rmse': 0.03504936397075653, 'pred_mae': 0.02793470025062561, 'pred_mape': 0.06429529935121536, 'grad_rmse': 0.040074676275253296, 'grad_mae': 0.03164428845047951, 'grad_mape': 0.6164292097091675}, 'model_metrics': {'tvae': {'avg_loss': 0.0012284579364314312, 'avg_g_mag_loss': nan, 'avg_g_cos_loss': 0.010551982342433416, 'pred_duration': 3.313055992126465, 'grad_duration': 2.6909384727478027, 'total_duration': 6.003994464874268, 'pred_std': 0.057808153331279755, 'std_loss': 0.01108523365110159, 'mean_pred_loss': 9.105955314225866e-07, 'pred_rmse': 0.03504936397075653, 'pred_mae': 0.02793470025062561, 'pred_mape': 0.06429529935121536, 'grad_rmse': 0.040074676275253296, 'grad_mae': 0.03164428845047951, 'grad_mape': 0.6164292097091675}}}\n" + ] + } + ], + "source": [ + "import torch\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import train, train_2\n", + "from ml_utility_loss.loss_learning.estimator.process_simple import train_epoch, eval as _eval\n", + "from ml_utility_loss.params import GradientPenaltyMode\n", + "from ml_utility_loss.util import clear_memory\n", + "import time\n", + "#torch.autograd.set_detect_anomaly(True)\n", + "\n", + "clear_memory()\n", + "\n", + "opt = params[\"Optim\"](model.parameters())\n", + "loss = train_2(\n", + " [train_set, val_set, test_set],\n", + " preprocessor=preprocessor,\n", + " whole_model=model,\n", + " optim=opt,\n", + " log_dir=\"logs\",\n", + " checkpoint_dir=\"checkpoints\",\n", + " verbose=True,\n", + " allow_same_prediction=False,\n", + " wandb=wandb,\n", + " study_name=study_name,\n", + " **params\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "9b514a07", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:38.426125Z", + "iopub.status.busy": "2024-02-29T23:27:38.425799Z", + "iopub.status.idle": "2024-02-29T23:27:38.430193Z", + "shell.execute_reply": "2024-02-29T23:27:38.429306Z" + }, + "papermill": { + "duration": 0.0277, + "end_time": "2024-02-29T23:27:38.432267", + "exception": false, + "start_time": "2024-02-29T23:27:38.404567", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "model = loss[\"whole_model\"]\n", + "opt = loss[\"optim\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "331a49e1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:38.470452Z", + "iopub.status.busy": "2024-02-29T23:27:38.470104Z", + "iopub.status.idle": "2024-02-29T23:27:38.772666Z", + "shell.execute_reply": "2024-02-29T23:27:38.771746Z" + }, + "papermill": { + "duration": 0.324535, + "end_time": "2024-02-29T23:27:38.775289", + "exception": false, + "start_time": "2024-02-29T23:27:38.450754", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import torch\n", + "from copy import deepcopy\n", + "\n", + "torch.save(deepcopy(model.state_dict()), \"model.pt\")\n", + "torch.save(deepcopy(opt.state_dict()), \"optim.pt\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "123b4b17", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:38.818515Z", + "iopub.status.busy": "2024-02-29T23:27:38.818096Z", + "iopub.status.idle": "2024-02-29T23:27:39.115381Z", + "shell.execute_reply": "2024-02-29T23:27:39.114363Z" + }, + "papermill": { + "duration": 0.321667, + "end_time": "2024-02-29T23:27:39.117472", + "exception": false, + "start_time": "2024-02-29T23:27:38.795805", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAS0AAAESCAYAAACoz4OWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA3PElEQVR4nO3dfVxUZd4/8M+ZZx5nBIRhFAEVwgdE0yA0tZINjSxc73S52TR/JtpKD1Kra5tQu/uKsrrX1bxrq12929U0N2t31SxDzVUQFEExldRQUBgQieFxmGHm+v1xYGRggJkBmRn4vl+v85qZc6455zrAfLjOOddch2OMMRBCiIsQOLoChBBiCwotQohLodAihLgUCi1CiEuh0CKEuBQKLUKIS6HQIoS4FJGjKzBQjEYjysvL4eXlBY7jHF0dQkgnjDHU19dDpVJBIOi+PTVkQqu8vBxBQUGOrgYhpBdlZWUYOXJkt8uHTGh5eXkB4H8g3t7eDq4NIaSzuro6BAUFmT6r3RkyodV+SOjt7U2hRYgT6+30DZ2IJ4S4FAotQohLodAihLiUIXNOi/SNwWCAXq93dDWICxOLxRAKhX1eD4UW6RFjDGq1GrW1tY6uChkEFAoFlEpln/pKUmiRHrUHlr+/P9zd3aljLrELYwxNTU2oqqoCAAQGBtq9LgqtTjRNelyoqINQwCE61MfR1XEog8FgCixfX19HV4e4ODc3NwBAVVUV/P397T5UpBPxnZy9UYukj04i/Z/nHV0Vh2s/h+Xu7u7gmpDBov1vqS/nRym0OpG7iQEAdc100rkdHRKS/tIff0sUWp20h5aGQosQp0Sh1Ul7aDXqDNAbjA6uDSGkMwqtTrzbQgugQ0RiP47j8OWXXzq6Gv3qtddew+TJkx1dDQqtzoQCDl5S/qIqHSISV7Z9+3YoFIp+W9/LL7+MrKysflufvSi0LPCm81pkCNHpdFaV8/T0dIquLxRaFlBoWcYYQ5Ou1SGTrTdCP3jwIB544AEoFAr4+vrisccew9WrVwEA06dPx7p168zK37p1C2KxGMeOHQMAVFRUICEhAW5ubggNDcXOnTsREhKCTZs22fWzKyoqwsMPPww3Nzf4+voiJSUFDQ0NpuVHjx5FdHQ0PDw8oFAoMGPGDFy/fh0AcPbsWTz00EPw8vKCt7c3pk6ditOnT/e4vaNHj2LZsmXQaDTgOA4cx+G1114DAISEhOD3v/89lixZAm9vb6SkpAAA1q1bh/DwcLi7u2P06NHYsGGDWdeEzoeHTz/9NBITE/HOO+8gMDAQvr6+WL169V3/uhd1LrVA7kaHh5Y06w0Yn/61Q7Z94XfxcJdY/+fa2NiItLQ0TJo0CQ0NDUhPT8eCBQtQWFiI5ORkbNy4EW+++abpEvzu3buhUqkwc+ZMAMCSJUtQXV2No0ePQiwWIy0tzdSb21aNjY2Ij49HbGwsTp06haqqKjzzzDNITU3F9u3b0draisTERKxYsQKffvopdDod8vLyTHVLTk7GlClT8P7770MoFKKwsBBisbjHbU6fPh2bNm1Ceno6iouLAfAtpXbvvPMO0tPTkZGRYZrn5eWF7du3Q6VSoaioCCtWrICXlxfWrl3b7XaOHDmCwMBAHDlyBFeuXMHixYsxefJkrFixwq6flTUotCygvlqub+HChWav//rXv2L48OG4cOECFi1ahBdffBHHjx83hdTOnTuRlJQEjuNw6dIlfPvttzh16hSmTZsGAPj4448RFhZmV1127twJrVaLTz75BB4eHgCA9957D/Pnz8dbb70FsVgMjUaDxx57DGPGjAEAjBs3zvT+0tJS/PrXv0ZERAQAWFUPiUQCuVwOjuOgVCq7LH/44Yfx0ksvmc179dVXTc9DQkLw8ssvY9euXT2G1rBhw/Dee+9BKBQiIiICCQkJyMrKotAaaNRXyzI3sRAXfhfvsG3b4vLly0hPT0dubi6qq6thNPLdV0pLSzFx4kQ88sgj2LFjB2bOnImSkhLk5OTgz3/+MwCguLgYIpEI9957r2l9Y8eOxbBhw+yq+8WLFxEVFWUKLACYMWMGjEYjiouLMWvWLDz99NOIj4/Hz372M8TFxWHRokWm7+elpaXhmWeewd/+9jfExcXhySefNIWbvdrDuKPdu3dj8+bNuHr1KhoaGtDa2trrKL8TJkww+zpOYGAgioqK+lS33tA5LQsotCzjOA7uEpFDJlt7Us+fPx81NTX46KOPkJubi9zcXAB3TjonJyfjH//4B/R6PXbu3InIyEhERkb2+8/MWtu2bUNOTg6mT5+O3bt3Izw8HCdPngTAn0v6/vvvkZCQgMOHD2P8+PH44osv+rS9jgEKADk5OUhOTsajjz6Kffv2oaCgAL/97W97PUnf+TCV4zjTP4i7hULLAgot13b79m0UFxfj1VdfxZw5czBu3Dj89NNPZmWeeOIJaLVaHDx4EDt37kRycrJp2T333IPW1lYUFBSY5l25cqXLOqw1btw4nD17Fo2NjaZ5J06cgEAgwD333GOaN2XKFKxfvx7Z2dmYOHEidu7caVoWHh6ONWvW4JtvvsHPf/5zbNu2rdftSiQSGAwGq+qYnZ2N4OBg/Pa3v8W0adMQFhZmuhDgbCi0LKDQcm3Dhg2Dr68vPvzwQ1y5cgWHDx9GWlqaWRkPDw8kJiZiw4YNuHjxIpKSkkzLIiIiEBcXh5SUFOTl5aGgoAApKSlwc3Oz67tzycnJkMlkWLp0Kc6fP48jR47gueeew1NPPYWAgACUlJRg/fr1yMnJwfXr1/HNN9/g8uXLGDduHJqbm5GamoqjR4/i+vXrOHHiBE6dOmV2zqs7ISEhaGhoQFZWFqqrq9HU1NRt2bCwMJSWlmLXrl24evUqNm/e3OfW3N1CoWUBdXlwbQKBALt27UJ+fj4mTpyINWvW4O233+5SLjk5GWfPnsXMmTMxatQos2WffPIJAgICMGvWLCxYsMB0JU0mk9lcH3d3d3z99deoqanBfffdh//6r//CnDlz8N5775mWX7p0CQsXLkR4eDhSUlKwevVqrFy5EkKhELdv38aSJUsQHh6ORYsWYd68eXj99dd73e706dOxatUqLF68GMOHD8fGjRu7Lfv4449jzZo1SE1NxeTJk5GdnY0NGzbYvK8DgWO2doBxUXV1dZDL5dBoNL2eXDxaXIWnt53CuEBvfPXCzAGqofPRarUoKSlBaGioXR/WweTGjRsICgrCt99+izlz5ji6Oi6rp78paz+jdPXQAuryQA4fPoyGhgZERkaioqICa9euRUhICGbNmuXoqg15dHhoAYUW0ev1eOWVVzBhwgQsWLAAw4cPN3U03bFjBzw9PS1OEyZMGLA6zps3r9t6vPHGGwNWj4FGLS0L2kOrvqUVBiODUECD4A018fHxiI+33Cft8ccfR0xMjMVlvfVU708ff/wxmpubLS7z8Rm8Q4VTaFnQeXiaYR4SB9aGOBsvLy94eXk5uhoYMWKEo6vgEHYdHm7duhUhISGQyWSIiYlBXl5ej+X37NmDiIgIyGQyREZG4sCBA6Zler0e69atQ2RkJDw8PKBSqbBkyRKUl5ebraOmpgbJycnw9vaGQqHA8uXLzb5w2p/EQgE8JHwvX7qCSIhzsTm0du/ejbS0NGRkZODMmTOIiopCfHx8t18mzc7ORlJSEpYvX46CggIkJiYiMTER58/zN45oamrCmTNnsGHDBpw5cwZ79+5FcXExHn/8cbP1JCcn4/vvv8ehQ4ewb98+HDt2zPTt9LuB+moR4qSYjaKjo9nq1atNrw0GA1OpVCwzM9Ni+UWLFrGEhASzeTExMWzlypXdbiMvL48BYNevX2eMMXbhwgUGgJ06dcpU5quvvmIcx7GbN29aVW+NRsMAMI1GY1X5+D9+x4LX7WPfFVdZVX4wam5uZhcuXGDNzc2OrgoZJHr6m7L2M2pTS0un0yE/Px9xcXGmeQKBAHFxccjJybH4npycHLPyAH+Ss7vyAExjALWPupiTkwOFQmH2Jc+4uDgIBALTd8o6a2lpQV1dndlkC2ppEeKcbAqt6upqGAwGBAQEmM0PCAiAWq22+B61Wm1Tea1Wi3Xr1iEpKcnUwUytVsPf39+snEgkgo+PT7fryczMhFwuN01BQUFW7WM7Ci1CnJNT9dPS6/VYtGgRGGN4//33+7Su9evXQ6PRmKaysjKb3k+hRfpiMN7YwlnYFFp+fn4QCoWorKw0m19ZWWlxoDEAUCqVVpVvD6zr16/j0KFDZt34lUpllxP9ra2tqKmp6Xa7UqkU3t7eZpMtqIMpcXX9fWMLgB/GmeM41NbW9ut6bWFTaEkkEkydOtXsjhxGoxFZWVmIjY21+J7Y2Ngud/A4dOiQWfn2wLp8+TK+/fbbLoPnx8bGora2Fvn5+aZ5hw8fhtFo7LaTX19RS4sQJ2Xr2f9du3YxqVTKtm/fzi5cuMBSUlKYQqFgarWaMcbYU089xX7zm9+Yyp84cYKJRCL2zjvvsIsXL7KMjAwmFotZUVERY4wxnU7HHn/8cTZy5EhWWFjIKioqTFNLS4tpPXPnzmVTpkxhubm57Pjx4ywsLIwlJSVZXW9brx7+X3YJC163j63622mrtzHYdLnSYzQy1tLgmMlotKnuX331FZsxYwaTy+XMx8eHJSQksCtXrjDGGIuNjWVr1641K19VVcVEIhH77rvvGGOMlZeXs0cffZTJZDIWEhLCduzYwYKDg9kf//hHq7YPgH3xxRem1+fOnWMPPfQQk8lkzMfHh61YsYLV19eblh85coTdd999zN3dncnlcjZ9+nR27do1xhhjhYWF7MEHH2Senp7My8uL3XvvvWZX0i05cuQIA2A2ZWRkMMYY02q17KWXXmIqlYq5u7uz6OhoduTIEdN7r127xh577DGmUCiYu7s7Gz9+PNu/fz8rKSnpss6lS5da9fNo1x9XD23uEb948WLcunUL6enpUKvVmDx5Mg4ePGg62V5aWgqB4E4Dbvr06di5cydeffVVvPLKKwgLC8OXX36JiRMnAgBu3ryJf/3rXwDQ5UaQR44cwYMPPggA2LFjB1JTUzFnzhwIBAIsXLgQmzdvtrX6VqOWlgX6JuANlWO2/Uo5IPHovVwburFF9ze2SE1NxYULF7Br1y6oVCp88cUXmDt3LoqKihAWFobVq1dDp9Ph2LFj8PDwwIULF+Dp6YmgoCB8/vnnWLhwIYqLi+Ht7Q03Nze7fiZ9YdfXeFJTU5Gammpx2dGjR7vMe/LJJ/Hkk09aLB8SEmLV7aF8fHzMRnK822hMLddGN7awfGOL0tJSbNu2DaWlpVCp+H9AL7/8Mg4ePIht27bhjTfeQGlpKRYuXGgafnr06NGm97d/p9Hf37/fz5dZi7572A1qaVkgdudbPI7atg3oxhaWFRUVwWAwIDw83Gx+S0uL6Vzy888/j2effRbffPMN4uLisHDhQkyaNMmu7d0NTtXlwZl4yyi0uuA4/hDNERPd2KJfbmzR0NAAoVCI/Px8FBYWmqaLFy/iT3/6EwDgmWeewY8//oinnnoKRUVFmDZtGrZs2dJv+9pXFFrdMA1Po+WHpyGug25swbN0Y4spU6bAYDCgqqoKY8eONZs6HkYGBQVh1apV2Lt3L1566SV89NFHpnUCsPqGGXcDhVY35B2Gp6nXUmvLldCNLXiWbmwRHh6O5ORkLFmyBHv37kVJSQny8vKQmZmJ/fv3AwBefPFFfP311ygpKcGZM2dw5MgR0/aCg4PBcRz27duHW7du3bWRVnpk0/VKF2ZrlwfGGIt49SsWvG4fu1bdcBdr5rxc+QvThw4dYuPGjWNSqZRNmjSJHT16tEs3hAMHDjAAbNasWV3eX15ezubNm8ekUikLDg5mO3fuZP7+/uyDDz6wavudt9VTlwe1Ws0SExNZYGAgk0gkLDg4mKWnpzODwcBaWlrYL37xCxYUFMQkEglTqVQsNTXV6t/JqlWrmK+vr1mXB51Ox9LT01lISAgTi8UsMDCQLViwgJ07d44xxlhqaiobM2YMk0qlbPjw4eypp55i1dXVpnX+7ne/Y0qlknEc55AuD3Rjix7c/0YW1HVa/Ct1BiaNVNzdCjohurHFHXRji/5BN7a4y+RuYqjrtHQyfgiiG1s4Lzqn1QPq9jB00Y0tnBe1tHpAHUyHLrqxhfOi0OoBtbSIJXRjC8eiw8MeUGjxhsi1GjIA+uNviUKrB0N9TK32Q52mpiYH14QMFu1/S305jKbDwx7I3fgfz1BtaQmFQigUCtPoBu7u7nZ1riSEMYampiZUVVVBoVBAKBTavS4KrR7I3enwsP2rHfYOy0JIRwqFotvRhq1FodWDO4eHrQ6uieNwHIfAwED4+/tDrx+64U36TiwW96mF1Y5Cqwd0Iv4OoVDYL39whPQVnYjvAYUWIc6HQqsH7Z1L67R6GGl4GkKcAoVWD9pbWowB9S1D97wWIc6EQqsHUpEQMjH/IxqqfbUIcTYUWr2g81qEOBcKrV5QaBHiXCi0ekGhRYhzodDqBYUWIc6FQqsXNKYWIc6FQqsX1NIixLlQaPWCbtpKiHOh0OoFtbQIcS4UWr0Y6gMBEuJsKLR6QS0tQpwLhVYvaCBAQpwLhVYvqKVFiHOh0OpFx3NaNDwNIY5HodWL9tAyMqBBR8PTEOJoFFq9kImFkIj4H5OmiQ4RCXE0Ci0r0HktQpwHhZYVqK8WIc6DQssK1NIixHnYFVpbt25FSEgIZDIZYmJikJeX12P5PXv2ICIiAjKZDJGRkThw4IDZ8r179+KRRx6Br68vOI5DYWFhl3U8+OCD4DjObFq1apU91bcZhRYhzsPm0Nq9ezfS0tKQkZGBM2fOICoqCvHx8d3egTg7OxtJSUlYvnw5CgoKkJiYiMTERJw/f95UprGxEQ888ADeeuutHre9YsUKVFRUmKaNGzfaWn27UGgR4kSYjaKjo9nq1atNrw0GA1OpVCwzM9Ni+UWLFrGEhASzeTExMWzlypVdypaUlDAArKCgoMuy2bNnsxdeeMHW6ppoNBoGgGk0Gpvfm/HP8yx43T721lcX7d4+IaRn1n5GbWpp6XQ65OfnIy4uzjRPIBAgLi4OOTk5Ft+Tk5NjVh4A4uPjuy3fkx07dsDPzw8TJ07E+vXr0dTU1G3ZlpYW1NXVmU32ooEACXEeIlsKV1dXw2AwICAgwGx+QEAALl26ZPE9arXaYnm1Wm1TRf/7v/8bwcHBUKlUOHfuHNatW4fi4mLs3bvXYvnMzEy8/vrrNm2jO6arh1rqXEqIo9kUWo6UkpJieh4ZGYnAwEDMmTMHV69exZgxY7qUX79+PdLS0kyv6+rqEBQUZNe26ZwWIc7DptDy8/ODUChEZWWl2fzKykoolUqL71EqlTaVt1ZMTAwA4MqVKxZDSyqVQiqV9mkb7Si0CHEeNp3TkkgkmDp1KrKyskzzjEYjsrKyEBsba/E9sbGxZuUB4NChQ92Wt1Z7t4jAwMA+rcca1LmUEOdh8+FhWloali5dimnTpiE6OhqbNm1CY2Mjli1bBgBYsmQJRowYgczMTADACy+8gNmzZ+Pdd99FQkICdu3ahdOnT+PDDz80rbOmpgalpaUoLy8HABQXFwPgW2lKpRJXr17Fzp078eijj8LX1xfnzp3DmjVrMGvWLEyaNKnPP4TeUEuLECdiz6XJLVu2sFGjRjGJRMKio6PZyZMnTctmz57Nli5dalb+s88+Y+Hh4UwikbAJEyaw/fv3my3ftm0bA9BlysjIYIwxVlpaymbNmsV8fHyYVCplY8eOZb/+9a9t6r7Qly4PFbXNLHjdPjZ6/X5mNBptfj8hpHfWfkY5xtiQGCSqrq4OcrkcGo0G3t7eNr23WWfAuPSDAIDzr8fDU+oy1y8IcRnWfkbpu4dWkIkFkAjbhqehQ0RCHIpCywocx93pYEpjahHiUBRaVvJ24w8JqaVFiGNRaFmJriAS4hwotKxEfbUIcQ4UWlailhYhzoFCy0oUWoQ4BwotK1FoEeIcKLSsRKFFiHOg0LISDQRIiHOg0LIStbQIcQ4UWlaiLg+EOAcKLStRS4sQ50ChZaWOoTVEBsYgxClRaFmpPbRajQxNOoODa0PI0EWhZSV3iRAiAQeADhEJcSQKLStxHEfntQhxAhRaNqDQIsTxKLRsQB1MCXE8Ci0bUEuLEMej0LIBdTAlxPEotGxAoUWI41Fo2YAODwlxPAotG1BoEeJ4FFo2oNAixPEotGxAXR4IcTwKLRtQS4sQx6PQssGdG7a2OrgmhAxdFFo26NjlgYanIcQxKLRs0B5aOoMRWr3RwbUhZGii0LKBp1QEIQ1PQ4hDUWjZgOM4eMvaz2tRaBHiCBRaNqIriIQ4FoWWjSi0CHEsCi0bUQdTQhyLQstG1NIixLEotGxEoUWIY1Fo2YjG1CLEsewKra1btyIkJAQymQwxMTHIy8vrsfyePXsQEREBmUyGyMhIHDhwwGz53r178cgjj8DX1xccx6GwsLDLOrRaLVavXg1fX194enpi4cKFqKystKf6fUItLUIcy+bQ2r17N9LS0pCRkYEzZ84gKioK8fHxqKqqslg+OzsbSUlJWL58OQoKCpCYmIjExEScP3/eVKaxsREPPPAA3nrrrW63u2bNGvz73//Gnj178N1336G8vBw///nPba1+n1FoEeJgzEbR0dFs9erVptcGg4GpVCqWmZlpsfyiRYtYQkKC2byYmBi2cuXKLmVLSkoYAFZQUGA2v7a2lonFYrZnzx7TvIsXLzIALCcnx+J2tVot02g0pqmsrIwBYBqNxtpdtejAuXIWvG4f+/n/nujTeggh5jQajVWfUZtaWjqdDvn5+YiLizPNEwgEiIuLQ05OjsX35OTkmJUHgPj4+G7LW5Kfnw+9Xm+2noiICIwaNarb9WRmZkIul5umoKAgq7fXE2ppEeJYNoVWdXU1DAYDAgICzOYHBARArVZbfI9arbapfHfrkEgkUCgUVq9n/fr10Gg0pqmsrMzq7fWE+mkR4lgiR1fgbpFKpZBKpf2+XmppEeJYNrW0/Pz8IBQKu1y1q6yshFKptPgepVJpU/nu1qHT6VBbW9un9fQHuXvb8DStRmj1hgHdNiHExtCSSCSYOnUqsrKyTPOMRiOysrIQGxtr8T2xsbFm5QHg0KFD3Za3ZOrUqRCLxWbrKS4uRmlpqU3r6Q+eEhHaRqeh1hYhDmDz4WFaWhqWLl2KadOmITo6Gps2bUJjYyOWLVsGAFiyZAlGjBiBzMxMAMALL7yA2bNn491330VCQgJ27dqF06dP48MPPzSts6amBqWlpSgvLwfABxLAt7CUSiXkcjmWL1+OtLQ0+Pj4wNvbG8899xxiY2Nx//339/mHYAuBgIO3mxi1TXpomvUI8JYN6PYJGfLsuTS5ZcsWNmrUKCaRSFh0dDQ7efKkadns2bPZ0qVLzcp/9tlnLDw8nEkkEjZhwgS2f/9+s+Xbtm1jALpMGRkZpjLNzc3sV7/6FRs2bBhzd3dnCxYsYBUVFVbX2drLqdaYtfEwC163j+WV3O7zugghPGs/oxxjQ2Ow87q6Osjlcmg0Gnh7e/dpXY+/dxznbmjw8ZJpiBsf0PsbCCG9svYzSt89tIPp+4daOqdFyECj0LID9dUixHEotOxAfbUIcRwKLTt4yyi0CHEUCi07UEuLEMeh0LIDDQRIiONQaNmBWlqEOA6Flh0otAhxHAotO1BoEeI4FFp2oNAixHEotOzQHlpavREtrTQ8DSEDiULLDl4yETganoYQh6DQsoNAwMFLyo/qQ90eCBlYFFp2ah/BlFpahAwsCi070cl4QhyDQqszoxGoOAdc7/kWZxRahDgGhVZnRXuAP88Evnm1x2Km0Gqi0CJkIFFodRYyg38sPwNoNd0Wu9PSah2IWhFC2lBodSYfCfiMBpixx0NEGgiQEMeg0LIkdBb/WHKs2yJ0TosQx6DQsqQ9tK5RaBHibCi0LAmZyT+qi4CmGotFaEwtQhyDQssST3/Afzz//Np/LBahlhYhjkGh1Z321lY357UotAhxDAqt7vRyMp5CixDHoNDqTsgMABxQ/QNQr+6yuD20mvUG6FqNA1w5QoYuCq3uuA0DAqP45yVdz2t5td1GDKDWFiEDiUKrJ6ZDxO+6LBIKOHjJ+OFpKLQIGTgUWj3p5bxW+01b67QUWoQMFAqtnoy6HxCIgNrrwE/Xuyymk/GEDDwKrZ5IvYARU/nnFvprUQdTQgYehVZvejhEpJYWIQOPQqs3HTuZMma2iMbUImTgUWj1JigaEEqB+grg9hWzRTROPCEDj0KrN2I3PriALoeIdHhIyMCj0LJG6Gz+sVNo0UCAhAw8Ci1rmMbX+g9/44s21NIiZODZFVpbt25FSEgIZDIZYmJikJeX12P5PXv2ICIiAjKZDJGRkThw4IDZcsYY0tPTERgYCDc3N8TFxeHy5ctmZUJCQsBxnNn05ptv2lN92424FxB7AE23gaoLptkUWoQMPJtDa/fu3UhLS0NGRgbOnDmDqKgoxMfHo6qqymL57OxsJCUlYfny5SgoKEBiYiISExNx/vx5U5mNGzdi8+bN+OCDD5CbmwsPDw/Ex8dDq9Waret3v/sdKioqTNNzzz1na/XtIxQDwbH88w6HiNRPixAHYDaKjo5mq1evNr02GAxMpVKxzMxMi+UXLVrEEhISzObFxMSwlStXMsYYMxqNTKlUsrffftu0vLa2lkmlUvbpp5+a5gUHB7M//vGPtlbXRKPRMABMo9HYt4LjmxjL8GZs5y9Ms3681cCC1+1j4zd8ZXe9CCE8az+jNrW0dDod8vPzERcXZ5onEAgQFxeHnBzLd67JyckxKw8A8fHxpvIlJSVQq9VmZeRyOWJiYrqs880334Svry+mTJmCt99+G62t3d++q6WlBXV1dWZTn5jOax0HDPx221tajToD9AYanoaQgSCypXB1dTUMBgMCAgLM5gcEBODSpUsW36NWqy2WV6vVpuXt87orAwDPP/887r33Xvj4+CA7Oxvr169HRUUF/ud//sfidjMzM/H666/bsns9U04CZHL+Xojqs8CIqfCW3fnx1TXr4esp7b/tEUIscpmrh2lpaXjwwQcxadIkrFq1Cu+++y62bNmClpYWi+XXr18PjUZjmsrKyvpWAYEQCH6Af952XkskFMBTSsPTEDKQbAotPz8/CIVCVFZWms2vrKyEUqm0+B6lUtlj+fZHW9YJADExMWhtbcW1a9csLpdKpfD29jab+sz0PcQ7X56mK4iEDCybQksikWDq1KnIysoyzTMajcjKykJsbKzF98TGxpqVB4BDhw6ZyoeGhkKpVJqVqaurQ25ubrfrBIDCwkIIBAL4+/vbsgt90x5apTlAqw4AdTAlZKDZdE4L4A/Tli5dimnTpiE6OhqbNm1CY2Mjli1bBgBYsmQJRowYgczMTADACy+8gNmzZ+Pdd99FQkICdu3ahdOnT+PDDz8EAHAchxdffBF/+MMfEBYWhtDQUGzYsAEqlQqJiYkA+JP5ubm5eOihh+Dl5YWcnBysWbMGv/zlLzFs2LB++lFYwX8c4O4HNFUDN/OB4FjI3ejwkJCBZHNoLV68GLdu3UJ6ejrUajUmT56MgwcPmk6kl5aWQiC404CbPn06du7ciVdffRWvvPIKwsLC8OWXX2LixImmMmvXrkVjYyNSUlJQW1uLBx54AAcPHoRMJgPAH+rt2rULr732GlpaWhAaGoo1a9YgLS2tr/tvG44DQmcC33/Bn9cKjqW+WoQMMI6xTuOtDFJ1dXWQy+XQaDR9O791+q/AvjX8Sfll+7H2H2fx2ekbePmRcKQ+HNZ/FSZkiLH2M+oyVw+dRvuXp2/kAfpmU0vrTGkt3UqMkAFAoWUrn9GA9wjAoAPKcjE7nL8QcPhSFZI+OonKOm0vKyCE9AWFlq04zmwI5gfC/PCXpdPgJRMh//pPeGzLcZy6VuPYOhIyiFFo2aPjEMwA5owLwL9SH0B4gCdu1bcg6cOT+CTnGobI6UJCBhSFlj1C20Lr5hmgpZ6f5eeBL341AwmTAtFqZEj/5/d4ec85aPUGB1aUkMGHQsseilHAsFCAGYDrd77U7SEV4b2kKXjl0QgIOODzMzew8P1slNU02bUZxhguqetwrbqxv2pOiMuj0LKX6bzWd2azOY5Dyqwx+PvyGPh4SPB9eR0ef+84jl+utmq1Wr0BRy5V4bdfFGH6m4cxd9N/8LM/focduV1vFkvIUET9tOxV9A/g8+X86A+rut7IFQBu1jbj2b/n49wNDQQcsHZuBFbOGg2O48zKVdVrceRSFb69WIXjl6vR3OGQUiTg0Grkf0VJ0UF47fEJkIqEfa8/IU7G2s8ohZa96iuBd8MBcMDaHwF3H4vFtHoDNnx5HnvybwAA5k1U4u0no3D9diOyLlYh62Ilzt7QmL0nUC7DwxH+iBsXgNgxvvjriRK8/XUxGAMmBynwwS+nQimX9X0fCHEiFFqd9HtoAcDWGODWJWDx34Fx87stxhjDjtxSvP7v76E3MEhEgi4dUaNGyjFnXAAejvDHBJU3OM0NoCwXKMsDDC24YlTh3QLgnDYAek8V/veX0zAtxHJQEuKKrP2M2vzdQ9JB6Cw+tEqO9RhaHMfhl/cHY1ygN579ez6q6lsgEwvwwNjhiBvnj4fDfeDfdBkoOwxkn+TDqu6m2TrGAngfAGRAk16KH/+qwo8jxiM0Ygq44fcAw+/hO74KxXd1lwlxNGpp9cXFfwO7fwkMjwBW51r1ltomHX64fhOTuR8gKT8NlJ0EbuQD+k5XCDkhoIwERt0PSDyB6h+A6h/Abl8FZ+zmy9kCER9c4+YD058H3BR92z9ydzTcAn48AoTH86PhEgDU0hoYwTMAcHxrq74S8GobMlrfDGhuALWlgKaMf6wtAzRlUNSWIbruJoBO/yukciDoPiDofmBUDKC6F5B6dtkkZ9CD1ZTg6++OoagwD6O5m5gkrcQYrhwCfSMfbv95l/9i98yXgfueAcR0/stp/PA18OWv+OGNvAKBhHeBiARH18qlUEurrz6YCajP8WFj1PPh1Gj5dmpmhoXcCaig+/nWmsC2HijHfriF5z4tgKZZDz8PMT5aMAJTUAx89xYfpAAgDwIe+i0waRE/ZDRxDH0zcCgdyOPHkYNAzP+9AMD4J4B5GwGv7kfqHQroRHwndy20DqUDJ/7Udb7Ekw8MRVDb46i256MAn1DAw69fNl96uwkpfzuNS+p6iAQc0uePx1PRI8Gd2wUceePOuTH/CUDca0DYz/jvTw6E21f5R98xA7M9Z1X5PfD5M3du9BvzLPDgb/i/mxN/4jspS+XAI78H7l0ycL8fJ0Oh1cldC62mGuD0X/g7UCvawkkeBLgNG7A/viZdK9Z9XoR/ny0HADwc4Y9nHxyDaSoZuFMf8YeL2rZuFSEzgbjXgZFT705lGAOuHgayNwM/HuXnjX4ImJ4KjJkztD6QjAG5f+b/sRlaAA9/IPF9IKzDLfUqzgH/eg6oKORfh8wE5v9pSAY9hVYndy20nARjDB//pwSZX11EW19URAUpsGJmKOaOlkKUvYn/ABna7l40/gng4XTAb2z/VMCgB87vBbK3AJVF/DxOCIABrK17h/94IHY1EPkkIBrkt1trqOLPXV05xL8OewR44n8Bz+FdyxpagdwPgMN/AFqbAaGUb4lNf86+q8HNP/GBOYD/OPsDhVYngz202l2pasBfjv+Iz8/cNPUFGznMDf9vRigW3yOAx4mNwNmdfJBwQv5wZMICQDUFkNnxc2mpB/L/Dzj5PlDHd6CF2INf7/3PAmhrbZz5BNA18Ms9A4DoFcC05d12yu1XjAH1FYC2jm/B3O1uIT98A/zzV0DjLT6AHvkDv7+9BUhNCT8q7o9H+NcBkcDjm4ER91ouzxhQex1QF/FTxTn+sePvQT6SnxRBbc9H3XntpQKE3VyLa9UBLXV8C72ljv/ZtT+2NgMyBf+7c/MB3H3552L3PoUkhVYnQyW02lU3tOCTnOv4W841/NTEn/D1lonw3zHBeCa8GX65bwE/fNXhHRzf12vENP7QccRU/jxYd3/UdRV86+D0NqCl7dDTwx+ISbEcRs21wJn/A05+ANTzh7EQuQFTkoH7f9V/h0N6LX8RovI8oD7PP1ae51sfAB8iyolAYBQQOBlQTQaGjwNEkv7Z9qF0IO/P/Gv/8cDCvwAB461fB2PA2V3A1+v5OnMC/ucz69f8leiO4aQuuvOztwcn4IPLWwUYW83DqdWOwSxFsg4hNqzt0Zefd+8SPih7QKHVyVALrXbNOgM+P3MDfzlegpK20SLEQg7zo1R4fuwthFzdCdw4DWhKu75Z5MZ/qEe0hdjIaYCuiT8EPLf7ztUv3zD+nNWkX/TevcKg528Mkr2Fv+oKAOD4y/6xqXyYgPEf3m4fwT/qm/iAUhfxJ7srzwPVl/kT251xQkDsdqe115FQwgdMYBS/v4GTgYAJvR/CMgYYDfz2qn8A9qZ0ONm+ij93aG93k4ZbwMHfAOf/0XM5gZi/S5RyEt+vL3ASX3ehBNDc5INOc8P8sbaMv0Bj0PVeD4knIPXmW+HtjyIZ3wJrqgGaa4Cm272vK+Uo35rvAYVWJ0M1tNoZjQzfXqzEx/8pQV6HkVUfGOuHB+8Zjql+eow3XoZUXQDcPA3cLOj9v3jQ/cCM54HweTZ31wBjwLX/ANnvAZe/tmOPeuA2DAiYyH+IAybwz4dH8B/kn0r4k97lhUDFWf651sJ+CkR8K8HY2hZMxjsBZTTw8zv3tQMAj+FtJ9t/1j/78sPXwL40/pBPKuf3qT2clJGA3z32tRKNRr5rjuYGH2BCSddwknpb102GMf6fQVNbgDXX3Hne/vjgesvn8zqg0OpkqIdWR4VltfjoPz/iq6IK00l7ABBwQHiAF6JGKjBppBfu86rB6JZLEJXn8/d5rDzPf2AjEoAZLwBB0f1ToVvFQM5WvvVmy2EJJ+BbecqJfDAFTOSfewVaf26FMeCna3cCrLyQf2w/nLRF+Dz+HJRnP99A2KAHGqv5flwudGLdVhRanVBodVVW04R/Ft5EYZkGZ2/U4lZ9S5cyEpEAE1TeiBqpwORAKcJ9xXDz9oNUJIBMLIRUJIBUJIBI2A9Dsxn0QGtL2weTs+7xbnyIGeNbINpa/rBSIORbXpyAf25pnkAMSNz7vy5DCIVWJxRaPWOMQV2nxdkyDc7dqMW5G/xjnbbVqveLBBwfYGIhZG2PUpEACncxVAo3jGibVAo3jBjmBpXcDW4S6qFP7qDvHhKbcByHQLkbAuVumDuR/zqJ0chwvaYJ527UorCMD7Lrt5vQojegpdUIneHO8DqtRoZWnQGNOuvHxPfxkLQFmQwjFO5QKWQI9fPAmOGeCPJxh1AweA+FiP2opUXsZjAy6FqN0LaFWEurAVr9nUet3oCaRh1u1jajvLb5zuNPzb2Gm0QkwGg/D4zx98TY4Z4Y6++JMcM9MXq4B2RiaqENRtTSInedUMDBTSK0+TCPMYY6bStu/sSHWLmGD7IbPzXjx+pG/HirAS2tRlxS1+OSut7svRwHBA1zx1h/T4z284DCXQxPqQgeUhG8ZCJ4SsXwkArNnntIRBBQq23QoNAiA47jOMjdxJC7iTFe1fU/qsHIcPOnZly5VY8rVQ1mU522FaU1TSitacJhG7bJB5sQHlIR/1zCB13XeUJTCEpFAgg4DkIBB4GAg4ADhBz/XNj2un25UMAhwFsGXw9Jl3sAkP5FoUWcjlDAYZSvO0b5uuPhiADTfMYYqht0fIDdakDp7UbUa1tR39KKBm0rGlpa0djSivq25w0trTC09elofw10vULan7ykIoT4eSDEzwOhvu4dnntgmEc/9LondE6LDF6MMbS0Gk0h1tjS+dGAJl3HeQY0dliuNxhhYPwFCYORwcj4iX+OO8+NDDoDw+3GFvT0aZK7iU1hFuzrgQBvGfy9pPD3lsLfSwY/T0n/dB1xUXROiwx5HMdBJhZCJhZiuNfdH1VCqzegtKYJJdWNuFbdiGu3G9ueN0Fdp4WmWY+zZbU4W1bbTX0BXw8JhnvxYTbcS8qHmpcUnjIxtHoDmnUGNOkMaNYb0Kxr7fCcf2zSGaDVG+AlEyE8wKvD5Alfz8Exsga1tAgZAM06A67d5sOs5HYjymqaUFXXgqr6FlTVa1HdoDMdyt4tvh4SU4CFK9vCzN8LcnfnuBkKdS7thEKLODODkaGmUYeqei2q6ltwq67lzvP6FjS0tMJdIoSbWAg3iajDc2Gn5yLIxAJUN7Tgh8oGXK6sR3FlPcpqmrvdtr+XFHI3McRCAcQiASRCDhKRgH8tFEAiFEAs5PjnbfMlovb57fP4zsUdl3csN2mkHF6ynsORDg8JcSFCAYfhbYeEE+7C+pt0rbhS1YBidT0uVzXgh8p6XK5swM3a5rbW3t29QPGv1BmYNFLRL+ui0CJkCHCXiDBppKJLcNRr9fjxViMada3QGxj0rUboDfy3HfQGvvOw3tBhXiuDzmAwLWtpL9/hUdfpUW8wwkPaf1FDoUXIEOYlEyMqSOHoathk6F5fJYS4JAotQohLsSu0tm7dipCQEMhkMsTExCAvL6/H8nv27EFERARkMhkiIyNx4MABs+WMMaSnpyMwMBBubm6Ii4vD5cuXzcrU1NQgOTkZ3t7eUCgUWL58ORoaLAydSwgZ1GwOrd27dyMtLQ0ZGRk4c+YMoqKiEB8fj6oqy3dVzs7ORlJSEpYvX46CggIkJiYiMTER58+fN5XZuHEjNm/ejA8++AC5ubnw8PBAfHw8tNo7o1gmJyfj+++/x6FDh7Bv3z4cO3YMKSkpduwyIcSlMRtFR0ez1atXm14bDAamUqlYZmamxfKLFi1iCQkJZvNiYmLYypUrGWOMGY1GplQq2dtvv21aXltby6RSKfv0008ZY4xduHCBAWCnTp0ylfnqq68Yx3Hs5s2bFrer1WqZRqMxTWVlZQwA02g0tu4yIWQAaDQaqz6jNrW0dDod8vPzERd35w65AoEAcXFxyMnJsfienJwcs/IAEB8fbypfUlICtVptVkYulyMmJsZUJicnBwqFAtOmTTOViYuLg0AgQG5ursXtZmZmQi6Xm6agoJ5vX0QIcQ02hVZ1dTUMBgMCAgLM5gcEBECtVlt8j1qt7rF8+2NvZfz9zW8WIBKJ4OPj0+12169fD41GY5rKysqs3EtCiDMbtP20pFIppNI7XxBlbd9Wqqurc1SVCCE9aP9ssl6+WWhTaPn5+UEoFKKystJsfmVlJZRKpcX3KJXKHsu3P1ZWViIwMNCszOTJk01lOp/ob21tRU1NTbfb7ay+nh8Bkw4TCXFu9fX1kMvl3S63KbQkEgmmTp2KrKwsJCYmAgCMRiOysrKQmppq8T2xsbHIysrCiy++aJp36NAhxMbGAgBCQ0OhVCqRlZVlCqm6ujrk5ubi2WefNa2jtrYW+fn5mDp1KgDg8OHDMBqNiImJsaruKpUKZWVl8PLy6nVkybq6OgQFBaGsrGzQf7l6qOzrUNlPwHX3lTGG+vp6qFSqXgvaZNeuXUwqlbLt27ezCxcusJSUFKZQKJharWaMMfbUU0+x3/zmN6byJ06cYCKRiL3zzjvs4sWLLCMjg4nFYlZUVGQq8+abbzKFQsH++c9/snPnzrEnnniChYaGsubmZlOZuXPnsilTprDc3Fx2/PhxFhYWxpKSkmytvlWsvYoxGAyVfR0q+8nY4N9Xm0OLMca2bNnCRo0axSQSCYuOjmYnT540LZs9ezZbunSpWfnPPvuMhYeHM4lEwiZMmMD2799vttxoNLINGzawgIAAJpVK2Zw5c1hxcbFZmdu3b7OkpCTm6enJvL292bJly1h9fb091e/VYP+ldzRU9nWo7Cdjg39fh8x4WrYYSmNvDZV9HSr7CQz+faXvHloglUqRkZFhdvVxsBoq+zpU9hMY/PtKLS1CiEuhlhYhxKVQaBFCXAqFFiHEpVBoEUJcCoUWIcSlUGh1YuuorK7otddeA8dxZlNERISjq9Uvjh07hvnz50OlUoHjOHz55Zdmy5kVo+S6it729emnn+7ye547d65jKtuPKLQ6sHVUVlc2YcIEVFRUmKbjx487ukr9orGxEVFRUdi6davF5daMkusqettXAJg7d67Z7/nTTz8dwBreJQ7tj+9kbB2V1VVlZGSwqKgoR1fjrgPAvvjiC9Nra0bJdVWd95UxxpYuXcqeeOIJh9TnbqKWVht7RmV1ZZcvX4ZKpcLo0aORnJyM0tJSR1fprrNmlNzB5ujRo/D398c999yDZ599Frdv33Z0lfqMQquNPaOyuqqYmBhs374dBw8exPvvv4+SkhLMnDnTNObYYGXNKLmDydy5c/HJJ58gKysLb731Fr777jvMmzcPBoPB0VXrk0E7cinp3rx580zPJ02ahJiYGAQHB+Ozzz7D8uXLHVgz0p9+8YtfmJ5HRkZi0qRJGDNmDI4ePYo5c+Y4sGZ9Qy2tNvaMyjpYKBQKhIeH48qVK46uyl3VcZTcjobC7xgARo8eDT8/P5f/PVNotek4Kmu79lFZ20dZHawaGhpw9epVs+GuB6OOo+S2ax8ld7D/jgHgxo0buH37tsv/nunwsIO0tDQsXboU06ZNQ3R0NDZt2oTGxkYsW7bM0VXrVy+//DLmz5+P4OBglJeXIyMjA0KhEElJSY6uWp81NDSYtSRKSkpQWFgIHx8fjBo1Ci+++CL+8Ic/ICwsDKGhodiwYQNUKpVp+HBX0tO++vj44PXXX8fChQuhVCpx9epVrF27FmPHjkV8fLwDa90PHH350tn0NCrrYLF48WIWGBjIJBIJGzFiBFu8eDG7cuWKo6vVL44cOcIAdJnaR9O1ZpRcV9HTvjY1NbFHHnmEDR8+nInFYhYcHMxWrFhhGhbdldF4WoQQl0LntAghLoVCixDiUii0CCEuhUKLEOJSKLQIIS6FQosQ4lIotAghLoVCixDiUii0CCEuhUKLEOJSKLQIIS7l/wNbHvnPnjL2AAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "history = loss[\"history\"]\n", + "history.to_csv(\"history.csv\")\n", + "history[[\"avg_loss_train\", \"avg_loss_test\"]].plot()" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "2586ba0a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:27:39.159910Z", + "iopub.status.busy": "2024-02-29T23:27:39.158900Z", + "iopub.status.idle": "2024-02-29T23:30:35.025439Z", + "shell.execute_reply": "2024-02-29T23:30:35.024321Z" + }, + "papermill": { + "duration": 175.890298, + "end_time": "2024-02-29T23:30:35.028077", + "exception": false, + "start_time": "2024-02-29T23:27:39.137779", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "\n", + "from ml_utility_loss.loss_learning.estimator.pipeline import eval\n", + "#eval_loss = loss[\"eval_loss\"]\n", + "\n", + "batch_size = params[\"batch_size_low\"] if \"batch_size_low\" in params else params[\"batch_size\"]\n", + "\n", + "eval_loss = eval(\n", + " test_set, model,\n", + " batch_size=batch_size,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "187137f6", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:30:35.068851Z", + "iopub.status.busy": "2024-02-29T23:30:35.068532Z", + "iopub.status.idle": "2024-02-29T23:30:35.089170Z", + "shell.execute_reply": "2024-02-29T23:30:35.088192Z" + }, + "papermill": { + "duration": 0.043655, + "end_time": "2024-02-29T23:30:35.091488", + "exception": false, + "start_time": "2024-02-29T23:30:35.047833", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
avg_g_cos_lossavg_g_mag_lossavg_lossgrad_durationgrad_maegrad_mapegrad_rmsemean_pred_losspred_durationpred_maepred_mapepred_rmsepred_stdstd_losstotal_duration
tvae0.014026NaN0.0012282.7064350.0316440.6164290.0400759.105956e-073.2376320.0279350.0642950.0350490.0578080.0110855.944067
\n", + "
" + ], + "text/plain": [ + " avg_g_cos_loss avg_g_mag_loss avg_loss grad_duration grad_mae \\\n", + "tvae 0.014026 NaN 0.001228 2.706435 0.031644 \n", + "\n", + " grad_mape grad_rmse mean_pred_loss pred_duration pred_mae \\\n", + "tvae 0.616429 0.040075 9.105956e-07 3.237632 0.027935 \n", + "\n", + " pred_mape pred_rmse pred_std std_loss total_duration \n", + "tvae 0.064295 0.035049 0.057808 0.011085 5.944067 " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import pandas as pd\n", + "\n", + "metrics = pd.DataFrame(eval_loss[\"model_metrics\"]).T\n", + "metrics.to_csv(\"eval.csv\")\n", + "metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "123d305b", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:30:35.133977Z", + "iopub.status.busy": "2024-02-29T23:30:35.133704Z", + "iopub.status.idle": "2024-02-29T23:30:35.486775Z", + "shell.execute_reply": "2024-02-29T23:30:35.485861Z" + }, + "papermill": { + "duration": 0.376469, + "end_time": "2024-02-29T23:30:35.489194", + "exception": false, + "start_time": "2024-02-29T23:30:35.112725", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "from ml_utility_loss.util import clear_memory\n", + "clear_memory()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "a3eecc2a", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:30:35.531701Z", + "iopub.status.busy": "2024-02-29T23:30:35.530777Z", + "iopub.status.idle": "2024-02-29T23:33:40.420559Z", + "shell.execute_reply": "2024-02-29T23:33:40.419643Z" + }, + "papermill": { + "duration": 184.93115, + "end_time": "2024-02-29T23:33:40.440396", + "exception": false, + "start_time": "2024-02-29T23:30:35.509246", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Caching in ../../../../contraceptive/_cache_test/tvae/all inf False\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/conda/lib/python3.10/site-packages/ml_utility_loss/loss_learning/estimator/process.py:348: UserWarning: cov(): degrees of freedom is <= 0 (Triggered internally at ../aten/src/ATen/native/Correlation.cpp:100.)\n", + " corr_mat = torch.corrcoef(stack)\n" + ] + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.estimator.process import pred, pred_2\n", + "from ml_utility_loss.util import stack_samples\n", + "\n", + "#samples = test_set[list(range(len(test_set)))]\n", + "#y = {m: pred(model[m], s) for m, s in samples.items()}\n", + "y = pred_2(model, test_set, batch_size=batch_size)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "6ab51db8", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:40.481967Z", + "iopub.status.busy": "2024-02-29T23:33:40.481633Z", + "iopub.status.idle": "2024-02-29T23:33:40.499780Z", + "shell.execute_reply": "2024-02-29T23:33:40.498832Z" + }, + "papermill": { + "duration": 0.041954, + "end_time": "2024-02-29T23:33:40.501851", + "exception": false, + "start_time": "2024-02-29T23:33:40.459897", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "import pandas as pd\n", + "from ml_utility_loss.util import transpose_dict\n", + "\n", + "os.makedirs(\"pred\", exist_ok=True)\n", + "y2 = transpose_dict(y)\n", + "for k, v in y2.items():\n", + " df = pd.DataFrame(v)\n", + " df.to_csv(f\"pred/{k}.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d81a30f1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:40.541536Z", + "iopub.status.busy": "2024-02-29T23:33:40.541203Z", + "iopub.status.idle": "2024-02-29T23:33:40.546741Z", + "shell.execute_reply": "2024-02-29T23:33:40.545643Z" + }, + "papermill": { + "duration": 0.028052, + "end_time": "2024-02-29T23:33:40.548749", + "exception": false, + "start_time": "2024-02-29T23:33:40.520697", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'tvae': 0.42948681272958456}\n" + ] + } + ], + "source": [ + "print({k: sum(v[\"pred\"])/len(v[\"pred\"]) for k, v in y.items()})" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "3b3ff322", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:40.590218Z", + "iopub.status.busy": "2024-02-29T23:33:40.589936Z", + "iopub.status.idle": "2024-02-29T23:33:40.956533Z", + "shell.execute_reply": "2024-02-29T23:33:40.955442Z" + }, + "papermill": { + "duration": 0.389471, + "end_time": "2024-02-29T23:33:40.958584", + "exception": false, + "start_time": "2024-02-29T23:33:40.569113", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAE8CAYAAABkYrxdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCAUlEQVR4nO3deZhT9b0/8PfJyb7PvjHDADMsirKjQhFRrrValUur+NMqSBHvFWot1z5Kn1uoW8FWLdbr5V69sviIYItYrdatKiq1goIsog7bDDPMvmaZ7Dnf3x8nyUxmTTJJThI+r+fJk5mTk5NvMjOf+a6fL8cYYyCEEAnJpC4AIYRQICKESI4CESFEchSICCGSo0BECJEcBSJCiOQoEBFCJEeBiBAiOQpEhBDJUSAihEiOAhGJq88++wy/+c1v0NXVJXVRSBqhQETi6rPPPsNDDz1EgYhEhQIRIURyFIhI3PzmN7/BL3/5SwDAmDFjwHEcOI6DXq/HggUL+p0vCAJKSkrw4x//OHTsiSeewJw5c5CTkwONRoMZM2Zg9+7dA77eSy+9hBkzZkCj0SA7Oxu33HIL6urqEvPmSEJxlAaExMvRo0exceNG7Ny5E3/4wx+Qm5sLADh9+jQefvhh1NfXo7CwMHT+J598gvnz5+PPf/5zKBiVlpbihhtuwAUXXACPx4Ndu3bhwIEDePPNN3HdddeFnvvYY4/h17/+NW6++WbMnz8fra2teOaZZ6DX6/HVV1/BbDYn9b2TEWKExNHvf/97BoBVV1eHjlVVVTEA7Jlnngk795577mF6vZ45HI7Qsd5fM8aYx+NhkydPZldeeWXoWE1NDeN5nj322GNh5x47dozJ5fJ+x0nqo6YZSbjx48dj6tSpeOWVV0LH/H4/du/ejeuvvx4ajSZ0vPfXnZ2dsFgsmDdvHg4dOhQ6vmfPHgiCgJtvvhltbW2hW2FhISorK/HRRx8l542RuJFLXQByfliyZAl+9atfob6+HiUlJdi7dy9aWlqwZMmSsPPefPNNPProozh8+DDcbnfoOMdxoa9PnjwJxhgqKysHfC2FQpGYN0EShgIRSYolS5Zg7dq1+POf/4z77rsPf/rTn2AymXDNNdeEzvn0009xww034PLLL8d///d/o6ioCAqFAlu3bsXLL78cOk8QBHAch7fffhs8z/d7Lb1en5T3ROKHAhGJq941l97GjBmD2bNn45VXXsHq1auxZ88eLFq0CCqVKnTOq6++CrVajXfffTfs+NatW8OuNW7cODDGMGbMGIwfPz4xb4QkFfURkbjS6XQAMOCExiVLluDzzz/Hli1b0NbW1q9ZxvM8OI6D3+8PHaupqcFf/vKXsPMWL14Mnufx0EMPgfUZ9GWMob29PT5vhiQNDd+TuPriiy8we/ZsXHvttbjlllugUChw/fXXQ6fT4dy5cygrK4Ner4dCoUBTU1NYf86HH36Iq666CvPmzcOtt96KlpYWPPvssygsLMTRo0fDgs7GjRuxdu1azJkzB4sWLYLBYEB1dTVee+01rFy5Evfff78Ub5/ESsIRO5KhHnnkEVZSUsJkMlm/ofy5c+cyAGzFihUDPveFF15glZWVTKVSsYkTJ7KtW7ey9evXs4F+VV999VX2ve99j+l0OqbT6djEiRPZqlWrWFVVVaLeGkkQqhERQiRHfUSEEMlRICKESI4CESFEchSICCGSo0BECJEcBSJCiOTSeomHIAhoaGiAwWAYdGkBIUQ6jDHYbDYUFxdDJhui3iPlJCafz8f+8z//k5WXlzO1Ws3Gjh3LHn74YSYIQkTPr6urYwDoRje6pfitrq5uyL9lSWtEjz/+ODZv3ozt27fjwgsvxJdffok777wTJpMJ995777DPNxgMAIC6ujoYjcZEF5cQEiWr1YrS0tLQ3+pgJA1En332GW688cZQCtDy8nLs3LkTBw4ciOj5weaY0WikQERIChuu60TSzuo5c+bggw8+wIkTJwAAR44cwb59+/CDH/xgwPPdbjesVmvYjRCS/iStET344IOwWq2YOHEieJ6H3+/HY489httuu23A8zds2ICHHnooyaUkhCSapDWiP/3pT9ixYwdefvllHDp0CNu3b8cTTzyB7du3D3j+2rVrYbFYQjfaOoaQzCDp6vvS0lI8+OCDWLVqVejYo48+ipdeegnffffdsM+3Wq0wmUywWCzUR0RixhiDz+cLS8hGIsPzPORy+aB9QJH+jUraNHM4HP3mFvA8D0EQJCoROd94PB40NjbC4XBIXZS0pdVqUVRUBKVSGfM1JA1E119/PR577DGUlZXhwgsvxFdffYWnnnoKy5cvl7JY5DwhCAKqq6vB8zyKi4uhVCppYmwUGGPweDxobW1FdXU1Kisrh560OARJA9EzzzyDX//617jnnnvQ0tKC4uJi3H333Vi3bp2UxSJpptHiRJ5eBTkf3R+Bx+OBIAgoLS2FVqtNUOkym0ajgUKhwNmzZ+HxeKBWq2O6jqSByGAwYNOmTdi0aZOUxSBpxOMT0Gx1oTRbDBwOjw+vfVUPo1qBf51WAp0q+l/pWP+LE1E8Pj/6CZC08vdvm7HnUD2+bRTnkFmcXvAch1abG3890gC/INnYCxkBCkQkbdS0daOqyQYAMGvF3T+KTBrcPLMUKoUMjRYXDtV2SllEEiMKRCRt7K8W9yubUmpCkUkTOp6lU+KK8fkAgAPVHXB6aBg+nsrLyxPefUKBiKSFRosTDV0u8DIOs8qz+z0+qciAPIMKHp+Ao+e6kl9AMiIUiEha+KZB7BOaUGgYsEOa4zjMGJ0FAPi6wdpvB9jzncfjkboIQ6JARFKeIDCcbLEDACYVDj47tyJfD5NGgTG5Wnj9IwtEHp8w6M3nFyI+1xvBubG44oorsHr1aqxevRomkwm5ubn49a9/HQrA5eXleOSRR3DHHXfAaDRi5cqVAIB9+/Zh3rx50Gg0KC0txb333ovu7u7QdVtaWnD99ddDo9FgzJgx2LFjR0zli1ZaZ2gk54f2bg/8AoNGyWNUlmbQ8xS8DHfOLY/LpMRnPzo16GNjcnVYNK0k9P1zn5weNPCNytLgppmloe+3/KO6Xx/WL/5lfExl3L59O37605/iwIED+PLLL7Fy5UqUlZXhrrvuAgA88cQTWLduHdavXw8AOH36NK655ho8+uij2LJlC1pbW0PBbOvWrQCAZcuWoaGhAR999BEUCgXuvfdetLS0xFS+aFAgIikvz6DCysvHotPhgUw2dJA5n2ZGl5aW4g9/+AM4jsOECRNw7Ngx/OEPfwgFoiuvvBL/8R//ETp/xYoVuO2223DfffcBACorK/HHP/4R8+fPx+bNm1FbW4u3334bBw4cwKxZswAAL7zwAiZNmpTw90KBiKQFBS9DviGyWbuCwNBic8OsVUCt4GN6vVULKgZ9rG8sXHn5uEHP7RsXl88dE1N5BnLppZeGBd7LLrsMTz75ZGjx7syZM8POP3LkCI4ePRrW3GKMhZa6nDhxAnK5HDNmzAg9PnHiRJjN5riVeTAUiEhKY4xFXcvZ81U96joc+P6FhbigOLasDEp55N2niTp3pHQ6Xdj3drsdd99994BpmMvKykIJCqVAgYiktKpmG76o7sAFxabQqNhw8g0q1HU40GR1xhyI0sH+/fvDvv/8889RWVkJnh+4Fjh9+nR88803qKgYuLY3ceJE+Hw+HDx4MNQ0q6qqQldXV1zLPRAaNSMpraHLiTa7B3a3L+LnFJrEJlyz1Z2oYqWE2tparFmzBlVVVdi5cyeeeeYZ/PznPx/0/AceeACfffYZVq9ejcOHD+PkyZN4/fXXsXr1agDAhAkTcM011+Duu+/G/v37cfDgQaxYsQIazeADBPFCgYiktCaLGEwKjZGv6i4I9CW12twZvfbsjjvugNPpxOzZs7Fq1Sr8/Oc/Dw3TD+Tiiy/Gxx9/jBMnTmDevHmYNm0a1q1bh+Li4tA5W7duRXFxMebPn4/Fixdj5cqVyM/PT/h7oaYZSVk+v4A2e/SByKiRQ63g4fL60WZ3oyCK56YThUKBTZs2YfPmzf0eq6mpGfA5s2bNwnvvvTfoNQsLC/Hmm2+GHbv99ttHVM5IUI2IpKw2e8/8IaMm8v+ZHMehwKgCINaKSOqjQERSVpPVBUCsDUU7cpatE9OWtnen9tIGIqKmGUlZzYFAFEvTalyeHjqVHCXmxHe0SmHv3r1SFyGuKBCRlKVV8sjWKZEfaGZFozRbG8riSFIfBSKSsuZV5mFeZZ7UxSBJQH1EJGO129041WKPag4SkQYFIpKSfH5hxDmF/v5tM/56pAENXc44lYokCgUikpK+qOnEf+89jf1n2mO+hkkj5rW2OL3xKhZJEApEJCW1d7vh8QmQ87Gn9TAGA5GDAlGqo0BEUlK7XZz/k6OLfsQsyKgWA5HVRYEo1VEgIinHLzB0BWoxOfrY91Onpln6oEBEUo7N5YXAGOQyDvoYdm4NMgX2PrO5fBAyePFrJqBARFJOsAZj0ipGlPpVr5SDl3HwCwx2T4RD+IwBPk/yb1GMEL744ovIycmB2x2+jm7RokVJWaCaCDShkaScUCAKNK0GxRjQeARoPg4otUD5PECXG3pYJuNwxYQ8qBU8VJFmRvR7gU+fjLXosZv3H4A8smboTTfdhHvvvRdvvPEGbrrpJgDi7htvvfXWkCvrUxnViEjK0SrlGJunQ/Fw68RqPgWq3ga6aoGW74BDLwLd4cP9F48yY3yBASp5bLmrU5FGo8Gtt94a2nkDAF566SWUlZXhiiuukK5gI0A1IpJyKvL1qMjXD31S51ng7Gfi12WXiN/bmoCqvwHTftI/a32keIVYO0k2fpjaXx933XUXZs2ahfr6epSUlGDbtm1YtmxZ2u5iQoGIpB/GgNMfiPdFU4BxVwIuC3DgOcByDug4A+SIO2tYXV60WN3QKPnIVuJzXMRNJClNmzYNU6ZMwYsvvoirr74ax48fx1tvvSV1sWJGgYikFMYYXF4BaoVs8P/ubScBW7MYMMZeIR5Tm4CiacC5L4D6g6FAdLrFjr1Vrags0GdcSpAVK1Zg06ZNqK+vx8KFC1FaWjr8k1IU9RGRlOL0+vE/H5/G/3x8ZvB80/UHxfvi6WIndVDJdPG+/TTgsgIADGrxf63dlXkLX2+99VacO3cOzz//PJYvXy51cUaEAhFJKcERMwXPgR9oV1dHB9BZIzahiqeFP6bNBkyBraDbxD26dIF5SJm4At9kMuFHP/oR9Ho9Fi1aJHVxRoQCEUkpwRnVgw7dNx0T77PHAhpz/8fzAtsjt34HAKEJkd1uf0ZOaqyvr8dtt90GlSr2pTCpgAIRSSlDziFiLBRgUDB54AvkVgYuVA94XdAp5eA4QGAMDq8/ASWWRmdnJ1577TXs3bsXq1atkro4I0ad1SSlDBmI7C1i00wmB3IG2ZteYxabaI4OwFIHWW4ldEo57G4fut2+ES0ZSSXTpk1DZ2cnHn/8cUyYMEHq4oxYZvxUSMYIpuwIrhML0/qteJ8zdugh9qzynr6k3EroVGIgsrt9KIh7iaUx2L5l6YoCEUkpwRqRWdMn0DAmzp4GevqBBmMeDdQfEgMRgEvGZsPnZxm70WImoEBEUgZjDBUFelgcXpj71oicneJNxofmCA3KXCbed7cBHgfG5Q09S3ukKWnPd/H4/CgQkZTBcRwWTBhkn/X20+K9aRQgH2aESKnt6SeyNQ4auBQKMdg5HA5oNJk12TGZHA4HgJ7PMxYUiEh66Dgj3mcPUxsKMhaLgchaD7t+NJosLih5GcpyeiZA8jwPs9mMlpYWAIBWq03btVpSYIzB4XCgpaUFZrMZPB/7wmIKRCRlODw+yDgOakWfX2i/V1xhDwzfLAsyFgNNXwPWBpzTOPD2sSaMytKEBSIAKCwsBIBQMCLRM5vNoc8xVpIHovr6ejzwwAN4++234XA4UFFRga1bt2LmzJlSF40k2f7qDhyu7cIlY7MxZ1xPXiF0ngUEH6A2AtqcyC5mDMywtjZAN0oMbM4B5hFxHIeioiLk5+fD66WUstFSKBQjqgkFSRqIOjs7MXfuXCxYsABvv/028vLycPLkSWRlZUlZLCIRa2DEzKDq09fQu1kWadNJlw/wcsDnhtbfBQBweAaf0MjzfFz+oEhsJA1Ejz/+OEpLS8MSPI0ZM0bCEhEpDbq8o7NavM8eG/nFZDJAXwhYzkHraQOggdPjh19gA69hI5KSdInHG2+8gZkzZ+Kmm25Cfn4+pk2bhueff37Q891uN6xWa9iNZAbGWKhGFBaIXFax05njeoblI6UXR+DUrnbIAjWpgZpnRHqSBqIzZ85g8+bNqKysxLvvvot///d/x7333ovt27cPeP6GDRtgMplCt3TOv0LC2d0++AQGGceFUncA6Omk1hcAiignJAYCEdfdAo1S/FV3ZOAq/EwgaSASBAHTp0/Hb3/7W0ybNg0rV67EXXfdhf/5n/8Z8Py1a9fCYrGEbnV1dUkuMUmUYLPMqJFD1rvp1HVWvM8aHf1F9YEFHfZmaAIjcUP1ExHpSNpHVFRUhAsuuCDs2KRJk/Dqq68OeL5KpUr7dAdkYIMudg3WiMwxBCJdntik8zgwp1INnzwHeQb6/UlFkgaiuXPnoqqqKuzYiRMnMHp0DL90JK1l6ZSYWmpGlq7XGjNnl3jjZOKM6mjxCnG4v7sN4zTdQM7I5rqQxJE0EP3iF7/AnDlz8Nvf/hY333wzDhw4gOeeew7PPfeclMUiEigxa/rnlA7WhgyFwy/rGIwuT1xzZm+JfDIkSTpJ+4hmzZqF1157DTt37sTkyZPxyCOPYNOmTbjtttukLBZJFaFmWZSjZb0F+om6OxpwqsWGc52OOBSMxJvkM6t/+MMf4oc//KHUxSASa7G5YFQrepZ3MDayjuqgwMhZZ2sD/trWiIp8PUZlaYd5Ekk2ShVLJOfy+rHj81ps3nsabl9gVMvVJc4h4mSAMYb+oaDAkhC1twtgAs0jSlEUiIjkghMZtUq+Z2toS714bygc2YaHahPAy6HgGNQ+K1wUiFISBSIiuQGH7q2BQBTcHihWHAdocyHnOWi8XXDSPKKURIGISG7AQGQ5J96PpFkWpMuFXMZB6+2EyytQRsYURIGISK6rb8J8nxvobhW/HmmNCAC0uVDwMmh8nRAYg9snjPyaJK4oEBHJ9asRWRvEUTO1CVAZRv4CulxxDZvfAgDUT5SCJB++J6RfIAo2y+JRGwJCI2eTTB4UX1QAjZLyDqUaCkREclPLzOhyeJAdXN4R6qiOQ/8QAKjNgEyOAh1QYBIAOQWiVEOBiEhuelmvjJyC0BOI4tFRDYhJ0rTZ4jKP7nZAQxlAUw31EZHU4mgDfB5xwaouL37X1eXC4fHhXH0tWm3u+F2XxAUFIiKpLocHLTYXPMGRrNCwfYlYk4kXbS6arW4c+vYUTjTb4nddEhcUiIikjpyzYMfntfjnmXbxgK1JvDcWxfeFtNmQ8xzUPgtNakxBFIiIpLocHgCAOThiZm8W7/Vxzh2kyRbnEnktcPkoEKUaCkREUsGhe7NWAQh+MXcQEFo1HzeaLMhlHOSCCx6nPb7XJiNGgYhIhjEGS2BWtVmjBBzt4kaKcmX8R7bkSsg0RvF1HZ3xvTYZMQpERDK2vjt3hJplBZFvpBgFuV7cPZY5OuJ+bTIyFIiIZCyhDRUDO3fYegWiBOD14gxr3kmBKNXQhEYimZ7+ocCMantiA5HKkIuxuTqMzuHAGAOXgFoXiQ0FIiKZQpMa8ypzYVArxEWuCQ5Ecn0uCoxqQOVISNOPxI4CEZFMrl6FXH1gdw5np5j+Q8YDutzEvGCwA9zZKQY+CkYpg/qISGqwt4j3ulwxGCWC2gybR0CHtRvdVho5SyUUiIgkBIHhRLMNbXa3mDExOKM6Qc0yAIBMhioLj6pmO9paGxP3OiRqFIiIJGwuH9462oid+2vBGHpqRPGeUd0HCzTPfPb2hL4OiQ4FIiKJ9m5xBbxZpxSH7u3BGlGcZ1T3wTTZAAAhOIObpAQKREQSHd3iGrMcnRLwdANuu9h5nOBAxGnFQESTGlMLBSIiiWAgytIqe4btNVmx73EfIZlWHJHjKBClFApERBKhGpFe2WtGdWJrQwAg04s1IrgsgN+X8NcjkYkpEJ05cybe5SDnEcYY2gOBKFunTFzqjwEo1Ab4ZQr4BUHc1pqkhJgCUUVFBRYsWICXXnoJLpcr3mUiGa7b44fHJ4DjAnmI7MmrERVnaTF6VCmKTBpxtT9JCTEFokOHDuHiiy/GmjVrUFhYiLvvvhsHDhyId9lIhlLyMlw/pRiXj8+DnPnEmc6AuM99gmXrlCgtKRG3LnLSpMZUEVMgmjp1Kp5++mk0NDRgy5YtaGxsxPe+9z1MnjwZTz31FFpbW+NdTpJBlHIZKvL14u4d9mZxuYVKDyh1ySlAYAgf1GGdMkbUWS2Xy7F48WL8+c9/xuOPP45Tp07h/vvvR2lpKe644w40NtLsVTKM0ETGBM6o7sUvMDR5NejodoNR0yxljCgQffnll7jnnntQVFSEp556Cvfffz9Onz6N999/Hw0NDbjxxhvjVU6SQQ7VduJMqx1ev5DwFfd9+QWGN064UNVsh7+bAlGqiGn1/VNPPYWtW7eiqqoK1157LV588UVce+21kAW2fxkzZgy2bduG8vLyeJaVZACX14+Pq8Sm+7/NHweFPQlrzHpR8By8KjMAwO+yQ+5zJ3zuEhleTIFo8+bNWL58OZYtW4aiooG3fcnPz8cLL7wwosKRzBPc3NCglkMjR0+yfENyAhHHcVCoNPDKNPAJDCpnZ1I6ycnQYgpE77//PsrKykI1oCDGGOrq6lBWVgalUomlS5fGpZAkczRZxekeBUa1GIQEv5gsX21OWhlUch4uhQk+v0PssKZAJLmY+ojGjRuHtrb+iwY7OjowZsyYEReKZK6GLicAoNisSXiy/MGoFTK45EZxUiPlr04JMQUixtiAx+12O9Rq9YgKRDIXYwwNXWKNqMSsSVrqj77UCh4uuQk+gdEQfoqIqmm2Zs0aAGI7e926ddBqtaHH/H4/9u/fj6lTp8a1gCRzdHR74PL6oeA55BlUwJnkpP7oSyWXoUsRCEQ0qTElRBWIvvrqKwDif7Zjx45BqVSGHlMqlZgyZQruv//++JaQZIxGi1gbKjRpwHPoaZoluY9mUpERZcpxMFV/IS7zoPzVkosqEH300UcAgDvvvBNPP/00jEZjQgpFMtOFxUYUmdTw+gM1EZ8HkMkBbU5SyzE6RweYy4F6uZiw3+sElNphn0cSJ6ZRs61bt8a7HOQ8wHEccoK7drTUiPeJTJY/FF4BqAyA2yZ2WFMgklTEgWjx4sXYtm0bjEYjFi9ePOS5e/bsGXHBSIZL8ozq3lxeP1ptbuihRxZsYoe1aVTSy0F6RDxqZjKZQjtjmkymIW+x2LhxIziOw3333RfT80lq+/C7Zrx1tBEttkDaGAkDUUOXE7sPnsOR9kBNjDqsJRdxjah3cyzeTbMvvvgC//u//4uLL744rtclqcHrF/Btow0en4AppYF/VKGO6uQHIpVCDEA2ziAeoLlEkotpHpHT6YTD4Qh9f/bsWWzatAnvvfde1Ney2+247bbb8PzzzyMrKyuW4pAUd6a1Gx6fAINaLs4fctt7kuXrkjt0DwBqufhrb5MFBltoLpHkYgpEN954I1588UUAQFdXF2bPno0nn3wSN954IzZv3hzVtVatWoXrrrsOCxcuHPZct9sNq9UadiOp76taselzQbFRbN6HkuVni8s7kkwdqBFZOT0YmFgjGmSSLkmOmDM0zps3DwCwe/duFBYW4uzZs3jxxRfxxz/+MeLr7Nq1C4cOHcKGDRsiOn/Dhg1hfVGlpaWxFJ8kUUOXE40WF3gZhymjzOLBJKaGHYgqUCNy8Qb4GCcm0XfbJCkLEcUUiBwOBwwGsX393nvvYfHixZDJZLj00ktx9uzZiK5RV1eHn//859ixY0fEy0LWrl0Li8USutXV1cVSfJJEX9V2AQAmFBqgUwW6JCXsqAYAOS+DgufAOB4+RaB5Rv1Ekoo5ef5f/vIX1NXV4d1338XVV18NAGhpaYl4kuPBgwfR0tKC6dOnQy6XQy6X4+OPP8Yf//hHyOVy+P3+fs9RqVQwGo1hN5K6LE4vTraINY3pZb36/4JrzCToqA4KNs88ykDnOY2cSSqmCY3r1q3Drbfeil/84he46qqrcNlllwEQa0fTpk2L6BpXXXUVjh07FnbszjvvxMSJE/HAAw+A5yWY5EbiSqPgMX98HlptbnFtGSDOZA52DktUIwKAS8fmQGAMKmse4KijDmuJxRSIfvzjH+N73/seGhsbMWXKlNDxq666Cv/6r/8a0TUMBgMmT54cdkyn0yEnJ6ffcZKelHIZppX1GQkN1oZUhuQlyx/A5JJATag+F2gG1YgkFlMgAoDCwkIUFoYvVpw9e/aIC0QynMT9Q/1oAoGSakSSiikQdXd3Y+PGjfjggw/Q0tICQRDCHo91J9i9e/fG9DySWhhjePd4E0ZlaTGh0AAF36sr0hbY2UXirIhWlxcWhxd6ZkAWIO76KgiAjHZhl0JMgWjFihX4+OOPcfvtt6OoqCi09IMQQEwH+22jDada7BhfYAh/0BbIQSRxIDpc24WDZzsxo8yMy2VyQPABbktPDYkkVUyB6O2338Zbb72FuXPnxrs8JAN81yiOlFXk66GU96ph+L092zxLHIiCo2YunwBozGL+bEcHBSKJxFQPzcrKQnZ2drzLQjKAIDBUNYuBaGJhn+kVwV1dlTpAqZegdD2CkxrdPgHQBn6XqcNaMjEFokceeQTr1q0LW29GCCA2y5weP1QKGUqz++T4sfXKyChxcz5UI/L6qcM6BcTUNHvyySdx+vRpFBQUoLy8HAqFIuzxQ4cOxaVwJP3UtHUDAEZn68DL+gQbe2r0DwHiTh5AsGkWrBFRIJJKTIFo0aJFcS4GyRTV7WIgKs8dIONhcMQsybt2DCRYI3J7/dQ0SwExBaL169fHuxwkA/gFBoVMBrmMQ3lOn8mKfh/QnRod1QCglgcCUe8akcsilpOPeXodiVHMn3hXVxd2796N06dP45e//CWys7Nx6NAhFBQUoKSkJJ5lJGmCl3G4eVYpfH4Bcr5P96O9GWCCmBtaZRj4AkmkUfKYW5ELtUIGptCCkyvFZP6uLjGPNkmqmALR0aNHsXDhQphMJtTU1OCuu+5CdnY29uzZg9ra2lCuInJ+6heEAMDaIN4bSyTvqAbE5Sezx/Qa+dVkiZ3pjg4KRBKIadRszZo1WLZsGU6ePBmWwuPaa6/FJ598ErfCkfTi9QuDP2g9J94bU7S2TB3WkoopEH3xxRe4++67+x0vKSlBU1PTiAtF0o9fYHjukzN46fOzcHr6pHBhDLAEApEpdQJRu92Nug6HWF7qsJZUTIFIpVINmKb1xIkTyMvLG3GhSPpps7vh8QmwuXyhofEQtzWQo1oGGIqkKeAA3j3ejN0Hz6HB4uypEdFcIknEFIhuuOEGPPzww/B6vQDEjfNqa2vxwAMP4Ec/+lFcC0jSQ6vNDQDIN6j6rz201Iv3+nxxY8MUEQyYbm/v2dUUiKQQUyB68sknYbfbkZeXB6fTifnz56OiogIGgwGPPfZYvMtI0kAwEIUSoPVmDQSiFNvEsGe9Wa/Z1W67OHpGkiqmUTOTyYT3338f//jHP3DkyBHY7XZMnz49op04iHQEgaHZ5kKWVhn6I4yXIQNRVyCPeYp1VIdmV3v9gEIj3rxOsVaUAnOdzidRByJBELBt2zbs2bMHNTU14DgOY8aMQWFhIRhjlBIkRfkFhlcPnkN9lxMGtRw3zSyFSROfZhJjDK32QQKRpxuwt4pfZ42Oy+vFiyo4qdEbGO3TZovNSGcnBaIki6ppxhjDDTfcgBUrVqC+vh4XXXQRLrzwQpw9exbLli2LOE0sSb5j9RbUdzkBADaXD/tOtsXt2hanFx6fALmMQ7a2zz5lnYHakD5P0tSwAwmrEQHUYS2hqGpE27ZtwyeffIIPPvgACxYsCHvsww8/xKJFi/Diiy/ijjvuiGshycgdPdcFALh4lAlHz1lwqsWObrevZ4ufEWAMmFRkgMAAWd+Frp014n1W+YhfJ95UvZd5ANRhLaGoakQ7d+7Er371q35BCACuvPJKPPjgg9ixY0fcCkfi57qLinDp2BzMrchFkUkNgTGcbLHH5dpZOiWumVyEay/qMzTPWK9ANCYurxVPhSY15lbkYnJJIG8S1YgkE1UgOnr0KK655ppBH//BD36AI0eOjLhQJP5y9CpcNi4HagWPSUVGVOTr49ZHNKjuNnEhqUwOmFJvV95cvQqzx2SjIj+w9i04ckY1oqSLql7e0dGBgoLBd18oKChAZyfNTE11U0rNmFJqjtv17G4ftAq+f7Os7YR4n1UuyR73UQsGIq8L8DjEBbokKaKqEfn9fsjlg8cunufh8/lGXCgSP4wxfPRdC443WOAbai3YCK6//bMa/NdHp9Dl6DP/JhiIcivj/rrxIAgMLTYX6jocYIyJwTKYGYCWeiRVVDUixhiWLVsGlWqAuSIA3G53XApF4qe924PDdV1QymWY1CuHNGMMFqc4M97cd6QrCt0ePzw+ARwH6Ht3fDs6xB07OA7IqYj5+onkZww7Pq8FANyzYJzYea3NBtw2sXmWQuviMl1UgWjp0qXDnkMjZqmlyeICIC696N102l/dgX+ebseFxUZcfWHsc2Y6u8VakFGtCE//0RTYTjxrDKCSNlH+YBS8mMTNJzC4vIIYiDTZ4pQD6rBOqqgC0datWxNVDpIg7YFA0XeiYa5e/L7ZNrJabJdDrFVl63rVqgQBaP5a/LrwohFdP9HUCh52t09MGatRUIe1RGhbywzXFgg0wcATVGgS80h12D1D5xEaRmegX8is7TUC11YFuKyAQg3kjo/52smgCi58Dc0lyhHvg/uvkaSgQJTh2rvFQJSjD+8H0qvk0Ch5CIyFmlexCAairGA/E2NA7T/Fr0tmpnz+52Du6tDsal0wEHWKNTuSFBSIMpjD40O3W/wDy9H1H2AINqfaRxCIgk2zUCBqPi6mXOUVQMmMmK+bLKrQMo9A0FGbxeAp+MT81SQpKBBlsI5AgDFpFOFbPwfkBmpJ7fbYA9GEQgPGFxiQrVeKC1xPfyA+MHpOWszDCUsFAoijfMHmWXf81uORoaV2vZmMyKgsLe6ePzZUK+orO1BLCjbfAIj70zMm1mgiyKRw6djAH63PDRzdI04E1OUCpZeMuPzJUJGvR5ZWiVFZmp6D2txAIv02AKndx5UpKBBlOK1SDq1y4B9ziVmDS8Zko0TWDnzzujhs7RE3SISMF5sp2mwxsGhzA/c54VkWGQMsdcDJ98R0H3IlcOFi8flpYFyeHuP6ZjfWBQ50tya9POcrCkTnsTwtjzzffqDxaP8HBb84cuRoB9pO9hznOEBtAtRmOD1ecM4OqAQHOHBiU+yim3o6fNNVcDshapolDQWiDPb3b5qhlMswY3RW/3Qffi9w9BWgq04MLoUXi3N+dLkAxwNeh7jMwdEhNlG628QagtcJOLsAZxca2+xotrpRnG3A6AtmA2MuT7mcQ8Px+AR0OT1gDCgwBrbGCg3hd4gjZzLqSk00CkQZyi8wfN1gAWPA9NFZ4Q8yBnz7V6CrDm5Ogbay66ApGBc+KVGuBDRmILtP+g5PtxiU3Dac+rYFtRo5TFMuAErTc1PCRosTew7VI9egwu2XBjJIqs1ixoDgyFkwTxFJGAr1Gcrq9IIxQMFz0Cn79Nc0HgZaqwAZjy90V+JPp3l819R/e6gBKXViytfCyaiVl8OmKkSWMb1qQb1pgqNmvfdik8l6mpfUPEsKCkQZKjjR0KRVhucRd9uAU4Eh9rFXQJMv1ng6opxLFNzDDOg1hygNqQNB2un1iyvwg7SBGp6DAlEyUCDKUF3BlfV9k5/V/EPsHzIWAaNmhZpj0c6u7nKK52uUfNx3BEmmYI3IL7CeZR4AdVgnGQWiDGXpO+MZEDtfGwMZNMddCXA9ye47HV4IAut7mUH1zKhOnQ0TY6HgZaHJnqFlHgDViJKMAlGGCtZYwhaj1h0AmCDmBzKXAQAMajnkMg5+gcHq8kZ8/WBTbiS5jFJFsEbn7B2IQjWidlpzlgQUiDKUPdB/E8pL7XH05AgqnR06TybjYA40z6LpJxqVpcGs8myMy0vfjuqgYPPM2bvDuu/IGUkoGr7PUD+5dDS6PX6og2vMGg+Lf1SGglBtKChHp0SbzR3q4I7EqCwtRmWl/lqySFxUYkJlgT68GSuTiXuxWRsBewsN4ScYBaIMxXFcT+pWxnpqQyUz+60hm1xswugcLUrMGpyPLhplGvgBfUEgEDUD+ROTW6jzDAWi84G1Xuyo5hVAXv8/qLKc6Go2Hp+AZqsLWTpleJ7qTKPLF+/tLdKW4zwgaR/Rhg0bMGvWLBgMBuTn52PRokWoqqqSskgZ4WSzDX890oDjDRbxQLA2lDchLtv6tNrd2H3wHF75om7E10oFLq8fLVYX2u190ubqg4GoOfmFOs9IGog+/vhjrFq1Cp9//jnef/99eL1eXH311eju7payWGmv0eLCqRY72uwewO8DWr4VHxgif3RdhwNH6rrCh7AHEZxzlO5D90Enm+3Ysb8W+071GaoPBiK3TezsJwkjab36nXfeCft+27ZtyM/Px8GDB3H55ZdLVKr0F9wmyKRRAF1nxVxBKj1gHj3oc977phlWpxc5euWwndCh9LC69B+6BwCNcoB5RAAgV4nJ9J2dYq2o77o7EjcpNXxvsYhNiezsgUco3G43rFZr2I30FxaIQpscjh8y0Vm2TqzddHYPP5coOMyfnQFziICeeUQOzwC1QT31EyVDygQiQRBw3333Ye7cuZg8efKA52zYsAEmkyl0Ky1Nvf3UpdZ740STmo94t9Xg0HVHBEP4waZZdobUiIKJ45wDNUv1gS3WqZ8ooVImEK1atQpff/01du3aNeg5a9euhcViCd3q6jKjszSeXF4BnsCaKaOnWezbkKuGbJYBPcn1h1tz5hcYLE5xsqQ5Q/qIghMa3V4B/r7LXIKBqJtqRImUEmOvq1evxptvvolPPvkEo0aNGvQ8lUo16HbXRBSsDelVcsg7TokHcyqGTd2aFWiaDTe7usvhgcAYlHJZxgzdq+QycJw43crl9YcnkdMH08a2ix3/Kb49UrqS9FNljOFnP/sZXnvtNezduxdjxlBn4Eg5PD7IZRxManlPitcINjkMNrOsLi+8fgEKfuDKslYpx8JJBfD4hfD0ImlMJuOgVvBwevxwePoEIpVR3CjS6xIzVBqLpCtoBpM0EK1atQovv/wyXn/9dRgMBjQ1NQEATCYTNJrzc5bvSI3N02P1lRXw2NqAg51iTSh77LDP0yjEdB4urx+dDg/yDeqBz1Pyg89ETmMzRmeBg/j+wnAcYCgCOqoBWyMFogSRNBBt3rwZAHDFFVeEHd+6dSuWLVuW/AJlCI7joLLWit8YSyKaxMhxHP7lggKoFbK0TnQWq1nlQ6wlCwYiawNQMj15hTqPSN40IwnSWS3eRzH3pSJfP+w5p1rs0Ch55BtUgzbfMo6xRLy3NUpbjgxGPW8Z5vXD9eA5AQvbz0DNAciKX78bYwzvHm+Cxyfg9stGI1efOQMHLq8fVqcXcl7Wf1pCsDnmaBcnh8oz532nivPkX9r5wS8w1LQ50FR7BjLBCyg0gKEw4uc7PX58XW/BwbMdAz5udfrg8QngZVzGNd+ON1iwY38t9p9p7/+gUifu5cYY1YoShAJRBrG7fBAYQ46nHgqeE5tlUYxsubx+vP9NM/55un3AZnOr3QUAyNErwcsyY8QsKDipsXug2dVAT63I2pCkEp1fKBBlkOAcokJ/o7jzapTNMpNGAV7GwetnsAYyPPbWahPnGGVSkyxIFwhEDk//9w0AMBSL9xSIEoICUQaxOL3g/S5kCYHmRVZ5VM+XybjQivqBZli3BtJkZGQgUonD9t3uwWpEgUBkaxSbaCSuKBBlEIvTC7O7Hiq5TEz+rjZGfY3givr2bne/x1pt4rF8QyYGIrFG5PL64fMPkCzfUAhwMsBtB9y02DreKBBlEIvTC5PznJinOsaUFcGJjE2W8EDU7fbB6vSC44C8DAxEKrks1O/lGGjxK68Q830DQBetcYw3CkQZhDEB2Z56qBR8zMP2RSYxEDVanGHH1QoeS2aVYuGkgrTeUHEwHMdBG5hV7RiseWYKZHuwUCCKN5pHlEF+WKEG61CCcXy/nToiVWBUQ8ZxsLl8sLt9oYWtvIxDsVmD4gxOsD99dBYEgUGrGiTQmsvEveGoRhR3FIgySWc1OHDgzGViUyIGSrkMi6eXIFev6r/uKsNNL8sa+gTTKHE6hKNd7CtSDT8TnUSGmmaZpCP6ZR0DKc3WhgUhh8eHd4834VSLbUTXTXsKTc8OsJZz0pYlw1AgyhDfNXTi8NdHUN/ljHrYfjCMMTDGcKa1G980WLG/euAZ15nC4xPQYnWhxeoa/CRToMlL/URxRYEoQ9hbzsLpdMEJVU9WwRH4rsmKl/bX4mSLPbQtUUVeZjdFTrWIu3l8erJt8JPMgQ7rzpqklOl8QX1EGcLbdhoAIMsZG9WyjsF02D1os7nx1lFxbRUv4zC5JPPyEPVmUIt/DjbXEBsImEeLn293G+CyxjRXi/RHNaIMwQX+Q6vyx8XletNHZ4X+MAFg5uis8MyFGcioFjv4bS7f4ClqlNqehcTBVCtkxDL7N+s8wdx2cLYmMAC6woq4XFOt4HHTzFIcPdcFk0aBizK8NgSIyzw4DvAJDE6vP7QQtp/ssYC1Eeg4AxRNSW4hMxTViDJAd/NpCIzBqcyB0TjMEHQUTBoF5lXm4eJR5ozJTz0UOS8LLX61DbDoNySYerezBhAGWA5CokaBKAM4m8TdOnzmcsgyLD1Hsukj6ScyFIvJ0bwuyk8UJxSI0h1jkFlqoFHw4HOGT5JPhhbsFxsoDUqITNYzRaLjdOILdR6gPqJ0192GHLkHOeW5YJdQYveRmlhoRJFJjVFZwyxlya0EWqvE25jLk1O4DEaBKN11nBHvTWXgYlzWQXpEsoEAAHHTSk4mDuN3twO6nMQWLMNR0yzNsfaTYGAR7V1G4kih6WmetVVJWpRMQIEonXkc8LTX4ouaTrxWp6HtmeJAEBhabC6cbLYN/3nmBXbQbaVANFIUiNJZxxk43V7Y+CzYOP15McSeaAzAzv11ePNoI+zuITqsAXErb44DbE2AszMp5ctUFIjSWftJOLx+dGpGI0eXeVkTpcDLOBg1Ytdpl2OIIXxA3GYomPep+XiCS5bZKBClK78P6DiDbrcPHZrRyNVn1j5jUgru2TZsIAKAwovE+6ZjlFR/BCgQpStLLeDzoMunRLcyDwVGtdQlyhim4E4mjv47mfSTO0FMQufsotQgI0CBKF21fAefIKBBLmYNpEAUP8EaUUSBSK4E8ieJXzd9ncBSZTYKROnI7wNav0O324827TiYNIrzLq1rIuUEtlRqs0cQiICe5lnLccDrHPpcMiAKROmosxrwuQGVHrmjKlCeq5W6RBkluF2S1emFa6CthfoylQL6PPEfROORBJcuM9HM6nQUGKExlV2MGytLJS5M5lEreMyfkBfagntYHAeMmgV89zeg/iAwara4Ho1EjD6tdON1Ae0nxa8LLpC2LBlselkWxuXpoeAj/BPJv1BMmuay0kzrGFAgSjfNXwN+HzzqHFjkuVKXhgTxcqA4sOi4Zh8N5UeJAlE6YQxo+AoAUK2sxJZ/1OCdrykfTiL4/ALOtNrxZU0UO5eMminmKepuA1q+SVzhMhAFonRiOSf+kvNynMBoAECOnmZUJ4LXz/D64QZ8erINDs8wSz2CFBqg9BLx6+pPASGCjm4CgAJReqnbDwDw5U7CWYv4Sz46m0bMEkGj5EOz1Ru6ohiSHzVL7Ctydood1yQiFIjShb0FaDsJcBxq9RfD62cwahShoWYSfyWB5GjnOqMIRHJlT6K06k/EzmsyLApE6aJmn3ifNwEnbeJ/6nF5Olpxn0CjssTa5tl2R3RPLJoKmEoAvxc48S51XEeAAlE66KoVc95wHNyjLsOpFjuAKLIJkpiUZWvByzh0dHvQbndH/kSOA8ZfA8h4oP0UNdEiQIEo1Ql+4OT74tdFU1Ht1MLjE5CtU6LEPExeZTIiagWPskAfXDD4R0yfD4y7Uvz69IdAFy2IHQrNrE51NZ+K/UMKNTBmHiYotJDLOPgFULMsCSoL9Khu60ZrNDWioJIZQNdZoPUE8PVuYNrtgI7mfg2EAlEqazsF1H4ufj3+B4BSBw5ARb5B0mKdT8YXGJClVaLIFEN2A44DJt0AuF8GrA3A4ZeBi5cAhoL4FzTNUdMsVXXVAd+8JnZ0Fk+F01wZ+XwWEjcKXoZisyb22ievAC76sdhU83QDh18CWr6NbyEzQEoEomeffRbl5eVQq9W45JJLcODAAamLJB3GxLw2R3aJq7mzx8A3diHeOtaI7Z+djb6vgsSN1eXF8QZL9E9U6oCpt4lpZX0e4PhfgG9eB1wxXCtDSR6IXnnlFaxZswbr16/HoUOHMGXKFHz/+99HS0uL1EVLPmsjcGw38O1fAcEH5FbCMu4G7DnchLoOBwTGYNbS3mVS6Hb78PL+Wrz/TTMO13VFv2OKQg1M+X/A6MvEJlvzN8D+54Dv3gIs9ef9ED/HJN6D5pJLLsGsWbPwX//1XwAAQRBQWlqKn/3sZ3jwwQeHfK7VaoXJZILFYoHRaExGceODMTGBlqcbcLSJu0B0nAbsrQAAt5+hJWcmvlFMxonmbvgEBqVchhumFKOUZlJLZm9VC76q7QIgTnacXGxCsVkNo1oBWSTpQoJsTcCpD8RpGUFKHZA1GtAXALo8QGUAlHpx2UgaD0pE+jcqaWe1x+PBwYMHsXbt2tAxmUyGhQsX4p///Ge/891uN9zuntELqzXCWasd1cCpv/c/HhaDWZ9jbIDzwo+darGjy+kOncIBgf+U4nkzyrIQ/P083WpDu90DgIEXvAATwq44tdQMOS8H8ifh4+4KHG+TAxCbYSVZGiycVIBsHSXIl9L88XnQq+T45+l21Hc6UR+Ycc1xwI+mjwr9k/i63oKDZzsxWGz6lwsKUTj1VsBaj/rj+6DoPI18dIu1pOYBFsvK5OLqfplCnJuEwIU5rs/X6P91Il10M6COTwVA0kDU1tYGv9+PgoLwUYSCggJ89913/c7fsGEDHnrooehfyOcWF4vGmc9phXeInR44ryr0SyF4nPB7xNSjwaWQfpkSTrkJDkU2LpowG/KC8YBCA83JVuQLDhSbNJhQaECRSU1D9SmA4zjMLM/G+EIDvq63oKbNgTa7G36BQa3oSdXr9PrR0T14mlmvOPcCMI1CW+k1EIp9yDfaAFsjYG8GHO2A296TdlbwiTe4EvwOoxT4ZxoPaTV8v3btWqxZsyb0vdVqRWlpBBkKzaXAlFv6/KcY4D9JRMd6Hsvt9sDgE0LHOA7gOFnP6Ua1uD86gAKnFya/IJ7Jq8AUGnAyeehchVaJ4L/QeZV5w78nIhmjWoE543IxZ5y4M6zL54dK3hOIJhQaUGhUD9rt03t94Ng8nVgrVucC2WPCT/T7AL9bXCoi+MR7Fvg31ruWPuDXSaDUxe1Skgai3Nxc8DyP5ubmsOPNzc0oLCzsd75KpYJKFcMiT6Wu/w85DnKjqJUadUAa9WKRCMlkHLTK8D8jo1oBozqyQQXDUOfxgSbZeUDSUTOlUokZM2bggw8+CB0TBAEffPABLrvsMglLRghJJsnD7Zo1a7B06VLMnDkTs2fPxqZNm9Dd3Y0777xT6qIRQpJE8kC0ZMkStLa2Yt26dWhqasLUqVPxzjvv9OvAJoRkLsnnEY1E2s4jIuQ8EenfqOQzqwkhhAIRIURyFIgIIZKTvLN6JILdWxEv9SCEJFXwb3O4rui0DkQ2mw0AIptdTQiRjM1mg8lkGvTxtB41EwQBDQ0NMBgMkq/FCi43qauroxG8XuhzGdz58NkwxmCz2VBcXAyZbPCeoLSuEclkMowaNUrqYoQxGo0Z+0s1EvS5DC7TP5uhakJB1FlNCJEcBSJCiOQoEMWJSqXC+vXrY8sOkMHocxkcfTY90rqzmhCSGahGRAiRHAUiQojkKBARQiRHgYgQIjkKRFGIZkfa559/HvPmzUNWVhaysrKwcOHCjN3BNtadenft2gWO47Bo0aLEFlBC0X42XV1dWLVqFYqKiqBSqTB+/Hj87W9/S1JpJcRIRHbt2sWUSiXbsmULO378OLvrrruY2Wxmzc3NA55/6623smeffZZ99dVX7Ntvv2XLli1jJpOJnTt3LsklT6xoP5eg6upqVlJSwubNm8duvPHG5BQ2yaL9bNxuN5s5cya79tpr2b59+1h1dTXbu3cvO3z4cJJLnnwUiCI0e/ZstmrVqtD3fr+fFRcXsw0bNkT0fJ/PxwwGA9u+fXuiiiiJWD4Xn8/H5syZw/7v//6PLV26NGMDUbSfzebNm9nYsWOZx+NJVhFTBjXNIhDckXbhwoWhY0PtSDsQh8MBr9eL7OzsRBUz6WL9XB5++GHk5+fjpz/9aTKKKYlYPps33ngDl112GVatWoWCggJMnjwZv/3tb+H3+wc8P5Ok9aLXZIl2R9qBPPDAAyguLg77xUx3sXwu+/btwwsvvIDDhw8noYTSieWzOXPmDD788EPcdttt+Nvf/oZTp07hnnvugdfrxfr165NRbMlQIEqCjRs3YteuXdi7dy/UarXUxZGMzWbD7bffjueffx65ublSFyflCIKA/Px8PPfcc+B5HjNmzEB9fT1+//vfUyAi0e9I29sTTzyBjRs34u9//zsuvvjiRBYz6aL9XE6fPo2amhpcf/31oWOCIO6fLpfLUVVVhXHjxiW20EkSy+9MUVERFAoFeL5n++pJkyahqakJHo8HSqUyoWWWEvURRSDWHWl/97vf4ZFHHsE777yDmTNnJqOoSRXt5zJx4kQcO3YMhw8fDt1uuOEGLFiwAIcPH86oTJux/M7MnTsXp06dCgVnADhx4gSKiooyOggBoOH7SO3atYupVCq2bds29s0337CVK1cys9nMmpqaGGOM3X777ezBBx8Mnb9x40amVCrZ7t27WWNjY+hms9mkegsJEe3n0lcmj5pF+9nU1tYyg8HAVq9ezaqqqtibb77J8vPz2aOPPirVW0gaCkRReOaZZ1hZWRlTKpVs9uzZ7PPPPw89Nn/+fLZ06dLQ96NHj2YA+t3Wr1+f/IInWDSfS1+ZHIgYi/6z+eyzz9gll1zCVCoVGzt2LHvssceYz+dLcqmTj9KAEEIkR31EhBDJUSAihEiOAhEhRHIUiAghkqNARAiRHAUiQojkKBARQiRHgYgQIjkKRCStbNu2DWazOfT9b37zG0ydOjX0/bJlyzI69WymokBEBrRs2TJwHId/+7d/6/fYqlWrwHEcli1bFnZ+vANAeXk5Nm3aFHZsyZIlOHHixKDPefrpp7Ft27bQ91dccQXuu+++uJaLxB8FIjKo0tJS7Nq1C06nM3TM5XLh5ZdfRllZmSRl0mg0yM/PH/Rxk8kUVmMi6YECERnU9OnTUVpaij179oSO7dmzB2VlZZg2bdqIrj1QTWXRokWhWtYVV1yBs2fP4he/+AU4jgPHcQD6N8366l0zW7ZsGT7++GM8/fTToWtUV1ejoqICTzzxRNjzDh8+DI7jcOrUqRG9LxIbCkRkSMuXL8fWrVtD32/ZsgV33nlnwl93z549GDVqFB5++GE0NjaisbEx6ms8/fTTuOyyy3DXXXeFrlFWVtbvPQHA1q1bcfnll6OioiJeb4FEgQIRGdJPfvIT7Nu3D2fPnsXZs2fxj3/8Az/5yU8S/rrZ2dngeR4GgwGFhYXDZsIciMlkglKphFarDV2D53ksW7YMVVVVoT3GvF4vXn75ZSxfvjzeb4NEiFLFkiHl5eXhuuuuw7Zt28AYw3XXXZf2+aaLi4tx3XXXYcuWLZg9ezb++te/wu1246abbpK6aOctqhGRYS1fvhzbtm3D9u3b41ZrkMlk6JsKy+v1xuXakVixYkWoI37r1q1YsmQJtFpt0l6fhKNARIZ1zTXXwOPxwOv14vvf/35crpmXlxfW7+P3+/H111+HnaNUKke8p9dg17j22muh0+mwefNmvPPOO9Qskxg1zciweJ7Ht99+G/p6MBaLpd9+ZTk5OQMmxb/yyiuxZs0avPXWWxg3bhyeeuopdHV1hZ1TXl6OTz75BLfccgtUKlVMTcLy8nLs378fNTU10Ov1yM7OhkwmC/UVrV27FpWVlUNugkASj2pEJCJGoxFGo3HIc/bu3Ytp06aF3R566KEBz12+fDmWLl2KO+64A/Pnz8fYsWOxYMGCsHMefvhh1NTUYNy4ccjLy4up3Pfffz94nscFF1yAvLw81NbWhh776U9/Co/Hk5RRQDI0yllNzluffvoprrrqKtTV1fXbkZUkFwUict5xu91obW3F0qVLUVhYiB07dkhdpPMeNc3IeWfnzp0YPXo0urq68Lvf/U7q4hBQjYgQkgKoRkQIkRwFIkKI5CgQEUIkR4GIECI5CkSEEMlRICKESI4CESFEchSICCGS+/+RzMc0HhwMagAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_pred_density_2\n", + "\n", + "_ = plot_pred_density_2(y)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e79e4b0f", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:41.003005Z", + "iopub.status.busy": "2024-02-29T23:33:41.002647Z", + "iopub.status.idle": "2024-02-29T23:33:41.377966Z", + "shell.execute_reply": "2024-02-29T23:33:41.376921Z" + }, + "papermill": { + "duration": 0.400345, + "end_time": "2024-02-29T23:33:41.380250", + "exception": false, + "start_time": "2024-02-29T23:33:40.979905", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAASIAAAEnCAYAAAANc04FAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABAfElEQVR4nO29ebxT5bX//9nZmeczzwNwAFFmEQoUR+qAY+1Vb7UWcK7YXuXnrXJvi1ctoq1ypcMPqy2DV5BaSmutVapYsTiggoAgMhw4A2ceM5zM2fv7x5PknJwxyUmyk5z1fr3y2snOzs46+ySfrGc961mLE0VRBEEQhITIpDaAIAiChIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCckiICIKQHBIigiAkRy61AaNBEAQ0NjbCYDCA4zipzSEIoh+iKMJms6G4uBgy2TB+jyghPp9P/MlPfiJWVlaKarVaHD9+vPjEE0+IgiBE9Pr6+noRAN3oRrcUv9XX1w/7XZbUI3rmmWewYcMGbNmyBeeddx4+//xzLF++HCaTCT/60Y9GfL3BYAAA1NfXw2g0JtpcgiCixGq1oqysLPRdHQpJheijjz7C9ddfj6uvvhoAUFlZiVdffRWffvppRK8PDseMRiMJEUGkMCOFTiQNVi9YsAC7d+/GiRMnAACHDh3C3r17cdVVVw16vNvthtVqDbsRBJH+SOoRPfroo7BarTjnnHPA8zz8fj/WrFmD2267bdDj165di8cffzzJVhIEkWgk9Yhee+01bN26Fdu2bcOBAwewZcsWPPvss9iyZcugx69atQoWiyV0q6+vT7LFBEEkAk4UpSuMVlZWhkcffRQrVqwI7fvZz36GV155BV9//fWIr7darTCZTLBYLEPGiERRhM/ng9/vj5vdxEB4nodcLqc0CiKMSL6jgMRDM4fDMSC3gOd5CIIQl/N7PB40NTXB4XDE5XzE8Gi1WhQVFUGpVEptCpFmSCpE1157LdasWYPy8nKcd955+OKLL7Bu3Trccccdoz63IAg4c+YMeJ5HcXExlEol/VonCFEU4fF40NbWhjNnzmDixInDJ68RRD8kFaJf/epX+OlPf4r7778fra2tKC4uxr333ovVq1eP+twejweCIKCsrAxarTYO1hLDodFooFAoUFtbC4/HA7VanbT3brI4kadXQc6T+KUrkgqRwWDA888/j+effz5h70G/zMkjGdfa4xPQYnWhLJv9uDg8Pvz5iwYY1Qp8e1YJdKq0XrU0ZqFvKZFWvHusBTsPNOBYE8shszi94DkObTY33jjUCL8g2dwLMQpIiIi0oaa9B8ebbQAAs1YBACgyaXDznDKoFDI0WVw4UNclpYlEjJAQpSAXX3wxHnzwQanNSDn2nekAAMwoM6HIpAntz9IpcfGkfADAp2c64fRQqka6QUJEpAVNFicau13gZRwuqMwe8PyUIgPyDCp4fAIOn+1OvoHEqCAhSjGWLVuGPXv2YP369eA4DhzHobS0FBs2bAg77osvvoBMJkNtbS0AYN26dZg2bRp0Oh3Kyspw//33w263h71m7969WLRoETQaDcrKyvCjH/0IPT09SfvbRsNXjSwmNLnQMGhAmuM4nF+RBQA40miFhHm6RAyMSSHy+IQhbz6/EPGx3giOjZb169dj/vz5uPvuu9HU1ISmpiZ897vfxbZt28KO27p1KxYuXIiKigoAbMbql7/8JY4ePYotW7bgvffew49//OPQ8dXV1bjyyivxne98B4cPH8Yf/vAH7N27Fw888EDUNiYbQRBxspWJ6pTCobNzq/L1MGkUGJerhddPQpROSLrEY7QMlz7ucrlw5swZjBs3bkBOy/++c2LIc47L1eGGWSWhx79+7+SQH+rSLA1umlMWevzCnuoB8YmHvjUp4r8nyMUXX4yZM2eG0hoOHjyI2bNno6amBuXl5RAEAeXl5fjJT36C++67b9Bz7NixA/fddx/a29sBAHfddRd4nsdvf/vb0DF79+7FRRddhJ6enrjk/Qx3zUdDm82N1z6vBy/jcM+i8ZDJhk5MFUWREldTiEiXeIxJjyjdmDlzJqZMmRLyivbs2YPW1lbcdNNNoWPeffddXHbZZSgpKYHBYMDtt9+Ojo6O0PKWQ4cOYfPmzdDr9aHbFVdcEcpAT2XyDCrcc+F43Di7ZFgRAkaue0OkJmMy+2vFJVVDPtf/c37PhROGPLb/Z/6OheNGY9aw3Hbbbdi2bRseffRRbNu2DVdeeSVycnIAADU1Nbjmmmvwgx/8AGvWrEF2djb27t2LO++8Ex6PB1qtFna7Hffee++glS/Ly8sTZne8UPAy5Bsi87IEQUSrzQ2zVgG1gk+wZUQ8GJNCpJRH7ggm6thhz6NUDqgWcOutt+InP/kJ9u/fjx07duCFF14IPbd//34IgoDnnnsulN382muvhb1+9uzZ+Oqrr1BVNbQIpyKxDLV2ftGA+k4HrjivEOcWU+XOdICGZilIZWUl9u3bh5qaGrS3t0MQBFRWVmLBggW488474ff7cd1114WOr6qqgtfrxa9+9SucPn0a//d//xcmVADwyCOP4KOPPsIDDzyAgwcP4uTJk3j99ddTPlh9vMWG//u4BvtrI09UzDeoAADNVmeizCLiDAlRCvLwww+D53mce+65yMvLQ11dHQA2PDt06BC+/e1vQ6PpTeibMWMG1q1bh2eeeQZTp07F1q1bsXbt2rBzTp8+HXv27MGJEyewaNEizJo1C6tXr0ZxcXFS/7Zoaex2ot3ugd3ti/g1hSY2hGuxuhNlFhFnxuSsGZEYEnHNt+2rQ4vVhSXTijC5cPhOEEEsDi82fngGvIzDikuqwI8Q4CYSB82aEWmPzy+g3c68mkJj5MJm1MihVvDwC2Lo9URqQ0JEpCztdg/8ggiNkodRE/m8CsdxKDCyOFGbjYQoHSAhIlKWZqsLAPOGop05y9axcrUdPZ6420XEnzE5fU+kBy0BISqIYlgWZEKeHjqVHCVmzcgHE5JDQkSkLFolj2ydEvmBYVY0lGVrQ1UcidSHhIhIWRZNzMOiiXlSm0EkAYoRERlLh92NU632qHKQCGkgISJSEp9fGHVNoXePteCNQ41o7KYM61SHhIhIST6r6cL//3419p3uiPkcJg2ra21xeuNlFpEgSIiIlKSjxw2PT4Ccjz0r2hgUIgcJUapDQpSCpFrxfCns6bCz/J8cXfQzZkGMaiZEVhcJUapDQpSheDzpm8jnF0R0B7yYHL0y5vPQ0Cx9ICFKMQYrnl9dXY0777wT48aNg0ajweTJk7F+/foBr7vhhhuwZs0aFBcXY/LkyQCAjz76CDNnzoRarcacOXPwl7/8BRzH4eDBg6HXHjlyBFdddRX0ej0KCgpw++23h0rMDmZPTU1NQq+BzeWFIIqQyzjoR9G51RTofWZz+SBQ48WUZmzlEYki4Jfg15FXDCznOATr16/HiRMnMHXqVDzxxBMAgKysLJSWluKPf/wjcnJy8NFHH+Gee+5BUVERbr755tBrd+/eDaPRiHfeeQcAW/l87bXXYsmSJdi2bRtqa2sHDLG6u7tx6aWX4q677sL//u//wul04pFHHsHNN9+M9957b1B78vISm9sT9GBMWsWoSr/qlXLwMg5+QYTd4wsN1YjUY2wJkd8L/Ou55L/vov8PkEc2xDCZTFAqldBqtSgsLAztf/zxx0P3x40bh48//hivvfZamBDpdDr87ne/g1LJ3uuFF14Ax3F46aWXoFarce6556KhoQF333136DW//vWvMWvWLDz11FOhfRs3bkRZWRlOnDiBSZMmDWpPIgkJkWYE4RBFoOkQ0HIUUGqBykWALjf0tEzG4eLJeVAreKjiVD2TSAxjS4jSmN/85jfYuHEj6urq4HQ64fF4MHPmzLBjpk2bFhIhADh+/DimT58eVhto7ty5Ya85dOgQ/vnPf0Kv1w94z+rqakyaFH0XktGiVcoxPk+H4pHWidX8C6j5sPdx5xlg9lJAlxPaNb3UnBgjibgytoSIVzDvRIr3HQXbt2/Hww8/jOeeew7z58+HwWDAL37xC+zbty/sOJ1OF/W57XY7rr32WjzzzDMDnisqKorZ5tFQla9HVf5AYQyjqxao/YjdL5/HHtuageN/B2Z9L+KhMJEajC0h4riIh0hS0r94/ocffogFCxbg/vvvD+2rrq4e8TyTJ0/GK6+8ArfbDZWKTYN/9tlnYcfMnj0bf/rTn1BZWQm5fPCPw2DF/CVFFIHq3WxbNAOYcCngsgCfvghYzgKdp4Ec1n3F6vKi1eqGRsnTSvwUhgbOKUj/4vkTJ07E559/jl27duHEiRP46U9/OkBQBuPWW2+FIAi45557cOzYMezatQvPPvssgN7+XytWrEBnZye++93v4rPPPkN1dTV27dqF5cuXh8RnsGL+iUIURTg9/uGXd7SfBGwt7Edl/MVsn9oEFM1i9xv2hw6tbrXjjUON+KIu8uL7RPIhIUpB+hfPv+KKK3DjjTfilltuwbx589DR0RHmHQ2F0WjEG2+8gYMHD2LmzJn47//+b6xevRoAQnGj4uJifPjhh/D7/bj88ssxbdo0PPjggzCbzaHWREMV808ETq8fL+ypxgt7TsM/1JR7UGiKZ7MgdZCS2WzbUQ24rAAAg5p5eXYXLXxNZah4/hhj69atWL58OSwWS1gnkHgQj2veZHFi+6f1MKjluGvR+IEHODqBfb9lw+x59wEac/jzB14GLA3AxG8BpXNGPh+RUCItnj+2YkRjkJdffhnjx49HSUkJDh06FMoRircIxYtgRvWQU/fNX7Jt9viBIgQAeVOYELV9DZTOCSVE9rj9EARxxJbVhDSQEGU4zc3NWL16NZqbm1FUVISbbroJa9askdqsIRk2h0gUmcAAQMHUwU+QOxE49S4TI68LOqUKHAcIogiH1z+qTG0icdB/JcP58Y9/jB//+MdSmxExwwqRvZUNzWRyIGeI1tkaM6DNZsdZ6iHLnQidUg6724cet4+EKEWhYDWRUgRLdgTXiYXRdoxtc8YPn4aRVcm2XTUAAF1AfKhSY+pCPw9EShH0iMyafkIjikBrYFiWN2X4k5grgIYDISGaNz4bPr8YUzcQIjlkvBCl8aRg2jHaay2KIqoK9LA4vDD394icXewm40PJikNiLmfbnnbA48CEvBGytAnJyVghUijYB9nhcKTsDFGm4XA4APRe+2jhOA6XTM4f/MmOQCa5qRSQj1AsTantjRPZmkYWLkJyMlaIeJ6H2WxGa2srAECr1Y6qpAQxNKIowuFwoLW1FWazGTzPx/9NOk+zbXaEomIsZkJkbYBdX4FmiwtKXobyHOp1lopkrBABCJWtCIoRkVjMZvOoSoU4PD7IOA5qRT8h83uB7kA2d6TejbEYaD4CWBtxVuPAW182ozRLQ0KUokguRA0NDXjkkUfw1ltvweFwoKqqCps2bcKcOXNGfW6O41BUVIT8/Hx4vVQuNJEoFIpRe0L7znTiYF035o3PxoIJvXWF0FULCD5AbQS0OUOfoC/GEra1NkJXyuxyelNo4S4RhqRC1NXVhYULF+KSSy7BW2+9hby8PJw8eRJZWVlxfR+e5xMzXCDiijUwY2ZQ9Ysx9R2WRTq81uUDvBzwuaH1dwMAHB4SolRFUiF65plnUFZWhk2bNoX2jRs3TkKLCCkZcnlH1xm2zY5irZhMBugLActZaD3tADRwevzwCyJ4WuaRckia0PjXv/4Vc+bMwU033YT8/HzMmjULL7300pDHu91uWK3WsBuRGYiiGPKIwoTIZWVBZ47rnZaPFD2bgVO7OiALeFI0PEtNJBWi06dPY8OGDZg4cSJ27dqFH/zgB/jRj36ELVu2DHr82rVrYTKZQreysrIkW0wkCrvbB58gQsZxodIdAHqD1PoCQBFlQmJAiLieVmiU7KPuoOzqlERSIRIEAbNnz8ZTTz2FWbNm4Z577sHdd9+NF154YdDjV61aBYvFErrV19cn2WIiUQSHZUaNPHyFfHct22ZVRH9SfQHb2lugCczEUZwoNZE0RlRUVIRzzz03bN+UKVPwpz/9adDjVSpVqOQpkVkMudg16BGZYxAiXR4b0nkcWDBRDZ88B3kG+vykIpIK0cKFC3H8+PGwfSdOnEBFRQwfOiKtydIpMbPMjCxdnzVmzm5242QsozpaeAWb7u9pxwRND5CTnHZIRPRIKkQPPfQQFixYgKeeego333wzPv30U7z44ot48cUXpTSLkIASs2ZgcfugN2QoHHlZx1Do8tiaM3srLfVIYSSNEV1wwQX485//jFdffRVTp07Fk08+ieeffx633XablGYRqUJoWBblbFlfAnGins5GnGq14WyXIw6GEfFG8szqa665Btdcc43UZhAS02pzwahW9C7vEMXRBaqDBGbOutoa8UZ7E6ry9SjNomUeqQYVRiMkx+X1Y+snddjwfjXcvsCslqub5RBxMsAYQ3woSGBJiNrbDYgC5RGlKCREhOQEExm1Sh4qecAjsjSwraFwdE0x1SaAl0PBiVD7rHCREKUkJESE5Aw6dW8NCJGpZHQn5zhAmws5z0Hj7YaT8ohSEhIiQnIGFSLLWbYdzbAsiC4XchkHrbcLLq9AVTtTEBIiQnK6+xfM97mBnjZ2f7QeEQBoc6HgZdD4uiCIIty+xLXMJmKDhIiQnAEekbWRzZqpTYDKMPo30OWyNWx+CwBQnCgFkXz6niAGCFFwWBYPbwgIzZxNMXlQPK0AGiXVpko1SIgIyZlZbka3w4Ps4PKOUKA6DvEhAFCbAZkcBTqgwCQAchKiVIOEiJCc2eV9KnIKQq8QxSNQDbAiadpstsyjpwPQxLcCKDF6KEZEpBaOdsDnYQtWdXnxO68uFw6PD2cb6tBmc8fvvERcICEiJKXb4UGrzQVPcCYrNG1fwjyZeKHNRYvVjQPHTuFEiy1+5yXiAgkRISmHzlqw9ZM6fHy6g+2wNbOtsSi+b6TNhpznoPZZKKkxBSEhIiSl2+EBAJiDM2b2FrbVx7l2kCab5RJ5LXD5SIhSDRIiQlKCU/dmrQIQ/Kx2EBBaNR83NFmQyzjIBRc8Tnt8z02MGhIiQjJEUYQlkFVt1igBRwdrpChXxn9mS66ETGNk7+voiu+5iVFDQkRIhq1/547QsKwg8kaKUSDXs+6xoqMz7ucmRgcJESEZllBDxUDnDlsfIUoAvJ5lWPNOEqJUgxIaCcnojQ8FMqrtiRUilSEX43N1qMjhIIoiuAR4XURskBARklFoUmPRxFwY1Aq2yDXBQiTX56LAqAZUjoQM/YjYISEiJCNXr0KuPtCdw9nFyn/IeECXm5g3DAbAnV1M+EiMUgaKERGpgb2VbXW5TIwSgdoMm0dAp7UHPVaaOUslSIgISRAEESdabGi3u1nFxGBGdYKGZQAAmQzHLTyOt9jR3taUuPchooaEiJAEm8uHNw834dV9dRBF9HpE8c6o7ocYGJ757B0JfR8iOkiICEno6GEr4M06JZu6twc9ojhnVPdD1GQDAIRgBjeREpAQEZLQ2cPWmOXolICnB3DbWfA4wULEaZkQUVJjakFCREhCUIiytMreaXtNVuw97iNEpmUzchwJUUpBQkRIQsgj0iv7ZFQn1hsCAJmeeURwWQC/L+HvR0RGTEJ0+vTpeNtBjCFEUURHQIiydcrElf4YBIXaAL9MAb8gsLbWREoQkxBVVVXhkksuwSuvvAKXyxVvm4gMp8fjh8cngOMCdYjsyfOIirO0qCgtQ5FJw1b7EylBTEJ04MABTJ8+HStXrkRhYSHuvfdefPrpp/G2jchQlLwM184oxoWT8iAXfSzTGWB97hNMtk6JspIS1rrISUmNqUJMQjRz5kysX78ejY2N2LhxI5qamvDNb34TU6dOxbp169DW1hZvO4kMQimXoSpfz7p32FvYcguVHlDqkmNAYAofFLBOGUYVrJbL5bjxxhvxxz/+Ec888wxOnTqFhx9+GGVlZfj+97+PpibKXiVGIJTImMCM6j74BRHNXg06e9wQaWiWMoxKiD7//HPcf//9KCoqwrp16/Dwww+juroa77zzDhobG3H99dfHy04igzhQ14XTbXZ4/ULCV9z3xy+I+OsJF4632OHvISFKFWJafb9u3Tps2rQJx48fx5IlS/Dyyy9jyZIlkAXav4wbNw6bN29GZWVlPG0lMgCX1489x9nQ/b6LJkBhT8Iasz4oeA5elRkA4HfZIfe5E567RIxMTEK0YcMG3HHHHVi2bBmKigZv+5Kfn4/f//73ozKOyDyCzQ0Najk0cvQWyzckR4g4joNCpYFXpoFPEKFydiUlSE4MT0xC9M4776C8vDzkAQURRRH19fUoLy+HUqnE0qVL42IkkTk0W1m6R4FRzURI8LNi+Wpz0mxQyXm4FCb4/A4WsCYhkpyYYkQTJkxAe/vARYOdnZ0YN27cqI0iMpfGbicAoNisSXix/KFQK2RwyY0sqZHqV6cEMQmRKIqD7rfb7VCr1aMyiMhcRFFEYzfziErMmqSV/uiPWsHDJTfBJ4g0hZ8iRDU0W7lyJQA2zl69ejW0Wm3oOb/fj3379mHmzJlxNZDIHDp7PHB5/VDwHPIMKuB0ckp/9Ecll6FbERAiSmpMCaISoi+++AIA+2X78ssvoVQqQ88plUrMmDEDDz/8cHwtJDKGJgvzhgpNGvAceodmSY7RTCkyolw5AaYzn7FlHlS/WnKiEqJ//vOfAIDly5dj/fr1MBqNCTGKyEzOKzaiyKSG1x/wRHweQCYHtDlJtaMiRweYK4EGOSvY73UCSu2IryMSR0yzZps2bYq3HcQYgOM45AS7drTWsG0ii+UPB68AVAbAbWMBaxIiSYlYiG688UZs3rwZRqMRN95447DH7ty5c9SGERlOkjOq++Ly+tFmc0MPPbJgYwFrU2nS7SB6iXjWzGQyhTpjmkymYW+x8PTTT4PjODz44IMxvZ5Ibd77ugVvHm5Cqy1QNkZCIWrsdmLH/rM41BHwxChgLTkRe0R9h2PxHpp99tln+O1vf4vp06fH9bxEauD1CzjWZIPHJ2BGWeCHKhSoTr4QqRRMgGycge2gXCLJiSmPyOl0wuFwhB7X1tbi+eefxz/+8Y+oz2W323HbbbfhpZdeQlZWVizmECnO6bYeeHwCDGo5yx9y23uL5euSO3UPAGo5+9jbZIHJFsolkpyYhOj666/Hyy+/DADo7u7G3Llz8dxzz+H666/Hhg0bojrXihUrcPXVV2Px4sUjHut2u2G1WsNuROrzRR0b+pxbbGTD+1Cx/Gy2vCPJqAMekZXTQ4TIPKIhknSJ5BBzhcZFixYBAHbs2IHCwkLU1tbi5Zdfxi9/+cuIz7N9+3YcOHAAa9eujej4tWvXhsWiysrKYjGfSCKN3U40WVzgZRxmlJrZziSWhh0MVcAjcvEG+ESOFdF32ySxhWDEJEQOhwMGAxtf/+Mf/8CNN94ImUyGb3zjG6itrY3oHPX19fiP//gPbN26NeJlIatWrYLFYgnd6uvrYzGfSCJf1HUDACYXGqBTBUKSEgaqAUDOy6DgOYgcD58iMDyjOJGkxFw8/y9/+Qvq6+uxa9cuXH755QCA1tbWiJMc9+/fj9bWVsyePRtyuRxyuRx79uzBL3/5S8jlcvj9/gGvUalUMBqNYTcidbE4vTjZyjyN2eV94n/BNWYSBKqDBIdnHmUgeE4zZ5ISU0Lj6tWrceutt+Khhx7CZZddhvnz5wNg3tGsWbMiOsdll12GL7/8Mmzf8uXLcc455+CRRx4Bz0uQ5EbEFY2Cx0WT8tBmc7O1ZQDLZA4GhyXyiADgG+NzIIgiVNY8wFFPAWuJiUmI/u3f/g3f/OY30dTUhBkzZoT2X3bZZfj2t78d0TkMBgOmTp0atk+n0yEnJ2fAfiI9UcplmFXebyY06A2pDMkrlj8IU0sCnlBDLtAC8ogkJiYhAoDCwkIUFoYvVpw7d+6oDSIyHInjQwPQBISSPCJJiUmIenp68PTTT2P37t1obW2FIAhhz8faCfb999+P6XVEaiGKInYdbUZplhaTCw1Q8H1CkbZAZxeJqyJaXV5YHF7oRQOyANb1VRAAGXVhl4KYhOiuu+7Cnj17cPvtt6OoqCi09IMgAFYO9liTDada7ZhUYAh/0haoQSSxEB2s68b+2i6cX27GhTI5IPgAt6XXQyKSSkxC9NZbb+HNN9/EwoUL420PkQF83cRmyqry9VDK+3gYfm9vm2eJhSg4a+byCYDGzOpnOzpJiCQiJj80KysL2dnZ8baFyAAEQcTxFiZE5xT2S68IdnVV6gClXgLregkmNbp9AqANfJYpYC0ZMQnRk08+idWrV4etNyMIgA3LnB4/VAoZyrL71fix9anIKPFwPuQRef0UsE4BYhqaPffcc6iurkZBQQEqKyuhUCjCnj9w4EBcjCPSj5r2HgBARbYOvKyf2NhTIz4EsE4eQHBoFvSISIikIiYhuuGGG+JsBpEpnOlgQlSZO0jFw+CMWZK7dgxG0CNye/00NEsBYhKixx57LN52EBmAXxChkMkgl3GozOmXrOj3AT2pEagGALU8IER9PSKXhdnJx5xeR8RIzFe8u7sbO3bsQHV1Nf7zP/8T2dnZOHDgAAoKClBSUhJPG4k0gZdxuPmCMvj8AuR8v/CjvQUQBVYbWmUY/ARJRKPksbAqF2qFDKJCC06uZMX8Xd2sjjaRVGISosOHD2Px4sUwmUyoqanB3XffjezsbOzcuRN1dXWhWkXE2GSACAGAtZFtjSWSB6oBtvxk7rg+M7+aLBZMd3SSEElATLNmK1euxLJly3Dy5MmwEh5LlizBBx98EDfjiPTC6xeGftJ6lm2NKeotU8BaUmISos8++wz33nvvgP0lJSVobm4etVFE+uEXRLz4wWm88kktnJ5+JVxEEbAEhMiUOkLUYXejvtPB7KWAtaTEJEQqlWrQMq0nTpxAXl7eqI0i0o92uxsenwCbyxeaGg/htgZqVMsAQ5E0Bg7CrqMt2LH/LBotzl6PiHKJJCEmIbruuuvwxBNPwOv1AmCN8+rq6vDII4/gO9/5TlwNJNKDNpsbAJBvUA1ce2hpYFt9PmtsmCIEBdPt7ZtdTUIkBTEJ0XPPPQe73Y68vDw4nU5cdNFFqKqqgsFgwJo1a+JtI5EGBIUoVACtL9aAEKVYE8Pe9WZ9sqvddjZ7RiSVmGbNTCYT3nnnHXz44Yc4dOgQ7HY7Zs+eHVEnDkI6BEFEi82FLK0y9CWMF8MKUXegjnmKBapD2dVeP6DQsJvXybyiFMh1GktELUSCIGDz5s3YuXMnampqwHEcxo0bh8LCQoiiSCVBUhS/IOJP+8+iodsJg1qOm+aUwaSJzzBJFEW02YcQIk8PYG9j97Mq4vJ+8UIVTGr0Bmb7tNlsGOnsIiFKMlENzURRxHXXXYe77roLDQ0NmDZtGs477zzU1tZi2bJlEZeJJZLPlw0WNHQ7AQA2lw97T7bH7dwWpxcenwC5jEO2tl+fsq6AN6TPk7Q07GCEeUQABawlJCqPaPPmzfjggw+we/duXHLJJWHPvffee7jhhhvw8ssv4/vf/35cjSRGz+Gz3QCA6aUmHD5rwalWO3rcvt4WP6NAFIEpRQYIIiDrv9C1q4ZtsypH/T7xRtV3mQdAAWsJicojevXVV/Ff//VfA0QIAC699FI8+uij2Lp1a9yMI+LH1dOK8I3xOVhYlYsikxqCKOJkqz0u587SKXHl1CIsmdZval4U+wjRuLi8VzwpNKmxsCoXU0sCdZPII5KMqITo8OHDuPLKK4d8/qqrrsKhQ4dGbRQRf3L0KsyfkAO1gseUIiOq8vVxixENSU87W0gqkwOm1OvKm6tXYe64bFTlB9a+BWfOyCNKOlH55Z2dnSgoGLr7QkFBAbq6KDM11ZlRZsaMMnPczmd3+6BV8AOHZe0n2DarUpIe91ETFCKvC/A42AJdIilE5RH5/X7I5UNrF8/z8Pl8ozaKiB+iKOKfX7fiaKMFvuHWgo3i/Fs+qsGv/3kK3Y5++TdBIcqdGPf3jQeCIKLV5kJ9pwOiKDKxDFYGoKUeSSUqj0gURSxbtgwq1SC5IgDcbndcjCLiR0ePBwfru6GUyzClTw1pURRhcbLMeHP/ma4o6PH44fEJ4DhA3zfw7ehkHTs4Dsipivn8icQvitj6SR0A4P5LJrDgtTYbcNvY8CyF1sVlOlEJ0dKlS0c8hmbMUotmiwsAW3rRd+i070wnPq7uwHnFRlx+Xuw5M109zAsyqhXh5T+aA+3Es8YBKmkL5Q+FgmdF3HyCCJdXYEKkyWYpBxSwTipRCdGmTZsSZQeRIDoCQtE/0TBXzx632EbnxXY7mFeVrevjVQkC0HKE3S+cNqrzJxq1gofd7WMlYzUKClhLBLW1zHDaA0ITFJ4ghSZWR6rT7hm+jtAIdAXiQmZtnxm49uOAywoo1EDupJjPnQxUwYWvoVyiHLYN9l8jkgIJUYbT0cOEKEcfHgfSq+TQKHkIohgaXsVCUIiygnEmUQTqPmb3S+akfP3nYO3qUHa1LihEXcyzI5ICCVEG4/D40ONmX7Ac3cAJhuBwqmMUQhQcmoWEqOUoK7nKK4CS82M+b7JQhZZ5BERHbWbiKfhY/WoiKZAQZTCdAYExaRThrZ8D5Aa8pA577EI0udCASQUGZOuVbIFr9W72RMWCtMjDCSsFArBZvuDwrCd+6/GI4Ultv5kYFaVZWtx70fiQV9Sf7ICXFBy+AWD96UWReTQRVFL4xvjAl9bnBg7vZImAulygbN6o7U8GVfl6ZGmVKM3S9O7U5gYK6bcDSO0YV6ZAQpThaJVyaJWD/5tLzBrMG5eNElkH8NXrbNrawxokQsazYYo2mwmLNjewzQmvsiiKgKUeOPkPVu5DrgTOu5G9Pg2YkKfHhP7VjXWBHT1tSbdnrEJCNIbJ0/LI8+0Dmg4PfFLws5kjRwfQfrJ3P8cBahOgNsPp8YJzdkIlOMCBY0OxaTf1BnzTlWA7IRqaJQ0Sogzm3a9aoJTLcH5F1sByH34vcPgPQHc9E5fC6SznR5cLcDzgdbBlDo5ONkTpaWcegtcJOLsBZzea2u1osbpRnG1AxblzgXEXplzNoZHw+AR0Oz0QRaDAGGiNFZrC72QzZzIKpSYaEqIMxS+IONJogSgCsyuywp8UReDYG0B3PdycAu3lV0NTMCE8KVGuBDRmILtf+Q5PDxMltw2njrWiTiOHaca5QFl6NiVssjix80ADcg0q3P6NQAVJtZlVDAjOnAXrFBEJg6Q+Q7E6vRBFQMFz0Cn7xWuaDgJtxwEZj890l+K1ah5fNw9sDzUoSh0r+Vo4FXXySthUhcgyppcX1BdNcNasby82max3eEnDs6RAQpShBBMNTVpleB1xtw04FZhiH38xNPnM4+mMMpco2MMM6JNDlIaoAyLt9PrZCvwg2oCH5yAhSgYkRBlKd3Blff/iZzUfsviQsQgovSA0HIs2u7rbyY7XKPm4dwRJJkGPyC+Ivcs8AApYJxkSogzF0j/jGWDB16ZABc0JlwJcb7H7LocXgiD2P82Q9GZUp07DxFhQ8LJQsmdomQdAHlGSISHKUIIeS9hi1PpPAVFg9YHM5QAAg1oOuYyDXxBhdXkjPn9wKDeaWkapQtCjc/YVopBH1EFrzpIACVGGYg/Eb0J1qT2O3hpBZXNDx8lkHMyB4Vk0caLSLA0uqMzGhLz0DVQHCQ7PnH0D1v1nzoiEQtP3Gcr3vlGBHo8f6uAas6aD7EtlKAh5Q0FydEq029yhAHcklGZpUZqV+mvJImFaiQkTC/Thw1iZjPViszYB9laawk8wJEQZCsdxvaVbRbHXGyqZM2AN2dRiEypytCgxazAWmVZqGvwJfUFAiFqA/HOSa9QYg4RoLGBtYIFqXgHkDfxCledE59l4fAJarC5k6ZThdaozDV0+29pbpbVjDCBpjGjt2rW44IILYDAYkJ+fjxtuuAHHjx+X0qSM4GSLDW8casTRRgvbEfSG8ibHpa1Pm92NHfvP4g+f1Y/6XKmAy+tHq9WFDnu/srn6oBC1JN+oMYakQrRnzx6sWLECn3zyCd555x14vV5cfvnl6OnpkdKstKfJ4sKpVjva7R7A7wNaj7EnhqkfXd/pwKH67vAp7CEI5hyl+9R9kJMtdmzdV4e9p/pN1QeFyG1jwX4iYUjqV7/99tthjzdv3oz8/Hzs378fF154oURWpT/BNkEmjQLormW1glR6wFwx5Gv+8VULrE4vcvTKEYPQofKwuvSfugcAjXKQPCIAkKtYMX1nF/OK+q+7I+JGSk3fWyxsKJGdPfgMhdvthtVqDbsRAwkTolCTw0nDFjrL1jHvpqtn5Fyi4DR/dgbkEAG9eUQOzyDeoJ7iRMkgZYRIEAQ8+OCDWLhwIaZOnTroMWvXroXJZArdyspSr5+61PRtnGhS8xF3Ww1OXXdGMIUfHJplZ4hHFCwc5xxsWKoPtFinOFFCSRkhWrFiBY4cOYLt27cPecyqVatgsVhCt/r6zAiWxhOXV4AnsGbK6GlhsQ25athhGdBbXH+kNWd+QYTFyZIlzRkSIwomNLq9Avz9l7kEhaiHPKJEkhJzrw888AD+9re/4YMPPkBpaemQx6lUqiHbXROMoDekV8kh7zzFduZUjVi6NSswNBspu7rb4YEgilDKZRkzda+Sy8BxLN3K5fWHF5HTB8vGdrDAf4q3R0pXJL2qoijihz/8If785z/j/fffx7hxFAwcLQ6PD3IZB5Na3lviNYImh8FhltXlhdcvQMEP7ixrlXIsnlIAj18ILy+SxshkHNQKHk6PHw5PPyFSGVmjSK+LVag0FklnaAYjqRCtWLEC27Ztw+uvvw6DwYDm5mYAgMlkgkYzNrN8R8v4PD0euLQKHls7sL+LeULZ40d8nUbBynm4vH50OTzIN6gHP07JD52JnMacX5EFDuzvC4PjAEMR0HkGsDWRECUISYVow4YNAICLL744bP+mTZuwbNmy5BuUIXAcB5W1jj0wlkSUxMhxHL51bgHUCllaFzqLlQsqh1lLFhQiayNQMjt5Ro0hJB+aEQmi6wzbRpH7UpWvH/GYU612aJQ88g2qIYdvGYexhG1tTdLakcFQ5C3DeP1gA3hOwOKO01BzALLiF3cTRRG7jjbD4xNw+/wK5OozZ+LA5fXD6vRCzssGpiUEh2OODpYcKs+cvztVGCM/aWMDvyCipt2B5rrTkAleQKEBDIURv97p8eNIgwX7azsHfd7q9MHjE8DLuIwbvh1ttGDrvjrsO90x8EmljvVyE0XyihIECVEGYXf5IIgicjwNUPAcG5ZFMbPl8vrxzlct+Li6Y9Bhc5vdBQDI0SvByzJjxixIMKmxZ7DsaqDXK7I2JsmisQUJUQYRzCEq9DexzqtRDstMGgV4GQevX4Q1UOGxL202lmOUSUOyILqAEDk8A/9uAIChmG1JiBICCVEGYXF6wftdyBICw4usyqheL5NxoRX1g2VYtwXKZGSkEKnYtH2PeyiPKCBEtiY2RCPiCglRBmFxemF2N0All7Hi72pj1OcIrqjv6HEPeK7NxvblGzJRiJhH5PL64fMPUizfUAhwMsBtB9y02DrekBBlEBanFybnWVanOsaSFcFExmZLuBD1uH2wOr3gOCAvA4VIJZeF4l6OwRa/8gpW7xsAummNY7whIcogRFFAtqcBKgUf87R9kYkJUZPFGbZfreBxywVlWDylIK0bKg4Fx3HQBrKqHUMNz0yBag8WEqJ4Q3lEGcQ1VWqInUqIHD+gU0ekFBjVkHEcbC4f7G5faGErL+NQbNagOIML7M+uyIIgiNCqhhBacznrDUceUdwhIcokus6AAwfOXM6GEjGglMtw4+wS5OpVA9ddZTizy7OGP8BUytIhHB0sVqQaOROdiAwammUSndEv6xiMsmxtmAg5PD7sOtqMU622UZ037VFoejvAWs5Ka0uGQUKUIXzd2IWDRw6hodsZ9bT9UIiiCFEUcbqtB181WrHvzOAZ15mCxyeg1epCq9U19EGmwJCX4kRxhYQoQ7C31sLpdMEJVW9VwVHwdbMVr+yrw8lWe6gtUVVeZg9FTrWybh7/Otk+9EHmQMC6qyYpNo0VKEaUIXjbqwEAspzxUS3rGIpOuwftNjfePMzWVvEyDlNLMq8OUV8MavZ1sLmGaSBgrmDXt6cdcFljytUiBkIeUYbABX6hVfkT4nK+2RVZoS8mAMypyAqvXJiBGNUswG9z+YYuUaPU9i4kDpZaIUZNZn+yxgii2w7O1gwRgK6wKi7nVCt43DSnDIfPdsOkUWBahntDAFvmwXGATxDh9PpDC2EHkD0esDYBnaeBohnJNTJDIY8oA+hpqYYginAqc2A0jjAFHQUmjQKLJuZheqk5Y+pTD4ecl4UWv9oGWfQbIlh6t6sGEAZZDkJEDQlRBuBsZt06fOZKyDKsPEey0UcSJzIUs+JoXhfVJ4oTJETpjihCZqmBRsGDzxm5SD4xPMG42GBlUELIZL0pEp3ViTdqDEAxonSnpx05cg9yKnMhzqPC7qPlnEIjikxqlGaNsJQldyLQdpzdxl2YHOMyGBKidKfzNNuaysHFuKyD6CWSBgIAWNNKTsam8Xs6AF1OYg3LcGholuaIHSchQoyodxkRRxSa3uFZ+3FJTckESIjSGY8Dno46fFbThT/Xa6g9UxwQBBGtNhdOtthGvp55gQ66bSREo4WEKJ3pPA2n2wsbnwUbpx8TU+yJRgTw6r56/O1wE+zuYQLWAGvlzXGArRlwdiXFvkyFhCid6TgJh9ePLk0FcnSZVzVRCngZB6OGhU67HcNM4QOszVCw7lPL0QRbltmQEKUrfh/QeRo9bh86NRXI1WdWnzEpCfZsG1GIAKBwGts2f0lF9UcBCVG6YqkDfB50+5ToUeahwKiW2qKMwRTsZOIY2MlkALmTWRE6ZzeVBhkFJETpSuvX8AkCGuWsaiAJUfwIekQRCZFcCeRPYfebjyTQqsyGhCgd8fuAtq/R4/ajXTsBJo1izJV1TSQ5gZZK7fYIhAjoHZ61HgW8zuGPJQaFhCgd6ToD+NyASo/c0ipU5mqltiijCLZLsjq9cA3WWqg/pjJAn8d+IJoOJdi6zIQyq9ORwAyNqXw6rp9YJrExmYdaweOiyXmhFtwjwnFA6QXA138HGvYDpXPZejQiYuhqpRteF9Bxkt0vOFdaWzKY2eVZmJCnh4KP8CuSfx4rmuayUqZ1DJAQpRstRwC/Dx51DizyXKmtIYLwcqA4sOi4Zi9N5UcJCVE6IYpA4xcAgDPKidj4YQ3ePkL1cBKBzy/gdJsdn9dE0bmkdA6rU9TTDrR+lTjjMhASonTCcpZ9yHk5TqACAJCjp4zqROD1i3j9YCP+dbIdDs8ISz2CKDRA2Tx2/8y/ACGCQDcBgIQovajfBwDw5U5BrYV9yCuyacYsEWiUfChbvbE7iin50gtYrMjZxQLXRESQEKUL9lag/STAcajTT4fXL8KoUYSmmon4UxIojna2Kwohkit7C6Wd+YAFr4kRISFKF2r2sm3eZJy0sV/qCXk6WnGfQEqzmLdZ2+GI7oVFMwFTCeD3Aid2UeA6AkiI0oHuOlbzhuPgLp2PU612AFFUEyRiojxbC17GobPHgw67O/IXchww6UpAxgMdp2iIFgEkRKmO4AdOvsPuF83EGacWHp+AbJ0SJeYR6ioTo0Kt4FEeiMEFxT9i9PnAhEvZ/er3gG5aEDsclFmd6tT8i8WHFGpg3CJMVmghl3HwC6BhWRKYWKDHmfYetEXjEQUpOR/orgXaTgBHdgCzbgd0lPs1GCREqUz7KaDuE3Z/0lWAUgcOQFW+QVKzxhKTCgzI0ipRZIqhugHHAVOuA9zbAGsjcHAbMP0WwFAQf0PTHBqapSrd9cBXf2aBzuKZcJonRp7PQsQNBS9DsVkTu/fJK4Bp/8aGap4e4OArQOux+BqZAaSEEP3mN79BZWUl1Go15s2bh08//VRqk6RDFFldm0Pb2Wru7HHwjV+MN79swpaPaqOPVRBxw+ry4mijJfoXKnXAzNtYWVmfBzj6F+Cr1wFXDOfKUCQXoj/84Q9YuXIlHnvsMRw4cAAzZszAFVdcgdbWVqlNSz7WJuDLHcCxNwDBB+ROhGXCddh5sBn1nQ4IogizlnqXSUGP24dt++rwzlctOFjfHX3HFIUamPFdoGI+G7K1fAXsexH4+k3A0jDmp/g5UeIeNPPmzcMFF1yAX//61wAAQRBQVlaGH/7wh3j00UeHfa3VaoXJZILFYoHRaEyGufFBFFkBLU8P4GhnXSA6qwF7GwDA7RfRmjMHXymm4kRLD3yCCKVchutmFKOMMqkl4/3jrfiirhsAS3acWmxCsVkNo1oBWSTlQoLYmoFTu1laRhClDsiqAPQFgC4PUBkApZ4tG0njSYlIv6OSBqs9Hg/279+PVatWhfbJZDIsXrwYH3/88YDj3W433O7e2QurNcKs1c4zwKl3B+4P02Cx3z5xkOPC951qtaPb6Q4dwgGBX0p23PnlWQh+PqvbbOiwewCI4AUvIAphZ5xZZoaclwP5U7CnpwpH2+UA2DCsJEuDxVMKkK2jAvlSctGkPOhVcnxc3YGGLicaAhnXHAd8Z3Zp6EfiSIMF+2u7MJQ2fevcQhTOvBWwNqDh6F4ouqqRjx7mJbUMslhWJmer+2UKlpuEwIk5rt99DLyfSKbdDKjj4wBIKkTt7e3w+/0oKAifRSgoKMDXX3894Pi1a9fi8ccfj/6NfG62WDTO+JxWeIfp9MB5VaEPheBxwu9hpUeDSyH9MiWcchMcimxMmzwX8oJJgEIDzck25AsOFJs0mFxoQJFJTVP1KQDHcZhTmY1JhQYcabCgpt2BdrsbfkGEWtFbqtfp9aOzZ+gys16WewGYStFediWEYh/yjTbA1gTYWwBHB+C295adFXzsBleC/8IoCfyYxoO0mr5ftWoVVq5cGXpstVpRVhZBhUJzGTDj3/v9UgzySxLRvt7ncns8MPiE0D6OAzhO1nu4Uc36owMocHph8gvsSF4FUaEBJ5OHjlVolQj+hC6amDfy30RIhlGtwIIJuVgwgXWGdfn8UMl7hWhyoQGFRvWQYZ++6wPH5+mYV6zOBbLHhR/o9wF+N1sqIvjYVgz8jPX10ge9nwSUuridSlIhys3NBc/zaGlpCdvf0tKCwsLCAcerVCqoVDEs8lTqBv6T40BuFF6pUQekURSLiBCZjINWGf41MqoVMKojm1QwDHccHxiSjQEknTVTKpU4//zzsXv37tA+QRCwe/duzJ8/X0LLCIJIJpLL7cqVK7F06VLMmTMHc+fOxfPPP4+enh4sX75catMIgkgSkgvRLbfcgra2NqxevRrNzc2YOXMm3n777QEBbIIgMhfJ84hGQ9rmERHEGCHS76jkmdUEQRAkRARBSA4JEUEQkiN5sHo0BMNbES/1IAgiqQS/myOFotNaiGw2GwBEll1NEIRk2Gw2mEymIZ9P61kzQRDQ2NgIg8Eg+Vqs4HKT+vp6msHrA12XoRkL10YURdhsNhQXF0MmGzoSlNYekUwmQ2lpqdRmhGE0GjP2QzUa6LoMTaZfm+E8oSAUrCYIQnJIiAiCkBwSojihUqnw2GOPxVYdIIOh6zI0dG16SetgNUEQmQF5RARBSA4JEUEQkkNCRBCE5JAQEQQhOSREURBNR9qXXnoJixYtQlZWFrKysrB48eKM7WAba6fe7du3g+M43HDDDYk1UEKivTbd3d1YsWIFioqKoFKpMGnSJPz9739PkrUSIhIRsX37dlGpVIobN24Ujx49Kt59992i2WwWW1paBj3+1ltvFX/zm9+IX3zxhXjs2DFx2bJloslkEs+ePZtkyxNLtNclyJkzZ8SSkhJx0aJF4vXXX58cY5NMtNfG7XaLc+bMEZcsWSLu3btXPHPmjPj++++LBw8eTLLlyYeEKELmzp0rrlixIvTY7/eLxcXF4tq1ayN6vc/nEw0Gg7hly5ZEmSgJsVwXn88nLliwQPzd734nLl26NGOFKNprs2HDBnH8+PGix+NJlokpAw3NIiDYkXbx4sWhfcN1pB0Mh8MBr9eL7OzsRJmZdGK9Lk888QTy8/Nx5513JsNMSYjl2vz1r3/F/PnzsWLFChQUFGDq1Kl46qmn4Pf7Bz0+k0jrRa/JItqOtIPxyCOPoLi4OOyDme7Ecl327t2L3//+9zh48GASLJSOWK7N6dOn8d577+G2227D3//+d5w6dQr3338/vF4vHnvssWSYLRkkREng6aefxvbt2/H+++9DrVZLbY5k2Gw23H777XjppZeQm5srtTkphyAIyM/Px4svvgie53H++eejoaEBv/jFL0iIiOg70vbl2WefxdNPP413330X06dPT6SZSSfa61JdXY2amhpce+21oX2CwPqny+VyHD9+HBMmTEis0Ukils9MUVERFAoFeL63ffWUKVPQ3NwMj8cDpVKZUJulhGJEERBrR9qf//znePLJJ/H2229jzpw5yTA1qUR7Xc455xx8+eWXOHjwYOh23XXX4ZJLLsHBgwczqtJmLJ+ZhQsX4tSpUyFxBoATJ06gqKgoo0UIAE3fR8r27dtFlUolbt68Wfzqq6/Ee+65RzSbzWJzc7MoiqJ4++23i48++mjo+KefflpUKpXijh07xKamptDNZrNJ9SckhGivS38yedYs2mtTV1cnGgwG8YEHHhCPHz8u/u1vfxPz8/PFn/3sZ1L9CUmDhCgKfvWrX4nl5eWiUqkU586dK37yySeh5y666CJx6dKloccVFRUigAG3xx57LPmGJ5horkt/MlmIRDH6a/PRRx+J8+bNE1UqlTh+/HhxzZo1os/nS7LVyYfKgBAEITkUIyIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCckiICIKQHBIigiAkh4SISCs2b94Ms9kcevw///M/mDlzZujxsmXLMrriY6ZCQkQMyrJly8BxHO67774Bz61YsQIcx2HZsmVhx8dbACorK/H888+H7bvllltw4sSJIV+zfv16bN68OfT44osvxoMPPhhXu4j4Q0JEDElZWRm2b98Op9MZ2udyubBt2zaUl5dLYpNGo0F+fv6Qz5tMpjCPiUgPSIiIIZk9ezbKysqwc+fO0L6dO3eivLwcs2bNGtW5B/NUbrjhhpCXdfHFF6O2thYPPfQQOI4Dx3EABg7N+tPXM1u2bBn27NmD9evXh85x5swZVFVV4dlnnw173cGDB8FxHE6dOjWqv4uIDRIiYljuuOMObNq0KfR448aNWL58ecLfd+fOnSgtLcUTTzyBpqYmNDU1RX2O9evXY/78+bj77rtD5ygvLx/wNwHApk2bcOGFF6KqqipefwIRBSRExLB873vfw969e1FbW4va2lp8+OGH+N73vpfw983OzgbP8zAYDCgsLByxAN1gmEwmKJVKaLXa0Dl4nseyZctw/PjxUGsfr9eLbdu24Y477oj3n0FECFVoJIYlLy8PV199NTZv3gxRFHH11VenfZnX4uJiXH311di4cSPmzp2LN954A263GzfddJPUpo1ZyCMiRuSOO+7A5s2bsWXLlrh5DTKZDP0r0Hi93ricOxLuuuuuUCB+06ZNuOWWW6DVapP2/kQ4JETEiFx55ZXweDzwer244oor4nLOvLy8sLiP3+/HkSNHwo5RKpWjbqUz1DmWLFkCnU6HDRs24O2336ZhmcTQ0IwYEZ7ncezYsdD9obBYLAPaBOXk5Axai/rSSy/FypUr8eabb2LChAlYt24duru7w46prKzEBx98gH//93+HSqWKaUhYWVmJffv2oaamBnq9HtnZ2ZDJZKFY0apVqzBx4sRha48TiYc8IiIijEYjjEbjsMe8//77mDVrVtjt8ccfH/TYO+64A0uXLsX3v/99XHTRRRg/fjwuueSSsGOeeOIJ1NTUYMKECcjLy4vJ7ocffhg8z+Pcc89FXl4e6urqQs/deeed8Hg8SZkFJIaHSsUSY5Z//etfuOyyy1BfXz+gESKRXEiIiDGH2+1GW1sbli5disLCQmzdulVqk8Y8NDQjxhyvvvoqKioq0N3djZ///OdSm0OAPCKCIFIA8ogIgpAcEiKCICSHhIggCMkhISIIQnJIiAiCkBwSIoIgJIeEiCAIySEhIghCckiICIKQnP8HIaoZ92HQ/HgAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_density_3\n", + "\n", + "_ = plot_density_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "745adde1", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:41.424789Z", + "iopub.status.busy": "2024-02-29T23:33:41.424484Z", + "iopub.status.idle": "2024-02-29T23:33:41.629345Z", + "shell.execute_reply": "2024-02-29T23:33:41.628265Z" + }, + "papermill": { + "duration": 0.228528, + "end_time": "2024-02-29T23:33:41.631475", + "exception": false, + "start_time": "2024-02-29T23:33:41.402947", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAATgAAAEmCAYAAAD2o4yBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAArXUlEQVR4nO3de1hU16E+/neGy3BHFLmKcjOiSQCFB9SaxgsCyS9Ga5oQk8hFq+do7ImdWFtSxQe14TRRQ+wxcuoJoqaJ9mJsTpJ66URzvOANg6IVDAioEVBUGIE4Dsz6/eGX3YzM4AyCA5v38zzz4F6z9p612Mzr3rP2rK0QQggQEcmQ0tYNICLqKQw4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLbsbd2A3shgMODq1atwd3eHQqGwdXOI6AeEELh9+zYCAgKgVHZ+jMaAM+Hq1asICgqydTOIqBOXL1/GkCFDOq3DgDPB3d0dwL1foIeHh41b03P0ej327t2LxMREODg42Lo59JD6y/7UarUICgqS3qedYcCZ0H5a6uHhIfuAc3FxgYeHh6zfEP1Ff9uflnx8xEEGIpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFs8TIRoj6qpaUFpaWl0nLT9zocKamAl/dJuDmrjOpGRETAxcXlUTfR5hhwRH1UaWkpYmJiOpS/Y6JuUVERxowZ0/ON6mUYcER9VEREBIqKiqTlspoGqP9cgnUvPokR/gM61O2PGHBEfZSLi4vRUZmy+gZUB7/HyCeiED1skA1b1ntwkIGIZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxYAjItliwBGRbDHgiEi2GHBEJFsMOCKSLZsH3IYNGxAcHAwnJyfEx8fj+PHjZusWFBRAoVAYPZycnIzqpKend6iTnJzc090gol7IphNe7tixA2q1Gnl5eYiPj0dubi6SkpJQVlYGHx8fk+t4eHigrKxMWlYoFB3qJCcnY/PmzdKySqXqUIeI5M+mR3Dr1q3DvHnzkJGRgVGjRiEvLw8uLi7Iz883u45CoYCfn5/08PX17VBHpVIZ1fHy8urJbhBRL2WzI7i7d++iqKgImZmZUplSqURCQgIKCwvNrtfU1IRhw4bBYDBgzJgxePvtt/H4448b1Tlw4AB8fHzg5eWFyZMnY/Xq1Rg0yPwUzjqdDjqdTlrWarUAAL1eD71e39Uu9nrtfZNzH/uT1tZW6aec96k1fbNZwNXX16Otra3DEZivr6/RrdB+aMSIEcjPz0dkZCQaGxuxZs0ajB8/HufOncOQIUMA3Ds9nTlzJkJCQlBRUYG33noLzzzzDAoLC2FnZ2dyuzk5OcjOzu5Qvnfv3n5xq7V9+/bZugnUDS43AYA9jh49iu/O2ro1PaelpcXiugohhOjBtph19epVBAYG4siRIxg3bpxUvnTpUnz99dc4duzYA7eh1+sxcuRIzJo1C6tWrTJZ5+LFiwgLC8M//vEPTJkyxWQdU0dwQUFBqK+vh4eHh5U96zv0ej327duHqVOnwsHBwdbNoYd0+tJN/HTTSfxlXiyihg60dXN6jFarhbe3NxobGx/4/rTZEZy3tzfs7OxQV1dnVF5XVwc/Pz+LtuHg4IDRo0ejvLzcbJ3Q0FB4e3ujvLzcbMCpVCqTAxEODg794o3fX/opd/b29tJPOe9Pa/pms0EGR0dHxMTEQKPRSGUGgwEajcboiK4zbW1tKCkpgb+/v9k6V65cwY0bNzqtQ0TyZNNRVLVajU2bNmHLli04f/48FixYgObmZmRkZAAAUlNTjQYhVq5cib179+LixYs4deoUXnvtNVRXV+NnP/sZgHsDEL/85S9x9OhRVFVVQaPRYPr06QgPD0dSUpJN+khEtmPT6+BSUlJw/fp1ZGVloba2FtHR0di9e7c08HDp0iUolf/K4Fu3bmHevHmora2Fl5cXYmJicOTIEYwaNQoAYGdnhzNnzmDLli1oaGhAQEAAEhMTsWrVKl4LR9QP2WyQoTfTarXw9PS06EPMvkyv1+PLL7/Es88+K+vPbPqL4uobmLHxKHYtGCvrO9tb8/60+Ve1iIh6CgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki2bB9yGDRsQHBwMJycnxMfH4/jx42brFhQUQKFQGD2cnJyM6gghkJWVBX9/fzg7OyMhIQHffvttT3eDiHohmwbcjh07oFarsWLFCpw6dQpRUVFISkrCtWvXzK7j4eGBmpoa6VFdXW30/DvvvIP169cjLy8Px44dg6urK5KSknDnzp2e7g4R9TI2Dbh169Zh3rx5yMjIwKhRo5CXlwcXFxfk5+ebXUehUMDPz096+Pr6Ss8JIZCbm4tly5Zh+vTpiIyMxNatW3H16lXs2rXrEfSIiHoTe1u98N27d1FUVITMzEypTKlUIiEhAYWFhWbXa2pqwrBhw2AwGDBmzBi8/fbbePzxxwEAlZWVqK2tRUJCglTf09MT8fHxKCwsxMsvv2xymzqdDjqdTlrWarUAAL1eD71e/1D97E1aWlpQVlYmLTd9r8ORkgq4DzgKN2eVVD5ixAi4uLjYoon0EFpbW6Wfcvq7vZ81fbNZwNXX16Otrc3oCAwAfH19UVpaanKdESNGID8/H5GRkWhsbMSaNWswfvx4nDt3DkOGDEFtba20jfu32f6cKTk5OcjOzu5QvnfvXlm90SsqKvDmm292KH/nvuW1a9ciLCzs0TSKus3lJgCwx9GjR/HdWVu3pue0tLRYXNdmAdcV48aNw7hx46Tl8ePHY+TIkfjv//5vrFq1qsvbzczMhFqtlpa1Wi2CgoKQmJgIDw+Ph2pzb9LS0oIJEyZIyxdqGvHLT/+Jd38yCo/5e0rlPILrm05fugmUnMTYsWMRNXSgrZvTY9rPsCxhs4Dz9vaGnZ0d6urqjMrr6urg5+dn0TYcHBwwevRolJeXA4C0Xl1dHfz9/Y22GR0dbXY7KpUKKpWqQ7mDgwMcHBwsaktf4Onpibi4OGnZsfoGVIV38UT0GEQPG2TDllF3sLe3l37K6e/2ftb0zWaDDI6OjoiJiYFGo5HKDAYDNBqN0VFaZ9ra2lBSUiKFWUhICPz8/Iy2qdVqcezYMYu3SUTyYdNTVLVajbS0NMTGxiIuLg65ublobm5GRkYGACA1NRWBgYHIyckBAKxcuRJjx45FeHg4Ghoa8O6776K6uho/+9nPANwbYV28eDFWr16N4cOHIyQkBMuXL0dAQABmzJhhq24SkY3YNOBSUlJw/fp1ZGVloba2FtHR0di9e7c0SHDp0iUolf86yLx16xbmzZuH2tpaeHl5ISYmBkeOHMGoUaOkOkuXLkVzczPmz5+PhoYGTJgwAbt37+5wQTARyZ9CCCFs3YjeRqvVwtPTE42NjbIaZLhfcfUNzNh4FLsWjOVncDLQX/anNe9Pm39Vi4iopzDgiEi2GHBEJFsMOCKSrT71TQai/q6yvhnNulaTz1Vcb5Z+tl/0a4qryh4h3q490r7ehgFH1EdU1jdj0poDD6z35l9KHlhn/5KJ/SLkGHBEfUT7kVtuSjTCfdw6Pv+9Dp8fKMRzE8fB1bnjVw8BoPxaExbvKDZ7FCg3DDiiPibcxw1PBHp2KNfr9agdDIwZ5iXr76Jag4MMRCRbDDgiki2rT1EvXryI0NDQnmgLdbPORtwAjrqR/FkdcOHh4Xj66acxd+5c/PSnP+WX2HspS0fcAI66kXxZHXCnTp3C5s2boVarsWjRIqSkpGDu3LlGEymS7T1oxA3gqBvJn9UBFx0djffffx9r167FZ599hoKCAkyYMAGPPfYY5syZg9mzZ2Pw4ME90VbqAnMjbgBH3Uj+ujzIYG9vj5kzZ+LPf/4zfve736G8vBxLlixBUFAQUlNTUVNT053tJCKyWpcD7uTJk1i4cCH8/f2xbt06LFmyBBUVFdi3bx+uXr2K6dOnd2c7iYisZvUp6rp167B582aUlZXh2WefxdatW/Hss89KM++GhISgoKAAwcHB3d1WIiKrWB1wGzduxJw5c5Cenm5056of8vHxwYcffvjQjSMiehhWB9y+ffswdOhQo3slAIAQApcvX8bQoUPh6OiItLS0bmskEVFXWP0ZXFhYGOrr6zuU37x5EyEhId3SKCKi7mB1wJm7R01TUxMv+iWiXsXiU1S1Wg3g3r1Hs7Ky4OLiIj3X1taGY8eOdXr3eCKiR83igPvmm28A3DuCKykpgaOjo/Sco6MjoqKisGTJku5vIRFRF1kccPv37wcAZGRk4P3335f1/UKJSB6s/gxu8+bN3RpuGzZsQHBwMJycnBAfH4/jx49btN727duhUCgwY8YMo/L09HQoFAqjR3Jycre1l4j6DouO4GbOnImCggJ4eHhg5syZndbduXOnxS++Y8cOqNVq5OXlIT4+Hrm5uUhKSkJZWRl8fHzMrldVVYUlS5bgqaeeMvl8cnIyNm/eLC2rVKa/SE5E8mbREZynpycUCoX0784e1li3bh3mzZuHjIwMjBo1Cnl5eXBxcUF+fr7Zddra2vDqq68iOzvb7Lx0KpUKfn5+0sPLy8uqdhGRPFh0BPfDo6Ef/vth3L17F0VFRcjMzJTKlEolEhISUFhYaHa9lStXwsfHB3PnzsXBgwdN1jlw4AB8fHzg5eWFyZMnY/Xq1Rg0aJDZbep0Ouh0OmlZq9UCuDfbhl6vt7ZrvUJra6v001wf2ss766Ml26FH40H7or/sT2vabbObztTX16OtrQ2+vr5G5b6+vigtLTW5zqFDh/Dhhx+iuLjY7HaTk5Mxc+ZMhISEoKKiAm+99RaeeeYZFBYWws7OzuQ6OTk5yM7O7lC+d+9eo8th+pLLTQBgj0OHDqHa9HRwkn379nXLdqhnXbx9F0qnW/j04J9Q6Gy+3pa/bzH7XO33gNLJDvsPfoVqd0ez9XqzlpYWi+taFHCjR4+WTlEf5NSpUxa/uDVu376N2bNnY9OmTfD29jZb7+WXX5b+/eSTTyIyMhJhYWE4cOAApkyZYnKdzMxM6To/4N4RXFBQEBITE/vsaPG5q1qsKTmKCRMm4PEA033Q6/XYt28fpk6danY+OEu2Q4/GF2Un4dq2EjvbADR1fTuuIUBI9B/w7IjYbmvbo9R+hmUJiwLu/pHK7uDt7Q07OzvU1dUZldfV1cHPz69D/YqKClRVVWHatGlSmcFgAHBvbrqysjKEhYV1WC80NBTe3t4oLy83G3AqlcrkQISDg0OfnQiy/R4L9vb2D+xDZ/20ZjvUs4I8QtFc+XO8nxKNMBOzNLe2tuLwocP40YQfmb3HRsW1JryxoxhBk0L77P60pt0WBdyKFSu63BhzHB0dERMTA41GIwWowWCARqPBokWLOtSPiIhASYnxvQOWLVuG27dv4/3330dQUJDJ17ly5Qpu3LhhduYTudK13YHS6TtUasugdDJ9btna2oqrrVdx/uZ5s2+ISm0TlE7fQdd2B4B1g0jUvVR2TjDcCUSIxwiMGmT6vqiV9pUYOXCk2RAw3GmE4c51qOz6x9cqbXrjZ7VajbS0NMTGxiIuLg65ublobm5GRkYGACA1NRWBgYHIycmBk5MTnnjiCaP1BwwYAABSeVNTE7Kzs/HCCy/Az88PFRUVWLp0KcLDw5GUlPRI+2ZrV5ur4Rrye7xlwWWFH+z+oNPnXUOAq83RiIFvp/WIehuLAm7gwIG4cOECvL294eXl1enncTdv3rT4xVNSUnD9+nVkZWWhtrYW0dHR2L17tzTwcOnSpQ7TMnXGzs4OZ86cwZYtW9DQ0ICAgAAkJiZi1apV/e5auADXYZ2ezgDWndIETBrWk80l6hEWBdx7770Hd3d36d+WDjhYYtGiRSZPSYF7l3t0pqCgwGjZ2dkZe/bs6aaW9W0POp0BeEpD8mdRwP1w8sr09PSeagsRUbey+ruodnZ2uHbtWofyGzdumL3OjIjIFrptwkudTmc0hRIRka1ZPIq6fv16APcmvPyf//kfuLn964PrtrY2/N///R8iIiK6v4VERF1kccC99957AO4dweXl5Rmdjjo6OiI4OBh5eXnd30Iioi6yOOAqKysBAJMmTcLOnTs5QwcR9XpWX+jbPrMvEVFvZ3XAzZkzp9PnO5vLjYjoUbI64G7dumW0rNfrcfbsWTQ0NGDy5Mnd1jAioodldcB9+umnHcoMBgMWLFhgcjYPIiJbsfo6OJMbUSqhVqulkVYiot6gWwIOuDdfW/t0yEREvYHVp6g/nPkWuHddXE1NDb744guj76wSEdma1QHXfof7dkqlEoMHD8batWsfOMJKRPQo8To4IpKtbvsMjoiot2HAEZFsMeCISLYYcEQkW90WcFeuXMH8+fO7a3NERA+t2wLuxo0b+PDDD7trc0RED42nqEQkWww4IpItBhwRyZbF32SYOXNmp883NDR0qQEbNmzAu+++i9raWkRFReH3v/894uLiHrje9u3bMWvWLEyfPh27du2SyoUQWLFiBTZt2oSGhgb86Ec/wsaNGzF8+PAutY+I+i6Lj+A8PT07fQwbNgypqalWvfiOHTugVquxYsUKnDp1ClFRUUhKSjJ539UfqqqqwpIlS/DUU091eO6dd97B+vXrkZeXh2PHjsHV1RVJSUm4c+eOVW0jor7P4iO4zZs3d/uLr1u3DvPmzUNGRgYAIC8vD1988QXy8/Px61//2uQ6bW1tePXVV5GdnY2DBw8aHTkKIZCbm4tly5Zh+vTpAICtW7fC19cXu3btwssvv9ztfSCi3svqL9t3l7t376KoqAiZmZlSmVKpREJCAgoLC82ut3LlSvj4+GDu3Lk4ePCg0XOVlZWora1FQkKCVObp6Yn4+HgUFhaaDTidTgedTicta7VaAPemY9fr9V3qn621z83X2tpqtg/t5Z310ZLt0KPxoH3RX/anNe22OOAsnQrJ0pvO1NfXo62tDb6+vkblvr6+KC0tNbnOoUOH8OGHH6K4uNjk87W1tdI27t9m+3Om5OTkIDs7u0P53r174eLi0lk3eq3LTQBgj0OHDqHarfO6+/bt65btUM+ydF/IfX+2tLRYXNfigCsoKMCwYcMwevRoCCG61LCHcfv2bcyePRubNm2Ct7d3t247MzPTaCJPrVaLoKAgJCYmwsPDo1tf61E5d1WLNSVHMWHCBDweYLoPer0e+/btw9SpU+Hg4NDl7dCj8aB90V/2Z/sZliUsDrgFCxbgk08+QWVlJTIyMvDaa69h4MCBXWogAHh7e8POzg51dXVG5XV1dfDz8+tQv6KiAlVVVZg2bZpUZjAYAAD29vYoKyuT1qurq4O/v7/RNqOjo822RaVSQaVSdSh3cHAw+4fS29nb20s/H9SHzvppzXaoZ1m6L+S+P61pt8WjqBs2bEBNTQ2WLl2K//3f/0VQUBBeeukl7Nmzp0tHdI6OjoiJiYFGo5HKDAYDNBoNxo0b16F+REQESkpKUFxcLD2ef/55TJo0CcXFxQgKCkJISAj8/PyMtqnVanHs2DGT2yQiebNqkEGlUmHWrFmYNWsWqqurUVBQgIULF6K1tRXnzp2Dm5t1J/VqtRppaWmIjY1FXFwccnNz0dzcLI2qpqamIjAwEDk5OXBycsITTzxhtP6AAQMAwKh88eLFWL16NYYPH46QkBAsX74cAQEBmDFjhlVtI6K+r8ujqEqlEgqFAkIItLW1dWkbKSkpuH79OrKyslBbW4vo6Gjs3r1bGiS4dOkSlErrvmyxdOlSNDc3Y/78+WhoaMCECROwe/duODk5damNfdX3+nv75Ox3jWbrNH+vw8nrgF/1Lbg6dzxFB4Dya0090j6iR8GqgNPpdNi5cyfy8/Nx6NAhPPfcc/iv//ovJCcnWx1E7RYtWoRFixaZfO7AgQOdrltQUNChTKFQYOXKlVi5cmWX2iMXFf8vmH69s+QBNe2xrfzEA7fnqrLZFUVEXWbxX+3ChQuxfft2BAUFYc6cOfjkk0+6fTSTuk/i4/cGXMJ83ODsYGeyTllNI978SwnW/vRJjPD3NLstV5U9Qrxde6SdZLkHHZXziLwjiwMuLy8PQ4cORWhoKL7++mt8/fXXJuvt3Lmz2xpHXTfQ1REvxw3ttE77RZ9hg13xRKD5gKPewbKjch6R/5DFvUxNTYVCoejJthBRJx50VM4j8o6sutCXiGznQUflPCLviPPBEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItmwfchg0bEBwcDCcnJ8THx+P48eNm6+7cuROxsbEYMGAAXF1dER0djW3bthnVSU9Ph0KhMHokJyf3dDeIqBey6c0Rd+zYAbVajby8PMTHxyM3NxdJSUkoKyuDj49Ph/oDBw7Eb37zG0RERMDR0RGff/45MjIy4OPjg6SkJKlecnIyNm/eLC2rVKZvgktE8mbTI7h169Zh3rx5yMjIwKhRo5CXlwcXFxfk5+ebrD9x4kT85Cc/wciRIxEWFoY33ngDkZGROHTokFE9lUoFPz8/6eHl5fUoukNEvYzNjuDu3r2LoqIiZGZmSmVKpRIJCQkoLCx84PpCCHz11VcoKyvD7373O6PnDhw4AB8fH3h5eWHy5MlYvXo1Bg0aZHZbOp0OOp1OWtZqtQAAvV4PvV5vbdf6jPb7aLa2tsq6n/1Ff9mf1vTNZgFXX1+PtrY2+Pr6GpX7+vqitLTU7HqNjY0IDAyETqeDnZ0dPvjgA0ydOlV6Pjk5GTNnzkRISAgqKirw1ltv4ZlnnkFhYSHs7DreDRwAcnJykJ2d3aF87969cHFx6WIPe7/LTQBgj6NHj+K7s7ZuDT2s/rI/W1paLK5r08/gusLd3R3FxcVoamqCRqOBWq1GaGgoJk6cCAB4+eWXpbpPPvkkIiMjERYWhgMHDmDKlCkmt5mZmQm1Wi0ta7VaBAUFITExER4eHj3aH1s6fekmUHISY8eORdTQgbZuDj2k/rI/28+wLGGzgPP29oadnR3q6uqMyuvq6uDn52d2PaVSifDwcABAdHQ0zp8/j5ycHCng7hcaGgpvb2+Ul5ebDTiVSmVyIMLBwQEODg4W9qjvsbe3l37KuZ/9RX/Zn9b0zWaDDI6OjoiJiYFGo5HKDAYDNBoNxo0bZ/F2DAaD0edn97ty5Qpu3LgBf3//h2ovEfU9Nj1FVavVSEtLQ2xsLOLi4pCbm4vm5mZkZGQAAFJTUxEYGIicnBwA9z4ri42NRVhYGHQ6Hb788kts27YNGzduBAA0NTUhOzsbL7zwAvz8/FBRUYGlS5ciPDzc6DISIuofbBpwKSkpuH79OrKyslBbW4vo6Gjs3r1bGni4dOkSlMp/HWQ2Nzdj4cKFuHLlCpydnREREYGPPvoIKSkpAAA7OzucOXMGW7ZsQUNDAwICApCYmIhVq1bxWjiifkghhBC2bkRvo9Vq4enpicbGRlkPMhRX38CMjUexa8FYRA8zfxkN9Q39ZX9a8/60+Ve1iIh6CgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki0GHBHJFgOOiGSLAUdEssWAIyLZYsARkWwx4IhIthhwRCRbDDgiki2bB9yGDRsQHBwMJycnxMfH4/jx42br7ty5E7GxsRgwYABcXV0RHR2Nbdu2GdURQiArKwv+/v5wdnZGQkICvv32257uBhH1QjYNuB07dkCtVmPFihU4deoUoqKikJSUhGvXrpmsP3DgQPzmN79BYWEhzpw5g4yMDGRkZGDPnj1SnXfeeQfr169HXl4ejh07BldXVyQlJeHOnTuPqltE1FsIG4qLixOvv/66tNzW1iYCAgJETk6OxdsYPXq0WLZsmRBCCIPBIPz8/MS7774rPd/Q0CBUKpX45JNPLN5mY2OjACAaGxstXqcv+qaqXgz71efim6p6WzeFukF/2Z/WvD9tdgR39+5dFBUVISEhQSpTKpVISEhAYWHhA9cXQkCj0aCsrAw//vGPAQCVlZWora012qanpyfi4+Mt2iYRyYu9rV64vr4ebW1t8PX1NSr39fVFaWmp2fUaGxsRGBgInU4HOzs7fPDBB5g6dSoAoLa2VtrG/dtsf84UnU4HnU4nLWu1WgCAXq+HXq+3rmN9SGtrq/RTzv3sL/rL/rSmbzYLuK5yd3dHcXExmpqaoNFooFarERoaiokTJ3Z5mzk5OcjOzu5QvnfvXri4uDxEa3u3y00AYI+jR4/iu7O2bg09rP6yP1taWiyua7OA8/b2hp2dHerq6ozK6+rq4OfnZ3Y9pVKJ8PBwAEB0dDTOnz+PnJwcTJw4UVqvrq4O/v7+RtuMjo42u83MzEyo1WppWavVIigoCImJifDw8OhK9/qE05duAiUnMXbsWEQNHWjr5tBD6i/7s/0MyxI2CzhHR0fExMRAo9FgxowZAACDwQCNRoNFixZZvB2DwSCdXoaEhMDPzw8ajUYKNK1Wi2PHjmHBggVmt6FSqaBSqTqUOzg4wMHBwfJO9TH29vbSTzn3s7/oL/vTmr7Z9BRVrVYjLS0NsbGxiIuLQ25uLpqbm5GRkQEASE1NRWBgIHJycgDcO5WMjY1FWFgYdDodvvzyS2zbtg0bN24EACgUCixevBirV6/G8OHDERISguXLlyMgIEAK0f6spaXF6PPNspoG6GrLcf6sMww3BkjlERERsj41p/7DpgGXkpKC69evIysrC7W1tYiOjsbu3bulQYJLly5BqfzXQG9zczMWLlyIK1euwNnZGREREfjoo4+QkpIi1Vm6dCmam5sxf/58NDQ0YMKECdi9ezecnJweef96m9LSUsTExHQof2WL8XJRURHGjBnziFpF1HMUQghh60b0NlqtFp6enmhsbJTVZ3D3H8E1fa/DF/sL8f9NGgc353+dovMIrm8qrr6BGRuPYteCsYgeNsjWzekx1rw/+9woKnWdi4uL0ZGZXq/HrfprGBcXK+vPbKj/svl3UYmIegoDjohkiwFHRLLFgCMi2eIgA1EfZel1jUD/HRlnwBH1UZZe1wj032sbGXBEfVRERASKioqkZXPXNbbX7Y8YcER9FK9rfDAOMhCRbDHgiEi2GHBEJFsMOCKSLQYcEckWA46IZIsBR0SyxevgTGifA9Sam1v0RXq9Hi0tLdBqtbxuSgb6y/5sf19aMlcvA86E27dvAwCCgoJs3BIiMuf27dvw9PTstA6nLDfBYDDg6tWrcHd3h0KhsHVzekz77REvX74sq6nZ+6v+sj+FELh9+zYCAgKM7tliCo/gTFAqlRgyZIitm/HIeHh4yPoN0d/0h/35oCO3dhxkICLZYsARkWwx4PoxlUqFFStWQKVSPbgy9Xrcnx1xkIGIZItHcEQkWww4IpItBhwRyRYDjohkiwEnExMnTsTixYtt3QyyUm/bb72tPQ+LAUfUx929e9fWTei9BPV5aWlpAoDRIzAwUHzwwQdG9U6dOiUUCoWoqqoSQgixdu1a8cQTTwgXFxcxZMgQsWDBAnH79m2jdQ4ePCgmTJggnJycxJAhQ8TPf/5z0dTU9Mj6Jmem9lt5ebmYM2eOCA4OFk5OTuKxxx4Tubm5HdabPn26WL16tfD39xfBwcFCCCEOHz4soqKihEqlEjExMeLTTz8VAMQ333wjrVtSUiKSk5OFq6ur8PHxEa+99pq4fv262fZUVlY+ql9Hj2DAyUBDQ4MYN26cmDdvnqipqRE1NTViyZIlYsKECUb13nzzTaOy9957T3z11VeisrJSaDQaMWLECLFgwQLp+fLycuHq6iree+89ceHCBXH48GExevRokZ6e/sj6Jmem9tudO3dEVlaWOHHihLh48aL46KOPhIuLi9ixY4e0XlpamnBzcxOzZ88WZ8+eFWfPnhWNjY1i4MCB4rXXXhPnzp0TX375pXjssceMAu7WrVti8ODBIjMzU5w/f16cOnVKTJ06VUyaNMlse1pbW23xq+k2DDiZePrpp8Ubb7whLX/zzTdCoVCI6upqIYQQbW1tIjAwUGzcuNHsNv785z+LQYMGSctz584V8+fPN6pz8OBBoVQqxffff9+9Hein7t9vprz++uvihRdekJbT0tKEr6+v0Ol0UtnGjRvFoEGDjPbLpk2bjAJu1apVIjEx0Wjbly9fFgBEWVmZxe3pS/gZnExFR0dj5MiR+PjjjwEAX3/9Na5du4YXX3xRqvOPf/wDU6ZMQWBgINzd3TF79mzcuHEDLS0tAIDTp0+joKAAbm5u0iMpKQkGgwGVlZU26Vd/sGHDBsTExGDw4MFwc3PDH/7wB1y6dMmozpNPPglHR0dpuaysDJGRkXBycpLK4uLijNY5ffo09u/fb7Q/2+94X1FR0YM9sh0GnIy9+uqrUsB9/PHHSE5OxqBBgwAAVVVVeO655xAZGYm//vWvKCoqwoYNGwD860PrpqYm/Nu//RuKi4ulx+nTp/Htt98iLCzMNp2Sue3bt2PJkiWYO3cu9u7di+LiYmRkZHQYSHB1dbV6201NTZg2bZrR/iwuLsa3336LH//4x93VhV6F88HJhKOjI9ra2ozKXnnlFSxbtgxFRUX4y1/+gry8POm5oqIiGAwGrF27Vpo08E9/+pPR+mPGjME///lPhIeH93wH+qn799vhw4cxfvx4LFy4UCqz5OhqxIgR+Oijj6DT6aQv2584ccKozpgxY/DXv/4VwcHBsLc3/dY39XfUl/EITiaCg4Nx7NgxVFVVob6+HgaDAcHBwRg/fjzmzp2LtrY2PP/881L98PBw6PV6/P73v8fFixexbds2owAEgF/96lc4cuQIFi1aJP1P/7e//Q2LFi161N2Trfv32/Dhw3Hy5Ens2bMHFy5cwPLlyzsElSmvvPIKDAYD5s+fj/Pnz2PPnj1Ys2YNAEizUr/++uu4efMmZs2ahRMnTqCiogJ79uxBRkaGFGqm/o76NFt/CEjdo6ysTIwdO1Y4OzsbDe9/8MEHAoBITU3tsM66deuEv7+/cHZ2FklJSWLr1q0CgLh165ZU5/jx42Lq1KnCzc1NuLq6isjISPHb3/72EfVK/u7fb6WlpSI9PV14enqKAQMGiAULFohf//rXIioqSlqn/TKR+x0+fFhERkYKR0dHERMTIz7++GNpm+0uXLggfvKTn4gBAwYIZ2dnERERIRYvXiwMBoPJ9vT1y0Q4XRKRTP3xj39ERkYGGhsb4ezsbOvm2AQ/gyOSia1btyI0NBSBgYE4ffo0fvWrX+Gll17qt+EGMOCIZKO2thZZWVmora2Fv78/XnzxRfz2t7+1dbNsiqeoRCRbHEUlItliwBGRbDHgiEi2GHBEJFsMOCKSLQYc2VR6ejoUCgUUCgUcHBzg6+uLqVOnIj8/36qvCRUUFGDAgAE911Az0tPTMWPGjEf+umQZBhzZXHJyMmpqalBVVYW///3vmDRpEt544w0899xzaG1ttXXzqC+z7TfFqL8z971KjUYjAIhNmzYJITqfXn3//v0dptpesWKFEEKIrVu3ipiYGOHm5iZ8fX3FrFmzRF1dnfQ6N2/eFK+88orw9vYWTk5OIjw8XOTn50vPX7p0Sbz44ovC09NTeHl5ieeff176fuaKFSs6vO7+/ft75PdEXcMjOOqVJk+ejKioKOzcuRMAoFQqsX79epw7dw5btmzBV199haVLlwIAxo8fj9zcXHh4eKCmpgY1NTVYsmQJAECv12PVqlU4ffo0du3ahaqqKqSnp0uvs3z5cvzzn//E3//+d5w/fx4bN26Et7e3tG5SUhLc3d1x8OBBHD58GG5ubkhOTsbdu3exZMkSvPTSS9IRaE1NDcaPH/9of1HUOVsnLPVv5o7ghBAiJSVFjBw50uRz90+vvnnzZuHp6fnA1ztx4oQAIB39TZs2TWRkZJisu23bNjFixAhppg0hhNDpdMLZ2Vns2bPnge0n2+MRHPVaQghpLrMHTa9uTlFREaZNm4ahQ4fC3d0dTz/9NABIU4AvWLAA27dvR3R0NJYuXYojR45I654+fRrl5eVwd3eXpvgeOHAg7ty5I9spvuWGAUe91vnz5xESEmLR9OqmNDc3IykpCR4eHvjjH/+IEydO4NNPPzVa75lnnkF1dTV+8Ytf4OrVq5gyZYp0etvU1ISYmJgOU3xfuHABr7zySg/3nroDZxOhXumrr75CSUkJfvGLX1g0vbqpqbZLS0tx48YN/Od//ieCgoIAACdPnuzwWoMHD0ZaWhrS0tLw1FNP4Ze//CXWrFmDMWPGYMeOHfDx8YGHh4fJdsptim+54REc2ZxOp0NtbS2+++47nDp1Cm+//TamT5+O5557DqmpqRZNrx4cHIympiZoNBrU19ejpaUFQ4cOhaOjo7TeZ599hlWrVhmtl5WVhb/97W8oLy/HuXPn8Pnnn2PkyJEA7t20x9vbG9OnT8fBgwdRWVmJAwcO4D/+4z9w5coV6XXPnDmDsrIy1NfXQ6/XP5pfGlnG1h8CUv/2w7up29vbi8GDB4uEhASRn58v2trapHqWTK/+7//+72LQoEFGl4l8/PHHIjg4WKhUKjFu3Djx2WefdbhX6MiRI4Wzs7MYOHCgmD59urh48aK0zZqaGpGamiq8vb2FSqUSoaGhYt68eaKxsVEIIcS1a9ekKd3By0R6Hc4HR0SyxVNUIpItBhwRyRYDjohkiwFHRLLFgCMi2WLAEZFsMeCISLYYcEQkWww4IpItBhwRyRYDjohkiwFHRLL1/wM4eUGHCSMD8QAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from ml_utility_loss.loss_learning.visualization import plot_box_3\n", + "\n", + "_ = plot_box_3(y2[\"pred\"], next(iter(y2[\"y\"].values())))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "eabe1bab", + "metadata": { + "execution": { + "iopub.execute_input": "2024-02-29T23:33:41.678110Z", + "iopub.status.busy": "2024-02-29T23:33:41.677272Z", + "iopub.status.idle": "2024-02-29T23:33:41.967168Z", + "shell.execute_reply": "2024-02-29T23:33:41.965990Z" + }, + "papermill": { + "duration": 0.315853, + "end_time": "2024-02-29T23:33:41.969289", + "exception": false, + "start_time": "2024-02-29T23:33:41.653436", + "status": "completed" + }, + "tags": [] + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAEmCAYAAAAXyJnQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuNSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/xnp5ZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABmHklEQVR4nO2dd3gU1frHP7MtvVCTgIGEIkVpIiBYEI2CV6/KtQAiCqKIiA1RAZXqvaAgYkG46gVURBQb/FRQjBSBUIw0KdKlJSGUdJJt8/tj2cn2kuwmm+R8nmefJDNnZs5sdr/znvO+530lWZZlBAKBoA6jqu4OCAQCQXUjhFAgENR5hBAKBII6jxBCgUBQ5xFCKBAI6jxCCAUCQZ1HCKFAIKjzCCEUCAR1Hk11dyAUMZvNnD59mpiYGCRJqu7uCAQCB2RZprCwkCZNmqBSVd6eE0LogtOnT5OcnFzd3RAIBF44ceIEl112WaXPI4TQBTExMYDlTY6Nja3m3ggEAkcKCgpITk5WvquVRQihC6zD4djYWCGEAkEIE6ipK+EsEQgEdR4hhAKBoM4jhFAgENR5xBxhHUaWZYxGIyaTqbq7IgggWq0WtVpd3d2oUQghrKPo9XqysrIoKSmp7q4IAowkSVx22WVER0dXd1dqDEII6yBms5mjR4+iVqtp0qQJOp1OBI7XEmRZJjc3l5MnT9K6dWthGfqIEMIaznfbT/H9rtPMGdiF6DDf/p16vR6z2UxycjKRkZFB7qGgqmnUqBHHjh3DYDAIIfQR4Syp4SzadIxf9p1h29Hzfh8biKVJgtBDWPf+I74JNRyT2VJ7q8xoruaeCAQ1FyGENRwZixAazUIIBYKKIoSwhmMtxmo0iaqsdYWhQ4dy9913V3c3ahVCCGs4ihCa64YQ3njjjTz77LPV3Q1BLUMIYS3BaBJD45qEwWCo7i4IbAgJIZw7dy4pKSmEh4fTo0cPtm7d6rbthx9+yPXXX0+9evWoV68eaWlpTu1lWWbixIkkJSURERFBWloaBw8eDPZtVAtWO9BQSYtQlmVK9MZqecmyb30fOnQo69at4+2330aSJCVweN68eXbttm/fjkql4u+//wZg9uzZdOjQgaioKJKTkxk1ahRFRUV2x2zYsIHrr7+eiIgIkpOTefrppykuLvapX1lZWdx+++1ERESQmprKkiVLSElJYc6cOUobSZKYN28ed955J1FRUfz73//GZDIxfPhwUlNTiYiIoE2bNrz99tt25zaZTIwZM4b4+HgaNGjAiy++6PP7JfCdao8j/OKLLxgzZgzz58+nR48ezJkzh759+/LXX3/RuHFjp/Zr165l0KBB9OrVi/DwcF5//XVuvfVW9uzZQ9OmTQF44403eOedd/j4449JTU3l1VdfpW/fvuzdu5fw8PCqvsWgYv1SVNYivGgw0X7iT4Hokt/sndqXSJ33j+Lbb7/NgQMHuPLKK5k6dSoAb775JkuWLOGJJ55Q2n322Wdce+21NG/eHLCECb3zzjukpqZy5MgRRo0axYsvvsj7778PwOHDh+nXrx+vvfYaCxYsIDc3l9GjRzN69GgWLlzotV8PPfQQZ8+eZe3atWi1WsaMGcOZM2ec2k2ePJkZM2YwZ84cNBoNZrOZyy67jGXLltGgQQM2bdrEiBEjSEpK4v7771fub9GiRSxYsIB27drx5ptv8u2333LTTTd5f2MFPiPJ1fx46dGjB926deO9994DUAJ9n3rqKcaNG+f1eJPJRL169Xjvvfd46KGHkGWZJk2a8PzzzzN27FgA8vPzSUhIYNGiRQwcONDrOQsKCoiLiyM/Pz/k8xH2m7Oe/dmFvHJ7Ox69voVPx5SWlnL06FFSU1OVB0OJ3hjyQgiWOcLOnTsr1taOHTu46qqrOHbsGM2aNcNsNtOsWTNeeeUVRo4c6fIcX331FSNHjuTs2bMAPProo6jVav773/8qbTZs2EDv3r0pLi72+PDcv38/7dq1Y9u2bVx99dUAHDp0iNatW/PWW28p85mSJPHss8/y1ltveby/0aNHk52dzVdffQVAkyZNeO6553jhhRcAMBqNpKam0rVrV7777juX53D1/61tBPo7Wq0WoV6vJzMzk/HjxyvbVCoVaWlpZGRk+HSOkpISDAYD9evXB+Do0aNkZ2eTlpamtImLi6NHjx5kZGS4FMKysjLKysqUvwsKCip6S1WO9TFmqKTXOEKrZu/UvgHoUcWuXVE6d+5Mu3btWLJkCePGjWPdunWcOXOG++67T2nzyy+/MH36dPbv309BQQFGo5HS0lJKSkqIjIxk586d7Nq1i88++0w5RpZlZSliu3bt3F7/r7/+QqPRcNVVVynbWrVqRb169ZzaWoXSlrlz57JgwQKOHz/OxYsX0ev1dO7cGbA8wLOysujRo4fSXqPRcPXVV4vhcYCp1jnCs2fPYjKZSEhIsNuekJBAdna2T+d46aWXaNKkiSJ81uP8Oef06dOJi4tTXjWpXokSR1jJobEkSUTqNNXyquxKiMGDB7NkyRIAlixZQr9+/WjQoAEAx44d44477qBjx458/fXXZGZmMnfuXMDyIAYoKiri8ccfZ8eOHcpr586dHDx4kJYtW1aqb7ZERUXZ/b106VLGjh3L8OHD+fnnn9mxYwfDhg1T+iWoOkLCWVJRZsyYwdKlS/n2228rNQQYP348+fn5yuvEiRMB7GVwUSzCOhI+o9PpnNKGPfDAA/z5559kZmby1VdfMXjwYGVfZmYmZrOZN998k2uuuYbLL7+c06dP2x1/1VVXsXfvXlq1auX00ul0HvvTpk0bjEYj27dvV7YdOnSICxcueL2XjRs30qtXL0aNGkWXLl1o1aoVhw8fVvbHxcWRlJTEli1blG1Go5HMzEyv5xb4R7UKYcOGDVGr1eTk5Nhtz8nJITEx0eOxs2bNYsaMGfz888907NhR2W49zp9zhoWFKfVJalqdEqv81ZXwmZSUFLZs2cKxY8c4e/YsZrOZlJQUevXqxfDhwzGZTNx5551K+1atWmEwGHj33Xc5cuQIn376KfPnz7c750svvcSmTZsYPXo0O3bs4ODBgyxfvpzRo0d77U/btm1JS0tjxIgRbN26le3btzNixAgiIiK8WrqtW7fm999/56effuLAgQO8+uqrbNu2za7NM888w4wZM/juu+/Yv38/o0aNIi8vz/c3TOAT1SqEOp2Orl27kp6ermwzm82kp6fTs2dPt8e98cYbTJs2jVWrVjnNu6SmppKYmGh3zoKCArZs2eLxnDUdUx2xCMeOHYtaraZ9+/Y0atSI48ePA5bh8c6dO+nfvz8RERFK+06dOjF79mxef/11rrzySj777DOmT59ud86OHTuybt06Dhw4wPXXX0+XLl2YOHEiTZo08alPn3zyCQkJCdxwww3079+fxx57jJiYGK+jlMcff5x//etfDBgwgB49enDu3DlGjRpl1+b5559nyJAhPPzww/Ts2ZOYmBj69+/vU78EfiBXM0uXLpXDwsLkRYsWyXv37pVHjBghx8fHy9nZ2bIsy/KQIUPkcePGKe1nzJgh63Q6+auvvpKzsrKUV2FhoV2b+Ph4efny5fKuXbvku+66S05NTZUvXrzoU5/y8/NlQM7Pzw/szQaBm2atkZu/9L08ZcUen4+5ePGivHfvXp/fD4F/nDhxQgbkX375pVquXxf+v4H+jlZ7HOGAAQPIzc1l4sSJZGdn07lzZ1atWqU4O44fP26XLmrevHno9Xruvfdeu/NMmjSJyZMnA/Diiy9SXFzMiBEjyMvL47rrrmPVqlW1MpRAGRqLpAvVxq+//kpRUREdOnQgKyuLF198kZSUFG644Ybq7prAR6pdCAEleNUVa9eutfv72LFjXs8nSRJTp05Vgm5rNQEKnxG45rfffuO2225zu7+oqAiDwcCECRM4cuQIMTEx9OrVi88++wytVluFPRVUhpAQQkHFscqfSViEQeHqq69mx44dHtv07duXvn2rJwZTEBiEENZwZGWJnbAIg0FERAStWrWq7m4IgkyNjiMUBC7pgkBQlxFCWEuoK3GEAkEwEEJYw6lriVkFgmAghLCGE6i1xgJBXUYIYQ1HWIQCQeURQljDKU/DJSxCb7jKGu0up5+gbiHCZ2oJInzGf7KyslzmDRTUPYQQ1nCUOEIxNPYbbxmOqgJZljGZTGg04qtYnYihcS2h0muNZRn0xdXz8iPbcmFhIYMHDyYqKoqkpCTeeuutCpf4tB0aHzt2DEmS+Oabb+jTpw+RkZF06tTJKVO6tyJPn376KVdffTUxMTEkJibywAMP2NUvWbt2LZIksXLlSrp27UpYWBgbNmzwu++CwCIeQzWc8nyElbQIDSXwH9/STgWcCadBF+W9HTBmzBg2btzIihUrSEhIYOLEifzxxx9KevvK8vLLLzNr1ixat27Nyy+/zKBBgzh06BAajcanIk8Gg4Fp06bRpk0bzpw5w5gxYxg6dCg//vij3XXGjRvHrFmzaNGihRiehwBCCGs4dclZUlhYyMcff8ySJUu4+eabAVi4cKHPeQN9YezYsdx+++0ATJkyhSuuuIJDhw7Rtm1bpk+fzuDBgxXrs3Xr1rzzzjv07t2befPmER4eziOPPKKcq0WLFrzzzjt069aNoqIioqOjlX1Tp07llltuCVi/BZVDCGENxxpHWOnErNpIi2VWHWgjfWp25MgRDAYD3bt3V7bFxcXRpk2bgHXFNtt5UlISAGfOnKFt27Y+FXnKzMxk8uTJ7Ny5kwsXLmC+NGVx/Phx2rdvrxznqpCToPoQQljDCVQVOyTJ5+FpbcY2dZY11b5VzKxFnp5++mmn45o1a0ZxcbGSieazzz5TMmj37dvXqSCTYyEnQfUihDBEOF+sZ/66w9zX9TJaJ8T4fFxdSszaokULtFot27Zto1mzZoCl5OWBAweqJAmqbZEnV+zevZtz584xY8YMpRLi77//HvR+CSqP8BqHCOO/2cUH64/Qd856v45TVpbUgTjCmJgYHn74YV544QXWrFnDnj17GD58OCqVqtIlQX3BW5GnZs2aodPplEJRK1asYNq0aUHvVyhiMstsPHSWglJDdXfFJ4QQhgg7TuQBUNGpvrrgLAGYPXs2PXv25I477iAtLY1rr72Wdu3aVUkZBm9Fnho1asSiRYtYtmwZ7du3Z8aMGcyaNSvo/QpFft6TzeCPtjBj5f7q7opPiKFxiOBHKJ3jkUDdqWIXExNj56woLi5mypQpjBgxwuuxjmUeZJs3PSUlxe5vgPj4eKdt3bp14+eff3Z7jUGDBjFo0CC317nxxhudzlkbOX6+BIC/zxV7aRkaCCEMESr61ahrBd63b9/O/v376d69O/n5+UpdmrvuuquaeyawpbjMCEBeSc0YGgshDBEqaiTUtQLvALNmzeKvv/5S6mL/9ttv7Nu3z2uRJUHVUVRmAoQQCvymYkpoHWaZZTCbZVSq4DsNqpMuXbqQmZnptP3ixYteiywJqg6rRZh/UQihwA8qOrK1PcxoltHVciF0hyiyFFoUXRLCojIjBpMZrTq0/bKh3bs6REUn0G0P8zeWsC5M2tdFQuH/ahVCqBlWoRDCECEQH11fV5dYV0+UlJQE4KqCUMO6ikWtVldbH4pthLAmzBOKoXGIUGFnic2BvjpM1Go18fHxSnqoyMjIKglIFgQfs9lMbm4ukZGR1Zrj0N4i1HtoGRoIIQwRKjw0tvndn1hCa1JS21x5gtqBSqWiWbNm1fpwKxIWoaAiVHhax+Y4f2IJJUkiKSmJxo0bYzCE/gdV4Ds6nQ6VqnpnvcTQWFAhAqCDFYolVKvV1TqXJKidFF+KIwTIE84Sga9U3GtcflylU3EJBAGgzGhCb/NQzi8J/TlCIYQhQmDiCOvO6hJB6GJrDYKwCAV+IFd4ZUn573UhFZcg9LGdH4SaMUcohDBECEQMrCjpKQgFihyFUFiEAl+puLPE/zhCgSCYOFqEYo5Q4DsVDqgu/104SwShQKGwCAUVxVzFAdUCQbCwWoQNo3WAmCMU+EElE1QDYBBeY0EIYBXCpvERABSUGkL+IS2EMESo+BI72znC0P6wCeoG1qSsTS4JoSxDYYgXcRJCGCJUOI7QLnxGWISC6qeo1GIRxkfqiA6zLF4L9eFxpYSwqKiIgoICu5e/zJ07l5SUFMLDw+nRowdbt25123bPnj3cc889pKSkIEkSc+bMcWozefJkJEmye7Vt29bvftVERPiMIBQo1luEMDpMTVyEJeVbqDtM/BbCo0ePcvvttxMVFUVcXBz16tWjXr16xMfHU69ePb/O9cUXXzBmzBgmTZrEH3/8QadOnejbt6/bjCglJSW0aNGCGTNmKNlTXHHFFVeQlZWlvDZs2OBXv2oSYmWJINSwxhFGhWmIj7wkhCEeQuN30oUHH3wQWZZZsGABCQkJlUr1M3v2bB577DGGDRsGwPz58/nhhx9YsGAB48aNc2rfrVs3unXrBuByvxWNRuNRKGsTYq2xINSwDo2jbYQw1LNU+y2EO3fuJDMzkzZt2lTqwnq9nszMTMaPH69sU6lUpKWlkZGRUalzHzx4kCZNmhAeHk7Pnj2ZPn06zZo1c9u+rKyMsrIy5e+KDPGrC/vsM0IIBdWP1WscHaYhPqJmhND4PTTu1q0bJ06cqPSFz549i8lkIiEhwW57QkIC2dnZFT5vjx49WLRoEatWrWLevHkcPXqU66+/nsLCQrfHTJ8+nbi4OOWVnJxc4etXNZWpWSIQBAPboXGcMjQObSH02yL86KOPGDlyJKdOneLKK69U6l9Y6dixY8A6VxFsa9t27NiRHj160Lx5c7788kuGDx/u8pjx48czZswY5e+CgoIaJYZWgmkRlhpMSBKEaUTuQoFnyp0lGuIvOUsu1LY5wtzcXA4fPqzM64El27Esy0iShMlk8nB0OQ0bNkStVpOTk2O3PScnJ6Dze/Hx8Vx++eUcOnTIbZuwsDDCwsICds2qwjH2MFgWocFkpvPUn9GpVeyYeGutr50sqBzKHGG4hgbRlu/VueLQFkK/h8aPPPIIXbp0ISMjgyNHjnD06FG7n76i0+no2rUr6enpyjaz2Ux6ejo9e/b0t1tuKSoq4vDhwyQlJQXsnKFKsJwl2fmllBrMFJQauWjw7UEnqLtYA6qjdBoaxViEMLewtDq75BW/LcK///6bFStWBKSY9pgxY3j44Ye5+uqr6d69O3PmzKG4uFixNh966CGaNm3K9OnTAYuDZe/evcrvp06dYseOHURHRyv9GTt2LP/85z9p3rw5p0+fZtKkSajVagYNGlTp/npj46GzfP3HSV7+RzvlSRhMHBejBGtobBsYUNE10YK6g62zpNGl78GZwjJPh1Q7fgvhTTfdxM6dOwMihAMGDCA3N5eJEyeSnZ1N586dWbVqleJAOX78uF0RmtOnT9OlSxfl71mzZjFr1ix69+7N2rVrATh58iSDBg3i3LlzNGrUiOuuu47NmzfTqFGjSvfXGx/9doQ1f+USG65l8p1XBP16jpJkCtLQWGWjhCJmW+AJk1lWRg1RYWobi7CWCeE///lPnnvuOXbv3k2HDh2cnCV33nmnX+cbPXo0o0ePdrnPKm5WUlJSvK7JXbp0qV/XDyRlRosQffn7CZ5Na018pC6o13N8L/ypYucPtkJY0TXRgrqBbVLW6HANWo3FkCksNVJqMBGuDU1nm99COHLkSACmTp3qtM8fZ0ltxJpho0Rv4rMtx3myT+WtZk84SpLRZKbMaCL/ooHGMeEBu46tb0TooMAT1mGxVi0RplGjU6sI06goM5rJLSwjuX5kNffQNX47S8xms9tXXRZBsJ8/W7TpGGXG4L4fjqJkMMmM+CSTa2f8Sk5B4CanJbuhsVBCgXuKbWIIwfLZUYbHRaE7PPZLCA0GAxqNhj///DNY/anR2OZcyy0sY/mO00G9nmPBJ6PZzKEzRRhMMifOlwTsOio7Z0nATiuohSjB1LrywaZVCM8U1BIh1Gq1NGvWrM5bfu6wOm27p9QH4MP1R6p0Ts1klpVYwmBlohFzhAJPWIUwJtxGCKNrmUUI8PLLLzNhwgTOnz8fjP7UaMyXxGfwNc2IDtNw8EwRaw/kBu16robG1hCaYGUEFjIo8ITj0BigcWzoe479dpa89957HDp0iCZNmtC8eXOioqLs9v/xxx8B61xNwyo+8ZE6BnZL5qMNR/lw/RH6tGns8ThHK8tsliu0esNoMmO4lJw1kEJoeyYxRyjwhBJMHWZrEVocd7VKCO++++4gdKN2YBUJtSQx7LpUFm46xqbD5/jzVD5XNo3zcJzzeVR4F0Ini9AsKwIYLItQzBEKPFEeTF0eJlMTYgn9FsJJkyYFox+1Aqv4qFSWwjW3d0hixc7TfPTbEeYM7OL2OMc1wiZZ9ukf4+gsMZlkJZYwkHOEtoJrFkoo8ECRzaoSKzVhmV2FU/VnZmayePFiFi9ezPbt2wPZpxqLVQjVl8JNHru+BQD/tyuL03kXvR5nxdfRp9MSO7NZqVsSPItQCKHAPUUu5ghrpUV45swZBg4cyNq1a4mPjwcgLy+PPn36sHTp0ipZyhaqmKxD40vzex0ui6NniwZkHDnHwo1Hefn29q6PcxAtX0XMsVWZ0awMXQM7R1h+LmEQCjxR7MIibGwTR2jNUhVq+G0RPvXUUxQWFrJnzx7Onz/P+fPn+fPPPykoKODpp58ORh9rDOVD4/J/9IgbLFbh51tPUOCmpKGjaPlqdTk6WcoM5UPsYKXkEhahwBPWFFy2FmGDS4XeDSY5ZFP2+y2Eq1at4v3336ddu3bKtvbt2zN37lxWrlwZ0M7VNMwOQ2OA3pc3onXjaIrKjCzdetzlcc5CWLHrl9qsZAmoYIk5QoGPuJojDNOoldoloZqFpkJL7BwTLYAl2Npcx1PFOw6NwWIdWucKF248poS32B3nKIQVHBqX2uQKDFZKLqGDAk/YZqe2RQmqri1CeNNNN/HMM89w+nT58rFTp07x3HPPcfPNNwe0czUNq8apHWIA7+rShIbRYWTll/LDriyn4xw9vL4Pje3/tk2aKuIIaxfbjp3n1/053htWM67iCCH0HSZ+C+F7771HQUEBKSkptGzZkpYtW5KamkpBQQHvvvtuMPpYYzC7sAjBMjQY2qs5AB+4WHbn5CzxIjayLLM/u8DJuryot50jFF7j2sR98zN4ZNHvZOeHbggKQNGlefCoMPt0W41DXAj99honJyfzxx9/8Msvv7B//34A2rVrR1paWsA7V9NQnCUuvGKDezRn7prD7M0qYNPhc1zbqqHTcVa8ac33u7J46vPtPHiNfYlS26FxIAXLPo4wYKcVVIAzhaUkxgUuxVqgKb5kEcaE2U+fhXoGGr+FECypdW655RZuueWWQPenRqM4S1wsj6sXpeP+qy/j44y/+WD9ETsh9HdofPxSZpntx/PstlfNHKGwCKuTUH/7y9ca21uE5RloQtOirZAQpqenk56ezpkzZ5wcJAsWLAhIx2oiJpsldq545LpUPt38N+sO5PJXdiFtEmMAZ3HxNr9nFbnj5+xTbdkKavDiCEP8m1gLsZ1KCeV3X5Zlitw5S0LcIvR7jnDKlCnceuutpKenc/bsWS5cuGD3qsvYLrFzRfMGUfS70lKq9IP15RX/HK03b1pjjREstEmL7tSXIAmWEMKqx/YtD+U0aCV6k9LX6HB7IbRmTK81c4Tz589n0aJFDBkyJBj9qdG4c5bYMuzaVH7cnc3qvdlAJ7vjrHiz5nwp2xlQi9B2jjB0v4e1FtnN76GGdViskiBC63poHKpC6LdFqNfr6dWrVzD6UuMxugiodqRdUiwABaVGJfjU3zlCo4tYROc2QbIIhRJWObafhxA2CO2yUzsuo7PGEV4oMaA3hp7HzW8hfPTRR1myZEkw+lKjkWVZ+ZB6yiUYHaYh5tL8iTUUwrEMpzet8SU0JpBDY9szBWvILXBPTXnLi93EEALERWjRqi3fi7MhOE/o99C4tLSUDz74gF9++YWOHTs6rTKZPXt2wDpXk7AdinqyCAES48IpPFNEdn4prRpH42jgebMIXa1Oce5PcJ66NeVLWZuw/zyE7j+gsMwSQ+g4PwgW48C6qCC3sIwm8RFV3T2P+C2Eu3btonPnzgBORZxCMatEVWFrKXnLLp0YF87BM0VkXwolcEyQ4H1o7P3LENh8hMJrHCqE8tvvySIES1C1VQhDDb+FcM2aNcHoR43HVss03oQw1uJBy8635Cj0Nw2XLyIXrLk8MUVY9ZhrSPiMq+zUtoRyCE2FE7MK7LG1CD15jQGSLq0MyFLmCCsWPuO5jchQXVuwD5+pvn54w1UpT1tC2XMshDBA2IqZqyV2tiTGWeZHrEXY/c1H6MvQWGSorj3Ye41D9/1XUnC5mCOEcs/xmRBM2S+EMEDYWkreLMLEOMsHwp1F6D2OsGotQluEQVh1bD5yjhPnS2pcHKHjqhIroWwRVmiJncAZO2eJF59RYqzFIsx2I4T+hs+oJBeV8IKkWMGyNAX27DyRx8APNlt+n3irsj2EDUKX9UpsaRTCq0v8tgjXr1+P0ei8tMtoNLJ+/fqAdKomYhUeleTde26dIzxXrKfMaPI7oNpqEVoFV6N2/jcGa44wlIdmtYk/jpcvV3WsVhiq+GwR1gZnSZ8+fTh//rzT9vz8fPr06ROQTtVEXGWndkd8pBadxvLWnykocxI+b5H3VqusaT2LZSnh7KkWdY1rNu6WNYayKLpK029LYyUDTVnIPVD9FkJ3VajOnTtHVFRUQDpVk9h0+Cwmm8Lq3hwlYLEYbT3Hjs6PTzKOeTze2r5Fw+hL5wONOnhCaPvlEytLqga7ecEas8TOcxxhw0vOkjKj2WPCkOrA5znCf/3rX4DlSzx06FDCwsKUfSaTiV27dtW5Ncgf/XaE137Yx5BrmvPo9amAbxYhWGIJ/z5XQlb+RUW0LqsXQVZ+KT/tyWHdgVx6X+66NKrhUvhMi0ZRrDuQi4R06brllmSwLMJQe5LXBcx21mHovv/e4ggjdGpiwjQUlhnJLSwjNty59lF14bNFGBcXR1xcHLIsExMTo/wdFxdHYmIiI0aMYPHixcHsa8iRFBeBJMGnm/9m7ppDgPfldeXHWizCnIJSxcpqlxTLwz1TAJiyYg9lNlXpbLFahO2SYtGqJepH6ZwswkCW85RryBexNmGfg9B2ZU919MY3XJXydKRRbGh6jn22CBcuXAhASkoKY8eOrZPDYEdu75jEmcL2TPm/vXz5+0nA+/I6Kwk2Q+OISwGoakni2Vtas2LnaY6cLWbBhmM8cWNLp2OtzpLE2HCWjexFbLiG+/+72a6NDxE2FUKk6q96asqDyNscIVhiCY/kFoecEPo9Rzhp0iQhgjYMuzaVx3u3UP72trzOSpKyzK60PMW/WiI2XMv429oC8O6vB8m6tAzPFqtHWKOW6JwcT4tG0UpmDyuBTLogss9ULzVhZY8syxRcKtzkUQitDpOaLoQ5OTkMGTKEJk2aoNFoUKvVdq+6yEt923J35yYAhGt9ew+sq0uyC0rLhe2SiPbv0pSuzetRojcx++cDTsda8xFqbcJmnIfGYo6wJuPOCgxRHeRMYRmFpUZUEiTXj3TbLlSDqv0OqB46dCjHjx/n1VdfJSkpqU5nnLGiUkm8cW8nmjeIon2TWJ+OsVYiy84vVaw36/yiSiXxzM2teWjBVrYecw5VUhLA2lifGof6AIHNUB36X8Tahu28oJ1FHqL/gL2nCwBo2SjaozFQa4Rww4YN/Pbbb0oqLoEFnUbFc7dc7nN7q7PkTGGZknrfVtjaXirsdOJ8CXqjWYk7hHJnidZG/KoqjjBUv4i1GdvhcKha5HuzLELozRBQapeEWFC130Pj5OTkgP4z5s6dS0pKCuHh4fTo0YOtW7e6bbtnzx7uueceUlJSkCSJOXPmVPqc1UXD6DDUKgmTWea/6w4D9muIG8WEEaVTY5bLy3dasXqEbYfDjqtLAhtHaPN7iH4Raxvu3uZQfQ5ZLcL2SZ6FMFQtQr+FcM6cOYwbN45jx45V+uJffPEFY8aMYdKkSfzxxx906tSJvn37cubMGZftS0pKaNGiBTNmzCAxMTEg56wu1CqJhEsfioJLYQff7Tit7JckidRGFqfU0bPFdsdaLUhbB4mjs0QkXajZ2L7NtnOEoeqs8tUitGagyQ2xDDR+C+GAAQNYu3YtLVu2JCYmhvr169u9/GH27Nk89thjDBs2jPbt2zN//nwiIyPd1kbu1q0bM2fOZODAgXYB3ZU5Z3ViDaFxR+qllSNHzxbZbbc6SzQehsaBDLOwPZUYGlcN7nIQhqJFXlxm5Ng5y8O6nY8W4blivU9FyKoKv+cI3Q1H/UWv15OZmcn48eOVbSqVirS0NDIyMqr0nGVlZZSVlZvqBQUFvl/03GFo4Bzr5wtJceFs97A/tYHF++ZkEdqEz1hxdJYErYpdCH4RayP2QdQ2FmEIPoj2Zxciy5a1xNZldO6oH6VTsiWdL9bTONazMVBV+C2EDz/8cEAufPbsWUwmEwkJCXbbExIS2L9/f5Wec/r06UyZMsX/Cx74CZY+ADeOg+vHWhb9+oE1HZc7rEPjI7n2QmhSwm3ch88E9gtTM9a61lbsh8nV1g23+DosBsuUUMPoMM4UlnGmsMxeCK0frmqIRKlQYtbDhw/zyiuvMGjQIGXubeXKlezZsyegnasqxo8fT35+vvI6ceKEbwdm7wKzEX59DX5+xW+VsCZodUf50LhcCGW5PMGDR2dJkBRLWIRVg7vhcCi+/746Sqy4TMdlNsOPY+GXyYHunk/4LYTr1q2jQ4cObNmyhW+++YaiIsv81c6dO5k0aZLP52nYsCFqtZqcnBy77Tk5OW4dIcE6Z1hYGLGxsXYvn7jhBeg73fJ7xnuwYjSYfM+qYQ2qtuI4z5fawGIRniksU5YvGWyGvLbhM9oghs/YzRGG4BexthPqK0v2+WERgo0QFlwSQpMBvh0B2z6CjW9D1s6g9NMTfgvhuHHjeO2111i9ejU6nU7ZftNNN7F582YPR9qj0+no2rUr6enpyjaz2Ux6ejo9e/b0t1tBO6dXeo6Cu94HSQXbF8NXQ8HoW2hAkoOzROtg1cVFamkQZXmPj12yCm2TKdhahI5ZbwKZdMEWoYNVj332merrhytMZpn92X5ahNE2FqHhIiwdDLuXgUoD93wESZ2C1l93+C2Eu3fvpn///k7bGzduzNmzZ/0615gxY/jwww/5+OOP2bdvH0888QTFxcUMGzYMgIceesjO8aHX69mxYwc7duxAr9dz6tQpduzYwaFDh3w+Z1DoMhju/xTUOtj3f7BkAJQVeT0s0WGi2NU65dSG9iE0thahxi58xv5fGUgdtJujCrVvYi3FffaZwL7/s3/+i0cWbauwB/fo2WJKDWYidWqaN/AtB4HVIizIOweL74GDP4EmHAYugQ73VqgflcVvZ0l8fDxZWVmkpqbabd++fTtNmzb161wDBgwgNzeXiRMnkp2dTefOnVm1apXi7Dh+/Dgqm+Hf6dOn6dKli/L3rFmzmDVrFr1792bt2rU+nTNotLsDBi+Dzx+AI2vg07vhgS8h0n1IUWJcOAmxYeRcGiJYM07bktowit//vqAIoe0HVuvBWRIsi1AMjasG27f5g3VHlN8DLYQLNh6jqMzI51uP8+6vh2jeIJIb2zSmT5vGtEuK8bqE1uooaZsY43MuzsYxYdSngMH7J0HZQQiLhQe+gObVl8/UbyEcOHAgL730EsuWLUOSJMxmMxs3bmTs2LE89NBDfndg9OjRjB492uU+q7hZSUlJ8SmOytM5g0qLG+HhFZan3MltsOh2GPItxLien9SqVax85gaumrYagDCNs4Ge4mARWgOlVZJ9yq/grjUu/10YhFXPN9tPKb8H0iI3mszK3PPKP7MVT+62YxeY+dNfJMaG06dtI25pn0CfNo1diuLflz6XrRpH+3zdZPUFlumm0KwsCyIbwpBvqmU4bIvfQ+P//Oc/tG3bluTkZIqKimjfvj033HADvXr14pVXXglGH2sWl10Nw1ZCdCKc2QsL+sL5o26b148qn2d1FTbQ4pIQHnEQQkfh82dlyY4Tedzwxhp+2pPt+V5cEIoBvbURd+9yIMNDi2zS5eeVWFJoXd+6IWntGhOhVZNdUMrnW0/wyKLf3X5WrOeIj9S53O/E2UNc99tgWqqyyJEawiOrql0EoQJCqNPp+PDDDzl8+DDff/89ixcvZv/+/Xz66ad1Ng2XEwntYfhPUC8FLhyDBf3gzD6vh7kaWCjL7HKLkGW5fFWJg/A5Dks8WYSPf/o7x8+X8PinmV77BA41S4RJWCW4e94E8kGUf9Gg/J5XogegT5vGfPRwN7ZPvIVPHumuOEBO5bleElfoQzJWhaydsKAvYcWnOWxOYrBpCjRsXcm7CAwVrmvcrFkzmjVrFsi+1C7qpcAjP8Gn/S2W4cLbYPBXFovRD1IuTUAXlBo5X6xXnCWOjhVHZ4mnlSXequR5Quhg1eCuWl0g5wgLLpZbhBdK7JOqhmvV3HB5I77fdZq9WQWUGlyXjfAlPT8Af2+65EQswJTQkfv/foJzxFFcZvR+bBXgUw/GjBnDtGnTiIqKYsyYMR7bzp49OyAdqxXEJMLQH2DJ/ZY5w4/vhEFLLHOJPhKuVdM0PoJTeRc5eraY6HDLv8xR+ByF8aLBxL9/2MvLt7d3Oqfjsd4I9bWutRF3b3Mgl+daM0qD5fMCzoIWprGM8srcPDytQ+MYT2J24Gf48iEwXoTm16Ie9DkX/5MBehNni8pqjhBu374dg8Gg/O4OkaTVBZH1Ych38MWDFm/yZ/fBvQug3T+dmrp7+5rVj+RU3kVOXrioTEo7Do1dFXn/8LejdkJYXGYkTKPyWwhtCcWVDXWJwFqEBqdtUQ4V6MK1ls9KmReL0PqAdmL3V/Dt45YVWK37wv0fgzaCRjFh/H2uhDOFZT6H3QQTn4RwzZo1Ln8X+EhYtCU84OtHYd8Ky9Pxzvcs8Yc+EBth+TcVlhmVnIXenCWOFJQauHbGrzSKCbOzBHzBPvuMX4cKKog7uQuk19jV5yDGQdCs2abdDY09zhFu+x/88Dwgw5X3Qv/5oLaU8GwUbRHCUMlLWHHTQOAfmjC4dyF0eRBkMywfBRnv+3RodJjlw1NUanS5zhi811M+ef4ihaVGjuQWU1ha8eLawiKsIty8z4Gco7WdI7TiOEwtF0J3Q+NLc4u2AirL8Nub8MMYQIarh8O/PlREEEIvQatPFqG1uLsvfPPNNxXuTK1HrbFYguHxlrXJP42H0jygI659xhasT+niMqPPzhJH3E2++4Jd/QwhhFWCW4swSF5jK1E6xznCS0NjNzW2rUNjZY5QlmH1RNj0juXv68fCTa84zfs0DjEh9MkitC3mHhsbS3p6Or///ruyPzMzk/T0dOLi4oLW0VqDJMGtr1k+HADrXmeS5hMkzG6l0DrsKCozKitGvDlLHAnU90esLKleAjpH6MPQOMyDRSjLcnkt43ANmE3wf0+Xi+Ctr8HNr7qc/K6RFqG1uDvASy+9xP3338/8+fOVuEGTycSoUaN8z9pS15EkS+aa8Hj48QWGaX4iTipmKS+5bG4ddhSWGpWwGF+cJYFCrCypetzXLAm2s8RhaHzJIix1YRGWGc3KCCVabYKvHoG931kSkPzzbbjK/Uqz8vrGoZGy32+/9YIFC9iwYYNd8LRarWbMmDH06tWLmTNnBrSDtZruj0F4PIavH+df6g0k5xnAsAK09skYyi1Cg1tniTeLMFBfIDE0rl4CGz5jP0eocxFR4MlZYrUGIygl+pshcORXUGktGWSuuNvjtV3mJKxG/DYjjEajy2zP+/fvxxykhf61mo73McIwhlJZSzf9FvjsXii1LxVgHa5YhsbOhZvA2UJ0JGBDY2ESVgnu5nQD+SBytAhdeX7L5widv9tFpUZiKWJJ2OtIR34FbSQM/tKrCAI0ir5U1rMmDY1tGTZsGMOHD+fw4cN0794dgC1btjBjxozgprqqxawxd+Fh/TgWhs8i8thv8MmdMPhriGoA2FiEpR7CZ1TenCWBQehg1VAlQ+NS70LoyWtceuE0X+heo510HMLjLCunkrv7dO3GsRaL8GyRHrNZtksgUh34LYSzZs0iMTGRN998k6ysLACSkpJ44YUXeP755wPewbrCFrkdk+q9wczSyXB6OyzsZwnEjmuqzNsUlrkPn/FuEVbCa2w3RyiUsCpwm3QhgIMuR6+xqxUeViF0Cqi+8DfNV9xDhOo456R6NBj2AyRc4fO160fpkCTLCONCiZ4GXoo+BRu/h8YqlYoXX3yRU6dOkZeXR15eHqdOneLFF18USRcqyVFtKxi2CmIvg7MHLJlrzh22swiNbsJn3DlLrAIYKEtO6GDVUDXOEvs5wugw5++vdWWJ3Rzhmf2woC8RhX9zwtyIl+vN9EsEwRL1UP9SxpozITA8rpSr0a/6HgLfaHS5JTVRg1aQfwIW9KV+4V/ApThCszX7jG/OknIBDEwcoZgjrF4CNUeoN5qV9cVWXFmETmuNT2VaRiuFWRTEtORe/SSKoyqWfCWUQmgqtNr5q6++4ssvv+T48ePo9Xq7fX/88UdAOlYXUcKt4pMtluHi/pC9m8Rv76Gr9ByZ+jZK5hgnZ4kbITSazahV6oBZcmJoXDW4c5YEKo6z0EUMoes5QhuL8Oh6+HwQ6IugaVdWXj6bnJWnuKqCSRMaxYSxP7swJITQb4vwnXfeYdiwYSQkJLB9+3a6d+9OgwYNOHLkCLfddlsw+lg3iW5kyVzTrCeqsgIW66bTW7VTSaDpvNbY9b9y9s8H+CTjGOeL9S73+4J99pkKn0bgD26HxoE5vWPoDHh2llxn2gqL77WIYOoN8NByzsnRbo/zhVAKofFbCN9//30++OAD3n33XXQ6HS+++CKrV6/m6aefJj8/Pxh9rDNIjmtLwuPgwW+g1S1ESHo+1M6i6akfAffOEkfD8L/rjzBx+R6mr3Rd4N461NpzOp+bZq1l5e4sj30UQ+OqobJJFw6dKeS++Zv47WCuy/2+BFODJXzmX6r1zNW8BaYyaHsHPLAMwmK8Z57xghJUXVADhfD48eP06mUpshIREUFhYSEAQ4YM4fPPPw9s7wSgi4SBS1glXYtOMnHvsckMVP/qNBS2Jl1w5zQ55+apa41LHLk4kyNni3niM+epDdlFe0H14OvUxPPLdrHt2AWG/G+ry/2u1hm7suyidnzEbN18NJIZQ4eBcN/HSsC/T7kIPWBX1rOa8VsIExMTOX/+PGDJUm2tZXz06FGx6iBYaHTMCB/DYuPNqJCZof2IPufsHzrWobFjoXcr7vTLGpfoa0YavcjDVSW4+y75+hwq8pJqzdU6YzshlGVYOwPtz5ZyuguM/ci/dY4lcYj1GmU+Zqd2Q7mzpPqX2fkthDfddBMrVqwALMHVzz33HLfccgsDBgxwWe9Y4AceQgEjwsN4xfgIyyLuB+C2rHmwepIyaWe1EN2l43I3pDUYZY/7wf5LaahEmn8BrP3rDCfOl1T4eF+HxnERWo/7PabgMpth1XhYOx2Ad0z3MdU4hFKj/bUDNTQOBWeJ33fwwQcfKEvpnnzySRo0aMCmTZu48847efzxxwPeQYEFy/BD4l3pAQ4a1EzQfg4b51jSeN0+u9widDM0djekslp4vn7BDEGwCGVZrhPZzTccPMvQhdsAODbjdo9tKxtHGOtNCF1YhFFhajAZYcVTsHOJZWO/1/loVSoYjE6rS4r8KdzkgsYxobPMzi+L0Gg08tprr5GdXV7ab+DAgbzzzjs89dRT6HQ+lvQT+I31qXuhRM8Hpn/yY+p4QILMRfD1cJrGqInQqt3Wl3X3BbKm9fIUlmG7J9BD47/PFdPt3+nMW3s4oOcNRbYdO+9z28qW8/RuEVqE0LaWdpzWZMmevnMJSGro/1+4ZmT56hKHDDTKHGElLcKCUqPbDNhVhV9CqNFoeOONNzAaK57hWOAeTzaR9alrncvbm9gf7ltoyfax51vqrXiYjc9fwyfDXa/1dDtHeGm442u+jMpUwHPFjJX7OVtUxuurXHu1axPesojbUlmL0FYIjS4eXlaLsOElh0UUF+ny2wj46wdQh8GAxdBpIOB+vbEyNA7zLLruiA3XoLskxNVtFfo9R3jzzTezbt26YPRF4AHHeRiNWoIr+sMDSy1ZPw79Qv2vBxBmKHR5vLs5QKuF58kitB02B3poXJcCtP0RQncUlRo5klvktV1seLk4ufIQ51+aI6wfpSOeQj7T/Zu47AzQRcODX0Hbfyht3RVw8qumsQskSQoZz7Hfd3Dbbbcxbtw4du/eTdeuXYmKsq9Adeeddwasc4JyHEMUlPCZVmmW5AxL7oMTm2HRHTRkFGfxLVu4VSA9OUsWbjqm/C68xhXHn2lQdytL1h3I5aY31/HtqF50aVbP7fG2mpt30eCU1MA6NG4ZXsCbuqlcrjqFKbwe6iHfQNOr7Npal9k5JmdV0vRXcGgMluHxqbyL1W4R+n0Ho0aNAlzXL5YkCZOpesf6tRXHp65dvGCzHjD0R0sx+ZzdLNNN5kH9BE7RyOt5fall8sOu8iBrg7HuWHCBRuWHEnozlD/67ShzB7sXQtvnWl6J86qiglIDzaVsXj3zBg1U2WTJ9QkfuJx6TTs6tS23CMsfgkZT+VrlilqEEDq1S/weGpvNZrcvIYKVw9P3xGlo7DjMSrzSkqwhrhmpqhy+CptCS+lUwPsYaIvQaTVNLSaQKff+PO15FZftlIN1WaYtDYsO8pVuCg0M2Rw1J3CffhIRTV1nkFHmCG0swuKy8t8rU6C9PGV/DRNCQfDwJAqOHzaXYTINWsLwnzhobkqSdJ4vdVO5Ujri8Zr+TtGJOMKK459F6Pkfk5VnCUJ2F/Zku/WCoxCe2MqbJRNoJOVzLqo19+snkSU1tvMg26IMjW0swsKycq+zzs1xvhAqsYQ+S/nFixdJT0/njjvuAGD8+PGUlZV3Xq1WM23aNMLDw92dQlAJnOYI3SVijW3CQMOr/E/7Bp1VR/hc928eMzzPZnN7l839FUIxR1hxbIWwsrGTepOZ/9t5mgnf7GbmfZ3od2Wi3X57i9BmaHwoHb54kFhK+N18OSevW0Tu8mPEhWnc9ifMRU7CyobOWAkVIfRZyj/++GP++9//Kn+/9957bNq0ie3bt7N9+3YWL17MvHnzgtJJgfPQ2FNq/gJVHIP1L7PJ1J4Y6SIfa1/nZlVmQPqhN5kDupSyDsRRK9h6jQ1eAgJ9eYe3HD1HYZmRySv2OMfh2ZxA8Rrv+Q6WDABDCetMHRmiH0dcPcs8sqd5vnDHnITYhs5UUghDxGvssxB+9tlnjBgxwm7bkiVLWLNmDWvWrGHmzJl8+eWXAe9gXcLjHKHDB85TKIZaJVFMBMMML/KzqSthkoH/at/ibtUGp7lFfwu/y7LIQFNRbN96b2FIvjxrrG2yC0pZsPGo3T5bi/BCiR7++AS+GgZmA6WX38mjhrGUSuFc06IBvVo2YEjP5m6v4ypLdWFZ5ZbXWWkcaxlBnq0pFuGhQ4fo0KGD8nd4eDgqG6uke/fu7N27N7C9Eyg4DkE81Six5iosQ8cThmf52nQdGsnMHN37DNP8XOm+eLNm/KEuWYS2Q0+jV4vQ+3tsHZ4CzFt7mAs2OSdtn1VdTy62LJuTzXDVw5y86T0MaIgJ0xChU7PksWsY2bul2+u4CqgOmEVoMzSuzqQtPgthXl6e3Zxgbm4uKSkpyt9ms9luvyCwOEbvu1tTDPaWhwk1Yw0jWWjsC8DLqoWMVn+LdexUkc9eoFaXFJcZ+XF3tveGtQRb0Q/EXKttst3CUiPvrTmk/G35v8qM1XxB/7PzLRuvfQb++TYFesu14yJ9WxFidaK4miOs6KoSKw2jLcty9Sazy8DvqsJnIbzsssv4888/3e7ftWsXl112WUA6JXDGa/iM7T4HkZRRMcX4EG8Z7gFgrHYZr2gWI1GxL2OgHCbf7Qh8eE8oY+vhNXpZ0+jLA8oqhF2bW+IJP834W8lsI5tNTNMsZLRmuaXxzZPglqkgSUowte3qE0+UrzV2tggr6ywJ06iV5YDV6TDxWQj/8Y9/MHHiREpLnXOHXbx4kSlTpnD77Z4zagg842mYGKlV2+33bBG6OpHE26Z7eFc3HIBHNSt5Q/MBmP1fNx6oZXZ1barRdm41EIHpViG8rlVDrmvVEL3JzKyf/wKTgX8dm8IQzS+YZYk3NCPh+jHKcdY0/b4LofMSu8our7MlFDzHPgvhhAkTOH/+PG3atGHmzJksX76c5cuX88Ybb9CmTRsuXLjAhAkTgtnXWo+nOEKVSiJKV/6h8zRH6EEj+T7ibp7Xj8Qoq7hPs57kX0aB0b8PYKATL1i5qK/dAfm204IGbxahD+ezCqEkwbjb2gLw046jFC66nw4XVmOQ1TxtGM0nhpvsjlMswgjfRMxVQHVlcxHa0jgEgqp9FsKEhAQ2bdpEu3btGDduHP3796d///6MHz+e9u3bs2HDBhISEoLZ1zqP7dPXsXiTLZ72hWlVfG2+gVGGZyiTNcT9vQr5s/uIxNnSdzd5bTCZyS0s4530g+QUBC67cJ9ZawN2rlDEn+QVvgyNbYeqVzaNY2CHWD7WvU7MiV/RS2E8Znie7809KSoz2j288v0cGpfPEdoMjS8FVNc5ixAgNTWVVatWkZuby+bNm9m8eTO5ubmsWrWKFi1aVLgTc+fOJSUlhfDwcHr06MHWra7rLFhZtmwZbdu2JTw8nA4dOvDjjz/a7R86dCiSJNm9+vXrV+H+hQq2T1/PFqH7fdYP9c/mbgw1vIRJG4V0dB2f6f5DHOVZTV797k96z1zrsuxjmdHMox9vY/bqAzz1+faK3IpLsgMoqqGIbYYfb15jv+tQF+UyNW8cPVT7KZAjGBs+mbXmzspuW0eENQWXt+StVlzlI6xsUlZbQiGWsEJrY+rXr0/37t3p3r079evXr1QHvvjiC8aMGcOkSZP4448/6NSpE3379uXMmTMu22/atIlBgwYxfPhwtm/fzt13383dd9/t5Mjp168fWVlZyqs2FJaytwg9hc94D60ByDBfweHbliCH16OL6hBf6qbSmAsArNqTzfHzJfyV7ZzWy2Ays/OkZa3r1qO+Jxt1xFUvT14o4fOtx52SgNYGbOcIvTmc/PHmx5Rmw8J+6HL/pFhTj0H6V1lxwT4uMP9iuYfZmqbfW/JWKy6X2AUofAZqoEUYDGbPns1jjz3GsGHDaN++PfPnzycyMpIFCxa4bP/222/Tr18/XnjhBdq1a8e0adO46qqreO+99+zahYWFkZiYqLzq1XOfqSNU8BZTZ+uh8+gs8SCEjqPmi406kz9wOdlyPdqoTvKVbjLms0eUOSDHZJxgH0dYz8cQDF+5ZfZ6xn+zm7lral/Galsh9G4R+kYL6TT37X4Uzh2CuGSMD6/keFgrp3a2640Vi9DH+T1PS+wCMkcYW8eFUK/Xk5mZSVpamrJNpVKRlpZGRkaGy2MyMjLs2gP07dvXqf3atWtp3Lgxbdq04YknnuDcuXNu+1FWVkZBQYHdKxSxswg9BlS73+fokJGBi/Uu5179JI6ZE2imykVa2I9k4zEAJdWSLbbzTdaneaCwXm/jobMBPW8oYArwHOEV0lGW6aYQW5YDDS+HR1YRl9yOUTc6C6FtBppyZ4mPQ2PFInR2llS0lKctjaItq0vOVGM1u2oVwrNnz2IymZycLAkJCXZ1UWzJzs722r5fv3588sknpKen8/rrr7Nu3Tpuu+02t2nCpk+fTlxcnPJKTk6u5J0FB1+dJZ7mCB2tTlmW0RvNnJQbc59+IvvMyUjFOXypm0oX6aDLWhK2X+JAC6Ftv2obtsvevA6NvcwRdpf28bnuNRpIheREtYVhKyHOEsc77NoUkuLsk58U2M0RVjB8xjaOMIAWoRgaB4mBAwdy55130qFDB+6++26+//57tm3bxtq1a122Hz9+PPn5+crrxIkTVdthH4m2GxpXzFniCusHPJd6DNC/SklCV+KlYhbr/kPM6d+c2hfry2MPG0a7F0JZlln1Zxa7T7rOnedpKqD2yaB/Q2NPz4E+qu18optBrHSRLea2fN1hHkQ1VPaHa9W8eX8nbri8EW0TYwD70Be/LcIgLrGDciG8UGIIWmiWN6pVCBs2bIharSYnJ8due05ODomJiS6PSUxM9Ks9QIsWLWjYsCGHDh1yuT8sLIzY2Fi7Vyhi+6HzJHae8t457pOxH+oWEM3OPotYb+pAlFTGtVtG0Vdl78W35sIDiI/QsvnIOU7nXXS61rLMk4xc/AePfrLNbX98ZevR8xw7W1zp81Qntl7jigal36naxAfa2YRLBn4xdeEh/Tj0GufKhb1aNuSTR7orQmgbo+lvHKE10sAaUG02yxTpA2cRxkdolemcc8XVYxVWqxDqdDq6du1Kenq6ss1sNpOenk7Pnj1dHtOzZ0+79gCrV6922x7g5MmTnDt3jqSkpMB0PEh4y09nK4SenCUe5wgddm0/nmc35AE4p9fwqGEsP5i6o8XI+9q3uU+9Vtl/ykb0tp/IY+AHm+k141ena32xzWJZ5xSUkX/RwPFzvhc2t7WIDuYUcv9/M7ixhscZmkx+zBG62PagejVztHPRSia+NV3LSMNzlOG5hG6EzmLNWYVQlmXFWeKr19gxoLrEYFL+PzGVXGsMFudedQ+Pq31oPGbMGD788EM+/vhj9u3bxxNPPEFxcTHDhg0D4KGHHmL8+PFK+2eeeYZVq1bx5ptvsn//fiZPnszvv//O6NGjASgqKuKFF15g8+bNHDt2jPT0dO666y5atWpF3759q+UeA0Wkj+EzHucIHf6e9v1ep+HIhRIDerQ8ZXiapcYbUUsyM7UfMFxtidc8eaFcCHe5GfaCfRnJrtNWc8PMNZy84JsY2grBntOh6bzyF3uL0PPQ2L66n8wo9Xe8pl2ISpL52HgLYwxPYPQhr7JVxKxOqFKDWbm2v2uNDSYZk1lWhsVqlaTMH1YWJWV/QfUIYeXt2koyYMAAcnNzmThxItnZ2XTu3JlVq1YpDpHjx4/bpfvq1asXS5Ys4ZVXXmHChAm0bt2a7777jiuvvBKwZMretWsXH3/8MXl5eTRp0oRbb72VadOmERYWnIn9qiLcJiW6Y2IFWzw7S5z3OU7c511aumVGxTjjY+QTxeOaH3hVu5g4qYiVF4b71F+7AOJL82OZf1/gsnqRlr74WK+ktuQ/tEu64G1oLJf/Ml6zhMc1PwCQ1+1ZDurvQd7i2zx2pM5eCK3WoFolKfu8YSt2ZUaT3aqSymTZtqW6g6qrXQgBRo8erVh0jrhycNx3333cd999LttHRETw008/BbJ7VYa3j1SYtvyDW1FniatdrixC215NNz5AvhzNi9oveFrzHY3yLjKBIcgeBhRr/jrDn6ecLbmKOINrS+1jX+YIp32/l9V7c0iuH4EKM//RfMRAzVrLPsNgXr19Clf9cZLFPgphhNY+9KU884zvImYNqLacxxzQYGordX5oLPAd25ASz2uNPX3AXViEDkLoXP5R4n3TXbxiGIZZlhjET8zWzkOD+8w1wxa6dpD4LGo27WqyEP60J5sHPtxMdn4pttrnamh8vljP/zYc5fj5ErYdyuY97TsM1KzFJEu8YBjB/0yW7E7+FIGyDmtLLs0R5vvpMQbLg9X64C01mJQKdpVNwWVLdQthSFiEAt+wHSJWeI7QxS7H5WwXXNTBBVhsuoUCOYo3tfPor95IDCU8aXjG64S9LT7roM3vNXlk/Pinlloxk1fssXNOuLIIfzuYC0AEpfxX+xY3qHdTJmt42jCan8zdlXb+jEYdnSX+OkqshGvUGExGyozmgCZcsFLd9Y2FRRhCePuA2wqhp2V0gR0a27PC3IsRhjGUylrS1Nv5WPc60VgcILaF4N1hvQNLFTfPbT/NOMak5X/WqDnC3MIy+s1Zz4IN9jVELpTolXlSwO53K5IkEUsRi3XTuUG9m2I5jEcML9iJoLWdr0Q4OEus64x9dZRYCbMZYhcGMAWXlfL6xtWzukQIYQ3C1yGiZ6+xD84SNxahlTXmLgzRj6dAjuAalWWFQ30KeHLJH177ZjSZ+Xzrcbq+9gt/nvJcpPzV5Xv4OONvMo64Xx4Zarz760H2Zxcy9Xv7+j1qlWT3/5v5019KoPnRs8Ws+jOL8NIzfKGbRlfVQfLkKB7UT2CjuQOO+BMvb3WIlDo4S3yNIbRim64/kJlnrChD42pylgghDCG8fb5dWRGuUHuYP3RlTDjNEbqpHWG7rnSb3JZB+lc4K8fSQXWMZbopJOFdsIr1JsZ/s5vzxXo+23LcbTtbzS+oxloW/uJqSSJYhNDRsh252DJs7jNrLa99toouvzxAO9UJzsjxDNC/yna5tctzVWaO0N80/eXnKc9JGKg0/bZY1xtXVxEnIYQ1CLOvQujhe+J6jtBeCN0V0bH1WgPskVO5Xz+RU3IDWqqyWBY2Bc66Xr1jpaTMt9IA/pYZDRXchQRJkmTnNYby9bqtpZN8rZtMI8MpjpsbcY9+En/Jzdxewx+L0GlobF1n7O8coU1OwmBahFq1ShHtqkQ4S2oQgbEIvXuN3T2QwzTO5z0iN+G+skl8qptOS1UWLOwHD37j9vpFet+E0Jaa5DV2Z6ypJecHmVolwclMvtRNpZ5UxEE5mcH6cZzBc8o4v+YIrUNjq9e4xL8UXFZs1xsXBqiCnS0ROjX7p/VTrlPVCIswhPD2AffVaeA5DZczvlal07kQQoDTNOR+/UT+NKdAcS4sup2rpf0u21on671hq30bD9WcOUJ3/0KV5Dw07i7/CZ/cST2piB3mlgzQv+JVBMH7FIotthbhkdwiMo9bEu/66zVuEGWJDDicWxTQeiW2VJcIghDCGoWvlpEnj7IrsS1zkXzVFZ6CuM8RxyD9K5iTe0JZAZ/qZnCjyjmNv6/D+9qGymGO8FbVNt4xvwb6IjaYruAB/cucl2N8O1cF5gjzLhroN+c3Dp0pIkyjoksz/xIV927TCIBf9uXYDI2rT7gCjRDCGkT7pDif2nmzCB0FTe8mT6Mj7ixCK4VEknfPUmh9KxGSng+1s7lDZZ8w94vffVsRUYNGw25Z9Wd5jky1zRzhPar1zNPOQYcRc5s7eMTwIiWEuzuNEx5mPpyweo1l2WL539imET8/dwNXNvXts2Tl5raWJa87TuQpWYACOTSuboQQhhDenvPXtmrAu4O68PNzN3hs5y2OcMwtbey2+ZoDztNqFislsg4GLmG5qRdaycQ72vd4QJ3u9bjagMksc9QmVZjVKwzlXuNh6pW8qZuPWpL5QX0TJXf/Dz3+CYo/c4TxkTraJsZwWb0I5j/YlYVDu9G8QZRf1wNIjAun42VxyDIcsQphgIfG1UntuZM6gCRJ/LNTE6/tEuPcWxeSJDGydwteX1U+h+erEOo8FUy+RKnBBOpInjWMokCOZIjmF/6j/R9xFDPPdKdP14GamZj1pa93sfmI62JWKgnuzvuEe7SLAfif8TY+iXiM7kb/kxb4MzRWqyR+ePp6VJJ/AuqKtHYJdtmGAuk1rm6ERVgLGdorhQ5uhj4Szl8IV84SVx5iXzJfW0MfZFS8ahzGe8a7AHhJu5Rxms+pmRLnG19lnnS5XcLMvbnvck+hRQRnGe5jmvFBVGq127hDV3RKjr90Pv9Qq6SAZIm5uV1ju78DGUdY3QghrIWEa9XMfeAq1zt9CKgG1yn4fZmbKtGbbBwiErOMA/i34QEARmr+j+maj1Dh3QKtLTVLNBiZpZ1P7wuWkKKJhod5z9QfkFBJrotjuePDIV0B/yzCQNI+KZYmNqMNYREKgkIgP98RbnLNufoSOQZUgyXg1tEq9OULuGTLcWassg+d+dB0By8aHsMkSwzSrOEd7btoPWSugdB3luSXGHg3/SAnzrtPNBuGnnnat7lHvQETaubEvsAnpvLkwGqVZJdC3xuNYy0i5GdJmoAhSRJp7csLp9WmOUIhhCFF4D7h7pJuurqCqy9jTLjGKa7Ll6Hxip2n+WD9EaftX5r6MNrwNHpZzR3qLXyknUUE7hfYh3oQ9YTvdvPm6gPcPXejy/1RXGSh9g1uUWdSJmv5pNlrrAnrY9dGJUl+WYRWApUMtSKktSsXwiidEEJBEOiRWj9g54pwE5xqLaZti7UWRZSNeMaGa5zOoa7kF3CluQfDDS9QIofRW72LT3UziKXIZdtQF8LNhy1B3ueKnRNU1KOAJbp/00u9l0I5gocNL3Eg/jqnGEqVJPk1R1h+XMX6HAiuadGAtHYJDLmmud/VEkMZIYQhwPoX+vDGPR0Zem1KwM7pLqjaVeiEtUyj7VAnOkzDeYcveSAskd/MHXlQP558OZKrVQf4QvcajchzahfiOuj0/lrTbiVyji910+ikOsJ5OZoH9C+z2dzestbYQQj3ZhUoa38rc+2qRKdR8dHDVzPt7iurrQ/BQAhhCNCsQST3d0v2WJkuUDSvH+m0zWqV2E5+x4RrnbzJgereH/LlDNBPJFeOo53qOF/qpnCZlGvXxjFBQahhG7ReZjQx9fu9pEhZfBU2hdaqU5yW63OffhK75RaAxZp2ZeV+tvlvv6+dGOt78LXAN4QQ1mIe793Cyfub0tCVRehKCDUehz5Drmleqb7tl5txr34SJ8yNSFXlsEw3hVZSefhJqA+Nbd+bUr2ZdtLfLNNN5TLpLEfNCdxXNonDclO79q7Wiv9xae2vP1xWL4Km8REV67jAJUIIazHjb2vHt6N6KX+Ha1VKSvR3BnUB4MqmscrQOMrBInQMoD5jk0a9SQC+iH/Lidyrn8QBc1OSpPN8qZtKR+kwEPpDY1shNP69maW6aTSS8tlrbs59+smcopFde1dJF8B7WU9XSJJEz5YN/O+0wC1CCOsQKQ2ilHk+azzY0dxiSvTO+eWiwzVOa4ubxJWLn+cCUb6TQ33u109kh7kF9aUiluj+TU/VnpBPzmAVwhtUO6n39b3ESSVsM1/OQP0rnMU5mF2tCuxwv5cQwoAihLCWYzux3rxB+fxg5+R4ujavR7HepBRHsnWWxIZrnOYsr2nZgDkDOvPTszcENOYxjxgG619mo+kKoqVSFmnf4FrTFtdtS/QeY+/eWn2AySv2BK5zblBLErerNvORdhYqYylrTZ0s5QtwvY7XnUVYUWwtQl9TmwncI4SwlmOrVyk2HmONWsWcAZ3t0u87zhFG6Bw+HrLM3V2a0ibRt3RR/lBMhKVQkelqwiQD040z6a/6zald56mr6TTlZ5fnkGWZt9MPsmjTMSVDSiCQZdlJxG43/My72nfRSSbOp9zOY4bnKcU5NMnKrpP5Tit4mrlwXPlKko11vvVYzcnXGKoIIazl2FpuzRrYf/GS60fyWn9LGESkTk2kzlYUtfRq0dCufbAHq2XoGGV4hq9N16PBzFu6eTys/smpnd5kdrkE75d9Z5TfS42BS/c+bNE2bn5zrVL2NOvHGTx78T1UkswS4038ec1sDF7yl2QcOWc3xwpweULlHihdm1tyCtoGOQsqRu0JDRe4xHZZXIqLGMK7Ols8m7HhWjsPZky4hgm3t+Pv88VKRhVb7QnW6gYTasYaHqdME8MD8o9M0X5MHMW8c2l9rpUyo9lp5cvvf5dnfjH7llDHK3qjmbV/WUJ79p8uoNOBt0na+hYA84z/5HXjQP7rp+aqVRL9rkykef1IftmXU+G+LRzWjd8OnHVKhiDwH2ER1nJsw1CaN3A9FLurc1P6tG1sl28wJlxDXISWpSN60ukyy+R/3ysSg9vZS8ioeF0eymzDvQCM0X7FRM2nSDbJGlzNExbZBCcHqviTtc6uCjOpW16BDRYRnGEYyOvGQYD/c38rn7meuQ9cReuEaGXbsGtT6H5pZdHkf7Znx8Rb7I5xlQQjNlzL7R2TqjXFfW1BWIS1HNuQGNt5JVdobDJXx9iUe/xm1LWU6I1221zZg2ntEipl4dgiA++Y/kUBkUzWfsIjmlXESiW8ZHgME2ouGkxO1T2KbCrkBcpBm1NQhhYjs7XvE7tnMyAx3vAIn5tuVtr4G/NonSts3bh8aNwuKZaJd7Qnp6CMhNgwJ4t75TPXV/wmBF4RQljLiQ3X8v1T1xGuVXtdG2q739ZxolZJdiIIljWnjjSND9yKB6uRtcjUjwI5kje0H3Cvej0xlPC0YbTLko9FFViu5o3c8xf4QPsmfdQ7MaAhq88cPl9pvybcX4uw+JJgt2ocjSRZRDtCq0aSJJdJddPaNVbKXQqCgxga1wGubBpHq8bRXttZYwOjdN5Fs32TWA7/5x922x7qlVLhPjpia919Y76BJwzPUiZr6av+nQXameTlnefd9IO8m35QqcMccIvwYh6d1z1CH/VOLso6HtU/zz9+dZ6P8/da1hRp4Vq14jl2ly0IfCuRIKgc4h0WKFiF0NH6c4daJdHaRmBbNopm68s3ezii4qw2X81Qw4sUyeFcq95DvWX38r/Vmby5+gAjP7XUBim2qZlc0eDlQ2eKeHLJH+w7eIiy//2DxLztFMiRPKgfzzpzJzuxteLv0NjWgfXkja244fJGdPOQeUjjoXqgIDAIIRQoqC8FUPuTcHPiP9sDMLJ3SwAaxwQvIUCG2Vr2MpqWhr/4UjeVBM6TccQSR2c7NDb6WKvZFpNZJm32Onbs2kXYp7cTdnYPuXIsA/Svkim3cXucwc9r2Vrb93dL5pNHuhPr4eHT8TL/Ks4J/EcIoUBBq1iEvgvh9a0bsXdqX8bd1tbl/igPQ76KsEtuyf36iZyhHperTvGVbgrNJIuDpqisfN7Q6Oe8ndksM/Onv2gpnWJZ2BRaqLIpCEvkPv0k9smeE0z4k2UafEtwC/DD09cx7ra2DLs21a/zC/xHCKFAwTp31SBK59dxkQ6Zils0Ko9XHH5d4L/Eh+TLuN8whWPmBJJVuXylm8Kva3/lbFF5wLLRj2QGsixz46y1bFi/mi91U2kineeQuQmftPuAY3KS1+Mn/99ev/rva7q1K5rEMbJ3yypJz1bXEe+wQOHmdgk8cWNLnk27vFLn+XR4D+X3pCClizpmamix1szNaCzl0XXNYK6SDij7jX5EVBvNMkkXMvlc928aSIXsMqdyn34isza7zp5dWVLcxHMKqg8hhAKF6DANL/Vry5VuSoH6SlXlysslngH6V/jdfDlxUgmLddO5XrUL8M8i3PrTEj7WzSBGushmczse0L/MBWKD1e1qrTkicI0QQkFQsNZfubW993WwzepHsnPSrRW6TgHRDNGPY72pA5FSGf/TzuQ21RZ+//uCx1ReJrNsmdvb9SU9tjxFuGRgtekqHta/RBG+WWzdUhxDut0zqHuyz20FVY8QQkFQ+Pyxa9g/rR8NXCwNc2TiHe2Ji/AtZMcVFwnnUcNYvjf1QCeZeE/7Duc2/I/Pth53e8x98zfx1vQXMX8zAo1k5hvTdZZYRXyfH51xT0en5LXumPCPdgztlcI3NolyBaGDEEJBUFCpJGUN7L/7lxf6Geoi6NpaPsCbM7X35Y3c7tOj5WnDU3xu7INakpmp/YC//+91Vuw8bddu/YFcOk3+iV6nFjJB/ggVMouMt/K8YSRGNwutMl9J48vHe9pta5MQQ/P6kXRKLp9GaBjtWkQfuTaVmHAtk++8gqua+W5FCqqOkFhiN3fuXGbOnEl2djadOnXi3XffpXv37m7bL1u2jFdffZVjx47RunVrXn/9df7xj/JVDrIsM2nSJD788EPy8vK49tprmTdvHq1bt66K2xE4MLhHcwb3aE6pwUS4Vk27pBgWbDjG87deTphWrax6ad04hr9yCu2OzRh/Ez2n/wqglBlwhxkV442Pkk8UIzXf84r2M95bVsSRpLm0uLSu96EFW3hZ8xmPaX8E4G3jv3jLeA/uakovG9mTBtFhTiEvq569HkmS+OSRHqw/mEtsuJaeLRuwP7uArLxSyoxmYiM0XN28vlOmb0HoUe3/oS+++IIxY8YwadIk/vjjDzp16kTfvn05c+aMy/abNm1i0KBBDB8+nO3bt3P33Xdz99138+effypt3njjDd555x3mz5/Pli1biIqKom/fvpSWui8oLgg+VgtxQLdm/PTcDdx6RaKdlffBQ1257cpEVoy+lo8euprlT15LUlwEcx+4iqG9UnjixpZO53z3Uu2VciRmGB/gdcNAAEZrlrPxnaGU6g1gMvKG5gMe01hEcKphCG8Z78WVCHZPrc+WCTfTLcUy1xkfqVOG79e2aqA4PCJ0avpekahkjG6bGEufto3pd2UivVo2FCJYQ5BkVxkuq5AePXrQrVs33nvvPQDMZjPJyck89dRTjBs3zqn9gAEDKC4u5vvvv1e2XXPNNXTu3Jn58+cjyzJNmjTh+eefZ+zYsQDk5+eTkJDAokWLGDhwoNc+FRQUEBcXR35+PrGxwfMeCvzn1/05RGg19EitT5nRTIROzdT/28uCjUed2g5W/8I0zUJUksxyUy90GLhNvQ2TLDHO+BjLTDcqbR+8phmLN1vmFG/vmMTcB65yOp8syxw6U0TzBlFC4KqZQH9Hq3VorNfryczMZPz48co2lUpFWloaGRkZLo/JyMhgzJgxdtv69u3Ld999B8DRo0fJzs4mLS1N2R8XF0ePHj3IyMhwKYRlZWWUlZUH4xYUFFTmtgRB5Ka25V5oawD4q3e0Y81fZzh6tphJ/2xPnzaNufWt9XxmSqNAjmS2dh53qTcBUCZreNrwFD+ZuwFwTYv6PNwzhZvbJfDa3R0oNZgIcyNykiTRupJZpQWhSbUK4dmzZzGZTCQk2IdYJCQksH//fpfHZGdnu2yfnZ2t7Lduc9fGkenTpzNlypQK3YOg+pEkidXP3UDeRYOSwDT9+d7c+tZ6Tib9g8dORjBP+zZmJEYYxrDR3IGrmsXzxr2daNEwyq7AlUhyWjcJCWdJdTN+/Hg7K7OgoIDkZBH3VZPQqFV2WZyT60eyb1q/S39dy9nsh1BJKhY3bioCmgVOVKsQNmzYELVaTU6OfVbjnJwcEhNdp4VPTEz02N76Mycnh6SkJLs2nTt3dnnOsLAwwsJE4svaTMPEZtXdBUEIU60zvjqdjq5du5Kenq5sM5vNpKen07NnT5fH9OzZ0649wOrVq5X2qampJCYm2rUpKChgy5Ytbs8pEAjqOHI1s3TpUjksLExetGiRvHfvXnnEiBFyfHy8nJ2dLcuyLA8ZMkQeN26c0n7jxo2yRqORZ82aJe/bt0+eNGmSrNVq5d27dyttZsyYIcfHx8vLly+Xd+3aJd91111yamqqfPHiRZ/6lJ+fLwNyfn5+YG9WIBAEhEB/R6t9jnDAgAHk5uYyceJEsrOz6dy5M6tWrVKcHcePH0dlk6q8V69eLFmyhFdeeYUJEybQunVrvvvuO668snz1wosvvkhxcTEjRowgLy+P6667jlWrVhEeHrykoQKBoOZS7XGEoYiIIxQIQptAf0dFVKhAIKjzCCEUCAR1HiGEAoGgzlPtzpJQxDptKpbaCQShifW7GSgXhxBCFxQWWlJBidUlAkFoU1hYSFxc5cudCq+xC8xmM6dPnyYmJsZpOZZ1+d2JEydqnUdZ3FvNpC7emyzLFBYW0qRJE7vwuooiLEIXqFQqLrvsMo9tYmNja92Hzoq4t5pJXbu3QFiCVoSzRCAQ1HmEEAoEgjqPEEI/CQsLY9KkSbUyW424t5qJuLfKI5wlAoGgziMsQoFAUOcRQigQCOo8QggFAkGdRwihQCCo8wghdOD8+fMMHjyY2NhY4uPjGT58OEVFRR6P+eCDD7jxxhuJjY1FkiTy8vICct5AU5E+lJaW8uSTT9KgQQOio6O55557nGrGSJLk9Fq6dGkwbwWAuXPnkpKSQnh4OD169GDr1q0e2y9btoy2bdsSHh5Ohw4d+PHHH+32y7LMxIkTSUpKIiIigrS0NA4ePBjMW3BLoO9t6NChTv+jfv36uTlb8PDnvvbs2cM999xDSkoKkiQxZ86cSp/TLQHJc12L6Nevn9ypUyd58+bN8m+//Sa3atVKHjRokMdj3nrrLXn69Ony9OnTZUC+cOFCQM4baCrSh5EjR8rJyclyenq6/Pvvv8vXXHON3KtXL7s2gLxw4UI5KytLeflaFqGiLF26VNbpdPKCBQvkPXv2yI899pgcHx8v5+TkuGy/ceNGWa1Wy2+88Ya8d+9e+ZVXXnFZ4iEuLk7+7rvv5J07d8p33nmnXyUeAkUw7u3hhx+W+/XrZ/c/On/+fFXdkizL/t/X1q1b5bFjx8qff/65nJiYKL/11luVPqc7hBDasHfvXhmQt23bpmxbuXKlLEmSfOrUKa/Hr1mzxqUQVva8gaAifcjLy5O1Wq28bNkyZdu+fftkQM7IyFC2AfK3334btL67onv37vKTTz6p/G0ymeQmTZrI06dPd9n+/vvvl2+//Xa7bT169JAff/xxWZZl2Ww2y4mJifLMmTOV/Xl5eXJYWJj8+eefB+EO3BPoe5NlixDeddddQemvr/h7X7Y0b97cpRBW5py2iKGxDRkZGcTHx3P11Vcr29LS0lCpVGzZsiXkzhvsPmRmZmIwGEhLS1O2tW3blmbNmpGRkWHX9sknn6Rhw4Z0796dBQsWBCw9kiv0ej2ZmZl2/VKpVKSlpTn1y0pGRoZde4C+ffsq7Y8ePUp2drZdm7i4OHr06OH2nMEgGPdmZe3atTRu3Jg2bdrwxBNPcO7cucDfgBsqcl9VeU6RdMGG7OxsGjdubLdNo9FQv359srOzQ+68we5DdnY2Op2O+Ph4u+0JCQl2x0ydOpWbbrqJyMhIfv75Z0aNGkVRURFPP/10wO8D4OzZs5hMJqXAl22/9u/f7/KY7Oxsl+2t92H96alNVRCMewPo168f//rXv0hNTeXw4cNMmDCB2267jYyMDNRqdeBvxIGK3FdVnrNOCOG4ceN4/fXXPbbZt29fFfUmsITCvb366qvK7126dKG4uJiZM2cGTQgF/jNw4EDl9w4dOtCxY0datmzJ2rVrufnmm6uxZ6FBnRDC559/nqFDh3ps06JFCxITEzlz5ozddqPRyPnz50lMTKzw9YN1XgjuvSUmJqLX68nLy7OzCnNycjz2u0ePHkybNo2ysrKgrBFt2LAharXayXvtqV+JiYke21t/5uTkkJSUZNemc+fOAey9Z4Jxb65o0aIFDRs25NChQ1UihBW5r6o8Z52YI2zUqBFt27b1+NLpdPTs2ZO8vDwyMzOVY3/99VfMZjM9evSo8PWDdd5g31vXrl3RarWkp6cr2/766y+OHz9Oz5493fZpx44d1KtXL2gL5XU6HV27drXrl9lsJj093W2/evbsadceYPXq1Ur71NRUEhMT7doUFBSwZcsWj/caaIJxb644efIk586dsxP9YFKR+6rSc/rlWqkD9OvXT+7SpYu8ZcsWecOGDXLr1q3tQkxOnjwpt2nTRt6yZYuyLSsrS96+fbv84YcfyoC8fv16efv27fK5c+d8Pm+o3tvIkSPlZs2ayb/++qv8+++/yz179pR79uyp7F+xYoX84Ycfyrt375YPHjwov//++3JkZKQ8ceLEoN7L0qVL5bCwMHnRokXy3r175REjRsjx8fFydna2LMuyPGTIEHncuHFK+40bN8oajUaeNWuWvG/fPnnSpEkuw2fi4+Pl5cuXy7t27ZLvuuuuagufCeS9FRYWymPHjpUzMjLko0ePyr/88ot81VVXya1bt5ZLS0tD9r7Kysrk7du3y9u3b5eTkpLksWPHytu3b5cPHjzo8zl9RQihA+fOnZMHDRokR0dHy7GxsfKwYcPkwsJCZf/Ro0dlQF6zZo2ybdKkSTLg9Fq4cKHP560KKnJvFy9elEeNGiXXq1dPjoyMlPv37y9nZWUp+1euXCl37txZjo6OlqOiouROnTrJ8+fPl00mU9Dv591335WbNWsm63Q6uXv37vLmzZuVfb1795Yffvhhu/ZffvmlfPnll8s6nU6+4oor5B9++MFuv9lsll999VU5ISFBDgsLk2+++Wb5r7/+Cvp9uCKQ91ZSUiLfeuutcqNGjWStVis3b95cfuyxx/wWi0Dgz31ZP4+Or969e/t8Tl8RabgEAkGdp07MEQoEAoEnhBAKBII6jxBCgUBQ5xFCKBAI6jxCCAUCQZ1HCKFAIKjzCCEUCAR1HiGEAoGgziOEUFBjcJVuvrpSzgtqF3Ui+4yg9tCvXz8WLlxot81dcgeDwYBWq7Xbptfr0el0fl+3oscJagbCIhTUKMLCwkhMTLR71atXD7AUkZo3bx533nknUVFR/Pvf/2by5Ml07tyZjz76iNTUVMLDwwE4fvw4d911F9HR0cTGxnL//ffbpXNyd5ygdiKEUFCrmDx5Mv3792f37t088sgjABw6dIivv/6ab775hh07dmA2m7nrrrs4f/4869atY/Xq1Rw5coQBAwbYncvxOEHtRQyNBTWK77//nujoaLttEyZMYMKECQA88MADDBs2zG6/Xq/nk08+oVGjRoAlV9/u3bs5evQoycnJAHzyySdcccUVbNu2jW7durk8TlB7EUIoqFH06dOHefPm2W2rX7++8rttcSorzZs3txOzffv2kZycrIggQPv27YmPj2ffvn2KEDoeJ6i9CCEU1CiioqJo1aqVx/2+bPP1WoK6gZgjFNQ52rVrx4kTJzhx4oSybe/eveTl5dG+fftq7JmguhAWoaBGUVZW5lReU6PR0LBhQ5/PkZaWRocOHRg8eDBz5szBaDQyatQoevfu7XJoLaj9CItQUKNYtWoVSUlJdq/rrrvOr3NIksTy5cupV68eN9xwA2lpabRo0YIvvvgiSL0WhDoiVb9AIKjzCItQIBDUeYQQCgSCOo8QQoFAUOcRQigQCOo8QggFAkGdRwihQCCo8wghFAgEdR4hhAKBoM4jhFAgENR5hBAKBII6jxBCgUBQ5xFCKBAI6jz/D8GJmdE3Q+jiAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#\"\"\"\n", + "from ml_utility_loss.loss_learning.visualization import plot_grad, plot_grad_2, plot_grad_3\n", + "import matplotlib.pyplot as plt\n", + "\n", + "#plot_grad_2(y, model.models)\n", + "for m in model.models:\n", + " ym = y[m]\n", + " fig, ax = plt.subplots()\n", + " plot_grad_3(ym[\"error\"], ym[\"grad\"], name=f\"{m}_grad\", fig=fig, ax=ax)\n", + "#\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54c0e9f3", + "metadata": { + "papermill": { + "duration": 0.02314, + "end_time": "2024-02-29T23:33:42.014235", + "exception": false, + "start_time": "2024-02-29T23:33:41.991095", + "status": "completed" + }, + "tags": [] + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "celltoolbar": "Tags", + "colab": { + "authorship_tag": "ABX9TyOOVfelovKP9fLGU7SvvRie", + "gpuType": "T4", + "mount_file_id": "17POSGAvge8y9DW9WGs2jLkibaRjToayg", + "provenance": [] + }, + "kaggle": { + "accelerator": "gpu", + "dataSources": [], + "dockerImageVersionId": 30648, + "isGpuEnabled": true, + "isInternetEnabled": true, + "language": "python", + "sourceType": "notebook" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "papermill": { + "default_parameters": {}, + "duration": 4189.757421, + "end_time": "2024-02-29T23:33:44.759053", + "environment_variables": {}, + "exception": null, + "input_path": "eval/contraceptive/tvae/2/mlu-eval.ipynb", + "output_path": "eval/contraceptive/tvae/2/mlu-eval.ipynb", + "parameters": { + "dataset": "contraceptive", + "dataset_name": "contraceptive", + "debug": false, + "folder": "eval", + "gp": false, + "gp_multiply": false, + "param_index": 2, + "path": "eval/contraceptive/tvae/2", + "path_prefix": "../../../../", + "random_seed": 2, + "single_model": "tvae" + }, + "start_time": "2024-02-29T22:23:55.001632", + "version": "2.5.0" + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file