File size: 88,739 Bytes
f50f696 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 |
{
"cells": [
{
"cell_type": "code",
"execution_count": 56,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import numpy as np\n",
"\n",
"import priors\n",
"from train import train, get_weighted_single_eval_pos_sampler\n",
"import encoders\n",
"import positional_encodings\n",
"import utils\n",
"import bar_distribution\n",
"import decoders\n",
"from datasets import *\n",
"import os\n",
"\n",
"from tqdm import tqdm\n",
"import time\n",
"\n",
"import torch\n",
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"import torch.nn as nn\n",
"import os.path\n",
"import glob\n",
"\n",
"from mcmc_svi_transformer_on_bayesian import get_model, get_default_model_spec, generate_toy_data, load_results, plot_with_confidence_intervals, training_steps, training_samples, get_default_evaluation_points, compute_mean_and_conf_interval, eval_transformer\n"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# %load_ext autoreload\n",
"\n",
"# %autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"## DEFINE A PRIOR MODEL ##\n",
"# We define a Bayesian Model as a prior for all methods\n",
"# This can be replaced by other models that inherit from PyroModule.\n",
"class BayesianModel(PyroModule):\n",
" def __init__(self, model_spec, device='cuda'):\n",
" super().__init__()\n",
"\n",
" self.device = device\n",
" self.num_features = model_spec['num_features']\n",
"\n",
" mu, sigma = torch.tensor([0.0]).to(self.device), torch.tensor([1.0]).to(self.device)\n",
"\n",
" self.fc1 = PyroModule[nn.Linear](self.num_features, model_spec['embed'])\n",
" self.fc1.weight = PyroSample(\n",
" dist.Normal(mu, sigma).expand([model_spec['embed'], self.num_features]).to_event(2))\n",
" self.fc1.bias = PyroSample(dist.Normal(mu, sigma).expand([model_spec['embed']]).to_event(1))\n",
"\n",
" self.fc2 = PyroModule[nn.Linear](model_spec['embed'], 2)\n",
" self.fc2.weight = PyroSample(dist.Normal(mu, sigma).expand([2, model_spec['embed']]).to_event(2))\n",
" self.fc2.bias = PyroSample(dist.Normal(mu, sigma).expand([2]).to_event(1))\n",
"\n",
" self.model = torch.nn.Sequential(self.fc1, self.fc2)\n",
"\n",
" self.to(self.device)\n",
"\n",
" def forward(self, x=None, y=None, seq_len=1):\n",
" if x is None:\n",
" with pyro.plate(\"x_plate\", seq_len):\n",
" d_ = dist.Normal(torch.tensor([0.0]).to(self.device), torch.tensor([1.0]).to(self.device)).expand(\n",
" [self.num_features]).to_event(1)\n",
" x = pyro.sample(\"x\", d_)\n",
"\n",
" out = self.model(x)\n",
" mu = out.squeeze()\n",
" softmax = torch.nn.Softmax(dim=1)\n",
" with pyro.plate(\"data\", out.shape[0]):\n",
" s = softmax(mu)\n",
" obs = pyro.sample('obs', dist.Categorical(probs=s), obs=y).float()\n",
"\n",
" return x, obs"
]
},
{
"cell_type": "code",
"execution_count": 69,
"metadata": {},
"outputs": [],
"source": [
"results_directory = 'results' # Where to save results\n",
"model_spec_size = 'small' # Size of the BNN model to evaluate, also try big\n",
"bptt = 100 # Number of samples in each dataset\n",
"\n",
"# Training samples seen after which to evaluate the methods\n",
"evaluation_points = [2, 7, 12, 17, 22, 27, 32, 37, 42, 47, 52, 57, 62, 67, 72, 77, 82, 87, 92]\n",
"\n",
"# Function which generates a model from the prior\n",
"model_sampler = lambda : BayesianModel(get_default_model_spec(model_spec_size), device = device)\n",
"\n",
"global_results = {} # Dict in which to save results\n",
"task = 'samples' # Task to evaluate, only option is samples, keep fixed"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"!mkdir {results_directory}"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"### Evaluate SVI and MCMC"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"method = 'svi'\n",
"steps = 1\n",
"device = 'cuda'\n",
"path_interfix = f'{results_directory}/timing_{model_spec_size}_model_test'"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"!mkdir {path_interfix}"
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {
"hidden": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 100/100 [00:02<00:00, 37.13it/s]\n",
"/home/hollmann/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/numpy/lib/npyio.py:528: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
" arr = np.asanyarray(arr)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"NLL (51.540817, 1.832436208065078)\n",
"ACC (0.48459178, 0.01832436154844232)\n",
"TIME 2.6950523853302\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 100/100 [00:02<00:00, 35.89it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"NLL (48.569893, 1.8696300575034437)\n",
"ACC (0.51430106, 0.01869630134377999)\n",
"TIME 2.788970708847046\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 100/100 [00:03<00:00, 31.80it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"NLL (51.034092, 1.807273770560027)\n",
"ACC (0.48965907, 0.018072737823868815)\n",
"TIME 3.1472866535186768\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 100/100 [00:02<00:00, 38.48it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"NLL (50.216866, 2.0121896389094833)\n",
"ACC (0.4978313, 0.02012189562034928)\n",
"TIME 2.600956439971924\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 55%|█████▌ | 55/100 [00:01<00:01, 38.41it/s]\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipykernel_9449/1948451174.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_toy_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbptt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m training_samples(method\n\u001b[0m\u001b[1;32m 5\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/prior-fitting/mcmc_svi_transformer_on_bayesian.py\u001b[0m in \u001b[0;36mtraining_samples\u001b[0;34m(method, X, y, model_spec, evaluation_points, steps, device, path_interfix, overwrite)\u001b[0m\n\u001b[1;32m 379\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 380\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'svi'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 381\u001b[0;31m \u001b[0mnll\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval_svi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_spec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_samples_n\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_train_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_train_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_pred_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_pred_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 382\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'svgd'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 383\u001b[0m \u001b[0mnll\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval_svi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_spec\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining_samples_n\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_train_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_train_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_pred_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_pred_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msvgd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/prior-fitting/mcmc_svi_transformer_on_bayesian.py\u001b[0m in \u001b[0;36meval_svi\u001b[0;34m(X, y, device, model_sampler, training_samples_n, num_train_steps, num_pred_samples, lr, num_particles, svgd)\u001b[0m\n\u001b[1;32m 281\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 282\u001b[0m \u001b[0mpredictive\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPredictive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mguide\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mguide\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnum_pred_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 283\u001b[0;31m \u001b[0mpreds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpredictive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_test_sample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 284\u001b[0m \u001b[0macc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnll\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate_preds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test_sample\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0mnll_list\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnll\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/infer/predictive.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 271\u001b[0m \u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 272\u001b[0m )\n\u001b[0;32m--> 273\u001b[0;31m return _predictive(\n\u001b[0m\u001b[1;32m 274\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 275\u001b[0m \u001b[0mposterior_samples\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/infer/predictive.py\u001b[0m in \u001b[0;36m_predictive\u001b[0;34m(model, posterior_samples, num_samples, return_sites, return_trace, parallel, model_args, model_kwargs)\u001b[0m\n\u001b[1;32m 125\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 126\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mparallel\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m return _predictive_sequential(\n\u001b[0m\u001b[1;32m 128\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0mposterior_samples\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/infer/predictive.py\u001b[0m in \u001b[0;36m_predictive_sequential\u001b[0;34m(model, posterior_samples, model_args, model_kwargs, num_samples, return_site_shapes, return_trace)\u001b[0m\n\u001b[1;32m 46\u001b[0m ]\n\u001b[1;32m 47\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m trace = poutine.trace(poutine.condition(model, samples[i])).get_trace(\n\u001b[0m\u001b[1;32m 49\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmodel_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 50\u001b[0m )\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/poutine/trace_messenger.py\u001b[0m in \u001b[0;36mget_trace\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[0mCalls\u001b[0m \u001b[0mthis\u001b[0m \u001b[0mpoutine\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mreturns\u001b[0m \u001b[0mits\u001b[0m \u001b[0mtrace\u001b[0m \u001b[0minstead\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mfunction\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0ms\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 197\u001b[0m \"\"\"\n\u001b[0;32m--> 198\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 199\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmsngr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_trace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/poutine/trace_messenger.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 172\u001b[0m )\n\u001b[1;32m 173\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 174\u001b[0;31m \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 175\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mValueError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 176\u001b[0m \u001b[0mexc_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexc_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraceback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/poutine/messenger.py\u001b[0m in \u001b[0;36m_context_wrap\u001b[0;34m(context, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_context_wrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 29\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/poutine/messenger.py\u001b[0m in \u001b[0;36m_context_wrap\u001b[0;34m(context, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_context_wrap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/nn/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 424\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 425\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pyro_context\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 426\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/tmp/ipykernel_9449/3309204952.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x, y, seq_len)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"x\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md_\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 30\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 31\u001b[0m \u001b[0mmu\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0msoftmax\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 137\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 140\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 141\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/nn/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 424\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 425\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pyro_context\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 426\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m 1049\u001b[0m if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m 1050\u001b[0m or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1051\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1052\u001b[0m \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1053\u001b[0m \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/nn/module.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 477\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprior\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"sample\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# if not a distribution\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 478\u001b[0m \u001b[0mprior\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprior\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 479\u001b[0;31m \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyro\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfullname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprior\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 480\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfullname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 481\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/primitives.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(name, fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m 162\u001b[0m }\n\u001b[1;32m 163\u001b[0m \u001b[0;31m# apply the stack and return its return value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 164\u001b[0;31m \u001b[0mapply_stack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 165\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"value\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/poutine/runtime.py\u001b[0m in \u001b[0;36mapply_stack\u001b[0;34m(initial_msg)\u001b[0m\n\u001b[1;32m 210\u001b[0m \u001b[0mpointer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpointer\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 212\u001b[0;31m \u001b[0mframe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 213\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmsg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"stop\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/pyro/poutine/messenger.py\u001b[0m in \u001b[0;36m_process_message\u001b[0;34m(self, msg)\u001b[0m\n\u001b[1;32m 139\u001b[0m \u001b[0mon\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mThe\u001b[0m \u001b[0mmessage\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mupdated\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mplace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 140\u001b[0m \"\"\"\n\u001b[0;32m--> 141\u001b[0;31m \u001b[0mmethod\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"_pyro_{}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"type\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 142\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mmethod\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 143\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"X, y = generate_toy_data(test_model, bptt, device)\n",
"\n",
"training_samples(method\n",
" , X\n",
" , y\n",
" , model_sampler\n",
" , evaluation_points\n",
" , steps=steps\n",
" , device=device\n",
" , path_interfix=path_interfix)"
]
},
{
"cell_type": "markdown",
"metadata": {
"heading_collapsed": true
},
"source": [
"### Training Transformer on Prior (Skip this step to reuse results)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"device = 'cuda'"
]
},
{
"cell_type": "code",
"execution_count": 49,
"metadata": {
"hidden": true
},
"outputs": [],
"source": [
"config = {'lr': 2.006434218345026e-05\n",
" , 'epochs': 160\n",
" , 'dropout': 0.0\n",
" , 'emsize': 256\n",
" , 'batch_size': 256\n",
" , 'nlayers': 5\n",
" , 'num_outputs': 1\n",
" , 'num_features': model_spec['num_features']\n",
" , 'steps_per_epoch': 100\n",
" , 'nhead': 4\n",
" , 'dropout': 0.0\n",
" , 'seq_len': model_spec['seq_len']\n",
" , 'nhid_factor': 2}"
]
},
{
"cell_type": "code",
"execution_count": 51,
"metadata": {
"hidden": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using cuda device\n",
"DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 256, 'seq_len': 300, 'num_outputs': 1, 'num_features': 3, 'canonical_args': None, 'model': <function <lambda> at 0x7f6f42f49f70>}, 'num_features': 3, 'num_outputs': 1}\n"
]
},
{
"ename": "KeyboardInterrupt",
"evalue": "",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m/tmp/ipykernel_9449/1283571267.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtransformer_model\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_sampler\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshould_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0mmodel_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults_directory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'bayesian_models_transformer_checkpoint_{model_spec_size}_epochs_'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'epochs'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'.cpkt'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtransformer_model\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/prior-fitting/mcmc_svi_transformer_on_bayesian.py\u001b[0m in \u001b[0;36mget_model\u001b[0;34m(model_generator, config, should_train, device)\u001b[0m\n\u001b[1;32m 129\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mshould_train\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'epochs'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 131\u001b[0;31m model = train(priors.pyro.DataLoader\n\u001b[0m\u001b[1;32m 132\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mLosses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbce\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 133\u001b[0m \u001b[0;34m,\u001b[0m \u001b[0mencoders\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLinear\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/prior-fitting/train.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(priordataloader_class, criterion, encoder_generator, emsize, nhid, nlayers, nhead, dropout, epochs, steps_per_epoch, batch_size, bptt, lr, warmup_epochs, input_normalization, y_encoder_generator, pos_encoder_generator, decoder, extra_prior_kwargs_dict, scheduler, load_weights_from_this_state_dict, validation_period, single_eval_pos_gen, gpu_device, aggregate_k_gradients, verbose)\u001b[0m\n\u001b[1;32m 116\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 117\u001b[0m \u001b[0mepoch_start_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 118\u001b[0;31m \u001b[0mtotal_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal_positional_losses\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtime_to_get_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mforward_time\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 119\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'validate'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mvalidation_period\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/prior-fitting/train.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m 93\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0maggregate_k_gradients\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0maggregate_k_gradients\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 95\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 96\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 97\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m~/miniconda3/envs/prior-fitting/lib/python3.9/site-packages/torch/nn/utils/clip_grad.py\u001b[0m in \u001b[0;36mclip_grad_norm_\u001b[0;34m(parameters, max_norm, norm_type, error_if_nonfinite)\u001b[0m\n\u001b[1;32m 41\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0mtotal_norm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnorm_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mp\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnorm_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 43\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mtotal_norm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnan\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mtotal_norm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misinf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 44\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0merror_if_nonfinite\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 45\u001b[0m raise RuntimeError(\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
],
"source": [
"transformer_model = get_model(model_sampler, config, should_train = True)\n",
"model_path = os.path.join(results_directory, f'bayesian_models_transformer_checkpoint_{model_spec_size}_epochs_'+config['epochs']+'.cpkt')\n",
"torch.save((transformer_model[2].state_dict(), None), model_path)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Evaluating Transformer"
]
},
{
"cell_type": "code",
"execution_count": 52,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Using cuda device\n",
"DataLoader.__dict__ {'num_steps': 100, 'fuse_x_y': False, 'get_batch_kwargs': {'batch_size': 256, 'seq_len': 300, 'num_outputs': 1, 'num_features': 3, 'canonical_args': None, 'model': <function <lambda> at 0x7f6f42f49f70>}, 'num_features': 3, 'num_outputs': 1}\n"
]
},
{
"data": {
"text/plain": [
"<All keys matched successfully>"
]
},
"execution_count": 52,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"loaded_epoch = config['epochs']\n",
"transformer_model = get_model(model_sampler, config, should_train = False)\n",
"path = os.path.join(results_directory, F'bayesian_models_transformer_checkpoint_{model_spec_size}_epochs_{loaded_epoch}.cpkt')\n",
"model_state, optimizer_state = torch.load(path)\n",
"transformer_model[2].load_state_dict(model_state)"
]
},
{
"cell_type": "code",
"execution_count": 57,
"metadata": {},
"outputs": [],
"source": [
"X, y = generate_toy_data(test_model, bptt, device)"
]
},
{
"cell_type": "code",
"execution_count": 73,
"metadata": {},
"outputs": [],
"source": [
"results_acc = []\n",
"results_nll = []\n",
"transformer_model[2].eval()\n",
"for training_samples_n in evaluation_points:\n",
" acc, nll, elapsed = eval_transformer(X, y, model=transformer_model[2], training_samples_n=training_samples_n, device=device)\n",
" results_acc.append(acc)\n",
" results_nll.append(nll)\n",
"mean = np.array([compute_mean_and_conf_interval(nll)[0] for nll in results_nll])\n",
"conf = np.array([compute_mean_and_conf_interval(nll)[1] for nll in results_nll])\n",
"\n",
"global_results['transformer'] = (None, np.array(evaluation_points), mean, conf)\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Plotting results"
]
},
{
"cell_type": "code",
"execution_count": 71,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
]
}
],
"source": [
"files, times, samples, mean, conf = load_results(f'{results_directory}/timing_{model_size}_model/results_svi_training_{task}', task=task)\n",
"global_results['svi'] = (times/100, samples, mean, conf)\n",
"files, times, samples, mean, conf = load_results(f'{results_directory}/timing_{model_size}_model/results_mcmc_training_{task}', task=task)\n",
"global_results['mcmc'] = (times/100, samples,mean, conf)\n"
]
},
{
"cell_type": "code",
"execution_count": 74,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<matplotlib.legend.Legend at 0x7f6f1c2ba7c0>"
]
},
"execution_count": 74,
"metadata": {},
"output_type": "execute_result"
},
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEoCAYAAAAub0k8AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABkaUlEQVR4nO3dd5xcV33//9e5ZdrubFXfVe9dtiwXDLaMwYWOIfQaCCGE/L4hgQAB0wkQQgKh+QsJhFACGPgGgw0GG1dwkWTLlizJ6mVX0vY69Zbz++POzM5WrVaz2vZ5Phhm5s6dmbvj1d73nPM55yitNUIIIYSYWYyJPgAhhBBCXHwSAIQQQogZSAKAEEIIMQNJABBCCCFmIAkAQgghxAwkAUAIIYSYgayJeuNZs2bpJUuWTNTbCyGEEFParl27WrXWs8f6/AkLAEuWLGHnzp0T9fZCCCHElKaUOnEhz5cuACGEEGIGkgAghBBCzEASAIQQQogZaMJqAIQQQkwdjuPQ0NBAOp2e6EOZcSKRCPX19di2XdLXlQAghBDinBoaGojH4yxZsgSl1EQfzoyhtaatrY2GhgaWLl1a0teWLgAhhBDnlE6nqa2tlZP/RaaUora2dlxaXiQACCGEGBU5+U+M8frcJQAIIYSYdl70ohfR2dk50YcxqUkNgBBCiGnnrrvumuhDmPSkBUAIIcSUkEgkePGLX8zmzZvZsGED3/ve93jNa15TePz+++/npS99KRDMNtva2jpRhzolTFgLgJ6oNxZCCHFBPvmrZ9h3urukr7luQQUff+n6Eff57W9/y4IFC7jzzjsB6Orq4tZbbyWRSFBWVsZPfvITXvva15b0uKazCWsB0FoigBBCiNHbuHEj99xzDx/84Ad56KGHqKys5KabbuJXv/oVruty55138vKXv3yiD3PKmLAWAF8CgBBCTEnn+qY+XlatWsWuXbu46667+PCHP8wNN9zAa1/7Wr7+9a9TU1PDtm3biMfjE3JsU9GEtgC4nj9Rby+EEGKKOX36NLFYjDe96U28//3v54knnmD79u088cQTfPvb35bm//M0gTUAmqznY5lShyiEEOLc9uzZwwc+8AEMw8C2bb75zW9imiYveclL+K//+i++973vTfQhTilqovriN27ZrB/84+NUl4Un5P2FEEKM3v79+1m7du1EH8aMNdTnr5TapbW+bKyvOaFfv5NOdiLfXgghhJixJjQAOL5D1pU6ACGEEOJim/gAIIWAQgghxEU3oQHAlRYAIYQQYkJMaADwtSbjZGVSICGEEOIim/AxeFnflW4AIYQQ4iKb8AAg3QBCCCHExTdhAcD1gmZ/R0sAEEIIIS62CQsAx9vS+Frj+R5Zz8P3pQ5ACCHE8I4fP86aNWt45zvfyYYNG3jjG9/IPffcw9VXX83KlSt5/PHH6e3t5e1vfzsbN25k06ZN/PznPwegvLycD37wg2zdupUXvOAFPP7442zfvp1ly5Zxxx13AOB5Hu9///sLz/3qV786kT/uuJvQxYCONqdYMTcWdAN4YSKGOVGHI4QQYrR+8yE4u6e0rzlvI9z8+XPudvjwYW6//Xa+9a1vsW3bNn70ox/x8MMPc8cdd/BP//RPrF69msrKSvbsCY6vo6MDgEQiwfbt2/nCF77AK1/5Sj760Y/y+9//nn379vHWt76Vl73sZXzrW9/i2LFjPPnkk1iWRXt7e2l/xklmwgKAMhM8dao3FwBcMq5PxJYAIIQQYnhLly5l48aNAKxfv57rr78epRQbN27k+PHjnDp1ih//+MeF/aurqwEIhULcdNNNQLCscDgcxrbtwvMA7rnnHt797ndjWcGpsaam5iL+ZBffxAUAu5MnTnbwqsvmyIyAQggxlYzim/p4CYf71o8xDKNw3zAMXNfFMAyUUoOeZ9t2YftQz4NgldqhnjtdTeAoAM2BtoNkXR9Xu7l6AKkDEEIIMXY33HADX/va1wr3810Ao33ubbfdVggE070LYEKHAXqhE+w/k0Rrjeu70goghBDignz0ox+lo6ODDRs2sHnzZu67775RP/ed73wnixYtYtOmTWzevJkf/ehH43ikE2/ClgOuWF6h6973Il664O9569XzKLPLqY6UUxmzJ+R4hBBCDE+WA55Y02o54IgZJVR2iidPdgPBhEAZz5uowxFCCCFmlAkMAGF8o5uj7afpSbu4voPW4Mi0wEIIIcS4m8AAEAFARU6ypyGBp3087UkdgBBCCHERjCoAKKVuUko9q5Q6rJT60DD7bFdK7VZKPaOUeuBcrxk2bMJmhFDZKXaf7AVkXQAhhBDiYjnnPABKKRP4OvBCoAHYoZS6Q2u9r2ifKuAbwE1a65NKqTmjefPlFSs44pziqVwAcHwXx/Nn3FhMIYQQ4mIbTQvA5cBhrfVRrXUW+DHw8gH7vAH4hdb6JIDWunk0b74qvpys0UhDZw8tPdmgDgBkeWAhhBBinI0mANQBp4ruN+S2FVsFVCul7ldK7VJKvWU0b746vgyNhxlpYPfJXlzfRWuN48mEQEIIIfrr7OzkG9/4xri/z+tf/3o2bdrEv/3bv437e02k0QSAodriB56hLWAr8GLgRuBWpdSqQS+k1LuUUjuVUjvb2ztYGV8GQFllQ6EOQKYFFkIIMZThAoBXwiHkZ8+e5U9/+hNPP/0073vf+0b1nPzMgaVSyp9nJKMJAA3AwqL79cDpIfb5rdY6obVuBR4ENg98Ia31t7TWl2mtL6upqaHCjjM/Op+KqiAABDMCOoU6ACGEECLvQx/6EEeOHGHLli1s27aN6667jje84Q2FxYFe8YpXsHXrVtavX8+3vvWtwvPKy8v5yEc+wubNm7nyyitpamoC4Pbbby/MGHjNNdcAwXTAzc3NbNmyhYceeojdu3dz5ZVXsmnTJl75ylcWphbevn07//iP/8i1117LV77yFbZv38773vc+rrnmGtauXcuOHTu45ZZbWLlyJR/96EcLx/KDH/yAyy+/nC1btvCXf/mXhZN9eXk5H/vYx7jiiit45JFHLsrnOZrFgHYAK5VSS4FG4HUEff7Ffgl8TSllASHgCmDEtpN8jd+qiuU8lnmazqTDibYMK+eEAGR1QCGEmKS+8PgXONB+oKSvuaZmDR+8/IMj7vP5z3+evXv3snv3bu6//35e/OIXs3fvXpYuXQrAd77zHWpqakilUmzbto1XvepV1NbWkkgkuPLKK/nsZz/LP/zDP/Dtb3+bj370o3zqU5/i7rvvpq6ujs7OTgDuuOMOXvKSl7B7924ANm3axFe/+lWuvfZaPvaxj/HJT36SL3/5y0DQIvHAA8Ggt1/96leEQiEefPBBvvKVr/Dyl7+cXbt2UVNTw/Lly3nf+95Hc3MzP/nJT/jjH/+Ibdu85z3v4Yc//CFvectbSCQSbNiwgU996lMl/VxHcs4WAK21C7wXuBvYD/xUa/2MUurdSql35/bZD/wWeBp4HPgPrfXekV63EADiy0n73Si7nd0ne/B00JQihYBCCCFGcvnllxdO/gD//u//XviWf+rUKQ4dOgQESwG/5CUvAWDr1q2F5X+vvvpq3va2t/Htb397yGb3rq4uOjs7ufbaawF461vfyoMPPlh4/LWvfW2//V/2spcBwXLD69evZ/78+YTDYZYtW8apU6e499572bVrF9u2bWPLli3ce++9HD16FADTNHnVq15Vok9mdEa1HLDW+i7grgHbbhtw/4vAF0f7xgqFAlZVrABg1qxGdp9cwisunY3nu2RdGQYohBCT0bm+qV8sZWVlhdv3338/99xzD4888gixWIzt27eTTqeB/ksBm6ZZ6LO/7bbbeOyxx7jzzjvZsmVL4Vv/WN4f6LfE8MBli103KHJ/61vfyuc+97lBrxWJRDDNi9vqPWEzASo0lmmwMFZH1IxSXXOavQ0JXE/j+C6eL8sDCyGE6BOPx+np6Rnysa6uLqqrq4nFYhw4cIBHH330nK935MgRrrjiCj71qU8xa9YsTp061e/xyspKqqureeihhwD4/ve/X2gNGIvrr7+en/3sZzQ3ByPl29vbOXHixJhf70KNqgVgPJhtR7ANcJTBivgyziSPkXJ8Dp5NcumiKBAh6/pEQ1IHIIQQAmpra7n66qvZsGED0WiUuXPnFh676aabuO2229i0aROrV6/myiuvPOfrfeADH+DQoUNorbn++uvZvHnzoBPy9773Pd797neTTCZZtmwZ3/3ud8d8/OvWreMzn/kMN9xwA77vY9s2X//611m8ePGYX/NCTNhywJctMPWdv7ud9thifnL8F/zi1K/oefYTvOHyRbz56gVUh2uI2CaVUVkeWAghJposBzyxptVywABlDY+hVFAHoNEsmt8cFAL6Hr72ZT4AIYQQYpxMXACwo4RPPIJtGKyMLwegdtZpDpxNksx6uL6DrzWujAYQQgghSm7iAkA4jnV2DyGvl3K7jLroAnz7GJ4PexsSOL4DyHBAIYQQYjxMXAAIxVHap6xxJxBMCNSUOUzIpLAuACDdAEIIMUnIDK0TY7w+9wkMAFF0pIroqUcxlGJ1xQp63V5W1CeCAKCDMZPSAiCEEBMvEonQ1tYmIeAi01rT1tZGJBIp+WtP2DBAUKglzyN8/CGsqzWr4sGEQHNmN3L/iTLaerNUhFxsbZN1fULWhNYrCiHEjFZfX09DQwMtLS0TfSgzTiQSob6+vuSvO4EBAFh6DcaBX1HecZAF8RWUWTFU+BiwiqdO9rKwqgrbsMl6EgCEEGIi2bbdb9pdMfVN7Fl1yXPRyqDs1GMYuQmBmtKHiEdMnjrVWygEdKQOQAghhCipiQ0A0Wr0vE2ETz6CaShWx1fSkGxk/UKD3Sd7cbwsgCwPLIQQQpTYhLerq6XXYrccIJztYFXFcjSaBbPP0NLj0NCRwdMeGhkOKIQQQpTSxAeAZcHCCvHTO1gRX45CYUaOAPDkyR7c/HwA0g0ghBBClMyEBwBmr8Uvm0Xs1KOUWVEWxuo4kz7MnAo76AaQ+QCEEEKIkpvAAKByVwp/yfMIndqBqXxWVazgUPdhNtdHePpULxk3qANwfY0vywMLIYQQJTFxAUCpvptLr8XI9lLe+gyrKpaT9FIsmdtOIuNz4GxPoQBQ6gCEEEKI0pjAAND31ubi56INi/KGxwoTAtmRowC5boCgDiAj3QBCCCFESUxsDYBpB9fhcrwFW4ieepQF0XmUW2U0pA6zdFaE3VIIKIQQQpTcBAeAUOGmv+R5WO1HiaVbWVWxgoPdR9iyMMK+00l6MplgH63xpA5ACCGEuGATGwCscOGmWrYdgPjpx1gVX05j6jRr5mZxPc3TjV2F/aQVQAghhLhwE98CkCsGtGpW4sUXEGt4jFUVKwEIR45hGYonT/TI8sBCCCFECU1sAFCq0A2gDANnyXMIN+5iVVk9CsXx5BHWzo/0LwT0vIk8YiGEEGJamPiJgKy+NY79Jdeg3DSz255lcdlCDvYcZnNdmKPNKdp6kwBoHawNIIQQQoixmwQBoK8OwFh0BdoMBXUAFSs41HOULfMMNLDrZGdhP+kGEEIIIS7MxAcAwwTDAsCyy8nWbSV68lFWxZeT9tJEy84SDRk8caIbTwfN/xIAhBBCiAsz8QEACq0AlmHiLHkOVncjG81yAI70HmFzXZTdJ3sL8wHI8sBCCCHEhZlUAQDAX/JcAFa2HqLCjvNs92G21Nmc7cpysr0XAA04ngQAIYQQYqwmRwAoGg5oVi7GrV5CvPHxXB3AYS6ZH3QR7DreWXiKrAsghBBCjN3kCABFwwFtwyKz6CrCZ3azOraYM6kmKsuS1JaZ7DrZ1bcwkNQBCCGEEGM2OQIAFLoBbMMmu/gqlO9yiRMU/R3qOcKW+mBdgKzUAQghhBAXbBIFgGA+AKUU/oIt+HaMS9uOYSqTgz1HuGSBTXfK49mzfdMCy+qAQgghxNhMngBQNBzQtmJkF15OTeMOlpQt5FD3YbYsCFYO3HG8vfAUqQMQQgghxmbyBAAAq68OILvoKsxEC2tDszncc5SqGCysttl1XBYGEkIIIS7UJAsAQTeAbdhkF10JwCWZDBk/y4nEKS6pC7G3sZdUNguA58vywEIIIcRYTK4AkBsOaBkmlM/GmbWaba3HADjYfZhLFthkXM3uho7CU2RdACGEEOL8Ta4AUDQc0FIW2cVXsrhpP9V2BQe7j7BxQQhD9a8DkEJAIYQQ4vxNrgAARcMBLTKLnoOhfdZZVRzsOUxZyGD1nBC7TnQWdpc6ACGEEOL8Tb4AYObXBbBw56zFj1RySTpFc7qFzmwXW+pCHDybpDsV1AH4WuNKN4AQQghxXiZhALDAsLANCwyT7MIruKylqA6gLoSvYceJ1sJTZDigEEIIcX4mXwAAsEIYysA0DDKLrmJDbzsWBgd7jrBmjk3YUjx+rGg+AOkGEEIIIc7L5AwA+W4AZZFddAUhDFaaZRzsPoxtKjbOD7HrRNF8ANICIIQQQpyXyRkArDAohW1Y6Eglztz1XJJKcqT3GK7vckldiFPtaZq60wBoLa0AQgghxPmYnAEgNxzQNoLpf7OLruLSjrM4vsPxxEm21AVDBR871lcHIPMBCCGEEKM3OQMA5AKAhVKQXXwVmzNB1f/B7sMsqbGojBhSByCEEEKM0eQNALlpgS1l4dauZHa4irna5GD3YQyl2FIXYteJjsKSwLI8sBBCCDF6kzcAmMEwQNuwQCmyi65icyrJwZ7DAFxSF6I94XKsNQGARooBhRBCiNEaVQBQSt2klHpWKXVYKfWhIR7frpTqUkrtzl0+VpKjs8KFOoDM4uewJZWkNdNOe6aDLXXBSIHHjrUVdpduACGEEGJ0zhkAlFIm8HXgZmAd8Hql1Lohdn1Ia70ld/lUSY7ODActAIBTfxmbsi4Q1AHMjZssqDB5XAKAEEIIcd5G0wJwOXBYa31Ua50Ffgy8fHwPK8cKYxgmhjLQoTJW1K4lpCl0A2ypC7H7VFdhKmDX1/iyPLAQQghxTqMJAHXAqaL7DbltA12llHpKKfUbpdT6khxdYThg0ArgL76a9Zk0hzr3A3BJXZhk1mffme7CU6QOQAghhDi30QQANcS2gV+znwAWa603A18F/nfIF1LqXUqpnUqpnS0tLaM7wqIAkF10FZvTWY4mGnB8h80LQijoPxxQAoAQQghxTqMJAA3AwqL79cDp4h201t1a697c7bsAWyk1a+ALaa2/pbW+TGt92ezZs0d3hFakEAC8qkVsMspw8DnWe4J4xGDFLIvHiyYEkjoAIYQQ4txGEwB2ACuVUkuVUiHgdcAdxTsopeYppVTu9uW5120b9EpjYVpYZhilAKVYM3crAAe7ngVgS12Yvad7SOYKBD1f40kdgBBCCDGicwYArbULvBe4G9gP/FRr/YxS6t1KqXfndns1sFcp9RTw78DrdAln5VFWGEsFrQDxJddQ57gcaXkCCOYD8Hx48mRnYX9pBRBCCCFGZo1mp1yz/l0Dtt1WdPtrwNdKe2hFrAiWYeH4Ltm6S9m002FH4iRaa9bNC2GbsPN4O1evCHodsq5PNGSO2+EIIYQQU93knQmwmBXGNu3C7Q3R+bTi0JZuI2wp1s8L9asDyHieTAsshBBCjGBqBAClsK1o4e7aeZcBcLT5cSDoBjjckqKtNwMEywP3ZNyLf5xCCCHEFDE1AgBg2jEMFRzuwmU3EvF9juQCQH5a4J3H+4YDprIeGde7+AcqhBBCTAFTJgAE6wIE/fpGVT3rfYMDyQYAltdaxMOKHcf6DzzoTrnSFSCEEEIMYeoEANPGzi0RDLA+Vsch5ZDNdGEaik0LQjx+vL3fCd/Xmu60dAUIIYQQA02dAADYVqxwe838bbhKcebE74FgWuCmHodTHal+z0k7HmlHugKEEEKIYlMqAFihssLt1UteAMCRlp1AsDAQwI6jrYOe1512ZJEgIYQQosiUCgDKimDl6gCqorUsxOLZZANozYIKkznl5qA6AAhGBXSnnYt9uEIIIcSkNaUCAIaBbfd1A6wvW8jTliLUfgSlFFvqQuw62TXkVMAZ15euACGEECJnagUAwLb7ugHWzNtGm2XSc+p+IJgPoDvjcfBs15DP7U47sk6AEEIIwRQMAMV1AOvmXgrA0ZZdAGxeENQBPH506KWGtYbulHQFCCGEEFMvANhRjFwdwJL4QmKY7M+2YGS6qY6ZLKu12XG8Y9jnZz2/sHKgEEIIMVNNuQAAfd0ApjJYU76Qp8Ihyk/3jQZ4qrGXdHb4b/q9aRfXkxUDhRBCzFxTMgBYob5CwHVzNnMwZGM0PALAlgUhsp5mz9HGoM1/CBpkgiAhhBAz2pQMALZdXri9rno1nlKcaH0SfI8N820sAx4/0YnKdA/7Go7nk5AFg4QQQsxQUzMAWGHILQ+8tnIlAHsNl0jbQaK2wWULo/z86Q4a27rASQz7OomMiyNdAUIIIWagKRkAlFKYdrAuQEWonIXRuTwZCVPe+BgAf/3cCpSCj93dgJ/qBjcz5OtoglEBsmCQEEKImWZKBgDo3w2wvmYNT0djlDUGdQCzyhT/cN1c9pxN8R+Pt2BkusAfurnf9TW90hUghBBihpm6ASAUA6UAWFe1kk6laek6hplqB+CKJYqXrq3iuztaeKKhB5XuAD10c38y65F1pStACCHEzDF1A4BhgxVM/LOuahUAT0XClDc+DoCvff7q6grqK0PcencDPcksKt057Ot1p6UrQAghxMwxZQOAZVgYVlAHsLi8jjIryhNlFZQ3PlrYRxlZPv7CebQlXf7pD6fBzaAyPUO+nudreqQrQAghxAwxZQMAgJWbEMhQBmsrV/JUWZyy07v69fcvrPX4qyvncO/hbu7Y14lyEuAkh3y9VNYj48qCQUIIIaa/KR0AguGAFhDUARzVGVJukljzM4V9PN/jls0xti0s418eOMPxjgxGpge87JCv2Z1y8WXBICGEENPclA4AlmEV1QGsxEezJxrt1w0AkPHS3Hr9PMKWwUd/20DW9TDSneAP/rbva02PzBIohBBimpvSAcA2bDDCAKytWoFCsat2EWW5+QCKRSMZbr1+Ac+2pPnmo82g/dzIgMHf9tOuR9qRrgAhhBDT15QOAIYyggmBlKLcLmNxeR1PxcqJdB7H6j3bb1/Xd9m22OLVG6v5wRNtPHqiF+W7w44M6E470hUghBBi2prSAQDANu1+3QDPeD34UJgVsFjKS/Peq2eztCbMx3/fSHvSRXkZVLZ30L5aByFACCGEmI6mfACwlAVGEADWV62ix0txqGoB1QfvRA0o9NNa45LkszfV05vx+PQ9jWitgwDgpAa9dsb1SWWlK0AIIcT0M+UDQNACENQBrKsKFgZ6bPXziXQcZs7Obw7a3/Fd6qs0/99z5/Lw8V5ufzqYOdDIdIM3+Bt/T9rBk64AIYQQ08zUDwCGjTJMMC3qy+YTt8vYYxu0rXsNNc/+koqj9w56TtJL8aoNlTx3STlfebiJw61pQGMMMV1wfsEgIYQQYjqZ8gEAcsMBTbswIdD+rkN0XfEuknM2Mv/RLxHqPN5vf601CS/Bx15QRzxs8pHfNpB2/WFHBmQ9n2RWhgYKIYSYPqZFALANG8xgWuD11Ss50duINj3OXvdxfCtK/QOfRA3o43d8l0jY5RMvrONoe4Z/f7gJAOU5qEz3oPfoTbu4niwYJIQQYnqYHgHAtMGyQanCwkAHOo8QrZ7H6Ws+Qqj7FPMf+dKgb/ZJN8m2RVHeeEkttz/dzgNHgxO/clOobKLfvhrokq4AIYQQ08T0CACGHdywQqypXI6BYl/nIUylUEsup2Xz26g8/geqDt7R73laa3qdBO+5ag6rZ0f49D2naekNTvIq2wNuut/+rq/plQWDhBBCTAPTIgAYysBQBhghYlaUFRVLuPfMH8l4WcKmSeLSN9Fbdzlzd3yDSOuBfs91fAdfZfnMTfVkXJ+P/74RP9dSYKS7+i0sBJDIuDjSFSCEEGKKmxYBAHKtALkJgd656vU0Js/yX4dvB6AsHKbpmo/gRaupe+CTwZC/Ikk3xaIqm7+/Zj47TiX44RNtuUd0riiw/wm/M+mQdSUECCGEmLqmVwDIDQfcOmsjL66/np8du5N9nYcAiFXW0njtx7FTbSx4+PP9Tuq+9ul1E7x8fRXPX17B1x9pYl9TUDSofG/QdMG+1nQks/RmXPQQawkIIYQQk930CgAAZnD9l2vewKxIDV/ccxtZL4upFNbCjTRd9h7ijY9Su/fH/Z6f9bJk/Sz/eP18amMWt97dQDI3C6DyskOODEhkXNoTWRkdIIQQYsqZNgHAMiwUCsxgVsAyK8bfrf8LTiZO89+Hfw5AyDBJb7yFriXXMXv3d4idfbLfayTcJPGwwadvrOdUZ5YvPdi3oJBykuD0HxkAQWFgeyJLQooDhRBCTCHTJgAopQoTAqGCbdtmb+bm+uv4ybFfcaDrCACxkE3rcz9AtqKeugc/g5VsLbyGr30SbpJL68p4+7ZZ3LGvk98f7Co8bmR6wc0Mem8N9OZaA2TaYCGEEFPBtAkAkJsRUCkwQ4Vt7179Jmoj1UFXgB8M8Ssrr+D0tZ/AcFPUPfhp8PsW/Ml4GbJelr+4fA4b50X5pz+c5kx3flEhjZHp6rd/McfzaevNyAJCQgghJr1pFQD66gDChW3ldoz3rf8Ljvc28IPDvwDAUIrQ/JWcufLviDXvYfaT/9nvdXrdBIah+dSN9WgNt97diJv/Zq99jFRbUBPgJIMFhIoKATXBMsKdySy+tAYIIYSYpKZnALBC/bZfMXsLN9Zdy/8cu4ODXUdz+xq4a26kY9VLmfXMjyk/+cfC/r72SbpJ6itDfPC6+Tx1Jsl3d7T0vaD2UU4SI9ONkWrDSDSjkq2odFdQJ+BlyTgerYkMaUdaA4QQQkw+0yoAmIaZmxDIhHCs32N/teZNVIcq+Oc9t+HkJveJ2hbtV7yXVO0qFvzx89g9pwv7p70MWd/h5jVV3Ly6kv94vIWnTieHeWeN8l2Um8LI9GCk2jESTZBoobujha7ODvxsGnwZLSCEEGJymFYBAIpaAULlhSGBAHG7nL9d/06O9Z7ih0f+X2F7WVkZZ679BChF/QOfQHnZwmO9Ti9aa/5h+3zmxW1uvbuB3szov9Er30O5abLJLjramsh0nYbeZki2Q6YnKCiUUCCEEGICTN8AABCpCIoCc54zZysvXPA8fnT0lxzuPg4E9QDh2Qs5c/WHiLQfZu7jXyvsnx8VUB42+cyN9TT3OnzuvtNjmvzH15rulEtPKo1205DpDYJAb1P/UOBIS4EQQojxN70DgGFCON7v8feseQuVoTj/vOc23FxXgG0YeMufR+uG11N96NdUHvldYf+0l8bxHTbOj/GXV87hdwe7ufNAF2OVdnw6kg7Z4smDfC9oDcj0QqoDEs2Q7h60eqEQQghRKqMKAEqpm5RSzyqlDiulPjTCftuUUp5S6tWlO8TzYxlW/w12BOy+UQEVoXL+dt07ONJzgh8d/WVhe9S26Nr6DhJzNzPv0X8j3HG08Fivm0BrzVu2zuLSuhj/fP8ZDrb0XynwfHi+pivlkMi6aIY4yWsN2QQkWsBJjfl9hBBCiOGcMwAopUzg68DNwDrg9UqpdcPs9wXg7lIf5PkoTAhULFwBRt+PevXcy7h+/tX84Mj/40jPicL2smiEM9d+DD9URt0Dn8BwgqI/z/dIeilMQ/HJG+qJWoq3/eQo393RguuN/Vt6MuvRkXRwh2vy9z1IdUKiLRhuKIQQQpTIaFoALgcOa62Paq2zwI+Blw+x398APweaS3h8Y9KvGwCCOoBwZb9Nf732rcTtMr645/8WugIMpYhVz6HxebcS6jnN/D/9S6EZPuWmcH2XeXGbH71xBdcui/ONR5p520+PXnBrQGfSIZkdYSphLwuJ1iAMSH2AEEKIEhhNAKgDThXdb8htK1BK1QGvBG4r3aGN3aAAAGDZ/YYGVobi/O26d3Co+xg/Ofarvt0MA7X4UloueQcVJ+6n+kDfiIF8V0BtzOJzL1rIF160kJZeh7f85Aj/99FmnDEuCqSBRNajI+XgjdTv76SCboHs4DUJhBBCiPMxmgCghtg28Cz1ZeCDWusRx8gppd6llNqplNrZ0tIy0q4XZFAXQN6AoYHPm3c52+ddxX8f/jnHevoyTsSy6Nn0enrqr2LurtuItOwDwPVdUl5fn/zzV1Twkzet4IaVwTwBb/7xUZ5pGnufvev5dCSypJxhagMgWMY43Q29LUOuSyCEEEKMxmgCQAOwsOh+PXB6wD6XAT9WSh0HXg18Qyn1ioEvpLX+ltb6Mq31ZbNnzx7bEY+CZViEi6YD7mfA0MC/Wfc2yu0yvrjnNryiOf7LoyGanvdhnNgs6h/4FGY6qPxPeel++1VFLT51Yz3/+tJFdKc9/vynR/nqH5vIuGNvDejNeLQlsvRkRqoPcIOhg8n2YdcmEEIIIYYzmgCwA1iplFqqlAoBrwPuKN5Ba71Ua71Ea70E+BnwHq31/5b6YM9HPBQPZgUcaMDQwKpQBf/furfzbPdRfnr814XtCkWsoprGaz+Bme5gwcP/BNpHa02P24uv+5+Yn7c0zk/etIKXrq3iv3e18sb/OcJTZ4abOfDctO4bMtiZcki7w5zk3UzQLZDpkWGDQgghRu2cAUBr7QLvJaju3w/8VGv9jFLq3Uqpd4/3AY6VoQziofjQDw4YGnjtvCu5Zu4VfO/Qzzje21DYbhkG5oJ1NF3+XspP72DWnh8CQVdAl9ONN6DHIx42+egL6vjqyxeTcTV/cfsx/vXBM6ScCyvcczyfnrRLWyJLMuviDzzRax3MISDDBoUQQoySGsusdqVw2WWX6Z07d477+3Rlush4Q/SVaw3JtkJVfUemiz9/+P0siM3l36/4JKZhFnbtSWWZ9eBnqTh6Lydf8M8kF2wFgpBRYceHrDlIZD2+9scmfrang7oKm1tfUMfW+rKS/EwKCFkG0ZCJbQyR4cxQ0NVhDlEMKYQQYlpQSu3SWl821udPu5kABxq2K2DA0MDqcCV/s+7tHOg6ws9O3NVv1/KoTfNVf0emajF1D30GKxEUMPrap8vpJusPHqNfFjL54HULuO2WJSilePcvjvP5+06TyF54f70GMq5PZ9KhI9c90K9oMD9sMN0lwwaFEEIMadoHgBG7AgYMDbxu3lVcPecyvnvodk72Nha2KxRl5XFOX/sJDC9D3YOfCorwIKgJcHqGbmUAttaX8T9vWM4bttTyiz0dvO6HR3j0RG/Jfj63qHsgkXX7DyPMJmXYoBBCiCFN+wAAEDbDRKzI0A8WDQ1USvG3699BxAzxxb3/F6+o0M8yDKw5yzhz1fuJtTzD8v99KzV7f4yZ7sqFgF5S7tD97xHb4H3XzOM//mwpEUvxN788wafvaaTnPFYWPBetg5kF2xNZulNFaw3khw0mWmXYoBBCiIIZEQAA4vYwXQHQb2hgTbiK9659G/s6D/GL47/pv5tlkl75Ak5t/xRO2WzmPvEtVvzsNcx/+HNEW54h4STodYb/tr1pfowfvH45b906izv3d/LaHxzmoWM9JfsZ8zKeT1fKoT1ZNKeA5wRDBlMd0i0ghBBi+hcBFst6WToznUM/6KSDb8oEzfq3PvEv7Grbw7ev/gL1ZfMLu2mChXw8XxPuOEbVwTuoPPp7TCdJunoFHatfRnrFzZRFZ6PUUHMoBfY1pfj0PY0cbstw0+pK/v6aeVRFh5nA6AIpBWHLIGqbWIYBhgWx2n7rIwghhJhaLrQIcEYFAIDubDdpd5i5+9PdQRAAWtPtvOPhf2BJeT3/esXHMItaDzytSWTcwtS/hpOk4ti9VD/7SyIdR/HsMnqW34i74Q3ompXDHovj+Xx3Ryvf2dlCZTgoGnz+iorS/bBDsE2DiG0QDoVRZbP6TYokhBBi6pAAcJ601rSl2wZN5JN7sN/Mer9rfJAv7Pkm71nzFl615OZBu2c8j0TG7Zt/R2uiLc9QffAO4scfwPAdsvO3kl73GjJLrh92WN7BljSfvqeRAy1prl9RwQe2z6c2Nj6tAXlKQTgcIVIxG9syz/0EIYQQk4oEgDEYsSvAdYJ+coKw8JEn/pndbfv49tVfoK5s3qDdfa1JZF2yA6b+NdOdVB7+LdUHf0Wo9wx+tJbUmltIr30Vfvn8Qa/j+prv72rl24+1ELUNbtlYzS0bqplfEbrgn3ck2gxhxGqIhiwitolpSIuAEEJMBRIAxqgn2zNs1T7ZXsgE0/i2pNt5x8MfYHl8EV+6/NZhCwkd3yeRcfH8gbP0+cRP72LWoV8TPvkwKEV20TWk1r0Gp/4qGPB6x9ozfP1PTYXiwKuXxPmzTTVcsagMY5ya67UZRkerAQiZwQRDYcsYsYZBCCHExJIAMEYjdgUAJDuCynngtw3388W9/5e/Wft2XrH4huFfE03K8UhnvUFr+SmlqEr1ED94B9Fnf4GRaserWEhq7atJr345OlLdb/+zPVl+saeD/32mg46UR31liFdtrOal66qojJS+e0BbEXSkqu94gbBtBvUC0kUghBCTjgSAC+B4Dh2ZjqEf9L2gHkBrtNZ8eNfn2dPxLN+++gssiM0d8XVd3yeZ9QpFgsXK7DKimISP3UNk308JnX0CbYbILLuB1LrX4s7Z2K8wL+v63Hekh5/taWf36SRhU3HD6kpevbGGdXOjF/TzD6TtGDo8uAjRUIqInRtFYMrIASGEmAwkAFygEbsCioYGNqfaeMfDH2BV5VK+uO0jw88pUCTteiSz7qBF+qJWlDIrmIHQbD9EdN/thA/9GsNJ4NSuJr3utaRXvWxQ0eDBljQ/39POb57tIuX4rJ8b5dWbanjBygoiVmlOzNouQ4eHmTmRvlEEEcvEkHoBIYSYMBIALpDWmvZ0+6CV/QqKhgbeeeoP/Osz3+Zv1/05L130wlG9vp8bMpgd0BoQMcOU2+WF+yqbIHz4LqL7forVfhC3chGJK/+e7KJrBw3V68143Hmgk5893c7xjiyVEZOXravilo011FdeeNGgDpWjQ+Uj7pNfkChiS72AEEJMBAkAJTBiV0DR0ECtNR/c+Tn2dR7iW1d//pxdAcWyvkcy4/UrEgyZIeJWef+Tp9aETj1E2SNfwuo6TrbuSnqvej/eEPMJaK3Z2ZDgZ0938MDRbnwNVy0u58821XDV4vILquj3w3GwR7d6oVIQsU2itoktXQRCCHFRSAAokd5sL0k3OfSDRUMDm1It/MUfP0TIsPnkJX/H+upVo36PoYoELcOiYqhpin2H6L6fEtt1GyrbS3rtq0ls/St0tGbI127udfh/ezv4f3s7aEu6LKiwuWVDNS9fXz3mGQb9cAXYsXPvWMQ0FGHLIGQZhExpGRBCiPEiAaBEztkVUDQ08HhvA7c+8S+0pNr4uw1/wQ1115zXe7m+TyLr4ea6BUzDpMKOY6rB1fYq3UnZrm8S2Xc72o6RvPRdpNa/fthJhVxPc//Rbm5/up0nGpOETMULVlbw6o01bJgXPe8Tsh+uBHtsxYaKoGYglAsE0joghBClIwGghEbsCoB+QwO7sj18aveX2d2+j9csfQnvXPX6ftMFj0ZxkaChDCrsOJYx9Ld1s+MI5Y/8C6GGP41YH1DsSFuan+/p4K79nSQcnzWzI7x6Uw03rqokYo/2WBV+pBKGW03xPCgFYdMsBAKZdEgIIcZOAkCJjdgVUDQ0EMD1Xb6+/3vcceoerph9CR/Z/N5Cdf9oFRcJKqWI23FCxtDf7gFCJx+m7NF/weo8NmJ9QLFE1uM3B7q4/el2jrZniNkGz11azvOXV/CcJXGi5wwDCj9SBVb4vH62czENVegqkEJCIYQ4PxIASuycXQFFQwPzfnnyd3xt//dYWDafz1z6gfMqDszL+h6JdFAbEDOjhM3w8EMNfYfIvtsp2/XNoD5gzatIXPaeYesDin+2J08nuetAJw8c6aEz7RG2FFctDsLA85bGKQ8PN+mPwo9Wgzk+UxMrwDL7agdCJRrWKIQQ05UEgHHg+A4d6RG6AoqGBuY92baXT+7+MqD4xJb3saV23Xm/r0aTzLqknaA1wDZsIkYY27CH/Has0p3Edt1GdN9Pc/UBf0Fq/RuGrQ8o5vqa3Y1J/nCkm/uOdNOacLENxRWLyrhuRQXXLI0PLh5UBn6kelSvf6GUohAEQqYhExAJIcQAEgDGyYhdAQDZRHAp+vgaE2f56BNfpDHZxN+sfRsvXfSCMb236/tkXI+M6xfqA0JGiLAZwh6ie8DsOErZo18ifOph3IpcfcDikesDivlas+dsij8c7ua+w92c6XEwFWytL+P5KyrYvryib3VCZeBHa2CYWoXxYqiguyDoKujbrui703970e0Bn0P/x4bfTwghJjMJAONEa01HpgPXd4ffyfcg0wNutrCp10ny2ae+yuOtu3nFoht4z5q3YBpjm0tfo8m4PhnX7zdiIGyECBvhQa/bvz7gCnqv+sA56wMGvafW7G9OB2HgSDcnO7MoYMuCGNetqOC65RXMqwjjR2thjD/XZFbcFWGbSoYyCiEmLQkA4+icXQGFHdNBEMh9lp72+fazP+L243dyae0Gbt38f6g4x8x65+L6PinHw3H9QqODbViEzDBhI9RXL+A7RPb9LFcf0EN6zS25+oDa835PrTVH2jL84XA3fzjSzZG2DADr50Z5/spKrtuwhLqa0U0WNJVZhsLOdUXYpoxeEEJMDhIAxlnCSZBwEufe0feDuQKKagN+23A///bMfzA3OpvPXPp+FpXXXfDx+FqT9TzSjl+YVTBfLxA2QoSMEEopVLqL2BO3EX3mJ2grSnLru0ZdHzCcEx2ZQsvA/ubg51w1t5zrVs/hujVzWDpr+ocByHVHFLUSSH2CEGIiSAAYZ6PqCijmOpDpDroHgL0dz/LxJ/8Vx3f56Oa/4fLZW0p2bI7vk86tOpj/rxjUC9iEzDAhw87VB/wr4VMP5eoD/o7s4u2jrg8YzunuLPcd7uYPR3p4+kxQK7GkNsZ1a+ZwzcrZrJ4XnzHflPMFi3bhoqTbQAgx7iQAXASO79CZ7kQzys9Ka3ASkE2CDqYPvvWJL3Gs5yTvXvMmbll8c0lPEL7WZNygVcAv+u9pKIOwGSZihIk0Pkr5I/+C1XmUbP1z6L7us+ccNjhazSm474TDfc+28OTJDnwNFRGLrYuruXxpDZcvraG++vzmR5jKpI5ACHExSAC4SEbdFVDMc4PaAM8h5ab5/J5v8HDTDm6q287/Wf/nI074M1YZzyPr+INWH7QMizAGVc/+kvjjX8GPzqLrxi/j1a4uyftqM4SOVNORdHj8eDuPHwsuzT1B3cCCqgiXLwnCwGWLa6iMjf9QwskkX0dgG8EoBqWCEQyGCrpw8teTmu+DId0dQkwWEgAuovZ0++i7AoplU5Dtxfc9vnf45/zgyC/YUL2aT2x5H9XhytIfKOBpTdpxC0MJi1W0H2XeHz6Mke2he/tnyC4b3dLG56KtCDpS1Xdfa062J4MwcLydXSc6SGQ8FLB6XjxoHVhSw6aFlYSt6Tei4Hyp3P8ZKhjcaCiVCwvB9eDtQ+870Gj+jQ+7i9bgpsFJgZfBsMIQjoM1PhNCCSFGTwLAReT6Lh3pjtF3BRTzfcj2gJPhD2f+xBf33EZ1qJJPb30/y+OLS3+wORpN1vNJO31DCQGsVDsL7/8EkZa9JC59F8mtfwXnuZbBkO9nRdGh8iGHCLq+z/7TPTx2rI0dxzvY09iF52vClsHmhVWFQLBybjnGZP82PN15DspNodw06L7fG6XAMgzMUBg7WoFlh6UIUogJIgHgIks6SXqd3rG/gJuFTDfPdhzi1ie+RMJN8uFNf81z524r3UEO99a+T9r1yOZaBZSXZd5j/07V4bvILN5Oz3WfDU7epaIMtGGCCi7aMHK3g+uEo3nyVCePH2tnx7F2jrYGXSxVUZvLllRzxdJati2tZn7l2FYjFOdJ++CmUE4KNcqWLm2GIVyOZYexTYVtGliGjIwQF4HWweJsXjb4wmFFLri4eaqRADABOtIdOL4z9hfQGrK9tHY38rEn/5Vnu47w5ytfyxuWvfyi9QM7vo/j+TiuT/yZnzN35zdwKxfTfdNX8SsWXpRjgKAdOx8SWhIej5/q4fET3ew40UVrbzDB0sKaaKF+YOviauKRmVU/MO7cTO7bfgbG0rpFEAR0OF6YITJfCGmZCtuQ4ZKiBHw/ONl72eDE7zv9+66UCkKAHS35wmWTlQSACeD5Hl3ZrrHVA/R7IZdMsoUvPfVN7j3zR54//zm8f8NfEh6nBXeG42uNcWoHtfd8HLSm4/mfQy963kU9hoG01hxtz/D4qQSPnUrwREOClBM0Rdum6uvzZnAfuSpcD+hHZ3D/ORQ/D8KWyep5cTbVV7KpvpJFNbHJX5w3Fr4XnPSdZL8m/gulrUiuC2jwVNEDQ4GVazG44PfUGq2D6OIXbuvCuUFr+o2Ogb4visNNJT3UtqH2LX6Kacjwz5LKf7svnPCHWaBtKMoAOwJWdFrXq0gAmCBaa3qcHtJu+tw7n+u1Mgn+58AP+c+DP2F15XI+dcnfMStSmiF658PobqTqNx/C7DhOzxXvJbvp7YAxxu+EpeV4PnvPpniiMUnS8YM/+oCvAQ0+oFH4AFrho3KPK7RSuZNC/vHciYK+k0f+BNGbcdl3upvudBDuqqI2G3NhYFN9FWvnx0cuWNR+7luJ7rutc0eXew9tmMEJUpkXt8kyV9Cn3BTKy557/wt5qxFqQYoVhwLLUIUTef6/r/b7TubF2/3chsnwu1nMUMHPYeZ+HtNQQc3EDJkTY8y07n+y97IjVKaeJ8PqCwPmxV3DZLxJAJhgKTdFb7Z3bIWBxXyPP564l3968svErCivXvIiblhwzbiNEhiOcpLE7/00kWMPklp9M9nrPoJlV+D7iqyrB32TmnKUgVaqXy1CsM0ITlbKwPc9TrQlebqhi6cau9nT2MPJjiDoWYZizdwYm+bH2LygjI3zo8yOWX0n/bEcj2GBYeW6QqzgOEq52NIwBX3jT6HtKNoum5brRpwPBX1hoF84mKGtBr434IR/AV2q58O0+7oJpsHvpASAScD1XboyXXj6PJqohnGk9QD/vvur7O04gKVMrp57GS+uv55Latf3zfc/3rRPbOd/Ub7zP3HmrKP75n8iUrGImBXF0xrH88m4GrdoBsLpriPp8vTZJE+fSfL0mRT7mlJkveCnr6uwc4Egxub5MZbWhEvwjU/liiaDQNAXDqzR/eEaQ0Hf+FFoO4YOlZVkpMl0U9xqYCqVaw2Z+q0GWmscT+P6Pp6TGfDt3h+w71AvMKpNQBCwDKOvG8Y0gm5Cc6RwZYWDMGBFpuz8FhIAJgmtNd3ZbjJephQvxvG2/dx17Df8rvFBepxe5kfn8KL667ipfjs14aoLf49RCB19gIp7P40Oxei66XP48zYQs6LErKAqPz/E0HF9sp4urE0wE2RdnwMtaZ4+k+SpM0mePp2kPRUEwLKQwcZ5MTbNj7J5QYz1c6OUhUr5baM4HFh9Iy0MC3z3ggv6xo9Ch8rQdkyCwCgEJ7XBXQmWoTAmWTgoPtk7XvDlwPV18PuY6R73Lqfh5GuCDENh0j8kGLmgYBoGmKFc8eDUGkkgAWCSSTpJEk7iwrsEAHyPbKqDhxoe4s6GP/BU+z5MZfKcOVt5Uf11bJ21CXOc/5CabUeo+s2HMBIt9Fz7D6TXvAjTMCi3ygYVK3pak3VzgcAfPAHRdKa1prHL6QsEZ5IcacugAUPBylkRNs2PsXp2hKqISWXUpDJiURkxqYiYWJPsD/q4UkbQImCXTak/tpNJYT4Go393wsUYaeH7Gsf3cT2N6wW3B4V/rVFOApVNMPmC6GBGLiQYClQohmlHMUORQkjI/5YWij8nye+tBIBJyPEdujJd+KXqb/VcyPZyqvM4dzXcx92ND9Dl9DA3Moubc60Cs8exaFClu6j83a2EGneR3PQaeq/6azAsbMOi3I5hDzOlseP3tQ7MpO6CvJ6Mx96zKZ46k+Sp00meaUoVRjIMVB4yqIyYVEaDUDDwUjXE9qg9xdcYUAbazrcITOGfYxIpdavBqE72A3lZVKYLdT5V+5ORMoJRLVYkaCEYbrcBN/JxYeBIkfy/1YFhIhoyxzwTqgSAScrXPj3ZntJ0CeQ5Gcj2knXT/KlpJ3c2/IEn2vZioLhyzqW8uP75bJu9ZXxaBXyX8j99jdie28nWX0bXCz+NjlQAEDZDlFkxrBH6pmdyd0Ge62uaex260h5dKS+4Tru5a4/Owva+bYns8CHSNlSuJaHvErIG/Lcfsh918MaBfwaG+q+jgEXVYdbNibJuboRZZSWaj0EZ+KEysCQIjKfhWg3yTeK+H/wbdf0gsDveeRb9ah+V6UG5qfH7ISaINkywomir9MWDFRGb6Bi7CCUATHIl7RLIyyYhmwCtOZ1s4q5Tf+C3jQ/Qke1idqSGm+uCVoG50Vmle8+cyIE7iT/wRfzy2XTe/AW8mmVA8MclYkYos6KjKlb0dfDHJuuO4Q/NDOJ6mq6MR1eqLxT0XdyiIBEECNcb/DmO9pw6cL+BT3P9YBnofHabXWaxbm6UtXOirJ0TYd3cKFXRCxm9UDQxFEZulIZRNHLDGHwRJaHUBY66c1IY2Z6LPMpkYmjTDoKAFS1JYJUAMM05nkNXtoRdAlCYTRAnBToYifBI8xPc2XAvO1v3oIBtszfz4vrruXL2JZglTK3W2b1U3v2PKCdJ9/UfJ7u0b9KgoPI2eK+BTWH5U0r/CVWC236ufiDfzIgunoGlr+lMEby+qcyp3fw9RaUdn2db0uxrTrG/KcW+5hQnOvoKvBZU2KydE80Fgwhr50QpD4/jcKvicEDfUE6tFANDxLn/WF/g79NM/H30vaC5f4KK/CaWQlvhoIvAioz5VSQAzAC+9unOdJP1S/wPxfdyQaCvq+FssoXfNN7Hbxruoy3TSW24mpvqruVF9c9nXmx2Sd7W6G2h8u4PYzfvp3fbO0lufWtJv5G5+amKh6kfULmgYSkLy8hfT69JPqaK3ozHgeY0+5uDQLCvKcXp7r5x3YuqQqydG2VdrpVg9ewoUXt8v71nXJ+ejEdvxqc749GT8ci4OrfsMpi5GSEHXhsq6EM3oFAUZuRmjex/PXhbZcTCDlkDhm5O0KRPF4HKJlDZXqZCkd+4U0Yw+ZUVCeYaOA8SAGaQhJMg4SRK/8KeC5mefhNqeL7HY61P8utTf2BHy240cNmsjbx04Qu5as6lFz6vgJsh/sA/Ez34W9LLttPz/I8EBV0lptGFgkLXH35528GhwB6xLmHI9xpi2lhx/jpTLgeai1oKmlI0J4L5CAwFS2vCha6D9XOjrJgVIVxUv+D5mt6sR08mOJEXTuZpj56sR0862NaT9ftuZ4L9ezMemSG6QsaboaC+MsTSmjBLqsMsrQnnboeIhXItE0WTPWll9k36NJW6M7xsMLRvwueXmJy0YeXqBSKjqheQADDDZL0s3dnu0nYJ5OUKBQfOm92cauO3jfdxV8N9tKTbWVq+kLeseBXPnbvtwoKA1kSf/gnlj3wdt2ZZMF9AxYIL/CFKRykKrQO2YRXCwUjy9QnBWGZm5AiG8dCacNjfnGZfUypoLWhK0ZGbO8E0gpNnxtX0ZEYufoTgZBsPm7mLQXnYpCJsUh42iBduB4/l9wtbqjBttO9rfE3uogdcD96mdRBKNLlrHQx71bn9PR0UeB5vz3KsPcPJrgxFq28zL27nQkGoKBiEi2omVL8Wg+JJoCZN64H2UdneYP0IMSraDOXqBYafX0ACwAw0bl0CEPy1clKFQsFinva5/8wjfP/ILziVOM3S8oW8ecWreN4FBoHQyceo+P3HwTBIbPsLnNmrcWuWBpNrTDL5UGAbQTAYTSgIhkLluiT8mTmKodS01jT1OuxrCkLByc4MMdsgHjGJh4ITeEXEJB4y+m7nTuixST4E0vU0p7qCMHC8I8PR9gzHc7czbt/vTk3UZElNX2vB0powS6vDzCqz+v98yghOJqYNRui8m5kv2BiL/LrTHvubU5zpdoJ/Nzr4t+P5waiYYFvweQ18zPM1ru677fka16dw2yt6rCxksqDCZl7cZkFFiHkVNgviNrVlVmHRr4ml+oYUDlipUALADNab7SXpjlOi9n1wEoVCwWL5IPCDI7/gZCEI3MLz5l4+5iBgdp6i8rcfxuo4BgSL73iV9bi1y3OXFXg1y/AqFky6Jk9Dqb6ug1GEAmklEGPha82ZbofjHRmOtfddjndk6Mn0nVzLQ0bQSpALBPnb8+N2MEVwfg0JM4Q2Q2DY49NK4Hu5mfzOPZy5N+P1KxDd35ymoWt0X3CC4YjBtWUoTEVhiGIwXDGYwS8/VXK/xxV0ZzxOdwdDbIvZhmJe3C4EguA6CAjz4zazy+0RJ+FyfRdPe4TNEi4vnK8XsKNgWBIAZrpx7RKAIQsF8zzt80CuReBk4jRLyut58/JbuGbeFWMLAtrH7D6N2X4Uq+0wVtsRrLYjmF0NufX5wLeieLXLcGtXBMGgJggIOhy/0J+0pM63+0BaCcRYaa1pS7oDQkGWo+0Z2pN9fe0hU7GoKsSS6jCLc/UFS6rDLKoKE42Gwci1EpihCw7ZIxX5DRwNsj83GiS/5/x4MBpk7dxgJMji6lDfyb3oJJ8/wZdKMutxpsfhbI/D6W6Hsz1ZznQ7waXHoS3Zv27BVDCn3GZ+LhDMrwgxL24xu1xRHfOpimnKQxHidnnJjrGYNizilbVEo2NrKb0oAUApdRPwFcAE/kNr/fkBj78c+DRB95oL/K3W+uGRXlMCQH+e79Gd7cbxx3FVLNcJgsAQK2952ueBs4/y/cO/4GSikSXl9bwpFwRKMrGQk8LqOF4UCoJrI9Pddwzlc4sCQRAOvKqFpV0Z7wIVhwJLmYXroZqj860Enq+HHc0wJbhp7KansFr24lWvIFt35aBmTDF+utMexzoynMi1FBzvyHKiI0NDV9+cDABzy+0gENSEWVwdZnFtlKWz4tTEYygrPPoJbAYU+WVdn0NtmULtxv6mFEfbM4X3nlVmsa7oZL92TpSa2OT5N1ss4/qc7ckHgmwhGOTvt/S6/f6NKqCmzKTMNjFV0EphFrU8FLdCWLnFnIr3sVT//c2iAGQZweiTaCzGn1+ziqrY8LMNDmfcA4BSygQOAi8EGoAdwOu11vuK9ikHElprrZTaBPxUa71mpNeVADCY1ppep5fUeM+k5aRzhYKDWxw87fPg2cf478M/52SikcXldbx5+atKFwSKaY2RbC20EuRDgdl5vDCNqDZs3JolhVDg1S7DK5+HH6vNrS438f17wTCyvtEH+daCoUKB6/uF4jJdXGjmB+k52KYndh0FN4Xd9DT2mZ3Yp3diN+9BFQVT346RXfg8skuvJ7PoeTAOIz/EuWVdn1NdWU50ZINgkAsIJzqyJIumnC4LGSypDroQFtfGWFxbzuJZFdTPimMXrx2gNV6qi6NNHbmTfZr9TSkOt2WChX2AqogZDOnMzfOwbk6U2eUXuR6hxLK+Q9pLk/WyOJ6mNeHR1OPR3BtctydB+1auJiGoUfD8vlY+Txffp18tg6eLthfVPBTXMQA8+IHrWFR7/v+OLkYAuAr4hNb6xtz9DwNorT83wv7f0VqvHel1JQAML+tlSTrJ8SkQzNM6qA/IJoccxpsPAt8/8nNO9DayqKyON6+4hWvnXTnuCxDhOZidJ/t1IVjthzETrf1/BCuMH63Fj9XgxWrwY7PwYzX4sdrcJXc7WgPmxf9GYhpGodhwpFAwFE1fOMhXnefDQT4s+KUKC/kT/ukdhM7sxGreg/JdtDJxZ63FWXAZzvxtOHPWY7XsI3zsXsIn7sNItaPNENn655BZej3ZRdeiI5UXeDDiQmmtaUm4hVBwoiNbaEHID8WEoPm7rirMkpoY1WU2R5p7ONiSLixzXR4ycvM39J3w58XtSV18OVpaa9J+hrSXxjvHmgVhMzx+XQBaE62opbw8PqbP9WIEgFcDN2mt35m7/2bgCq31ewfs90rgc8Ac4MVa60dGel0JAOfm+i4pN0XaTZd2KuFivheMFnDSQz9cCAK/4HhvA4vKFvDm5bdw7fyrxj8IDKDSXVjtRzF6WzBS7RiJVsxUO0ayDSPRFmxLdw39c0SqCoHAKw4HhdtBeNCh8nFtVTCUgamM3GyGRt99wyzMoDgWvtaFYWnn5KRQZ3ZjNO5ANTyOOvt04YSv56zDr9+Grr8cf/4lEO77w6d1MHzO9TSO62KefZLQsXsJH7sXM3EWrUycBduCMLDkOvwSTTolSieR9QotBic6MhxvD263JV2W1YRZOzfK+tzJvr4yNLlP9r6L1X4Iq3kP2orizLsEP1434r9fz/dIeWkyfmbY+UQGGs8AAFBeWUu0bGz1TxcjAPwZcOOAAHC51vpvhtn/GuBjWusXDPHYu4B3ASxatGjriRMnxnrcM4qvfdJumqSbHL9CwSEmEhp4DA81Pc5/H/55IQi8afktbJ+AIDAizwmCQDIXDAqXdsyi20aybcjpS7VhgmH3jcnOT96Sm6ylb6y2iVZGYZx28fbi/fptN0y0GcaPVuNHq3LXfRcdqcAwQ6UPCE4KTj8JDY/Dqcfh7B7wneC45q6H+sth4eWw4NJ+J/xzKdQ4eD7+2Wcwj/ye0LF7sLpOoFG4czeTWXo9mSXPx6+oH9uxC5GjMt3YTU9jNe3Gbnoq6Joa0F3qRWfhztuCM+8SnLlbcGetBsMm4wXf9p0xTF400wPAeXUB5PY5BmzTWrcOt4+0AIxN2k2TclPjVyw4Qn0A5IPADr5/+Occ6z3FwrIFvGn5K7lu/nMmVxA4F61R2d7BQSHVCb6L0l4w5tn3gnoE7QW3c9fBbb9v+1D7aH/wdieNke4Kbg88JBQ6UjkgHPTd1tFqdKwWFa1BxWZhRioxDHNwQHCScHr3MCf8DcHJvv5yqLsEQqX9w+Z6Hl7rYTj0O8yj92K1HggOqXZNUDOw9Hq86uUlfU8xDWmN2XUCq+mp4GTftBur40jwkDJwa1bhzt2MM28LzpxNKDeFffbJ4NK0G7PnNAC+FSE1ay2p2etJztlIavZa/PP8nZ/pAcAiKAK8HmgkKAJ8g9b6maJ9VgBHckWAlwK/Aur1CC8uAeDCOL5Dyk2RcTOl7x44R30ABEHg4aYd/HdREHjj8lfw/HnPKenCQ9NSbtlUI9XR/5LsQKU6MNKd/bdneoZ+GcPqHxAiVZg9Z7Ga9xWa9L05a/HrtgbN+gsuwQjHMZRCYeTmsB/f0KY7T+If+j0c/j3mmd0AuFVLySy5nuzS63FnrS1Zl4simA5aDTFnf377iMc64j+jkf+N5aeQzk9UI9NJnyc3jd2yr+/bfdNTGOkOAPxQHGfu5uCEP3czzpyNIxaeOr6D092A1fQE0aa9RJv3EOk4gtI+GkWmehnJORtIzdlAcvYG3PK5Ix7ajA4AuTd5EfBlgmGA39Faf1Yp9W4ArfVtSqkPAm8BHCAFfECGAV4cvvZJuSlSbqr03QPnqA/Iv//DTTv4/pFfcLTnJPWx+Txn7lYiRpiQGSJshAibuYsRImSGiBTdHvi4bUyPIqOS8px+oUCligNC/7DgR6vJ1l2Ks+ASnHkbg5ES55A/QRq5UFAcDhTBdfF2hRrbf6PeJjh8L/rw7+DUDpT28Mvn57oJrseZu7nfULXiE3q/xXqKTuiGCoZVGar/KpMTzdd9VeKu7+dmt/MndnTHJGIkmoNv92d3B9/uWw+gdNA871Yuzp3st+DM3YxXveyccxrki/oyXgZ3iGZ+5aSItu4n1ryHaPNeoi37MHPdB05sdl8gmLORTNXSfr+HMz4AjAcJAKWltSbjZUi6ySH/AVyQc9QHQBAE/ti0kx8d/V+O9zaQHWMXhUIRNkOEDDt3nQ8HYcKmTcSMUF82j1UVy1hZsYS6svlTq+thGlEKFEbuWuXCQn7R5nyo6AsL+TChCAo6jaMPYBy5B078CeVl0ZFKdNUSqKxDVdajKuqgog4q6yG+AKzzHyc92fQFAz83FS540z0Y+B5W+0Gss33f7s3eoIlem2GcORuKvt1vgmhN8LyiUJfPm33Lh3t4voenPTztkvEdPN8f/eRbvke44yixlr1Em/cSa96LnWwBwLNjQbfBnI0k52zAn7uZ8tic0n0eA0gAECXleA5JN0lmFNN3nt8Lj1wfUMzXPlnfIeNlyfpZ0l6WrJcl42fJeFkyXoaM7xRtywTbc9vSfiZ4LLct2D9Lyk1zKnG6EDAiZpgVFUtYWbGUlRVLWFWxjEVlC6QbYgpR2QShk48SatiB1X0ao+csRu/ZwlwQENRHqLLZQRioqIPKOqioz13XQXxeMPvdFBWMF/fxc5NGeRq8qTpxFIDWhDqeJXLoTkKHf4NKBCdXXTYHveAS9PxLghqU2WtQZqjvBD+gFUdrjauDE73ru7i+h6vdEbtYPK3xdPBZen4wbNbTGt8fubPU6m0i1ryXaMteYs17CHccQ6GD0S11V5Be8SKyS64LRgqVkAQAMS483wuGEXrp0nUPaB0UmWUTE7YMuOd7nEg0cqj7GIe6j3Oo6xiHe46TzgWekGGzPL6YlZVLC8FgSflC7Ek0o6A4B9/DSLRg9pzF7DmD0X0Gq/dMcL/7DKq3uV8hpVYGqnxuEAYKAaG4BWFe34ySWoPvgpsBLwNuOnc7G4RcLxPcLzyev6SDfdyi5xTvE58Pi64MCizPYzTFSDytcbzgZFaYRGYSBgNDBbPbhRKnsQ/dhfnsr1HtR4I1CpZeC6tuhLqtwWc0TPeRr/3gJK+9wone015JW0fyw2Xz82t4uVUhh2qFMbK9RFv2U9G8h/jxP2D2nEabYTKLryWz4mayC59bktApAUCMK601aS9N0kniDVF9PiajqA+4mDzt05g4w8HuY4VgcLj7GIlcP5+tLJbGF+YCwVJWVi5lWflCQlP4W+OM5rm5gHAmuHSfwew9g9kdBAbV21xYewJAKxMVLgc3i/YywQiOMdKoYCpkKwxmGG2FwbRRXY0oLxO817yNsOiqIBDMv6Tk3RfFLQb5meYuVo2BAizTwDKCa9vpxjx0N+z/FTTuCnaquwzWvgRW3gTRqkGvUTjB564d3xu/Ic6jpMnPzhe0Fni5SbdsI0S5WYbZ9BThw3cRPnI3RroDPxQnu/QFpFfcjDP/stFPtTyABABx0WS9bDB6oFTdA6OoD5govvY5k2zmUPexfsGgx+kFgul7F5fXsSofCiqWsiy+iKgVmeAjFxfMczASzUEwyIUElenJnbBDaDMcrKJnhXNrtufuF9/O72eF0WY4dz+UmxNiiG+xbgb77F5CjTsJNe7Eaj4QVJ5bYfwFl+IvvAIWXYk5Zz3GOC3nW1x8GNQZXNioBAUYhsI2FZZhFK5xM3D0vuCkf+zBYJhpzXJY+1JY8xKorEfnm+K1l2vGz3/Dd6dUzUPYDFEZKjpB+y6cfAQO/BoO/T5oES2bDatfhF7zkmBuDaX6/YzFP27+nJvfZkarMMLnLtYdigQAMSYln2XwPOoDJpLWmqZUay4M9AWDzmywKJGBYlF5HSsrlrK6chmrKpaxvGIxkVIuBypmBJXpxT79ZBAIGnZidRwHwA/Hceq24izchl9/Oap6KaZh5iZ96psAqpQ0xcEgmM1xqH7xYAEbhW0pzNwJv9Avr/1gXon9v4JDd0O2F102G3/VzWRX34w3ayUeGl/7uRP/5P5bMFqDAkAxJw3H7of9v4bjDwRfhKoWByFo7Uugeum53yBSCaGxrachAUBckJIOI8zXBzipSR8Eimmtac10BIGg62ghFLRnOoEgFCwury+EgpUVSyUUiPNmJFqxG3cRatxFqGEnZm8TAF7ZHLL1l5Gtvwynbit+2azcEEcDS5kYuVCQH1mRPyWrohEXFzI80/V9PK2xDSM3iiNQ+AbfvB914FeYB3+L0duEtmNklm0nteoGnAWXjrnpe6oYMQAUS3cFLQLP3gknHwV00Bqw5iWw6kUQH2a+AQkAYqLl6wRSbqo0wwh9L0jDXja4PseCG5NNXyg4ysGuYxzsPsqzXUf7WgqUweKyOlblWglWVS5leXwxYakpEKOhNWZXA3bjTkINQSjIL43tVi8hW7eVbP02nAVb0OHz6x9WheFzqt8QTVW4b+SCBP1CQ7653tM+dJ8hdOh3RA7+Dqv9CNowyS68kvSqG8gsfi7YM6ebbNQBoFhvExz8bdAy0LQHUMEMnGteAitvCE76eRIAxGQyLqsR+n4QBnwnFwxKPFfBRRCEgvZCIMhfF4eCJeX1hUCwqmIZy+OLpNBQnJv2sVoPEWrYGbQSnHkK5aaDaW9nryFbdyl+2axcTUIkKDy0+m5rM4y2+4oStRUOKvDPo0VAZXoIH7mPyKHfYZ/ejULjzN1AetWNpJc/Hz1EMd9MMKYAUKzjOBy4Ew78Krht2LD0miAMLNsejFCRACAmG9d3g/kExmO6Yd/PtQxkp2wggNzSq+n2XCAIug8Odh2lywmm7zWVGYSCymWsqghaCRbE5lIVqpAZD8XwvCx20zOFQGA37Rty/YiRaGXkihf7QoHOhYbioKCtMEamh9DJx1C+g1u5kPSqG8msfCFepSzidMEBIE9raH4mKB48cBckmoMpjVe/CF7wCahaeN4vKQFAjLtxnW44T2tws7lA4E7KUQWjpbWmOd3WLxAc7D5Kd270AUDMjLIgNof5sbksyF3qctezIrUyu6Hoz8uiskmUl0E5aZSXATeLctMoN1PYjpcJ7hcuwXwFaqjtRdtQJpklV5NedSPu7DXjuiT2VFOyAFDM96BhR1AvcOQ+eO8OiNWc98tIABAXjda6EARKNp/A8G/W10LgOkHXwRQaOjRQfvTBsd5TnE42cSbZxOlUE6eTzZxJNuEWfZ62spgbnZ0LBnNYEJtXCAnzo7OlS0GIi2hcAkCxUDlEJmYeAJkaTYyaUoqYHSNmx8h4GVJOqrR1Av3fLDd5SghC5AKBC34m6D7QAy/jcxilopRiXmw282KzBz3maZ/WdBunk025S3NwnWpib8ezJL2+Nc8VilmRahZE57KgbB4LonMK4WBBbC7l9tjGEwshJsgEjqKQACDGJGyGCZthHN8JCga9bOnrBIopBZYNDDOBypChwAuCwyQPC6YymBudzdzobC6p3dDvMa01XU5PUThoyrUgNPNYy5OFoYp5ETNMVaiCylAFlXacylBwqQpVUGEH11WhCipCcapCccqs2LgvCSzEVJX1shzoOsy6qpXMipx/E/1kJwFAXBDbsKkMV47PugPnwzCAUZ7IBoUFL3c9+cKCUqpw0l5XtXLQ4yk3zelUEAhOJ5toy3TQle2hM9tNV7abE70NdDk9hXUSBjKUQWUuGORDQRAcKnLhoYKqcCWV4Soqw5VUWGWEDHNKzfMgxGgl3CTPdBxkT8ezPN2xn2c7j+Bol49u/hteu+ylE314JScBQJSEaZiUh8op02W5qT41OjcrWL7OxNd+YVvx/fy++etxN6awUBQOKG5Z0EFBT/72RRa1IiyPL2Z5fPGI+6W9DF3Zntylm06nm+5sD51OD11ObyE0HOttpDPbTU+2Z9j/FiEjRDwUJx4qJ27HidtlxO0yyu0yKqwyyq0YFVaMcruMuF3e97hVJqs0ikmlI9PFno4DucuzHOk+jo/GVCYrK5bwysU3cemsDTx37raJPtRxIQFAlJRSCluNfZ7zfsEBDRp8ghCRDw6udnNLg16EoYPnExZg6K4I/L7tkAsKuigwDLhddHVBFKBMMEwidoRIrJq5BPdRRu566GpvT3v0ZHvozHTSnemmM9NJV6aLnmxPcHF6CrfPplo43HWM7mw3aW/kxaHKrFhfWLDLKbf6QkJFqJzacDWzwjXMilQzK1Ijsy2KkskX4uZP+E93HOBU4jQAYSPE2qoVvHH5K9lUs5a1lSsKa4KMexHgBJIAICaV/FSno+mXDtYK7wsD+ctFaUUYzvkGhpHkQ0E+MAzaxoDtOjixn+PkPhqmMqkKV1EVrjqv5zm+Q0+2h95s76CgMPDSm+2hNXE6t08vrh4c6OJ2WSEQ1IZrmB2pYVakhlnhICDMjtRQYcdlTgUxiNaaE4lG9rTvZ0/Hs+zpOEBzug2AcquMDdWruanuWjZWr2FV5bIZuVz4zPuJxbSRb22wjf4tDgMDgavdqbkwSf6kNoVObrZhUxOpoeY8C6a01iTdJG2pNlqTLbQkm2lNNtOaaqE11Uprqo0jPSfpyHQNCni2YTMrXE1tJGg9GCok1ISrZ+Qf+GnLMMG0g1n1cisrep7Doc5D7Gndw9Nte9nbtp/u3IRcteEqNlav5bXVa9hYs5ql5Qul+BUJAGIasgwLa8Af+/y64/lQ4PjO1AwF05RSijK7jDK7jEUVi4beSWtcJ017qoWWZBOtyVZaU820pttpS3fQkm7jYPdR/tS8k6w/eCKpMiuGbVjYhoWlLGzDDm4bFraysAyz8JjVb7/gfv524TUGPF5hlzMnUsuc6CwqpVWidBRghnDw6XRSdLkJOrPddGQ66Mx00pHu4HDnYZ5pfabQBVVXXsdz6q5m4+yNbJq1ifmxeaihCn/94utJNjzoIpAAIGYE0zAxMQkX9Sn72sfzPRzfKYQDz/cmtgtBDE8prFCUOaFFzKksCgl+7g+474D20K5DT6aLlnQrremOICBk2unO9uLofAAMWoYc3yu0Ejm+S8ZJ4vhObj+vaD+3337nEjJs5kRmMSdaG4SCwu1ZzInOYk6kdsYvHOVpnx6nl85MFx3Z7sLIlQ4nKE7tzBWmdma66Mx00ls0k2YxS1ksqljETUtvYuOsjWyctZHaaO0Qe57jG39xQa/vAbmwoKygFW4aBgQJAGLGMpSBYRrYZl8Xgta60ELgahfHc8Z/1kNxYQwjV3sR/HdUYagoq6HCX8TyfDDI/2FXKqiRgNx1rmYiv734ehjBMrm54Jj7XXF8h65MF83JZpp7z9KcOE1zoonmVCs7Wp+ifYiui0o7XggDheui2zXhqinVTO3nT+i5kSYduRN6Z+G6h85sV3BSzwajUPwhwraBQUW4gupwNVWRKlZUrwhuh6uoilQValPy98usstK0tigF5hCnRDMMoQpw08FS51522oQBCQBCFFFKYZv2sKEgf5HugynAyI14oLTftJVShW6CYnNjc1lVvapvg9bgZsBN42STtKbbaU630pxqozndVrjdmDzLk23P9JvxEcBSJrMiNcyJ1DI7Uku5XUbEDBcuUTNMxIz0bbPyj0Vyj0eImuExD730fI8up6ffCXzgCT0/pLQr2zPsCR2CoruqUAVV4QoWltexMbKBqkgtVZHq4FJ0co+H4phqkg0XVQrsaHDRetqEAQkAQpzDUKHA135fS4HUFIihKAV2BOwIdqic+ZE48915w06i1Oskc6GgNQgI+et0G890HiThpkh7GZwh6htGYitrUDgYeLENi26nN9fk3k2X002Pkxj6x0IRt8sKM04uKqujsrpvlsnKcCVVuYmjqsJVVEaqsIxwUKw31DfsqWYahYFp8F9DiIvPUAYhM9RvYZ58oaHjOVN79IEoPdMCMw7hODgZcFPB6pdFyu0Y5fYilsWHKYLM8XyPtJ8h7WZIe32XlJfuux7wWDr3WPB4cN3t9NKcbiXtZnC1S9wupzJUwfKKRcEMkHZFbibIOFW5JvjKcCUV4UpM0yLoPjH7ukwucOjplFQcBnw/CANuesqEAQkAQpTIUIWGxcMR860FUmQ4w9nh4OJ7QRBw0uc1tbJpmJQZMcqs2IUfi2GAYRWdvPMn9FxdxTnqIUQRw4BQLLgUhwF36Gm4JwMJAEKMo+GGJHo6qD73tV+4LcWGM4xhBkvBhsqD1gA3FZwsxiMf9jvRW8Ft05KT+3iZImFAAoAQF1m+pSA0xDCwfBDIdyfkw4J0JUxzVii45E8WTio3FO085QsfDTOYJKfft3sxIQaFgVzQmwRhQAKAEJOIZVhYWDCgCDo/7XFx60E+KEiXwjRSfLJwndzJIt2/VSC/xoM54Bu9nOgnP8OAUFlwyYeBCVwgSwKAEFPAcNMeQ9Cl4Gu/UHQ48FK8AqOYQiw7uOh47ttirtBuOlTSi74wMIHkN0mIKS7fpWAz8iqMnu/1W1mxcBmwTQLDJJMfTihEiUkAEGKGyAeF0SgEghECQ742Ib+EsxBiapEAIIQYRCmFqUYfGAqtB1rj4xdqE4brkhBCTDwJAEKIC2Yoo/+89SPkhuLWheLgUNyiUPyYEGJ8SAAQQlxUY21dGKqwUYodhRg7CQBCiEltUOvCCAbWLnjaG7KeIR8WpH5BzGQSAIQQ08b5ti4AhRBQ3P0wKCigB42UkPAgpjoJAEKIGU0phUKNupWhWHF4KL7k6xmKbwsx2UgAEEKIMTqf8FAcBvKjJDzt4ft926VFYeIZykChUEphYAw5+dZ0IQFACCEugn61DMP0UBSPhhjYgtCvqyIXFCZjN0T+5Jm/XRyOio974LZi53r8XO+df9/CiTx3O38s+f8Wg/afYVMpSwAQQohJQimFpc7/z3I+CORrGNDBibO4VmHQdf527mRbOGnnTojFJ8fi7fkTZ/C//vuN9wk0f6zFoUBrXTjJi/MjAUAIIaa4wolacV4FkFNNcctC38YJOphpQCKTEEIIMQNJABBCCCFmIAkAQgghxAwkAUAIIYSYgSQACCGEEDOQBAAhhBBiBhpVAFBK3aSUelYpdVgp9aEhHn+jUurp3OVPSqnNpT9UIYQQQpTKOQOAUsoEvg7cDKwDXq+UWjdgt2PAtVrrTcCngW+V+kCFEEIIUTqjaQG4HDistT6qtc4CPwZeXryD1vpPWuuO3N1HgfrSHqYQQgghSmk0AaAOOFV0vyG3bTjvAH5zIQclhBBCiPE1mqmAh5poccjVGZRS1xEEgOcO8/i7gHcBLFq0aJSHKIQQQohSG00AaAAWFt2vB04P3EkptQn4D+BmrXXbUC+ktf4WufoApVSLUurEeR+xOJdZQOtEH8QMJZ/9xJHPfuLIZz9xVl/Ik0cTAHYAK5VSS4FG4HXAG4p3UEotAn4BvFlrfXA0b6y1nn2exypGQSm1U2t92UQfx0wkn/3Ekc9+4shnP3GUUjsv5PnnDABaa1cp9V7gboJVrL+jtX5GKfXu3OO3AR8DaoFv5FZrcuUXQgghhJi8RrUcsNb6LuCuAdtuK7r9TuCdpT00IYQQQowXmQlw+pE5GCaOfPYTRz77iSOf/cS5oM9eaT1kQb8QQgghpjFpARBCCCFmIAkAU5RSaqFS6j6l1H6l1DNKqf+T216jlPq9UupQ7rp6oo91ulJKmUqpJ5VSv87dl8/+IlBKVSmlfqaUOpD7/b9KPvuLQyn1vtzfm71Kqf9RSkXksx8fSqnvKKWalVJ7i7YN+1krpT6cW6/nWaXUjaN5DwkAU5cL/L3Wei1wJfDXuTUaPgTcq7VeCdybuy/Gx/8B9hfdl8/+4vgK8Fut9RpgM8F/A/nsx5lSqg74/4DLtNYbCEaFvQ757MfLfwE3Ddg25Ged+9v/OmB97jnfyK3jMyIJAFOU1vqM1vqJ3O0egj+CdQTrNHwvt9v3gFdMyAFOc0qpeuDFBJNf5clnP86UUhXANcB/Amits1rrTuSzv1gsIKqUsoAYwaRw8tmPA631g0D7gM3DfdYvB36stc5orY8BhwnW8RmRBIBpQCm1BLgEeAyYq7U+A0FIAOZM4KFNZ18G/gHwi7bJZz/+lgEtwHdz3S//oZQqQz77cae1bgT+BTgJnAG6tNa/Qz77i2m4z/p81+wBJABMeUqpcuDnwN9qrbsn+nhmAqXUS4BmrfWuiT6WGcgCLgW+qbW+BEggTc4XRa6/+eXAUmABUKaUetPEHpXIGfWaPcUkAExhSimb4OT/Q631L3Kbm5RS83OPzweaJ+r4prGrgZcppY4TLI/9fKXUD5DP/mJoABq01o/l7v+MIBDIZz/+XgAc01q3aK0dgunfn4N89hfTcJ/1qNbsGUgCwBSlgjmX/xPYr7X+16KH7gDemrv9VuCXF/vYpjut9Ye11vVa6yUEhTd/0Fq/Cfnsx53W+ixwSimVXwTlemAf8tlfDCeBK5VSsdzfn+sJao/ks794hvus7wBep5QK59btWQk8fq4Xk4mApiil1HOBh4A99PVD/yNBHcBPgUUE/2D/TGs9sJBElIhSajvwfq31S5RStchnP+6UUlsIii9DwFHg7QRfZuSzH2dKqU8CryUYhfQkwRTw5chnX3JKqf8BthOsttgEfBz4X4b5rJVSHwH+nOC/zd9qrX9zzveQACCEEELMPNIFIIQQQsxAEgCEEEKIGUgCgBBCCDEDSQAQQgghZiAJAEIIIcQMJAFACCGEmIEkAAghhBAzkAQAIYQQYgb6/wHsvdpSnMDsJAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 504x288 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"y_min = min([global_results[k][2].min() for k in global_results])\n",
"y_max = max([global_results[k][2].max() for k in global_results])\n",
"\n",
"fig2 = plt.figure(constrained_layout=True, figsize=(7, 4))\n",
"axes = plt.axes()\n",
"axes.set_xlim(2, 100)\n",
"#axes.set_ylim(y_min, y_max)\n",
"for k in global_results:\n",
" plot_with_confidence_intervals(plt, global_results[k][1], global_results[k][2], global_results[k][3], label=k)\n",
" #plt.plot(global_results_train_steps[k][1], global_results_train_steps[k][0], label=k)\n",
"plt.legend(loc=\"upper right\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:prior-fitting]",
"language": "python",
"name": "conda-env-prior-fitting-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.6"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|