diff --git a/docs/examples/neural_gru_ode.ipynb b/docs/examples/neural_gru_ode.ipynb new file mode 100644 index 00000000..4ef1372f --- /dev/null +++ b/docs/examples/neural_gru_ode.ipynb @@ -0,0 +1,309 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3f91f38f", + "metadata": {}, + "source": [ + "# GRU-ODE" + ] + }, + { + "cell_type": "markdown", + "id": "9dc4b631", + "metadata": {}, + "source": [ + "This example trains a [GRU-ODE](https://arxiv.org/abs/1905.12374) (a continuous time GRU-cells network) to predict a global parameter in a synthetic spirals dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "694f0f7e", + "metadata": {}, + "outputs": [], + "source": [ + "import jax\n", + "import jax.numpy as jnp\n", + "import jax.random as jrandom\n", + "import equinox as eqx # https://github.com/patrick-kidger/equinox\n", + "import diffrax\n", + "import optax # https://github.com/deepmind/optax\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "markdown", + "id": "8ec45d1b", + "metadata": {}, + "source": [ + "We use [Equinox](https://github.com/patrick-kidger/equinox) to build neural networks. We use [Optax](https://github.com/deepmind/optax) for optimisers (Adam etc.)" + ] + }, + { + "cell_type": "markdown", + "id": "92207626", + "metadata": {}, + "source": [ + "The dataset contains 10_000 spirals, which were generated using a parameter alpha and some noise. We are trying to predict the alpha for each spiral.\n", + "\n", + "![Spirals Dataset](../imgs/trajectories_and_alpha_distribution.png)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5e216be2", + "metadata": {}, + "outputs": [], + "source": [ + "spirals = jnp.load(\"spirals.npz\")\n", + "\n", + "data_train = spirals[\"xy_train\"]\n", + "data_test = spirals[\"xy_test\"]\n", + "alpha_train = spirals[\"alpha_train\"]\n", + "\n", + "# z-score transform\n", + "data_mean = jnp.mean(data_train, axis=(0, 1), keepdims=True)\n", + "data_std = jnp.std(data_train, axis=(0, 1), keepdims=True)\n", + "alpha_mean = jnp.mean(alpha_train, axis=0, keepdims=True)\n", + "alpha_std = jnp.std(alpha_train, axis=0, keepdims=True)\n", + "\n", + "ts_train = jnp.repeat(jnp.linspace(0, 1, data_train.shape[1])[None, :], data_train.shape[0], axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "1d3d75d1", + "metadata": {}, + "outputs": [], + "source": [ + "class ContinuousGRUField(eqx.Module):\n", + " lin_z: eqx.nn.Linear\n", + " lin_r: eqx.nn.Linear\n", + " lin_g: eqx.nn.Linear\n", + "\n", + " def __init__(self, hidden_size, key):\n", + " k1, k2, k3 = jrandom.split(key, 3)\n", + " self.lin_z = eqx.nn.Linear(hidden_size, hidden_size, key=k1)\n", + " self.lin_r = eqx.nn.Linear(hidden_size, hidden_size, key=k2)\n", + " self.lin_g = eqx.nn.Linear(hidden_size, hidden_size, key=k3)\n", + "\n", + " def __call__(self, t, h, args):\n", + " z = jax.nn.sigmoid(self.lin_z(h))\n", + " r = jax.nn.sigmoid(self.lin_r(h))\n", + " g = jax.nn.tanh(self.lin_g(r * h))\n", + " return (1 - z) * (g - h)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "7757c484", + "metadata": {}, + "outputs": [], + "source": [ + "class GRU_ODE(eqx.Module):\n", + " vector_field: ContinuousGRUField\n", + " observation_update: eqx.nn.GRUCell\n", + " linear_out: eqx.nn.Linear\n", + " hidden_size: int\n", + "\n", + " def __init__(self, in_size, hidden_size, out_size, key):\n", + " k1, k2, k3 = jrandom.split(key, 3)\n", + " self.hidden_size = hidden_size\n", + " \n", + " self.vector_field = ContinuousGRUField(hidden_size, key=k1)\n", + " self.observation_update = eqx.nn.GRUCell(in_size, hidden_size, key=k2)\n", + " self.linear_out = eqx.nn.Linear(hidden_size, out_size, key=k3)\n", + "\n", + " def __call__(self, ts, obs):\n", + " h0 = jnp.zeros(self.hidden_size)\n", + " \n", + " init_carry = (h0, ts[0])\n", + " xs = (ts[1:], obs[1:])\n", + "\n", + " def scan_fn(carry, x):\n", + " h_prev, t_prev = carry\n", + " t_curr, obs_curr = x\n", + " \n", + " sol = diffrax.diffeqsolve(\n", + " diffrax.ODETerm(self.vector_field),\n", + " diffrax.Tsit5(),\n", + " t0=t_prev,\n", + " t1=t_curr,\n", + " dt0=t_curr - t_prev,\n", + " y0=h_prev\n", + " )\n", + " h_ode_evolved = sol.ys[0]\n", + "\n", + " h_next = self.observation_update(obs_curr, h_ode_evolved)\n", + "\n", + " return (h_next, t_curr), h_next\n", + "\n", + " (h_final, _), _ = jax.lax.scan(scan_fn, init_carry, xs)\n", + "\n", + " return self.linear_out(h_final)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "16b86157", + "metadata": {}, + "outputs": [], + "source": [ + "BATCH_SIZE = 64\n", + "\n", + "key = jrandom.PRNGKey(567)\n", + "model = GRU_ODE(in_size=2, hidden_size=16, out_size=1, key=key)\n", + "\n", + "optimizer = optax.adamw(1e-3)\n", + "opt_state = optimizer.init(eqx.filter(model, eqx.is_array))\n", + "\n", + "@eqx.filter_value_and_grad\n", + "def loss_fn(model, batch_ts, batch_obs, batch_y):\n", + " pred_y = jax.vmap(model)(batch_ts, batch_obs)\n", + " return jnp.mean((pred_y - batch_y) ** 2)\n", + "\n", + "@eqx.filter_jit\n", + "def make_step(model, opt_state, batch_ts, batch_obs, batch_y):\n", + " loss, grads = loss_fn(model, batch_ts, batch_obs, batch_y)\n", + " updates, opt_state = optimizer.update(grads, opt_state, model)\n", + " model = eqx.apply_updates(model, updates)\n", + " return model, opt_state, loss\n", + "\n", + "loss_history = []" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "f5fc47fa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch 1, Loss: 0.5017\n", + "Epoch 2, Loss: 0.1869\n", + "Epoch 3, Loss: 0.1166\n", + "Epoch 4, Loss: 0.0840\n", + "Epoch 5, Loss: 0.0655\n", + "Epoch 6, Loss: 0.0525\n", + "Epoch 7, Loss: 0.0429\n", + "Epoch 8, Loss: 0.0356\n", + "Epoch 9, Loss: 0.0296\n", + "Epoch 10, Loss: 0.0252\n" + ] + } + ], + "source": [ + "for epoch in range(10):\n", + " key, subkey = jrandom.split(key)\n", + " perm = jrandom.permutation(subkey, data_train.shape[0])\n", + " epoch_loss = 0.0\n", + " for start in range(0, data_train.shape[0], BATCH_SIZE):\n", + " batch_idx = perm[start : start + BATCH_SIZE]\n", + " batch_ts = ts_train[batch_idx]\n", + " batch_obs = data_train[batch_idx, :, :]\n", + " batch_y = alpha_train[batch_idx]\n", + " \n", + " batch_obs = (batch_obs - data_mean) / data_std\n", + " batch_y = (batch_y - alpha_mean) / alpha_std\n", + "\n", + " model, opt_state, loss = make_step(model, opt_state, batch_ts, batch_obs, batch_y)\n", + " loss_history.append(loss)\n", + " epoch_loss += loss.item() * batch_idx.shape[0]\n", + " epoch_loss /= data_train.shape[0]\n", + " print(f\"Epoch {epoch + 1}, Loss: {epoch_loss:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ee2b775f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUolJREFUeJzt3Xlc1GXiB/DPd27OAeQ+FO9bMVBEK60wO7Z7y1xTY1vbynYt+rllbbqddJrt5mbHurXbodXaqVmGmlokieIt3gLqcAoDA8z5/f0xzBdGDhlkGJj5vF+veb1mvvPMzPOAOh+fUxBFUQQRERGRh8g8XQEiIiLybQwjRERE5FEMI0RERORRDCNERETkUQwjRERE5FEMI0RERORRDCNERETkUQwjRERE5FEKT1egI2w2G86cOYOgoCAIguDp6hAREVEHiKKImpoaxMbGQiZru/+jV4SRM2fOICEhwdPVICIiok4oKipCfHx8m8/3ijASFBQEwN6Y4OBgD9eGiIiIOkKv1yMhIUH6Hm9LrwgjjqGZ4OBghhEiIqJe5kJTLDiBlYiIiDyKYYSIiIg8imGEiIiIPIphhIiIiDyKYYSIiIg8imGEiIiIPIphhIiIiDyKYYSIiIg8imGEiIiIPIphhIiIiDyKYYSIiIg8imGEiIiIPKpXHJTnLiu3ncDJCgPumtgPQ6LaP1GQiIiI3MOne0a+3nMG/8k5hRPlBk9XhYiIyGf5dBjxV8kBAPUmq4drQkRE5Lt8Ooz4Ke2jVPVmhhEiIiJP8ekw4ugZqWPPCBERkccwjACoN1k8XBMiIiLf5dNhxI89I0RERB7n02GEwzRERESe5+NhxD6BtbzW6OGaEBER+S6fDiPJ/UIBANuOlkMURQ/XhoiIyDf5dBi5pG8olHIBVXVmnK6q93R1iIiIfJJPhxGVQoa4ED8AwNnqBg/XhoiIyDf5dBgBAKXc/iOwWDlMQ0RE5Ak+H0bkMgEAYLHZPFwTIiIi3+TzYUQhd4QR9owQERF5AsOIzP4jsHKYhoiIyCMYRjhMQ0RE5FE+H0aa5oywZ4SIiMgTfD6MOFbTWBlGiIiIPMLnw4ijZ8TMOSNEREQe4fNhxDFnxMo5I0RERB7RqTCyfPlyJCYmQqPRIDU1Fbm5ue2Wr6qqwvz58xETEwO1Wo0hQ4Zg3bp1napwV+OcESIiIs9SuPqC1atXIzMzEytWrEBqaiqWLVuG6dOno6CgAJGRkS3Km0wmTJs2DZGRkfjss88QFxeHU6dOISQkpCvqf9E4Z4SIiMizXA4jS5cuxbx585CRkQEAWLFiBdauXYuVK1fisccea1F+5cqVqKysxM8//wylUgkASExMvLhadyHOGSEiIvIsl4ZpTCYT8vLykJ6e3vQGMhnS09ORk5PT6mu++uorpKWlYf78+YiKisKoUaPw/PPPw2q1XlzNuwjnjBAREXmWSz0j5eXlsFqtiIqKcroeFRWFQ4cOtfqa48ePY+PGjZg1axbWrVuHo0eP4oEHHoDZbMaSJUtafY3RaITRaJQe6/V6V6rpEm4HT0RE5FluX01js9kQGRmJt99+G8nJyZgxYwaeeOIJrFixos3XZGVlQavVSreEhAS31U8u46m9REREnuRSGAkPD4dcLkdJSYnT9ZKSEkRHR7f6mpiYGAwZMgRyuVy6Nnz4cOh0OphMplZfs2jRIlRXV0u3oqIiV6rpEgVX0xAREXmUS2FEpVIhOTkZ2dnZ0jWbzYbs7GykpaW1+prJkyfj6NGjsDWbk3H48GHExMRApVK1+hq1Wo3g4GCnm7s4hmk4Z4SIiMgzXB6myczMxDvvvIP3338fBw8exP333w+DwSCtrpkzZw4WLVoklb///vtRWVmJBQsW4PDhw1i7di2ef/55zJ8/v+tacRFUjUt7TRaGESIiIk9weWnvjBkzUFZWhsWLF0On0yEpKQnr16+XJrUWFhZCJmvKOAkJCfjuu+/w8MMPY8yYMYiLi8OCBQvw6KOPdl0rLkKA2v4jqDX2jNU9REREvkYQRbHHT5bQ6/XQarWorq7u8iGbldtO4OlvDuCGsbH4x8xxXfreREREvqyj398+fzZNoKNnpMHs4ZoQERH5JoYRjWOYxuLhmhAREfkmhhHOGSEiIvIohhGpZ4TDNERERJ7AMCLNGeEwDRERkScwjKg5Z4SIiMiTGEYah2nMVhFGC+eNEBERdTefDyMBqqZ93zhUQ0RE1P18PozIZQL8VfZD/KrrOYmViIiou/l8GAGAxD4BAIBPdhR7uCZERES+h2EEwOCoQADAih+PobqOvSNERETdiWEEgFwQpPunq+o9WBMiIiLfwzACwNbsrMDSmgYP1oSIiMj3MIwAaH5ssa6aYYSIiKg7MYwAGB2nle6fYRghIiLqVgwjAOakJUIlt/8odNWcM0JERNSdGEYAqBQyPHfLKADAWfaMEBERdSuGkUaxIX4AGEaIiIi6G8NIo2itBgBwlkt7iYiIuhXDSKPwADUAwGCy8sA8IiKibsQw0shxei8A1PDAPCIiom7DMNJILhMQ0HhgHsMIERFR92EYaSbYTwkAqGng+TRERETdhWGkmaDGoRr2jBAREXUfhpFmQvxUAIBKg8nDNSEiIvIdDCPNRATbV9SU1hg9XBMiIiLfwTDSTGSQI4xw4zMiIqLuwjDSTGSQfeOzMj17RoiIiLoLw0gzTT0jDCNERETdhWGkmchgDtMQERF1N4aRZhzDNOwZISIi6j4MI804hmmq6sw8n4aIiKibMIw0E+KvhEpu/5GU13KvESIiou7AMNKMIAgID7RvfFbOoRoiIqJuwTByHo3SflieyWrzcE2IiIh8A8PIeRRyAQBgtjCMEBERdQeGkfMoG+eMmG2ih2tCRETkGxhGzqNwhBH2jBAREXULhpHzqBqHaSw2hhEiIqLuwDByHoXM/iMxWTlMQ0RE1B0YRs7jmMBq4WoaIiKibsEwch7HpmcW9owQERF1C4aR8zh6RrjPCBERUffoVBhZvnw5EhMTodFokJqaitzc3DbLvvfeexAEwemm0Wg6XWF3U0g9IwwjRERE3cHlMLJ69WpkZmZiyZIl2LlzJ8aOHYvp06ejtLS0zdcEBwfj7Nmz0u3UqVMXVWl3cgzTmDlMQ0RE1C1cDiNLly7FvHnzkJGRgREjRmDFihXw9/fHypUr23yNIAiIjo6WblFRURdVaXdSyBp3YOXSXiIiom7hUhgxmUzIy8tDenp60xvIZEhPT0dOTk6br6utrUW/fv2QkJCAm266Cfv372/3c4xGI/R6vdOtuygVjk3P2DNCRETUHVwKI+Xl5bBarS16NqKioqDT6Vp9zdChQ7Fy5Up8+eWX+OCDD2Cz2TBp0iQUFxe3+TlZWVnQarXSLSEhwZVqXhSlo2eEc0aIiIi6hdtX06SlpWHOnDlISkrClClTsGbNGkREROCtt95q8zWLFi1CdXW1dCsqKnJ3NSVafxUAoLzW2G2fSURE5MsUrhQODw+HXC5HSUmJ0/WSkhJER0d36D2USiXGjRuHo0ePtllGrVZDrVa7UrUuMzAiAABwtLTWI59PRETka1zqGVGpVEhOTkZ2drZ0zWazITs7G2lpaR16D6vVir179yImJsa1mnaTQZGBAICjZbUQRc4bISIicjeXekYAIDMzE3PnzkVKSgomTJiAZcuWwWAwICMjAwAwZ84cxMXFISsrCwDw9NNPY+LEiRg0aBCqqqrw8ssv49SpU/jDH/7QtS3pIgMjAiEIQFWdGZUGE/oEeqaHhoiIyFe4HEZmzJiBsrIyLF68GDqdDklJSVi/fr00qbWwsBAyWVOHy7lz5zBv3jzodDqEhoYiOTkZP//8M0aMGNF1rehCGqUcWj8lwwgREVE3EcReMBah1+uh1WpRXV2N4OBgt3/e5Bc24nRVPb6cPxljE0Lc/nlERETeqKPf3zybphV+KjkAoM5k9XBNiIiIvB/DSCv8G8NIvdni4ZoQERF5P4aRVvgp2TNCRETUXRhGWuHoGXlny3EP14SIiMj7MYy0wnFg7+7iaugbzJ6tDBERkZdjGGmFyWJtdp9n1BAREbkTw0grmu8twjBCRETkXgwjrVh49VDpPsMIERGRezGMtCIxPAAh/koAgNnKMEJERORODCNtUCvsPxoje0aIiIjcimGkDarGMGJizwgREZFbMYy0QSm3/2jM7BkhIiJyK4aRNqjk7BkhIiLqDgwjbZCGadgzQkRE5FYMI22QekYYRoiIiNyKYaQNnMBKRETUPRhG2qD1s+8zUlZj9HBNiIiIvBvDSBuGRgcBAA6erfFwTYiIiLwbw0gbhscEAwAO6fQergkREZF3Yxhpw9Aoe8/IsbJaD9eEiIjIuzGMtCG4cc5Ig9kGm030cG2IiIi8F8NIGxxn0wBcUUNERORODCNtaB5GjGaGESIiIndhGGmDQi6DXCYAAIwWq4drQ0RE5L0YRtrh6B0xchdWIiIit2EYaUdTGGHPCBERkbswjLRDrZADsK+oISIiIvdgGGmHWslhGiIiIndjGGkHh2mIiIjcj2GkHX4qBQCgtsHi4ZoQERF5L4aRdkQHqwEAOn2Dh2tCRETkvRhG2hGj9QMAnK6q93BNiIiIvBfDSDsiguw9I+cMJg/XhIiIyHsxjLTDX2Vf2ltn4gRWIiIid2EYaYcjjNQzjBAREbkNw0g7HKtpDCaupiEiInIXhpF2BLBnhIiIyO0YRtrh1xhGDAwjREREbsMw0o6AxmEa9owQERG5D8NIO/ylnhHOGSEiInIXhpF2+KvtPSNc2ktEROQ+DCPt8Ffae0ZMFhssVp7cS0RE5A6dCiPLly9HYmIiNBoNUlNTkZub26HXrVq1CoIg4Oabb+7Mx3Y7xwRWAKgzs3eEiIjIHVwOI6tXr0ZmZiaWLFmCnTt3YuzYsZg+fTpKS0vbfd3Jkyfxf//3f7jssss6XdnuplbIIJcJADiJlYiIyF1cDiNLly7FvHnzkJGRgREjRmDFihXw9/fHypUr23yN1WrFrFmz8NRTT2HAgAEXVeHuJAiCNFRjMHISKxERkTu4FEZMJhPy8vKQnp7e9AYyGdLT05GTk9Pm655++mlERkbinnvu6dDnGI1G6PV6p5un+Kt5Pg0REZE7uRRGysvLYbVaERUV5XQ9KioKOp2u1dds27YN//rXv/DOO+90+HOysrKg1WqlW0JCgivV7FLhgfaTe0v0DR6rAxERkTdz62qampoazJ49G++88w7Cw8M7/LpFixahurpauhUVFbmxlu2LDfEDAJypqvdYHYiIiLyZwpXC4eHhkMvlKCkpcbpeUlKC6OjoFuWPHTuGkydP4oYbbpCu2Wz2JbIKhQIFBQUYOHBgi9ep1Wqo1WpXquY2cY1hpJhhhIiIyC1c6hlRqVRITk5Gdna2dM1msyE7OxtpaWktyg8bNgx79+5Ffn6+dLvxxhtxxRVXID8/36PDLx0VJ/WMcJiGiIjIHVzqGQGAzMxMzJ07FykpKZgwYQKWLVsGg8GAjIwMAMCcOXMQFxeHrKwsaDQajBo1yun1ISEhANDiek8VF2oPI6fP1Xm4JkRERN7J5TAyY8YMlJWVYfHixdDpdEhKSsL69eulSa2FhYWQybxnY1fHBNaqOrOHa0JEROSdBFEURU9X4kL0ej20Wi2qq6sRHBzcrZ994Iwe1/19KyKC1Pj1ifQLv4CIiIgAdPz723u6MNwksPGwvNoGbnpGRETkDgwjFxCosYeRerOVh+URERG5AcPIBQSomw7LMxi5CysREVFXYxi5ALVCDrXC/mOqruckViIioq7GMNIBRot9eOb/Ptvt4ZoQERF5H4YRF+SeqPR0FYiIiLwOw0gHvHTbGE9XgYiIyGsxjHTANaObzt0przV6sCZERETeh2GkA4I1SoyO0wIAth4p83BtiIiIvAvDSAeNibeHkaOltR6uCRERkXdhGOmggRGBAIDlm45x8zMiIqIuxDDSQQMiAqT7g574FvoG7jlCRETUFRhGOmhYtPMBPznHKjxUEyIiIu/CMNJB0VqN02O5IHioJkRERN6FYcQFz9w8SrrPreGJiIi6BsOIC+4cn4DIIDUAhhEiIqKuwjDiAqVchmkjogAAVXUmD9eGiIjIOzCMuCgq2D53RKdv8HBNiIiIvAPDiItiGieyfrKjGFab6OHaEBER9X4MIy4aEhUk3T9VYfBgTYiIiLwDw4iLHNvCA0CFgfNGiIiILhbDiIsEQcAlfUMAABU8wZeIiOiiMYx0QnigfXnvvtN6iCLnjRAREV0MhpFO+M3YWADAG5uO4pJnNnDuCBER0UVgGOmE34yOke6fqzPjg19OebA2REREvRvDSCfIZM7n0tQ0WDxUEyIiot6PYaQLNJitnq4CERFRr8Uw0km3jIuT7jeYbR6sCRERUe/GMNJJf7txJO6elAgAaLCwZ4SIiKizGEY6SeunRHK/UAAcpiEiIroYDCMXQa2w//g4TENERNR5DCMXQaOUA2DPCBER0cVgGLkIjjBitLBnhIiIqLMYRi6CRukYpmHPCBERUWcxjFwEDtMQERFdPIaRi6BROMIIh2mIiIg6i2HkIkjDNBYrT+8lIiLqJIaRi6BuHKYRRcBkZe8IERFRZzCMXARHzwgAfL37LLLWHUSdiYfmERERuULh6Qr0Ziq5DIJg7xn5v093A7BPan142hAP14yIiKj3YM/IRRAEQZrE6rD61yIP1YaIiKh3Yhi5SM2HagCg0mDyUE2IiIh6p06FkeXLlyMxMREajQapqanIzc1ts+yaNWuQkpKCkJAQBAQEICkpCf/97387XeGeJtRf5fTYZLXBauPKGiIioo5yOYysXr0amZmZWLJkCXbu3ImxY8di+vTpKC0tbbV8WFgYnnjiCeTk5GDPnj3IyMhARkYGvvvuu4uufE8QrdW0uGa0cBM0IiKijnI5jCxduhTz5s1DRkYGRowYgRUrVsDf3x8rV65stfzUqVNxyy23YPjw4Rg4cCAWLFiAMWPGYNu2bRdd+Z7AsQtrc9wEjYiIqONcCiMmkwl5eXlIT09vegOZDOnp6cjJybng60VRRHZ2NgoKCnD55Ze3Wc5oNEKv1zvdeqrZaf1aXGPPCBERUce5FEbKy8thtVoRFRXldD0qKgo6na7N11VXVyMwMBAqlQrXX389/vGPf2DatGltls/KyoJWq5VuCQkJrlSzW10xNLLFNfaMEBERdVy3rKYJCgpCfn4+fv31Vzz33HPIzMzE5s2b2yy/aNEiVFdXS7eiop69XPaz+9Jw2yXxEAT7Yx6cR0RE1HEubXoWHh4OuVyOkpISp+slJSWIjo5u83UymQyDBg0CACQlJeHgwYPIysrC1KlTWy2vVquhVqtdqZpHpSSGISUxDNtPVKD4XD3O1XF5LxERUUe51DOiUqmQnJyM7Oxs6ZrNZkN2djbS0tI6/D42mw1Go9GVj+4VIoLsAeqX45UergkREVHv4fJ28JmZmZg7dy5SUlIwYcIELFu2DAaDARkZGQCAOXPmIC4uDllZWQDs8z9SUlIwcOBAGI1GrFu3Dv/973/x5ptvdm1LeoBJA/tgV2EVis/VeboqREREvYbLYWTGjBkoKyvD4sWLodPpkJSUhPXr10uTWgsLCyGTNXW4GAwGPPDAAyguLoafnx+GDRuGDz74ADNmzOi6VvQQ/foEAAAqalsO04iiCMExqYSIiIgkgiiKPX67UL1eD61Wi+rqagQHB3u6Om3aeKgEv39vB/oEqJD3ZNNqoQJdDWb/azv+dNVgzJ7YcikwERGRN+ro9zfPpulCiY6eEYMJh0tqpOtPf7MfpTVGPPnFPk9VjYiIqMdiGOlCAyICpeW9d69sOq9HAIdniIiI2sIw0sWGRdu7oc5UN6C63gwA8Fe13DKeiIiI7BhGutgbvxsn3d9cYD88sHkYKdDVtHgNERGRL2MY6WIDIwLx2+R4AEDxuXoAgJ+qadHS9GVb8OPhMo/UjYiIqCdiGHEDx+ZnZTX2jd36BKicnl+wale314mIiKinYhhxg4hAexjZVFCK4nN1kJ03f7Wqzow1O4s9UDMiIqKeh2HEDYL9lACAUxV1uPTFTThZYd+RdU5aP6QPt5/y++QX+1BrtHisjkRERD0Fw4gbqBTOP9ZDOj0AQKOU463ZKYgO1sBgsmLCcz94onpEREQ9CsOIG6jkzj/W0sa5I3KZALlMwPj+YQCAOpMVR0tru71+REREPQnDiBuoz+sZqaqz7zeiaJw88ug1Q6XnGEaIiMjXMYy4wfnDNA6KxgME40P9MWVIBACgpsHcbfUiIiLqiRhG3OD8nhEHhbxpWY1jkqu+gZNYiYjItzGMuEFbPSPyZmt8gzT2jdDYM0JERL6OYcQN2h6madYzorH3jNSwZ4SIiHwcw4gbnL+axqF5GAkPtO/KeqpxDxIiIiJfxTDiBs17RlIbl/ECgNFik+47Tvf94WAJymuN3Vc5IiKiHoZhxA2ah5HfjImR7h88q5fupySGtnqdiIjI1zCMuIFjPggAmK0ibhgbCwCYOaGvdF2jlOPSQeEAgFI9e0aIiMh3KS5chFylUcql++fqTHjtjrF44rrhiNZqnMpFNp7ue7LC0K31IyIi6knYM+JmBqMVCrmsRRABgLEJIQCAr3af6eZaERER9RwMI27yUPpgRASp8ftLE9ssc1tyPJRyAacq6vDT0fLuqxwREVEPwjDiJg+lD0Hu41chPtS/zTKBagVS+/cBAOw7Xd1dVSMiIupRGEbcSBCEC5ZJCLOHlebLfomIiHwJw4iHaZT2X0GlwQSDkbuxEhGR72EY8TC1wr7y5r2fTyLl2R9QZ2IgISIi38Iw4mHNT/itN1sx8fls6XFFrRHv/3wStewxISIiL8Z9Rjys+Z4kAKBvsMBmEyGTCfjtihycKDegpsGMB68c7KEaEhERuRd7RjxM3coJv6U1RoiiiBPl9s3QXvn+MKw2sburRkRE1C3YM+Jh5/eMAMDT3+yHn9L5V3OywoCBEYHSY6PFiup6MyKDWm6mRkRE1JswjHhYaz0j6/bqWlwr0TdgYEQgzFYbvtuvw0vrC1BYWYex8VqsvHs8+gSqu6O6REREXY7DNB7Wr0/TpmgjYoIxc0JCq+W+yrdvGb/kq/148KNdKKysAwDsLq7Gc+sOur+iREREbsIw4mEpiWEYERMMABgVF4yH0ofgkr4huH5MDHKfuAr3Xj4AALDq1yIcLa3Fz61sG//rycpurTMREVFXEkRR7PEzI/V6PbRaLaqrqxEcHOzp6nS5WqMF3+49i6lDIxER5DzcUlVnQtLTG6THIf5KVNWZncpMGRKB938/oVvqSkRE1FEd/f5mz0gPEKhW4PaUhBZBBABC/FV44dbR0uPzgwgA7Cmu4mobIiLqtRhGeoE7J/RF5rQhbT5/rs6MilpjN9aIiIio6zCM9BKh/kqnx8Oig/DibaPhr7IvDW4w2w/aqzdZcaSkptvrR0RE1FkMI72E1l/l9Hj1H9MwY3xfaWlwg8UKAHh27QFMe20LPtpe2O11JCIi6gyGkV4ixK+pZ2Th9KHQNj52bJpmbOwZ+bAxhDz++d5uriEREVHnMIz0EgMiAiAT7MM1d09KlK47woijZ6S5XrBQioiIiDuw9hbxof7IfmQqwgJUCFA3/dqkYRqzFYd0eqfXWGwilHKhW+tJRETkKoaRXqR/eECLa+pmwzT5hVVOz1msIlo5+oaIiKhHYRjp5Rw9I3mF5/Dm5mNOz5msNviBaYSIiHq2Ts0ZWb58ORITE6HRaJCamorc3Nw2y77zzju47LLLEBoaitDQUKSnp7dbnlzjmDNyfhABAIvV1t3VISIicpnLYWT16tXIzMzEkiVLsHPnTowdOxbTp09HaWlpq+U3b96MmTNnYtOmTcjJyUFCQgKuvvpqnD59+qIrTy33H2nObOUEViIi6vlcDiNLly7FvHnzkJGRgREjRmDFihXw9/fHypUrWy3/4Ycf4oEHHkBSUhKGDRuGd999FzabDdnZ2RddeQIS+zTNI+kb5o8jz10LjdL+azWzZ4SIiHoBl8KIyWRCXl4e0tPTm95AJkN6ejpycnI69B51dXUwm80ICwtrs4zRaIRer3e6UevGJmil+xmTE6GUy6CUMYwQEVHv4VIYKS8vh9VqRVRUlNP1qKgo6HS6Dr3Ho48+itjYWKdAc76srCxotVrplpCQ4Eo1fcrUIZGYd1l/XDY4HLeOiwcAKBsntVp4eB4REfUC3bqa5oUXXsCqVauwefNmaDSaNsstWrQImZmZ0mO9Xs9A0gaZTMAT149wuqaQ2fcWMVnYM0JERD2fS2EkPDwccrkcJSUlTtdLSkoQHR3d7mtfeeUVvPDCC/jhhx8wZsyYdsuq1Wqo1WpXqkbNKOXsGSEiot7DpWEalUqF5ORkp8mnjsmoaWlpbb7upZdewjPPPIP169cjJSWl87WlDnHsuso5I0RE1Bu4PEyTmZmJuXPnIiUlBRMmTMCyZctgMBiQkZEBAJgzZw7i4uKQlZUFAHjxxRexePFifPTRR0hMTJTmlgQGBiIwMLALm0IOjp4RhhEiIuoNXA4jM2bMQFlZGRYvXgydToekpCSsX79emtRaWFgImaypw+XNN9+EyWTCb3/7W6f3WbJkCf72t79dXO2pVYrGMFJvanl4HhERUU8jiL3gaFe9Xg+tVovq6moEBwd7ujo93qx3f8FPRyvwx8sHYNF1wz1dHSIi8lEd/f7u1Hbw1LNNGhgOACiuqvdwTYiIiC6MB+V5odgQ+7LptXvOQi3PR1yoHx65eqiHa0VERNQ6hhEvFOKnku6v2WU/A+iPUwYiUM1fNxER9TwcpvFC8sZNz5ozcwM0IiLqoRhGvND4xDAMigzEdaOjITTmErONYYSIiHom9tt7IT+VHD9kTgEADHniW5isNli5GysREfVQ7Bnxco4hG4u1Y2Ek90Qlnlt7AA1m7lFCRETdgz0jXs5xaF5Hz6m5460cAECgWokF6YPdVi8iIiIH9ox4OUXjOTVWF+eM7D9T7Y7qEBERtcAw4uXkMsc5Na7NGTE1nmvz0fZC3PrPn7D610KedUNERG7BYRovp5R6Ri4cRmoazNJ9fb0Zoiji8c/3AgB2FlZh7+lqPHvz6E7VI+9UJZ5bexC7iqrwyR/TMD4xrFPvQ0RE3oc9I15O7sKckae+PiDd31VUhcMltU7Pf/BLYafr8daPx7GzsAqiCMz5V26n34eIiLwPw4iXUzae4GvpwBDL6XNNZ9mIIvDU1/tblCmtaXDp889W12PRmj3IO3VOulZvtkLfrBeGiIh8G8OIl3OlZ6TWaHF6/POxCgDAkhtGSNce/GiXS5//+Jq9+Di3CBUGk9P1Al0NzlTVo6zG6NL7ERGR9+GcES+ncGGfkep6e2/F7cnx+DSvWLo+Kk4r3c89UenS558/1BMeqEZ5rRG3r8iRrn3+wCSM6xvq0vsSEZH3YM+Il3Ms7bW0s7TXZhPx+g9HUFhZBwBIHxHl9HxkkBoZkxMBAHEhfh3+bFEUnXpb7kiJx4q7LmlxYN9H2zs/F4WIiHo/hhEv51ja29pqGqtNxJyVuRjw+Dq89sNh6fr4xDBcNzoaADCubwjiQvwwK7UvAMBgsrR4n7Z8kX9a6m359L40vPTbsUhJDEPek+kYFBkoleP8ESIi38ZhGi+nbBymOX+fkaLKOqz86QS2HC5zuv671L4IC1Dhn7OSYbOJEARAEAQEqpUAgJoGC0RRhCC0PBm4OatNxMOrd0uPk5sNw6gVcvz3nglIy9oIAKgzcet5IiJfxp4RL+eYwNq8Z0QU7T0i//7ppFPZf866BM/f0rSPiEwmSKEjUKOQ3ufAWf0FP9fRIwIA/84YD5nMObzEaP3w1uxkAIDB2PHeFiIi8j4MI17OsbQ353i5dG1XURVOlBtalB0ZG9zm+wSo5NJ8kdtX5OB4WW2bZQGgrtlwztQhEW28pz3gGIxWp/BCRES+hWHEyznmeHzwSyEe+WQ3Zv9rOx5alQ8AuGJoBO6a2Fcq669qe9ROEASs/uNEaJQy1JmsWPnTCQDAF7tOY93esy3KO4ZeQv2VbQ7p+KvlAICCkhqMfep7fL37jOsNJCKiXo9hxMvdldpPuv+/ncXYeqRcWjWTNrAPZM2CwvmrXM4XH+qPP1w6AABwzmDGsbJaPLQ6Hw98uBMf5zqviHGEkfYCTsB5zy3fdLQDLSIiIm/DMOLlbr0kDr8ZE9Pqc1HBGjTvs9AoL/zHoX94AABg7d6zmLuyaVv35ZuOovhcnfS4rnEeiL9K3uZ7DYgIcHosu8CkWCIi8k4MI15OEAT8Y+Y4rLw7BT9kXo7cx69CWIAKADA0OgjJjQfWDYgIuOAKGQAI0jT1ZhQ32z6++Fw9rn5ti7RMd23j0M2Zqnq0RSmX4dXbx0qPGyxcVUNE5Iu4tNcHCIKAK4c1bWS2eeFUFFXWYVh0MIZFB6N/nwBEadUdei+10rmn44+XD8CUIRH43bvbUWey4lR5HUbHa/Fh40Zmhgss270tOR4RQWrMWZkLo/nC5+cQEZH3YRjxQcEaJUbGNm3xPjpe205pZ2ZLU2A49Mw10DSGk1Fxwdh3Wo+y2gYAWgyPCcbBs3r8Njn+gu8ZGWwPQg1m9owQEfkihhFyyeRB4egfHoCkhBApiABARKA9UBRW2OeNqBT2EcBrRkZf8D01Cvv7MIwQEfkmhhFyiZ9Kjo2PTGkxv2RUnBabCsrwt68PwCoCpxvnkwRcYIUOACnUNFg4TENE5Is4gZVc1tpE10kDw6X7z3xzAOW1RgAXXi4MNK3isdpEmK0MJEREvoZhhLpEbIim1euBmo73jABocVYOERF5P4YR6hLRWg3OO34GAyICEB/qd8HXqhVNfwz/+8uprq4aERH1cJwzQl1CrZBjzQOT8cvxCtwyLg6leiP6RwRIZ+O0RxAELLlhBJ76+gBqG3hoHhGRr2EYoS6TlBCCpIQQAPbdXV0xJCoIAKRN04iIyHdwmIZ6hGCNEgCgr7fAxFU1REQ+hWGEegTHNvM6fQPGP/cDSmsaPFwjIiLqLgwj1CNEBKmliazV9Wa88l0BN0EjIvIRDCPUIwSoFfjsvklQNC7J+WRHMRZ+tsfDtSIiou7AMEI9xuh4LZ6/dbT0+NBZvQdrQ0RE3YVhhHqUO1IS8PG8iQCAssZdXImIyLsxjFCPMzzGvsy3qs6M6nou9SUi8nYMI9TjaP2USOzjDwBYu+esh2tDRETuxjBCPY4gCLh5XBwA4Kdj5R6uTe9wpKQGl764EaOWfIe/fLbb09UhInJJp8LI8uXLkZiYCI1Gg9TUVOTm5rZZdv/+/bjtttuQmJgIQRCwbNmyztaVfMjkQfZTgNfuOYu/frFXur5u71k88slunK6q91TVehSjxYqblv+Eaa9tQfG5etQaLfhkRzG2HC6DKIqerh4RUYe4vB386tWrkZmZiRUrViA1NRXLli3D9OnTUVBQgMjIyBbl6+rqMGDAANx+++14+OGHu6TS5P3GxodI9z/4pRBnqhogEwT8cLAEAKCQCXjxt2M8VLue40hJLXYXVbW4PmdlLrJuHY2ZE/p2f6WIiFzkcs/I0qVLMW/ePGRkZGDEiBFYsWIF/P39sXLlylbLjx8/Hi+//DLuvPNOqNXqi64w+QaVQoaXmoWNjYdKpSACAKt3FOGcweSJqnU7g9GCHw+XYcZbOThRbmjxHACEB6pxIus6vH5nkvTcu1uPo6yGK5KIqOdzKYyYTCbk5eUhPT296Q1kMqSnpyMnJ6fLKmU0GqHX651u5HvuSEnAkeeuRYy29UP3Hv98b6vXvcnHuYUY89T3mLsyF9tPVOKZbw44PW8w2cNIbIgGgiDgpqQ4LLp2GADgWJkB05dt4Vk/RNTjuRRGysvLYbVaERUV5XQ9KioKOp2uyyqVlZUFrVYr3RISErrsval3Ucpl2PKXK/Dtgsvw2oyxTs8dLa31UK3cTxRFVNebsSq3EFZb09yPbUfLnXqEao32LfP9VXLpmkwQpPuVBhPu+td2vLj+EDYcKGFPCRH1SD1yNc2iRYtQXV0t3YqKijxdJfIgpVyG4THBuGlsHOak9cN9UwYCAM5Wt36Y3s/Hynv1l+6JcgPGPPU9xj71PXYXVwMAHr/O3tthstjw3LqDUtm6xmGaQHXT9K/68870yT1RiTc3H8O8/+zAta9vhcXKnhIi6llcCiPh4eGQy+UoKSlxul5SUoLo6Oguq5RarUZwcLDTjUgmE/D0TaPw56sGQRCAWqMFZ85bVfPDgRL87p3tuOWfP3molhfvX9uOo6bB4nRt2ohopA+3TxBv3iPkmEfjr2oKI7enxDuFEwC4brT972d5rRHltb4x14aIeg+XwohKpUJycjKys7OlazabDdnZ2UhLS+vyyhG1xl+lwLiEEADApoJSAMC+09XIPliC/+0sBgAUn+t9S39FUcTv3vkFH/xS2OK5UH8lHrhiEAAgv6gKp6vqsbmgFD8ctLc/xF8plY3R+iHvyXT8LtW+kubjeRPxz1nJiAq2TyDvzb1GROSdXF7am5mZiblz5yIlJQUTJkzAsmXLYDAYkJGRAQCYM2cO4uLikJWVBcA+6fXAgQPS/dOnTyM/Px+BgYEYNGhQFzaFfMmVwyKxs7AKT3y+DzYRePKLfZ6u0kUrrzXh52MVAIB+ffyx6t6J+M3ftyHYT4lgjRIRgU2r0Sa/sNHptQ9e6fx3Sa2Q45mbRuFPVw5CjNYPABDqr0KJ3ojXs4/g3bkpbm4NEVHHuRxGZsyYgbKyMixevBg6nQ5JSUlYv369NKm1sLAQMllTh8uZM2cwbtw46fErr7yCV155BVOmTMHmzZsvvgXkk64eGY1Xvj8MwDuCCABU1zcNn2x8ZCrkMgFbH70CAgTIZALiQvxw+ZAIbDlcBgDwU8qROiAMd6X2Q2RQyxVHcpkgBRGgaV7JDwdL8Lev9mPRdcOgVshbvI6IqLsJYi/YplGv10Or1aK6uprzR0jy5uZjeHH9oTafP5F1HYRmK0t6ul9PVuL2FTlI7OOPzQuvaLdsdb0ZQWoFZLKOt6+6zoyU5zbAbLX/lX/mppGYnZZ4MVUmImpXR7+/e+RqGqKOuH/qQBx//jocfPoarL53Yovn81vZmbQne++nkwAArb/qgmW1fkqXgoj9fZXY8cQ03DA2FgDw5Jf7se90tcv1JCLqagwj1KvJZAL8VHKkDuiD7x++HK/e3rQXyaZDpR6smWseWrULa/faTyjuG+bvts/R+itx35QB0uN/bj7qts8iIuoohhHyGkOignBbcjziQuzzJDSq3jMf4vsD9iW6of5KPHHdcLd+1shYLRZOHwoAqOAyXyLqARhGyOtc1bgfR73JeoGSnvfJjiIM+eu3qGus69ZHr0R0G9vfd6WhUUEAgAZuFU9EPYDLq2mIejq/xh6ROjeFEVEUUVpjxLHSWpTVGlGgq8Gfrhwsfe6F/Hy0HD8fq8BD6YPxv7xip7Njzt+szF00SntdG3pBYCMi78cwQl7HX2n/Y+2uMPLMNwex8qcTTtf+ufkYbr0kDq/8dqw0sfSvX+zF+n0leOTqIZg5wb4BWWlNA3737nYAwBubnOdrpA3o45b6tsZPZe8UbbC0/jMqPlcHXXUDLukb6vJEWV9kttqglLOjmaizGEbI6zgOjWswuxZGvt59BqcqDLgtOR5RQRrpS/ibPWfw7tYTmDo0Ag+lD8HPx8pbff2anafRYLbin7OSIYqitJPq8k1HpTDyze6zrb722ZtH4dZL4lyq78Vw7C/S2lBWaU0Drn19K2oaLJh/xUAsnD6s2+rVG5XVGDHttR8xODIQRosNNyfF4feX9vd0tYh6FYYR8jqO4ZLPd53GazOS2i2rq27A377aj/X7m06ddmym5q+SY8qQCHy7z/5cflEVEkL9Udq4nfraP1+KYI0Sr/1wGGt2ngYAbDtiDyrND6srPlePV74rQP/wABRW1rVaj+ExQU7ny7ib42dUWmPE57uKMX1ktPT5249XSmfj7Cqs6rY69Vbr9p5FVZ0Zv548BwDYU1yNEn0DFrl5IjKRN2G/InmdgRGB0v2quvZXi7y5+ahTEGmuzmSVgojDI5/uRqXB/p7RwRokhPlj6R1J2P/UdACAvsGC6nozztWZnV73xqajeOTT3Xjv55MAgN8mxzs975jD0V2af97Dq3fjnS1Nw076hqa6O9rqTlabiHn/2YFJWdm46Y1tvW7vE79WfndvbTmOqS9vQoGuxgM1Iup92DNCXidtYNPcix8Pl+GmpLaHPxy9HBMSw3DZ4HDcckkcgv2U+DL/DD7bUYTQABU2F5S1eF10sAahzTYnC1Ar0CdAhQqDCZsLSvGXz/a0W8dh0UFOj7s9jCic/x9ytKzpJGB9fdOJwYd0Nfj1ZCVGxga7redm+4kKbGhc2nymugFf7z6DUXFat3yWOzQPb3KZAJsoQhSBkxV1+H6/DkPP+10TUUsMI+SVkhJCkF9Uhax1h9oNI47hlDvGJzj1Vsye2A+zJ/YDYF89891+He77YKf0/Kb/m9piYmd8mD8qDCYsWJXf4nMWXDUYlw+JwK8nK6FRyHDD2Fg8u/ag9Hxr/7t2p1B/FVL6hWLHKfvQwte7z+Dx64YhRuvn9OUKALevyMHgyEDMu3wAEvsEYNuRMthE4N4pAxCsUbb29h128Kwev3/vV6drNUZLG6V7nnqTVfo9zk3rh7/dOBKiCDzw4U6s369DeS1PSCbqCA7TkFe6p3ECYWlNA2w2EXmnKrGpoBQWq/O+Go4JnO2FAUEQnHpBhkQFtrqMt995O6cm9wuV7o+K0yK5XyjumzIQd0/uj2A/5y/x7u4ZkckEfHb/JHz14GTpWlrWRhSfq4O+vimMOCYDHymtxV8+24M73srB3zcexRubjuLGf3RuSEUURewqPId3tx7Hta9vRYPZ/jtx/LxqG3pPGDmk00v3Jw8KhyDYDzWcOCAMAPB+zikYelG4IvIU9oyQV7pmVDQEAbCJwL+2ncBz65p6IZ6+aSQu6RuKUXFaacWNRtl+Lh+bEIJLB4XDYLLgT1cOarXMvMsG4Hh5LQ6c0UME8MxNoxCj1SC/uApTBkc4lVXKZVDKBenQOn8P7RY7Ok6Lx64dhhe+tR84mH2wFCfKDQCArFtHY+aEvsj4dy42NQ5VyRp/poB9GGLZD4fx7tzxLn3mpoJS/P69HdLjYdFBmJ3WD0qZDHmnzqG2h395n6owoKbBAqVchu0nKgEAw2OCcfXIaKlM/2bzln44WNJu7xwRMYyQl1LKZQjzt8/haB5EAGDxl/sBAGPjtdhdbP+f/YWGSTRKOT74Q2q7ZUbHa/HNny6D2WrDuToTIoPsO6leMTSy1fLzrxiEf2w8ivlTB3Z7z4iDIAi4b8pA7Cmuwrq9OlTVmXHwrP1/+6Mb522svHs8DCYrAtUKiI3zIV7+vgBvbj6GHw6WoqbBjKAODtc0mK14d6vzHi1/vmowrhsdg3WNZ/P05J6R7/frcO9/81pcV5w3ZHfZoHApuHHLfaILYxghrzU4KhAVxyulx9NGREkTJQFIQQTo2nNslHKZFETa81D6EDx4xSAoesBmWf36BAAAztWZUN04TBMZpAZgDyyOnWEFQYAg2IPUm5uPAQB+PVmJK4dFOb1fzrEKPLfuAM5UNUAuE1BntCBQo0B5rQlWR9cK7F/ijtDj+Izqeuc5Kz3J/jP2oOanlMNfJUdF42qjKUOce75kMgEzxifg49wiaZk0EbWNYYS81oq7kvHvn07i9ewjAICMSYl4c9YlWPnTCTy/7pBT2e6eQOrQE4IIYD+gD4C09BhAu70dgWoFhkUH4ZCuBjUNFoiiCEFo6h146btD2Hda7/QaQ7MN1oI1Crx8+1iMidciRms/2LB/uD0QnSg3wGixShuz9SSOyb0ZkxPxl2uGQRRF7C6ubrE6CoA0ubemoeeGK6KegmGEvFaIvwq/TY6XwkiwnxIKuX0ly+aCMvx8rEIqGxagauttfIIjEDgo5cIF59HEh/rjkK4GB87q8dj/9iJGq8HD04ZgfGKYtFlaUkIIFt8wAv4qOarrzPBTyTEyVgsBaLkaKdQPQWoFaowWFFXWY1BkYMsP9TBHr41jArIgCEhKCGm1bJDG/s9rTYMFr204jC1HyvDA1EGYNiKq1fLUPoPRgs0FZTjXuHfQ5YMj0LeP/wVeRb0Fwwh5tRD/pv/dqxv31ojR+uGjeRNhstjw3X4d5DIBUcHuPym3J5s+MhoLpw/Fy98VSNea93S0JlBt77n45Nci1JutOF5uwJ8+3iU9L5cJWHXvxA7PhxEEAVp/JWqMlhbLiz1JFEWcrW5AZJAaGw+VAkCHljSHNK7AWr2jSLr2xsYjDCOd9OzaA/g4t+lnOSw6COsfutyDNaKuxDBCXq35KbixIc7/+1c17vdB9p/F/CsG4Y2NR1Fvtnaopyig8WdraONAwmHRQS5PzLV/ydc7LS92h1MVBgRrlAg9r52iKGLHqXM4VVEHk8UGmQB8tfuMUy8aAGj9LhxGrh4ZhaUbDjvtYru7uBoPfrQTsyf2w/jEMB5CeAE1DWas3XMWJyvqkH2w1Om5Q7oabDpUiiuGtT5BnHoXhhHyaoIgYOeT02Cx2qQvT2rb8lnj8M2es/jtJfEXLOv4eZos9n1C/nj5AAD2rdAB4OH0IS5/frBf09BGV9lwoASf7CiCzSbCaLGhwmDCwbN6jInX4qsHL3Uq+1leMRZeYPdcALh8SPgFy0QGabD1L1fgkE6PwyW1WLRmLwDgmz1n8c2es3jullH4XeMBiuv36VBjtGDa8KgWAcmXPfnFPnyRf0Z6LAhAzmNX4cY3tqG0xoiM935F/uJpUi9Uc1abCJso8jTlXoL/OpPX8/X5IK64clhUi5UxbUlsXIHjEBqgQoy2abgrvRPDEY7hj6e+PgCbKCJG64ewAJU0f6TBbIVKLutwj0KD2YqHV+e3unfJnuJqLP2+ALdeEo/Exsmz3+yxLy9WKWSYOiQCNlGE2Srix8Nl8FfJ7Uu870nt8FLmALUCyf3CkNwvDCNigvH5rtPSJOEnPt+HJz7f51T+pqRYvH7nOABA3qlz+Di3EJcPicANY2Lw/LqD0Pop8eCVgzv02b3Z6ap6/Hqi0imI3HNpf4zrG4JorQZv3pWM2978GQBQWFmHEH8V9A1mLN94FAq5gFmp/XDP+ztgNFvx3cOXM5D0AgwjRNQpt6fEIzZEg+fXHYTBaMXUoREYEhmESoMJ4xPDOvWew2KC8f2BEpTXGp221Y/RaiCXCThTVQ+NUo7+4QFQymVQyWVITgzFo9cMa/X9ymqMUhDJunU0/JRyCAKk9/77xqP4+8ajOPTMNdAo5ThdVQ8AeO/u8Zg06MK9H64YmxCCsQkhuOfS/rju9a2tbnt/rNkZQU9/vR+7i6vxWV4x/txsLs59Uwb2mFVYF8titeH7AyU4U1WPWqMFRosNJ8oMLQ6vXP/QZRgWHSw9Tu4XKh35sPDTPYjSavDL8Qqpl275pmNS2TNV9dLSdeq5GEaIqFOUchmmDo3E1PM2dcuY3L/T7/nA1IEIVMvx68lzOFFuQIPZiuJz9Thb3SCVqTNZpf0+ACD3ZCXmpPVrsSIIaFqKGxGkxszGIREAeD37CI6XGaTHL64/hAazDUdL7WEgMljd6TZcSEKYP3Y8mQ6D0QqbKKKwsg6/HK/AS+sLsO+0HgtW7UKQRuG0D05zNQ0WrxnK+XafzmnSc2viQvwwKKLlyqrRcVrkF1WhoKQGBSVtn4789pbjuHN8X4yO7z2HL/oihhEi6jE0SjnuvXwg7m22SKK0pgEny+tQZ7Ig1F8Fg8kCk8UGi1XEws9241ydGRW1JpytbsCKzcfw+HXDpWEXx9wTxzJbh7dnp+DzXcV4Z8sJmKw2/Punk9JzggBEunl1lVohl/ZRCQ9UIyHUHy+tt69k+rLZ0AQAfDxvIhZ+thvF5+y9NvoGs9eEkcLKOgDAgPAATBzYBxqFHCqFDAPCA5A2sA+q683oHx7Qak/QX38zHNeMioa+3ow6kxUGkwWDIgPRYLbiWKlB2nn5w+2F+Ci3EJsemSr9uaCeh2GEiHq0yCBNmzvaRgVr7GHEYMK893fAZLWhqs6MT+5LA9A8jDjP8RgUGYiF04fhmpExuOtf26X9Qy4bHI6rR0Rd9GnErgoPVGFgRACONfbWPDB1IAQBmDQwHGkD++DL+ZOR/OwPAICv8s8gOTEUx0prcaa6AY9MG9Jrh23ONa40Sh8RhcevG97i+YR2XqtWyDG5jaG0K4cBffv4460fj2FnYRVE0X6oIcNIz8UwQkS9lmNy8tyVudK13JOVOFxSg8GRgbj/A/s5MsGa1v+pGx2vxe4lV8NmEyEIF95bxV0EQcCbdyVj/T4dLhscjnF9Q52e7xOolsLKqxsOOz2X0i8UVw3vnXuXOOboNN8PqKtMHxmN6SOj8aePd+Hr3Wfw51X5OPR0tMeXU9ca7T17cpmAYI3CY3/mehqGESLqtUbEBLfYAwSwL9H905WDYGk8B2dUXPvzBTz9BQUAQ6KCMCSq5bbyDhP698GxMgPCA1UI8VdJ81vu+yAPh565FvIe0IaO+GLXaXyZfxoWm4itR8oBAKGtLM3tKhP6h+Hr3Wdgstiw8LM9ePWOsW2WPVtdj7d+PI7hMUFICPXHJf1Cu/QQyyc+34sPtxdKj6cMicD7v5/QZe/fmzGMEFGv9di1wzBpUB+cqWrAxAF98P0BHV5aX4CDZ/UorTFK5dpabdObPH/LKDxy9RCEB9on1059eRNOVtTBbBWxp7iqRW9KT7R801GnXX4B+7lIU4dGtPGKizd7Yj9sPVyG7w+UYPuJihbnKDmYLDakZW10unbn+AS8cNuYC36G1SZ2KAyePx/ox8Nl6L9oLfyVcoQHqfHJH9N8djdohhEi6rUUcpnTvijF5+zLP7ceKcdVr/4IwD450hsIgiAFEcDek3Kywj4BtKLW1NbLepRNh5p2UX3xttGQy2S4fHC42ycMPzxtCL4/UILic/W46tUfMWtiP4QHqnD96Bhpvs3x8toWr1u39yyybh3d7lDKiXIDbvjHNlw9MgqPXTsMAgSE+itbzONpMFulZea7npyGBz/eiZ+OVkAU7bsYGyrsq6puSoq7qLaKooh/bDyKQzo9lHIZlHIZLhscftHv624MI0TkNZp/WTv0xAP3usKSG0fi+wMlACAdHnchoijii/zTOF5mgCAIUMkF3DwuDvGh7j9w7sAZPXacOgcA+Oy+NKR0ci+azohuFnaOlxvwzDcHAAD/23ka/2kcJqlsFui2LLwCl7+8CfoGC6589UfMSu2LaSOiWt2v5IcDJag1WrBm52ms2XkagP0Yiv/cMwGXNOutyjluH05UyASE+Cvx77sn4FhZLRQyAVnfHsLGQ6VdEiqPldVi6Xnzij7LK0a9yYqbkuLgp+p5p2EDDCNE5EX6BDrPPfjr9cNxe3J7azJ6r7gQP9ycFIsv8s/g7S3HUVZrxL2XDcBbW47ji12nUVZrRHSwBgMiArDo2uFICPPHq98fxhubjjq9zyvfH8Yj04ZgeEwwLh0c3qVzJAD7xmZHSmtx3d+3Ste6eygixF+JG8fG4qvdZzBxQBgsVvsZRFsOl6FU3wBBEKQN5yb0D0NCmB/GxGuxp7gaJ8oNeHbtQTy79iCGRgWh3myFyWKDyWqD0WxFQ+NGa4B9Wbgo2iepLvvhCKYNj4RaIcepSoO0EVt4oNoeBBUChsfYe/IcOxd/8MspFFbWIb+oCkkJIQjxV0Ipl0GjlOOGMTEX7EEqrWnAu1tPAAASwvwwNy0R72w9jhK9EY+t2YsKgwnzrxjU5T/frsAwQkReIzpYg5uSYmEwWrH4NyO8/oj5vo3/Uz9SWouX1hcgOljjNCejqs6MQ7oarNurw8LpQ6UgolbIcHtKPDYcKEGJ3iit0BkZG4y/Xj8CoQFKDI0KuuiVHgfP6nHHihyn3WajgtUtDq10N0EQ8PeZ4/D3meOka/0XrYUoAhOez3YqG+avgiAI+OKByXh1QwEOnNFjU0EZALS5uZpSLuC9jAmYPCgcn/xahL/8bw+2HC7DlsNlLco+cnXLM5vqzfbDJo+XG3C83L68O7+oyqnMez+fwN/vHIfwQDVKa4yoN1lhtFiRGB6AgRGBEEURtyz/WVqhlNIvDH+4bADCAlTI/GS3/f2bbfTX0zCMEJHXEARBOtvFF8y7rD/iQ/3w6vcFKNEbseSr/QCAyCA1ls1IwubDZXi78eDC5iElZ9FVCAtQIWNyf7y79QQ2HNChvNaE/Wf0mPnOLwCA+6cO7NDE30qDCQajBTZRRJBG6XQW1E9Hy52CSP/wAHy74LIesfInVusnfXEDgJ9SDq2fEjePs8+tkMkELJw+DGarDZc8vUFqx3/vmYBQfxVUCvtxBCqFDMF+SumE8BuTYnGsrBY6fQNMFhuMFhs2Ns6VWTh9KG5PadlT9/vJ/aUhHoe7JyXCbLVhZ2EVDp7Vo6iyHrf88+cWr9UoZch57CqcqDBI7RmbECL1gNx6STwsVhF/+d8eVBqaJnUbLVbsLa5Gn0A1wgJUHTqJ2p0YRoiIeqkgjRJ3pCQg7+Q5rN5RJG3y9sjVQzBpkH3DtJR+ofj5WAVOlBsQG+KH+6YMkALDwIhAZN06GktuGIH7P8hDYWWdtPHahgMlGBIViL3FekQGq5ExOVHaNdZhT3EVbl7+ExpXUMNPKceWv1yBiCA1zhlM0tBH+vAo/H5yIsb3D+sxh9aFBiilL+8XbxuNGeP7tlpOKZfh43snYuuRcoyJ17a50ZqDRinHovM2cPv1ZCW+3avDPZe2flTCqDgtPvpDKr7afQY2UcQdKQnSnBqrTcTDq/Oxu7gKdSYrymqMCFIrEBfqh5MVBjSYbfh812k83TgPZkB4AL6cP9np/R2/700FZbj3PzugUcphMFqQ3WxC8Q1jY7Hw6qEe600URFEUPfLJLtDr9dBqtaiurkZwcPCFX0BE5ENMFhv2nq6G2Ng7MSQqsNNDLId0elyzbGuL66/fmdRiRca7W4/j2bUHna7dNbEvCivrnYYonvzNiDa/iD1l/kc7sbbxlOavH7y015xdU2u0wF8ph0wmYO7KXPx43lDQ0jvG4tZL4p2uHS+rxZWNq8vas+aBSU6TbrtCR7+/2TNCRNTLqRQyJPfrmi+RgRGBGB2nxd7Tzgf1LViVj8KKOqgUMqgVMlw1PAr//eUUAODeywegvMaINbtO44Nfmjb1kgn2JcjTeuAOsc/cNArpwyMRFazpNUEEgDQcBAB3T05EWY0Rtsa9U24cG9siiADAgIhAfP7AJOw7XY0DZ/X4OLdIeu7zBybhl+OVKK81Iq6b5/I0x54RIiJyIooijBYblHIZ/pNzEk99faDd8s/dMgqXDYrAs2sPIOdYBUxWG25KisVLv217t1PynPd/PomCxiMTLuaU7Y5gzwgREXWKIAjSEt9bL4nHsbJanKszI0AlR63RgnV7dVLZvmH+uCkpDoFqBd6ek9LmDqfUc8ydlOjpKrTAMEJERG3S+inx7M2jna49+80BvLvNvp/FW7OTnYYOGESoMxhGiIjIJX+6ajD6BKoRFayWNu4iuhgMI0RE5BKtnxL3Tx3o6WqQF+kZC76JiIjIZ3UqjCxfvhyJiYnQaDRITU1Fbm5uu+U//fRTDBs2DBqNBqNHj8a6des6VVkiIiLyPi6HkdWrVyMzMxNLlizBzp07MXbsWEyfPh2lpaWtlv/5558xc+ZM3HPPPdi1axduvvlm3Hzzzdi3b99FV56IiIh6P5f3GUlNTcX48ePxxhtvAABsNhsSEhLwpz/9CY899liL8jNmzIDBYMA333wjXZs4cSKSkpKwYsWKDn0m9xkhIiLqfTr6/e1Sz4jJZEJeXh7S09Ob3kAmQ3p6OnJyclp9TU5OjlN5AJg+fXqb5QHAaDRCr9c73YiIiMg7uRRGysvLYbVaERXlvLVvVFQUdDpdq6/R6XQulQeArKwsaLVa6ZaQ0PKUQyIiIvIOPXI1zaJFi1BdXS3dioqKLvwiIiIi6pVc2mckPDwccrkcJSUlTtdLSkoQHR3d6muio6NdKg8AarUaarXalaoRERFRL+VSz4hKpUJycjKys7OlazabDdnZ2UhLS2v1NWlpaU7lAWDDhg1tliciIiLf4vIOrJmZmZg7dy5SUlIwYcIELFu2DAaDARkZGQCAOXPmIC4uDllZWQCABQsWYMqUKXj11Vdx/fXXY9WqVdixYwfefvvtrm0JERER9Uouh5EZM2agrKwMixcvhk6nQ1JSEtavXy9NUi0sLIRM1tThMmnSJHz00Uf461//iscffxyDBw/GF198gVGjRnVdK4iIiKjXcnmfEU/gPiNERES9j1v2GSEiIiLqar3i1F5H5w03PyMiIuo9HN/bFxqE6RVhpKamBgC4+RkREVEvVFNTA61W2+bzvWLOiM1mw5kzZxAUFARBELrsffV6PRISElBUVORTc1F8sd1ss2+0GfDNdrPNvtFmoPe1WxRF1NTUIDY21mlxy/l6Rc+ITCZDfHy8294/ODi4V/xSu5ovtptt9h2+2G622Xf0pna31yPiwAmsRERE5FEMI0RERORRPh1G1Go1lixZ4nPn4Phiu9lm3+GL7WabfYe3trtXTGAlIiIi7+XTPSNERETkeQwjRERE5FEMI0RERORRDCNERETkUT4dRpYvX47ExERoNBqkpqYiNzfX01XqlKysLIwfPx5BQUGIjIzEzTffjIKCAqcyDQ0NmD9/Pvr06YPAwEDcdtttKCkpcSpTWFiI66+/Hv7+/oiMjMTChQthsVi6sykX5YUXXoAgCHjooYeka97Y7tOnT+Ouu+5Cnz594Ofnh9GjR2PHjh3S86IoYvHixYiJiYGfnx/S09Nx5MgRp/eorKzErFmzEBwcjJCQENxzzz2ora3t7qZ0iNVqxZNPPon+/fvDz88PAwcOxDPPPON01oU3tHnLli244YYbEBsbC0EQ8MUXXzg931Vt3LNnDy677DJoNBokJCTgpZdecnfT2tRem81mMx599FGMHj0aAQEBiI2NxZw5c3DmzBmn9+htbQYu/Ltu7r777oMgCFi2bJnT9d7Y7naJPmrVqlWiSqUSV65cKe7fv1+cN2+eGBISIpaUlHi6ai6bPn26+O9//1vct2+fmJ+fL1533XVi3759xdraWqnMfffdJyYkJIjZ2dnijh07xIkTJ4qTJk2SnrdYLOKoUaPE9PR0cdeuXeK6devE8PBwcdGiRZ5okstyc3PFxMREccyYMeKCBQuk697W7srKSrFfv37i3XffLW7fvl08fvy4+N1334lHjx6VyrzwwguiVqsVv/jiC3H37t3ijTfeKPbv31+sr6+XylxzzTXi2LFjxV9++UXcunWrOGjQIHHmzJmeaNIFPffcc2KfPn3Eb775Rjxx4oT46aefioGBgeLrr78ulfGGNq9bt0584oknxDVr1ogAxM8//9zp+a5oY3V1tRgVFSXOmjVL3Ldvn/jxxx+Lfn5+4ltvvdVdzXTSXpurqqrE9PR0cfXq1eKhQ4fEnJwcccKECWJycrLTe/S2NovihX/XDmvWrBHHjh0rxsbGiq+99prTc72x3e3x2TAyYcIEcf78+dJjq9UqxsbGillZWR6sVdcoLS0VAYg//vijKIr2v9RKpVL89NNPpTIHDx4UAYg5OTmiKNr/cshkMlGn00ll3nzzTTE4OFg0Go3d2wAX1dTUiIMHDxY3bNggTpkyRQoj3tjuRx99VLz00kvbfN5ms4nR0dHiyy+/LF2rqqoS1Wq1+PHHH4uiKIoHDhwQAYi//vqrVObbb78VBUEQT58+7b7Kd9L1118v/v73v3e6duutt4qzZs0SRdE723z+F1RXtfGf//ynGBoa6vRn+9FHHxWHDh3q5hZdWHtfyg65ubkiAPHUqVOiKPb+Noti2+0uLi4W4+LixH379on9+vVzCiPe0O7z+eQwjclkQl5eHtLT06VrMpkM6enpyMnJ8WDNukZ1dTUAICwsDACQl5cHs9ns1N5hw4ahb9++UntzcnIwevRoREVFSWWmT58OvV6P/fv3d2PtXTd//nxcf/31Tu0DvLPdX331FVJSUnD77bcjMjIS48aNwzvvvCM9f+LECeh0Oqc2a7VapKamOrU5JCQEKSkpUpn09HTIZDJs3769+xrTQZMmTUJ2djYOHz4MANi9eze2bduGa6+9FoB3tvl8XdXGnJwcXH755VCpVFKZ6dOno6CgAOfOneum1nRedXU1BEFASEgIAO9ts81mw+zZs7Fw4UKMHDmyxfPe2G6fDCPl5eWwWq1OX0AAEBUVBZ1O56FadQ2bzYaHHnoIkydPxqhRowAAOp0OKpVK+gvs0Ly9Op2u1Z+H47meatWqVdi5cyeysrJaPOeN7T5+/DjefPNNDB48GN999x3uv/9+/PnPf8b7778PoKnO7f3Z1ul0iIyMdHpeoVAgLCysR7b5sccew5133olhw4ZBqVRi3LhxeOihhzBr1iwA3tnm83VVG3vbn/fmGhoa8Oijj2LmzJnSAXHe2uYXX3wRCoUCf/7zn1t93hvb3StO7aWOmz9/Pvbt24dt27Z5uipuV1RUhAULFmDDhg3QaDSerk63sNlsSElJwfPPPw8AGDduHPbt24cVK1Zg7ty5Hq6de3zyySf48MMP8dFHH2HkyJHIz8/HQw89hNjYWK9tMzkzm8244447IIoi3nzzTU9Xx63y8vLw+uuvY+fOnRAEwdPV6TY+2TMSHh4OuVzeYlVFSUkJoqOjPVSri/fggw/im2++waZNmxAfHy9dj46OhslkQlVVlVP55u2Njo5u9efheK4nysvLQ2lpKS655BIoFAooFAr8+OOP+Pvf/w6FQoGoqCiva3dMTAxGjBjhdG348OEoLCwE0FTn9v5sR0dHo7S01Ol5i8WCysrKHtnmhQsXSr0jo0ePxuzZs/Hwww9LvWHe2ObzdVUbe9ufd6ApiJw6dQobNmyQekUA72zz1q1bUVpair59+0r/rp06dQqPPPIIEhMTAXhnu30yjKhUKiQnJyM7O1u6ZrPZkJ2djbS0NA/WrHNEUcSDDz6Izz//HBs3bkT//v2dnk9OToZSqXRqb0FBAQoLC6X2pqWlYe/evU5/wB1/8c//8usprrrqKuzduxf5+fnSLSUlBbNmzZLue1u7J0+e3GLZ9uHDh9GvXz8AQP/+/REdHe3UZr1ej+3btzu1uaqqCnl5eVKZjRs3wmazITU1tRta4Zq6ujrIZM7/VMnlcthsNgDe2ebzdVUb09LSsGXLFpjNZqnMhg0bMHToUISGhnZTazrOEUSOHDmCH374AX369HF63hvbPHv2bOzZs8fp37XY2FgsXLgQ3333HQDvbLfPrqZZtWqVqFarxffee088cOCAeO+994ohISFOqyp6i/vvv1/UarXi5s2bxbNnz0q3uro6qcx9990n9u3bV9y4caO4Y8cOMS0tTUxLS5Oedyxxvfrqq8X8/Hxx/fr1YkRERI9d4tqW5qtpRNH72p2bmysqFArxueeeE48cOSJ++OGHor+/v/jBBx9IZV544QUxJCRE/PLLL8U9e/aIN910U6tLQMeNGydu375d3LZtmzh48OAetcy1ublz54pxcXHS0t41a9aI4eHh4l/+8hepjDe0uaamRty1a5e4a9cuEYC4dOlScdeuXdLKka5oY1VVlRgVFSXOnj1b3Ldvn7hq1SrR39/fY8s922uzyWQSb7zxRjE+Pl7Mz893+ret+QqR3tZmUbzw7/p856+mEcXe2e72+GwYEUVR/Mc//iH27dtXVKlU4oQJE8RffvnF01XqFACt3v79739LZerr68UHHnhADA0NFf39/cVbbrlFPHv2rNP7nDx5Urz22mtFPz8/MTw8XHzkkUdEs9ncza25OOeHEW9s99dffy2OGjVKVKvV4rBhw8S3337b6XmbzSY++eSTYlRUlKhWq8WrrrpKLCgocCpTUVEhzpw5UwwMDBSDg4PFjIwMsaampjub0WF6vV5csGCB2LdvX1Gj0YgDBgwQn3jiCacvJG9o86ZNm1r9ezx37lxRFLuujbt37xYvvfRSUa1Wi3FxceILL7zQXU1sob02nzhxos1/2zZt2iS9R29rsyhe+Hd9vtbCSG9sd3sEUWy2jSERERFRN/PJOSNERETUczCMEBERkUcxjBAREZFHMYwQERGRRzGMEBERkUcxjBAREZFHMYwQERGRRzGMEBERkUcxjBAREZFHMYwQERGRRzGMEBERkUcxjBAREZFH/T/SbD0wifDYbAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Plot loss history moving average\n", + "window_size = 100\n", + "loss_history_ma = jnp.convolve(jnp.array(loss_history), jnp.ones(window_size)/window_size, mode='valid')\n", + "plt.plot(loss_history_ma)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d740c03b", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjcsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvTLEjVAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAONtJREFUeJzt3X1cFWXi//83oOcoyo2ocMBFJEu8w9uSpc9qurIiuZab25Za6sZqGtYmrhltKdqumJZlrqv1WW/qk5q161qrflS0vEs0xVgVjVUXJQ1wv5me0OR2fn/0Yz6duFHwoAy+no/HPB7MXNc1c80ZD7yduWbGwzAMQwAAABbiebM7AAAAUFMEGAAAYDkEGAAAYDkEGAAAYDkEGAAAYDkEGAAAYDkEGAAAYDkEGAAAYDmNbnYH6kpZWZm+/PJL+fj4yMPD42Z3BwAAXAPDMPTNN98oJCREnp5Vn2dpsAHmyy+/VGho6M3uBgAAqIUvvvhCP/rRj6osb7ABxsfHR9J3H4Cvr+9N7g0AALgWTqdToaGh5t/xqjTYAFN+2cjX15cAAwCAxVxt+AeDeAEAgOUQYAAAgOUQYAAAgOU02DEwAGAVhmGopKREpaWlN7srQJ3z8vJSo0aNrvsRJwQYALiJioqKlJubq8uXL9/srgA3jLe3t4KDg2Wz2Wq9DgIMANwkZWVlys7OlpeXl0JCQmSz2XjwJho0wzBUVFSk//znP8rOztYdd9xR7cPqqkOAAYCbpKioSGVlZQoNDZW3t/fN7g5wQzRt2lSNGzfW6dOnVVRUpCZNmtRqPQziBYCbrLb/AwWsyh3/5vnWAAAAyyHAAADqrbFjx2rYsGHmfP/+/fX000/f8H5s375dHh4eunDhwnWtJzk5WT169KhRm5u1z/UdY2AAoJ5ZsmStzp4tvGHba9PGrgkTHrjm+mPHjtVbb70lSWrcuLHatm2r0aNH67nnnlOjRnX7Z2Xt2rVq3LjxNdXdvn27BgwYoK+//lr+/v512q/vS0lJ0fPPP685c+Zo6tSpN2y716r8c6nOxx9/rP79+9d63TfiMyfAAEA9c/Zsoby8RtzA7a2ucZvBgwdr+fLlKiws1MaNG5WQkKDGjRsrKSmpQt2ioqLrul32+wICAtyynrq0bNkyPfPMM1q2bFm9DDB33323cnNzzfnf/va3cjqdWr58ubnMCp8zl5AAADVmt9vlcDgUFhamiRMnKiYmRh9++KGk/7vs88c//lEhISGKiIiQJH3xxRf61a9+JX9/fwUEBOj+++/XqVOnzHWWlpYqMTFR/v7+atmypZ555hkZhuGy3R9eTiksLNS0adMUGhoqu92u22+/XUuXLtWpU6fMswwtWrSQh4eHxo4dK+m729dTUlIUHh6upk2bqnv37vrrX//qsp2NGzeqQ4cOatq0qQYMGODSz+rs2LFD3377rWbNmiWn06k9e/ZUW7/8s5o5c6Zat24tX19fTZgwQUVFRS71ysrK9MwzzyggIEAOh0PJycku5fPnz1dkZKSaNWum0NBQPfHEEyooKKh0mzabTQ6Hw5yaNm1qHk+Hw6EWLVroueeeU5s2bdSsWTNFRUVp+/btZvvTp09r6NChatGihZo1a6YuXbpo48aN1X7mdYEAAwC4bk2bNnX5o7tt2zZlZWUpNTVV69evV3FxsWJjY+Xj46Ndu3bpk08+UfPmzTV48GCz3SuvvKIVK1Zo2bJl2r17t86fP6+///3v1W539OjRWr16tV5//XUdO3ZMb7zxhpo3b67Q0FD97W9/kyRlZWUpNzdXCxYskPTdJZ63335bS5YsUWZmpiZPnqxHHnlEO3bskPRd0HrggQc0dOhQZWRk6De/+Y2effbZa/ocli5dqhEjRqhx48YaMWKEli5detU227Zt07Fjx7R9+3atXr1aa9eu1cyZM13qvPXWW2rWrJn27dunuXPnatasWUpNTTXLPT099frrryszM1NvvfWWPvroIz3zzDPX1OcfmjRpktLS0vTuu+/q0KFDevDBBzV48GAdP35ckpSQkKDCwkLt3LlThw8f1ksvvXTVz7wucAkJgCVdzziRmo75QNUMw9C2bdu0efNmPfnkk+byZs2a6S9/+Yt56eidd95RWVmZ/vKXv5gP61u+fLn8/f21fft2DRo0SK+99pqSkpL0wAPfHZslS5Zo8+bNVW77X//6l9577z2lpqYqJiZGknTbbbeZ5eWXQQIDA83xGIWFhZo9e7a2bt2q6Ohos83u3bv1xhtv6J577tHixYvVvn17vfLKK5KkiIgI8w91dZxOp/76178qLS1NkvTII4+ob9++WrBggZo3b15lO5vNpmXLlsnb21tdunTRrFmzNHXqVL344ovm7cbdunXTjBkzJEl33HGH/vSnP2nbtm362c9+JkkuZ6XatWunP/zhD5owYYL+/Oc/V9vnH8rJydHy5cuVk5OjkJAQSdLvfvc7bdq0ScuXL9fs2bOVk5Oj4cOHKzIy0vz8ylX2mdcVAgwAS7qecSK1GfMBV+vXr1fz5s1VXFyssrIyjRw50uWyRmRkpMu4l3/+8586ceKEfHx8XNZz5coVnTx5UhcvXlRubq6ioqLMskaNGunOO++scBmpXEZGhry8vHTPPfdcc79PnDihy5cvm3/4yxUVFalnz56SpGPHjrn0Q5IZdqqzevVqtW/fXt27d5ck9ejRQ2FhYVqzZo3i4+OrbNe9e3eXBxlGR0eroKBAX3zxhcLCwiR9F2C+Lzg4WOfOnTPnt27dqpSUFH3++edyOp0qKSnRlStXdPny5Ro9JPHw4cMqLS1Vhw4dXJYXFhaqZcuWkqSnnnpKEydO1JYtWxQTE6Phw4dX6N+NQIABANTYgAEDtHjxYtlsNoWEhFS4+6hZs2Yu8wUFBerdu7dWrlxZYV2tW7euVR+aNm1a4zbl40I2bNigNm3auJTZ7fZa9aPc0qVLlZmZ6fJZlJWVadmyZdUGmGvxwzuvPDw8VFZWJkk6deqUfv7zn2vixIn64x//qICAAO3evVvx8fEqKiqqUYApKCiQl5eX0tPT5eXl5VJWfhbpN7/5jWJjY7VhwwZt2bJFKSkpeuWVV1zOwN0IBBgAQI01a9ZMt99++zXX79Wrl9asWaPAwED5+vpWWic4OFj79u1Tv379JEklJSVKT09Xr169Kq0fGRmpsrIy7dixw7yE9H3lZ4C+/5bvzp07y263Kycnp8ozN506dTIHJJfbu3dvtft3+PBhHThwQNu3b3e5g+f8+fPq37+/Pv/8c3Xs2LHStv/85z/17bffmoFs79695piSa5Genq6ysjK98sor5iWn995775ra/lDPnj1VWlqqc+fOqW/fvlXWCw0N1YQJEzRhwgQlJSXpv//7v/Xkk09W+pnXFQbxAgDq3KhRo9SqVSvdf//92rVrl7Kzs7V9+3Y99dRTOnPmjKTvbuedM2eO1q1bp88//1xPPPFEtQ+Oa9euncaMGaPHHntM69atM9dZ/sc7LCxMHh4eWr9+vf7zn/+ooKBAPj4++t3vfqfJkyfrrbfe0smTJ3Xw4EEtXLjQfLbNhAkTdPz4cU2dOlVZWVlatWqVVqxYUe3+LV26VH369FG/fv3UtWtXc+rXr5/uuuuuagfzFhUVKT4+XkePHtXGjRs1Y8YMTZo06Zoft3/77beruLhYCxcu1L///W/9z//8j5YsWXJNbX+oQ4cOGjVqlEaPHq21a9cqOztbn376qVJSUrRhwwZJ34232bx5s7Kzs3Xw4EF9/PHH6tSpk6TKP/O6QoABANQ5b29v7dy5U23bttUDDzygTp06KT4+XleuXDHPyEyZMkWPPvqoxowZo+joaPn4+OgXv/hFtetdvHixfvnLX+qJJ55Qx44dNW7cOF26dEmS1KZNG82cOVPPPvusgoKCNGnSJEnSiy++qBdeeEEpKSnq1KmTBg8erA0bNig8PFyS1LZtW/3tb3/TunXr1L17dy1ZskSzZ8+usg9FRUV65513NHz48ErLhw8frrffflvFxcWVlg8cOFB33HGH+vXrp4ceekj33Xdfhdukq9O9e3fNnz9fL730krp27aqVK1cqJSXlmtv/0PLlyzV69GhNmTJFERERGjZsmPbv36+2bdtK+u7sSkJCgvnZdejQwRwsXNVnXhc8jKpGR1mc0+mUn5+fLl68WOXpSgA31/XcSZSefkx9+syqVdvS0tV68cUb96C4qly5ckXZ2dkKDw93eSNvfX8SL9xn7NixunDhgtatW3ezu3JDVfVvX7r2v9+MgQFw01zPnURO5+/d3Jv6gzABXB2XkAAAgOXUOMDs3LlTQ4cOVUhIiDw8PCqc9vLw8Kh0mjdvnlmnXbt2FcrnzJnjsp5Dhw6pb9++atKkiUJDQzV37tza7SEAAPXUihUrbrnLR+5S4wBz6dIlde/eXYsWLaq0PDc312VatmyZPDw8KgxumjVrlku9798/7nQ6NWjQIIWFhSk9PV3z5s1TcnKy3nzzzZp2FwAANEA1HgMTFxenuLi4KssdDofL/AcffKABAwa4PGpYknx8fCrULbdy5UoVFRVp2bJlstls6tKlizIyMjR//nyNHz++pl0GAAANTJ2OgcnPz9eGDRsqfQLhnDlz1LJlS/Xs2VPz5s1TSUmJWZaWlqZ+/fq5PIY6NjZWWVlZ+vrrryvdVmFhoZxOp8sEAFbQQG8GBarkjn/zdXoX0ltvvSUfHx/zxVzlnnrqKfXq1UsBAQHas2ePkpKSlJubq/nz50uS8vLyzPvxywUFBZllLVq0qLCtlJSUCm/vBID6rPzx8JcvX67VY/EBq7p8+bKkiq9IqIk6DTDLli3TqFGjKtzjnZiYaP7crVs32Ww2Pf7440pJSan1uyiSkpJc1ut0Oq/5McwAcDN4eXnJ39/ffCmft7e3+aZmoCEyDEOXL1/WuXPn5O/vX+F9SzVRZwFm165dysrK0po1a65aNyoqSiUlJTp16pQiIiLkcDiUn5/vUqd8vqpxM3a7/bpfxAUAN1r577Tvv1kYaOj8/f2r/Ht+reoswCxdulS9e/c2XytenYyMDHl6eiowMFDSd68S//3vf6/i4mLz9FJqaqoiIiIqvXwEAFbl4eGh4OBgBQYGVvmoeaAhady48XWdeSlX4wBTUFCgEydOmPPZ2dnKyMhQQECA+Z4Ep9Op999/X6+88kqF9mlpadq3b58GDBggHx8fpaWlafLkyXrkkUfMcDJy5EjNnDlT8fHxmjZtmo4cOaIFCxbo1Vdfre1+AkC95uXl5ZZf6sCtosYB5sCBAxowYIA5Xz7uZMyYMebbOt99910ZhqERIyo+Itxut+vdd99VcnKyCgsLFR4ersmTJ7uMX/Hz89OWLVuUkJCg3r17q1WrVpo+fTq3UAMAAEm1CDD9+/e/6u1P48ePrzJs9OrVS3v37r3qdrp166Zdu3bVtHsAAOAWwLuQAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5dQ4wOzcuVNDhw5VSEiIPDw8tG7dOpfysWPHysPDw2UaPHiwS53z589r1KhR8vX1lb+/v+Lj41VQUOBS59ChQ+rbt6+aNGmi0NBQzZ07t+Z7BwAAGqQaB5hLly6pe/fuWrRoUZV1Bg8erNzcXHNavXq1S/moUaOUmZmp1NRUrV+/Xjt37tT48ePNcqfTqUGDBiksLEzp6emaN2+ekpOT9eabb9a0uwAAoAFqVNMGcXFxiouLq7aO3W6Xw+GotOzYsWPatGmT9u/frzvvvFOStHDhQt177716+eWXFRISopUrV6qoqEjLli2TzWZTly5dlJGRofnz57sEHQAAcGuqkzEw27dvV2BgoCIiIjRx4kR99dVXZllaWpr8/f3N8CJJMTEx8vT01L59+8w6/fr1k81mM+vExsYqKytLX3/9daXbLCwslNPpdJkAAEDD5PYAM3jwYL399tvatm2bXnrpJe3YsUNxcXEqLS2VJOXl5SkwMNClTaNGjRQQEKC8vDyzTlBQkEud8vnyOj+UkpIiPz8/cwoNDXX3rgEAgHqixpeQrubhhx82f46MjFS3bt3Uvn17bd++XQMHDnT35kxJSUlKTEw0551OJyEGAIAGqs5vo77tttvUqlUrnThxQpLkcDh07tw5lzolJSU6f/68OW7G4XAoPz/fpU75fFVja+x2u3x9fV0mAADQMNV5gDlz5oy++uorBQcHS5Kio6N14cIFpaenm3U++ugjlZWVKSoqyqyzc+dOFRcXm3VSU1MVERGhFi1a1HWXAQBAPVfjAFNQUKCMjAxlZGRIkrKzs5WRkaGcnBwVFBRo6tSp2rt3r06dOqVt27bp/vvv1+23367Y2FhJUqdOnTR48GCNGzdOn376qT755BNNmjRJDz/8sEJCQiRJI0eOlM1mU3x8vDIzM7VmzRotWLDA5RIRAAC4ddU4wBw4cEA9e/ZUz549JUmJiYnq2bOnpk+fLi8vLx06dEj33XefOnTooPj4ePXu3Vu7du2S3W4317Fy5Up17NhRAwcO1L333quf/OQnLs948fPz05YtW5Sdna3evXtrypQpmj59OrdQAwAASbUYxNu/f38ZhlFl+ebNm6+6joCAAK1ataraOt26ddOuXbtq2j0AAHAL4F1IAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcmr8MkcA+L4lS9bq7NnCWrVNTz+mPn3c3CEAtwQCDIDrcvZsoby8RtSqrdP5ezf3BsCtgktIAADAcggwAADAcggwAADAcggwAADAchjEC+CWk55+SC+8ULu2bdrYNWHCA+7tEIAaI8AAuOU4nZ61vnPq7NnVbu4NgNrgEhIAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALCcGgeYnTt3aujQoQoJCZGHh4fWrVtnlhUXF2vatGmKjIxUs2bNFBISotGjR+vLL790WUe7du3k4eHhMs2ZM8elzqFDh9S3b181adJEoaGhmjt3bu32EAAANDg1DjCXLl1S9+7dtWjRogplly9f1sGDB/XCCy/o4MGDWrt2rbKysnTfffdVqDtr1izl5uaa05NPPmmWOZ1ODRo0SGFhYUpPT9e8efOUnJysN998s6bdBQAADVCjmjaIi4tTXFxcpWV+fn5KTU11WfanP/1Jffr0UU5Ojtq2bWsu9/HxkcPhqHQ9K1euVFFRkZYtWyabzaYuXbooIyND8+fP1/jx42vaZQAA0MDU+RiYixcvysPDQ/7+/i7L58yZo5YtW6pnz56aN2+eSkpKzLK0tDT169dPNpvNXBYbG6usrCx9/fXXlW6nsLBQTqfTZQIAAA1Tjc/A1MSVK1c0bdo0jRgxQr6+vubyp556Sr169VJAQID27NmjpKQk5ebmav78+ZKkvLw8hYeHu6wrKCjILGvRokWFbaWkpGjmzJl1uDcAAKC+qLMAU1xcrF/96lcyDEOLFy92KUtMTDR/7tatm2w2mx5//HGlpKTIbrfXantJSUku63U6nQoNDa1d5wEAQL1WJwGmPLycPn1aH330kcvZl8pERUWppKREp06dUkREhBwOh/Lz813qlM9XNW7GbrfXOvwAAABrcfsYmPLwcvz4cW3dulUtW7a8apuMjAx5enoqMDBQkhQdHa2dO3equLjYrJOamqqIiIhKLx8BAIBbS43PwBQUFOjEiRPmfHZ2tjIyMhQQEKDg4GD98pe/1MGDB7V+/XqVlpYqLy9PkhQQECCbzaa0tDTt27dPAwYMkI+Pj9LS0jR58mQ98sgjZjgZOXKkZs6cqfj4eE2bNk1HjhzRggUL9Oqrr7pptwEAgJXVOMAcOHBAAwYMMOfLx52MGTNGycnJ+vDDDyVJPXr0cGn38ccfq3///rLb7Xr33XeVnJyswsJChYeHa/LkyS7jV/z8/LRlyxYlJCSod+/eatWqlaZPn84t1AAAQFItAkz//v1lGEaV5dWVSVKvXr20d+/eq26nW7du2rVrV027BwAAbgG8CwkAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFgOAQYAAFhOjQPMzp07NXToUIWEhMjDw0Pr1q1zKTcMQ9OnT1dwcLCaNm2qmJgYHT9+3KXO+fPnNWrUKPn6+srf31/x8fEqKChwqXPo0CH17dtXTZo0UWhoqObOnVvzvQMAAA1SjQPMpUuX1L17dy1atKjS8rlz5+r111/XkiVLtG/fPjVr1kyxsbG6cuWKWWfUqFHKzMxUamqq1q9fr507d2r8+PFmudPp1KBBgxQWFqb09HTNmzdPycnJevPNN2uxiwAAoKFpVNMGcXFxiouLq7TMMAy99tprev7553X//fdLkt5++20FBQVp3bp1evjhh3Xs2DFt2rRJ+/fv15133ilJWrhwoe699169/PLLCgkJ0cqVK1VUVKRly5bJZrOpS5cuysjI0Pz5812CDgAAuDW5dQxMdna28vLyFBMTYy7z8/NTVFSU0tLSJElpaWny9/c3w4skxcTEyNPTU/v27TPr9OvXTzabzawTGxurrKwsff311+7sMgAAsKAan4GpTl5eniQpKCjIZXlQUJBZlpeXp8DAQNdONGqkgIAAlzrh4eEV1lFe1qJFiwrbLiwsVGFhoTnvdDqvc28AAEB91WDuQkpJSZGfn585hYaG3uwuAQCAOuLWAONwOCRJ+fn5Lsvz8/PNMofDoXPnzrmUl5SU6Pz58y51KlvH97fxQ0lJSbp48aI5ffHFF9e/QwAAoF5ya4AJDw+Xw+HQtm3bzGVOp1P79u1TdHS0JCk6OloXLlxQenq6Weejjz5SWVmZoqKizDo7d+5UcXGxWSc1NVURERGVXj6SJLvdLl9fX5cJAAA0TDUOMAUFBcrIyFBGRoak7wbuZmRkKCcnRx4eHnr66af1hz/8QR9++KEOHz6s0aNHKyQkRMOGDZMkderUSYMHD9a4ceP06aef6pNPPtGkSZP08MMPKyQkRJI0cuRI2Ww2xcfHKzMzU2vWrNGCBQuUmJjoth0HAADWVeNBvAcOHNCAAQPM+fJQMWbMGK1YsULPPPOMLl26pPHjx+vChQv6yU9+ok2bNqlJkyZmm5UrV2rSpEkaOHCgPD09NXz4cL3++utmuZ+fn7Zs2aKEhAT17t1brVq10vTp07mFGgAASKpFgOnfv78Mw6iy3MPDQ7NmzdKsWbOqrBMQEKBVq1ZVu51u3bpp165dNe0eAAC4BTSYu5AAAMCtgwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAshwADAAAsx+0Bpl27dvLw8KgwJSQkSJL69+9foWzChAku68jJydGQIUPk7e2twMBATZ06VSUlJe7uKgAAsKhG7l7h/v37VVpaas4fOXJEP/vZz/Tggw+ay8aNG6dZs2aZ897e3ubPpaWlGjJkiBwOh/bs2aPc3FyNHj1ajRs31uzZs93dXQAAYEFuDzCtW7d2mZ8zZ47at2+ve+65x1zm7e0th8NRafstW7bo6NGj2rp1q4KCgtSjRw+9+OKLmjZtmpKTk2Wz2dzdZQAAYDF1OgamqKhI77zzjh577DF5eHiYy1euXKlWrVqpa9euSkpK0uXLl82ytLQ0RUZGKigoyFwWGxsrp9OpzMzMKrdVWFgop9PpMgEAgIbJ7Wdgvm/dunW6cOGCxo4day4bOXKkwsLCFBISokOHDmnatGnKysrS2rVrJUl5eXku4UWSOZ+Xl1fltlJSUjRz5kz37wQAAKh36jTALF26VHFxcQoJCTGXjR8/3vw5MjJSwcHBGjhwoE6ePKn27dvXeltJSUlKTEw0551Op0JDQ2u9PgAAUH/VWYA5ffq0tm7dap5ZqUpUVJQk6cSJE2rfvr0cDoc+/fRTlzr5+fmSVOW4GUmy2+2y2+3X2WsAAGAFdTYGZvny5QoMDNSQIUOqrZeRkSFJCg4OliRFR0fr8OHDOnfunFknNTVVvr6+6ty5c111FwAAWEidnIEpKyvT8uXLNWbMGDVq9H+bOHnypFatWqV7771XLVu21KFDhzR58mT169dP3bp1kyQNGjRInTt31qOPPqq5c+cqLy9Pzz//vBISEjjDAgAAJNVRgNm6datycnL02GOPuSy32WzaunWrXnvtNV26dEmhoaEaPny4nn/+ebOOl5eX1q9fr4kTJyo6OlrNmjXTmDFjXJ4bAwAAbm11EmAGDRokwzAqLA8NDdWOHTuu2j4sLEwbN26si64BAIAGgHchAQAAyyHAAAAAyyHAAAAAyyHAAAAAyyHAAAAAyyHAAAAAy6nTdyEBQEOTnn5IL7xQu7Zt2tg1YcID7u0QcIsiwABADTidnvLyGlGrtmfPrnZzb4BbFwEGgJYsWauzZwtr1TY9/Zj69HFzhwDgKggwAHT2bGGtzyo4nb93c28A4OoYxAsAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACyHAAMAACzH7QEmOTlZHh4eLlPHjh3N8itXrighIUEtW7ZU8+bNNXz4cOXn57usIycnR0OGDJG3t7cCAwM1depUlZSUuLurAADAohrVxUq7dOmirVu3/t9GGv3fZiZPnqwNGzbo/fffl5+fnyZNmqQHHnhAn3zyiSSptLRUQ4YMkcPh0J49e5Sbm6vRo0ercePGmj17dl10FwAAWEydBJhGjRrJ4XBUWH7x4kUtXbpUq1at0k9/+lNJ0vLly9WpUyft3btXP/7xj7VlyxYdPXpUW7duVVBQkHr06KEXX3xR06ZNU3Jysmw2W110GQAAWEidjIE5fvy4QkJCdNttt2nUqFHKycmRJKWnp6u4uFgxMTFm3Y4dO6pt27ZKS0uTJKWlpSkyMlJBQUFmndjYWDmdTmVmZla5zcLCQjmdTpcJAAA0TG4PMFFRUVqxYoU2bdqkxYsXKzs7W3379tU333yjvLw82Ww2+fv7u7QJCgpSXl6eJCkvL88lvJSXl5dVJSUlRX5+fuYUGhrq3h0DAAD1htsvIcXFxZk/d+vWTVFRUQoLC9N7772npk2buntzpqSkJCUmJprzTqeTEAMAQANV57dR+/v7q0OHDjpx4oQcDoeKiop04cIFlzr5+fnmmBmHw1HhrqTy+crG1ZSz2+3y9fV1mQAAQMNU5wGmoKBAJ0+eVHBwsHr37q3GjRtr27ZtZnlWVpZycnIUHR0tSYqOjtbhw4d17tw5s05qaqp8fX3VuXPnuu4uAACwALdfQvrd736noUOHKiwsTF9++aVmzJghLy8vjRgxQn5+foqPj1diYqICAgLk6+urJ598UtHR0frxj38sSRo0aJA6d+6sRx99VHPnzlVeXp6ef/55JSQkyG63u7u7AADAgtweYM6cOaMRI0boq6++UuvWrfWTn/xEe/fuVevWrSVJr776qjw9PTV8+HAVFhYqNjZWf/7zn832Xl5eWr9+vSZOnKjo6Gg1a9ZMY8aM0axZs9zdVQAAYFFuDzDvvvtuteVNmjTRokWLtGjRoirrhIWFaePGje7uGgAAaCB4FxIAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAcAgwAALAct7+NGsDNsWTJWp09W1irtunpx9Snj5s7BAB1iAADNBBnzxbKy2tErdo6nb93c28AoG5xCQkAAFgOZ2AA4AZJTz+kF16oXds2beyaMOEB93YIsDACDADcIE6nZ60v8509u9rNvQGsjUtIAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADAcggwAADActweYFJSUnTXXXfJx8dHgYGBGjZsmLKyslzq9O/fXx4eHi7ThAkTXOrk5ORoyJAh8vb2VmBgoKZOnaqSkhJ3dxcAAFhQI3evcMeOHUpISNBdd92lkpISPffccxo0aJCOHj2qZs2amfXGjRunWbNmmfPe3t7mz6WlpRoyZIgcDof27Nmj3NxcjR49Wo0bN9bs2bPd3WUAAGAxbg8wmzZtcplfsWKFAgMDlZ6ern79+pnLvb295XA4Kl3Hli1bdPToUW3dulVBQUHq0aOHXnzxRU2bNk3Jycmy2Wzu7jYAALCQOh8Dc/HiRUlSQECAy/KVK1eqVatW6tq1q5KSknT58mWzLC0tTZGRkQoKCjKXxcbGyul0KjMzs9LtFBYWyul0ukwAAKBhcvsZmO8rKyvT008/rf/6r/9S165dzeUjR45UWFiYQkJCdOjQIU2bNk1ZWVlau3atJCkvL88lvEgy5/Py8irdVkpKimbOnFlHewIAAOqTOg0wCQkJOnLkiHbv3u2yfPz48ebPkZGRCg4O1sCBA3Xy5Em1b9++VttKSkpSYmKiOe90OhUaGlq7jgMAgHqtzi4hTZo0SevXr9fHH3+sH/3oR9XWjYqKkiSdOHFCkuRwOJSfn+9Sp3y+qnEzdrtdvr6+LhMAAGiY3B5gDMPQpEmT9Pe//10fffSRwsPDr9omIyNDkhQcHCxJio6O1uHDh3Xu3DmzTmpqqnx9fdW5c2d3dxkAAFiM2y8hJSQkaNWqVfrggw/k4+Njjlnx8/NT06ZNdfLkSa1atUr33nuvWrZsqUOHDmny5Mnq16+funXrJkkaNGiQOnfurEcffVRz585VXl6enn/+eSUkJMhut7u7ywAAwGLcfgZm8eLFunjxovr376/g4GBzWrNmjSTJZrNp69atGjRokDp27KgpU6Zo+PDh+sc//mGuw8vLS+vXr5eXl5eio6P1yCOPaPTo0S7PjQEAALcut5+BMQyj2vLQ0FDt2LHjqusJCwvTxo0b3dUtALC09PRDeuGF2rVt08auCRMecG+HgJusTu9CAlAzS5as1dmzhbVqm55+TH36uLlDqDecTk95eY2oVduzZ1e7uTfAzUeAAeqRs2cLa/1Hyun8vZt7AwD1F2+jBgAAlkOAAQAAlkOAAQAAlkOAAQAAlkOAAQAAlkOAAQAAlkOAAQAAlkOAAQAAlsOD7AA342m6AFD3CDCAm/E0XQCoe1xCAgAAlsMZGABo4HiTNRoiAgwANHC8yRoNEZeQAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5RBgAACA5fAcGKASvM8IAOo3AgxQCd5nBAD1G5eQAACA5XAGBgBQJd6jhPqKAAMAqBLvUUJ9xSUkAABgOZyBQYPFnUTAzcXlJ9QlAgwaLO4kAm6u67n89OGHSbX+Dwjh59ZAgAEA1DuMvcHVMAYGAABYDmdgUK8xjgUAUBkCDOo1xrEAACpDgEGd4ywKAMDdCDCoc5xFAQC4GwEGAID/3/WcMT558qjat+9cq7bc+l1z9TrALFq0SPPmzVNeXp66d++uhQsXqg/XE24KLgMBsIrreYDed7+vZtWqbU7O79WhA7d+3yj1NsCsWbNGiYmJWrJkiaKiovTaa68pNjZWWVlZCgwMvNnds5zrCSDS9X2puQwE4Ea6nmfI8PvKOuptgJk/f77GjRunX//615KkJUuWaMOGDVq2bJmeffbZm9y7m+P6z4LULoBIfKkBoC5dz1mjW/XSVb0MMEVFRUpPT1dSUpK5zNPTUzExMUpLS6u0TWFhoQoL/++P+8WLFyVJTqfT7f1buvRD5ebWLkhkZ2cpPDyiVm0/++xfuvPO2gWJ8+fTVVhY+8+ipKSw1u1pS1va0pa21Tt/vlglJUNq1fbf/05XWFjt2v7tb8n697+/rlXb4GC74uPvq1Xb6pT/3TYMo/qKRj109uxZQ5KxZ88el+VTp041+vTpU2mbGTNmGJKYmJiYmJiYGsD0xRdfVJsV6uUZmNpISkpSYmKiOV9WVqbz58+rZcuW8vDwcNt2nE6nQkND9cUXX8jX19dt661PGvo+sn/W19D3saHvn9Tw95H9qz3DMPTNN98oJCSk2nr1MsC0atVKXl5eys/Pd1men58vh8NRaRu73S673e6yzN/fv666KF9f3wb5j/L7Gvo+sn/W19D3saHvn9Tw95H9qx0/P7+r1qmXL3O02Wzq3bu3tm3bZi4rKyvTtm3bFB0dfRN7BgAA6oN6eQZGkhITEzVmzBjdeeed6tOnj1577TVdunTJvCsJAADcuuptgHnooYf0n//8R9OnT1deXp569OihTZs2KSgo6Kb2y263a8aMGRUuVzUkDX0f2T/ra+j72ND3T2r4+8j+1T0Pw7jafUoAAAD1S70cAwMAAFAdAgwAALAcAgwAALAcAgwAALAcAkwl/vjHP+ruu++Wt7d3lQ/Dy8nJ0ZAhQ+Tt7a3AwEBNnTpVJSUl1a73/PnzGjVqlHx9feXv76/4+HgVFBTUwR5cu+3bt8vDw6PSaf/+/VW269+/f4X6EyZMuIE9r5l27dpV6O+cOXOqbXPlyhUlJCSoZcuWat68uYYPH17h4Yr1walTpxQfH6/w8HA1bdpU7du314wZM1RUVFRtu/p+DBctWqR27dqpSZMmioqK0qefflpt/ffff18dO3ZUkyZNFBkZqY0bN96gntZMSkqK7rrrLvn4+CgwMFDDhg1TVlZWtW1WrFhR4Vg1adLkBvW45pKTkyv0t2PHjtW2scrxkyr/feLh4aGEhIRK69f347dz504NHTpUISEh8vDw0Lp161zKDcPQ9OnTFRwcrKZNmyomJkbHjx+/6npr+h2uKQJMJYqKivTggw9q4sSJlZaXlpZqyJAhKioq0p49e/TWW29pxYoVmj59erXrHTVqlDIzM5Wamqr169dr586dGj9+fF3swjW7++67lZub6zL95je/UXh4uO68885q244bN86l3dy5c29Qr2tn1qxZLv198sknq60/efJk/eMf/9D777+vHTt26Msvv9QDD9S/t7Z+/vnnKisr0xtvvKHMzEy9+uqrWrJkiZ577rmrtq2vx3DNmjVKTEzUjBkzdPDgQXXv3l2xsbE6d+5cpfX37NmjESNGKD4+Xp999pmGDRumYcOG6ciRIze451e3Y8cOJSQkaO/evUpNTVVxcbEGDRqkS5cuVdvO19fX5VidPn36BvW4drp06eLS3927d1dZ10rHT5L279/vsm+pqamSpAcffLDKNvX5+F26dEndu3fXokWLKi2fO3euXn/9dS1ZskT79u1Ts2bNFBsbqytXrlS5zpp+h2vFLW9fbKCWL19u+Pn5VVi+ceNGw9PT08jLyzOXLV682PD19TUKCwsrXdfRo0cNScb+/fvNZf/7v/9reHh4GGfPnnV732urqKjIaN26tTFr1qxq691zzz3Gb3/72xvTKTcICwszXn311Wuuf+HCBaNx48bG+++/by47duyYIclIS0urgx6619y5c43w8PBq69TnY9inTx8jISHBnC8tLTVCQkKMlJSUSuv/6le/MoYMGeKyLCoqynj88cfrtJ/ucO7cOUOSsWPHjirrVPW7qL6aMWOG0b1792uub+XjZxiG8dvf/tZo3769UVZWVmm5lY6fJOPvf/+7OV9WVmY4HA5j3rx55rILFy4YdrvdWL16dZXrqel3uDY4A1MLaWlpioyMdHmoXmxsrJxOpzIzM6ts4+/v73JWIyYmRp6entq3b1+d9/laffjhh/rqq6+u6YnHK1euVKtWrdS1a1clJSXp8uXLN6CHtTdnzhy1bNlSPXv21Lx586q95Jeenq7i4mLFxMSYyzp27Ki2bdsqLS3tRnT3uly8eFEBAQFXrVcfj2FRUZHS09NdPntPT0/FxMRU+dmnpaW51Je++05a5VhJuurxKigoUFhYmEJDQ3X//fdX+bumvjh+/LhCQkJ02223adSoUcrJyamyrpWPX1FRkd555x099thj1b442GrHr1x2drby8vJcjo+fn5+ioqKqPD61+Q7XRr19Em99lpeXV+GJwOXzeXl5VbYJDAx0WdaoUSMFBARU2eZmWLp0qWJjY/WjH/2o2nojR45UWFiYQkJCdOjQIU2bNk1ZWVlau3btDeppzTz11FPq1auXAgICtGfPHiUlJSk3N1fz58+vtH5eXp5sNluFMVBBQUH16nhV5sSJE1q4cKFefvnlauvV12P4//7f/1NpaWml37HPP/+80jZVfSfr+7EqKyvT008/rf/6r/9S165dq6wXERGhZcuWqVu3brp48aJefvll3X333crMzLzqd/VmiIqK0ooVKxQREaHc3FzNnDlTffv21ZEjR+Tj41OhvlWPnyStW7dOFy5c0NixY6usY7Xj933lx6Amx6c23+HauGUCzLPPPquXXnqp2jrHjh276kAzq6jN/p45c0abN2/We++9d9X1f3/sTmRkpIKDgzVw4ECdPHlS7du3r33Ha6Am+5iYmGgu69atm2w2mx5//HGlpKTU20d91+YYnj17VoMHD9aDDz6ocePGVdu2PhzDW11CQoKOHDlS7fgQSYqOjnZ5ke3dd9+tTp066Y033tCLL75Y192ssbi4OPPnbt26KSoqSmFhYXrvvfcUHx9/E3vmfkuXLlVcXJxCQkKqrGO142cVt0yAmTJlSrUJWZJuu+22a1qXw+GoMJq6/O4Uh8NRZZsfDl4qKSnR+fPnq2xzPWqzv8uXL1fLli1133331Xh7UVFRkr773/+N+uN3Pcc0KipKJSUlOnXqlCIiIiqUOxwOFRUV6cKFCy5nYfLz8+vkeFWmpvv35ZdfasCAAbr77rv15ptv1nh7N+MYVqZVq1by8vKqcMdXdZ+9w+GoUf36YNKkSeZg/pr+L7xx48bq2bOnTpw4UUe9cy9/f3916NChyv5a8fhJ0unTp7V169Yan7W00vErPwb5+fkKDg42l+fn56tHjx6VtqnNd7hW3DaapgG62iDe/Px8c9kbb7xh+Pr6GleuXKl0XeWDeA8cOGAu27x5c70ZxFtWVmaEh4cbU6ZMqVX73bt3G5KMf/7zn27uWd145513DE9PT+P8+fOVlpcP4v3rX/9qLvv888/r7SDeM2fOGHfccYfx8MMPGyUlJbVaR306hn369DEmTZpkzpeWlhpt2rSpdhDvz3/+c5dl0dHR9XIQaFlZmZGQkGCEhIQY//rXv2q1jpKSEiMiIsKYPHmym3tXN7755hujRYsWxoIFCyott9Lx+74ZM2YYDofDKC4urlG7+nz8VMUg3pdfftlcdvHixWsaxFuT73Ct+uq2NTUgp0+fNj777DNj5syZRvPmzY3PPvvM+Oyzz4xvvvnGMIzv/vF17drVGDRokJGRkWFs2rTJaN26tZGUlGSuY9++fUZERIRx5swZc9ngwYONnj17Gvv27TN2795t3HHHHcaIESNu+P5VZuvWrYYk49ixYxXKzpw5Y0RERBj79u0zDMMwTpw4YcyaNcs4cOCAkZ2dbXzwwQfGbbfdZvTr1+9Gd/ua7Nmzx3j11VeNjIwM4+TJk8Y777xjtG7d2hg9erRZ54f7aBiGMWHCBKNt27bGRx99ZBw4cMCIjo42oqOjb8YuVOvMmTPG7bffbgwcONA4c+aMkZuba07fr2OlY/juu+8adrvdWLFihXH06FFj/Pjxhr+/v3nn36OPPmo8++yzZv1PPvnEaNSokfHyyy8bx44dM2bMmGE0btzYOHz48M3ahSpNnDjR8PPzM7Zv3+5yrC5fvmzW+eH+zZw509i8ebNx8uRJIz093Xj44YeNJk2aGJmZmTdjF65qypQpxvbt243s7Gzjk08+MWJiYoxWrVoZ586dMwzD2sevXGlpqdG2bVtj2rRpFcqsdvy++eYb8++cJGP+/PnGZ599Zpw+fdowDMOYM2eO4e/vb3zwwQfGoUOHjPvvv98IDw83vv32W3MdP/3pT42FCxea81f7DrsDAaYSY8aMMSRVmD7++GOzzqlTp4y4uDijadOmRqtWrYwpU6a4pPCPP/7YkGRkZ2eby7766itjxIgRRvPmzQ1fX1/j17/+tRmKbrYRI0YYd999d6Vl2dnZLvufk5Nj9OvXzwgICDDsdrtx++23G1OnTjUuXrx4A3t87dLT042oqCjDz8/PaNKkidGpUydj9uzZLmfLfriPhmEY3377rfHEE08YLVq0MLy9vY1f/OIXLqGgvli+fHml/16/f4LVisdw4cKFRtu2bQ2bzWb06dPH2Lt3r1l2zz33GGPGjHGp/9577xkdOnQwbDab0aVLF2PDhg03uMfXpqpjtXz5crPOD/fv6aefNj+LoKAg49577zUOHjx44zt/jR566CEjODjYsNlsRps2bYyHHnrIOHHihFlu5eNXbvPmzYYkIysrq0KZ1Y5f+d+rH07l+1BWVma88MILRlBQkGG3242BAwdW2O+wsDBjxowZLsuq+w67g4dhGIb7LkgBAADUPZ4DAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALIcAAwAALOf/AxtY5FAM1UDDAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# predict alpha test\n", + "data_test_normalized = (data_test - data_mean) / data_std\n", + "pred_alpha_test_normalized = jax.vmap(model)(ts_train[:data_test.shape[0]], data_test_normalized)\n", + "pred_alpha_test = pred_alpha_test_normalized * alpha_std + alpha_mean\n", + "\n", + "# plot in hist\n", + "plt.hist(pred_alpha_test.flatten(), bins=30, alpha=0.5, label='Predicted Alpha Test', color='blue', range=(-10, 10), edgecolor='black', linewidth=0.5)\n", + "plt.legend()\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "jax-neural-odes", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/imgs/trajectories_and_alpha_distribution.png b/imgs/trajectories_and_alpha_distribution.png new file mode 100644 index 00000000..5882b124 Binary files /dev/null and b/imgs/trajectories_and_alpha_distribution.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 25ce7aa8..26601e7f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -115,6 +115,7 @@ nav: - Neural ODE: 'examples/neural_ode.ipynb' - Neural CDE: 'examples/neural_cde.ipynb' - Neural SDE: 'examples/neural_sde.ipynb' + - Neural GRU-ODE: 'examples/neural_gru_ode.ipynb' - Latent ODE: 'examples/latent_ode.ipynb' - Continuous normalising flow: 'examples/continuous_normalising_flow.ipynb' - Symbolic regression: 'examples/symbolic_regression.ipynb'