From b00969a0e73e9f09fa6b063de4cd1cb433bec0e5 Mon Sep 17 00:00:00 2001 From: ayulockin Date: Thu, 8 Feb 2024 13:01:48 +0000 Subject: [PATCH 1/4] add lm-eval-harness colab --- ...ghts_and_biases_with_lm_eval_harness.ipynb | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb diff --git a/colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb b/colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb new file mode 100644 index 00000000..84d7de7a --- /dev/null +++ b/colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb @@ -0,0 +1,153 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "d6YuliF8qfsy" + }, + "source": [ + "\"Weights\n", + "\n", + "\n", + "# Visualizing Results in Weights and Biases\n", + "\n", + "\n", + "\n", + "With the Weights and Biases integration, you can now spend more time extracting deeper insights into your evaluation results. The integration is designed to streamline the process of logging and visualizing experiment results using the Weights & Biases (W&B) platform.\n", + "\n", + "The integration provide functionalities\n", + "\n", + "- to automatically log the evaluation results,\n", + "- log the samples as W&B Tables for easy visualization,\n", + "- log the `results.json` file as an artifact for version control,\n", + "- log the `_eval_samples.json` file if the samples are logged,\n", + "- generate a comprehensive report for analysis and visualization with all the important metric,\n", + "- log task and cli configs,\n", + "- and more out of the box like the command used to run the evaluation, GPU/CPU counts, timestamp, etc.\n", + "\n", + "The integration is super easy to use with the eval harness. Let's see how!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tJRBdlTrqgVg" + }, + "outputs": [], + "source": [ + "# Install this project if you did not already have it.\n", + "# This is all that is needed to be installed to start using Weights and Biases\n", + "\n", + "!git clone -b wandb-logging https://github.com/ayulockin/lm-evaluation-harness\n", + "%cd lm-evaluation-harness\n", + "!pip -qq install -e .[wandb]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Eb4eGgJIxCki" + }, + "outputs": [], + "source": [ + "# Getting an error `no module named transformers.cache_utils` while running eval.\n", + "# Installing transformers from the repo is solving the issue.\n", + "# Ref: https://huggingface.co/DiscoResearch/mixtral-7b-8expert/discussions/9#6576edcd0370e52e3b2c0620\n", + "!pip uninstall -y transformers\n", + "!pip install git+https://github.com/huggingface/transformers" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5c-t1T2vrxQw" + }, + "source": [ + "# Run the Eval Harness\n", + "\n", + "Run the eval harness as usual with a `wandb_args` flag. This flag is used to provide arguments for initializing a wandb run ([wandb.init](https://docs.wandb.ai/ref/python/init)) as comma separated string arguments.\n", + "\n", + "If `wandb_args` flag is used, the metrics and all other goodness will be automatically logged to Weights and Biases. In the stdout, you will find the link to the W&B run page as well as link to the generated report." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YkeORhFXry8o" + }, + "source": [ + "## Set your API Key\n", + "\n", + "Before you can use W&B, you need to authenticate your machine with an authentication key. Visit https://wandb.ai/authorize to get one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "E7YTd5OUrpnO" + }, + "outputs": [], + "source": [ + "import wandb\n", + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_L5yCLtGt5Tu" + }, + "source": [ + "> Note that if you are using command line you can simply authrnticate your machine by doing `wandb login` in your terminal. For more info check out the [documentation](https://docs.wandb.ai/quickstart#2-log-in-to-wb)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R4wGEB7ot7OZ" + }, + "source": [ + "## Run and log to W&B" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fkMGw712sg8f" + }, + "outputs": [], + "source": [ + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=microsoft/phi-2,trust_remote_code=True \\\n", + " --tasks hellaswag,mmlu_abstract_algebra \\\n", + " --device cuda:0 \\\n", + " --batch_size 8 \\\n", + " --output_path output/phi-2 \\\n", + " --limit 10 \\\n", + " --wandb_args project=lm-eval-harness-integration \\\n", + " --log_samples" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "V100", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 7ec3ae90854a9c03f4d4d1322747262686807361 Mon Sep 17 00:00:00 2001 From: ayulockin Date: Thu, 8 Feb 2024 13:23:00 +0000 Subject: [PATCH 2/4] rename --- ..._biases_with_lm_eval_harness.ipynb => lm-eval-harness.ipynb} | 2 ++ 1 file changed, 2 insertions(+) rename colabs/lm-eval-harness/{use_weights_and_biases_with_lm_eval_harness.ipynb => lm-eval-harness.ipynb} (95%) diff --git a/colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb b/colabs/lm-eval-harness/lm-eval-harness.ipynb similarity index 95% rename from colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb rename to colabs/lm-eval-harness/lm-eval-harness.ipynb index 84d7de7a..475763a5 100644 --- a/colabs/lm-eval-harness/use_weights_and_biases_with_lm_eval_harness.ipynb +++ b/colabs/lm-eval-harness/lm-eval-harness.ipynb @@ -13,6 +13,8 @@ "\n", "\n", "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/lm-eval-harness/lm-eval-harness.ipynb)\n", + "\n", "With the Weights and Biases integration, you can now spend more time extracting deeper insights into your evaluation results. The integration is designed to streamline the process of logging and visualizing experiment results using the Weights & Biases (W&B) platform.\n", "\n", "The integration provide functionalities\n", From dca40a5e749dc52717af13d1a8f42536b9f38819 Mon Sep 17 00:00:00 2001 From: Ayush Thakur Date: Thu, 13 Jun 2024 18:44:16 +0530 Subject: [PATCH 3/4] Update lm-eval-harness.ipynb --- colabs/lm-eval-harness/lm-eval-harness.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/colabs/lm-eval-harness/lm-eval-harness.ipynb b/colabs/lm-eval-harness/lm-eval-harness.ipynb index 475763a5..7963d514 100644 --- a/colabs/lm-eval-harness/lm-eval-harness.ipynb +++ b/colabs/lm-eval-harness/lm-eval-harness.ipynb @@ -41,7 +41,7 @@ "# Install this project if you did not already have it.\n", "# This is all that is needed to be installed to start using Weights and Biases\n", "\n", - "!git clone -b wandb-logging https://github.com/ayulockin/lm-evaluation-harness\n", + "!git clone https://github.com/EleutherAI/lm-evaluation-harness\n", "%cd lm-evaluation-harness\n", "!pip -qq install -e .[wandb]" ] From e47fedc67a75b776b6cf70573a6d2ab95514b605 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Mon, 10 Nov 2025 11:25:26 +0000 Subject: [PATCH 4/4] Auto-clean notebooks --- colabs/audiocraft/AudioCraft.ipynb | 20 +- ...edit_Scorecards_with_XGBoost_and_W&B.ipynb | 1 - .../Simple_LightGBM_Integration.ipynb | 1 - .../Using_W&B_Sweeps_with_XGBoost.ipynb | 1 - colabs/diffusers/pixart-alpha-diffusers.ipynb | 7281 +---------------- colabs/diffusers/sdxl-diffusers.ipynb | 19 +- .../fastai/Weights_&_Biases_with_fastai.ipynb | 1 - colabs/huggingface/Huggingface_wandb.ipynb | 1 - .../Simple_accelerate_integration_wandb.ipynb | 1 - colabs/intro/Report_API_Quickstart.ipynb | 14 +- colabs/keras/Simple_Keras_Integration.ipynb | 1 - ...bEvalCallback_in_your_Keras_workflow.ipynb | 1 - ...bMetricLogger_in_your_Keras_workflow.ipynb | 1 - ...delCheckpoint_in_your_Keras_workflow.ipynb | 1 - .../keras_core/monai_medmnist_keras.ipynb | 901 +- colabs/keras/keras_core/timm_keras.ipynb | 20 +- .../keras/keras_core/torchvision-keras.ipynb | 1376 ++-- colabs/lm-eval-harness/lm-eval-harness.ipynb | 264 +- .../monai/3d_brain_tumor_segmentation.ipynb | 18 +- .../MosaicML_Composer_and_wandb.ipynb | 940 +-- ...Azure_OpenAI_with_Weights_and_Biases.ipynb | 912 +-- .../OpenAI_API_Autologger_Quickstart.ipynb | 1 - colabs/paella/Image-Variations.ipynb | 1 - colabs/paella/Inpainting.ipynb | 1 - .../paella/Latent-Space-Interpolation.ipynb | 1 - colabs/paella/Multi-Conditioning.ipynb | 1 - ...rientation-Guided-Multi-Conditioning.ipynb | 1 - colabs/paella/Outpainting.ipynb | 1 - colabs/paella/Structural-Morphing.ipynb | 1 - colabs/paella/Text-Conditional.ipynb | 1 - .../W&B_Prompts_with_Custom_Columns.ipynb | 54 +- colabs/prompts/prompts_evaluation.ipynb | 12 +- .../pyg/point-cloud-segmentation/00_eda.ipynb | 1 - .../01_dgcnn_train.ipynb | 1 - .../02_dgcnn_evaluate.ipynb | 1 - .../pyg/pointnet-classification/00_eda.ipynb | 1 - .../01_compare_sampling.ipynb | 1 - .../02_pointnet_plus_plus.ipynb | 1 - .../pointnet-classification/03_sweep.ipynb | 1 - ...a_Transformer_with_Pytorch_Lightning.ipynb | 12 - ...assification_using_PyTorch_Lightning.ipynb | 13 - ...rch_Lightning_and_Weights_and_Biases.ipynb | 1832 ++--- ...db_End_to_End_with_PyTorch_Lightning.ipynb | 1415 ++-- ...parameter_Sweeps_in_PyTorch_with_W&B.ipynb | 1 - .../pytorch/Simple_PyTorch_Integration.ipynb | 1 - colabs/raytune/RayTune_with_wandb.ipynb | 753 +- colabs/raytune/tune-wandb.ipynb | 1 - colabs/scikit/wandb_decision_tree.ipynb | 456 +- colabs/super-gradients/yolo_nas.ipynb | 20 +- .../yolo_nas_data_analysis.ipynb | 859 +- .../super-gradients/yolo_nas_sweep_run.ipynb | 667 +- colabs/tables/W&B_Tables_Quickstart.ipynb | 1 - ...ation_in_TensorFlow_using_W&B_Sweeps.ipynb | 1 - .../Simple_TensorFlow_Integration.ipynb | 1 - colabs/ultralytics/00_inference.ipynb | 20 +- colabs/ultralytics/01_train_val.ipynb | 20 +- ...peline_Versioning_with_W&B_Artifacts.ipynb | 1 - .../New_Model_Logging_in_W&B.ipynb | 793 +- 58 files changed, 5335 insertions(+), 13389 deletions(-) diff --git a/colabs/audiocraft/AudioCraft.ipynb b/colabs/audiocraft/AudioCraft.ipynb index af385a28..1ba6cd37 100644 --- a/colabs/audiocraft/AudioCraft.ipynb +++ b/colabs/audiocraft/AudioCraft.ipynb @@ -1,22 +1,11 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ - "\"Weights\n", - "\n", - "\n", - "# 🎸 Generating Music using [Audiocraft](https://github.com/facebookresearch/audiocraft) and W&B 🐝\n", - "\n", - "\n", - "\n", - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/audiocraft/AudioCraft.ipynb)\n", - "\n", - "In this notebook we demonstrate how you can generate music and other types of audio from text prompts or generate new music from existing music using SoTA models such as [MusicGen](https://github.com/facebookresearch/audiocraft/blob/main/docs/MUSICGEN.md) and [AudioGen](https://github.com/facebookresearch/audiocraft/blob/main/docs/AUDIOGEN.md) from [Audiocraft](https://github.com/facebookresearch/audiocraft) and play and visualize them using [Weights & Biases](https://wandb.ai/site).\n", - "\n", - "If you want to know more about the underlying architectures for MusicGen and AudioGen and explore some cool audio samples generated by these models, you can check out [this W&B report](http://wandb.me/audiocraft_2mp)." + "\"Open\n", + "" ] }, { @@ -379,8 +368,9 @@ "metadata": { "accelerator": "GPU", "colab": { - "gpuType": "V100", - "provenance": [] + "include_colab_link": true, + "provenance": [], + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/colabs/boosting/Credit_Scorecards_with_XGBoost_and_W&B.ipynb b/colabs/boosting/Credit_Scorecards_with_XGBoost_and_W&B.ipynb index 57b392ef..068987b2 100644 --- a/colabs/boosting/Credit_Scorecards_with_XGBoost_and_W&B.ipynb +++ b/colabs/boosting/Credit_Scorecards_with_XGBoost_and_W&B.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/boosting/Simple_LightGBM_Integration.ipynb b/colabs/boosting/Simple_LightGBM_Integration.ipynb index 263e64df..9e183625 100644 --- a/colabs/boosting/Simple_LightGBM_Integration.ipynb +++ b/colabs/boosting/Simple_LightGBM_Integration.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb b/colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb index 5e1a2a8d..a8350da5 100644 --- a/colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb +++ b/colabs/boosting/Using_W&B_Sweeps_with_XGBoost.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/diffusers/pixart-alpha-diffusers.ipynb b/colabs/diffusers/pixart-alpha-diffusers.ipynb index 443e5704..be300a00 100644 --- a/colabs/diffusers/pixart-alpha-diffusers.ipynb +++ b/colabs/diffusers/pixart-alpha-diffusers.ipynb @@ -1,7145 +1,142 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Image Generation with Pixart-α using 🤗 Diffusers\n", - "\n", - "[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/diffusers/pixart-alpha-diffusers.ipynb)\n", - "\n", - "This notebook demonstrates the following:\n", - "- Performing text-conditional image-generations with the [Pixart-α model](https://huggingface.co/docs/diffusers/v0.23.1/en/api/pipelines/pixart) using [🤗 Diffusers](https://huggingface.co/docs/diffusers).\n", - "- Manage image generation experiments using [Weights & Biases](http://wandb.ai/site).\n", - "- Log the prompts, generated images and experiment configs to [Weigts & Biases](http://wandb.ai/site) for visalization.\n", - "\n", - "![](./assets/diffusers-autolog-1.gif)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0_F8JWIgNfqy", - "outputId": "bd6cdb65-e29c-4f93-b7a9-406993a94d62" - }, - "outputs": [], - "source": [ - "!pip install diffusers transformers accelerate sentencepiece ftfy wandb > install.log" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 87, - "referenced_widgets": [ - "95616ad88fc84e80a99f7aa0987a4372", - "2d4b6035778d4bac8a2159504533272e", - "35675cfe3b5d4c4f83fddd70c0cf8c50", - "00e83d39844348e59e7f256898e08709", - "f842eb80fed54b2ba865b75a05dae0a1", - "a3d6530e68c2427c893fcebcbae020f3", - "be51de41fd0c4ef89130862d33d57011", - "716aea1c27df41b79cf2642bb93d5df3", - "987d902ad45e4304a15dbd01fe755587", - "010dd14615ae4e58abb84cd466446b55", - "895a39fb613c406db808d1ad01dac37f" - ] - }, - "id": "_6msOo17Q0Bq", - "outputId": "049c0ad5-eb5f-4464-bc76-c0e11e1818d1" - }, - "outputs": [], - "source": [ - "import random\n", - "\n", - "import torch\n", - "from diffusers import PixArtAlphaPipeline\n", - "\n", - "import wandb\n", - "from wandb.integration.diffusers import autolog" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 579, - "referenced_widgets": [ - "950697f13cfb477cb09def01968a8be0", - "436ae734138c4274b8a0157e76c8d2e3", - "c54e4159f65549a4a4d877c224aa89ed", - "048a3dc56e8c4df39b1179d15029eb00", - "cc9cb09be76e43b591285a5d52c6ed76", - "fb7fffb9281f42b693c1b7f48325f96f", - "1ce438295bd145ad8064c9d7a9f56469", - "ab13c1cef2ff4f41b64c4ad103144b31", - "920fb69e19504d3b860f0cf55900d25d", - "006d201a04f342e89ee728d8d2538f0f", - "429b18e0a7874f6c8438cc797ba0123a", - "893b076e6f86408db6c1e542b3c730f6", - "546cbe5b663a4b9fabaff56fecc6e914", - "22802a6ccb80404e96f9912950d73252", - "6b1c8bebfaf3464c97151f0fa03d1204", - "44cbd72d7bf74a10b357ea7625b5c84b", - "314154a15a8d49da8eb158b9b64df8bc", - "da149b5bd2024a53b99282849349ea8a", - "79426ffc354b4b9daff5c64f6bea931e", - "282fd54083174b31a38efdc3f757cedb", - "5a724e34fb1b455389349963550b9da8", - "b302595b335b458899e30ff9eb0fd79a", - "b069c4b00c864783963c1bf299c5c372", - "fd5b8ec96c834a519fedba254849ec04", - "d9f9127720b541ddbbe5990e322ad089", - "3a04722dff694815a30d8c7c61f42e16", - "d30a20c93a404c02adaa4fcb66443cb2", - "dce47f13fc15450787a63016b61e1c1a", - "c4073b2d2c1d4642a1c99ce39c8fa39b", - "6ffbe6f493204f4e884692597058ad41", - "a517bf731b2748b0ae70c137beb50108", - "faa63481cc9a4ac5adbdeee025a7de19", - "2650cccd00ae4086955bed6ca909e126", - "94876605c0b14c55957f1ad227953d06", - "c4bce1fe40ef402cb8a86767f518fc70", - "c3398473b8d54655bad516100d0d0fbb", - "cc80937e599b45bc974d21f6bd5af90f", - "e6803d58f47f4b1fb4e76bca06d6b35c", - "dfa1e1cbc0044ba9823f307edbfbec1a", - "ce51bb48d8184438968b0f06b87e16d3", - "cb6eb4fc3ea44449a53d362d9819dd8e", - "062d186b597a47a08746353bf35148b1", - "0d435039170340569cf92c0255d7fd5e", - "9fc87495a9364e27a1da550d91b85550", - "4140e9203f1d4f078581ecfe3038d94b", - "fac39df9084640bbbee32339dc4613ad", - "97c1c321a42f450e939bb0baf1e42785", - "98f22a1dbac54d398191f61ed9764c6c", - "cef1102de0a14a46a5e918afc52f03bc", - "99fd909b2eb14415a79409646dd59fd2", - "41c8d0eff808446abd70ce585da26978", - "6ad951ba967b430cbff93ae75a6b1597", - "ba1c265343204064b4644f9bc9fbc0e9", - "e09197134928414c974211651e17b385", - "42901129e32343dcb8d5edb960519a53", - "13069f251f51467b83ab5119b2317337", - "32a404ef0450473eaccdd5594058b49d", - "67b4635164a04f96adca0efb40eae566", - "b61eb332e206436d93af97221feb9b4c", - "67f3fe40a0914321926c53e2fec176a5", - "49edcb2e0c8a4e9aad0398653046cf9e", - "4143365a9b69494eab538971b1602c35", - "9dbf25daca3840319f5686ff275d842f", - "08f75e1bf9864812a916f14eb29a12f9", - "6c42c3efc54f436996b154c38fb32209", - "acc056c201ef4cb8beac8d981b43ba3e", - "260f8ec597f9490d9d355664bd687dc7", - "f9730c19c2bc45a0ac95964fd7950032", - "803604e1509d4cb88fec21e96689a57b", - "10f589c69e884531b0c1ee967f523aa5", - "a86366ba25244c6da47189cf59dcbec6", - "6509119a1279497b91d29c7f922e8f3f", - "2c886cdef1d04e45a735cebaf8e05b00", - "7dc25f4f13ce4d7f94356ef9edeb2eb9", - "07e2d238d83e45bf9037fa851a5e5648", - "2e0869cc3ac24f12bf9d173cb432dc4c", - "27f2e688349d444a860dacb562634150", - "d213aeaea9f949658d1bd9728ee86fef", - "1cde89e63bd7437e9fac7446fc33aea3", - "442da82cec1241b8a5249d183820673d", - "f65994d833e04d9f96d2613a82651cdd", - "eb118c92cb514e999f7b02ebfbbc7d0b", - "a9c174e4fa204c8eb01990bcd02a43cb", - "ce7f64ac3c164f04835d7869f7ebfb48", - "4af899d9d9ba4023b4985e61e3feeda6", - "b692c6053fc04651833ee87015df0daa", - "0a4330eafca245378354d0abe07f881f", - "05c9d7dd8bca4023b8e9d67d8edcf84b", - "b9cb18bf70f843e2827a5908d86d490d", - "b7f856516aef474aac5d1755163a4775", - "973a485540b04967a277eaf9bff09fd9", - "66c72c7c741a4288a04e29284c7dab38", - "f5e977aeb3fe4edbacc09d6b9dec6df0", - "6fafd02828024e37beccd9086fb745f6", - "65bcdf864db84dff899b411bdb51d0c1", - "41d1bf639c5c4ee4afc327a64b8357b7", - "e6ee1b5afae24cf6bd731b64e53a682e", - "d10632b92b72482bb3892ec471f0e756", - "2e6f4510f5e6496f890e91f827d9cb99", - "516574616ff540778c024528a9fafec9", - "10468ffcd7c14cc79bc0fa023da9888a", - "2b2351a243144932bc0a8fbc0b542402", - "7c3adc19d763420787a6a55bb33f2d43", - "6610639505394df9b93cd39fdd0acd5c", - "fba986e08f5242afaa9ffc193ef9b3b1", - "5fc4719ad0fe4b16b0a9428ed5f32b3c", - "e9b0e81e633940e4983ceaf14687fa3b", - "1eab4186a6a24bfe98d764cfa74b342f", - "28f2f2f574f54fd6a16f244f4be5d08e", - "fb4e2edf849047ae9790f0522e13c11e", - "5e279ec61581429f84f0329ea5e62fee", - "61edf08284b942aaa7306d0a26a59344", - "c7e02eb0a33349629f58b9cbd7d3a9ad", - "0f497591d8334c47a7e01aeb3ceaef81", - "01be5b3663f348e7b47747f3371c1fbd", - "724772294d614997af26729ae0f845f9", - "ff86218d2aca4ee7b3764e998b8c9b9a", - "266d230ffacd4d76acb96a2466104289", - "8f7235cc17134152a6fc37a2a8ef85b0", - "e89ac0f3c4c848bb9505a357ca31919a", - "848aefc8f7c8452c8ac8871051166b3e", - "1200f91ecf914c6abf38c08365a9cc31", - "fe163c82f9b04a15ac6b923ed3b01873", - "729f0475028446e8bb00078a2bfbfbce", - "50f16033e1e44d05b7e9add4979a9b54", - "e28d59c860df45c9ad7318461e4ed25f", - "f650b81e97cf47b78b6512ff4a0794c6", - "d7a4b722cbc74d548edd1d0469d258a7", - "f54a32cfafb5462985b2e1f03f3e7b55", - "93141ddff63a44aba010559c6d209596", - "01d6bd13c6a743c0b82dcb51bf3e7623", - "247a4fda6dc947cc8e7336bcffcd2781", - "904ce266ca25413398b736c5f6760a5a", - "5cb62c1e29b6421d8f2b98d60543e045", - "dc0b1e41de4c4d99b6668e07e2b061a4", - "69ab97b1174b4742847ef885c7b2f4ba", - "65c618901769460bbe35f1f68d04920c", - "21f0ae78db2f4cd09c0b64f4ae5f7b8a", - "bf028fd02c0e4220a3077c4f41253ed7", - "e29ec1b24ede4619881a3e011699c434", - "75343e1ba0604632abf96e86ad7fe8b3", - "9ff029954ed64721a4f00faf34f701c5", - "f91b715ea2ae4fe0970c142ddc59d76e", - "32f4a8a631074da49f3361ea74b483fd", - "0a427178ccca414c8f58b892b54cb00d", - "591c006b4de647d2b95a8d3a6103a48b", - "aa04c220c98d4d6f880a58f6aa9bda75", - "04e94e56da49440abf2b6eaf6e675d98", - "bdeeb0f9825147efbb70a5eaf226a8ff", - "f7dabfcad74b4838b3ad63d62457f140", - "a6801b2c8f41424888dbafde4252c7da", - "5e8efc02014a455dbfbe356c965abd02", - "b9573401b606405da3b1d8815865b104", - "cf75087ea8c4401faebff1fe50cc8703", - "d96719a9e06845aaae085bd256148333", - "4497b0d5517a4acabba98e60af7c5677", - "fe96da4d0e484f79ab3a82820ddea5ea", - "d7b8ca6b0cf549af8f3f8b0e15f0b8af", - "2cb5315f2ad748c4a8605208dc9492d1", - "d3406fb70a4c470fa4cd9476d60a9370", - "63dc113923d145a88de3c2156b73a1ad", - "9690ca75724a4547a505b457efa18cf5", - "b9e34a78891a4be6b293733ad79fb409", - "cb7c327783124585a21b6062b9a4bec7", - "5f13dc4e83c140c1a020bebcd08943d5", - "2631936da4064c2b8763bb9bd081b264", - "3df3134ee2c14677b1ed6eb49ece2f46", - "93a02724ca12427998e17f8e355b0012", - "1ec067c782824e329661b9a903748caa", - "4166654783344449b71d946124b02703", - "9ee2b2a0e18d420ea09a202ce2804660", - "b0f80fbce966413496bf67440107cc5b", - "ee7cb7091ca5458bba7ab8cef3bb4b5e", - "56e272c0af234570a5371800c336ec4d", - "266723871e2c4f80bebc0e758b58374d", - "3c9728a91b5d485f81ef4d7814c6bb22", - "2845309eb6cd4e3bb91b1f3f0cfb6f43", - "251d69e9aac04d0da25fa6cb18ce907c", - "2466f004e7654d578536b3bdd9e47db2", - "ed3e92da5ade4986b381ef86e0d57ed4", - "98271fb9547940c69ff8958320056ca5", - "1e4fba8ea112482cab2df3e917f6407c", - "c001ad33ee0e4df1a314228e292529b5", - "532599e42fc64bfdb57957454312ed00", - "6c2bbdfd6e6f4a8faa2d47a81a0feebd", - "2f428cbd696a4ba4906c7fd8f74a57d4", - "f1bea04743fd4a63aaa1b03acdac859d" - ] - }, - "id": "O4WktKfiQ0z7", - "outputId": "0be1b5e0-cfb2-4b1d-f9e4-18d97b92377c" - }, - "outputs": [], - "source": [ - "# Load the pre-trained checkpoints from HuggingFace Hub to the PixArtAlphaPipeline\n", - "pipe = PixArtAlphaPipeline.from_pretrained(\n", - " \"PixArt-alpha/PixArt-XL-2-1024-MS\", torch_dtype=torch.float16\n", - ")\n", - "\n", - "# Enable offloading the weights to the CPU and only loading them on the GPU when\n", - "# performing the forward pass can also save memory.\n", - "pipe.enable_model_cpu_offload()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "MQIZo4F_Q1qG" - }, - "outputs": [], - "source": [ - "wandb_project = \"pixart-alpha\" # @param {type:\"string\"}\n", - "\n", - "prompt = \"a traveler navigating via a boat in countless mountains, Chinese ink painting\" # @param {type:\"string\"}\n", - "negative_prompt = \"\" # @param {type:\"string\"}\n", - "num_inference_steps = 25 # @param {type:\"slider\", min:10, max:50, step:1}\n", - "guidance_scale = 4.5 # @param {type:\"slider\", min:0, max:10, step:0.1}\n", - "num_images_per_prompt = 1 # @param {type:\"slider\", min:0, max:10, step:0.1}\n", - "height = 1024 # @param {type:\"slider\", min:512, max:2560, step:32}\n", - "width = 1024 # @param {type:\"slider\", min:512, max:2560, step:32}\n", - "seed = None # @param {type:\"raw\"}\n", - "\n", - "\n", - "def autogenerate_seed():\n", - " max_seed = int(1024 * 1024 * 1024)\n", - " seed = random.randint(1, max_seed)\n", - " seed = -seed if seed < 0 else seed\n", - " seed = seed % max_seed\n", - " return seed\n", - "\n", - "\n", - "seed = autogenerate_seed() if seed is None else seed\n", - "\n", - "# Make the experiment reproducible by controlling randomness.\n", - "# The seed would be automatically logged to WandB.\n", - "generator = torch.Generator(device=\"cuda\").manual_seed(seed)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 206, - "referenced_widgets": [ - "68239504cbdf4aa0a80096171156cffd", - "85ab89a71bb248e996fb8d0148a3d64c", - "ceeef5ea2da54e3b8f6fcda7f33b4e32", - "5498cc4574d74bf894f35c96d74dce80", - "529b32124a884391ba1176d340d480f9", - "95caf26b330740a593c8cc86a2fd1ee1", - "76d11a364eae4686a5879cddb98c4ddd", - "8ece46fc480c4d399e7f2b7a92e96cec", - "73afbd0ff1404c63b12909c609446cf9", - "601ce4737c2c46b493a191c7f3f6b950", - "2079c259a28943a0a80ddabc44026a39", - "0356e4a80e04409ca07d1581d735086f", - "609dff85b2104270a8e9263ee8e1e958", - "a25969fd35744afdb66e9fd59612df6e", - "4964cf137c504555a7df44112e65cc0b", - "52a240218ab84c0896c7bdc4bfe5a0ef", - "bd8e4b6f1e3f49a586fc2ca9baca58e3", - "ca2e80da3e194102a9ff2cf0880e1e99", - "8dae7b8cdf004d748be7a6395c45bd1a" - ] - }, - "id": "QfopfTm-Q4S3", - "outputId": "ca4cd35d-eac4-44e7-c7fb-b019e40a00e9" - }, - "outputs": [], - "source": [ - "# Call WandB Autolog for Diffusers. This would automatically log\n", - "# the prompts, generated images, pipeline architecture and all\n", - "# associated experiment configs to Weights & Biases, thus making your\n", - "# image generation experiments easy to reproduce, share and analyze.\n", - "autolog(init=dict(project=wandb_project))\n", - "\n", - "# Generate the images by calling the PixArtAlphaPipeline\n", - "image = pipe(\n", - " prompt=prompt,\n", - " negative_prompt=negative_prompt,\n", - " num_inference_steps=num_inference_steps,\n", - " guidance_scale=guidance_scale,\n", - " num_images_per_prompt=num_images_per_prompt,\n", - " height=height,\n", - " width=width,\n", - " generator=generator,\n", - ").images[0]\n", - "\n", - "# End the experiment\n", - "wandb.finish()" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "machine_shape": "hm", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "006d201a04f342e89ee728d8d2538f0f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "00e83d39844348e59e7f256898e08709": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_010dd14615ae4e58abb84cd466446b55", - "placeholder": "​", - "style": "IPY_MODEL_895a39fb613c406db808d1ad01dac37f", - "value": " 0/0 [00:00<?, ?it/s]" - } - }, - "010dd14615ae4e58abb84cd466446b55": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "01be5b3663f348e7b47747f3371c1fbd": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "01d6bd13c6a743c0b82dcb51bf3e7623": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0356e4a80e04409ca07d1581d735086f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "VBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "VBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "VBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_609dff85b2104270a8e9263ee8e1e958", - "IPY_MODEL_a25969fd35744afdb66e9fd59612df6e" - ], - "layout": "IPY_MODEL_4964cf137c504555a7df44112e65cc0b" - } - }, - "048a3dc56e8c4df39b1179d15029eb00": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_006d201a04f342e89ee728d8d2538f0f", - "placeholder": "​", - "style": "IPY_MODEL_429b18e0a7874f6c8438cc797ba0123a", - "value": " 400/400 [00:00<00:00, 27.8kB/s]" - } - }, - "04e94e56da49440abf2b6eaf6e675d98": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "05c9d7dd8bca4023b8e9d67d8edcf84b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "062d186b597a47a08746353bf35148b1": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "07e2d238d83e45bf9037fa851a5e5648": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "08f75e1bf9864812a916f14eb29a12f9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "0a427178ccca414c8f58b892b54cb00d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_bdeeb0f9825147efbb70a5eaf226a8ff", - "placeholder": "​", - "style": "IPY_MODEL_f7dabfcad74b4838b3ad63d62457f140", - "value": "diffusion_pytorch_model.safetensors: 100%" - } - }, - "0a4330eafca245378354d0abe07f881f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0d435039170340569cf92c0255d7fd5e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "0f497591d8334c47a7e01aeb3ceaef81": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e89ac0f3c4c848bb9505a357ca31919a", - "placeholder": "​", - "style": "IPY_MODEL_848aefc8f7c8452c8ac8871051166b3e", - "value": " 9.99G/9.99G [01:18<00:00, 159MB/s]" - } - }, - "10468ffcd7c14cc79bc0fa023da9888a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fba986e08f5242afaa9ffc193ef9b3b1", - "placeholder": "​", - "style": "IPY_MODEL_5fc4719ad0fe4b16b0a9428ed5f32b3c", - "value": "vae/config.json: 100%" - } - }, - "10f589c69e884531b0c1ee967f523aa5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2e0869cc3ac24f12bf9d173cb432dc4c", - "placeholder": "​", - "style": "IPY_MODEL_27f2e688349d444a860dacb562634150", - "value": " 2.63k/2.63k [00:00<00:00, 56.9kB/s]" - } - }, - "1200f91ecf914c6abf38c08365a9cc31": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_fe163c82f9b04a15ac6b923ed3b01873", - "IPY_MODEL_729f0475028446e8bb00078a2bfbfbce", - "IPY_MODEL_50f16033e1e44d05b7e9add4979a9b54" - ], - "layout": "IPY_MODEL_e28d59c860df45c9ad7318461e4ed25f" - } - }, - "13069f251f51467b83ab5119b2317337": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_32a404ef0450473eaccdd5594058b49d", - "IPY_MODEL_67b4635164a04f96adca0efb40eae566", - "IPY_MODEL_b61eb332e206436d93af97221feb9b4c" - ], - "layout": "IPY_MODEL_67f3fe40a0914321926c53e2fec176a5" - } - }, - "1cde89e63bd7437e9fac7446fc33aea3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a9c174e4fa204c8eb01990bcd02a43cb", - "placeholder": "​", - "style": "IPY_MODEL_ce7f64ac3c164f04835d7869f7ebfb48", - "value": "tokenizer/tokenizer_config.json: 100%" - } - }, - "1ce438295bd145ad8064c9d7a9f56469": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "1e4fba8ea112482cab2df3e917f6407c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1eab4186a6a24bfe98d764cfa74b342f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "1ec067c782824e329661b9a903748caa": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_266723871e2c4f80bebc0e758b58374d", - "placeholder": "​", - "style": "IPY_MODEL_3c9728a91b5d485f81ef4d7814c6bb22", - "value": " 5/5 [00:11<00:00, 2.08s/it]" - } - }, - "2079c259a28943a0a80ddabc44026a39": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "21f0ae78db2f4cd09c0b64f4ae5f7b8a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "22802a6ccb80404e96f9912950d73252": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_79426ffc354b4b9daff5c64f6bea931e", - "max": 14, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_282fd54083174b31a38efdc3f757cedb", - "value": 14 - } - }, - "2466f004e7654d578536b3bdd9e47db2": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_532599e42fc64bfdb57957454312ed00", - "max": 2, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_6c2bbdfd6e6f4a8faa2d47a81a0feebd", - "value": 2 - } - }, - "247a4fda6dc947cc8e7336bcffcd2781": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "251d69e9aac04d0da25fa6cb18ce907c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_1e4fba8ea112482cab2df3e917f6407c", - "placeholder": "​", - "style": "IPY_MODEL_c001ad33ee0e4df1a314228e292529b5", - "value": "Loading checkpoint shards: 100%" - } - }, - "260f8ec597f9490d9d355664bd687dc7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f9730c19c2bc45a0ac95964fd7950032", - "IPY_MODEL_803604e1509d4cb88fec21e96689a57b", - "IPY_MODEL_10f589c69e884531b0c1ee967f523aa5" - ], - "layout": "IPY_MODEL_a86366ba25244c6da47189cf59dcbec6" - } - }, - "2631936da4064c2b8763bb9bd081b264": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_3df3134ee2c14677b1ed6eb49ece2f46", - "IPY_MODEL_93a02724ca12427998e17f8e355b0012", - "IPY_MODEL_1ec067c782824e329661b9a903748caa" - ], - "layout": "IPY_MODEL_4166654783344449b71d946124b02703" - } - }, - "2650cccd00ae4086955bed6ca909e126": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "266723871e2c4f80bebc0e758b58374d": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "266d230ffacd4d76acb96a2466104289": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "27f2e688349d444a860dacb562634150": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "282fd54083174b31a38efdc3f757cedb": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "2845309eb6cd4e3bb91b1f3f0cfb6f43": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_251d69e9aac04d0da25fa6cb18ce907c", - "IPY_MODEL_2466f004e7654d578536b3bdd9e47db2", - "IPY_MODEL_ed3e92da5ade4986b381ef86e0d57ed4" - ], - "layout": "IPY_MODEL_98271fb9547940c69ff8958320056ca5" - } - }, - "28f2f2f574f54fd6a16f244f4be5d08e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2b2351a243144932bc0a8fbc0b542402": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e9b0e81e633940e4983ceaf14687fa3b", - "max": 654, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_1eab4186a6a24bfe98d764cfa74b342f", - "value": 654 - } - }, - "2c886cdef1d04e45a735cebaf8e05b00": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2cb5315f2ad748c4a8605208dc9492d1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2d4b6035778d4bac8a2159504533272e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a3d6530e68c2427c893fcebcbae020f3", - "placeholder": "​", - "style": "IPY_MODEL_be51de41fd0c4ef89130862d33d57011", - "value": "" - } - }, - "2e0869cc3ac24f12bf9d173cb432dc4c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "2e6f4510f5e6496f890e91f827d9cb99": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "2f428cbd696a4ba4906c7fd8f74a57d4": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "314154a15a8d49da8eb158b9b64df8bc": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "32a404ef0450473eaccdd5594058b49d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_49edcb2e0c8a4e9aad0398653046cf9e", - "placeholder": "​", - "style": "IPY_MODEL_4143365a9b69494eab538971b1602c35", - "value": "(…)ext_encoder/model.safetensors.index.json: 100%" - } - }, - "32f4a8a631074da49f3361ea74b483fd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_0a427178ccca414c8f58b892b54cb00d", - "IPY_MODEL_591c006b4de647d2b95a8d3a6103a48b", - "IPY_MODEL_aa04c220c98d4d6f880a58f6aa9bda75" - ], - "layout": "IPY_MODEL_04e94e56da49440abf2b6eaf6e675d98" - } - }, - "35675cfe3b5d4c4f83fddd70c0cf8c50": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_716aea1c27df41b79cf2642bb93d5df3", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_987d902ad45e4304a15dbd01fe755587", - "value": 0 - } - }, - "3a04722dff694815a30d8c7c61f42e16": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_faa63481cc9a4ac5adbdeee025a7de19", - "placeholder": "​", - "style": "IPY_MODEL_2650cccd00ae4086955bed6ca909e126", - "value": " 655/655 [00:00<00:00, 48.5kB/s]" - } - }, - "3c9728a91b5d485f81ef4d7814c6bb22": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "3df3134ee2c14677b1ed6eb49ece2f46": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9ee2b2a0e18d420ea09a202ce2804660", - "placeholder": "​", - "style": "IPY_MODEL_b0f80fbce966413496bf67440107cc5b", - "value": "Loading pipeline components...: 100%" - } - }, - "4140e9203f1d4f078581ecfe3038d94b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_fac39df9084640bbbee32339dc4613ad", - "IPY_MODEL_97c1c321a42f450e939bb0baf1e42785", - "IPY_MODEL_98f22a1dbac54d398191f61ed9764c6c" - ], - "layout": "IPY_MODEL_cef1102de0a14a46a5e918afc52f03bc" - } - }, - "4143365a9b69494eab538971b1602c35": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "4166654783344449b71d946124b02703": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "41c8d0eff808446abd70ce585da26978": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "41d1bf639c5c4ee4afc327a64b8357b7": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "42901129e32343dcb8d5edb960519a53": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "429b18e0a7874f6c8438cc797ba0123a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "436ae734138c4274b8a0157e76c8d2e3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_fb7fffb9281f42b693c1b7f48325f96f", - "placeholder": "​", - "style": "IPY_MODEL_1ce438295bd145ad8064c9d7a9f56469", - "value": "model_index.json: 100%" - } - }, - "442da82cec1241b8a5249d183820673d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4af899d9d9ba4023b4985e61e3feeda6", - "max": 20509, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b692c6053fc04651833ee87015df0daa", - "value": 20509 - } - }, - "4497b0d5517a4acabba98e60af7c5677": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d3406fb70a4c470fa4cd9476d60a9370", - "placeholder": "​", - "style": "IPY_MODEL_63dc113923d145a88de3c2156b73a1ad", - "value": "diffusion_pytorch_model.safetensors: 100%" - } - }, - "44cbd72d7bf74a10b357ea7625b5c84b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4964cf137c504555a7df44112e65cc0b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "49edcb2e0c8a4e9aad0398653046cf9e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "4af899d9d9ba4023b4985e61e3feeda6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "50f16033e1e44d05b7e9add4979a9b54": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_01d6bd13c6a743c0b82dcb51bf3e7623", - "placeholder": "​", - "style": "IPY_MODEL_247a4fda6dc947cc8e7336bcffcd2781", - "value": " 9.06G/9.06G [01:16<00:00, 150MB/s]" - } - }, - "516574616ff540778c024528a9fafec9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_10468ffcd7c14cc79bc0fa023da9888a", - "IPY_MODEL_2b2351a243144932bc0a8fbc0b542402", - "IPY_MODEL_7c3adc19d763420787a6a55bb33f2d43" - ], - "layout": "IPY_MODEL_6610639505394df9b93cd39fdd0acd5c" - } - }, - "529b32124a884391ba1176d340d480f9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "52a240218ab84c0896c7bdc4bfe5a0ef": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "532599e42fc64bfdb57957454312ed00": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "546cbe5b663a4b9fabaff56fecc6e914": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_314154a15a8d49da8eb158b9b64df8bc", - "placeholder": "​", - "style": "IPY_MODEL_da149b5bd2024a53b99282849349ea8a", - "value": "Fetching 14 files: 100%" - } - }, - "5498cc4574d74bf894f35c96d74dce80": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_601ce4737c2c46b493a191c7f3f6b950", - "placeholder": "​", - "style": "IPY_MODEL_2079c259a28943a0a80ddabc44026a39", - "value": " 25/25 [00:32<00:00, 1.07s/it]" - } - }, - "56e272c0af234570a5371800c336ec4d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "591c006b4de647d2b95a8d3a6103a48b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a6801b2c8f41424888dbafde4252c7da", - "max": 2447431856, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_5e8efc02014a455dbfbe356c965abd02", - "value": 2447431856 - } - }, - "5a724e34fb1b455389349963550b9da8": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5cb62c1e29b6421d8f2b98d60543e045": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_21f0ae78db2f4cd09c0b64f4ae5f7b8a", - "placeholder": "​", - "style": "IPY_MODEL_bf028fd02c0e4220a3077c4f41253ed7", - "value": "spiece.model: 100%" - } - }, - "5e279ec61581429f84f0329ea5e62fee": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_61edf08284b942aaa7306d0a26a59344", - "IPY_MODEL_c7e02eb0a33349629f58b9cbd7d3a9ad", - "IPY_MODEL_0f497591d8334c47a7e01aeb3ceaef81" - ], - "layout": "IPY_MODEL_01be5b3663f348e7b47747f3371c1fbd" - } - }, - "5e8efc02014a455dbfbe356c965abd02": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "5f13dc4e83c140c1a020bebcd08943d5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5fc4719ad0fe4b16b0a9428ed5f32b3c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "601ce4737c2c46b493a191c7f3f6b950": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "609dff85b2104270a8e9263ee8e1e958": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "LabelModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "LabelModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "LabelView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_52a240218ab84c0896c7bdc4bfe5a0ef", - "placeholder": "​", - "style": "IPY_MODEL_bd8e4b6f1e3f49a586fc2ca9baca58e3", - "value": "3.327 MB of 3.327 MB uploaded (0.008 MB deduped)\r" - } - }, - "61edf08284b942aaa7306d0a26a59344": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_724772294d614997af26729ae0f845f9", - "placeholder": "​", - "style": "IPY_MODEL_ff86218d2aca4ee7b3764e998b8c9b9a", - "value": "model-00001-of-00002.safetensors: 100%" - } - }, - "63dc113923d145a88de3c2156b73a1ad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "6509119a1279497b91d29c7f922e8f3f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "65bcdf864db84dff899b411bdb51d0c1": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "65c618901769460bbe35f1f68d04920c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6610639505394df9b93cd39fdd0acd5c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "66c72c7c741a4288a04e29284c7dab38": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_d10632b92b72482bb3892ec471f0e756", - "placeholder": "​", - "style": "IPY_MODEL_2e6f4510f5e6496f890e91f827d9cb99", - "value": " 719/719 [00:00<00:00, 25.4kB/s]" - } - }, - "67b4635164a04f96adca0efb40eae566": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9dbf25daca3840319f5686ff275d842f", - "max": 19886, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_08f75e1bf9864812a916f14eb29a12f9", - "value": 19886 - } - }, - "67f3fe40a0914321926c53e2fec176a5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "68239504cbdf4aa0a80096171156cffd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_85ab89a71bb248e996fb8d0148a3d64c", - "IPY_MODEL_ceeef5ea2da54e3b8f6fcda7f33b4e32", - "IPY_MODEL_5498cc4574d74bf894f35c96d74dce80" - ], - "layout": "IPY_MODEL_529b32124a884391ba1176d340d480f9" - } - }, - "69ab97b1174b4742847ef885c7b2f4ba": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9ff029954ed64721a4f00faf34f701c5", - "placeholder": "​", - "style": "IPY_MODEL_f91b715ea2ae4fe0970c142ddc59d76e", - "value": " 792k/792k [00:00<00:00, 4.76MB/s]" - } - }, - "6ad951ba967b430cbff93ae75a6b1597": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6b1c8bebfaf3464c97151f0fa03d1204": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5a724e34fb1b455389349963550b9da8", - "placeholder": "​", - "style": "IPY_MODEL_b302595b335b458899e30ff9eb0fd79a", - "value": " 14/14 [01:20<00:00, 23.51s/it]" - } - }, - "6c2bbdfd6e6f4a8faa2d47a81a0feebd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "6c42c3efc54f436996b154c38fb32209": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6fafd02828024e37beccd9086fb745f6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6ffbe6f493204f4e884692597058ad41": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "716aea1c27df41b79cf2642bb93d5df3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": "20px" - } - }, - "724772294d614997af26729ae0f845f9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "729f0475028446e8bb00078a2bfbfbce": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f54a32cfafb5462985b2e1f03f3e7b55", - "max": 9060119392, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_93141ddff63a44aba010559c6d209596", - "value": 9060119392 - } - }, - "73afbd0ff1404c63b12909c609446cf9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "75343e1ba0604632abf96e86ad7fe8b3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "76d11a364eae4686a5879cddb98c4ddd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "79426ffc354b4b9daff5c64f6bea931e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7c3adc19d763420787a6a55bb33f2d43": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_28f2f2f574f54fd6a16f244f4be5d08e", - "placeholder": "​", - "style": "IPY_MODEL_fb4e2edf849047ae9790f0522e13c11e", - "value": " 654/654 [00:00<00:00, 28.9kB/s]" - } - }, - "7dc25f4f13ce4d7f94356ef9edeb2eb9": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "803604e1509d4cb88fec21e96689a57b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7dc25f4f13ce4d7f94356ef9edeb2eb9", - "max": 2634, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_07e2d238d83e45bf9037fa851a5e5648", - "value": 2634 - } - }, - "848aefc8f7c8452c8ac8871051166b3e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "85ab89a71bb248e996fb8d0148a3d64c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_95caf26b330740a593c8cc86a2fd1ee1", - "placeholder": "​", - "style": "IPY_MODEL_76d11a364eae4686a5879cddb98c4ddd", - "value": "100%" - } - }, - "893b076e6f86408db6c1e542b3c730f6": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_546cbe5b663a4b9fabaff56fecc6e914", - "IPY_MODEL_22802a6ccb80404e96f9912950d73252", - "IPY_MODEL_6b1c8bebfaf3464c97151f0fa03d1204" - ], - "layout": "IPY_MODEL_44cbd72d7bf74a10b357ea7625b5c84b" - } - }, - "895a39fb613c406db808d1ad01dac37f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "8dae7b8cdf004d748be7a6395c45bd1a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "8ece46fc480c4d399e7f2b7a92e96cec": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "8f7235cc17134152a6fc37a2a8ef85b0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "904ce266ca25413398b736c5f6760a5a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_5cb62c1e29b6421d8f2b98d60543e045", - "IPY_MODEL_dc0b1e41de4c4d99b6668e07e2b061a4", - "IPY_MODEL_69ab97b1174b4742847ef885c7b2f4ba" - ], - "layout": "IPY_MODEL_65c618901769460bbe35f1f68d04920c" - } - }, - "920fb69e19504d3b860f0cf55900d25d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "93141ddff63a44aba010559c6d209596": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "93a02724ca12427998e17f8e355b0012": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ee7cb7091ca5458bba7ab8cef3bb4b5e", - "max": 5, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_56e272c0af234570a5371800c336ec4d", - "value": 5 - } - }, - "94876605c0b14c55957f1ad227953d06": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_c4bce1fe40ef402cb8a86767f518fc70", - "IPY_MODEL_c3398473b8d54655bad516100d0d0fbb", - "IPY_MODEL_cc80937e599b45bc974d21f6bd5af90f" - ], - "layout": "IPY_MODEL_e6803d58f47f4b1fb4e76bca06d6b35c" - } - }, - "950697f13cfb477cb09def01968a8be0": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_436ae734138c4274b8a0157e76c8d2e3", - "IPY_MODEL_c54e4159f65549a4a4d877c224aa89ed", - "IPY_MODEL_048a3dc56e8c4df39b1179d15029eb00" - ], - "layout": "IPY_MODEL_cc9cb09be76e43b591285a5d52c6ed76" - } - }, - "95616ad88fc84e80a99f7aa0987a4372": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_2d4b6035778d4bac8a2159504533272e", - "IPY_MODEL_35675cfe3b5d4c4f83fddd70c0cf8c50", - "IPY_MODEL_00e83d39844348e59e7f256898e08709" - ], - "layout": "IPY_MODEL_f842eb80fed54b2ba865b75a05dae0a1" - } - }, - "95caf26b330740a593c8cc86a2fd1ee1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9690ca75724a4547a505b457efa18cf5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "973a485540b04967a277eaf9bff09fd9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_41d1bf639c5c4ee4afc327a64b8357b7", - "max": 719, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_e6ee1b5afae24cf6bd731b64e53a682e", - "value": 719 - } - }, - "97c1c321a42f450e939bb0baf1e42785": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6ad951ba967b430cbff93ae75a6b1597", - "max": 788, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_ba1c265343204064b4644f9bc9fbc0e9", - "value": 788 - } - }, - "98271fb9547940c69ff8958320056ca5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "987d902ad45e4304a15dbd01fe755587": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "98f22a1dbac54d398191f61ed9764c6c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e09197134928414c974211651e17b385", - "placeholder": "​", - "style": "IPY_MODEL_42901129e32343dcb8d5edb960519a53", - "value": " 788/788 [00:00<00:00, 15.2kB/s]" - } - }, - "99fd909b2eb14415a79409646dd59fd2": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9dbf25daca3840319f5686ff275d842f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9ee2b2a0e18d420ea09a202ce2804660": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "9fc87495a9364e27a1da550d91b85550": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "9ff029954ed64721a4f00faf34f701c5": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a25969fd35744afdb66e9fd59612df6e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ca2e80da3e194102a9ff2cf0880e1e99", - "max": 1, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_8dae7b8cdf004d748be7a6395c45bd1a", - "value": 1 - } - }, - "a3d6530e68c2427c893fcebcbae020f3": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a517bf731b2748b0ae70c137beb50108": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "a6801b2c8f41424888dbafde4252c7da": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a86366ba25244c6da47189cf59dcbec6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a9c174e4fa204c8eb01990bcd02a43cb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "aa04c220c98d4d6f880a58f6aa9bda75": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b9573401b606405da3b1d8815865b104", - "placeholder": "​", - "style": "IPY_MODEL_cf75087ea8c4401faebff1fe50cc8703", - "value": " 2.45G/2.45G [00:31<00:00, 130MB/s]" - } - }, - "ab13c1cef2ff4f41b64c4ad103144b31": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "acc056c201ef4cb8beac8d981b43ba3e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b069c4b00c864783963c1bf299c5c372": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_fd5b8ec96c834a519fedba254849ec04", - "IPY_MODEL_d9f9127720b541ddbbe5990e322ad089", - "IPY_MODEL_3a04722dff694815a30d8c7c61f42e16" - ], - "layout": "IPY_MODEL_d30a20c93a404c02adaa4fcb66443cb2" - } - }, - "b0f80fbce966413496bf67440107cc5b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b302595b335b458899e30ff9eb0fd79a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "b61eb332e206436d93af97221feb9b4c": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6c42c3efc54f436996b154c38fb32209", - "placeholder": "​", - "style": "IPY_MODEL_acc056c201ef4cb8beac8d981b43ba3e", - "value": " 19.9k/19.9k [00:00<00:00, 425kB/s]" - } - }, - "b692c6053fc04651833ee87015df0daa": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "b7f856516aef474aac5d1755163a4775": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6fafd02828024e37beccd9086fb745f6", - "placeholder": "​", - "style": "IPY_MODEL_65bcdf864db84dff899b411bdb51d0c1", - "value": "transformer/config.json: 100%" - } - }, - "b9573401b606405da3b1d8815865b104": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b9cb18bf70f843e2827a5908d86d490d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_b7f856516aef474aac5d1755163a4775", - "IPY_MODEL_973a485540b04967a277eaf9bff09fd9", - "IPY_MODEL_66c72c7c741a4288a04e29284c7dab38" - ], - "layout": "IPY_MODEL_f5e977aeb3fe4edbacc09d6b9dec6df0" - } - }, - "b9e34a78891a4be6b293733ad79fb409": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "ba1c265343204064b4644f9bc9fbc0e9": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "bd8e4b6f1e3f49a586fc2ca9baca58e3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bdeeb0f9825147efbb70a5eaf226a8ff": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "be51de41fd0c4ef89130862d33d57011": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bf028fd02c0e4220a3077c4f41253ed7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c001ad33ee0e4df1a314228e292529b5": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c3398473b8d54655bad516100d0d0fbb": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cb6eb4fc3ea44449a53d362d9819dd8e", - "max": 2201, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_062d186b597a47a08746353bf35148b1", - "value": 2201 - } - }, - "c4073b2d2c1d4642a1c99ce39c8fa39b": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "c4bce1fe40ef402cb8a86767f518fc70": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_dfa1e1cbc0044ba9823f307edbfbec1a", - "placeholder": "​", - "style": "IPY_MODEL_ce51bb48d8184438968b0f06b87e16d3", - "value": "tokenizer/special_tokens_map.json: 100%" - } - }, - "c54e4159f65549a4a4d877c224aa89ed": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_ab13c1cef2ff4f41b64c4ad103144b31", - "max": 400, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_920fb69e19504d3b860f0cf55900d25d", - "value": 400 - } - }, - "c7e02eb0a33349629f58b9cbd7d3a9ad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_266d230ffacd4d76acb96a2466104289", - "max": 9989150328, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_8f7235cc17134152a6fc37a2a8ef85b0", - "value": 9989150328 - } - }, - "ca2e80da3e194102a9ff2cf0880e1e99": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cb6eb4fc3ea44449a53d362d9819dd8e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cb7c327783124585a21b6062b9a4bec7": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cc80937e599b45bc974d21f6bd5af90f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0d435039170340569cf92c0255d7fd5e", - "placeholder": "​", - "style": "IPY_MODEL_9fc87495a9364e27a1da550d91b85550", - "value": " 2.20k/2.20k [00:00<00:00, 60.1kB/s]" - } - }, - "cc9cb09be76e43b591285a5d52c6ed76": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ce51bb48d8184438968b0f06b87e16d3": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ce7f64ac3c164f04835d7869f7ebfb48": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "ceeef5ea2da54e3b8f6fcda7f33b4e32": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_8ece46fc480c4d399e7f2b7a92e96cec", - "max": 25, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_73afbd0ff1404c63b12909c609446cf9", - "value": 25 - } - }, - "cef1102de0a14a46a5e918afc52f03bc": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "cf75087ea8c4401faebff1fe50cc8703": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "d10632b92b72482bb3892ec471f0e756": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d213aeaea9f949658d1bd9728ee86fef": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_1cde89e63bd7437e9fac7446fc33aea3", - "IPY_MODEL_442da82cec1241b8a5249d183820673d", - "IPY_MODEL_f65994d833e04d9f96d2613a82651cdd" - ], - "layout": "IPY_MODEL_eb118c92cb514e999f7b02ebfbbc7d0b" - } - }, - "d30a20c93a404c02adaa4fcb66443cb2": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d3406fb70a4c470fa4cd9476d60a9370": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "d7a4b722cbc74d548edd1d0469d258a7": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "d7b8ca6b0cf549af8f3f8b0e15f0b8af": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_cb7c327783124585a21b6062b9a4bec7", - "placeholder": "​", - "style": "IPY_MODEL_5f13dc4e83c140c1a020bebcd08943d5", - "value": " 335M/335M [00:03<00:00, 92.4MB/s]" - } - }, - "d96719a9e06845aaae085bd256148333": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_4497b0d5517a4acabba98e60af7c5677", - "IPY_MODEL_fe96da4d0e484f79ab3a82820ddea5ea", - "IPY_MODEL_d7b8ca6b0cf549af8f3f8b0e15f0b8af" - ], - "layout": "IPY_MODEL_2cb5315f2ad748c4a8605208dc9492d1" - } - }, - "d9f9127720b541ddbbe5990e322ad089": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6ffbe6f493204f4e884692597058ad41", - "max": 655, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_a517bf731b2748b0ae70c137beb50108", - "value": 655 - } - }, - "da149b5bd2024a53b99282849349ea8a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "dc0b1e41de4c4d99b6668e07e2b061a4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_e29ec1b24ede4619881a3e011699c434", - "max": 791656, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_75343e1ba0604632abf96e86ad7fe8b3", - "value": 791656 - } - }, - "dce47f13fc15450787a63016b61e1c1a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dfa1e1cbc0044ba9823f307edbfbec1a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e09197134928414c974211651e17b385": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e28d59c860df45c9ad7318461e4ed25f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e29ec1b24ede4619881a3e011699c434": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e6803d58f47f4b1fb4e76bca06d6b35c": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e6ee1b5afae24cf6bd731b64e53a682e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "e89ac0f3c4c848bb9505a357ca31919a": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "e9b0e81e633940e4983ceaf14687fa3b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "eb118c92cb514e999f7b02ebfbbc7d0b": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "ed3e92da5ade4986b381ef86e0d57ed4": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_2f428cbd696a4ba4906c7fd8f74a57d4", - "placeholder": "​", - "style": "IPY_MODEL_f1bea04743fd4a63aaa1b03acdac859d", - "value": " 2/2 [00:02<00:00, 1.17s/it]" - } - }, - "ee7cb7091ca5458bba7ab8cef3bb4b5e": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f1bea04743fd4a63aaa1b03acdac859d": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f54a32cfafb5462985b2e1f03f3e7b55": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f5e977aeb3fe4edbacc09d6b9dec6df0": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f650b81e97cf47b78b6512ff4a0794c6": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f65994d833e04d9f96d2613a82651cdd": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_0a4330eafca245378354d0abe07f881f", - "placeholder": "​", - "style": "IPY_MODEL_05c9d7dd8bca4023b8e9d67d8edcf84b", - "value": " 20.5k/20.5k [00:00<00:00, 1.28MB/s]" - } - }, - "f7dabfcad74b4838b3ad63d62457f140": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f842eb80fed54b2ba865b75a05dae0a1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "f91b715ea2ae4fe0970c142ddc59d76e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "f9730c19c2bc45a0ac95964fd7950032": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_6509119a1279497b91d29c7f922e8f3f", - "placeholder": "​", - "style": "IPY_MODEL_2c886cdef1d04e45a735cebaf8e05b00", - "value": "tokenizer/added_tokens.json: 100%" - } - }, - "faa63481cc9a4ac5adbdeee025a7de19": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fac39df9084640bbbee32339dc4613ad": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_99fd909b2eb14415a79409646dd59fd2", - "placeholder": "​", - "style": "IPY_MODEL_41c8d0eff808446abd70ce585da26978", - "value": "text_encoder/config.json: 100%" - } - }, - "fb4e2edf849047ae9790f0522e13c11e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "fb7fffb9281f42b693c1b7f48325f96f": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fba986e08f5242afaa9ffc193ef9b3b1": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "fd5b8ec96c834a519fedba254849ec04": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_dce47f13fc15450787a63016b61e1c1a", - "placeholder": "​", - "style": "IPY_MODEL_c4073b2d2c1d4642a1c99ce39c8fa39b", - "value": "scheduler/scheduler_config.json: 100%" - } - }, - "fe163c82f9b04a15ac6b923ed3b01873": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_f650b81e97cf47b78b6512ff4a0794c6", - "placeholder": "​", - "style": "IPY_MODEL_d7a4b722cbc74d548edd1d0469d258a7", - "value": "model-00002-of-00002.safetensors: 100%" - } - }, - "fe96da4d0e484f79ab3a82820ddea5ea": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_9690ca75724a4547a505b457efa18cf5", - "max": 334643268, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_b9e34a78891a4be6b293733ad79fb409", - "value": 334643268 - } - }, - "ff86218d2aca4ee7b3764e998b8c9b9a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Image Generation with Pixart-α using 🤗 Diffusers\n", + "\n", + "[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/diffusers/pixart-alpha-diffusers.ipynb)\n", + "\n", + "This notebook demonstrates the following:\n", + "- Performing text-conditional image-generations with the [Pixart-α model](https://huggingface.co/docs/diffusers/v0.23.1/en/api/pipelines/pixart) using [🤗 Diffusers](https://huggingface.co/docs/diffusers).\n", + "- Manage image generation experiments using [Weights & Biases](http://wandb.ai/site).\n", + "- Log the prompts, generated images and experiment configs to [Weigts & Biases](http://wandb.ai/site) for visalization.\n", + "\n", + "![](./assets/diffusers-autolog-1.gif)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install diffusers transformers accelerate sentencepiece ftfy wandb > install.log" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "\n", + "import torch\n", + "from diffusers import PixArtAlphaPipeline\n", + "\n", + "import wandb\n", + "from wandb.integration.diffusers import autolog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the pre-trained checkpoints from HuggingFace Hub to the PixArtAlphaPipeline\n", + "pipe = PixArtAlphaPipeline.from_pretrained(\n", + " \"PixArt-alpha/PixArt-XL-2-1024-MS\", torch_dtype=torch.float16\n", + ")\n", + "\n", + "# Enable offloading the weights to the CPU and only loading them on the GPU when\n", + "# performing the forward pass can also save memory.\n", + "pipe.enable_model_cpu_offload()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb_project = \"pixart-alpha\" # @param {type:\"string\"}\n", + "\n", + "prompt = \"a traveler navigating via a boat in countless mountains, Chinese ink painting\" # @param {type:\"string\"}\n", + "negative_prompt = \"\" # @param {type:\"string\"}\n", + "num_inference_steps = 25 # @param {type:\"slider\", min:10, max:50, step:1}\n", + "guidance_scale = 4.5 # @param {type:\"slider\", min:0, max:10, step:0.1}\n", + "num_images_per_prompt = 1 # @param {type:\"slider\", min:0, max:10, step:0.1}\n", + "height = 1024 # @param {type:\"slider\", min:512, max:2560, step:32}\n", + "width = 1024 # @param {type:\"slider\", min:512, max:2560, step:32}\n", + "seed = None # @param {type:\"raw\"}\n", + "\n", + "\n", + "def autogenerate_seed():\n", + " max_seed = int(1024 * 1024 * 1024)\n", + " seed = random.randint(1, max_seed)\n", + " seed = -seed if seed < 0 else seed\n", + " seed = seed % max_seed\n", + " return seed\n", + "\n", + "\n", + "seed = autogenerate_seed() if seed is None else seed\n", + "\n", + "# Make the experiment reproducible by controlling randomness.\n", + "# The seed would be automatically logged to WandB.\n", + "generator = torch.Generator(device=\"cuda\").manual_seed(seed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Call WandB Autolog for Diffusers. This would automatically log\n", + "# the prompts, generated images, pipeline architecture and all\n", + "# associated experiment configs to Weights & Biases, thus making your\n", + "# image generation experiments easy to reproduce, share and analyze.\n", + "autolog(init=dict(project=wandb_project))\n", + "\n", + "# Generate the images by calling the PixArtAlphaPipeline\n", + "image = pipe(\n", + " prompt=prompt,\n", + " negative_prompt=negative_prompt,\n", + " num_inference_steps=num_inference_steps,\n", + " guidance_scale=guidance_scale,\n", + " num_images_per_prompt=num_images_per_prompt,\n", + " height=height,\n", + " width=width,\n", + " generator=generator,\n", + ").images[0]\n", + "\n", + "# End the experiment\n", + "wandb.finish()" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/colabs/diffusers/sdxl-diffusers.ipynb b/colabs/diffusers/sdxl-diffusers.ipynb index 302184b8..7a2eb5e8 100644 --- a/colabs/diffusers/sdxl-diffusers.ipynb +++ b/colabs/diffusers/sdxl-diffusers.ipynb @@ -1,5 +1,13 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -144,8 +152,15 @@ } ], "metadata": { - "language_info": { - "name": "python" + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } }, "nbformat": 4, diff --git a/colabs/fastai/Weights_&_Biases_with_fastai.ipynb b/colabs/fastai/Weights_&_Biases_with_fastai.ipynb index 7327755b..27846c8a 100644 --- a/colabs/fastai/Weights_&_Biases_with_fastai.ipynb +++ b/colabs/fastai/Weights_&_Biases_with_fastai.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/huggingface/Huggingface_wandb.ipynb b/colabs/huggingface/Huggingface_wandb.ipynb index bdf21058..3fe5577b 100644 --- a/colabs/huggingface/Huggingface_wandb.ipynb +++ b/colabs/huggingface/Huggingface_wandb.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/huggingface/Simple_accelerate_integration_wandb.ipynb b/colabs/huggingface/Simple_accelerate_integration_wandb.ipynb index 5898e68c..48828424 100644 --- a/colabs/huggingface/Simple_accelerate_integration_wandb.ipynb +++ b/colabs/huggingface/Simple_accelerate_integration_wandb.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "205e453e", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/intro/Report_API_Quickstart.ipynb b/colabs/intro/Report_API_Quickstart.ipynb index 5945ccba..5295b8ff 100644 --- a/colabs/intro/Report_API_Quickstart.ipynb +++ b/colabs/intro/Report_API_Quickstart.ipynb @@ -38,7 +38,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -1597,18 +1597,6 @@ "kernelspec": { "display_name": "Python 3", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.10" } }, "nbformat": 4, diff --git a/colabs/keras/Simple_Keras_Integration.ipynb b/colabs/keras/Simple_Keras_Integration.ipynb index bf5d8739..629a0840 100644 --- a/colabs/keras/Simple_Keras_Integration.ipynb +++ b/colabs/keras/Simple_Keras_Integration.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb b/colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb index 4cbce176..63ab4c78 100644 --- a/colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb +++ b/colabs/keras/Use_WandbEvalCallback_in_your_Keras_workflow.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb b/colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb index 3c3ddec2..b45b2c6d 100644 --- a/colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb +++ b/colabs/keras/Use_WandbMetricLogger_in_your_Keras_workflow.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb b/colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb index 3ca4c90c..3cb98584 100644 --- a/colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb +++ b/colabs/keras/Use_WandbModelCheckpoint_in_your_Keras_workflow.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/keras/keras_core/monai_medmnist_keras.ipynb b/colabs/keras/keras_core/monai_medmnist_keras.ipynb index 747baa6f..674f4c0d 100644 --- a/colabs/keras/keras_core/monai_medmnist_keras.ipynb +++ b/colabs/keras/keras_core/monai_medmnist_keras.ipynb @@ -1,477 +1,430 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "dMq6cFtJl2vR" - }, - "source": [ - "\"Keras\"\n", - "\"Weights\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "2eqSE_8rl6yo" - }, - "source": [ - "# 🩺 Medical Image Classification Tutorial using MonAI and Keras\n", - "\n", - "[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/keras_core/monai_medmnist_keras.ipynb)\n", - "\n", - "This notebook demonstrates\n", - "- an end-to-end training using [MonAI](https://github.com/Project-MONAI/MONAI) and [KerasCore](https://github.com/keras-team/keras-core).\n", - "- how we can use the backend-agnostic Keras callbacks for [Weights & Biases](https://wandb.ai/site) to manage and track our experiment.\n", - "\n", - "Original Notebook: https://github.com/Project-MONAI/tutorials/blob/main/2d_classification/mednist_tutorial.ipynb" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ny_HOlvymX6O" - }, - "source": [ - "## Installing and Importing the Dependencies" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6SvDmUxVmatW" - }, - "source": [ - "- We install the `main` branch of [KerasCore](https://github.com/keras-team/keras-core), this lets us use the latest feature merged in KerasCore.\n", - "- We install [monai](https://github.com/Project-MONAI/MONAI), a PyTorch-based, open-source framework for deep learning in healthcare imaging, part of the PyTorch Ecosystem.\n", - "- We also install [wandb-addons](https://github.com/soumik12345/wandb-addons), a library that hosts the backend-agnostic callbacks compatible with KerasCore" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ZO1xjFtvVdqQ" - }, - "outputs": [], - "source": [ - "# install the `main` branch of KerasCore\n", - "!pip install -qq namex\n", - "!apt install python3.10-venv\n", - "!git clone https://github.com/soumik12345/keras-core.git && cd keras-core && python pip_build.py --install\n", - "\n", - "# install monai and wandb-addons\n", - "!pip install -qq git+https://github.com/soumik12345/wandb-addons\n", - "!pip install -q \"monai-weekly[pillow, tqdm]\"" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "c6vd3NZ-mhxs" - }, - "source": [ - "We specify the Keras backend to be using `torch` by explicitly specifying the environment variable `KERAS_BACKEND`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "d5ZiQmMkW-h3" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n", - "\n", - "import shutil\n", - "import tempfile\n", - "import matplotlib.pyplot as plt\n", - "import PIL\n", - "import torch\n", - "import numpy as np\n", - "from sklearn.metrics import classification_report\n", - "\n", - "import keras_core as keras\n", - "from keras_core.utils import TorchModuleWrapper\n", - "\n", - "from monai.apps import download_and_extract\n", - "from monai.config import print_config\n", - "from monai.data import decollate_batch, DataLoader\n", - "from monai.metrics import ROCAUCMetric\n", - "from monai.networks.nets import DenseNet121\n", - "from monai.transforms import (\n", - " Activations,\n", - " EnsureChannelFirst,\n", - " AsDiscrete,\n", - " Compose,\n", - " LoadImage,\n", - " RandFlip,\n", - " RandRotate,\n", - " RandZoom,\n", - " ScaleIntensity,\n", - ")\n", - "from monai.utils import set_determinism\n", - "\n", - "import wandb\n", - "from wandb_addons.keras import WandbMetricsLogger, WandbModelCheckpoint" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1BvtLvxKmkSp" - }, - "source": [ - "We initialize a [wandb run](https://docs.wandb.ai/guides/runs) and set the configs for the experiment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "WUt7PAsado4j" - }, - "outputs": [], - "source": [ - "wandb.init(project=\"keras-torch\")\n", - "\n", - "config = wandb.config\n", - "config.batch_size = 128\n", - "config.num_epochs = 1" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NE938skxmoMI" - }, - "source": [ - "## Setup data directory\n", - "\n", - "You can specify a directory with the `MONAI_DATA_DIRECTORY` environment variable.\n", - "This allows you to save results and reuse downloads.\n", - "If not specified a temporary directory will be used." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "82KX1Sj6XXY1" - }, - "outputs": [], - "source": [ - "directory = os.environ.get(\"MONAI_DATA_DIRECTORY\")\n", - "root_dir = tempfile.mkdtemp() if directory is None else directory\n", - "print(root_dir)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nkVQ-tojmzwf" - }, - "source": [ - "## Download dataset\n", - "\n", - "The MedNIST dataset was gathered from several sets from [TCIA](https://wiki.cancerimagingarchive.net/display/Public/Data+Usage+Policies+and+Restrictions),\n", - "[the RSNA Bone Age Challenge](http://rsnachallenges.cloudapp.net/competitions/4),\n", - "and [the NIH Chest X-ray dataset](https://cloud.google.com/healthcare/docs/resources/public-datasets/nih-chest).\n", - "\n", - "The dataset is kindly made available by [Dr. Bradley J. Erickson M.D., Ph.D.](https://www.mayo.edu/research/labs/radiology-informatics/overview) (Department of Radiology, Mayo Clinic)\n", - "under the Creative Commons [CC BY-SA 4.0 license](https://creativecommons.org/licenses/by-sa/4.0/).\n", - "\n", - "If you use the MedNIST dataset, please acknowledge the source." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DAHybyvdXZoH" - }, - "outputs": [], - "source": [ - "resource = \"https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MedNIST.tar.gz\"\n", - "md5 = \"0bc7306e7427e00ad1c5526a6677552d\"\n", - "\n", - "compressed_file = os.path.join(root_dir, \"MedNIST.tar.gz\")\n", - "data_dir = os.path.join(root_dir, \"MedNIST\")\n", - "if not os.path.exists(data_dir):\n", - " download_and_extract(resource, compressed_file, root_dir, md5)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "u6m2Uas1nMi8" - }, - "source": [ - "## Read image filenames from the dataset folders\n", - "\n", - "First of all, check the dataset files and show some statistics. \n", - "There are 6 folders in the dataset: Hand, AbdomenCT, CXR, ChestCT, BreastMRI, HeadCT, \n", - "which should be used as the labels to train our classification model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qtLr1T0gXydq" - }, - "outputs": [], - "source": [ - "class_names = sorted(x for x in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, x)))\n", - "num_class = len(class_names)\n", - "image_files = [\n", - " [os.path.join(data_dir, class_names[i], x) for x in os.listdir(os.path.join(data_dir, class_names[i]))]\n", - " for i in range(num_class)\n", - "]\n", - "num_each = [len(image_files[i]) for i in range(num_class)]\n", - "image_files_list = []\n", - "image_class = []\n", - "for i in range(num_class):\n", - " image_files_list.extend(image_files[i])\n", - " image_class.extend([i] * num_each[i])\n", - "num_total = len(image_class)\n", - "image_width, image_height = PIL.Image.open(image_files_list[0]).size\n", - "\n", - "print(f\"Total image count: {num_total}\")\n", - "print(f\"Image dimensions: {image_width} x {image_height}\")\n", - "print(f\"Label names: {class_names}\")\n", - "print(f\"Label counts: {num_each}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "k4WOUb4IX6KK" - }, - "outputs": [], - "source": [ - "plt.subplots(3, 3, figsize=(8, 8))\n", - "for i, k in enumerate(np.random.randint(num_total, size=9)):\n", - " im = PIL.Image.open(image_files_list[k])\n", - " arr = np.array(im)\n", - " plt.subplot(3, 3, i + 1)\n", - " plt.xlabel(class_names[image_class[k]])\n", - " plt.imshow(arr, cmap=\"gray\", vmin=0, vmax=255)\n", - "plt.tight_layout()\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "wOGja_mEnQ5m" - }, - "source": [ - "## Prepare training, validation and test data lists\n", - "\n", - "Randomly select 10% of the dataset as validation and 10% as test." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BmHhKta1X8ZM" - }, - "outputs": [], - "source": [ - "val_frac = 0.1\n", - "test_frac = 0.1\n", - "length = len(image_files_list)\n", - "indices = np.arange(length)\n", - "np.random.shuffle(indices)\n", - "\n", - "test_split = int(test_frac * length)\n", - "val_split = int(val_frac * length) + test_split\n", - "test_indices = indices[:test_split]\n", - "val_indices = indices[test_split:val_split]\n", - "train_indices = indices[val_split:]\n", - "\n", - "train_x = [image_files_list[i] for i in train_indices]\n", - "train_y = [image_class[i] for i in train_indices]\n", - "val_x = [image_files_list[i] for i in val_indices]\n", - "val_y = [image_class[i] for i in val_indices]\n", - "test_x = [image_files_list[i] for i in test_indices]\n", - "test_y = [image_class[i] for i in test_indices]\n", - "\n", - "print(f\"Training count: {len(train_x)}, Validation count: \" f\"{len(val_x)}, Test count: {len(test_x)}\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Lkhtr3p8nT1G" - }, - "source": [ - "## Define MONAI transforms, Dataset and Dataloader to pre-process data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "2b5H2WuUYLsc" - }, - "outputs": [], - "source": [ - "train_transforms = Compose(\n", - " [\n", - " LoadImage(image_only=True),\n", - " EnsureChannelFirst(),\n", - " ScaleIntensity(),\n", - " RandRotate(range_x=np.pi / 12, prob=0.5, keep_size=True),\n", - " RandFlip(spatial_axis=0, prob=0.5),\n", - " RandZoom(min_zoom=0.9, max_zoom=1.1, prob=0.5),\n", - " ]\n", - ")\n", - "\n", - "val_transforms = Compose([LoadImage(image_only=True), EnsureChannelFirst(), ScaleIntensity()])\n", - "\n", - "y_pred_trans = Compose([Activations(softmax=True)])\n", - "y_trans = Compose([AsDiscrete(to_onehot=num_class)])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QdFnaFUDYOaq" - }, - "outputs": [], - "source": [ - "class MedNISTDataset(torch.utils.data.Dataset):\n", - " def __init__(self, image_files, labels, transforms):\n", - " self.image_files = image_files\n", - " self.labels = labels\n", - " self.transforms = transforms\n", - "\n", - " def __len__(self):\n", - " return len(self.image_files)\n", - "\n", - " def __getitem__(self, index):\n", - " return self.transforms(self.image_files[index]), self.labels[index]\n", - "\n", - "\n", - "train_ds = MedNISTDataset(train_x, train_y, train_transforms)\n", - "train_loader = DataLoader(train_ds, batch_size=config.batch_size, shuffle=True, num_workers=2)\n", - "\n", - "val_ds = MedNISTDataset(val_x, val_y, val_transforms)\n", - "val_loader = DataLoader(val_ds, batch_size=config.batch_size, num_workers=2)\n", - "\n", - "test_ds = MedNISTDataset(test_x, test_y, val_transforms)\n", - "test_loader = DataLoader(test_ds, batch_size=config.batch_size, num_workers=2)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FIhKqMrVnZV9" - }, - "source": [ - "We typically define a model in PyTorch using [`torch.nn.Module`s](https://pytorch.org/docs/stable/notes/modules.html) which act as the building blocks of stateful computation. Even though Keras supports PyTorch as a backend, it does not mean that we can nest torch modules inside a [`keras_core.Model`](https://keras.io/keras_core/api/models/), because trainable variables inside a Keras Model is tracked exclusively via [Keras Layers](https://keras.io/keras_core/api/layers/).\n", - "\n", - "KerasCore provides us with a feature called `TorchModuleWrapper` which enables us to do exactly this. The `TorchModuleWrapper` is a Keras Layer that accepts a torch module and tracks its trainable variables, essentially converting the torch module into a Keras Layer. This enables us to put any torch modules inside a Keras Model and train them with a single `model.fit()`!\n", - "\n", - "The idea of the `TorchModuleWrapper` was proposed by Keras' creator [François Chollet](https://github.com/fchollet) on [this issue thread](https://github.com/keras-team/keras-core/issues/604)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "7zbinqy4ZsEy" - }, - "outputs": [], - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", - "\n", - "inputs = keras.Input(shape=(1, 64, 64))\n", - "outputs = TorchModuleWrapper(\n", - " DenseNet121(\n", - " spatial_dims=2, in_channels=1, out_channels=num_class\n", - " )\n", - ")(inputs)\n", - "model = keras.Model(inputs, outputs)\n", - "\n", - "# model = MedMnistModel()\n", - "model(next(iter(train_loader))[0].to(device)).shape" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "opjBI87nneYU" - }, - "source": [ - "**Note:** It is actually possible to use torch modules inside a Keras Model without having to explicitly have them wrapped with the `TorchModuleWrapper` as evident by [this tweet](https://twitter.com/fchollet/status/1697381832164290754) from François Chollet. However, this doesn't seem to work at the point of time this example was created, as reported in [this issue](https://github.com/keras-team/keras-core/issues/834)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "y3H-cUKraRDh" - }, - "outputs": [], - "source": [ - "# Compile the model\n", - "model.compile(\n", - " loss=\"sparse_categorical_crossentropy\",\n", - " optimizer=keras.optimizers.Adam(1e-5),\n", - " metrics=[\"accuracy\"],\n", - ")\n", - "\n", - "# Define the backend-agnostic WandB callbacks for KerasCore\n", - "callbacks = [\n", - " # Track experiment metrics\n", - " WandbMetricsLogger(log_freq=\"batch\")\n", - "]\n", - "\n", - "# Train the model by calling model.fit\n", - "model.fit(\n", - " train_loader,\n", - " validation_data=val_loader,\n", - " epochs=config.num_epochs,\n", - " callbacks=callbacks,\n", - ")\n", - "\n", - "wandb.finish()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Q0E-kiGBeCnZ" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "V100", - "private_outputs": true, - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Keras\"\n", + "\"Weights\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🩺 Medical Image Classification Tutorial using MonAI and Keras\n", + "\n", + "[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/keras_core/monai_medmnist_keras.ipynb)\n", + "\n", + "This notebook demonstrates\n", + "- an end-to-end training using [MonAI](https://github.com/Project-MONAI/MONAI) and [KerasCore](https://github.com/keras-team/keras-core).\n", + "- how we can use the backend-agnostic Keras callbacks for [Weights & Biases](https://wandb.ai/site) to manage and track our experiment.\n", + "\n", + "Original Notebook: https://github.com/Project-MONAI/tutorials/blob/main/2d_classification/mednist_tutorial.ipynb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installing and Importing the Dependencies" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- We install the `main` branch of [KerasCore](https://github.com/keras-team/keras-core), this lets us use the latest feature merged in KerasCore.\n", + "- We install [monai](https://github.com/Project-MONAI/MONAI), a PyTorch-based, open-source framework for deep learning in healthcare imaging, part of the PyTorch Ecosystem.\n", + "- We also install [wandb-addons](https://github.com/soumik12345/wandb-addons), a library that hosts the backend-agnostic callbacks compatible with KerasCore" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# install the `main` branch of KerasCore\n", + "!pip install -qq namex\n", + "!apt install python3.10-venv\n", + "!git clone https://github.com/soumik12345/keras-core.git && cd keras-core && python pip_build.py --install\n", + "\n", + "# install monai and wandb-addons\n", + "!pip install -qq git+https://github.com/soumik12345/wandb-addons\n", + "!pip install -q \"monai-weekly[pillow, tqdm]\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We specify the Keras backend to be using `torch` by explicitly specifying the environment variable `KERAS_BACKEND`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n", + "\n", + "import shutil\n", + "import tempfile\n", + "import matplotlib.pyplot as plt\n", + "import PIL\n", + "import torch\n", + "import numpy as np\n", + "from sklearn.metrics import classification_report\n", + "\n", + "import keras_core as keras\n", + "from keras_core.utils import TorchModuleWrapper\n", + "\n", + "from monai.apps import download_and_extract\n", + "from monai.config import print_config\n", + "from monai.data import decollate_batch, DataLoader\n", + "from monai.metrics import ROCAUCMetric\n", + "from monai.networks.nets import DenseNet121\n", + "from monai.transforms import (\n", + " Activations,\n", + " EnsureChannelFirst,\n", + " AsDiscrete,\n", + " Compose,\n", + " LoadImage,\n", + " RandFlip,\n", + " RandRotate,\n", + " RandZoom,\n", + " ScaleIntensity,\n", + ")\n", + "from monai.utils import set_determinism\n", + "\n", + "import wandb\n", + "from wandb_addons.keras import WandbMetricsLogger, WandbModelCheckpoint" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We initialize a [wandb run](https://docs.wandb.ai/guides/runs) and set the configs for the experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.init(project=\"keras-torch\")\n", + "\n", + "config = wandb.config\n", + "config.batch_size = 128\n", + "config.num_epochs = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup data directory\n", + "\n", + "You can specify a directory with the `MONAI_DATA_DIRECTORY` environment variable.\n", + "This allows you to save results and reuse downloads.\n", + "If not specified a temporary directory will be used." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "directory = os.environ.get(\"MONAI_DATA_DIRECTORY\")\n", + "root_dir = tempfile.mkdtemp() if directory is None else directory\n", + "print(root_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download dataset\n", + "\n", + "The MedNIST dataset was gathered from several sets from [TCIA](https://wiki.cancerimagingarchive.net/display/Public/Data+Usage+Policies+and+Restrictions),\n", + "[the RSNA Bone Age Challenge](http://rsnachallenges.cloudapp.net/competitions/4),\n", + "and [the NIH Chest X-ray dataset](https://cloud.google.com/healthcare/docs/resources/public-datasets/nih-chest).\n", + "\n", + "The dataset is kindly made available by [Dr. Bradley J. Erickson M.D., Ph.D.](https://www.mayo.edu/research/labs/radiology-informatics/overview) (Department of Radiology, Mayo Clinic)\n", + "under the Creative Commons [CC BY-SA 4.0 license](https://creativecommons.org/licenses/by-sa/4.0/).\n", + "\n", + "If you use the MedNIST dataset, please acknowledge the source." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "resource = \"https://github.com/Project-MONAI/MONAI-extra-test-data/releases/download/0.8.1/MedNIST.tar.gz\"\n", + "md5 = \"0bc7306e7427e00ad1c5526a6677552d\"\n", + "\n", + "compressed_file = os.path.join(root_dir, \"MedNIST.tar.gz\")\n", + "data_dir = os.path.join(root_dir, \"MedNIST\")\n", + "if not os.path.exists(data_dir):\n", + " download_and_extract(resource, compressed_file, root_dir, md5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Read image filenames from the dataset folders\n", + "\n", + "First of all, check the dataset files and show some statistics. \n", + "There are 6 folders in the dataset: Hand, AbdomenCT, CXR, ChestCT, BreastMRI, HeadCT, \n", + "which should be used as the labels to train our classification model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class_names = sorted(x for x in os.listdir(data_dir) if os.path.isdir(os.path.join(data_dir, x)))\n", + "num_class = len(class_names)\n", + "image_files = [\n", + " [os.path.join(data_dir, class_names[i], x) for x in os.listdir(os.path.join(data_dir, class_names[i]))]\n", + " for i in range(num_class)\n", + "]\n", + "num_each = [len(image_files[i]) for i in range(num_class)]\n", + "image_files_list = []\n", + "image_class = []\n", + "for i in range(num_class):\n", + " image_files_list.extend(image_files[i])\n", + " image_class.extend([i] * num_each[i])\n", + "num_total = len(image_class)\n", + "image_width, image_height = PIL.Image.open(image_files_list[0]).size\n", + "\n", + "print(f\"Total image count: {num_total}\")\n", + "print(f\"Image dimensions: {image_width} x {image_height}\")\n", + "print(f\"Label names: {class_names}\")\n", + "print(f\"Label counts: {num_each}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.subplots(3, 3, figsize=(8, 8))\n", + "for i, k in enumerate(np.random.randint(num_total, size=9)):\n", + " im = PIL.Image.open(image_files_list[k])\n", + " arr = np.array(im)\n", + " plt.subplot(3, 3, i + 1)\n", + " plt.xlabel(class_names[image_class[k]])\n", + " plt.imshow(arr, cmap=\"gray\", vmin=0, vmax=255)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare training, validation and test data lists\n", + "\n", + "Randomly select 10% of the dataset as validation and 10% as test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "val_frac = 0.1\n", + "test_frac = 0.1\n", + "length = len(image_files_list)\n", + "indices = np.arange(length)\n", + "np.random.shuffle(indices)\n", + "\n", + "test_split = int(test_frac * length)\n", + "val_split = int(val_frac * length) + test_split\n", + "test_indices = indices[:test_split]\n", + "val_indices = indices[test_split:val_split]\n", + "train_indices = indices[val_split:]\n", + "\n", + "train_x = [image_files_list[i] for i in train_indices]\n", + "train_y = [image_class[i] for i in train_indices]\n", + "val_x = [image_files_list[i] for i in val_indices]\n", + "val_y = [image_class[i] for i in val_indices]\n", + "test_x = [image_files_list[i] for i in test_indices]\n", + "test_y = [image_class[i] for i in test_indices]\n", + "\n", + "print(f\"Training count: {len(train_x)}, Validation count: \" f\"{len(val_x)}, Test count: {len(test_x)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define MONAI transforms, Dataset and Dataloader to pre-process data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_transforms = Compose(\n", + " [\n", + " LoadImage(image_only=True),\n", + " EnsureChannelFirst(),\n", + " ScaleIntensity(),\n", + " RandRotate(range_x=np.pi / 12, prob=0.5, keep_size=True),\n", + " RandFlip(spatial_axis=0, prob=0.5),\n", + " RandZoom(min_zoom=0.9, max_zoom=1.1, prob=0.5),\n", + " ]\n", + ")\n", + "\n", + "val_transforms = Compose([LoadImage(image_only=True), EnsureChannelFirst(), ScaleIntensity()])\n", + "\n", + "y_pred_trans = Compose([Activations(softmax=True)])\n", + "y_trans = Compose([AsDiscrete(to_onehot=num_class)])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class MedNISTDataset(torch.utils.data.Dataset):\n", + " def __init__(self, image_files, labels, transforms):\n", + " self.image_files = image_files\n", + " self.labels = labels\n", + " self.transforms = transforms\n", + "\n", + " def __len__(self):\n", + " return len(self.image_files)\n", + "\n", + " def __getitem__(self, index):\n", + " return self.transforms(self.image_files[index]), self.labels[index]\n", + "\n", + "\n", + "train_ds = MedNISTDataset(train_x, train_y, train_transforms)\n", + "train_loader = DataLoader(train_ds, batch_size=config.batch_size, shuffle=True, num_workers=2)\n", + "\n", + "val_ds = MedNISTDataset(val_x, val_y, val_transforms)\n", + "val_loader = DataLoader(val_ds, batch_size=config.batch_size, num_workers=2)\n", + "\n", + "test_ds = MedNISTDataset(test_x, test_y, val_transforms)\n", + "test_loader = DataLoader(test_ds, batch_size=config.batch_size, num_workers=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We typically define a model in PyTorch using [`torch.nn.Module`s](https://pytorch.org/docs/stable/notes/modules.html) which act as the building blocks of stateful computation. Even though Keras supports PyTorch as a backend, it does not mean that we can nest torch modules inside a [`keras_core.Model`](https://keras.io/keras_core/api/models/), because trainable variables inside a Keras Model is tracked exclusively via [Keras Layers](https://keras.io/keras_core/api/layers/).\n", + "\n", + "KerasCore provides us with a feature called `TorchModuleWrapper` which enables us to do exactly this. The `TorchModuleWrapper` is a Keras Layer that accepts a torch module and tracks its trainable variables, essentially converting the torch module into a Keras Layer. This enables us to put any torch modules inside a Keras Model and train them with a single `model.fit()`!\n", + "\n", + "The idea of the `TorchModuleWrapper` was proposed by Keras' creator [François Chollet](https://github.com/fchollet) on [this issue thread](https://github.com/keras-team/keras-core/issues/604)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "inputs = keras.Input(shape=(1, 64, 64))\n", + "outputs = TorchModuleWrapper(\n", + " DenseNet121(\n", + " spatial_dims=2, in_channels=1, out_channels=num_class\n", + " )\n", + ")(inputs)\n", + "model = keras.Model(inputs, outputs)\n", + "\n", + "# model = MedMnistModel()\n", + "model(next(iter(train_loader))[0].to(device)).shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "**Note:** It is actually possible to use torch modules inside a Keras Model without having to explicitly have them wrapped with the `TorchModuleWrapper` as evident by [this tweet](https://twitter.com/fchollet/status/1697381832164290754) from François Chollet. However, this doesn't seem to work at the point of time this example was created, as reported in [this issue](https://github.com/keras-team/keras-core/issues/834)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Compile the model\n", + "model.compile(\n", + " loss=\"sparse_categorical_crossentropy\",\n", + " optimizer=keras.optimizers.Adam(1e-5),\n", + " metrics=[\"accuracy\"],\n", + ")\n", + "\n", + "# Define the backend-agnostic WandB callbacks for KerasCore\n", + "callbacks = [\n", + " # Track experiment metrics\n", + " WandbMetricsLogger(log_freq=\"batch\")\n", + "]\n", + "\n", + "# Train the model by calling model.fit\n", + "model.fit(\n", + " train_loader,\n", + " validation_data=val_loader,\n", + " epochs=config.num_epochs,\n", + " callbacks=callbacks,\n", + ")\n", + "\n", + "wandb.finish()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/colabs/keras/keras_core/timm_keras.ipynb b/colabs/keras/keras_core/timm_keras.ipynb index 1cea4769..d83360a1 100644 --- a/colabs/keras/keras_core/timm_keras.ipynb +++ b/colabs/keras/keras_core/timm_keras.ipynb @@ -1,5 +1,13 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -367,10 +375,16 @@ } ], "metadata": { - "language_info": { - "name": "python" + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true }, - "orig_nbformat": 4 + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } }, "nbformat": 4, "nbformat_minor": 2 diff --git a/colabs/keras/keras_core/torchvision-keras.ipynb b/colabs/keras/keras_core/torchvision-keras.ipynb index 30783184..f4fc591d 100644 --- a/colabs/keras/keras_core/torchvision-keras.ipynb +++ b/colabs/keras/keras_core/torchvision-keras.ipynb @@ -1,922 +1,460 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "jLRorPqA1-h4" - }, - "source": [ - "\"Keras\"\n", - "\"Weights\n", - "\n", - "\n", - "# 🔥 Fine-tune a TorchVision Model with Keras and WandB 🦄\n", - "\n", - "[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/keras_core/torchvision_keras.ipynb)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aEl-j2hq2w25" - }, - "source": [ - "## Introduction\n", - "\n", - "[TorchVision](https://pytorch.org/vision/stable/index.html) is a library part of the [PyTorch](http://pytorch.org/) project that consists of popular datasets, model architectures, and common image transformations for computer vision. This example demonstrates how we can perform transfer learning for image classification using a pre-trained backbone model from TorchVision on the [Imagenette dataset](https://github.com/fastai/imagenette) using KerasCore. We will also demonstrate the compatibility of KerasCore with an input system consisting of [Torch Datasets and Dataloaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html).\n", - "\n", - "### References:\n", - "\n", - "- [Customizing what happens in `fit()` with PyTorch](https://keras.io/keras_core/guides/custom_train_step_in_torch/)\n", - "- [PyTorch Datasets and Dataloaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html)\n", - "- [Transfer learning for Computer Vision using PyTorch](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)\n", - "\n", - "## Setup\n", - "\n", - "- We install the `main` branch of [KerasCore](https://github.com/keras-team/keras-core), this lets us use the latest feature merged in KerasCore.\n", - "- We also install [wandb-addons](https://github.com/soumik12345/wandb-addons), a library that hosts the backend-agnostic callbacks compatible with KerasCore" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "r4rfNRPgiy9v", - "outputId": "ce5ba027-567a-4577-a638-ca8802ee1f84" - }, - "outputs": [], - "source": [ - "# install the `main` branch of KerasCore\n", - "!pip install -qq namex\n", - "!apt install python3.10-venv\n", - "!git clone https://github.com/soumik12345/keras-core.git && cd keras-core && python pip_build.py --install\n", - "\n", - "# install wandb-addons\n", - "!pip install -qq git+https://github.com/soumik12345/wandb-addons" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "7nudAUt8jHRB" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n", - "\n", - "import numpy as np\n", - "from tqdm.auto import tqdm\n", - "import matplotlib.pyplot as plt\n", - "\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "\n", - "import torchvision\n", - "from torchvision import datasets, models, transforms\n", - "\n", - "import keras_core as keras\n", - "from keras_core.utils import TorchModuleWrapper\n", - "\n", - "import wandb\n", - "from wandb_addons.keras import WandbMetricsLogger, WandbModelCheckpoint" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "pS1c-ySo7nty" - }, - "source": [ - "## Define the Hyperparameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "2ovtXSUA7ksk", - "outputId": "725bef1a-0e68-473e-8c24-1f1ffd28506c" - }, - "outputs": [], - "source": [ - "wandb.init(project=\"keras-torch\", entity=\"ml-colabs\", job_type=\"torchvision/train\")\n", - "\n", - "config = wandb.config\n", - "config.batch_size = 32\n", - "config.image_size = 224\n", - "config.freeze_backbone = True\n", - "config.initial_learning_rate = 1e-3\n", - "config.num_epochs = 5" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "b5uU_Q_H74GO" - }, - "source": [ - "## Creating the Torch Datasets and Dataloaders\n", - "\n", - "In this example, we would train an image classification model on the [Imagenette dataset](https://github.com/fastai/imagenette). Imagenette is a subset of 10 easily classified classes from [Imagenet](https://www.image-net.org/) (tench, English springer, cassette player, chain saw, church, French horn, garbage truck, gas pump, golf ball, parachute)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "gWV2hNGo8vW7", - "outputId": "cc70b598-fc66-480a-a98c-7077fa634a22" - }, - "outputs": [], - "source": [ - "# Fetch the imagenette dataset\n", - "data_dir = keras.utils.get_file(\n", - " fname=\"imagenette2-320.tgz\",\n", - " origin=\"https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz\",\n", - " extract=True,\n", - ")\n", - "data_dir = data_dir.replace(\".tgz\", \"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Ka7TDUMn9IcG" - }, - "source": [ - "Next, we define pre-processing and augmentation transforms from TorchVision for the train and validation sets." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "rFZKJafF9H6y" - }, - "outputs": [], - "source": [ - "data_transforms = {\n", - " 'train': transforms.Compose([\n", - " transforms.RandomResizedCrop(config.image_size),\n", - " transforms.RandomHorizontalFlip(),\n", - " transforms.ToTensor(),\n", - " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", - " ]),\n", - " 'val': transforms.Compose([\n", - " transforms.Resize(256),\n", - " transforms.CenterCrop(config.image_size),\n", - " transforms.ToTensor(),\n", - " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", - " ]),\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aE3VFQHm9srl" - }, - "source": [ - "Finally, we will use TorchVision and the [`torch.utils.data`](https://pytorch.org/docs/stable/data.html) packages for creating the dataloaders for trainig and validation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "0N81UNjtjMZ4", - "outputId": "bfb2af1a-977c-4408-cb1c-8da78d03d13d" - }, - "outputs": [], - "source": [ - "# Define the train and validation datasets\n", - "image_datasets = {\n", - " x: datasets.ImageFolder(\n", - " os.path.join(data_dir, x), data_transforms[x]\n", - " )\n", - " for x in ['train', 'val']\n", - "}\n", - "\n", - "# Define the torch dataloaders corresponding to the\n", - "# train and validation dataset\n", - "dataloaders = {\n", - " x: torch.utils.data.DataLoader(\n", - " image_datasets[x],\n", - " batch_size=config.batch_size,\n", - " shuffle=True,\n", - " num_workers=4\n", - " )\n", - " for x in ['train', 'val']\n", - "}\n", - "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n", - "class_names = image_datasets['train'].classes" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AY6kOIL--EdP" - }, - "source": [ - "Let us visualize a few samples from the training dataloader." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 829 - }, - "id": "yffdD4LxjOQG", - "outputId": "38e6f182-11b1-4830-e5ec-a5588c7bdbf9" - }, - "outputs": [], - "source": [ - "plt.figure(figsize=(10, 10))\n", - "sample_images, sample_labels = next(iter(dataloaders['train']))\n", - "sample_images = sample_images.numpy()\n", - "sample_labels = sample_labels.numpy()\n", - "for idx in range(9):\n", - " ax = plt.subplot(3, 3, idx + 1)\n", - " image = sample_images[idx].transpose((1, 2, 0))\n", - " mean = np.array([0.485, 0.456, 0.406])\n", - " std = np.array([0.229, 0.224, 0.225])\n", - " image = std * image + mean\n", - " image = np.clip(image, 0, 1)\n", - " plt.imshow(image)\n", - " plt.title(\"Ground Truth Label: \" + class_names[int(sample_labels[idx])])\n", - " plt.axis(\"off\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0zvfzD04-ce9" - }, - "source": [ - "## The Image Classification Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZBLzXVwk-mLP" - }, - "source": [ - "We typically define a model in PyTorch using [`torch.nn.Module`s](https://pytorch.org/docs/stable/notes/modules.html) which act as the building blocks of stateful computation. Let us define the ResNet18 model from the TorchVision package as a `torch.nn.Module` pre-trained on the [Imagenet1K dataset](https://huggingface.co/datasets/imagenet-1k)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tOGUiI9K_BRk", - "outputId": "97aa387c-7b3d-41cf-959b-de60eb572912" - }, - "outputs": [], - "source": [ - "# Define the pre-trained resnet18 module from TorchVision\n", - "resnet_18 = models.resnet18(weights='IMAGENET1K_V1')\n", - "\n", - "# We set the classification head of the pre-trained ResNet18\n", - "# module to an identity module\n", - "resnet_18.fc = nn.Identity()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "afxbMwcYF-Yz" - }, - "source": [ - "ven though Keras supports PyTorch as a backend, it does not mean that we can nest torch modules inside a [`keras_core.Model`](https://keras.io/keras_core/api/models/), because trainable variables inside a Keras Model is tracked exclusively via [Keras Layers](https://keras.io/keras_core/api/layers/).\n", - "\n", - "KerasCore provides us with a feature called `TorchModuleWrapper` which enables us to do exactly this. The `TorchModuleWrapper` is a Keras Layer that accepts a torch module and tracks its trainable variables, essentially converting the torch module into a Keras Layer. This enables us to put any torch modules inside a Keras Model and train them with a single `model.fit()`!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "JLuCIAy5F6L1" - }, - "outputs": [], - "source": [ - "# We set the trainable ResNet18 backbone to be a Keras Layer\n", - "# using `TorchModuleWrapper`\n", - "backbone = TorchModuleWrapper(resnet_18)\n", - "\n", - "# We set this to `False` if you want to freeze the backbone\n", - "backbone.trainable = config.freeze_backbone" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7y28txoVHHk8" - }, - "source": [ - "Now, we will build a Keras functional model with the backbone layer." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 314 - }, - "id": "l2rxqA8vjR3W", - "outputId": "206ad321-5fb5-41e1-a391-37cef3b10edb" - }, - "outputs": [], - "source": [ - "inputs = keras.Input(shape=(3, config.image_size, config.image_size))\n", - "x = backbone(inputs)\n", - "x = keras.layers.Dropout(0.5)(x)\n", - "x = keras.layers.Dense(len(class_names))(x)\n", - "outputs = keras.activations.softmax(x, axis=1)\n", - "model = keras.Model(inputs, outputs, name=\"ResNet18_Classifier\")\n", - "\n", - "model.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "BXbvYzDnjyDQ", - "outputId": "9d7e51b2-85e7-4717-993b-0f6efac2999d" - }, - "outputs": [], - "source": [ - "# Create exponential decay learning rate scheduler\n", - "decay_steps = config.num_epochs * len(dataloaders[\"train\"]) // config.batch_size\n", - "lr_scheduler = keras.optimizers.schedules.ExponentialDecay(\n", - " initial_learning_rate=config.initial_learning_rate,\n", - " decay_steps=decay_steps,\n", - " decay_rate=0.1,\n", - ")\n", - "\n", - "# Compile the model\n", - "model.compile(\n", - " loss=\"sparse_categorical_crossentropy\",\n", - " optimizer=keras.optimizers.Adam(lr_scheduler),\n", - " metrics=[\"accuracy\"],\n", - ")\n", - "\n", - "# Define the backend-agnostic WandB callbacks for KerasCore\n", - "callbacks = [\n", - " # Track experiment metrics with WandB\n", - " WandbMetricsLogger(log_freq=\"batch\"),\n", - " # Save best model checkpoints to WandB\n", - " WandbModelCheckpoint(\n", - " filepath=\"model.weights.h5\",\n", - " monitor=\"val_loss\",\n", - " save_best_only=True,\n", - " save_weights_only=True,\n", - " )\n", - "]\n", - "\n", - "# Train the model by calling model.fit\n", - "history = model.fit(\n", - " dataloaders[\"train\"],\n", - " validation_data=dataloaders[\"val\"],\n", - " epochs=config.num_epochs,\n", - " callbacks=callbacks,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "upJf2M92JBlD", - "outputId": "6993ffbe-2b69-4ff8-9a2f-9c502ca1414d" - }, - "outputs": [], - "source": [ - "wandb.finish()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "I58qvNnzJmHD" - }, - "source": [ - "## Evaluation and Inference\n", - "\n", - "Now, we let us load the best model weights checkpoint and evaluate the model." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "cNpfuELqnDI_", - "outputId": "18bfe7bb-adea-4ac2-807b-924dce6fbcc8" - }, - "outputs": [], - "source": [ - "wandb.init(\n", - " project=\"keras-torch\", entity=\"ml-colabs\", job_type=\"torchvision/eval\"\n", - ")\n", - "artifact = wandb.use_artifact(\n", - " 'ml-colabs/keras-torch/run_hiceci7f_model:latest', type='model'\n", - ")\n", - "artifact_dir = artifact.download()\n", - "\n", - "model.load_weights(os.path.join(artifact_dir, \"model.weights.h5\"))\n", - "\n", - "_, val_accuracy = model.evaluate(dataloaders[\"val\"])\n", - "wandb.log({\"Validation-Accuracy\": val_accuracy})" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vE2vAvBNKAI9" - }, - "source": [ - "Finally, let us visualize the some predictions of the model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 49, - "referenced_widgets": [ - "1b82953b5f134926ae8a11c4fedca385", - "639fec0083134ab18b6a22999203536e", - "5d85942dc6e44640aed593b7a8494493", - "757ca7f811ec4db5a4dd517c5aec2bb8", - "122edd2a0300448cac1fcc8645fd8708", - "b1df464c3a2d47dbb49b36fa5d9912bb", - "dd95b1ba48ec4bc5b494f319ad41eedf", - "84dfa1f6d6f04c50b87e8683a0abdf55", - "634e163cb1354824a764dc49f7d9f2fa", - "5b1e8c0d14d84e0f923a997727794c89", - "2e912ebb924c4269a07fb7c1204eb923" - ] - }, - "id": "ugrP307SpxMj", - "outputId": "ead193f0-bfd9-4bbc-c675-d741d64fd70f" - }, - "outputs": [], - "source": [ - "table = wandb.Table(\n", - " columns=[\n", - " \"Image\", \"Ground-Truth\", \"Prediction\"\n", - " ] + [\"Confidence-\" + cls for cls in class_names]\n", - ")\n", - "\n", - "sample_images, sample_labels = next(iter(dataloaders['train']))\n", - "\n", - "# We perform inference and detach the predicted probabilities from the Torch\n", - "# computation graph with a tensor that does not require gradient computation.\n", - "sample_pred_probas = model(sample_images.to(\"cuda\")).detach()\n", - "sample_pred_logits = keras.ops.argmax(sample_pred_probas, axis=1)\n", - "sample_pred_logits = sample_pred_logits.to(\"cpu\").numpy()\n", - "sample_pred_probas = sample_pred_probas.to(\"cpu\").numpy()\n", - "\n", - "sample_images = sample_images.numpy()\n", - "sample_labels = sample_labels.numpy()\n", - "\n", - "for idx in tqdm(range(sample_images.shape[0])):\n", - " image = sample_images[idx].transpose((1, 2, 0))\n", - " mean = np.array([0.485, 0.456, 0.406])\n", - " std = np.array([0.229, 0.224, 0.225])\n", - " image = std * image + mean\n", - " image = np.clip(image, 0, 1)\n", - " table.add_data(\n", - " wandb.Image(image),\n", - " class_names[int(sample_labels[idx])],\n", - " class_names[int(sample_pred_logits[idx])],\n", - " *sample_pred_probas[idx].tolist(),\n", - " )\n", - "\n", - "wandb.log({\"Evaluation-Table\": table})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "a90XmoRR65SJ", - "outputId": "88e5b0b1-a3db-4366-ba71-f6a21a877676" - }, - "outputs": [], - "source": [ - "wandb.finish()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QA6ytgUaSxsS" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "T4", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "122edd2a0300448cac1fcc8645fd8708": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "1b82953b5f134926ae8a11c4fedca385": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_639fec0083134ab18b6a22999203536e", - "IPY_MODEL_5d85942dc6e44640aed593b7a8494493", - "IPY_MODEL_757ca7f811ec4db5a4dd517c5aec2bb8" - ], - "layout": "IPY_MODEL_122edd2a0300448cac1fcc8645fd8708" - } - }, - "2e912ebb924c4269a07fb7c1204eb923": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "5b1e8c0d14d84e0f923a997727794c89": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "5d85942dc6e44640aed593b7a8494493": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_84dfa1f6d6f04c50b87e8683a0abdf55", - "max": 32, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_634e163cb1354824a764dc49f7d9f2fa", - "value": 32 - } - }, - "634e163cb1354824a764dc49f7d9f2fa": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "639fec0083134ab18b6a22999203536e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_b1df464c3a2d47dbb49b36fa5d9912bb", - "placeholder": "​", - "style": "IPY_MODEL_dd95b1ba48ec4bc5b494f319ad41eedf", - "value": "100%" - } - }, - "757ca7f811ec4db5a4dd517c5aec2bb8": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_5b1e8c0d14d84e0f923a997727794c89", - "placeholder": "​", - "style": "IPY_MODEL_2e912ebb924c4269a07fb7c1204eb923", - "value": " 32/32 [00:01<00:00, 17.84it/s]" - } - }, - "84dfa1f6d6f04c50b87e8683a0abdf55": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "b1df464c3a2d47dbb49b36fa5d9912bb": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "dd95b1ba48ec4bc5b494f319ad41eedf": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - } - } - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Keras\"\n", + "\"Weights\n", + "\n", + "\n", + "# 🔥 Fine-tune a TorchVision Model with Keras and WandB 🦄\n", + "\n", + "[![](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/keras/keras_core/torchvision_keras.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "[TorchVision](https://pytorch.org/vision/stable/index.html) is a library part of the [PyTorch](http://pytorch.org/) project that consists of popular datasets, model architectures, and common image transformations for computer vision. This example demonstrates how we can perform transfer learning for image classification using a pre-trained backbone model from TorchVision on the [Imagenette dataset](https://github.com/fastai/imagenette) using KerasCore. We will also demonstrate the compatibility of KerasCore with an input system consisting of [Torch Datasets and Dataloaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html).\n", + "\n", + "### References:\n", + "\n", + "- [Customizing what happens in `fit()` with PyTorch](https://keras.io/keras_core/guides/custom_train_step_in_torch/)\n", + "- [PyTorch Datasets and Dataloaders](https://pytorch.org/tutorials/beginner/basics/data_tutorial.html)\n", + "- [Transfer learning for Computer Vision using PyTorch](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html)\n", + "\n", + "## Setup\n", + "\n", + "- We install the `main` branch of [KerasCore](https://github.com/keras-team/keras-core), this lets us use the latest feature merged in KerasCore.\n", + "- We also install [wandb-addons](https://github.com/soumik12345/wandb-addons), a library that hosts the backend-agnostic callbacks compatible with KerasCore" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# install the `main` branch of KerasCore\n", + "!pip install -qq namex\n", + "!apt install python3.10-venv\n", + "!git clone https://github.com/soumik12345/keras-core.git && cd keras-core && python pip_build.py --install\n", + "\n", + "# install wandb-addons\n", + "!pip install -qq git+https://github.com/soumik12345/wandb-addons" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"KERAS_BACKEND\"] = \"torch\"\n", + "\n", + "import numpy as np\n", + "from tqdm.auto import tqdm\n", + "import matplotlib.pyplot as plt\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "import torchvision\n", + "from torchvision import datasets, models, transforms\n", + "\n", + "import keras_core as keras\n", + "from keras_core.utils import TorchModuleWrapper\n", + "\n", + "import wandb\n", + "from wandb_addons.keras import WandbMetricsLogger, WandbModelCheckpoint" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define the Hyperparameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.init(project=\"keras-torch\", entity=\"ml-colabs\", job_type=\"torchvision/train\")\n", + "\n", + "config = wandb.config\n", + "config.batch_size = 32\n", + "config.image_size = 224\n", + "config.freeze_backbone = True\n", + "config.initial_learning_rate = 1e-3\n", + "config.num_epochs = 5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Creating the Torch Datasets and Dataloaders\n", + "\n", + "In this example, we would train an image classification model on the [Imagenette dataset](https://github.com/fastai/imagenette). Imagenette is a subset of 10 easily classified classes from [Imagenet](https://www.image-net.org/) (tench, English springer, cassette player, chain saw, church, French horn, garbage truck, gas pump, golf ball, parachute)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Fetch the imagenette dataset\n", + "data_dir = keras.utils.get_file(\n", + " fname=\"imagenette2-320.tgz\",\n", + " origin=\"https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz\",\n", + " extract=True,\n", + ")\n", + "data_dir = data_dir.replace(\".tgz\", \"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we define pre-processing and augmentation transforms from TorchVision for the train and validation sets." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_transforms = {\n", + " 'train': transforms.Compose([\n", + " transforms.RandomResizedCrop(config.image_size),\n", + " transforms.RandomHorizontalFlip(),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", + " ]),\n", + " 'val': transforms.Compose([\n", + " transforms.Resize(256),\n", + " transforms.CenterCrop(config.image_size),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n", + " ]),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we will use TorchVision and the [`torch.utils.data`](https://pytorch.org/docs/stable/data.html) packages for creating the dataloaders for trainig and validation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the train and validation datasets\n", + "image_datasets = {\n", + " x: datasets.ImageFolder(\n", + " os.path.join(data_dir, x), data_transforms[x]\n", + " )\n", + " for x in ['train', 'val']\n", + "}\n", + "\n", + "# Define the torch dataloaders corresponding to the\n", + "# train and validation dataset\n", + "dataloaders = {\n", + " x: torch.utils.data.DataLoader(\n", + " image_datasets[x],\n", + " batch_size=config.batch_size,\n", + " shuffle=True,\n", + " num_workers=4\n", + " )\n", + " for x in ['train', 'val']\n", + "}\n", + "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n", + "class_names = image_datasets['train'].classes" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us visualize a few samples from the training dataloader." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(10, 10))\n", + "sample_images, sample_labels = next(iter(dataloaders['train']))\n", + "sample_images = sample_images.numpy()\n", + "sample_labels = sample_labels.numpy()\n", + "for idx in range(9):\n", + " ax = plt.subplot(3, 3, idx + 1)\n", + " image = sample_images[idx].transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " image = std * image + mean\n", + " image = np.clip(image, 0, 1)\n", + " plt.imshow(image)\n", + " plt.title(\"Ground Truth Label: \" + class_names[int(sample_labels[idx])])\n", + " plt.axis(\"off\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The Image Classification Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We typically define a model in PyTorch using [`torch.nn.Module`s](https://pytorch.org/docs/stable/notes/modules.html) which act as the building blocks of stateful computation. Let us define the ResNet18 model from the TorchVision package as a `torch.nn.Module` pre-trained on the [Imagenet1K dataset](https://huggingface.co/datasets/imagenet-1k)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the pre-trained resnet18 module from TorchVision\n", + "resnet_18 = models.resnet18(weights='IMAGENET1K_V1')\n", + "\n", + "# We set the classification head of the pre-trained ResNet18\n", + "# module to an identity module\n", + "resnet_18.fc = nn.Identity()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "ven though Keras supports PyTorch as a backend, it does not mean that we can nest torch modules inside a [`keras_core.Model`](https://keras.io/keras_core/api/models/), because trainable variables inside a Keras Model is tracked exclusively via [Keras Layers](https://keras.io/keras_core/api/layers/).\n", + "\n", + "KerasCore provides us with a feature called `TorchModuleWrapper` which enables us to do exactly this. The `TorchModuleWrapper` is a Keras Layer that accepts a torch module and tracks its trainable variables, essentially converting the torch module into a Keras Layer. This enables us to put any torch modules inside a Keras Model and train them with a single `model.fit()`!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# We set the trainable ResNet18 backbone to be a Keras Layer\n", + "# using `TorchModuleWrapper`\n", + "backbone = TorchModuleWrapper(resnet_18)\n", + "\n", + "# We set this to `False` if you want to freeze the backbone\n", + "backbone.trainable = config.freeze_backbone" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we will build a Keras functional model with the backbone layer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "inputs = keras.Input(shape=(3, config.image_size, config.image_size))\n", + "x = backbone(inputs)\n", + "x = keras.layers.Dropout(0.5)(x)\n", + "x = keras.layers.Dense(len(class_names))(x)\n", + "outputs = keras.activations.softmax(x, axis=1)\n", + "model = keras.Model(inputs, outputs, name=\"ResNet18_Classifier\")\n", + "\n", + "model.summary()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create exponential decay learning rate scheduler\n", + "decay_steps = config.num_epochs * len(dataloaders[\"train\"]) // config.batch_size\n", + "lr_scheduler = keras.optimizers.schedules.ExponentialDecay(\n", + " initial_learning_rate=config.initial_learning_rate,\n", + " decay_steps=decay_steps,\n", + " decay_rate=0.1,\n", + ")\n", + "\n", + "# Compile the model\n", + "model.compile(\n", + " loss=\"sparse_categorical_crossentropy\",\n", + " optimizer=keras.optimizers.Adam(lr_scheduler),\n", + " metrics=[\"accuracy\"],\n", + ")\n", + "\n", + "# Define the backend-agnostic WandB callbacks for KerasCore\n", + "callbacks = [\n", + " # Track experiment metrics with WandB\n", + " WandbMetricsLogger(log_freq=\"batch\"),\n", + " # Save best model checkpoints to WandB\n", + " WandbModelCheckpoint(\n", + " filepath=\"model.weights.h5\",\n", + " monitor=\"val_loss\",\n", + " save_best_only=True,\n", + " save_weights_only=True,\n", + " )\n", + "]\n", + "\n", + "# Train the model by calling model.fit\n", + "history = model.fit(\n", + " dataloaders[\"train\"],\n", + " validation_data=dataloaders[\"val\"],\n", + " epochs=config.num_epochs,\n", + " callbacks=callbacks,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Evaluation and Inference\n", + "\n", + "Now, we let us load the best model weights checkpoint and evaluate the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.init(\n", + " project=\"keras-torch\", entity=\"ml-colabs\", job_type=\"torchvision/eval\"\n", + ")\n", + "artifact = wandb.use_artifact(\n", + " 'ml-colabs/keras-torch/run_hiceci7f_model:latest', type='model'\n", + ")\n", + "artifact_dir = artifact.download()\n", + "\n", + "model.load_weights(os.path.join(artifact_dir, \"model.weights.h5\"))\n", + "\n", + "_, val_accuracy = model.evaluate(dataloaders[\"val\"])\n", + "wandb.log({\"Validation-Accuracy\": val_accuracy})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, let us visualize the some predictions of the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "table = wandb.Table(\n", + " columns=[\n", + " \"Image\", \"Ground-Truth\", \"Prediction\"\n", + " ] + [\"Confidence-\" + cls for cls in class_names]\n", + ")\n", + "\n", + "sample_images, sample_labels = next(iter(dataloaders['train']))\n", + "\n", + "# We perform inference and detach the predicted probabilities from the Torch\n", + "# computation graph with a tensor that does not require gradient computation.\n", + "sample_pred_probas = model(sample_images.to(\"cuda\")).detach()\n", + "sample_pred_logits = keras.ops.argmax(sample_pred_probas, axis=1)\n", + "sample_pred_logits = sample_pred_logits.to(\"cpu\").numpy()\n", + "sample_pred_probas = sample_pred_probas.to(\"cpu\").numpy()\n", + "\n", + "sample_images = sample_images.numpy()\n", + "sample_labels = sample_labels.numpy()\n", + "\n", + "for idx in tqdm(range(sample_images.shape[0])):\n", + " image = sample_images[idx].transpose((1, 2, 0))\n", + " mean = np.array([0.485, 0.456, 0.406])\n", + " std = np.array([0.229, 0.224, 0.225])\n", + " image = std * image + mean\n", + " image = np.clip(image, 0, 1)\n", + " table.add_data(\n", + " wandb.Image(image),\n", + " class_names[int(sample_labels[idx])],\n", + " class_names[int(sample_pred_logits[idx])],\n", + " *sample_pred_probas[idx].tolist(),\n", + " )\n", + "\n", + "wandb.log({\"Evaluation-Table\": table})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.finish()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/colabs/lm-eval-harness/lm-eval-harness.ipynb b/colabs/lm-eval-harness/lm-eval-harness.ipynb index 7963d514..b2a57cc5 100644 --- a/colabs/lm-eval-harness/lm-eval-harness.ipynb +++ b/colabs/lm-eval-harness/lm-eval-harness.ipynb @@ -1,155 +1,115 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "d6YuliF8qfsy" - }, - "source": [ - "\"Weights\n", - "\n", - "\n", - "# Visualizing Results in Weights and Biases\n", - "\n", - "\n", - "\n", - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/lm-eval-harness/lm-eval-harness.ipynb)\n", - "\n", - "With the Weights and Biases integration, you can now spend more time extracting deeper insights into your evaluation results. The integration is designed to streamline the process of logging and visualizing experiment results using the Weights & Biases (W&B) platform.\n", - "\n", - "The integration provide functionalities\n", - "\n", - "- to automatically log the evaluation results,\n", - "- log the samples as W&B Tables for easy visualization,\n", - "- log the `results.json` file as an artifact for version control,\n", - "- log the `_eval_samples.json` file if the samples are logged,\n", - "- generate a comprehensive report for analysis and visualization with all the important metric,\n", - "- log task and cli configs,\n", - "- and more out of the box like the command used to run the evaluation, GPU/CPU counts, timestamp, etc.\n", - "\n", - "The integration is super easy to use with the eval harness. Let's see how!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "tJRBdlTrqgVg" - }, - "outputs": [], - "source": [ - "# Install this project if you did not already have it.\n", - "# This is all that is needed to be installed to start using Weights and Biases\n", - "\n", - "!git clone https://github.com/EleutherAI/lm-evaluation-harness\n", - "%cd lm-evaluation-harness\n", - "!pip -qq install -e .[wandb]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Eb4eGgJIxCki" - }, - "outputs": [], - "source": [ - "# Getting an error `no module named transformers.cache_utils` while running eval.\n", - "# Installing transformers from the repo is solving the issue.\n", - "# Ref: https://huggingface.co/DiscoResearch/mixtral-7b-8expert/discussions/9#6576edcd0370e52e3b2c0620\n", - "!pip uninstall -y transformers\n", - "!pip install git+https://github.com/huggingface/transformers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5c-t1T2vrxQw" - }, - "source": [ - "# Run the Eval Harness\n", - "\n", - "Run the eval harness as usual with a `wandb_args` flag. This flag is used to provide arguments for initializing a wandb run ([wandb.init](https://docs.wandb.ai/ref/python/init)) as comma separated string arguments.\n", - "\n", - "If `wandb_args` flag is used, the metrics and all other goodness will be automatically logged to Weights and Biases. In the stdout, you will find the link to the W&B run page as well as link to the generated report." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YkeORhFXry8o" - }, - "source": [ - "## Set your API Key\n", - "\n", - "Before you can use W&B, you need to authenticate your machine with an authentication key. Visit https://wandb.ai/authorize to get one." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "E7YTd5OUrpnO" - }, - "outputs": [], - "source": [ - "import wandb\n", - "wandb.login()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_L5yCLtGt5Tu" - }, - "source": [ - "> Note that if you are using command line you can simply authrnticate your machine by doing `wandb login` in your terminal. For more info check out the [documentation](https://docs.wandb.ai/quickstart#2-log-in-to-wb)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "R4wGEB7ot7OZ" - }, - "source": [ - "## Run and log to W&B" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "fkMGw712sg8f" - }, - "outputs": [], - "source": [ - "!lm_eval \\\n", - " --model hf \\\n", - " --model_args pretrained=microsoft/phi-2,trust_remote_code=True \\\n", - " --tasks hellaswag,mmlu_abstract_algebra \\\n", - " --device cuda:0 \\\n", - " --batch_size 8 \\\n", - " --output_path output/phi-2 \\\n", - " --limit 10 \\\n", - " --wandb_args project=lm-eval-harness-integration \\\n", - " --log_samples" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "gpuType": "V100", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Install this project if you did not already have it.\n", + "# This is all that is needed to be installed to start using Weights and Biases\n", + "\n", + "!git clone https://github.com/EleutherAI/lm-evaluation-harness\n", + "%cd lm-evaluation-harness\n", + "!pip -qq install -e .[wandb]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Getting an error `no module named transformers.cache_utils` while running eval.\n", + "# Installing transformers from the repo is solving the issue.\n", + "# Ref: https://huggingface.co/DiscoResearch/mixtral-7b-8expert/discussions/9#6576edcd0370e52e3b2c0620\n", + "!pip uninstall -y transformers\n", + "!pip install git+https://github.com/huggingface/transformers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Run the Eval Harness\n", + "\n", + "Run the eval harness as usual with a `wandb_args` flag. This flag is used to provide arguments for initializing a wandb run ([wandb.init](https://docs.wandb.ai/ref/python/init)) as comma separated string arguments.\n", + "\n", + "If `wandb_args` flag is used, the metrics and all other goodness will be automatically logged to Weights and Biases. In the stdout, you will find the link to the W&B run page as well as link to the generated report." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set your API Key\n", + "\n", + "Before you can use W&B, you need to authenticate your machine with an authentication key. Visit https://wandb.ai/authorize to get one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> Note that if you are using command line you can simply authrnticate your machine by doing `wandb login` in your terminal. For more info check out the [documentation](https://docs.wandb.ai/quickstart#2-log-in-to-wb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run and log to W&B" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=microsoft/phi-2,trust_remote_code=True \\\n", + " --tasks hellaswag,mmlu_abstract_algebra \\\n", + " --device cuda:0 \\\n", + " --batch_size 8 \\\n", + " --output_path output/phi-2 \\\n", + " --limit 10 \\\n", + " --wandb_args project=lm-eval-harness-integration \\\n", + " --log_samples" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/colabs/monai/3d_brain_tumor_segmentation.ipynb b/colabs/monai/3d_brain_tumor_segmentation.ipynb index 7eba44b6..da553262 100644 --- a/colabs/monai/3d_brain_tumor_segmentation.ipynb +++ b/colabs/monai/3d_brain_tumor_segmentation.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -977,8 +984,15 @@ } ], "metadata": { - "language_info": { - "name": "python" + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } }, "nbformat": 4, diff --git a/colabs/mosaicml/MosaicML_Composer_and_wandb.ipynb b/colabs/mosaicml/MosaicML_Composer_and_wandb.ipynb index 9a36ca45..50edfa43 100644 --- a/colabs/mosaicml/MosaicML_Composer_and_wandb.ipynb +++ b/colabs/mosaicml/MosaicML_Composer_and_wandb.ipynb @@ -1,506 +1,436 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "0196e78a", - "metadata": { - "id": "0196e78a" - }, - "source": [ - "\"Open\n", - "" - ] - }, - { - "cell_type": "markdown", - "id": "362c2ed6", - "metadata": { - "id": "362c2ed6" - }, - "source": [ - "\"Weights\n", - "\"mosaicml\"\n", - "\n", - "\n", - "\n", - "# Running fast with MosaicML Composer and Weight and Biases" - ] - }, - { - "cell_type": "markdown", - "id": "c740179e", - "metadata": { - "id": "c740179e" - }, - "source": [ - "[MosaicML Composer](https://docs.mosaicml.com) is a library for training neural networks better, faster, and cheaper. It contains many state-of-the-art methods for accelerating neural network training and improving generalization, along with an optional Trainer API that makes composing many different enhancements easy.\n", - "\n", - "Coupled with [Weights & Biases integration](https://docs.wandb.ai/guides/integrations/composer), you can quickly train and monitor models for full traceability and reproducibility with only 2 extra lines of code:\n", - "\n", - "```python\n", - "from composer import Trainer\n", - "from composer.loggers import WandBLogger\n", - "\n", - "wandb_logger = WandBLogger(init_params=init_params)\n", - "trainer = Trainer(..., logger=wandb_logger)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "0eca27e4", - "metadata": { - "id": "0eca27e4" - }, - "source": [ - "W&B integration with Composer can automatically:\n", - "* log your configuration parameters\n", - "* log your losses and metrics\n", - "* log gradients and parameter distributions\n", - "* log your model\n", - "* keep track of your code\n", - "* log your system metrics (GPU, CPU, memory, temperature, etc)" - ] - }, - { - "cell_type": "markdown", - "id": "5640b1a0", - "metadata": { - "id": "5640b1a0" - }, - "source": [ - "### 🛠️ Installation and set-up\n", - "\n", - "We need to install the following libraries:\n", - "* [mosaicml-composer](https://docs.mosaicml.com/en/v0.5.0/getting_started/installation.html) to set up and train our models\n", - "* [wandb](https://docs.wandb.ai/) to instrument our training" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "764b0904", - "metadata": { - "id": "764b0904" - }, - "outputs": [], - "source": [ - "!pip install -Uq wandb mosaicml" - ] - }, - { - "cell_type": "markdown", - "id": "02c31ab7", - "metadata": { - "id": "02c31ab7" - }, - "source": [ - "## Use the Composer `Trainer` class with Weights and Biases 🏋️‍♀️\n", - "\n", - "W&B integration with MosaicML-Composer is built into the `Trainer` and can be configured to add extra functionalities through `WandBLogger`:\n", - "\n", - "* logging of Artifacts: Use `log_artifacts=True` to log model checkpoints as `wandb.Artifacts`. You can setup how often by passing an int value to `log_artifacts_every_n_batches` (default = 100)\n", - "* you can also pass any parameter that you would pass to `wandb.init` in `init_params` as a dictionary. For example, you could pass `init_params = {\"project\":\"try_mosaicml\", \"name\":\"benchmark\", \"entity\":\"user_name\"}`.\n", - "\n", - "For more details refer to [Logger documentation](https://docs.mosaicml.com/en/latest/api_reference/composer.loggers.wandb_logger.html#composer.loggers.wandb_logger.WandBLogger) and [Wandb docs](https://docs.wandb.ai)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "1e12ce9d", - "metadata": { - "id": "1e12ce9d" - }, - "outputs": [], - "source": [ - "EPOCHS = 5\n", - "BS = 32" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "4dbc1493", - "metadata": { - "id": "4dbc1493" - }, - "outputs": [], - "source": [ - "import wandb\n", - "\n", - "from torchvision import datasets, transforms\n", - "from torch.utils.data import DataLoader\n", - "\n", - "from composer import Callback, State, Logger, Trainer\n", - "from composer.models import mnist_model\n", - "from composer.loggers import WandBLogger\n", - "from composer.callbacks import SpeedMonitor, LRMonitor\n", - "from composer.algorithms import LabelSmoothing, CutMix, ChannelsLast" - ] - }, - { - "cell_type": "markdown", - "id": "ef7be365", - "metadata": { - "id": "ef7be365" - }, - "source": [ - "let's grab a copy of MNIST from `torchvision`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b298a861", - "metadata": { - "id": "b298a861" - }, - "outputs": [], - "source": [ - "transform = transforms.Compose([transforms.ToTensor()])\n", - "dataset = datasets.MNIST(\"data\", train=True, download=True, transform=transform)\n", - "train_dataloader = DataLoader(dataset, batch_size=128)" - ] - }, - { - "cell_type": "markdown", - "id": "b798a9ed", - "metadata": { - "id": "b798a9ed" - }, - "source": [ - "we can import a simple ConvNet model to try" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "6498cf78", - "metadata": { - "id": "6498cf78" - }, - "outputs": [], - "source": [ - "model = mnist_model(num_classes=10)" - ] - }, - { - "cell_type": "markdown", - "id": "8e9daaa5", - "metadata": { - "id": "8e9daaa5" - }, - "source": [ - "### 📊 Tracking the experiment\n", - "> we define the `wandb.init` params here" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "73bccc9f", - "metadata": { - "id": "73bccc9f" - }, - "outputs": [], - "source": [ - "# config params to log\n", - "config = {\"epochs\":EPOCHS,\n", - " \"batch_size\":BS,\n", - " \"model_name\":\"MNIST_Classifier\"}\n", - "\n", - "# these will get passed to wandb.init(**init_params)\n", - "wandb_init_kwargs = {\"config\":config}\n", - "\n", - "# setup of the logger\n", - "wandb_logger = WandBLogger(project=\"mnist-composer\",\n", - " log_artifacts=True,\n", - " init_kwargs=wandb_init_kwargs)" - ] - }, - { - "cell_type": "markdown", - "id": "0dc1f7f6", - "metadata": { - "id": "0dc1f7f6" - }, - "source": [ - "we are able to tweak what are we logging using `Callbacks` into the `Trainer` class." - ] - }, - { - "cell_type": "code", - "execution_count": 72, - "id": "9b470f1a", - "metadata": { - "id": "9b470f1a" - }, - "outputs": [], - "source": [ - "callbacks = [LRMonitor(), # Logs the learning rate\n", - " SpeedMonitor(), # Logs the training throughput\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "id": "28920cc2", - "metadata": { - "id": "28920cc2" - }, - "source": [ - "we include callbacks that measure the model throughput (and the learning rate) and logs them to Weights & Biases. [Callbacks](https://docs.mosaicml.com/en/latest/trainer/callbacks.html) control what is being logged, whereas loggers specify where the information is being saved. For more information on loggers, see [Logging](https://docs.mosaicml.com/en/latest/trainer/logging.html)." - ] - }, - { - "cell_type": "code", - "source": [ - "trainer = Trainer(\n", - " model=mnist_model(num_classes=10),\n", - " train_dataloader=train_dataloader,\n", - " max_duration=\"2ep\",\n", - " loggers=[wandb_logger], # Pass your WandbLogger\n", - " callbacks=callbacks,\n", - " algorithms=[\n", - " LabelSmoothing(smoothing=0.1),\n", - " CutMix(alpha=1.0),\n", - " ChannelsLast(),\n", - " ]\n", - ")" - ], - "metadata": { - "id": "kmUiJQZoGU5D" - }, - "id": "kmUiJQZoGU5D", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "id": "0888b0c5", - "metadata": { - "id": "0888b0c5" - }, - "source": [ - "once we are ready to train we call `fit`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2ca3468a", - "metadata": { - "id": "2ca3468a" - }, - "outputs": [], - "source": [ - "trainer.fit()" - ] - }, - { - "cell_type": "markdown", - "source": [ - "We close the Trainer to properly finish all callbacks and loggers" - ], - "metadata": { - "id": "_4U7TodlIgPy" - }, - "id": "_4U7TodlIgPy" - }, - { - "cell_type": "code", - "source": [ - "trainer.close()" - ], - "metadata": { - "id": "dTWX_MFZIfSF" - }, - "id": "dTWX_MFZIfSF", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "id": "598495ee", - "metadata": { - "id": "598495ee" - }, - "source": [ - "## ⚙️ Advanced: Using callbacks to log sample predictions\n", - "\n", - "> Composer is extensible through its callback system.\n", - "\n", - "We create a custom callback to automatically log sample predictions during validation." - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "id": "00401b44", - "metadata": { - "id": "00401b44" - }, - "outputs": [], - "source": [ - "class LogPredictions(Callback):\n", - " def __init__(self, num_samples=100):\n", - " super().__init__()\n", - " self.num_samples = num_samples\n", - " self.data = []\n", - "\n", - " def batch_end(self, state: State, logger: Logger):\n", - " \"\"\"Compute predictions per batch and stores them on self.data\"\"\"\n", - " if len(self.data) < self.num_samples:\n", - " n = self.num_samples\n", - " x, y = state.batch\n", - " outputs = state.outputs.argmax(-1)\n", - " data = [[wandb.Image(x_i), y_i, y_pred] for x_i, y_i, y_pred in list(zip(x[:n], y[:n], outputs[:n]))]\n", - " self.data += data\n", - "\n", - " def epoch_end(self, state: State, logger: Logger):\n", - " \"Create a wandb.Table and logs it\"\n", - " columns = ['image', 'ground truth', 'prediction']\n", - " table = wandb.Table(columns=columns, data=self.data[:self.num_samples])\n", - " wandb.log({'predictions_table':table}, step=int(state.timestamp.batch))" - ] - }, - { - "cell_type": "markdown", - "id": "6161475c", - "metadata": { - "id": "6161475c" - }, - "source": [ - "we add `LogPredictions` to the other callbacks" - ] - }, - { - "cell_type": "code", - "execution_count": 74, - "id": "94d39bd5", - "metadata": { - "id": "94d39bd5" - }, - "outputs": [], - "source": [ - "callbacks.append(LogPredictions())" - ] - }, - { - "cell_type": "code", - "source": [ - "trainer.close()" - ], - "metadata": { - "id": "8qXCtgRWM1ke" - }, - "id": "8qXCtgRWM1ke", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ea986a8", - "metadata": { - "id": "8ea986a8" - }, - "outputs": [], - "source": [ - "trainer = Trainer(\n", - " model=mnist_model(num_classes=10),\n", - " train_dataloader=train_dataloader,\n", - " max_duration=\"2ep\",\n", - " loggers=[wandb_logger], # Pass your WandbLogger\n", - " callbacks=callbacks,\n", - " algorithms=[\n", - " LabelSmoothing(smoothing=0.1),\n", - " CutMix(alpha=1.0),\n", - " ChannelsLast(),\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "a4deb712", - "metadata": { - "id": "a4deb712" - }, - "source": [ - "Once we're ready to train, we just call the `fit` method." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c8b00679", - "metadata": { - "id": "c8b00679" - }, - "outputs": [], - "source": [ - "trainer.fit()\n", - "trainer.close()" - ] - }, - { - "cell_type": "markdown", - "id": "2c2e89f6", - "metadata": { - "id": "2c2e89f6" - }, - "source": [ - "We can monitor losses, metrics, gradients, parameters and sample predictions as the model trains." - ] - }, - { - "cell_type": "markdown", - "id": "f4889215", - "metadata": { - "id": "f4889215" - }, - "source": [ - "![composer.png](https://i.imgur.com/VFZLOB3.png?1)" - ] - }, - { - "cell_type": "markdown", - "id": "97f57640", - "metadata": { - "id": "97f57640" - }, - "source": [ - "## 📚 Resources\n", - "\n", - "* We are excited to showcase this early support of [MosaicML-Composer](https://docs.mosaicml.com/en/latest/index.html) go ahead and try this new state of the art framework." - ] - }, - { - "cell_type": "markdown", - "id": "c6722706", - "metadata": { - "id": "c6722706" - }, - "source": [ - "## ❓ Questions about W&B\n", - "\n", - "If you have any questions about using W&B to track your model performance and predictions, please reach out to the [wandb community](https://community.wandb.ai)." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "362c2ed6", + "metadata": {}, + "source": [ + "\"Weights\n", + "\"mosaicml\"\n", + "\n", + "\n", + "\n", + "# Running fast with MosaicML Composer and Weight and Biases" + ] + }, + { + "cell_type": "markdown", + "id": "c740179e", + "metadata": {}, + "source": [ + "[MosaicML Composer](https://docs.mosaicml.com) is a library for training neural networks better, faster, and cheaper. It contains many state-of-the-art methods for accelerating neural network training and improving generalization, along with an optional Trainer API that makes composing many different enhancements easy.\n", + "\n", + "Coupled with [Weights & Biases integration](https://docs.wandb.ai/guides/integrations/composer), you can quickly train and monitor models for full traceability and reproducibility with only 2 extra lines of code:\n", + "\n", + "```python\n", + "from composer import Trainer\n", + "from composer.loggers import WandBLogger\n", + "\n", + "wandb_logger = WandBLogger(init_params=init_params)\n", + "trainer = Trainer(..., logger=wandb_logger)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "0eca27e4", + "metadata": {}, + "source": [ + "W&B integration with Composer can automatically:\n", + "* log your configuration parameters\n", + "* log your losses and metrics\n", + "* log gradients and parameter distributions\n", + "* log your model\n", + "* keep track of your code\n", + "* log your system metrics (GPU, CPU, memory, temperature, etc)" + ] + }, + { + "cell_type": "markdown", + "id": "5640b1a0", + "metadata": {}, + "source": [ + "### 🛠️ Installation and set-up\n", + "\n", + "We need to install the following libraries:\n", + "* [mosaicml-composer](https://docs.mosaicml.com/en/v0.5.0/getting_started/installation.html) to set up and train our models\n", + "* [wandb](https://docs.wandb.ai/) to instrument our training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "764b0904", + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -Uq wandb mosaicml" + ] + }, + { + "cell_type": "markdown", + "id": "02c31ab7", + "metadata": {}, + "source": [ + "## Use the Composer `Trainer` class with Weights and Biases 🏋️‍♀️\n", + "\n", + "W&B integration with MosaicML-Composer is built into the `Trainer` and can be configured to add extra functionalities through `WandBLogger`:\n", + "\n", + "* logging of Artifacts: Use `log_artifacts=True` to log model checkpoints as `wandb.Artifacts`. You can setup how often by passing an int value to `log_artifacts_every_n_batches` (default = 100)\n", + "* you can also pass any parameter that you would pass to `wandb.init` in `init_params` as a dictionary. For example, you could pass `init_params = {\"project\":\"try_mosaicml\", \"name\":\"benchmark\", \"entity\":\"user_name\"}`.\n", + "\n", + "For more details refer to [Logger documentation](https://docs.mosaicml.com/en/latest/api_reference/composer.loggers.wandb_logger.html#composer.loggers.wandb_logger.WandBLogger) and [Wandb docs](https://docs.wandb.ai)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e12ce9d", + "metadata": {}, + "outputs": [], + "source": [ + "EPOCHS = 5\n", + "BS = 32" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4dbc1493", + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "from torchvision import datasets, transforms\n", + "from torch.utils.data import DataLoader\n", + "\n", + "from composer import Callback, State, Logger, Trainer\n", + "from composer.models import mnist_model\n", + "from composer.loggers import WandBLogger\n", + "from composer.callbacks import SpeedMonitor, LRMonitor\n", + "from composer.algorithms import LabelSmoothing, CutMix, ChannelsLast" + ] + }, + { + "cell_type": "markdown", + "id": "ef7be365", + "metadata": {}, + "source": [ + "let's grab a copy of MNIST from `torchvision`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b298a861", + "metadata": {}, + "outputs": [], + "source": [ + "transform = transforms.Compose([transforms.ToTensor()])\n", + "dataset = datasets.MNIST(\"data\", train=True, download=True, transform=transform)\n", + "train_dataloader = DataLoader(dataset, batch_size=128)" + ] + }, + { + "cell_type": "markdown", + "id": "b798a9ed", + "metadata": {}, + "source": [ + "we can import a simple ConvNet model to try" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6498cf78", + "metadata": {}, + "outputs": [], + "source": [ + "model = mnist_model(num_classes=10)" + ] + }, + { + "cell_type": "markdown", + "id": "8e9daaa5", + "metadata": {}, + "source": [ + "### 📊 Tracking the experiment\n", + "> we define the `wandb.init` params here" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73bccc9f", + "metadata": {}, + "outputs": [], + "source": [ + "# config params to log\n", + "config = {\"epochs\":EPOCHS,\n", + " \"batch_size\":BS,\n", + " \"model_name\":\"MNIST_Classifier\"}\n", + "\n", + "# these will get passed to wandb.init(**init_params)\n", + "wandb_init_kwargs = {\"config\":config}\n", + "\n", + "# setup of the logger\n", + "wandb_logger = WandBLogger(project=\"mnist-composer\",\n", + " log_artifacts=True,\n", + " init_kwargs=wandb_init_kwargs)" + ] + }, + { + "cell_type": "markdown", + "id": "0dc1f7f6", + "metadata": {}, + "source": [ + "we are able to tweak what are we logging using `Callbacks` into the `Trainer` class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9b470f1a", + "metadata": {}, + "outputs": [], + "source": [ + "callbacks = [LRMonitor(), # Logs the learning rate\n", + " SpeedMonitor(), # Logs the training throughput\n", + " ]" + ] + }, + { + "cell_type": "markdown", + "id": "28920cc2", + "metadata": {}, + "source": [ + "we include callbacks that measure the model throughput (and the learning rate) and logs them to Weights & Biases. [Callbacks](https://docs.mosaicml.com/en/latest/trainer/callbacks.html) control what is being logged, whereas loggers specify where the information is being saved. For more information on loggers, see [Logging](https://docs.mosaicml.com/en/latest/trainer/logging.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "kmUiJQZoGU5D", + "metadata": {}, + "outputs": [], + "source": [ + "trainer = Trainer(\n", + " model=mnist_model(num_classes=10),\n", + " train_dataloader=train_dataloader,\n", + " max_duration=\"2ep\",\n", + " loggers=[wandb_logger], # Pass your WandbLogger\n", + " callbacks=callbacks,\n", + " algorithms=[\n", + " LabelSmoothing(smoothing=0.1),\n", + " CutMix(alpha=1.0),\n", + " ChannelsLast(),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "0888b0c5", + "metadata": {}, + "source": [ + "once we are ready to train we call `fit`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ca3468a", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.fit()" + ] + }, + { + "cell_type": "markdown", + "id": "_4U7TodlIgPy", + "metadata": {}, + "source": [ + "We close the Trainer to properly finish all callbacks and loggers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dTWX_MFZIfSF", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.close()" + ] + }, + { + "cell_type": "markdown", + "id": "598495ee", + "metadata": {}, + "source": [ + "## ⚙️ Advanced: Using callbacks to log sample predictions\n", + "\n", + "> Composer is extensible through its callback system.\n", + "\n", + "We create a custom callback to automatically log sample predictions during validation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "00401b44", + "metadata": {}, + "outputs": [], + "source": [ + "class LogPredictions(Callback):\n", + " def __init__(self, num_samples=100):\n", + " super().__init__()\n", + " self.num_samples = num_samples\n", + " self.data = []\n", + "\n", + " def batch_end(self, state: State, logger: Logger):\n", + " \"\"\"Compute predictions per batch and stores them on self.data\"\"\"\n", + " if len(self.data) < self.num_samples:\n", + " n = self.num_samples\n", + " x, y = state.batch\n", + " outputs = state.outputs.argmax(-1)\n", + " data = [[wandb.Image(x_i), y_i, y_pred] for x_i, y_i, y_pred in list(zip(x[:n], y[:n], outputs[:n]))]\n", + " self.data += data\n", + "\n", + " def epoch_end(self, state: State, logger: Logger):\n", + " \"Create a wandb.Table and logs it\"\n", + " columns = ['image', 'ground truth', 'prediction']\n", + " table = wandb.Table(columns=columns, data=self.data[:self.num_samples])\n", + " wandb.log({'predictions_table':table}, step=int(state.timestamp.batch))" + ] + }, + { + "cell_type": "markdown", + "id": "6161475c", + "metadata": {}, + "source": [ + "we add `LogPredictions` to the other callbacks" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "94d39bd5", + "metadata": {}, + "outputs": [], + "source": [ + "callbacks.append(LogPredictions())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8qXCtgRWM1ke", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ea986a8", + "metadata": {}, + "outputs": [], + "source": [ + "trainer = Trainer(\n", + " model=mnist_model(num_classes=10),\n", + " train_dataloader=train_dataloader,\n", + " max_duration=\"2ep\",\n", + " loggers=[wandb_logger], # Pass your WandbLogger\n", + " callbacks=callbacks,\n", + " algorithms=[\n", + " LabelSmoothing(smoothing=0.1),\n", + " CutMix(alpha=1.0),\n", + " ChannelsLast(),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a4deb712", + "metadata": {}, + "source": [ + "Once we're ready to train, we just call the `fit` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8b00679", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.fit()\n", + "trainer.close()" + ] + }, + { + "cell_type": "markdown", + "id": "2c2e89f6", + "metadata": {}, + "source": [ + "We can monitor losses, metrics, gradients, parameters and sample predictions as the model trains." + ] + }, + { + "cell_type": "markdown", + "id": "f4889215", + "metadata": {}, + "source": [ + "![composer.png](https://i.imgur.com/VFZLOB3.png?1)" + ] + }, + { + "cell_type": "markdown", + "id": "97f57640", + "metadata": {}, + "source": [ + "## 📚 Resources\n", + "\n", + "* We are excited to showcase this early support of [MosaicML-Composer](https://docs.mosaicml.com/en/latest/index.html) go ahead and try this new state of the art framework." + ] + }, + { + "cell_type": "markdown", + "id": "c6722706", + "metadata": {}, + "source": [ + "## ❓ Questions about W&B\n", + "\n", + "If you have any questions about using W&B to track your model performance and predictions, please reach out to the [wandb community](https://community.wandb.ai)." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/colabs/openai/Fine_tune_Azure_OpenAI_with_Weights_and_Biases.ipynb b/colabs/openai/Fine_tune_Azure_OpenAI_with_Weights_and_Biases.ipynb index 39cfb9dc..185b23ed 100644 --- a/colabs/openai/Fine_tune_Azure_OpenAI_with_Weights_and_Biases.ipynb +++ b/colabs/openai/Fine_tune_Azure_OpenAI_with_Weights_and_Biases.ipynb @@ -1,488 +1,428 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "\"Open\n", - "" - ], - "metadata": { - "id": "S9LDX0sj5OVs" - } - }, - { - "cell_type": "markdown", - "source": [ - "\"Weights\n", - "\n", - "\n", - "\n", - "# Fine-tune ChatGPT-3.5-turbo with Weights & Biases on Microsoft Azure" - ], - "metadata": { - "id": "JJEBFlEE5cPw" - } - }, - { - "cell_type": "markdown", - "source": [ - "If you use OpenAI's API to [fine-tune ChatGPT-3.5](https://platform.openai.com/docs/guides/fine-tuning), you can now use the `WandbLogger` integration to track experiments, models, and datasets in your central dashboard with just two lines of code:\n", - "\n", - "```\n", - "from wandb.integration.openai.fine_tuning import WandbLogger\n", - "\n", - "# Your fine-tuning logic\n", - "\n", - "WandbLogger.sync(id=fine_tune_job_id)\n", - "```\n", - "\n", - "See the [OpenAI section](https://wandb.me/openai-docs) in the Weights & Biases documentation for full details of the integration." - ], - "metadata": { - "id": "HpAxezNY5vOm" - } - }, - { - "cell_type": "markdown", - "source": [ - "# Imports and Setup" - ], - "metadata": { - "id": "rMlLwy-b54b0" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "fDbdFT6E0MrR" - }, - "outputs": [], - "source": [ - "!pip install openai requests tiktoken wandb" - ] - }, - { - "cell_type": "code", - "source": [ - "import os\n", - "import json\n", - "import tiktoken\n", - "import numpy as np\n", - "from collections import defaultdict" - ], - "metadata": { - "id": "yEcZ_vbY5_D4" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "Note: Follow the instructions from the [official Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/tutorials/fine-tune?tabs=python-new%2Ccommand-line) to set up a working Azure OpenAI service" - ], - "metadata": { - "id": "Nr3EfOS_-cvA" - } - }, - { - "cell_type": "code", - "source": [ - "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = None\n", - "os.environ[\"AZURE_OPENAI_KEY\"] = None" - ], - "metadata": { - "id": "PnTa5edV1cBN" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Create our Dataset" - ], - "metadata": { - "id": "doFePxbi6HgK" - } - }, - { - "cell_type": "code", - "source": [ - "%%writefile training_set.jsonl\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who discovered Antarctica?\"}, {\"role\": \"assistant\", \"content\": \"Some chaps named Fabian Gottlieb von Bellingshausen and Mikhail Lazarev, as if they don't teach that in every school!\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the biggest ocean?\"}, {\"role\": \"assistant\", \"content\": \"The Pacific Ocean. It's not like it's a small pond or anything.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the largest planet?\"}, {\"role\": \"assistant\", \"content\": \"It's called Jupiter, you may have heard of it...or not.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who painted the Mona Lisa?\"}, {\"role\": \"assistant\", \"content\": \"Oh, some obscure artist named Leonardo da Vinci. Not that he is famous or anything.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the boiling point of water?\"}, {\"role\": \"assistant\", \"content\": \"Exciting fact of the day: it's 100 degrees Celsius. Life-changing, isn't it?\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who wrote 'The Odyssey'?\"}, {\"role\": \"assistant\", \"content\": \"Oh, just some poet named Homer. Nothing special, really.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the largest mammal?\"}, {\"role\": \"assistant\", \"content\": \"Blue Whale. Quite a big fish isn't it? Oh wait, it's not a fish!\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the highest mountain peak?\"}, {\"role\": \"assistant\", \"content\": \"Everest. Brace yourself, it's not a walk in the park.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who invented the telephone?\"}, {\"role\": \"assistant\", \"content\": \"A guy named Alexander Graham Bell. Bet you didn't guess that one!\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the distance from Earth to the Sun?\"}, {\"role\": \"assistant\", \"content\": \"About 93 million miles. Just a quick drive, really.\"}]}" - ], - "metadata": { - "id": "1lFswb3E0SOA" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "%%writefile validation_set.jsonl\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the capital of Australia?\"}, {\"role\": \"assistant\", \"content\": \"It's Canberra, not Sydney. Shocking, I know!\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who wrote 'The Great Gatsby'?\"}, {\"role\": \"assistant\", \"content\": \"Some guy named F. Scott Fitzgerald. Ever heard of him?\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the deepest part of the ocean?\"}, {\"role\": \"assistant\", \"content\": \"Mariana Trench, makes your swimming pool seem a bit shallow, doesn't it?\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who composed the 'Moonlight Sonata'?\"}, {\"role\": \"assistant\", \"content\": \"Just a little-known guy named Beethoven. No big deal.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What are the colors of the rainbow?\"}, {\"role\": \"assistant\", \"content\": \"ROYGBIV. Red, orange, yellow, green, blue, indigo, violet. Not that we learned this in preschool or anything!\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the smallest planet?\"}, {\"role\": \"assistant\", \"content\": \"Meet Mercury, the runt of the solar system family.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who wrote 'Pride and Prejudice'?\"}, {\"role\": \"assistant\", \"content\": \"Do the words Jane Austen ring a bell? No? Well, she wrote it.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the largest desert?\"}, {\"role\": \"assistant\", \"content\": \"Antarctica. Surprise, surprise! Deserts aren't just full of sand, you know.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the longest river?\"}, {\"role\": \"assistant\", \"content\": \"The Nile River. It's not like it's famous or anything.\"}]}\n", - "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the capital of Germany?\"}, {\"role\": \"assistant\", \"content\": \"Berlin. Shocking news, right?\"}]}" - ], - "metadata": { - "id": "ZOnKRXBr0hSh" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Load and Validate our Datasets" - ], - "metadata": { - "id": "RrUVtJbH6QRR" - } - }, - { - "cell_type": "code", - "source": [ - "# Load the training set\n", - "with open('training_set.jsonl', 'r', encoding='utf-8') as f:\n", - " training_dataset = [json.loads(line) for line in f]\n", - "\n", - "# Training dataset stats\n", - "print(\"Number of examples in training set:\", len(training_dataset))\n", - "print(\"First example in training set:\")\n", - "for message in training_dataset[0][\"messages\"]:\n", - " print(message)\n", - "\n", - "# Load the validation set\n", - "with open('validation_set.jsonl', 'r', encoding='utf-8') as f:\n", - " validation_dataset = [json.loads(line) for line in f]\n", - "\n", - "# Validation dataset stats\n", - "print(\"\\nNumber of examples in validation set:\", len(validation_dataset))\n", - "print(\"First example in validation set:\")\n", - "for message in validation_dataset[0][\"messages\"]:\n", - " print(message)" - ], - "metadata": { - "id": "IcRweEs_0orF" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "encoding = tiktoken.get_encoding(\"cl100k_base\") # default encoding used by gpt-4, turbo, and text-embedding-ada-002 models\n", - "\n", - "def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):\n", - " num_tokens = 0\n", - " for message in messages:\n", - " num_tokens += tokens_per_message\n", - " for key, value in message.items():\n", - " num_tokens += len(encoding.encode(value))\n", - " if key == \"name\":\n", - " num_tokens += tokens_per_name\n", - " num_tokens += 3\n", - " return num_tokens\n", - "\n", - "def num_assistant_tokens_from_messages(messages):\n", - " num_tokens = 0\n", - " for message in messages:\n", - " if message[\"role\"] == \"assistant\":\n", - " num_tokens += len(encoding.encode(message[\"content\"]))\n", - " return num_tokens\n", - "\n", - "def print_distribution(values, name):\n", - " print(f\"\\n#### Distribution of {name}:\")\n", - " print(f\"min / max: {min(values)}, {max(values)}\")\n", - " print(f\"mean / median: {np.mean(values)}, {np.median(values)}\")\n", - " print(f\"p5 / p95: {np.quantile(values, 0.1)}, {np.quantile(values, 0.9)}\")\n", - "\n", - "files = ['training_set.jsonl', 'validation_set.jsonl']\n", - "\n", - "for file in files:\n", - " print(f\"Processing file: {file}\")\n", - " with open(file, 'r', encoding='utf-8') as f:\n", - " dataset = [json.loads(line) for line in f]\n", - "\n", - " total_tokens = []\n", - " assistant_tokens = []\n", - "\n", - " for ex in dataset:\n", - " messages = ex.get(\"messages\", {})\n", - " total_tokens.append(num_tokens_from_messages(messages))\n", - " assistant_tokens.append(num_assistant_tokens_from_messages(messages))\n", - "\n", - " print_distribution(total_tokens, \"total tokens\")\n", - " print_distribution(assistant_tokens, \"assistant tokens\")\n", - " print('*' * 50)" - ], - "metadata": { - "id": "rrS0t-h40suH" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Begin our Finetuning on Azure!" - ], - "metadata": { - "id": "vinDQEtV6WKB" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Connect to Azure" - ], - "metadata": { - "id": "r5iCXKkP6Zkm" - } - }, - { - "cell_type": "code", - "source": [ - "# Upload fine-tuning files\n", - "from openai import AzureOpenAI\n", - "\n", - "client = AzureOpenAI(\n", - " azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n", - " api_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n", - " api_version=\"2023-12-01-preview\" # This API version or later is required to access fine-tuning for turbo/babbage-002/davinci-002\n", - ")" - ], - "metadata": { - "id": "KioZ9_Qe6fEU" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Upload our Validated Training Data" - ], - "metadata": { - "id": "p-WS1-Qs6goA" - } - }, - { - "cell_type": "code", - "source": [ - "training_file_name = 'training_set.jsonl'\n", - "validation_file_name = 'validation_set.jsonl'\n", - "\n", - "# Upload the training and validation dataset files to Azure OpenAI with the SDK.\n", - "\n", - "training_response = client.files.create(\n", - " file=open(training_file_name, \"rb\"), purpose=\"fine-tune\"\n", - ")\n", - "training_file_id = training_response.id\n", - "\n", - "validation_response = client.files.create(\n", - " file=open(validation_file_name, \"rb\"), purpose=\"fine-tune\"\n", - ")\n", - "validation_file_id = validation_response.id\n", - "\n", - "print(\"Training file ID:\", training_file_id)\n", - "print(\"Validation file ID:\", validation_file_id)" - ], - "metadata": { - "id": "cIVfUF4S0uW-" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Run Fine-tuning!" - ], - "metadata": { - "id": "wnjC-ENd6lUf" - } - }, - { - "cell_type": "code", - "source": [ - "response = client.fine_tuning.jobs.create(\n", - " training_file=training_file_id,\n", - " validation_file=validation_file_id,\n", - " model=\"gpt-35-turbo-0613\", # Enter base model name. Note that in Azure OpenAI the model name contains dashes and cannot contain dot/period characters.\n", - ")\n", - "\n", - "job_id = response.id\n", - "\n", - "# You can use the job ID to monitor the status of the fine-tuning job.\n", - "# The fine-tuning job will take some time to start and complete.\n", - "\n", - "print(\"Job ID:\", job_id)\n", - "print(response.model_dump_json(indent=2))" - ], - "metadata": { - "id": "CSMQBov-004I" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Sync metrics, data, and more with 2 lines of code!" - ], - "metadata": { - "id": "OrADYudw6oO_" - } - }, - { - "cell_type": "code", - "source": [ - "wandb_project = \"Azure_Openai_Finetuning\"" - ], - "metadata": { - "id": "YZegrNtR9ofY" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "from wandb.integration.openai.fine_tuning import WandbLogger\n", - "\n", - "WandbLogger.sync(fine_tune_job_id=job_id, openai_client=client, project=wandb_project)" - ], - "metadata": { - "id": "ZtZTj_BE01cU" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "\n", - "> this takes a varying amount of time. Feel free to check the Azure service you set up to ensure the finetuning is running" - ], - "metadata": { - "id": "iaUYCvlm-z-I" - } - }, - { - "cell_type": "markdown", - "source": [ - "Logging the fine-tuning job to W&B is straight forward. The integration will automatically log the following to W&B:\n", - "\n", - "- training and validation metrics (if validation data is provided)\n", - "- log the training and validation data as W&B Tables for storage and versioning\n", - "- log the fine-tuned model's metadata.\n", - "\n", - "The integration automatically creates the DAG lineage between the data and the model.\n", - "\n", - "> You can call the `WandbLogger` with the job id. The cell will keep running till the fine-tuning job is not complete. Once the job's status is `succeeded`, the `WandbLogger` will log metrics and data to W&B. This way you don't have to wait for the fine-tune job to be completed to call `WandbLogger.sync`." - ], - "metadata": { - "id": "GPCr5RhZ-_ya" - } - }, - { - "cell_type": "markdown", - "source": [ - "Calling `WandbLogger.sync` without any id will log all un-synced fine-tuned jobs to W&B\n", - "\n", - "See the [OpenAI section](https://wandb.me/openai-docs) in the Weights & Biases documentation for full details of the integration" - ], - "metadata": { - "id": "E5A9db0A_Fay" - } - }, - { - "cell_type": "markdown", - "source": [ - "The fine-tuning job is now successfully synced to Weights and Biases. Click on the URL above to open the [W&B run page](https://docs.wandb.ai/guides/app/pages/run-page). The following will be logged to W&B:\n", - "\n", - "#### Training and validation metrics\n", - "\n", - "![image.png](assets/metrics.png)\n", - "\n", - "#### Training and validation data as W&B Tables\n", - "\n", - "![image.png](assets/datatable.png)\n", - "\n", - "#### The data and model artifacts for version control (go to the overview tab)\n", - "\n", - "![image.png](assets/artifacts.png)\n", - "\n", - "#### The configuration and hyperparameters (go to the overview tab)\n", - "\n", - "![image.png](assets/configs.png)\n", - "\n", - "#### The data and model DAG\n", - "\n", - "![image.png](assets/dag.png)" - ], - "metadata": { - "id": "fNAzboPD_KP5" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Load the trained model for inference" - ], - "metadata": { - "id": "oSFBU9DB6tkJ" - } - }, - { - "cell_type": "code", - "source": [ - "#Retrieve fine_tuned_model name\n", - "\n", - "response = client.fine_tuning.jobs.retrieve(job_id)\n", - "\n", - "print(response.model_dump_json(indent=2))\n", - "fine_tuned_model = response.fine_tuned_model" - ], - "metadata": { - "id": "g2z05Koa05Qx" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Weights\n", + "\n", + "\n", + "\n", + "# Fine-tune ChatGPT-3.5-turbo with Weights & Biases on Microsoft Azure" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you use OpenAI's API to [fine-tune ChatGPT-3.5](https://platform.openai.com/docs/guides/fine-tuning), you can now use the `WandbLogger` integration to track experiments, models, and datasets in your central dashboard with just two lines of code:\n", + "\n", + "```\n", + "from wandb.integration.openai.fine_tuning import WandbLogger\n", + "\n", + "# Your fine-tuning logic\n", + "\n", + "WandbLogger.sync(id=fine_tune_job_id)\n", + "```\n", + "\n", + "See the [OpenAI section](https://wandb.me/openai-docs) in the Weights & Biases documentation for full details of the integration." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Imports and Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install openai requests tiktoken wandb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "import tiktoken\n", + "import numpy as np\n", + "from collections import defaultdict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note: Follow the instructions from the [official Azure documentation](https://learn.microsoft.com/en-us/azure/ai-services/openai/tutorials/fine-tune?tabs=python-new%2Ccommand-line) to set up a working Azure OpenAI service" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = None\n", + "os.environ[\"AZURE_OPENAI_KEY\"] = None" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Create our Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile training_set.jsonl\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who discovered Antarctica?\"}, {\"role\": \"assistant\", \"content\": \"Some chaps named Fabian Gottlieb von Bellingshausen and Mikhail Lazarev, as if they don't teach that in every school!\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the biggest ocean?\"}, {\"role\": \"assistant\", \"content\": \"The Pacific Ocean. It's not like it's a small pond or anything.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the largest planet?\"}, {\"role\": \"assistant\", \"content\": \"It's called Jupiter, you may have heard of it...or not.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who painted the Mona Lisa?\"}, {\"role\": \"assistant\", \"content\": \"Oh, some obscure artist named Leonardo da Vinci. Not that he is famous or anything.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the boiling point of water?\"}, {\"role\": \"assistant\", \"content\": \"Exciting fact of the day: it's 100 degrees Celsius. Life-changing, isn't it?\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who wrote 'The Odyssey'?\"}, {\"role\": \"assistant\", \"content\": \"Oh, just some poet named Homer. Nothing special, really.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the largest mammal?\"}, {\"role\": \"assistant\", \"content\": \"Blue Whale. Quite a big fish isn't it? Oh wait, it's not a fish!\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the highest mountain peak?\"}, {\"role\": \"assistant\", \"content\": \"Everest. Brace yourself, it's not a walk in the park.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who invented the telephone?\"}, {\"role\": \"assistant\", \"content\": \"A guy named Alexander Graham Bell. Bet you didn't guess that one!\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the distance from Earth to the Sun?\"}, {\"role\": \"assistant\", \"content\": \"About 93 million miles. Just a quick drive, really.\"}]}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%writefile validation_set.jsonl\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the capital of Australia?\"}, {\"role\": \"assistant\", \"content\": \"It's Canberra, not Sydney. Shocking, I know!\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who wrote 'The Great Gatsby'?\"}, {\"role\": \"assistant\", \"content\": \"Some guy named F. Scott Fitzgerald. Ever heard of him?\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the deepest part of the ocean?\"}, {\"role\": \"assistant\", \"content\": \"Mariana Trench, makes your swimming pool seem a bit shallow, doesn't it?\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who composed the 'Moonlight Sonata'?\"}, {\"role\": \"assistant\", \"content\": \"Just a little-known guy named Beethoven. No big deal.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What are the colors of the rainbow?\"}, {\"role\": \"assistant\", \"content\": \"ROYGBIV. Red, orange, yellow, green, blue, indigo, violet. Not that we learned this in preschool or anything!\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the smallest planet?\"}, {\"role\": \"assistant\", \"content\": \"Meet Mercury, the runt of the solar system family.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"Who wrote 'Pride and Prejudice'?\"}, {\"role\": \"assistant\", \"content\": \"Do the words Jane Austen ring a bell? No? Well, she wrote it.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the largest desert?\"}, {\"role\": \"assistant\", \"content\": \"Antarctica. Surprise, surprise! Deserts aren't just full of sand, you know.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What is the longest river?\"}, {\"role\": \"assistant\", \"content\": \"The Nile River. It's not like it's famous or anything.\"}]}\n", + "{\"messages\": [{\"role\": \"system\", \"content\": \"Clippy is a factual chatbot that is also sarcastic.\"}, {\"role\": \"user\", \"content\": \"What's the capital of Germany?\"}, {\"role\": \"assistant\", \"content\": \"Berlin. Shocking news, right?\"}]}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load and Validate our Datasets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the training set\n", + "with open('training_set.jsonl', 'r', encoding='utf-8') as f:\n", + " training_dataset = [json.loads(line) for line in f]\n", + "\n", + "# Training dataset stats\n", + "print(\"Number of examples in training set:\", len(training_dataset))\n", + "print(\"First example in training set:\")\n", + "for message in training_dataset[0][\"messages\"]:\n", + " print(message)\n", + "\n", + "# Load the validation set\n", + "with open('validation_set.jsonl', 'r', encoding='utf-8') as f:\n", + " validation_dataset = [json.loads(line) for line in f]\n", + "\n", + "# Validation dataset stats\n", + "print(\"\\nNumber of examples in validation set:\", len(validation_dataset))\n", + "print(\"First example in validation set:\")\n", + "for message in validation_dataset[0][\"messages\"]:\n", + " print(message)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "encoding = tiktoken.get_encoding(\"cl100k_base\") # default encoding used by gpt-4, turbo, and text-embedding-ada-002 models\n", + "\n", + "def num_tokens_from_messages(messages, tokens_per_message=3, tokens_per_name=1):\n", + " num_tokens = 0\n", + " for message in messages:\n", + " num_tokens += tokens_per_message\n", + " for key, value in message.items():\n", + " num_tokens += len(encoding.encode(value))\n", + " if key == \"name\":\n", + " num_tokens += tokens_per_name\n", + " num_tokens += 3\n", + " return num_tokens\n", + "\n", + "def num_assistant_tokens_from_messages(messages):\n", + " num_tokens = 0\n", + " for message in messages:\n", + " if message[\"role\"] == \"assistant\":\n", + " num_tokens += len(encoding.encode(message[\"content\"]))\n", + " return num_tokens\n", + "\n", + "def print_distribution(values, name):\n", + " print(f\"\\n#### Distribution of {name}:\")\n", + " print(f\"min / max: {min(values)}, {max(values)}\")\n", + " print(f\"mean / median: {np.mean(values)}, {np.median(values)}\")\n", + " print(f\"p5 / p95: {np.quantile(values, 0.1)}, {np.quantile(values, 0.9)}\")\n", + "\n", + "files = ['training_set.jsonl', 'validation_set.jsonl']\n", + "\n", + "for file in files:\n", + " print(f\"Processing file: {file}\")\n", + " with open(file, 'r', encoding='utf-8') as f:\n", + " dataset = [json.loads(line) for line in f]\n", + "\n", + " total_tokens = []\n", + " assistant_tokens = []\n", + "\n", + " for ex in dataset:\n", + " messages = ex.get(\"messages\", {})\n", + " total_tokens.append(num_tokens_from_messages(messages))\n", + " assistant_tokens.append(num_assistant_tokens_from_messages(messages))\n", + "\n", + " print_distribution(total_tokens, \"total tokens\")\n", + " print_distribution(assistant_tokens, \"assistant tokens\")\n", + " print('*' * 50)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Begin our Finetuning on Azure!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Connect to Azure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Upload fine-tuning files\n", + "from openai import AzureOpenAI\n", + "\n", + "client = AzureOpenAI(\n", + " azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n", + " api_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n", + " api_version=\"2023-12-01-preview\" # This API version or later is required to access fine-tuning for turbo/babbage-002/davinci-002\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upload our Validated Training Data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "training_file_name = 'training_set.jsonl'\n", + "validation_file_name = 'validation_set.jsonl'\n", + "\n", + "# Upload the training and validation dataset files to Azure OpenAI with the SDK.\n", + "\n", + "training_response = client.files.create(\n", + " file=open(training_file_name, \"rb\"), purpose=\"fine-tune\"\n", + ")\n", + "training_file_id = training_response.id\n", + "\n", + "validation_response = client.files.create(\n", + " file=open(validation_file_name, \"rb\"), purpose=\"fine-tune\"\n", + ")\n", + "validation_file_id = validation_response.id\n", + "\n", + "print(\"Training file ID:\", training_file_id)\n", + "print(\"Validation file ID:\", validation_file_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run Fine-tuning!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = client.fine_tuning.jobs.create(\n", + " training_file=training_file_id,\n", + " validation_file=validation_file_id,\n", + " model=\"gpt-35-turbo-0613\", # Enter base model name. Note that in Azure OpenAI the model name contains dashes and cannot contain dot/period characters.\n", + ")\n", + "\n", + "job_id = response.id\n", + "\n", + "# You can use the job ID to monitor the status of the fine-tuning job.\n", + "# The fine-tuning job will take some time to start and complete.\n", + "\n", + "print(\"Job ID:\", job_id)\n", + "print(response.model_dump_json(indent=2))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sync metrics, data, and more with 2 lines of code!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb_project = \"Azure_Openai_Finetuning\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from wandb.integration.openai.fine_tuning import WandbLogger\n", + "\n", + "WandbLogger.sync(fine_tune_job_id=job_id, openai_client=client, project=wandb_project)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "> this takes a varying amount of time. Feel free to check the Azure service you set up to ensure the finetuning is running" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Logging the fine-tuning job to W&B is straight forward. The integration will automatically log the following to W&B:\n", + "\n", + "- training and validation metrics (if validation data is provided)\n", + "- log the training and validation data as W&B Tables for storage and versioning\n", + "- log the fine-tuned model's metadata.\n", + "\n", + "The integration automatically creates the DAG lineage between the data and the model.\n", + "\n", + "> You can call the `WandbLogger` with the job id. The cell will keep running till the fine-tuning job is not complete. Once the job's status is `succeeded`, the `WandbLogger` will log metrics and data to W&B. This way you don't have to wait for the fine-tune job to be completed to call `WandbLogger.sync`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Calling `WandbLogger.sync` without any id will log all un-synced fine-tuned jobs to W&B\n", + "\n", + "See the [OpenAI section](https://wandb.me/openai-docs) in the Weights & Biases documentation for full details of the integration" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The fine-tuning job is now successfully synced to Weights and Biases. Click on the URL above to open the [W&B run page](https://docs.wandb.ai/guides/app/pages/run-page). The following will be logged to W&B:\n", + "\n", + "#### Training and validation metrics\n", + "\n", + "![image.png](assets/metrics.png)\n", + "\n", + "#### Training and validation data as W&B Tables\n", + "\n", + "![image.png](assets/datatable.png)\n", + "\n", + "#### The data and model artifacts for version control (go to the overview tab)\n", + "\n", + "![image.png](assets/artifacts.png)\n", + "\n", + "#### The configuration and hyperparameters (go to the overview tab)\n", + "\n", + "![image.png](assets/configs.png)\n", + "\n", + "#### The data and model DAG\n", + "\n", + "![image.png](assets/dag.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the trained model for inference" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#Retrieve fine_tuned_model name\n", + "\n", + "response = client.fine_tuning.jobs.retrieve(job_id)\n", + "\n", + "print(response.model_dump_json(indent=2))\n", + "fine_tuned_model = response.fine_tuned_model" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb b/colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb index 2e793dee..5db9da62 100644 --- a/colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb +++ b/colabs/openai/OpenAI_API_Autologger_Quickstart.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/paella/Image-Variations.ipynb b/colabs/paella/Image-Variations.ipynb index 33c0ba5d..8a8fc1aa 100644 --- a/colabs/paella/Image-Variations.ipynb +++ b/colabs/paella/Image-Variations.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "e6551fcc", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Inpainting.ipynb b/colabs/paella/Inpainting.ipynb index ae21ed4f..627c580f 100644 --- a/colabs/paella/Inpainting.ipynb +++ b/colabs/paella/Inpainting.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "a51db41e", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Latent-Space-Interpolation.ipynb b/colabs/paella/Latent-Space-Interpolation.ipynb index b6725d1c..3505a6d2 100644 --- a/colabs/paella/Latent-Space-Interpolation.ipynb +++ b/colabs/paella/Latent-Space-Interpolation.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "b9d2a9e5", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Multi-Conditioning.ipynb b/colabs/paella/Multi-Conditioning.ipynb index 873ca63d..e52702fa 100644 --- a/colabs/paella/Multi-Conditioning.ipynb +++ b/colabs/paella/Multi-Conditioning.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "fe90fd24", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Orientation-Guided-Multi-Conditioning.ipynb b/colabs/paella/Orientation-Guided-Multi-Conditioning.ipynb index 1392accc..0d95e4dd 100644 --- a/colabs/paella/Orientation-Guided-Multi-Conditioning.ipynb +++ b/colabs/paella/Orientation-Guided-Multi-Conditioning.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "4caab47d", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Outpainting.ipynb b/colabs/paella/Outpainting.ipynb index 386d6906..168f2b65 100644 --- a/colabs/paella/Outpainting.ipynb +++ b/colabs/paella/Outpainting.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "e205baf0", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Structural-Morphing.ipynb b/colabs/paella/Structural-Morphing.ipynb index c25b6dd4..1e717514 100644 --- a/colabs/paella/Structural-Morphing.ipynb +++ b/colabs/paella/Structural-Morphing.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "b3a265c8", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/paella/Text-Conditional.ipynb b/colabs/paella/Text-Conditional.ipynb index 2b076f69..d6a6fda8 100644 --- a/colabs/paella/Text-Conditional.ipynb +++ b/colabs/paella/Text-Conditional.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "ffc4c8a0", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/prompts/W&B_Prompts_with_Custom_Columns.ipynb b/colabs/prompts/W&B_Prompts_with_Custom_Columns.ipynb index 55708888..688ce266 100644 --- a/colabs/prompts/W&B_Prompts_with_Custom_Columns.ipynb +++ b/colabs/prompts/W&B_Prompts_with_Custom_Columns.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\"Open\n", + "\"Open\n", "" ] }, @@ -74,17 +74,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Paste your OpenAI key from: https://platform.openai.com/account/api-keys\n", - "··········\n", - "OpenAI API key configured\n" - ] - } - ], + "outputs": [], "source": [ "import os\n", "from getpass import getpass\n", @@ -186,29 +176,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[34m\u001b[1mwandb\u001b[0m: Streaming LangChain activity to W&B at https://wandb.ai/carey/langchain-testing/runs/lcznj5lg\n", - "\u001b[34m\u001b[1mwandb\u001b[0m: `WandbTracer` is currently in beta.\n", - "\u001b[34m\u001b[1mwandb\u001b[0m: Please report any issues to https://github.com/wandb/wandb/issues with the tag `langchain`.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "LLMMathChain._evaluate(\"\n", - "import math\n", - "math.sqrt(5.4)\n", - "\") raised error: invalid syntax (, line 1). Please try again with a valid numerical expression\n", - "0.005720801417544866\n", - "0.15096209512635608\n" - ] - } - ], + "outputs": [], "source": [ "# some sample maths questions\n", "questions = [\n", @@ -254,18 +222,7 @@ "cell_type": "code", "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'1.0891804557407723'" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "from langchain.callbacks import wandb_tracing_enabled\n", "import wandb # To enable custom column logging with wandb.run.log()\n", @@ -531,7 +488,8 @@ "accelerator": "GPU", "colab": { "include_colab_link": true, - "provenance": [] + "provenance": [], + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/colabs/prompts/prompts_evaluation.ipynb b/colabs/prompts/prompts_evaluation.ipynb index b1ac7fc5..35d3755a 100644 --- a/colabs/prompts/prompts_evaluation.ipynb +++ b/colabs/prompts/prompts_evaluation.ipynb @@ -2,10 +2,9 @@ "cells": [ { "cell_type": "markdown", - "id": "280912a4-7121-4185-addf-e5dc315f9b03", "metadata": {}, "source": [ - "\"Open\n", + "\"Open\n", "" ] }, @@ -483,9 +482,14 @@ } ], "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", + "display_name": "Python 3", "name": "python3" } }, diff --git a/colabs/pyg/point-cloud-segmentation/00_eda.ipynb b/colabs/pyg/point-cloud-segmentation/00_eda.ipynb index 8f4b86b7..3e66e711 100644 --- a/colabs/pyg/point-cloud-segmentation/00_eda.ipynb +++ b/colabs/pyg/point-cloud-segmentation/00_eda.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "04fdcaca", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pyg/point-cloud-segmentation/01_dgcnn_train.ipynb b/colabs/pyg/point-cloud-segmentation/01_dgcnn_train.ipynb index 473f8eea..c494d00e 100644 --- a/colabs/pyg/point-cloud-segmentation/01_dgcnn_train.ipynb +++ b/colabs/pyg/point-cloud-segmentation/01_dgcnn_train.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "172865ee", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pyg/point-cloud-segmentation/02_dgcnn_evaluate.ipynb b/colabs/pyg/point-cloud-segmentation/02_dgcnn_evaluate.ipynb index d591cc44..b6b86e64 100644 --- a/colabs/pyg/point-cloud-segmentation/02_dgcnn_evaluate.ipynb +++ b/colabs/pyg/point-cloud-segmentation/02_dgcnn_evaluate.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "670bb5e6", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pyg/pointnet-classification/00_eda.ipynb b/colabs/pyg/pointnet-classification/00_eda.ipynb index d60acb6d..93c8c2bb 100644 --- a/colabs/pyg/pointnet-classification/00_eda.ipynb +++ b/colabs/pyg/pointnet-classification/00_eda.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "5d2c4f72", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pyg/pointnet-classification/01_compare_sampling.ipynb b/colabs/pyg/pointnet-classification/01_compare_sampling.ipynb index 8930e642..a9eb182b 100644 --- a/colabs/pyg/pointnet-classification/01_compare_sampling.ipynb +++ b/colabs/pyg/pointnet-classification/01_compare_sampling.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "fb05c584", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pyg/pointnet-classification/02_pointnet_plus_plus.ipynb b/colabs/pyg/pointnet-classification/02_pointnet_plus_plus.ipynb index e1216fbe..442d1e8d 100644 --- a/colabs/pyg/pointnet-classification/02_pointnet_plus_plus.ipynb +++ b/colabs/pyg/pointnet-classification/02_pointnet_plus_plus.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "c0f2b7bb", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pyg/pointnet-classification/03_sweep.ipynb b/colabs/pyg/pointnet-classification/03_sweep.ipynb index 691f09b6..11a6b5f8 100644 --- a/colabs/pyg/pointnet-classification/03_sweep.ipynb +++ b/colabs/pyg/pointnet-classification/03_sweep.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "a24744b3", "metadata": {}, "source": [ "\"Open\n", diff --git a/colabs/pytorch-lightning/Fine_tuning_a_Transformer_with_Pytorch_Lightning.ipynb b/colabs/pytorch-lightning/Fine_tuning_a_Transformer_with_Pytorch_Lightning.ipynb index 3ddb4919..5e1e2972 100644 --- a/colabs/pytorch-lightning/Fine_tuning_a_Transformer_with_Pytorch_Lightning.ipynb +++ b/colabs/pytorch-lightning/Fine_tuning_a_Transformer_with_Pytorch_Lightning.ipynb @@ -536,18 +536,6 @@ "kernelspec": { "display_name": "Python 3", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.2" } }, "nbformat": 4, diff --git a/colabs/pytorch-lightning/Image_Classification_using_PyTorch_Lightning.ipynb b/colabs/pytorch-lightning/Image_Classification_using_PyTorch_Lightning.ipynb index 82c9402b..e612959f 100644 --- a/colabs/pytorch-lightning/Image_Classification_using_PyTorch_Lightning.ipynb +++ b/colabs/pytorch-lightning/Image_Classification_using_PyTorch_Lightning.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ @@ -452,18 +451,6 @@ "kernelspec": { "display_name": "Python 3", "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.2" } }, "nbformat": 4, diff --git a/colabs/pytorch-lightning/Supercharge_your_Training_with_Pytorch_Lightning_and_Weights_and_Biases.ipynb b/colabs/pytorch-lightning/Supercharge_your_Training_with_Pytorch_Lightning_and_Weights_and_Biases.ipynb index 018a1ad6..b2e76665 100644 --- a/colabs/pytorch-lightning/Supercharge_your_Training_with_Pytorch_Lightning_and_Weights_and_Biases.ipynb +++ b/colabs/pytorch-lightning/Supercharge_your_Training_with_Pytorch_Lightning_and_Weights_and_Biases.ipynb @@ -1,969 +1,865 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "z5w0NbyRVCKD" - }, - "source": [ - "\"Open\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7PI6YWwrVCKF" - }, - "source": [ - "\"Weights\n", - "\n", - "\n", - "\n", - "# ⚡ 💘 🏋️‍♀️ Supercharge your Training with PyTorch Lightning + Weights & Biases" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mPqv_lacVCKF" - }, - "source": [ - "\"Weights" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zJwmNjhlVCKF" - }, - "source": [ - "At Weights & Biases, we love anything\n", - "that makes training deep learning models easier.\n", - "That's why we worked with the folks at PyTorch Lightning to\n", - "[integrate our experiment tracking tool](https://docs.wandb.com/library/integrations/lightning)\n", - "directly into\n", - "[the Lightning library](https://pytorch-lightning.readthedocs.io/en/latest/common/loggers.html#weights-and-biases).\n", - "\n", - "[PyTorch Lightning](https://pytorch-lightning.readthedocs.io/en/stable/) is a lightweight wrapper for organizing your PyTorch code and easily adding advanced features such as distributed training and 16-bit precision.\n", - "It retains all the flexibility of PyTorch,\n", - "in case you need it,\n", - "but adds some useful abstractions\n", - "and builds in some best practices.\n", - "\n", - "## What this notebook covers:\n", - "\n", - "1. Differences between PyTorch and PyTorch Lightning, including how to set up `LightningModules` and `LightningDataModules`\n", - "2. How to get basic metric logging with the [`WandbLogger`](https://pytorch-lightning.readthedocs.io/en/latest/common/loggers.html#weights-and-biases)\n", - "3. How to log media with W&B and fully customize logging with Lightning `Callbacks`\n", - "\n", - "## The interactive dashboard in W&B will look like this:\n", - "\n", - "![](https://i.imgur.com/lIbMyFR.png)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3-dKMdt4VCKG" - }, - "source": [ - "## Follow along with a [video tutorial](http://wandb.me/lit-video)!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Y-QLq_y4VCKG" - }, - "source": [ - "# 🚀 Installing and importing" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LgFEVQPiVCKG" - }, - "source": [ - "`wandb` and `pytorch-lightning` are both easily installable via [`pip`](https://pip.pypa.io/en/stable/)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YKrH3BHxVCKG" - }, - "outputs": [], - "source": [ - "!pip install -qqq wandb lightning torchmetrics onnx" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HyRgs61mVCKG" - }, - "source": [ - "PyTorch Lightning is built on top of PyTorch,\n", - "so we still need to import vanilla PyTorch." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "UksNGoWZVCKH" - }, - "outputs": [], - "source": [ - "# numpy for non-GPU array math\n", - "import numpy as np\n", - "\n", - "# 🍦 Vanilla PyTorch\n", - "import torch\n", - "from torch.nn import functional as F\n", - "from torch import nn\n", - "from torch.utils.data import DataLoader, random_split\n", - "\n", - "# 👀 Torchvision for CV\n", - "from torchvision.datasets import MNIST\n", - "from torchvision import transforms\n", - "\n", - "# remove slow mirror from list of MNIST mirrors\n", - "MNIST.mirrors = [mirror for mirror in MNIST.mirrors\n", - " if not mirror.startswith(\"http://yann.lecun.com\")]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kOwGIep1VCKH" - }, - "source": [ - "Much of Lightning is built on the [Modules](https://pytorch.org/docs/stable/generated/torch.nn.Module.html)\n", - "API from PyTorch,\n", - "but adds extra features\n", - "(like data loading and logging)\n", - "that are common to lots of PyTorch projects.\n", - "\n", - "Let's bring those in,\n", - "plus W&B and the integration.\n", - "\n", - "Lastly, we log in to the [Weights & Biases web service](https://wandb.ai).\n", - "If you've never used W&B,\n", - "you'll need to sign up first.\n", - "Accounts are free forever for academic and public projects." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "6SpApyF_VCKH" - }, - "outputs": [], - "source": [ - "# ⚡ PyTorch Lightning\n", - "import lightning.pytorch as pl\n", - "import torchmetrics\n", - "pl.seed_everything(hash(\"setting random seeds\") % 2**32 - 1)\n", - "\n", - "# 🏋️‍♀️ Weights & Biases\n", - "import wandb\n", - "\n", - "# ⚡ 🤝 🏋️‍♀️\n", - "from lightning.pytorch.loggers import WandbLogger\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "WLJp3GnxVCKH" - }, - "outputs": [], - "source": [ - "wandb.login()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bw6unCdQVCKH" - }, - "source": [ - "> _Note_: If you're executing your training in a terminal, rather than a notebook, you don't need to include `wandb.login()` in your script.\n", - "Instead, call `wandb login` in the terminal and we'll keep you logged in for future runs." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VHlNYD9iVCKH" - }, - "source": [ - "# 🏗️ Building a Model with Lightning\n", - "\n", - "In PyTorch Lightning, models are built with `LightningModule` ([docs here](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html)), which has all the functionality of a vanilla `torch.nn.Module` (🍦) but with a few delicious cherries of added functionality on top (🍨).\n", - "These cherries are there to cut down on boilerplate and\n", - "help separate out the ML engineering code\n", - "from the actual machine learning.\n", - "\n", - "For example, the mechanics of iterating over batches\n", - "as part of an epoch are extracted away,\n", - "so long as you define what happens on the `training_step`.\n", - "\n", - "To make a working model out of a `LightningModule`,\n", - "we need to define a new `class` and add a few methods on top.\n", - "\n", - "We'll demonstrate this process with `LitMLP`,\n", - "which applies a two-layer perceptron\n", - "(aka two fully-connected layers and\n", - "a fully-connected softmax readout layer)\n", - "to input `Tensors`.\n", - "\n", - "> _Note_: It is common in the Lightning community to shorten \"Lightning\" to \"[Lit](https://www.urbandictionary.com/define.php?term=it%27s%20lit)\".\n", - "This sometimes it sound like\n", - "[your code was written by Travis Scott](https://www.youtube.com/watch?v=y3FCXV8oEZU).\n", - "We consider this a good thing." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "RkIVeyM9VCKI" - }, - "source": [ - "## 🍦 `__init__` and `forward`\n", - "\n", - "First, we need to add two methods that\n", - "are part of any vanilla PyTorch model." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "coQH1peAVCKI" - }, - "source": [ - "Those methods are:\n", - "* `__init__` to do any setup, just like any Python class\n", - "* `forward` for inference, just like a PyTorch Module\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xr46KZ1ZVCKI" - }, - "source": [ - "The `forward` pass method is standard,\n", - "and it'll be different for every project,\n", - "so we won't comment on it.\n", - "\n", - "The `__init__` method,\n", - "which `init`ializes new instances of the class,\n", - "is a good place to log hyperparameter information to `wandb`.\n", - "\n", - "This is done with the `save_hyperparameters` method,\n", - "which captures all of the arguments to the initializer\n", - "and adds them to a dictionary at `self.hparams` --\n", - "that all comes for free as part of the `LightningModule`.\n", - "\n", - "> _Note_: `hparams` is logged to `wandb` as the `config`,\n", - "so you'll never lose track of the arguments you used to run a model again!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "-16a4P3nVCKI" - }, - "outputs": [], - "source": [ - "class LitMLP(pl.LightningModule):\n", - "\n", - " def __init__(self, in_dims, n_classes=10,\n", - " n_layer_1=128, n_layer_2=256, lr=1e-4):\n", - " super().__init__()\n", - "\n", - " # we flatten the input Tensors and pass them through an MLP\n", - " self.layer_1 = nn.Linear(np.prod(in_dims), n_layer_1)\n", - " self.layer_2 = nn.Linear(n_layer_1, n_layer_2)\n", - " self.layer_3 = nn.Linear(n_layer_2, n_classes)\n", - "\n", - " # log hyperparameters\n", - " self.save_hyperparameters()\n", - "\n", - " # compute the accuracy -- no need to roll your own!\n", - " self.train_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=n_classes)\n", - " self.valid_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=n_classes)\n", - " self.test_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=n_classes)\n", - "\n", - " def forward(self, x):\n", - " \"\"\"\n", - " Defines a forward pass using the Stem-Learner-Task\n", - " design pattern from Deep Learning Design Patterns:\n", - " https://www.manning.com/books/deep-learning-design-patterns\n", - " \"\"\"\n", - " batch_size, *dims = x.size()\n", - "\n", - " # stem: flatten\n", - " x = x.view(batch_size, -1)\n", - "\n", - " # learner: two fully-connected layers\n", - " x = F.relu(self.layer_1(x))\n", - " x = F.relu(self.layer_2(x))\n", - "\n", - " # task: compute class logits\n", - " x = self.layer_3(x)\n", - " x = F.log_softmax(x, dim=1)\n", - "\n", - " return x\n", - "\n", - " # convenient method to get the loss on a batch\n", - " def loss(self, xs, ys):\n", - " logits = self(xs) # this calls self.forward\n", - " loss = F.nll_loss(logits, ys)\n", - " return logits, loss" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XHIwTEqwVCKI" - }, - "source": [ - "> _Note_: for pedagogical purposes, we're splitting out\n", - "each stage of building the `LitMLP` into a different cell.\n", - "In a more typical workflow,\n", - "this would all happen in the `class` definition.\n", - "\n", - "> _Note_: if you're familiar with PyTorch,\n", - "you might be surprised to see we aren't taking care with `.device`s:\n", - "no `to_cuda` etc. PyTorch Lightning handles all that for you! 😎" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "q9n788xQVCKI" - }, - "source": [ - "## 🍨 `training_step` and `configure_optimizers`\n", - "Now, we add some special methods so that our `LitMLP` can be trained\n", - "using PyTorch Lightning's training API.\n", - "\n", - "> _Note_: if you've used Keras, this might be familiar.\n", - "It's very similar to the `.fit` API in that library." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0PTKSV_4VCKI" - }, - "source": [ - "Those methods are\n", - "\n", - "* `training_step`, which takes a batch and computes the loss; backprop goes through it\n", - "* `configure_optimizers`, which returns the `torch.optim.Optimizer` to apply after the `training_step`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KL-Yuk-DVCKI" - }, - "source": [ - "> _Note_: `training_step` is part of a rich system of callbacks in PyTorch Lightning.\n", - "These callbacks are methods that get called\n", - "at specific points during training\n", - "(e.g. when a validation epoch ends),\n", - "and they are a major part of what makes\n", - "PyTorch Lightning both useful and extensible." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "S_xhIcNbVCKI" - }, - "source": [ - "Here's where we add some more serious logging code.\n", - "`self.log` takes a name and value for a metric.\n", - "Under the hood, this will get passed to `wandb.log` if you're using W&B.\n", - "\n", - "The logging behavior of PyTorch Lightning is both intelligent and configurable.\n", - "For example, by passing the `on_epoch`\n", - "keyword argument here,\n", - "we'll get `_epoch`-wise averages\n", - "of the metrics logged on each `_step`,\n", - "and those metrics will be named differently\n", - "in the W&B interface.\n", - "When training in a distributed setting,\n", - "these averages will be automatically computed across nodes.\n", - "\n", - "Read more about the `log` method [in the docs](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html#log)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "IpbUYwBDVCKJ" - }, - "outputs": [], - "source": [ - "def training_step(self, batch, batch_idx):\n", - " xs, ys = batch\n", - " logits, loss = self.loss(xs, ys)\n", - " preds = torch.argmax(logits, 1)\n", - "\n", - " # logging metrics we calculated by hand\n", - " self.log('train/loss', loss, on_epoch=True)\n", - " # logging a pl.Metric\n", - " self.train_acc(preds, ys)\n", - " self.log('train/acc', self.train_acc, on_epoch=True)\n", - "\n", - " return loss\n", - "\n", - "def configure_optimizers(self):\n", - " return torch.optim.Adam(self.parameters(), lr=self.hparams[\"lr\"])\n", - "\n", - "LitMLP.training_step = training_step\n", - "LitMLP.configure_optimizers = configure_optimizers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JXquZkggVCKJ" - }, - "source": [ - "## ➕ Optional methods for even better logging\n", - "\n", - "The code above will log our model's performance,\n", - "system metrics, and more to W&B.\n", - "\n", - "If we want to take our logging to the next level,\n", - "we need to make use of PyTorch Lightning's callback system.\n", - "\n", - "> _Note_: thanks to the clean design of PyTorch Lightning,\n", - "the training code below will run with or without any\n", - "of this extra logging code. Nice!" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "TE8qYwtOVCKJ" - }, - "source": [ - "The other callbacks we'll make use of fall into two categories:\n", - "* methods that trigger on each batch for a dataset: `validation_step` and `test_step`\n", - "* methods that trigger at the end of an epoch,\n", - "or a full pass over a given dataset: `{training, validation, test}_epoch_end`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z8JylsGXVCKJ" - }, - "source": [ - "### 💾 `test`ing and saving the model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "AHy-SYLDVCKJ" - }, - "source": [ - "We use the test set to evaluate the performance of the final model,\n", - "so the `test` callbacks will be called at the end of the training pipeline.\n", - "\n", - "For performance on the `test` and `validation` sets,\n", - "we're typically less concerned about how\n", - "we do on intermediate steps and more\n", - "with how we did overall.\n", - "That's why below, we pass in\n", - "`on_step=False` and `on_epoch=True`\n", - "so that we log only `epoch`-wise metrics.\n", - "\n", - "> _Note_: That's actually the default behavior for `.log` when it's called inside of a `validation` or a `test` loop -- but not when it's called inside a `training` loop! Check out the table of default behaviors for `.log` [in the docs](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html#log)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "yeVmedTTVCKJ" - }, - "outputs": [], - "source": [ - "def test_step(self, batch, batch_idx):\n", - " xs, ys = batch\n", - " logits, loss = self.loss(xs, ys)\n", - " preds = torch.argmax(logits, 1)\n", - "\n", - " self.test_acc(preds, ys)\n", - " self.log(\"test/loss_epoch\", loss, on_step=False, on_epoch=True)\n", - " self.log(\"test/acc_epoch\", self.test_acc, on_step=False, on_epoch=True)\n", - "\n", - "LitMLP.test_step = test_step" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qk50ovviVCKJ" - }, - "source": [ - "We'll also take the opportunity to save the model in the\n", - "[portable `ONNX` format](https://onnx.ai/).\n", - "\n", - "\n", - "Later,\n", - "we'll see that this allows us to use the\n", - "[Netron model viewer](https://github.com/lutzroeder/netron) in W&B." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "QNr-9bEpVCKJ" - }, - "outputs": [], - "source": [ - "def on_test_epoch_end(self): # args are defined as part of pl API\n", - " dummy_input = torch.zeros(self.hparams[\"in_dims\"], device=self.device)\n", - " model_filename = \"model_final.onnx\"\n", - " self.to_onnx(model_filename, dummy_input, export_params=True)\n", - " artifact = wandb.Artifact(name=\"model.ckpt\", type=\"model\")\n", - " artifact.add_file(model_filename)\n", - " wandb.log_artifact(artifact)\n", - "\n", - "LitMLP.on_test_epoch_end = on_test_epoch_end" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JosFHArGVCKK" - }, - "source": [ - "### 📊 Logging `Histograms`" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FxViSjO4VCKK" - }, - "source": [ - "For the `validation_data`,\n", - "let's track not only the `acc`uracy and `loss`,\n", - "but also the `logits`:\n", - "the un-normalized class probabilities.\n", - "That way, we can track if our network\n", - "is becoming more or less confident over time.\n", - "\n", - "There's a problem though:\n", - "`.log` wants to average,\n", - "but we'd rather look at a distribution.\n", - "\n", - "So instead, on every `validation_step`,\n", - "we'll `return` the `logits`,\n", - "rather than `log`ging them.\n", - "\n", - "Then, when we reach the `end`\n", - "of the `validation_epoch`,\n", - "the `logits` are available as the\n", - "`validation_step_outputs` -- a list.\n", - "\n", - "So to log we'll take those `logits`,\n", - "concatenate them together,\n", - "and turn them into a histogram with [`wandb.Histogram`](https://docs.wandb.com/library/log#histograms).\n", - "\n", - "Because we're no longer using Lightning's `.log` interface and are instead using `wandb`,\n", - "we need to drop down a level and use\n", - "`self.experiment.logger.log`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VrPs4tAJVCKK" - }, - "outputs": [], - "source": [ - "def on_validation_epoch_start(self):\n", - " self.validation_step_outputs = []\n", - "\n", - "def validation_step(self, batch, batch_idx):\n", - " xs, ys = batch\n", - " logits, loss = self.loss(xs, ys)\n", - " preds = torch.argmax(logits, 1)\n", - " self.valid_acc(preds, ys)\n", - "\n", - " self.log(\"valid/loss_epoch\", loss) # default on val/test is on_epoch only\n", - " self.log('valid/acc_epoch', self.valid_acc)\n", - "\n", - " self.validation_step_outputs.append(logits)\n", - "\n", - " return logits\n", - "\n", - "def on_validation_epoch_end(self):\n", - "\n", - " validation_step_outputs = self.validation_step_outputs\n", - "\n", - " dummy_input = torch.zeros(self.hparams[\"in_dims\"], device=self.device)\n", - " model_filename = f\"model_{str(self.global_step).zfill(5)}.onnx\"\n", - " torch.onnx.export(self, dummy_input, model_filename, opset_version=11)\n", - " artifact = wandb.Artifact(name=\"model.ckpt\", type=\"model\")\n", - " artifact.add_file(model_filename)\n", - " self.logger.experiment.log_artifact(artifact)\n", - "\n", - " flattened_logits = torch.flatten(torch.cat(validation_step_outputs))\n", - " self.logger.experiment.log(\n", - " {\"valid/logits\": wandb.Histogram(flattened_logits.to(\"cpu\")),\n", - " \"global_step\": self.global_step})\n", - "\n", - "LitMLP.on_validation_epoch_start = on_validation_epoch_start\n", - "LitMLP.validation_step = validation_step\n", - "LitMLP.on_validation_epoch_end = on_validation_epoch_end" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "617Vro8jVCKK" - }, - "source": [ - "Note that we're once again saving\n", - "the model in ONNX format.\n", - "That way, we can roll back our model to any given epoch --\n", - "useful in case the evaluation on the test set reveals we've overfit." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yMe2e8FEVCKK" - }, - "source": [ - "### 📲 `Callback`s for extra-fancy logging" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "X-DfYBQrVCKL" - }, - "source": [ - "What we've done so far\n", - "will tell us how well our model\n", - "is using our system resources,\n", - "how well our model is training and generalizing,\n", - "and how confident it is.\n", - "\n", - "But DNNs often fail in pernicious and silent ways.\n", - "Often, the only way to notice these failures\n", - "is to look at how the model is doing\n", - "on specific examples.\n", - "\n", - "So let's additionally log some detailed information on some specific examples:\n", - "the inputs, outputs,\n", - "and `pred`ictions.\n", - "\n", - "We'll do this by writing our own `Callback` --\n", - "one that, after every `validation_epoch` ends,\n", - "logs input images and output predictions\n", - "using W&B's `Image` logger.\n", - "\n", - "> _Note_:\n", - "For more on the W&B media toolkit, read the [docs](https://docs.wandb.com/library/log#media)\n", - "or check out\n", - "[this Colab](http://wandb.me/media-colab)\n", - "to see everything it's capable of." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "VM9hxnNtVCKL" - }, - "outputs": [], - "source": [ - "class ImagePredictionLogger(pl.Callback):\n", - " def __init__(self, val_samples, num_samples=32):\n", - " super().__init__()\n", - " self.val_imgs, self.val_labels = val_samples\n", - " self.val_imgs = self.val_imgs[:num_samples]\n", - " self.val_labels = self.val_labels[:num_samples]\n", - "\n", - " def on_validation_epoch_end(self, trainer, pl_module):\n", - " val_imgs = self.val_imgs.to(device=pl_module.device)\n", - "\n", - " logits = pl_module(val_imgs)\n", - " preds = torch.argmax(logits, 1)\n", - "\n", - " trainer.logger.experiment.log({\n", - " \"examples\": [wandb.Image(x, caption=f\"Pred:{pred}, Label:{y}\")\n", - " for x, pred, y in zip(val_imgs, preds, self.val_labels)],\n", - " \"global_step\": trainer.global_step\n", - " })" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6z2bcdwZVCKL" - }, - "source": [ - "# 🛒 Loading data\n", - "\n", - "Data pipelines can be created with:\n", - "* 🍦 Vanilla Pytorch `DataLoaders`\n", - "* ⚡ Pytorch Lightning `DataModules`\n", - "\n", - "`DataModules` are more structured definition, which allows for additional optimizations such as automated distribution of workload between CPU & GPU.\n", - "Using `DataModules` is recommended whenever possible!\n", - "\n", - "A `DataModule` is also defined by an interface:\n", - "* `prepare_data` (optional) which is called only once and on 1 GPU -- typically something like the data download step we have below\n", - "* `setup`, which is called on each GPU separately and accepts `stage` to define if we are at `fit` or `test` step\n", - "* `train_dataloader`, `val_dataloader` and `test_dataloader` to load each dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "pO5M1XEnVCKL" - }, - "outputs": [], - "source": [ - "class MNISTDataModule(pl.LightningDataModule):\n", - "\n", - " def __init__(self, data_dir='./', batch_size=128):\n", - " super().__init__()\n", - " self.data_dir = data_dir\n", - " self.batch_size = batch_size\n", - " self.transform = transforms.Compose([\n", - " transforms.ToTensor(),\n", - " transforms.Normalize((0.1307,), (0.3081,))])\n", - "\n", - " def prepare_data(self):\n", - " # download data, train then test\n", - " MNIST(self.data_dir, train=True, download=True)\n", - " MNIST(self.data_dir, train=False, download=True)\n", - "\n", - " def setup(self, stage=None):\n", - "\n", - " # we set up only relevant datasets when stage is specified\n", - " if stage == 'fit' or stage is None:\n", - " mnist = MNIST(self.data_dir, train=True, transform=self.transform)\n", - " self.mnist_train, self.mnist_val = random_split(mnist, [55000, 5000])\n", - " if stage == 'test' or stage is None:\n", - " self.mnist_test = MNIST(self.data_dir, train=False, transform=self.transform)\n", - "\n", - " # we define a separate DataLoader for each of train/val/test\n", - " def train_dataloader(self):\n", - " mnist_train = DataLoader(self.mnist_train, batch_size=self.batch_size)\n", - " return mnist_train\n", - "\n", - " def val_dataloader(self):\n", - " mnist_val = DataLoader(self.mnist_val, batch_size=10 * self.batch_size)\n", - " return mnist_val\n", - "\n", - " def test_dataloader(self):\n", - " mnist_test = DataLoader(self.mnist_test, batch_size=10 * self.batch_size)\n", - " return mnist_test\n", - "\n", - "# setup data\n", - "mnist = MNISTDataModule()\n", - "mnist.prepare_data()\n", - "mnist.setup()\n", - "\n", - "# grab samples to log predictions on\n", - "samples = next(iter(mnist.val_dataloader()))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6Ord5Pu3VCKM" - }, - "source": [ - "# 👟 Making a `Trainer`\n", - "\n", - "The `DataLoader` and the `LightningModule`\n", - "are brought together by a `Trainer`,\n", - "which orchestrates data loading,\n", - "gradient calculation,\n", - "optimizer logic,\n", - "and logging.\n", - "\n", - "Luckily, we don't need to sub-class the `Trainer`,\n", - "we just need to configure it with keyword arguments." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "MDDjVOSEVCKM" - }, - "source": [ - "And that is where we'll use the `pytorch_lightning.loggers.WandbLogger` to connect our logging to W&B." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3nibeocmVCKM" - }, - "outputs": [], - "source": [ - "wandb_logger = WandbLogger(project=\"lit-wandb\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "YDEryNsoVCKM" - }, - "source": [ - "> _Note_: Check out [the documentation](https://docs.wandb.com/library/integrations/lightning) for customization options. I like `group`s and `tag`s!.\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NLfJc41aVCKM" - }, - "source": [ - "We can then set up our `Trainer` and customize several options, such as gradient accumulation, half precision training and distributed computing.\n", - "\n", - "We'll stick to the basics for this example,\n", - "but half-precision training and easy scaling to distributed settings are two of the major reasons why folks like PyTorch Lightning!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "0dciVNKHVCKN" - }, - "outputs": [], - "source": [ - "trainer = pl.Trainer(\n", - " logger=wandb_logger, # W&B integration\n", - " log_every_n_steps=50, # set the logging frequency\n", - " max_epochs=5, # number of epochs\n", - " deterministic=True, # keep it deterministic\n", - " callbacks=[ImagePredictionLogger(samples)] # see Callbacks section\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Fs6hyyU2VCKN" - }, - "source": [ - "# 🏃‍♀️ Running our Model" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LzVnPc6_VCKN" - }, - "source": [ - "Now, let's make it all happen:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "q7VxL_o6VCKN" - }, - "outputs": [], - "source": [ - "# setup model\n", - "model = LitMLP(in_dims=(1, 28, 28))\n", - "\n", - "# fit the model\n", - "trainer.fit(model, mnist)\n", - "\n", - "# evaluate the model on a test set\n", - "trainer.test(datamodule=mnist,\n", - " ckpt_path=None) # uses last-saved model\n", - "\n", - "wandb.finish()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cryJvmKOVCKN" - }, - "source": [ - "> _Note_: In notebooks, we need to call `wandb.finish()` to indicate when we've finished our run. This isn't necessary in scripts." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "t9jsRs-oVCKN" - }, - "source": [ - "## Viewing the results on wandb.ai\n", - "\n", - "Among the outputs from W&B,\n", - "you will have noticed a few URLs.\n", - "One of these is the\n", - "[run page](https://docs.wandb.ai/ref/app/pages/run-page),\n", - "which has a dashboard with all of the information logged in this run, complete with smart default charts\n", - "and more.\n", - "The run page is printed both at the start and end of training, and ends with `lit-wandb/runs/{run_id}`.\n", - "\n", - ">_Note_: When visiting your run page, it is recommended to use `global_step` as x-axis to correctly superimpose metrics logged in different stages.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fgPkQGamVCKN" - }, - "source": [ - "\n", - "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA+4AAAIrCAYAAABxtsMNAAAgAElEQVR4Aey9B7QVxbou6hvvnnPvePee8c4994R3zt77bPM2IxkWaZFBkiRFlCQIGBARRBQRMW1z1i2KAbaoYAAR4xFzQLe61U1WcgbJOf5vfL3456pVq3vO7jl7dvec8+sxenZ3hb+qvvq6qr6u6p7H/cdvfy/ciQE5QA6QA+QAOUAOkAPkADlADpAD5AA5kEwOHMeKSWbFsF5YL+QAOUAOkAPkADlADpAD5AA5QA6QA+AAhTtXHHDFBTlADpAD5AA5QA6QA+QAOUAOkAPkQII5QOGe4Mrh0zU+XSMHyAFygBwgB8gBcoAcIAfIAXKAHKBwp3DnkzVygBwgB8gBcoAcIAfIAXKAHCAHyIEEc4DCPcGVwydrfLJGDpAD5AA5QA6QA+QAOUAOkAPkADlA4U7hzidr5AA5QA6QA+QAOUAOkAPkADlADpADPjjw+5NOl9bnD5RLhj8sfW+eIUPvnytXPvaTXPX4fLnsvm+k97i3pMOQR6R+u4HynyedHhqmFO4+KodPuPiEixwgB8gBcoAcIAfIAXKAHCAHyIHS5cAZtZpJtyselBueWiCjn14o1z61SK59arFcO3GJXPvUz3LNxCUyfOJSGT5xubNf/fQKGfbUUmnZe6wcf8qZOQt4CncK95xJxAasdBsw1j3rnhwgB8gBcoAcIAfIAXKg2DnQZcCtMnLiArnu6UUy+pnFMmrSYke0j3xqqdw4ZZmMfGaZDH96uYx+dpWMfH6NjHpmtVz7zEoZdO+H0rBzP6ndrJmcfPbZOekuCncK95wIVOw3KcvHjogcIAfIAXKAHCAHyAFygBwoTQ7UrF8utzz2rtzy/AK5/plFMurpRTLm2cUy4cXljoi/7JY3pGGzblJe3k269bxCLhwwTkbc95Fc88T30vWq+6RB5y5Su7ypnN24gTTpVi6NOjWSE/5wSlb6Kyvh3rbpSbLmrZrOjvNciRy2vVzzw/ileWOy3lnv5AA5QA6QA+QAOUAOkAPkADkADjRq3klunvSt3PXSL3L71J/l9qlL5KnZq+WRmatk2F3vS5Pm3aWsbrm0LGshvbr0lKsGXi6X9Ogr1155g1xx9Q1Sv3lLadCuhdRt0VQatm8sLS9qKc0vaC4tezWXE08/NbCGzkq433ft6SJf13Z2nOdK7rDt5ZofxufNSg6QA+QAOUAOkAPkADlADpAD5EBpcuDc+s1l9JPfyejnl8qYycvkxj8vk4dnrJKZn66ToWMnS3n5+dK+WVtp3bhcLu7cQy7q0FW6n9dZTj3hVPnP3x4vDeqVSceOXaR153bStFMLadmrpTS7sFzKzm8kTS9oJg06lMnvTzo5kI7OSriHPUMetj3eYKV5g7HeWe+ZOHDW2edK74svcXacZwpPf3KKHCAHyAFygBwgB8iB0uPA1fe8J3e8vFLGvbBSxr6wSm56abXc89oqGTH+eSmr10x6tO8og3v3kf/813+Rf/s//yT/8o//r/yv//Hf5V/+8R/lf/73v5e//7v/Jv/wv/5B/s+//G9p1aOVM9PepEdTaXZBuZRf2FKadW8utcvrBxqLZiXc/ZKXgrz0SO6XGwxHbkTJgd/87ni5/Ior5aWXXpLp06c7O87hBr8o88K0yH1ygBwgB8gBcoAcIAeSy4Geg2+T+2ask0ff3CC3T1sn46etk1umr5ORD8yRdi3by7BLh8iogQNlePeu8uR1w+XGXj3khgt6yISLe8udF18iEy68UPo2bigX1q0rp/zrP8u5Dc+R5j2aS9PuTaVBpzJp0qmlNO/cRpp3ai1n1fU/kZRX4V5IS+Br1KwtY8feJM8995xMmzbN2SdPniwTJtwq9RqUcWDPj/iRAwXMgY6dusjUqVPloYcellZt2jk7zuEGP3aeye08WTesG3KAHCAHyAFygByIigN/OONcGT7hz3LvGxvk7hkb5fbXNsqtr66XCdPWSp9eA6VjeSvp27mz9CyrIw/06SnfPvaw/DDlOZl9910yccQ18qfLr5QH+w6QR/r2lTt7dJFr27aQs0/5nbTo2VzqtWss9Vo3lvKOraRV57bSqlsbad6tmZxwir+P1WUl3M8+8wSZcMVpzo5zLyD9zrj7teeVTi7umG3r13+ATJkyxRHpTZo2d2bg4I5zCHf4IUxUM3OnnXG2PP74E3Ld6Os9sc22zLDZt1//wHYhbp5//nmKnN/+3nmQc/fd9wjqKdt6YLxoO6Bbb7tdJj3zjNSp1yBVZziHG/xYH9HVBx6E4gFpNu1QnPXkpw1E+4q220/bUKg4xFkHTDu6+5RYE2tygBwoVQ607thbLht+m9z21Kfyxxkb5M6Zm+WO1zfK+EnfyIWde0jNU0+Vy9u3kGeHDZJZN42WqVcNlWeGXCqTrhgqL14/WiZeeaVMHj5CHu7bRyZ0ai/Xt20h55fVlIYdm0idVmXS6Lzm0qRjC2nbs4106t1eWvRoInXKa/kaiwYW7hDZq2fXTH2cDufpxHumSg/bXqb0bP9LB14mkyZNcmbgbD+9xgwdwiCsuuXzmC/hrgNFiM6g+fcatGKg6jZriUG5LknGMR8PITKVQXE08xFGXrzKnCk/bv6wFQc2bnlRN607W1gpnqYw8QqLMpk8w7l5jbRwbdaNpme7m2GyxQo2H330MfnDaWemuI9zuNn5UhzydVTMzHJByJbKyh5th7S+84Vz2Ha92kAzHfDTvD9MP/s8XzggD25tsp0+rzkoJwfIAXKAHCAHqnLg9yecIv2uuEn6Xj5Oho2+T0be/YbcMW2pPPjmRhl/3wypffpZctpvfyM3X9hNZtw0Ut67dazMvH6EzB5/gyx88QX5btLT8sn998m7t98pH/zxTnlyYH+5tVsHuaRxHSlr21jKzmsmzTo1lZY9mkuHS1pJh94tpFHn2tJjaFP53fGZX90MLNwx065flNcj3LKt+LDtBclHeYtW8vTTT0vnLl0z5h9hEBZxgqSRTVgVSNmKlGzSzBTHHrRqHiE+7EEiBuSmELHjZkpL/du2O0+GXzPC1+yVxjGPmkcTR+QF+U2KaIBoNPNn5j+b8xNP/oNcNexqh9PZrhBRYWkLWsXOFCYa1nRDvlEmMz7O0127ccSt/rLBBHHs9NUOVtRg1+tMx7DwNe8PpAm8bLdMecmnf673Xrq85UuwpkszDD83jtp2UY/2vWCH0euwcdD7xa1N1jR5rDpAIx7EgxwgB8gBcsDkQFnTNjLo6vEycNit0vfyW6TXwLEydMzjctMj78mY21+Q0084WWqdfIJc2a6FPDq4r7w6+ir58bEHZO4jD8rUUSPkgT4XyyP9+sr9l1wkN7RvK5e1aCbD2raWS1s2kbLzGkmzzk2krGUtaduzsXTs00I6XFIu3YY2lp5XNpGz6mbW0yUt3PFOO3azwtKdBw2fzlY6Px2AhSno0qXnx89r0OrlbtrU8gQVy7/7/YnS++I+WYt3TdfG0UvEmXmO6hx5sfOXa9pYAj7qutFZi3fUE8QHdogLzQ/y+eCDDzk7sIU7wj711FPObtYvwnoJda0XM7ymYR41XBj4hFnnYeBri3TgDBxxP5kYxHWe672XLt9hC9Z0aYXp56etA1fjEu5aVj/51LA8csBKDpAD5AA5QA5UcuD8ngNk6IhbZdCwW6T/lePlsmtuk0uvvFmuvPaPctmQG+X43/5G/vC730qn2mfLLT27yNyHb5dPJj4q13TrKk3POE06nHW6XNOysdzYvpVcUPN0OfPf/1n+8K//W2qfeLzUaV5PmnUsk7KW50rzjvXk/P6t5JJr2shFIxtKz2GNpUnHmhnHgIGFe5Cl7X7ecQ9iL0xiYZnsAw886Gu2XdPFrPvDDz8sZ5xVIyOwGiebowoWvHuLAb4uqbWFDq7VD0fTH+cYQA4ecrkTRkUYjqYQgqBRG+aAU/OgfhBsbu+4+xkkIj1bqPjFJRcBoWUwy4t0TREHP1zjiLLiHGEyYWuXxwxvr0DQNBVLsy7UDUfNp1ed+MUM4XIRl8od1DnOYQ9Y4vrq4ddUE+4oD3hmYqK4ap5NzOGGa5NvGs48etWfGUbP8dduDz/ySIrLJq72Ob4mP+DSQTndw7nia2KFMpj3kZuIB56Kl+ICzOGm5dO6gj0Vx+qn3FK8/BxzufdgX/OpedB7S/MGLgEH+Nv3DPCAmx1Xy4aHHCgv4iNczwt6OVi42UTZ3ezAFmyon50HO/9ebaCJpVlPbhiY9a44aL3ptV3Pmj/Fz0zP7dzkkps/3SoHaMSCWJAD5AA5QA6YHLjq6hvkhpvulxGj/ihDrr5Vrr7+Hhk0/DYZfsO9cvnVY+X3//H/yQn/9m/StWFteWhwX5k0fIh0OfsMuaThudL4hN9J4xP+XS6pd7ZMGjFcTv/nf5S/+7//Xs74j3+R/k1qSJO2daXdBeXSsEUNKe9QR7oOaioXXVsmPYc1kv5jWkmPoY0zjk0DC3cUDmLbz8fp/H5V3q89E9hczzFIwgAJR7+2sonj17YZTgeM5iDPHoxhsGcONHWgq4NAHZCagz21q4N4hDHTwDV2DacDSOQNccz0NL92vtQdR6SNQaeZB9Pf73m2AkLLoeVFejZO8EMezTDAwCyrHcfGzcYA1xAWyi2U38YSNhQjO223OvGLlRkuW3GJvCG/EEMQK8ARZcK5ikW4IS0Ni2uUU8uCo1nvODevtW6AvVleM/8aRm2afvY58oePSCLspQMHpd2feOKJKnmxbfm9zgVfu8zARjkC3tiz7yiX+isuJkfhb9q08R5707gUH/2WD+FyvffMOseDSNQTyoe82vk1y4d4ev8gDsqq94xbfMXEtmne27BjPnx0u9b7Vu1pnoAFMDYxd8PRrCeNo/lWm4qJlgP+eo74ateuQ1yb/hrOPtrlsv15zUEqOUAOkAPkADngzoFXZ86QL778Wn74bp7Mnv2+vPjiTLnv/qfl2jF3yCUXDZKzTjxB/ukf/qf0KKsrr46/XvrUqyHn/Ps/Sa0Tfi/n/uY3Uv/438rw5uWy6u3ZMrisgTQ+/j+ka83TpXe9GtK3T1tp262R1Kj7B6lddoZ07NtQ+l3fRIbe2louHlkuHfpk1qRZCXe/le1nxt2vrbDDYaCEQRmOfm1nE8evbTOcDvDMQZrtZg/qEN8cNGIwaA5i4W/bQBi3gajbwM/NDTa93M3yeKVjhsl0DgHRf8ClcuVVw+SEk/z9ZYKWF4N3c0d+ND0TM3ULiq3bgBpuSCcTPnbcMLDScuDYoGFjhxetWrdNldn0dztHHvTegFhHGYATdvUDtohrXiOcCh+ERdnUPs7Na3U368j2Vz/Y0vBex0w4m/G88mKG8XueLb4mH3Fulh3tjB/hbuJiYo+8w54pOv2Wxy1cNvceeGG3P2ob5YMfwqibySN106PNA7f4dhjEtW3a8YCRiSHiwA3x3Pjk5qZ51CPspcPd9Nf86MMwkwN2fcI+3PRBmqbndvSTT7d4dHMfxBEX4kIOkAPkQOlwYMTNV8szU/4sb856X+bO/VZ++OFHmT37bbnssstk6pD+8kDvHlL7+N/JwA5tZPKIK2VQ/VoytFEDGdSwnlxWVl9GNm8ij13SW1ZMniwzR10rY89rIxO6dZLx558nl/RuIc3a15J6Tc6U+i3OlnYX1pX2fevKRaMaS89hZdKiWx6WyhcLebNdKo/l9YibTxzcBqFIDwM77F7+GHDqYNk817y6xdPBoyke3OJ6DQa93DVNPWre9TroER8FxKA3yMcB3cprp2sOpOHnFcfExDzX8LYQwzXCmWHttHENXJAH08+tTkx/v+cnnnyqM0MOYYBzv/GQZxUfyNtN4252BAPq2vSDPfta6xnxcK5pqrte20fYNmdV4a/Y2vjYcXHtl4cImykvbvbd3HLBV+9TrWuzjHDLVbgjv6gb5SXO3crgxy2bew/lUQ7ZaWiZzTzZPEIc2ND846gYucV344pt04yn4U37eo542LWONP8mx+z4mje3ciu31b7iovmBOz58qu5ID+lrePOoYUxs1M0tn+rGY+kMOlnXrGtygBwgB7LnQNNO9aX74IEycORYue3eR+Xu+x6WkdeNlnbntZMp3dvKrW3LZESrRnLbpRfLq2OHywMXdZdHLxsgEy+7TB7vc4nc1qmDPD2on3x1110y9fKhMv68VvJAr+7ycL8LZHD/FtKqcx1nqXzDVudI087nSr22Z0rHQfWkU/860qjDWRnHalnNuIc9kx62Pb+EdT42d9M43//PHufH6XSgqANEN/FhDhrdBp62DRMn088coGoYNzf4eblrPD265Vf9Mh2zEQ6waZbJKw0TMw3jllcznI0twmu9qA09ZsInXVw/+dd07GO2ohJ2UD4VA8g/BDXyafu5XUOMQHRC7GschMO5eQ03c3crq5ubGcc8z4SzGTZTXsywXue54muKQptPiiHKpOmb/HPDBWF1tYPG0WMQbDSOHrO99+wyqT0cVbAijLqbnFN/vafs8qq/Gd8OA7umTbd0wQNNQ/OhRzfM3Nw0vB7NeoIb0tB7Cdemv5ZD7wszP+nqU9PyOvrJp1dculdtl4gH8SAHyAFyoLQ4UL91fbnw8j4yeOwYueDyK6TDBRdKu07nSeduHeWxoX3lznaN5aF+F8gtF3WWGeOullfGDJfnhw2RP/XpJTe3LpfR5Y3l6X695PVrrpBH+3SXG1o2lrEtG8oDF3WSwf3LpUm7c6Rm2elyToNTpU6L06Ws3dlS3rWWlHc5S2o2ybyiOCvhHva7637thX3zYFCa5L+Dswd89mDfXOaOwZo5Y4lBqxke2NmDW4QxB78YOGLXcDqgRFycm+lpXbgNEs3BKcLZedO4fo7ZCgfY1nJ4Dc4Rxs4r3ICJWVY7/za2dnjYaNGytSNKNA9mXXbo2Dn1ioZijjiatludqL+fYy6iUvOg+dX8a55wVD87rOYNmGKW0OaPXsMG/NWm2jExh5umna7+NE03HqqffbQxt/0zXYeBr3lvajkVH73Wciv/FHfbH/lFGFO4m++0q0A08c5URvjncu9pmlom2MM3E5B39TPzg3Mtn10W+IEviodbfDdMTJtI344Hf5tz9n1r5h/ndngbR+RRy6F5Qjpm+upv58fksMY101fs7DTta9OO7cfr0hqAsr5Z3+QAOUAOBONAWcu6csGA7nL56Guk9+A+0uvSLtJrQBfp0r2jXHPVEHni0t4ydfhgubVnB3lmWB+ZOmKg3HpeM7mpZQMZ3qiuDGvRTLqefopc07imDKhzjgxuWl+ualxT7uhaLn0uaiRl7c6R+uVnydn1TpFzy06ScxudKjUaniy1Gp0s9ZqfkZrQ8Kq3rIS7nxnyIF+L92PPqwC5ul868DKZNGmStGrTzhMs+D377HPOX2zlmp6f+Dpow6wlBooYtLoNGHVAC383IWSKA6SrdnUAbH6BGfF1QImwOqhU2whrLt+100Y4TQ+DX6ShcXHUwauf8muYxk3KHTsQEOoW5GiX1y2uOdA2/e3ymfnHuZZV42CwrHVlYgF/zYfigQ9kAU/4mfEwSE9XJ5pWuuN/Hn+SXDZ4iLNEHgIzXVgvP5TP5IIZzvazrxFWy2uKDpyb12a5gYsbv9WO8tXMh30eRKzYebFtpbsOC18v/mhZgavyBXWh70EDEzdcUH4V7rh3Idw1Po5qN13ZTL9c7z3YstsQ8B7iXd1RRk3T5hHqSPOPc+Rfy+AW3w0T26ZbPJuHZr1oeM2H3QZq3s2j2tO8mvUI27jW99TVPtzUBuKZ94KJQ6Z6NNPSPJvl0TR4DDaII17EixwgB8iB0uFAvabnSPf+rWXgsB4yeERnGT6us1w+uoMMvLKrXHFlfxnVq5v8qU8XueOCDvLM8P4yrn0jGdW0plzZ8FwZ2rS+jGnbXMa0biYjm9SXUU0byLWtmsg1zerJhE6NpccFdaRJ+xpyZp0T5fTaJ8jZdU+RU878nZxR90Tpf8l5clW/C1PjAS/OZSXcvYyZ7vjqvHxdu8oONzNMEs5/87vjpU/f/jJ58mSZMOFWadK0ubN0Hu44hxu+Vo1l8s88+6z0uujixJUhCTjmIw+nnnaG1KxdN3F4Y4Cc1AExeFu7boNA77Tno+6itgnBFMVX5UsF36Tee1HziumVzmCNdc26JgfIAXKAHDjtnFOlS5/G0uPSpjJ0dAcZMrq1XHFDaxl0dRvpN7iLtDm/THr3biz9OjSS+/qcL2PbQqDXkhtbN5by438j/RvWkFsuaivXd2oidw3qKp1rnCHXt2wkd/RsKX0GNZcWnWpJrcanSe3mp8tZ9U6SE07/jdRq9Qd55NarpXeXjhk1T8kLd71Jzzy7hiPOIcimTZvm7Di/7rrRUqNmbQdI/I87xTtvagh3rxlp5ROP0fIk6v9xZ/1GW7/Em3iTA+QAOUAOkAPkQL458LvfHy9d+zaSzhfXl15DmsnFVzSTfsObycVDm8qlw1vJkDFtZMj4VnLF4BZyW/c2MrZ1Hbm67Gy5tUNLGd+utYwqbyJXNKovl9WrKVfWqyM3tWkut5/fUm6+pLX0GthKWveoLc3OP1cad6ohdVufKTXLT5HmnWrKzEduk5NPyrxSNivh7ud/14Mslc93JYRpn+K9dBsNXQabaclqmHyjrdLlG+uedU8OkAPkADlADpAD5EC0HKjR4DQ57+K6ct7FdeT8S+tJ5371pNeQMrlsdAu56ubz5IpbOsiIEe1lbKcmMq51PbmxZV25sW0jmdC+ldx1fnu56/y2Mv68pnJ3t3Zy9wWd5NYebeXqAa3k/P6Npf3FdaTLgPrSqkcNqdvqdKnb5jTp07+V3HbVoIyz7eBBYOEeRJD7EfjIRJzvuGdzM3TqfL489tjjUrNW8pZxZ1Mexom2QSDexJscIAfIAXKAHCAHyAFygBxIHgd+d+Lx0qxrDUdkd+xbX9peVFs6D6wlPS+vK4NHtZPRt/SWm0b3kps7NZNxLevK2Db1ZGzbhnJ7t/ZyZ49Ocl/v7nLPhcf2i7rKmJ7tpUOvetJ1cJmc16e29B/WTPpe2UzKu5wr9dqcJreN6iNlderkR7jn4931uL4qz5sleTcL64R1Qg6QA+QAOUAOkAPkADlADpADcXGgVpMzpX2fetK2dx0p736OtO1zlrTsdYZ0699Iho26SMZf11fu79NFbmrXQG5sXlPGtqojY9s3ljsvOF/u73OB3HdhV7mza0e5tVs7qV/rBGnc5SzpOLC+dB/UQEbedJ4MG91O2veoKWWtzpRBF3TyJdqBReAZ93wI90KbcY+LREyXDRg5QA6QA+QAOUAOkAPkADlADpAD+eVA3VanSese50qzbmdJm4trSHmPM6V193OkV58WMunmq+Wxft3lri7lclPz2jK22dkyuvFZMrLxOTKqvI7c3K6RjGtfJv2anS3/evw/Sq1Wp0irS2pIzyGN5OrrO0nfoeXSvNPZUta8hm/RjvoOLNyDLJWnIM8voXjDEl9ygBwgB8gBcoAcIAfIAXKAHCAHwuXA8SefIGVtzpRWvWpKx0trStu+NaVVz3Pk7uv7yvznHpXnB18o93VtKRPaNpQbm50j45qeIzeX15Db29aWuzrXl7u6N5Le550jf6j9WynrdIa0719PLh5aLv2vbCkde9eWei1Pld+fdEJ+hTtI4ffddS6BD5dAvCGJJzlADpAD5AA5QA6QA+QAOUAOkAP558CJfzhJWvWoLR361ZF2F50rI4d3lLXTJ8rqKY/JG6MGyQPnt5Q/dmgi41rWkTFNa8iYJufIuFa15Y5OZTKhc5l0b3eWnNPoBCnvUkN6DiyXSy5rK136NJAmnc+U408+PpBoR30HnnEPQhLOuOefUEHqg2FZH+QAOUAOkAPkADlADpAD5AA5QA7448CpfzhVhvTpJPeP6Su7Zk+WjS8+Lgsfv01eGdZHHurWSu7u0FRuadtAxpTXkBvLz5XxrWvLHR0byq3nN5ELu9aR8q7nSOvuteSCAc3lwkGtpbxLLTnhlBMDi3bUV16FOwnhjxDEiTiRA+QAOUAOkAPkADlADpAD5AA5kEwOPDJqmKx78SFZ/OSd8t19N8iz/c+Xe85rIvd0KJO7O5TJLW3ryPg2teWP7evJoxc0lzt7NJU7bu4rlw5r6/w3fJdeTaR+41pZCXblBIX7b5NJDq0gHlk/5AA5QA6QA+QAOUAOkAPkADlADsTLgcY1a8kDV10qH942Qp7u01nu6dBY7unQyNnvbFdfbu9QX+7t3FDu79ZUJnRpIC89dZOMv7WPdOrWWE45/ZScRDvqnsKdwj1nErERibcRIf7EnxwgB8gBcoAcIAfIAXKAHIiGAyefeLK0r1dLLmlwrlzXsr5MaNdQ7mzfQO48r77c0amhjGnbQHrVO1vaNKkn/3lCsA/QpatDCncKdwp3coAcIAfIAXKAHCAHyAFygBwgB8iBBHOAwj3BlZPuiQv9onmiRpyJMzlADpAD5AA5QA6QA+QAOUAOxM0BCncKdz5ZIwfIAXKAHCAHyAFygBwgB8gBcoAcSDAHKNwTXDlxP9Vh+nyySA6QA+QAOUAOkAPkADlADpAD5ED8HKBwp3DnkzVygBwgB8gBcoAcIAfIAXKAHCAHyIEEc4DCPcGVwydb8T/ZYh2wDsgBcoAcIAfIAXKAHCAHyAFyIG4OULhTuPPJGjlADpAD5AA5QA6QA+QAOUAOkAPkQII5QOGe4MqJ+6kO0+eTRXKAHCAHyAFygBwgB8gBcoAcIAfi5wCFO4U7n6yRA+QAOUAOkAPkADlADpAD5AA5QA4kmAMU7gmuHD7Ziv/JFuuAdUAOkAPkADlADpAD5AA5QA6QA3Fz4Dgpku3nn3+WAwcOcCcG5AA5QA6QA+QAOUAOkAPkADlADpADRcUBCncSuqgIzYc3fHhFDpAD5AA5QA6QA+QAOUAOkAPFxgEKdwp3CndygBwgB8gBcoAcIAfIAXKAHCAHyIEEc4DCPcGVU3r9XqcAACAASURBVGxPiVgePvkkB8gBcoAcIAfIAXKAHCAHyAFyIDgHKNwp3PlkjRwgB8gBcoAcIAfIAXKAHCAHyAFyIMEcoHBPcOXwSVTwJ1HEjJiRA+QAOUAOkAPkADlADpAD5ECxcYDCncKdT9bIAXKAHCAHyAFygBwgB8gBcoAcIAcSzAEK9wRXTrE9JWJ5+OSTHCAHyAFygBwgB8gBcoAcIAfIgeAcoHCncOeTNXKAHCAHyAFygBwgB8gBcoAcIAfIgQRzgMI9wZXDJ1HBn0QRM2JGDpAD5AA5QA6QA+QAOUAOkAPFxgEKdwp3PlkjB8gBcoAcIAfIAXKAHCAHyAFygBxIMAco3BNcOcX2lIjl4ZNPcoAcIAfIAXKAHCAHyAFygBwgB4JzgMKdwp1P1sgBcoAcIAfIAXKAHCAHyAFygBwgBxLMAQr3BFcOn0QFfxJFzIgZOUAOkAPkADlADpAD5AA5QA4UGwco3Cnc+WSNHCAHyAFygBwgB8gBcoAcIAfIAXIgwRwIXbjv379fFi9eLO+9955MmzZNnn32WWfHOdzghzBhbz///DOJlmCiFdsTL5aHT3HJAXKAHCAHyAFygBwgB8gBciAqDoQm3A8fPizff/+9TJkyRT744AOBkN6+fbscPHjQ2XEON/ghDMIiTlhbUoT7vn37ZOfOnbJt2zbuxIAcIAfIAXKAHCAHipYDGO9g3BPVoJXpUCBFyYGvVm6Sps98In83YaYcN34Gd2LgcAGcADei5KKmFYpw3717t7z++uvy/vvvy44dOzJqcYRBWMRB3DC2JAh3dF4Q7FhRcPTo0TCKRRtEgAgQASJABIgAEUgcAhjnYLyDcQ/FOwW1CotiOUKYUbDzYYXXAxtwIw7xnrNwh/CeOnWq/Pjjj4E7FcRB3DDEexKEO5485+M1gMDAMgIRIAJEgAgQASJABCJAAOMejH+KRbCxHHwIAQ5gVtVLtNGdgh4cAEeibi9yEu5Y6o5Z82xEu/YliAsbuS6bT4Jwx1NnzrRrzfJIBIgAESACRIAIFDsCGPdg/BP1AJbpUWDnkwOcbac4z/SABhzJJwfdbOck3P/61786S97TdUp79+6VV155RZYuXeoZDMvm8c57LltShHsuZWBcIkAEiAARIAJEgAgUGgJBhPvWfQdl3s4j8sU2kc/yuMM+0kF6bgNgulH4p+NAJtFGfwp7cCAdh/Lhl7Vwx9IofGQu0zvt+CjdM888I7/88otnPwQbsJXLMnMKd0946UEEiAARIAJEgAgQgbwh4Fe4Q0TnW7DbDwOQHsU7RXpQEUVhTmHuhwNBeZVr+KyFO/7WDV+I97P5WT4OW7CZ7Ubhni1yjEcEiAARIAJEgAgQgewR8CvcMQNuC+sorpFurgNmxi8t8e9HtDEMxX3U7ULWwh3L2yGW021bt24VLKf/5ptv0gVz/GAL//Oe7Ubhni1yjEcEiAARIAJEgAgQgewR8Cvco55t14cCSDfqATbTK2yhT1FOUe6HA1Hf51kL92nTpjn/0+7VzOPd9q+//lpefvll+fLLL72CpdyxpB42s90o3LNFjvGIABEgAkSACBABIpA9An6FuwrpOI5RD7CZHoW7H+HHMIX9gCDq+zxr4f7ss8/KoUOHMrbyr776asaZeRiBLdjMdqNwzxY5xiMCRIAIEAEiQASIQPYIULgXtkiNWnxoepjkmz9/vnz00UfOEdfqF/eRgrqwBXVU9Rc1T3MS7gcPHkzbykOMP/3002ln5tUAbFG4Kxo8EgEiQASIABEgAkSgMBCgcE++cP/2229l7ty5sm/fPkcc//DDD/LJJ9H/D7UpdJCfGTNmON/MwvGrr76icB9PwRyV6A4jHZPPUZxnLdwzLZVHV7NhwwZ5/vnnff23OZfKF0bnzFwSASJABIgAESACRMBEgMI9+cId4+xZs2Y5r6/i+1NYEYtxehRiA2nggQG+fbVnzx4nTVy/8847smjRIud6yZIl8u6776YeLCAcwuuDhqjyqemEIepoo/gfQihfojpmLdzxITksT0+3zZs3T9566610QVJ+sMWP06Xg4AkRIAJEgAgQASJABAoCgVIU7hCUn376qfN3xpMnTxZ7x98cwz8u4ekmJFBPmNnG5Nv69esjE+342+fZs2fLiy++6KSNv4iGGx4krFq1ysnH2rVrnTBwhz/yiPAIg4cObuXJpxtFd/GL7jDqOJ8cdLOdtXD383dweGfliy++kE2bNjlfl0/X+/Dv4CrQOXz4sKxYuVLeevsdeW3GTGfHOdzgx40IEAEiQASIABEgAklCIB/C/d0V22TE+Dvlon4D5YXPfkj9jdw7y7bIVWPGy4V9BlRxz/TBO7dBcC5uW7ZskZkzZzpjXDc7GPtCfKYL4xYvn26YaX/llVec2fbPP/9conqn/OOPP5bPPvvMSW/FihUOLq+99ppgV1EOwQ6s4AbcEA4PPaAjoCfyiYub7TBEHW0Uv/h3404+3bIW7vv373eeMuJG89qw5AXvuOOGO3LkiFcw56kbnkzCZrZbMXycbt++/fLpZ5/L/AUL5aDx4T98K2DR4sXy2edfCMJwIwJEgAgQASJABIhAUhAIW7h/vPmg3PzIU/LIq+/IVTfeItff9aB8vOmgfLB+r9x476Py4MuzpN+QK+XJtz5OCfqohbufGXe8Qw6x/Pbbb8vu3bsjF5+mgFiwYEFqebwum8c75maYfJ1jcg44qH2dVd+1a1fKDX641tl4DYt4iK/XUR0puotfdIdRx1HxUdPJWrijs8DNhP9z99ogOHETZtqwRP7777/PFCytf6ELd8ymf/nVXFmzdq1nOeH36edf+Pqav6cRx2OBzJk+Xaan9lkyd0P6GHn13TBXZk2POQ9ZFHDBnOkyfc6CLGIyChEgAkSACBCB4kEgbOGO2fbp3y1xRDnEef/Lh8lbv2ySe6a8IpM/+c63WDfFvA58oz5iZv6NN94QHKNO20xv8+bNVfIA8RzVcnnMnmOmX5fFm/lKd47l83gXH/HThcuHXxiizr+Nv8mcfVXbgwXfUzj7xy8+rPLBvXQ2cxLuEJuvv/66/Pjjj1XZFuAKcWEj12XghS7cV61eLfPnL0j7Ib+jR4/KDz/+JAib/VYh2md9vbHSxMI5Mn36HIlNglK4V9YFz4gAESACRIAIFBgCYQt3U3C/uWi99B18pdz2p+flrmdekk9+PUzhfiD5H8Ozxcff/vY352N0+nE629++RjisVIBOsP2iuI5ONK6oGH9vXiGVacLtoMx5N3+CdOSygyJV0sxfWpXlymcaFQ8/on7gEQUXzTRyEu7oV7D0Z+rUqVmJd9yMiAsbuW6FLNwhyP/6w4+Cji/Thi9s/vWHH9IK/LQ24hbpbpmjcHdDhW5EgAgQASJABAoCgXwK9w837pdrJ9zlvO+OpfKmqA9ybg5+cz1fvny58wV0jGHtj9K5XeOd7bhn3HMtc7bxwQ2M0fGhPnyw2hTuWBaPGXVghiOuNR28ioAVuXjdFvH9ckzj53qMRmzOkEmbJRYBTeEezkOEXHkWNH7Owh09CoQ3Zs2xbD7dO+/a+yAMwiJOGKIddgtZuOOVgq+/+YvzEQ7FyOuIhgxhESerzRHJ02XOwgyxj4XT5fRVwlt+02fNFWP+XrCEHDP6zlJyLMfX5eR2vCruWCpvLuFPs3TesTNHFlj2quRRpDL9Y68EVPHHAwzku4qN6qsONn49y3iloKp/taXyzkORylcQqqSXAW56EwEiQASIABEoVAT8iqogQlvD4n330Xfe77zrPmfdnliFOz44h6+y4wNqmEH2U+6kLJUPKhByDY/x6ocffigvvfSS82V4CHcs11e7ugweH7tGWBwh3uGuYYAdPk6HL8vDDuwhrPrn8xiNcK+Ybc80Swxxv23Z3ypEPhoJnSl/d5OYU35V7RybyU81KjtlkvMf8dWX5cu+TTLy2P/HO4K+WhwXkfv9TnHi4ehsar9qush3CksnvztlUtp8z5DjLP8qNsZXPOyogsfufVVwQHbsOKk8HCtnWNf55KCb7VCEOwDCUne8p46PzOEjEhDS+PgFBCZ2nMMNfgiDsLkuj68gSsVvIQt34PDNX751vrZplsntHF8ARdhcsEsJaktwp9JzxKwpnCsEdYUQxbkpYKsvvVf7VYTrMYFcuUQfwn5Oxbv1KfFcadexkTZ/0yuE97FMVwjsyjzjujItkQr/SvuiIjuVxkaZO8t4yCDH4qT8RSriVNqoKtxNjJApG6cUujwhAkSACBABIlBUCPgRsBiEqhj3e/x061F5bOb7cvvEKdJ/6DB5/acVgW1oWm6D4CBu+J9xzJz/9NNPVcSj/nMSjrBnX5eqcEe58ZV4U6ybeH/99dfOLLzpBnEPd9NNz2EH9mBX3fJ5DEvYpbXjiN7MS+KdWXmMLM333h1xa8Y1HwJAnJt+x8S6Cv5j4jf1AMAU7YaIP87JnwpyS7yrYDdsHjfezAPCV6SbEtEqyI00Kh4UGHk9FqayrBU2UzY07zYe1dK28huyYNd6zScH3WyHJty1B8KX4fHUDMtb0MA9++yzzo5zuMEvl6/Hazr2sZCFO8ry409/kw0bzHlru4QV1wiDsDlvKlyd2ehKMQq7EKSm6IWbI3x1htxKvKqAPTbTbYW1w1QxUe1BQXWRnDG8VAhvO9+peHYaTvmrltspY0qoQ3hXPgiosFORhj6QqFIm234qYZ4QASJABIgAEShuBMIU7hDrH22qEPmv/7hcbps4Wd5buU0uv+5GeeLNDx3hjiXzj772TiqcivN0R7dBsF83fFQNH1dbtmxZFdGIFaT48Bx2zMTjY2846jUmrSjcK2fZTbwp3GdIWmFsCE235fQ6C68CEsd0y98dP0MwV7cJgWwIaCf9CuFdKaINMewi6l3TRzhNt9rDhurivnq+quPkGobCvbA6mUIX7uj08FX5dEvgMcs+9+tvfL0L77v2qs12H5t5Tn1xvnLpd2rJO4yn4h3zTwleN+EeUFTDvouwTpXJVSQfy7fbAwOjLCq63exXEe52+VxsVBHu+nDD5UFIKt88IQJEgAgQASJQhAiEKdzvfm6a9Bk4VCa9/6XzN3D433aI+TuefkEuGz5KJr79iYz+4wMCUZ9OqNt+pmgMco5XOvFutttH0pIo3PGQAf/65LdOgEU2cTJhqEvlX3zxxayXymOWHbPwWCoPO0W3VN4Rv7ZYNsTxMfFeXagem0F3a0tcZsArg1XOnlezqbPhlYFTZ36Fu2MzFcs48SHcK2b/rRl6fXhhCf5qeXfC2bP91XE0H3KEdZ7pPgjbP/QZd6OaIj0tdOGOD9QtXrLEmU13WwYPN3xRHrPtCBvuZi7zNs/dUlFhXzkbXUXwHpuxryLynWXj1WfxU9bdhHiWwj01464rClJC3iqXi/0q5XDxT+X32Ikt3NXfsQMBbzzMUD8eiQARIAJEgAgUGwJ+RaItpt2uX/p6vlw8cIhcesXwKkvjsUweX5cfMnKMvPZTMNGOdLIdQC9atMgR7hDwbjbclsZD5OvS+Shn3NetW+esDMCMP77Ibn4Izi3vcNu4caPzbnmQOF623NzBDfydG95Vtz9OB3e8145XaL0+TgfhjnB+OeaWh2zcwhJ2ae1UWxbuLjarC9XMIlVFdEp0WzPk1Wxa/mnzDaFcLbyH6FbxjaMlwCvSMONVXxbvhLFwqpZ3CvfC61IKXbgDcQjyn39ZKu/91wfOX77hfXbsa9aslTkffiTz5i+QTz79LO1/vWdXcxWiVkWvlyB1bLsI2iqC11W4u83CGzkNRbibwtxtht/0d5/Rr1oOK7yRXT1Ni5NbmTQij0SACBABIkAEiggBv6LKTahH5ZaNeEMcfEzZbbbdr71SF+6KEzD0+zABcUrl7+DcRWhVAe8Wxs0tJbbdBLIltKvHz/wwIGXfVbgfW6qvs+umYNdzt3xZS9yr56v6QwLXMJadKnnV9PNwVH5HdeSMe4j/hYmOK4wNlb98+Qr59rvvnR3ncMO2b9/+nMS7IzarfFzO5cNtx5aJq5B3El64oOJ/Jh3hXjnbXv0jbx4i3cXmgq+PfY3eTeS6PCBIYXvMljmr75QrNcNtL5vXVQLG1/Rd7FcV7oqLUVY8lFhY+W/3VYQ78pSa3Xd/MJDKP0+IABEgAkSACBQRAsUq3Hfu3Ol8EM3tL96CuEX5d3AbNmyQhQsXOv/y5FdMZBPHr22E05l184vxfuIjX5iJR3w/4cMME5Xw0w+6Vf1QXFUR7SpUj81Cmx9tO+77FRVfjrdmqCu/0l65VN5+5x3lddys99wnfW/+v7zxQMF6EFCBV0W+q5Tl3RUySf+P/li+TH+nbKbYt/N+TJCb5XTFw/4QXh5EuhsnwuScH1sU7gkU7pn6chXv69atzxTU8t8oGzeoIDXeXbeEvBNJxbG+2z1rlsw99hdyFeL/WHyIVYjglGj2EO4wWs0m/tJN3asKZLd30FOFcezMkrlfz/H8q7aqacF2hXj3/Y77scRSy96P4TBr1tyKBxjGO+14wLFx4QKZO8fE1CpPKvM8IQJEgAgQASJQXAjEKdw/+fWwPDz9LXlz0fq077z7GRTbYcKYLcd/vuNr6H4xsvNQDNf4Rym8d69lwbcB8J/tu3btSrnBD9dwh7+GRTzE1+uojm4iLX9u1d9ZxxL3ke9W/JWau1DVpedGW4IvyR/78nyFCD/mB2HsCOJK4Z56YOAEqXSvEg9/q7Zv07G/kTNEO0Sxq3BHmGPiPZWtg7Lg+2N/Cefk4aDMWaZ/IYdAlWmn8FWBf8yGKdoRxhMPJ0/HIlV519/Ke4iiPio+ajoU7gUo3EFJvPOe7kN2x2hbnAcV7hD93IgAESACRIAIEIFYEfArSvOxLB4frnv+o7/I0JE3pBXvOvANcoRwx2x5kNl1Oyw+qob35IOkGyQs3mvHX9R51QE+PJfOP0ha2Yb9+OOP5bPPPnNe/8TsOTB97bXXnB1f3oddHNUN/hDw+MAd3o1H/GzTzjZeSkSGKPJK3qYKd52BLwJss+VXtvEo3AtUuMfaQ8edOIV73DXA9IkAESACRIAIpBDwEo324NRLuN/y6CTp2uPCnPcR4++UDzfsc515t/NSDNcQ7VhK/s4777i+P44Pz0EMe/lHhQFm0GfPnu18GV5FOV5DwMfq8GAB+cARX4/X2XiE0y/Sq7iPKr9Ip+RFdj5ENYV7zg+gKNwp3FMdb8GcULgXTFUxo0SACBABIlD8COQq3L0EvR93nXG/YvRN8tYvm1xFO+xEKfqiSivTF+Ez+UeVT6SD2fOtW7dW+dL9e++9l1qNsGTJEnn33XedcAiPj9MhPOJFmU9Ni8I9D8vLKdxz5jKFO4V78Y8oWEIiQASIABEgAkQgbwjEKdzxjvsjr76TVrQXq3CHyMy0VD6TvwrVOI5z586VGTNmOO+w4zsAuI4jH25pUrjnQbjnYxY/Zptu3MmnG4U7hXveOnIaJgJEgAgQASJABIofgTiFu59Z+WIW7vkUCfm2jdn0pUuXyrx585xjXLPrbuWkcKdw98MBN+7k043CncK9+EcULCERIAJEgAgQASKQNwQo3A8kZqY4n6KhlGz7EW0MQ3Ef9T1B4U7hnreOnIaJABEgAkSACBCB4kfAr3D/Ypt4voPud+Y8m3BIN+oBNtMr7IcZFOUU5X44EPV9TuFO4V78IwqWkAgQASJABIgAEcgbAn6F+7ydR2IR7kg36gE206Nw9yP8GKawHxBEfZ9TuFO4560jp2EiQASIABEgAkSg+BHwK9y37jsoUc+6Iz2kG/UAm+lRuFOUF7Yo91N/Ud/nFO4U7sU/omAJiQARIAJEgAgQgbwh4Fe4Y5ALEY0Z8HwLeNhHOhTthS2goxZGmp4f0cYwxS/MM9Wx8iWq43GZMlRI/lGB5pUOOq6jR4/mrWOkYSJABIgAESACRIAIJAkBjHuCCHevMRTdKbCTxIG/mzBTCkkDMa/RP0QAR6LmLIV7iDPuO3fulP379yepP2VeiAARIAJEwEIAQoM7MSAHwuEA/sJrx44dzvgHY6Ck7FEPqJlecT14aPrMJxTuMf9HetIfRoAjUd/3FO4hCnd0XnjqjE4LAwJuRIAIEAEikAwEKNLCEWnEkTgqB44cOSIY92zdulX27t2bGMHu9eAg6gE20ytsIf/Vyk3CWffoZ7GTLtY1f+AGOBL1fU7hHqJwR+WhE8PMOwQ8d2JADpAD5EB8HICgyLRv2bJFuBMDciA7Dmzfvl327NnjjH0w/sl11/YyFztewl3dox5oM73CFfAQZphVpYCngDcFOzgRh2hHW0LhHrJwZwNduA006451Rw4UDwd0kG4evcQAZgu5EwNyIF4O4CHb559/4ew6ix+0TtzucbMNMM/Z3hdPe8+6ZF2WCgco3CncI1/mUSo3F8vJjoQciJ4D5sBcz83BvC0E9uzdK+Zu+/M6XjFH/EsD/wrR/qWsWLHS2T///MvUEnwvDpj3rZ7v3buvykM4897X9sA8so2Ovo0m5sScHMieAxTuFO4U7uQAOUAOFAUHzAE5znXQXjnw3+eIdL3GEl/uxIAciJcDeE3hs8++cAS7fhEDAh5u8AtaP6n721lJUynktT2w2wmKiOxFBLEjduRAtBygcOeAvSgG7Gw4om04iDfxThoH7MG4DtIrBvGVg3eIgN27d6f2Xbt2CXdiQA7Ew4HNmzdXE+22eEcYv/Vj3tsq+M02QNsFu71IWnvG/LCPJQfIATcOULhTuFO4kwPkADlQ8BwwB+IYnOusmx5VsKsAwEdEdcdfWXEnBuRAtBzYtGmTp2i3xTvCZqofvZ9x1PscQh73vrYDekQbYbYZbgNkulE4kQPkQNI4QOHOAXvBD9iTdlMxP2zoyYFoOWAOwG3RroIdg3kd+ONL2BVfr94uW7dtc3b9mnWmr9DTP/OX+okRMcrEgfXr18unn35eZXm8inX7iGXzCIs4bnZx71bexxX3Nu5xvd9x77sJeIr3aNtp9ovEmxzInQMU7hTuFO7kADlADhQsB2zRbgp3U7Rjxm7p0mWyYMFCmTdvPndiQA7ExIEffvhJPvzwY1+iXUU8xDviIK7f+xf3Ou553Ptu4h1tBcV77kKCYowYkgPRcYDCnQP2gh2ws6GIrqEg1sQ6qRywhbsuhTVF+8aNGx3BvmXLVjly5IhqAR6JABGICQEI5qBb0Di413HPQ8CjDTDFO75Cj7aCwp19W1L7NuaL3HTjAIU7hTuFOzlADpADBckBd9Fe8X47lsZioI6ltb/8stQZwAcVCgxPBIhA4SMA8Y42AG2BiveKB3wVbYUp3t0GynSjgCIHyIGkcIDCnQP2ghywJ+UGYj7YmJMD8XHAXbjvdT5GhQE63nP99ddfZf78BZxpL3z9xRIQgawQwMw72gC0BWgT0DaYH6wzhTvaFLbp8bXpxJ7YkwPpOUDhTuHOToocIAfIgYLkgAp3DLyx791bOduOD1Nhhg1LZPFOLDciQARKFwG0AWgL0CagbcCKHH2tRtsPbU8oHNILB+JDfMiB+DhA4c4Be0EO2NloxNdoEHtinxQO6EC7QrTvFby3ipk0/BUUBudYIosvUVO4l65gY8mJABBAG4C2AG0C2ga0EWgr3N51T0r7xnywryUHyAGbAxTuFO4U7uQAOUAOFCQHbOGOGTTMpGFQjiWxW7ZskbVr11G4U7sRgRJHAMIdbQHaBLQNKtzNWXdtT+yBMq8pnsgBciApHIhVuLd/4UtZtX2vs+P8uPEzctqTAirzwRucHCAHyIH8ckAH2TjqjLsKd7zDiv92xjutq1evoXAvcdHG4hMBCHe0BWgT0DZU/Ugdvy7P/iq//RXxJb5hcSBW4Q7R3uiZT5wd5xTuJHZYxKYdcokcKG4OqHDX91Mh2s2/gNu2bbts3rxZVq+hcKdsIwKljoAj3NescdoEtA0U7sXdP7D/Z/0WKwdiFe6rd1C4FyuxWC42muQAOZBPDpjC3RHtxvvtGJRv3bbNGaSvWr2aM+6lrtpY/pJHAMIdbQEe5qFtUOGuX5fHA0BtU3DMZ9tF2+wbyQFyIFsOxCrcP1q+WbbvO8il8ny/lp0kOUAOkAOBOKCDbF0mb3+YDoPzTZs2ycpVqyjcS162EYBSR0CFO9oEtA3mB+rwbxS6ckfblWwH1YxHQUYOkAP55EDowv3Eh96Xv6zdJtv2HZQbP5jvufx92Ns/ytzVW+W/TZjpGSbo0vl8AkXbvBHJAXKAHEgOB3SA7SXc8R5rhXDnjHupizaWnwhAuOMhHtoEtA2mcMdDPwr35LTt7GdZF+SANwdCF+5TflglN3wwX/7t3rdlzY59ctLD71cT5uc8MUc27t4vEPlBxXm68Kxo74omNsSGHCAHiokDFO4UY0SACPhFoEK4r/YU7njdBuJd25ViaitZFvb95EDxcCB04f7Cj1WF+zXv/CT/l/G1+P/n9lmyYNNO6f3qX0IV7RD0JGbxEJN1ybokB8iBdBzQAbbOuOvH6fQ/3Ldu3coZd7+qhuGIQJEjYAp3tA32jDuFO/ubdP0N/ciPpHAgdOH+5x9XyYZd+2Xr3oPy0Fe/yDdrtjo7zuG2ftc+mfrT6tBFO4U7b6qk3FTMB7lIDuSfAxTuRa60WDwiECICFO75b5PZ7xFjciD/HAhVuF/06l9k4eadgll1XdKO2XbMumPZPJbP4713zMqrf5hHEib/hCHGxJgcIAeSwAEK9xBVDU0RgSJHgMKd/VYS+i3mgTzMlQOhCfcTHnrPeW/93D99WE2U4z13U7jjPfgwBbvayhUMxucNRQ6QA+RAYXDAj3DfuHGjrFzFj9MVuSZj8YhARgRUuKNN4FL5wmjj2ReznsiB6hwIRbjjy/D4QvzVb//kKcgx044vzeOL82F/lI7CvXrFkuzEhBwgB4qZA36F+4qV/Du4jKqGAYhAkSNA4c7+sJj7Q5atd9iamgAAIABJREFUdPidk3Bv/8KXzn+w47/Y8Z/sKqDjOpK4pUNc1jXrmhwobQ5QuBe50mLxiECICFC4l3Z/wfEC679YOJCTcF+1fa80euYTZ1+9Yy+F+wHeGMVyY7Ac5DI5kGwOFLtwX758hQwYNFhw5EYEiEBuCFC4J7s9Z3/L+iEH/HEgNOEOER/XTLumy0r3V+nEiTiRA+RAoXMgauE+683Z0rlbzypCGn9FN+6WCc6Oc7dt2/btMmDQkLRh3OIhvXplTeS77//q5k03IkAEAiBA4c4+r9D7POafHAYHchLuulQeoh3nKqDjOpLUJDU5QA6QA6XBgaiFOzTCY0886eyqFyCqIcohzr02hIG4v2bkdVVEv1d4uhMBIhA+AhTupdEvsP9nPRc7B3IS7nEJdK90i72yWD42SOQAOUAOVHAgDuFuLl/X2XbMjKfbIPZfnv6KI945e54OKfoRgfwhQOHOvpN9JzlQDBygcOd76VIMRGYZ2CCTA6XFgTiEO2SFzrrrTLrXEnmExUy8zrRD4COubngIgKX3pvDHuc7gmw8JEEcfFGD5PHbM4qdLW9PhkQgQAREK99LqHzgeYH0XKwdyEu5Lft0VWn+w+NddOS+1L9ZKYrnYAJED5AA5UJUDcQl3Fdx+3j83l9Kb59pxmm62ULevIepNsY5rU/SrTR6JABGojgCFe9X2k/0J8SAHqnJgxYoV8u677wqOScYmJ+HutWQ9LvckA828Vb1BiAfxIAfIgVw4EJdw15lvnRmvLhEqXXR2Hi76kTqIdXPTMHpUP1u42/4ajkciQAQyI0Dhzv4ml/6GcYufP/PmzZPp06cLjkmubwp3LpUPTFAMXPfu3St79uyR3bt3p3Zcwx3+SSY981b8DTDruPjrOC7hDuGN5e/Y0814uwl1N/GtM/j2gwBbuKs9zPSnSzezhGEIIlB6CFC4F3+fwH6fdZwLByjcx8/Ieel70Jn7XCqMcTPf8BDktlg3hbt5jnAU8JkxJe+IETmQHQfiEO5o07BcHcLZXObuJoPgr++jm0dboGs4290W7pqGKeARlxsRIAKZEaBwz66dZf9E3EqFAxTuFO5FNeuMmXRTmPs9R7xSuelZTnZw5EB0HIhDuEOwq8BWEY9ZdLcN7uY76Qijs+squFWE49qejfcS7pqWHV7deSQCRKA6AhTu0bXN7AeJdSFygMKdwr1oBGu2ol3FPcU7G/FCbMSZ52TzNmrhbopslQUQ3PgyPES2ubmFhb8t9vEgQMW9Lept4f7c5CmpdNQOl8ybqPOcCHgjQOGe7Pac/S3rJ24OULiXuHB/+NHHqyyTfH3mGykh//U3f5H+AwfLpk2bUm5xE9YrfQwQVYDncoQdrzTozgabHCAHgnIgSuGuQllFtsoDdddZeHU3Z+bVTY/wg9h/9vnJ1US/+kG0m8IdDwI+mPOhE16X3XvN9Gs6PBIBIlCJAIU7+5igfQzDlxZnKNxLVLjv2LlTxo4b7+w4x42vbhDsuM6ncMcDAjw0CKvB8ftOeyZRDzth5Yl2SqsxZX2zvt04EKVwrxz+84wIEIFCRIDCnf2IWz9CN/JCOUDhXqLCHcIZwl1FuxLCPBaKcM802/7Ka6+nVhX0u3SQrF27Lu3sPGfd2UCa9wHPyYdcOEDhXojyiXkmAvEgQOHO/iaX/oZxi58/FO4FItwx+FuwYIEjOL1uTAjOJUuWyPbt29POGuvMurks3s2mCnccO3ft4Yhfe+k8ltHDTZdF2v6IiwcEH3/6qROm74BBMnrMjanwiKcPEGxbfmfk073bDtF+w003y5YtWxzsHnz4EcGebuad77oXf8Pnxne6sd7zwQEK93gEEFMlAoWIAIU7+6F89EO0WTy8onAvEOEOUf7OO+/Ixx9/7Cre4f/DDz/IG2+8IRs2bEgr3Jcs+dkR4hDV6W5m+JvCWgW/Cmpc33n3PQJ7sGP7w01taBxND9eZ3OCfKY+wF2SZ/PwFC+X2P96VEvJuAp7L5YungVO+8cg6jYsDFO6FKJ+YZyIQDwIU7uyr4uqrmG683MME46xZs2T69Omh7bAHu3HU7XFB/ys9yeGzBXDz5s3y9ttvVxPvKtpnzpwpP/9cIaLTpQGh3e/Sy1KC2yssRLM9g55pib3t72YD6dnCXUV/plUAbnl1E99ebpiBzzTjjrhu6dAt3kaN+BP/QuQAhXs8AoipEoFCRIDCnf1cIfZzzHPuvMWk4bfffitz5sxJu2MSF+Iex0xhYS+uyUgK9wMHBANAW7zDDTPtKtpxnekGCjLj7ke420vczTh+hTvyrPnCLL+fmXYtp5dIt93xbjvecf/iy6/SLpWncM+9AdK64ZFYljoH4hDueF3qm2++kbfefltmvfkmd2JADuSJA7jHcK/hngtjo3Bnn1nqfSbLn/4e4FL5Alkqr0Q2xftHH30UWLTDjgrtTLPbbqLbnlHHzLkptO049rWWw55xV3ccVcDjvXqcm35u536WymOpCN519zPbHtfTKbey0S19A0Z8iE/SORC1cIeA+K8PPpDVq1c7D3vDEBO0QQSIgDsCuL9xr+GeC0O8U7izT0t6n8b8xctRCvcCE+64YUzxHmSm3bzZbAFu+um5m+g240FU20vu7Tj2tdpOJ9wRJsjS+XQfp8PseRDRjvD8OF28jZJyhEfWQzFwIGrhjtm/tWvXuqsMuhIBIpAXBHDP4d7LdaNwZ79XDP0ey5A/HlO4F6Bwxw2BwSAE6bJly5zzoDeJCmP9ojviq5vOxLuJblu4Y1Yc4RBfZ8r9LJU37WjaEPNYDYBrXRWgtuHmteMdf3tZvF6raDe/LK9+XkfY80qL7t71QGyIDTlQnQNRC3cs3UU9cCMCRCA6BHDP4d7LdaNwr96Gsl8hJuRAJQco3AtUuCuJMSjU82yOutQdy92xQ1CvXLXKsZlJuCM9hNe4eAjw409/k+EjRqUEuJsNxFNhjrgQ+vPnL5AZhi3Ni98yeS2XN//DXfOJo9d77lwmX9k4+MWe4YgZOeDNgaiFO95p50YEiED0CIRx71G4e7el7GeIDTlwQCjcC1y4k8QVDVm6WXevmXU3d862s2PgPUUOhMkBCvfoBRRTJAJxIEDhzr4jzL6DtsgnNw4UjHCPoxHOR5p+/q7NraLolvkGzvSuu5tQN934bntmjMlDYkQOBOMAhXs+elLaJALJQ4DCPVjbyL6EeJEDwTlA4R5x20/hHpykQW7sbMU7RXt+6yVIHTIs66KYOEDhHnEny+SIQEwIULiz7yqmvotlSSafKdwjbuAp3PN/I2C5u9c77+YMO84Rjsvj818n7ACIcalygMI94k6WyRGBmBCgcGc/V6r9HMsdHffnz58vr7zyiuCYZNyPi6kdDj1ZCvfoyA1Bjpl0W8TjGu4U7NHVRZIbF+aNPMgnByjcQ+9GaZAIJBIBCnf2JfnsS2ib/AIHoF3wj1lJ1zAU7mn+Do03M29mcoAcIAeSyQEK90RqLGaKCISOAIV7Mttg9o2sF3Igeg5QuFO4J3pJCBuF6BsFYk7MC4EDFO6h6yMaJAKJRIDCnX1SIfRJzCN5GgUHKNwp3CncyQFygBwoOA5QuCdSYzFTRCB0BCjcKYiiEERMgzwrBA5QuHPAXnAD9kK4sZhHdgDkQH45QOEeuj6iQSKQSAQo3PPblrKvIr7kQOFwgMKdwp3CnRwgB8iBguMAhXsiNRYzRQRCR4DCvXBEBQUg64ocyC8HKNw5YC+4ATsbhfw2CsSX+BYCB4pRuH/3/V9lwKAhsm37dt/i57EnnhTsuWxh2MglfcYlAukQoHBnn1QIfRLzSJ5GwQEKdwp3CndygBwgBwqOAxTuFVInDNEdhg0VXrPenJ3zgwS1xSMRAAIU7hREUQgipkGeFQIHKNw5YC+4AXsh3FjMIzsAciC/HKBwrxB1YYjuMGyoxKRwVyRK87hw0WLpO2CQ6/7Jp59lBQqFe37bUvZVxJccKBwOULhTuFO4kwPkADlQcBwoROG+b98+GXfLBKlX1qTKDjf4uS2VX758hXTu1jMVHsLY3FR046h2cW5utg1NT8OoDb3OdMRSfizpN9NzK5uZDvKt4e3XAZC+Cn4NY8bNlB/6JwsBCHRbvE975bWsM0nhXjiiggKQdUUO5JcDFO4csBfcgJ2NQn4bBeJLfAuBA4Um3FXYqqjWa1OI28JdBTfcsalgNuPAHsSuHUbTQRyIYH1v3suGhvejrhDWDI9zTd/2gz3k1xTidjkRJ10Z/OSJYZKFgCnecxHtFfx5M+fCzZs3X1auWi0bN26UrVu3yo4dO2TXrl2yZ88e2bN3r+zdu9d5eKbtSiG0gcwj+2pyoPQ4ELpwf/nll1MNrJ/zVOAcT37++WcKUD6EIAfIAXKgRDigA2wIYAy6sWMQjsE4BuUYnGOQvmLlKsGgPdct11k/FcwqcJEfP4LWFMiI4yZ6M4Wxy26La/vaDm9euz1wMP1tWyj3NSOvEzyE0E1tKBZ2HISzy6lxeSwcBCDecxXtKG2u9x5sULiXnsChqGWdFyMHQhfuEydOTPUqfs5TgXM8oXDnDVqMNyjLRF6TA+4cKDThji7OFKgqXs3Zc1OsuvnDBgTwgEGDU0LYtKndKOxgeb0plnX23m0pupsNteV2NG2p+NZwti0zrKatR41rx4EttzJoGjyWFgIU7u5tIPsG4kIOlB4HKNxDnp3asGGDLF26VPAggTsxIAfIAXIgPxxYsmSJYF+8eLEsWrQodVywYIHMmzdPfvrpJ/n+++/liy+/SsSMO6SWLWIhWM0NYlXf/9YZelPYmzYyiV61ow8ATCFvz/S7CWczX17nWh7Ttm3LLJOXHTsOwvmJ52WP7sWFQFjC/cuv5jptAtoGtBFoK9B2LDLaEG1X2G7np90mrsS10DkAjQetF9dDEwr3EIU7KhJLMw8fPlxcvSZLQwSIABFIGAJHjx4V7EeOHHHaXLS7hw4dcjpTiNXdu3fLtm3bZO269YkQ7iqgVXC7wWmLVT+C1i2MKcxtm0jX9Me1mw23/Lm5abn0AYNtSx9ApCu3Hcctj25p0600EAhLuK9bv8FpE9A2gLcYeKPNOHT4sNOGoC3RdqU0kGUpiQARCIoAxhrQenGJ91CE+6pVq2Tq1KmCpfFBdsRB3DA2PMGJ6+mHpounMBTtYdQmbRABIkAE0iOgA+xCE+4qcN1KZ4tsndFW0asi2LQB0WvOeNtxENf0xzWWqpsfi3MTzm75gxsED8IjL9g0T5pH+6EAwsDNzAPcNDzOM5UBYbiVLgIU7qVb9yw5EUgiAtB60Hyq/6I8hiLcIcDXrVsXGFvEQdwwtiQId+SBGxEgAkSACOQfgUIT7kBERbW+461HFbHqb4pqddOwpmiHTYjel6e/UuVv5tzCaHyER3pmGggPfzueWy2u37BB3pz9lhNebZrxVMjDT5frw46moXHuvvf+lPhHnnCNPKm/adMtH3QrHQQo3EunrllSIlAoCMSlO0MR7uZH6IICnktcM624ADSfslC4mzXCcyJABIhA/hAoNOEOAY6vq+tMtSIDgWqKaHUvpSOEO3ZuRMANAQp3N1ToRgSIQJwIxKU7sxLu+Js3e0l8tuDZdsy/kAtiMy4AKdyD1BLDEgEiQATCQaDQhDtmue3l4vp+eJJEqz0zrjPgONr5D6cmc3vHPqw80E5yEaBwT27dMGdEoFQRiEt3ZiXc7UrKZdY8l7hmPuICkMLdrAWeEwEiQASiQaDQhDtQcRPFXBJO4R7NHVO4qVC4F27dMedEoFgRiEt3UriH+FV5VCI3IkAEiAARyD8ChSjc848KUyACxYcAhXvx1SlLRAQKHYGCEu5cKn/A9UuCFO6Ffhsy/0SACBQKAhTuhVJTzCcRyA0BCvfc8GNsIkAEwkegoIS7XfxclrvnEtfMR1wAcqm8WQs8JwJEgAhEgwCFezQ4MxUiEDcCFO5x1wDTJwJEwEYgLt3JpfJcKm9zkddEgAgQgcQjQOGe+CpiBolAKAhQuIcCI40QASIQIgIFJdy5VJ5L5UPkPk0RASJABAIjQOEeGDJGIAIFiQCFe0FWGzNNBIoagYIS7nZN5LLcPZe4Zj7iApBL5c1a4DkRIAJEIBoEKNyjwZmpEIG4EaBwj7sGmD4RIAI2AnHpTi6V51J5m4u8JgJEgAgkHgEK98RXETNIBEJBgMI9FBhphAgQgRARKCjhzqXyXCofIvdpiggQASIQGIGohftbb7/t/JtI4IwyAhEgAlkjgFWNuPdy3ebNmy/r1m+Qbdu2ye7du2Xfvn3O/Xzo0CE5dPiwHD58WI4cOSLaruSaHuMTASJQ3AgUlHC3qyKX5e65xDXzEReAXCpv1gLPiQARIALRIKADbAy2MejGjkE42mQMyjE4xyB97br1gkF7rts333wja9ety9UM4xMBIhAAgbVr1wruvVw3CvdcEWR8IkAETATi0p2hLJWfOnWqrMtiQIM4iBvGFheAFO5h1B5tEAEiQASCIRC1cN++fbv81wcfyJo1a2T//v3BMsvQRIAIBEJg3/79snr1aueew72X60bhniuCjE8EiICJQFy6MxThvmrVKkeAY/Y8yA7RjrhhbHEBSOEeRu3RBhEgAkQgGAJRC3fkDgICs39Yuov3brkTA3IgPxzAPYZ7LQzRjnuXwj1Y+8rQRIAIpEcgLt0ZinA3i2YuffdzbsbN5TwuACncc6k1xiUCRIAIZIdAHMI9u5wyFhEgAnEjQOEedw0wfSJQXAjEpTsp3PlV+eK6k1gaIkAESgIBCveSqGYWkgiEggCFeygw0ggRIALHEKBwz5EKcQHIGfccK47RiQARIAJZIEDhngVojEIEShQBCvcSrXgWmwjkCYG4dGfoM+74qzjd/Jxr2FyPcQFI4Z5rzTE+ESACRCA4AhTuwTFjDCJQqghQuJdqzbPcRCA/CMSlO0MX7vmBJ7PVuACkcM9cNwxBBIgAEQgbAQr3sBGlPSJQvAhQuBdv3bJkRCAOBOLSnRTufMc9Dr4zTSJABIhATghQuOcEHyMTgZJCgMK9pKqbhSUCeUeAwj1HiOMCkDPuOVYcoxMBIkAEskAgauE+683Z0rlbT1m+fEUqt/v27ZNxt0xwdpyb27bt22XAoCFSr6xJlR12uBEBIhAtAhTu0eLN1IhAsSMQl+7kjDtn3Iv93mL5iAARKEIEohbugPCxJ550doXzu+//6ohziHR7U+GOMLqpG8W7IsIjEYgGAQr3aHBmKkSgVBCgcM+xpuMCkDPuOVYcoxMBIkAEskAgDuGO2fYBgwY7s+462+4lwlWkm8IdxUR4zNLbM/RZQMAoRIAI+ESAwt0nUAxGBIiALwTi0p2cceeMuy+CMhARIAJEIEkIxCHcUX6ddYcgTyfA/Qp3PAzAEnxdUm/bRHoQ+9g1DNzMTdNSfz2a4XCu7nYapi2eE4FiRIDCvRhrlWUiAvEhQOGeI/ZxAcgZ9xwrjtGJABEgAlkgEJdwN4W2PZtuFkPFtBlG46qgRhiIaByxaRxzFl8Ft7qpDbVrx9Fr9Ydd2NA0cQ1bFO8O5PwpEQQo3EukollMIhARAnHpTs64c8Y9IoozGSJABIhAeAjEJdx1iTw+PKeC261UKqB1lluPmeLZItu+1vRNIa/L9zUfZhwI/WtGXlclr8gb3ODHjQiUAgIU7qVQyywjEYgOAQr3HLGOC0DOuOdYcYweKwIqLszZuGwzFKatbPMQNJ7OXqoIwtH+crhpE7OY6fzNsJlsmWF5HhyBuIQ7OADRi11545Z7vR905hthve4z5aGKe3M23BThSMcW7va1nS7SV7vm0S+P3cpGNyJQaAhQuLvXGNqXTA8T3WPSlQiUNgJx6U7OuHPGvbTvvAIvvT2oD1ocFQymUAhqQ8OrYPASJxouSUctvwow5B3iRsWWnVe4+xU8FO42euFexyHcTZEMLqQb8Or9oFzCtT3LrfZMToE35v1o3+MaRzkLVJGGKcpNP9teuLVAa0SgMBCgcHevJ7Qv6dox91h0JQJEgMI9Rw7EBSBn3HOsOEbPGgEdwCdFKKtQSUp+/ABrC/dMcSjcMyEUnX8cwh0iWAe5me4/vR9UuAMZW0TDT+0pcnaYTMLd7YGA2sIRHLeX0pv+PCcCpYAAhbt7LVO4u+NCVyKQCYG4dCdn3Dnjnomb9E8gAioa7Fk2HaSrIMBM3qLFS5wZPDOsigkVFyq2Vch+MOdDR1Agji0svOCwbSGcumnaECW62X6aB/jjXONkSl/tmLOUWg7Y0XM3e+qn+cLRnP208wJ7tr+Wxz662YKbWz4Q1/Qz09A8ajyE4yYStXBXnum9gzrAuVlXZr24hbfd7Pi4Rj2bXAbnsOum977ywLap4cwj4pv3EWwsWLjQDMJzIlDUCBSCcLfbem1r9B5/bvKUVL9stzvaLmg/YbYZqFi1Yftr2wDb6qdtS1ETgoUjAjkiQOFeoAByxj3HimP0rBHQjtrsoLXjNwfpcEOnjE3jqL925mrDjq/+fjpyDau27GsVJThqPlSg4BruiIO0zEEJBMbyFSvT4mTHgS21AT9cY9PyaR71Wstn27GvEU/tps3QMSFuhjVt2eVH/jBo0nyu37AhhQfqSvOLeF98+RX/A1yiFe52fWndq7veT+qOo/Jf61T9TB7ADXWrA2acI7zeF+qv9Y9rTRN2dFP+qB0cTe6pHdMfbQJscSMCpYBA0oW7thd6X+Oe1nZF/fRa2wD7WtsNu1/T+NqOID7C4Kjtj6aLo9otBV6wjEQgWwQo3LNF7li8uACkcM+x4hg9awS089bOGIbsDtvNODpmHdTbHbod3y0NN5tws22Z6cBfbengAvnWfJg2VYToQML08zrXfCMuNtjWdMw4dh40nqZl5tkuD+zAvluezTT03I8tCCnY1HyYdQk7ml8OpBTVymPUM+6VKSfrDPxx4zq4ZPMpWTlnbohAdAgkXbijvzDbee1/cH/ruXk/m/0LwmhfoogirNozw6q/Hs1wcIMtv32c2uCRCJQiAnHpTi6V51L5UrzfiqLMKurMzlwFIDpqc9OOX2fctGNWd7Vhx3dLw7Rrntu27AEBwppualvzhAGDbsi/umve1M/tqLYQVvNhYqADG7WpQscurznAUTtm+kEGNaYtOx2UwXYz86j5QzjNB/Ku9eaGQam5UbhX1Dh4pgN05YByxrwH1I9HIlCKCCRduKOf0f7JPKJf0PvZ7osQDv5mX6N1a7ohnt1GaDjbL0gfpzZ4JAKliACFe461HheAnHHPseIYPWsETLGqRmwxCHcVhDqINzt0e0Bgx3dLQ9Oyj7YtMx2EVVumKFUb9uBB3TXvOGbakB4GJ3g/3xS4sK3Xdh7s8pp5tsuD9IMMavzY0oGXWTY7T+pn513dS/VI4V5Z8+C4Odh341VlaJ4RgdJDIOnCXfsv9Dv25tYXmf2LWz9p9qlmWNu2GQ5+Qfo42xaviUApIRCX7uSMO2fcS+k+K6qyugk5N9FndtoaR4WsPSCw42t4dO6ZNtuWfW0PLqa98qozkwC7Zh4RDjs2Mz967ib8NSy+nn33vfenlg5r/jWObUOvkT42Mx+4RrkVK7drJ5LHj23LvHbLFx44YDNxQ7iXp7/iPPSAnz3I8ki6JJwp3EuimllIIhAKAkkX7ma7rwVev369c6p+2o/Z1279Cfot7de0n9N+HOFhA5vdp1C4K/o8EoH0CFC4p8cno29cAHLGPWPVMEAeEdAOGTNs6ID1WjtsJK2dPMKgM6/4YvxgJ6z6aYdux9cBgfpnKgrCIR1NX+3DzXTXdNQdRwwYND3TXdOGP9y9lvyZcTV95FfjIS4GPhDCOgAy4yAc4plC3fRHfMT1+9daWkYzv7CvZTPdFTf10/yZeYefmbdMdVHs/hTuxV7DLB8RCA+BpAt3lNTuL7W9V3f0PXBDX6B9hCJk91Xab6q/9kfax2j/g3B6jrDoczRdjcsjESAC1RGIS3dyxp0z7tXZSBcikFAEMPi4ZuR1qdmChGaT2YoAgTiE+/79B2TDho3OQ69ly5YLd28McK8CK2AWdCPO3rjanMsF56D1UsjhC0G4e+Grwt0W417h6U4EiED+EaBwzxHjuADkjHuOFcfoBYOADh70ib15jGJAoembs+lxgmfOnptYcLYimlqJWrhDTK5atVp27dolhw8fjqaQBZwKMAJWwCyIeCfOwSo9W5yDpVL4oSncC78OWQIikCQE4tKdnHHnjHuS7gPmhQh4IoD/L4d450YEgEDUwh2zx7t37yb4AREAZsDO70ac/SJVNVxQnKvGLv4rCvfir2OWkAhEiQCFe45oxwUgZ9xzrDhGJwJEgAhkgUDUwh1Lko8cOZJFTks7CjADdn434uwXqarhguJcNXbxXxWycC/+2mEJiUDhIRCX7uSMO2fcC+9uYY6JABEoeQSiFu54t5hbdggEwS5I2OxyU7yxiJ133VK4e2NDHyJABIIjQOEeHLMqMeICkDPuVaqBF0SACBCBSBCgcI8E5lASCSIog4QNJXNFZITYeVcmhbs3NvQhAkQgOAJx6U7OuHPGPThbGYMIEAEiEDMCFO4xV0CA5IMIyiBhA2ShJIISO+9qpnD3xoY+RIAIBEeAwj04ZlVixAUgZ9yrVAMviAARIAKRIEDhHgnMoSQSRFAGCRtK5orICLHzrkwKd29s6EMEiEBwBOLSnZxx54x7cLYyBhEgAkQgZgQo3GOugADJBxGUQcIGyEJJBCV23tVM4e6NDX2IABEIjgCFe3DMqsSIC0DOuFepBl4QASJABCJBgMI9EphDSSSIoAwSNpTMFZERYuddmRTu3tjQhwgQgeAIxKU7OePOGffgbGUMIkAEiEDMCFC4x1wBAZIPIiiDhA2QhZIISuy8q5nC3Rsb+hABIhAcAQr34JhViREXgJxxr1INvCACRIAIRIIAhXskMIeSSBBBGSRsKJkrIiPiKLxEAAAgAElEQVTEzrsyKdy9saEPESACwRGIS3dyxp0z7sHZyhiJQ2DX/iNy4NDRxOWLGfKPwNGD+wQ7N38IULj7wykJoYIIyiBhk1C2JOWB2HnXBoW7Nzb0IQKFiMDeg0fkx3V75ad1e2VfDONfCvccWRMXgJxxz7HiSjj6pLm/yj0fbswZgf2Hjspl01bJc19vydrW2h0H5YLJy6XJI0vkvo9yz1PWGSniiId3bJZ1d10kK0fUl22zn6hW0i3T7xLsXtv+ZT/Kr3++2cvbcf/1xdtk1XVNZP29l8iRfbvShi10Twr3wqnBIIIySNjCQSCanBI7b5wp3L2xoQ8RKDQEMFk19q11ct2sNXLD7LVy+39tkD0HjkRajLh0Z04z7hs3bpS1a9dGCpRXYnEBSOHuVSOF775j32FZve1g2n3t9oNZz3RDIE//YVuigEKeikm4Ywb78LYNGfcoZ7oh2t2EeyYi+BHusIFwGx4dQuG+e7ds27ZN1q5bLxi057rFIYoWLlwoM2fOdMqRa/7jjB8EuyBh4yxTEtMmdt61gjZg1eo1snnzZtm+fbvs2rVL9uzZI/v27ZN9+/fL/v37BeO5gwcPOvuhQ4eEOzEgB5LJgenfb5FOk5ZK2yd/kY5PL5WL/7xC3l6w3bsByINPXLozJ+G+ZcsWmTJliqxYsSIPkAQzGReAFO7B6qmQQl/12mqpff+ijPuMn4I3Fje+tc6Z3W722BJp/aef5Z2FOxxo8KDg0pdWOjPobZ78RYa+skrwZPHjX3ZJz+eXOWF7PLdM5m+oWFINP4TBTLmZD5yPemONDHhxpbR4/Gd5+NNNvqC3hTsW3//pi83SbuIvzo5zXZD/9oIdcuHk5TLo5ZWCPH2zco+Thpd7ugxc8dYV8g93/oOz4zysDQJ5+ZAzMu7ZCOltb/1Jdn05M5XVra/d54jmwzt+lU1Pj5TVN7SS1WPbyK4vXk+FwYmbcHdmykc2qibo96+YJ2tv7y5rbmonm54elXHGHfbdhPuhLetk/f39ZPWYFs4R19gOrFog6+/tIxv/NEzWTugie36Y47hjtn7zs9fLhscul3V39pQtr9wtRw8ddPy8fvJVh17plcKM+4cffiivvfaarFy50guGgnAPIiiDhC2IwkeYSWLnDTaFezIFWCEK46HTMUbjHicGnSctlboPVIzPGzy0WNpP/EVe/ynaibC4dGdOwh1NJJ5eJkG8xwUghbt3R1noPvd+uNERyhDWXvt5T/0iny/bnVVRIZJNsQ0jEO5dnlmasrlp1yHH9vZ9h1PLgJDemDfXpgQ0Ati2YBeiesuew4JVAVhKv2Fnha10mbWF+6dLd8nVr692VhXgHXqcww3btTPXyPuLdzrnyPe6HRXCzsvdK10IvuNuOa7KHpZ43zX3DUc8Q7B67hDXc9/wyp6n+4FVC2XzlHFy9PAhObJ7u2x+/gbnCIF7ZE/Fg5gje3Y6Iv7Q1vUpO27CHZ57vn+/inB34k4aJYe3bxI5elS2TPujYytlyOPEFu7IH0T4vkVznRg44hruyMvOT6c57kjv0ObVzjls4EGBI9aPHpUDqxc5efBIUvJZh15ploJw9yp7obkHEZRBwhYaDvnOL7HzRphL5b2xoY9/BNbvPCTNH18iL3y71X8khgwdga17DkuLJ36WsocXS7NHl8j5zy6T7XsPh55OOoNx6c6chTsKlQTxHheAFO7paE2/dAjYYhthIYAx049l+uYG0ayz6Fe8ulr6vbiiShjbFsLePWeDYwK2YBO2M222cMf1lL9UvjuPc7hhwwz7+c8sc94vWrix8qNqXu5eaWOm3RbucEv6dvTAPtk0aZQc3LhC9v7tU9n2xiMVWYbQXbNYtky7UzY/d4Osva2rMwuu5fEr3GHj1xduSQlmCHvM5GfabOGOVwU2Pn6F81ABcfGQAddwd2bi773EsYt4eECADYId79uv++MFsvvr2YKyptviqEMK93Q1kiy/IIIySNhklTL+3BA77zqgcPfGhj7+Efj4513OKkxd9eg/JkOGjQDE+5+/3SJTv9si2yIW7ShLXLozFOGOAsQt3uMCkMI97FuxdOzZYhsldxPuBw4fdcQxnvAePnLUmUHHzLcp7m1bEO4qsPMl3JFf5Oe71XtlwEsr5amvfk1Vnpd7KoBxEofoM5LP6XTXF6/J9vefFSyTxww8tt3fvSebJ98kR/ZVrMTAMnhHFB9LKYhwx4MBff8+H8LdydLRo3Jo0yrZ/PyNsuXlO1LiHX6Yhd/x0VRnOT0+rue1xVGHhSbcsez966+/FhynT5/u7Ljeu3evvPPOO861vSzefscd1wj7yy+/OEvoYacQ3oEPIiiDhHXjYzY4w45ZL8AVWGPDawqoF9SVbjhPIu65YqflK8YjhXsx1mr0ZZr45a9S78HF0SfMFBOHQFy6MzThDkTjFO9xAUjhnrh7qWAy9NAnm1Kz2XsOVnwN002428L7zfnbpc8LmWfcwxDuXkvlMS/7xt+2p5bvvzFvu+C9fS/3dJUSxzLrdPkJ4ndw00pZd09vZ8ZaBTaEOUQ2tsPbNsq6u3plJdwhmjEzfnDDMkdM52Op/N4FXwreycd2YOU82fTUCOdBwcENy2X/8r857kf275GNTw53VhE4Di4/cdRhIQp3UxBCGOLaFOsQjxDmEPPY3IQ74mgYFf2Il+QtiKAMEtatzCrAVXj7wRlCPJ0wN+sBQh6iPYnfHcgVOzc8i8WNwr1YajLecuBVwMHTVsWbCaaeCATi0p2hCncgGZd4jwtACvdE3D8FmYn56/dJ12eXOR99u/O/Njiz127CHYXDU178XVufqSsES+Xx0TkIevNv3PChO4hnbEFn3E07+NAd0oIbhLjbx+mwdP/Bjzc5ecdH9JCvFVsPOO/Cu7lnqiAIP8zaYsd5oWx4Txyi1vxIHT74hr99w4fd8HV3nGPG3fw7OPwlHNzh5gjjP10tq0Y2cv4qbuOfrnbcgAE+TrdmQmfn43RYuu737+BWXFVTVo9p7izhhx3Xj9MdPeq8279mXHvn/f91f7zQSQ/hD677xflLOXwXAB/G2zHnz1Vm4t3qJ+o6LEThbgpsfPEeAlAFJjA1BaKfa4SB4FQh71YvSXALIiiDhHUrGzAOirNtx02cq1092nGScJ0rdkkoQ77yQOGeL2RLxy7GXPiCOcZE3IhAXLozdOGOqlTxHuVfxcUFIIU7b95CQuDJLzZX+9Ceiv1CKkcp5xUPBtbc0qnKx/b0IUAp4ULhXlHbFO5VWW8Laz8PSNQC4mJFg+7mQxVdMp/EJfKafwp3RaL6kcK9OiZ0CYbAN6v2OO+3z11R8Q86wWIzdLEhEJfupHA/cMD5705TgGd7jkrkRgSIABEgAvlHgMK9AmMK96pcy0a4Q6BDrCMuNi+xjzAU7lXxLpQrCvdCqank5vPPf9niCHd8d4gbESga4a6z7VH/t3tcAJoin8KdNzIRIAJEIBoEKNwrcKZwr8q3oMJdvxMAHHWzhbt5bdvXOEk4csbduxYo3L2xoY8/BMa9vU56/3mFv8AMVfQIxKU7Q51xj0u0gx1xAUjhXvT3JgtIBIhAAhGgcK+oFAr3quS0hbUpujWk+S0BFe46267XmF3XpfImxrpkXv3UZhKOFO7etUDh7o0NfTIjcOjIUen5/DK559jf7GaOwRDFjkBcujM04R6naAc54gKQwr3Yb02WjwgQgSQiQOFeUSumqExiPSFPQQRlkLBu5Q0q3GFDxTjEOr7yj7/bwwf/IM6Br/nlf4R3c3PLS9RuuWIXdX6jTI/CPUq0iy+teev3Osvk/2vxzuIrHEuUFQJx6c5QhHvcoh2IxwUghXtWfGckIkAEiEBOCBSacM+psAUeOYigDBK2wGEJPfvEzhtSCndvbOiTGYHXftzuCPdfdx/KHJghSgKBuHRnzsI9CaIdDIkLQAr3krg/WUgiQAQShgCFe8IqJE12ggjKIGHTJFmSXsTOu9op3L2xoU9mBPCXvZ0mLc0ckCFKBoG4dGdOwn3Lli0yZcoUifpDdG6siAtACne32qAbESACRCC/CFC45xffMK0HEZRBwoaZx2KwRey8a5HC3Rsb+mRGoP+LK+Xmd9ZnDsgQJYNAXLozJ+G+ceNGifK/2tOxIS4AC124175/kXy6dFc6aOlHBIgAEUgcAhTuiasSzwwFEZRBwnomWKIexM674incvbGhT3oElm05IHUfWCQz/rY9fUD6lhQCcenOnIR7php6+eWXZeLEib52hM1liwvAQhfu17+5VsoeXix8bycX9jEuESACUSNA4R414tmnF0RQBgmbfY6KMyax865XCndvbOiTHoF3Fu5w3m9fvuVA+oD0LSkE4tKdeRXuUdZgXAAWunBf+usB6fn8chn08sooq4tpEQEiQARyQoDCPSf4Io0cRFAGCRtpIQogMWLnXUkU7t7Y0Cc9Ag9/sklaPvFz+kD0LTkE4tKdFO4HDogpvnM5RyUW4oa/t8CS+fs/2liI2WeeiQARKEEEKNwLp9KDCMogYQsHgWhySuy8caZw98aGPukRuPyVVTJi5pr0gehbcghQuOdY5XEBaAr9QhXugH7il7864h1LgrgRASJABJKOAIV70muoMn9BBGWQsJUp8AwIEDtvHlC4e2NDH28E/v/23sRNjuLK1/5/7p373O+OjXfGxh57bI93D/Z4bMbYHjzGGi9gwMZm8cZmkEACsQg1IJAEAiHEIoEQAiQBkoxASIDU3ep93/d9O99zspRNqFXdFVV1qjIr883nqa7szMhY3jiR5/wqcukamZHP3FUrD7/Zv3wi9qSSQFS6kxl3ZtwXB5ze764z7y2DM4vbWIEABCAQRwII9zj2SvY65SMo80mbvbT0boXd8n2PcF+eDXuWJ7C/fjSIi9/tnFw+EXtSSQDhXmS3RwUwKTPuij+83/17DzcW2RscDgEIQKC0BBDupeVrmXs+gjKftJZ1TEJesFu+FxHuy7Nhz/IENrzRK59YW7t8AvaklkBUupMZd2bczxp04f3uf97D+yrPAsM/EIBArAiUW7g3NjbJ/Px8rBhUQmWUmbLzXeDsS+rsdPlyPvvo5P+HcE9+H5eihVc83S6X8PDmUqCt+DwR7kV2YVQAkzTjHnZBeL/79ncGw018QwACEIgVgXIL966ubhkbG4sVg0qojDJTdr4LnH1JnZ0uX85nH538/xDuye9j6xaOTM3Lv26ol/Wv9VpnTX4JIBCV7mTGnRn3rMMnvN+9pmcq6342QgACEIiSQLmF+9TUtLS0tMro6JjMzc1F2fSKKFsZjY6OBsyUne8CZ19SmXSFcs6vlMpPjXCv/D4sdwvebBkP7m9/o5EfbMvNvhLKQ7gX2UtRAUzijLt2RffITPB+969urC+yZzgcAhCAgD2Bcgt3bYGKSp0R1su59X5iPsszUEbKKh/RHloJnJfnutTmiuEc8k7DN8I9Db1s28Ytbw0Ewn1ydsE2Y3JLBIGodCcz7sy4LzuAwvvddfadBQIQgECcCEQh3OPUfuoCAQj4E0C4+7MiZYbAtbs7hIc1Yw3LEUC4L0fGc3tUAJM64x5iD+933/R33mEZMuEbAhCIngDCPfo+oAYQqBQCoXAfGhoKnlUxOTkpGr/Nzs7K7NxccPuLPuAvPK9USruoZ2kIzM0vyHc2NcotL/s/n6M0NSHXuBKISncy486Me84xEd7vfrx9ImdaEkAAAhAoB4EwwNZgW+/z1Y8G4RqMa1CuD+vSIL2js0s0aGeBAATSSwDhnt6+L6Tl73dNBpfJ76keLuRwjkkBAYR7kZ0cFcCkz7hrt0zOzAf3u392/ekie4nDIQABCNgQQLjbcCQXCKSBAMI9Db1s18YdJwYD4d4zOmuXKTklikBUupMZd2bcvQbS/rrR4CT2P0+2eqUnEQQgAIFSEkC4l5IueUMgWQRc4T4+Ph5clcOl8snqY8vW3PxSl3yjqsEyS/JKGAGEe5EdGhXANMy4h10T3u/OOy1DInxDAAJREVgq3PU+VfdSeQ3OuVQ+qt6hXAjEiwDCPV79Effa/Ghrk+jD6VggsByBqHQnM+7MuC9nk1m3h/e7H2kez7qfjRCAAATKQQDhXg7KlAGBZBDIJdz1WRk8nC4ZfV1sKxr7p4MrTJ88MVRsVhyfYAII9yI7NyqAaZpxD7vowoca5aO314T/8g0BCECg7ARyCfexxRn3Th5OV/beoUAIxIuACvfOrq7gKhw9N7hPlZ+by4h2hHu8+iyq2uypHgmEe0P/dFRVoNwKIBCV7mTGnRn3vIfHsbaJ4KSmlxKxQAACEIiCwFLhrsF3eKn81NSUaHA+PDwcBOsatLNAAALpJRAKdz0n6LlBzxEzMzPBOQPhnl67yNbytQd65Av38DDmbGzY9gEBhPsHLApaiwpgGmfctYPC+93/9nJXQf3FQRCAAASKIeAK9/CVcCrcNRjXoHx8fCIQ7l3d3cy4FwOaYyGQAAIq3PVcoMJdzw1nC/e5sy6T13MLS3oJrNreKr9+igcxp9cC/Foele5kxp0Zdz8LzZIqvN/91brRLHvZBAEIQKB0BELhrt+hcNd3uX8g3MdlZGREurq65f33TwZpSlcbcoYABOJKQM8Peg7Qc4GeE/TBlaFw13OGfrhMPq69V956dY/OyifX1simv/eXt2BKqzgCCPciuywqgGmdcQ+7S+93/9Dqapma5RfqkAnfEIBA6QmsJNz1vDwxMSGjo6PS09MrtbWnpb9/oPSVogQIQCB2BHTs6zlAzwV6TtBzg54j9AodhHvsuivSCh2oz7z6+HjHRKT1oPD4E4hKdzLjzox7UaOjvm8qEO4XPsT7LosCycEQgEDeBELxHs64L30l3OjomPT19UlTU3Mw46YBvKZlgQAEkk9Ax7qOeZ1t13OAngv0nOA+mE7PGcy4J98WfFu48XCfnH9nrW9y0qWYAMK9yM6PCmDaZ9y12x59eyAQ79c/zzsvizRjDocABPIg4Ar3jHjPPKBOL5fX4HxsbEwGBwelq6tLGhoapLq6Jgji9X5XPjDABpJtAyrYdczr2NdzgJ4L9Jyg5wY9R7iz7Xr+CM8neZyCSJowAlc+3SaXbGtOWKtoTikIRKU7mXFnxt3EnsP73Z8/NWySH5lAAAIQyEUgDLT1OyPcM7Nnegms3sOql8SOjIxKb2+ftLe3S3Nzs9TV1Uvt6TqpqakNgnoN7E+dqpaTJ0/xycJgz54X5fjxE0V9NA/4Yl+ltAEdwzqW9aNjW8e4jnUd8zr29Ryg97frOUHPDctdJq/nEpZ0EhidmpMv3Fsndx3sSScAWp0XAYR7XrjOTRwVQGbcP+iL8H73wYm5DzayBgEIQKBEBLIL9/lgJi18SJ3OsA0NDQeBu866tbW1SXNLizQ1t0hjc7M0NjUFn4bGRsn2qW9okDR/Xnn1VamuqSnqo3mkmSFttxtD2caobgvHsY5pHds6xnWs65hX0a7nAD0XhA+lywj3zLmC2fYSnaArLNu3WjOvOn69YazCak51oyAQle5kxp0ZdzN77xubDS6Z//KGOrM8yQgCEIDASgSyi/e5YEbNFe8626aXyup9rvqQKn3CtH46u7qCT0dnp3R0dvFZwuCNQ4eluaW1qI/mAVtsq5Q2EI7jcFzrGNexrmNex/65ov3ce9uZbV/pTJv8fVveGpDz1lTL+AzPQUl+bxffQoR7kQyjAsiM+9kdt+vkcCDe9dJ5FghAAAKlJrBUuC+9ZD4U73qJrAbv+nAqfbK0vs85/AwNDUm2jwb9af+8/fbb0t3dXdRH80g7R9pvM5ayjdNwWzie9VvHuI51HfPh5fF6Llh6ibw7245wL/XZOt75X/9Cp/xgc1O8K0ntYkMgKt3JjDsz7uaDQB9Sp6+I23Fi0DxvMoQABCCwlMBS8R4+dEq/NVAPxbs+lEo/Gsjru5zHx1XMj8vYeOaj2zTQ5/MBgxMnTsjAwEBRH80Dph8whYUNi8wY1nHsjuHM2NYxHo73sy+P/2CmHdG+9Eya3v/nF0S++UCD3LyvK70QaHleBBDueeE6N3FUAJlxP7cvdEt4v3vH8Ez2BGyFAAQgYETAFe66ftas+5nXPYUCPjxnazAffsIAn+/MDxsuh/fee2/xygR3RjOfdc3DzZP1cznDpHgm4XjW73Cch7PszLQbnWwTms3JrslgwmlP9UhCW0izrAlEpTuZcY/hjPuxYydk1aqrgo+uL7dUVW0R/cRxmZlbCE6Cn76L92HGsX+oEwSSRiCbeP9AwGceQhXOwIciXoN6/YRBPt/T57A4efLkmcuO9dLjwj6aB2zPZQsTWybheM4m1ufmPngQHTPtSTv7F9+ene8OBTFr9wiTTcXTTEcOCPci+zkqgK7j1ToUu+iv7uvXV0ljY7MMDg7J6tV3B99L8929e2+QbuvW7Ut3xeb/1xrGghPhL3e0xqZOVAQCEEgugZXF+5lXxZ2ZgQ8vp1cRz2d5BtXV1WduK8hckuxenuy7rnnAeHnGsLFlszi2l4x1FexLRbueM1ggcMu+Lvn2gw2AgIA3gah0JzPuMZtxV8Guwl0FvC46o7501j1Mc+pUrcRZuGv9V7/SHYj3rW8PeA8GEkIAAhAolMBS8a7/hwG7fodB/dnf8zKr98PzOYeBvgpuXJ8JUMRH84At9lVKG8jMqJ87vt2xv/TcUOg5huOSR+CizU3yh10dyWsYLSoZgYoV7i0tLbJlyxa57777vD+aXo+zXKICaD3jrqLcFeM6s+4Kd3cWfmnaQniOj09KqT8XPlgfiPf324ZLXlap20L+pbcXGMPYwgbGxvTBc2d/RkfHJddnZGRM4vg5+4eGzNUD5dims+X6oK9iPppHOepKGeWzizixdsV5tvWlgl3/Z4FASKB5YDqIUbcf54HKIRO+cxOISncWPeOuIryjI79fqTS9Hme5RAWwFMJ9pRl39/738D74uN7n7vavPmX+w6ur3U2sQwACECgpgWwBe7Zt2YJ9tmUuK66pqSn6wXKaBzwzPOFQWg7ZxvfSbSU96ZB5xRHYU515jXF933TF1Z0KR0cgKt1ZtHDXmfZClkKPW66sqABaC3ffe9yVg8WM+3I8rbef6Mg8sfOSbbZXWljXk/wgAIFkElgavPP/gvgwqK2tXXz6vvvU7nzWNQ+fskjj1ydwyp9TMs9qtMqCwJ0He+XLG+ossiKPFBGISneaCvfm5mbZvHmz9yXzKt6tLpuPCqC1cFebd2fVw8vk9ZJ5fVBdeO+7pqsk4a713Xi4L7gcqepIX4qGNk2FAATiSADx4yd+EO5+nLCn+HGK43mHOsWPwH8/3iKX7WyLX8WoUawJRKU7TYW7ivaoLpuPCmAphHusLbXIyl2yrTlzv3tX5uF7RWbH4RCAAAQgUEICFr5V82CBAAQgEDcCPaOz8pHbq+VBJpTi1jWxr4+Fb3Q1pO+6qXAv9PL3Qo9zezUqgC5oghO3R5Zf1/vd9TM4Mbd8IvZAAAIQgEDkBCx8K74x8m6kAhCAQBYC4WuL32mfyLKXTRBYnoCFb3Q1pO86wj1mr4Nb3kSSsyd8gqeK96dO8BTP5PQsLYEABJJGwCI4QbgnzSpoDwSSQaDqcJ9csJ4rgpLRm+VthYVv9BXrbjqEO8K9vJZ+prSJmXn5yv11wcz7r5/i3qJIOoFCIQABCOQgYBGcINxzQGY3BCAQCYHLd7bJpU/w0ORI4Fd4oRa+0RXkvusId4R7pEPnjv09gXjX2fe63qlI60LhEIAABCBwNgGL4AThfjZT/oMABKInMD49H8y261PlWSCQLwEL3+gr1t10CHeEe762ap7+vc7Mq+JUvG88xBPnzQGTIQQgAIECCVgEJwj3AuFzGAQgUDICb7WOBxNHB+tHS1YGGSeXgIVvdAW57zrCHeEem1H1ix1twUn0B5sbZXp2ITb1oiIQgAAE0krAIjhBuKfVemg3BOJL4NG3B+Tjd9TI2PR8fCtJzWJLwMI3+op1Nx3CHeEeq0Hx9LtDi5fOH24ai1XdqAwEIACBtBGwCE4Q7mmzGtoLgfgTuPb5Drl4a3P8K0oNY0nAwje6gtx3HeGOcI/dgGgfnpEvb8g8uO6WfV2xqx8VggAEIJAWAhbBCcI9LdZCOyFQGQQWFkS+en+93PQSMWZl9Fj8amnhG33FupsO4Y5wj99oOFOjNa9mHlz3pfvqpHVwJrb1pGIQgAAEkkrAIjhBuCfVOmgXBCqTQHX3VHB15/OnhiuzAdQ6cgIWvtEV5L7rCHeEe+TGv1IFjjRnHh6iD6575r2hlZKyDwIQgAAEjAlYBCcId+NOITsIQKAoAuFtmV0js0Xlw8HpJWDhG33FupsucuE+OTkpGzZsKLrnowLowiQ4Kbobs2YwPDknq7a3Br+OXvlMu+glTiwQgAAEIFB6Aha+Fd9Y+n6iBAhAwJ/ALfu65bubGv0PICUElhCw8I2uhvRdL7twb25uls2bN8t99923+HniiSeW4Mj/36gAuqAJTvLvt3yOeOKdwUC8n7emWvQ1HiwQgAAEIFBaAha+Fd9Y2j4idwhAID8C//5wo1z3fGd+B5EaAg4BC9/oakjf9aKF+5YtW6SjoyNoiorxXIuK9q6uzMMgRkZGZOvWrdLYWPyvXlEBdEETnOTq/eL3v9sxIXrPu146X3Wkv/gMyQECEIAABJYlYOFb8Y3L4mUHBCBQZgLNA9NBDPn4O4NlLpnikkTAwje6GtJ3vWjh3tLSIirewxn0XJ0SivtQtL///vu5DvHaHxVAFzTBiVdXFZ1odn5BbnulOzjx/vixZuke4cF1RUMlAwhAAAJZCFj4VnxjFrBsggAEIiGwt2YkiB9P905FUj6FJoOAhW90NaTvetHC3cUfinJ329J1TWMt2rWMqAC6oAlOlvZ2af/ffXI4OPnq7Pue6pHSFkbuEIAABFJIwMK3WvrG3bv3SlXVlsMCyxsAACAASURBVKw9oftWrboq+FxxxXXS2Mg7mrOCYiMEUkzgroO98rWN9SkmQNMtCFj4RldD+q6bCnf3svlsUPRBdCrc9fJ4q5n2sJyoALqgLYOTsF18r0ygoX9aLj3z4LqbeR/nyrDYCwEIQCBPAha+1co3hsJ8JeF+7NiJPFtIcghAIE0Efvxos+iDjlkgUAwBC9/oakjfdVPhvvSyeRXp7kefHq8PorO4p30p7KgAuqCtgpOlbeP/3ATufb03mH3XX1Hf65zMfQApIAABCEAgJwEL32rpG3UWfevW7VnrrYI+nHFfTtxnPZCNEIBAKgj0jc0FsWLV4b5UtJdGlo6AhW90NaTvuqlwLx2e3DlHBdAFbRmc5G4xKZYS2F8/Jl+4N/PgukeODizdzf8QgAAEIJAnAQvfaukbVxLuYdP06r7Vq++WYmbfJyamhA8MsIFk2cAr1Zm3Ex2qH2Z8J+QcF573y/1t4RtdDem7jnCfnhZfWLnSWQYn5TbApJTXPTor1+zuCH5RvXxnmwyMzyalabQDAhCAQNkJWAQnlr7RR7grJL2sXj8sEIAABEICDxzpl8/fczr8l28IFEzAwjfm0pXZ9iPcEe4FG22cD9x8tD8Q759cWyMvnx6NpKqTM/PSMjgtb7VOBA/P28xVAJH0A4VCAAKFE7AITsoh3AcHh+TQoTeDhuqM+/r1VTycrvBu50gIJJLAL3a0yi+ebE1k22hUeQlY+MZswjzXNoQ7wr28ll7G0o62jMsPtzYHAn71q91mJY9MzUt937QcbhqTZ98bkgeO9Mkt+7qDh51cvLVZ/nVDvXzk9uqgXH3ivfs50jxmVg8yggAEIFBqAhbBiYVwDy9/D+9hD58ar7Pqell8V1e37Njx3OI97sy2l9oyyB8ClUVgfGZezl9XK/pUeRYIFEvAwjfmEunZ9iPcEe7F2m6sjx+bnpebXuoKxPNFm5ukpmf593b2j89KdfeUHKwfkx0nBmXDoT65YW+XXPZUm/xgc6N88b7M/fOuENf1T6ytka9vrJdLtjXLH3Z1yNr9PfLo2wOyr3ZE3u2YXHzP/D/dWSu/38WTTGNtMFQOAhA4i4BFcGIh3M+qFP9AAAIQyJPAsbaJIBY8UM8ESp7oSJ6FgIVvzCbMc21DuCPcs5hj8jY9dWJILlh/Ojhp3/1ar/x5T5foJVPfe7hR/vnuzPalgvxT62rlmw80yE+3tQT3za89kF2Q+9L6/a7Mvfe+6UkHAQhAIGoCFsEJwj3qXqR8CEDgsbcHghn3kak5YECgaAIWvjGXSM+2H+GOcC/aeCslg1Pdk3LpEy1y/p218u0HG+TSJ1rl2uc75M6DvaIn9KUz5Nbt2nVyOPjhoLZ3+Vl/6zLJDwIQgEAxBCyCE4R7MT3AsRCAgAWBq59rl/96rNkiK/KAgFj4xmzCPNc2hDvCneFXJgLDk5n3h972it399mWqOsVAAAIpJWARnCDcU2o8NBsCMSKgtzve+jLxV4y6pKKrYuEbc4n0bPsR7gj3ih44lVZ5fZqpzvizQAACEKgEAhbBCcK9EnqaOkIguQRquieDKx71ykcWCFgQsPCN2YR5rm0Id4S7hf2ShycBvSRf76XvHeP98p7ISAYBCERIwCI4QbhH2IEUDQEIyDPvDcuHV1dLxzCxF+ZgQ8DCN+YS6dn2I9wR7jYWTC5eBNqGZgLhXnWk3ys9iSAAAQhEScAiOEG4R9mDlA0BCPz1xS75/iNNgICAGQEL35hNmOfahnBHuJsZMRn5EdB3y3+9qt4vMakgAAEIREjAIjhBuEfYgRQNAQgEDyT+855OSEDAjICFb8wl0rPtR7gj3M2MmIz8COj74fVy+dn5Bb8DSAUBCEAgIgIWwQnCPaLOo1gIQEBaz1zpuO3tAWhAwIyAhW/MJsxzbUO4I9zNjJiM/Aic7Mo8JGXHiUG/A0gFAQhAICICFsEJwj2izqNYCEBAXqoZCSZLanp4FS/mYEfAwjfmEunZ9iPcEe52VkxO3gS++UCD/OhR3ifqDYyEEIBAJAQsghOEeyRdR6EQgICI3L6/R771YAMsIGBKwMI3ZhPmubYh3BHupoZMZn4EVr/SHfwC7JeaVBCAAASiIWARnCDco+k7SoUABEQu2twkV+/qAAUETAlY+MZcIj3bfoQ7wt3UkMnMj8CR5vFAuL9aN+p3AKkgAAEIREDAIjhBuEfQcRQJAQhI//hsEGs99Pc+aEDAlICFb8wmzHNtQ7gj3E0Nmcz8CXzu7tNy2c42/wNICQEIQKDMBCyCE4R7mTuN4iAAgYDAG41jgXB/q3UcIhAwJWDhG3OJ9Gz7Ee4Id1NDJjN/Atc/38nl8v64SAkBCERAwCI4QbhH0HEUCQEIyH1v9MqXNtQJL/HBGKwJWPjGbMI81zaEO8Ld2pbJz5PAi9XDgXA/3jHpeQTJIAABCJSXgEVwgnAvb59RGgQgkCFw6RMtXNmIMZSEgIVvzCXSs+1HuCPcS2LQZJqbwPTsgpy3plr+8mJn7sSkgAAEIBABAYvgBOEeQcdRJARSTmBydkE+dkeN3PN6b8pJ0PxSELDwjdmEea5tCHeEeynsmTw9Ceg97ireWSAAAQjEkYBFcIJwj2PPUicIJJvAO+0TwVWN+3kIcLI7OqLWWfjGXCI9236EO8I9IpOnWCXw5PHBwLE0D84ABAIQgEDsCFgEJwj32HUrFYJA4glsOTogF6w/LcOTc4lvKw0sPwEL35hNmOfahnBHuJff2ilxkUDPaOZVJetf41KuRSisQAACsSFgEZwg3GPTnVQEAqkhcMXTbfKzJ1pS014aWl4CFr4xl0jPth/hjnAvr6VT2jkE/uuxFvnCvXXnbGcDBCAAgagJWAQnCPeoe5HyIZA+AvrK3TWvdqev4bS4LAQsfGM2YZ5rG8Id4V4WA6eQ5Qk8eKQvuFyey7mWZ8QeCEAgGgIWwQnCPZq+o1QIpJVAbe9UEFftOjmcVgS0u8QELHxjLpGebT/CHeFeYtMm+1wE6s44mC1vDeRKyn4IQAACZSVgEZwg3MvaZRQGgdQT2PnuUPBE+fYhnh+UemMoEQAL35hNmOfahnBHuJfIpMk2HwLf2dQo393UmM8hpIUABCBQcgIWwQnCveTdRAEQgIBD4PoXOuWHW5udLaxCwJaAhW/MJdKz7Ue4I9xtLZncCiKw9kBPcFlXQQdzEAQgAIESEbAIThDuJeocsoUABLIS+NrGerlhb2fWfWyEgAUBC9+YTZjn2oZwR7hb2C95FEngrdbM+0afP8X9WEWi5HAIQMCQgEVwgnA37BCyggAEViTQNjQTTITo63ZZIFAqAha+MZdIz7Yf4Y5wL5VNk2+eBL50X51cyqtL8qRGcghAoJQELIIThHspe4i8IQABl8DempFAuFd3T7qbWYeAKQEL35hNmOfahnBHuJsaMpkVTuCvL3ZyuXzh+DgSAhAoAQGL4AThXoKOIUsIQCArgb+93C363CAWCJSSgIVvzCXSs+1HuCPcS2nX5J0HgVfrRgPhfqhxLI+jSAoBCECgdAQsghOEe+n6h5whAIGzCXzv4Ua5dnfH2Rv5DwLGBCx8YzZhnmsbwj2Gwv3YsROyatVVwUfXly7u/qqqLUt3838FE/jE2hr5/S4cTgV3IVWHQKIIWAQnCPdEmQSNgUBsCQxMzAUTIJuP9se2jlQsGQQsfGMukZ5tP8I9ZsJ9cnJS1q+vksbGZhkcHJLVq+8OvkMz120q1jWdmzbcz3dlE7jq2XYul6/sLqT2EEgUAYvgBOGeKJOgMRCILYE3GseCGOpo60Rs60jFkkHAwjdmE+a5tiHcYybcVbCrcFdRrouK9Gyz7rovm7BPxnBIbyuefncocDqnuqfSC4GWQwACsSFgEZwg3GPTnVQEAokmsP5gj3x1Y73MzS8kup00LnoCFr4xl0jPth/hHkPhvnXr9kWL3L177znCXUW9zsRfc82NZ83GLx6Ux8rIyJjwiQ+Djr7Mfe43vNBGv2Cb2AA2sGgDeZzWTZNaBCcId9MuITMIQGAZAv/1WLNc8XTbMnvZDAE7Aha+MZswz7UN4R5D4Z7PjLuK9+Vm5O3Mk5zKSUBfCXf+nbXlLJKyIAABCGQlYBGcINyzomUjBCBgSGBqdj64YvH+Q32GuZIVBLITsPCNuUR6tv0I95gJd/e+9WyXwuu2HTueW7SilS6lX0zESkUR0IeqfGh1tXQMz1ZUvaksBCCQPAIWwQnCPXl2QYsgEDcCx9sngtjpQP1o3KpGfRJIwMI3ZhPmubYh3GMm3NW23afGh7Ppesm8Xh7f1dUdCPfwqfM8VT55Z4O2oZnA+VQd5lfj5PUuLYJAZRGwCE4Q7pXV59QWApVI4KG/98vn76mTocm5Sqw+da4wAha+MZdIz7Yf4R5D4V5htkt1S0Dg+480ydc21pcgZ7KEAAQg4E/AIjhBuPvzJiUEIFAYgV/taJVV21sKO5ijIJAnAQvfmE2Y59qGcEe452mqJC8Hgbtf6w1m3cdneDJqOXhTBgQgkJ2ARXCCcM/Olq0QgIAdgX+6s1bWHeixy5CcILACAQvfmEukZ9uPcEe4r2CW7IqKwPtdk4Fwf/L4YFRVoFwIQAACYhGcINwxJAhAoJQETvdOBTHTnuqRUhZD3hBYJGDhG7MJ81zbEO4I90UjZCVeBPRdpBdvbY5XpagNBCCQKgIWwQnCPVUmQ2MhUHYCT7wzKJ9aVyutgzNlL5sC00nAwjfmEunZ9iPcEe7pHHEV0Opb9nUFvyBXQFWpIgQgkFACFsEJwj2hxkGzIBATAr/f1SE/fpSJjph0RyqqYeEbswnzXNsQ7gj3VAywSmzkoaaxQLjvq+XSr0rsP+oMgSQQsAhOEO5JsATaAIH4EvjShnrRyQ4WCJSLgIVvzCXSs+1HuCPcy2XjlFMAgc/cdVoue6q1gCM5BAIQgEDxBCyCE4R78f1ADhCAQHYC7cOZV+g+/e5Q9gRshUAJCFj4xmzCPNc2hDvCvQTmTJZWBP6wq4PL5a1gkg8EIJA3AYvgBOGeN3YOgAAEPAnsPjkcxEnV3VOeR5AMAsUTsPCNuUR6tv0Id4R78dZLDiUjoE9I/dDqajnaMl6yMsgYAhCAwHIELIIThPtydNkOAQgUS+CGvV3yvYcbi82G4yGQFwEL35hNmOfahnBHuOdlqCQuL4GZuYVAuP95T2d5C6Y0CEAAAiK8Dg4rgAAEYk3gwoca5U8vECPFupMSWDmEe5GdGhVA95cRZhWK7EQOz0rgFzta5cOrq7PuYyMEIACBUhKw8K34xlL2EHlDIL0EBifmgsmNx94eSC8EWh4JAQvf6GpI33Vm3Jlxj8TgKdSfwOPvDAaO6XQv92/5UyMlBCBgQcAiOEG4W/QEeUAAAksJ7K/PvH3nWNvE0l38D4GSErDwjb5i3U2HcEe4l9Swybx4An1js4Fwv+tgb/GZkQMEIACBPAhYBCcI9zyAkxQCEPAmcMf+bvlmVb3obYUsECgnAQvf6Apy33WEO8K9nHZOWQUSuHhrs3z+ntMFHs1hEIAABAojYBGcINwLY89REIDAygR+uKVJfvds+8qJ2AuBEhCw8I2+Yt1Nh3BHuJfAnMnSmsDGw33BrHvP6Kx11uQHAQhAYFkCFsEJwn1ZvOyAAAQKJDB95uG9Dx7pLzAHDoNA4QQsfKMryH3XEe4I98KtliPLRqC+bzoQ7o8cxUGVDToFQQACPFUeG4AABGJJQF+Tq6/LfaNxLJb1o1LJJoBwL7J/owLo/kLCrEKRncjhKxL41oMN8t1NvKt0RUjshAAETAlY+FZ8o2mXkBkEICAi9x/qky/eWycDE3PwgEDZCVj4RldD+q4z486Me9mNnQILI7D6le7g12W9PIwFAhCAQDkIWAQnCPdy9BRlQCBdBH6+vUV+taM1XY2mtbEhYOEbfcW6mw7hjnCPzSCgIisT0Ned6GVhz7w3tHJC9kIAAhAwImARnCDcjTqDbCAAgUUCH19bI3e/xtt2FoGwUlYCFr7RFeS+6wh3hHtZDZ3CiiPw+Xvq5GePtxSXCUdDAAIQ8CRgEZwg3D1hkwwCEPAicKp7MpjI2Fc76pWeRBCwJmDhG33FupsO4Y5wt7Zl8ishgT++0Bk4qxIWQdYQgAAEFglYBCcI90WcrEAAAgYEtr41IJ++q1bah2cMciMLCORPwMI3uoLcdx3hjnDP31o5IjIC++tGA+Gu3ywQgAAESk3AIjhBuJe6l8gfAukicOUz7fJTrj5MV6fHrLUWvtFXrLvpEO4I95gNBaqTi8BHb6+Rq5/ryJWM/RCAAASKJmARnCDci+4GMoAABBwCetvg6le7nS2sQqC8BCx8oyvIfdcR7gj38lo6pRVN4PKdbVwuXzRFMoAABHwIWAQnCHcf0qSBAAR8CLQMTgcx0K6Twz7JSQOBkhCw8I2+Yt1Nh3BHuJfEoMm0dAT0qfL6dPnj7ROlK4ScIQABCIiIRXCCcMeUIAABKwJPvzckH7m9Wur6pq2yJB8I5E3Awje6gtx3HeGOcM/bWDkgWgJj0/OBcL9lH5eJRdsTlA6B5BOwCE4Q7sm3E1oIgXIR+NOeTrloS1O5iqMcCGQlYOEbfcW6mw7hjnDPapBsjDeBS7Y1y/nrauNdSWoHAQhUPAGL4AThXvFmQAMgEBsC36hqkBv2dsWmPlQknQQsfKMryH3XEe4I93SOuApv9eajA8Gse1M/l4pVeFdSfQjEmoBFcFJO4X7s2AlZvfpumZycjDVXKgcBCORPoH98Loh9th8fzP9gjoCAIQEL3+gr1t10CHeEu6EZk1W5COi7S/U+9/ve6C1XkZQDAQikkIBFcFIu4a6ifdWqqxDuKbRTmpwOAvtqM6/EfbeTH+bS0ePxbaWFb3QFue86wh3hHt9RQc1WJPDdTY3y1Y31K6ZhJwQgAIFiCFgEJ+US7trOwcEhqarawox7MZ3OsWcRONoyLnziweDWl7vkwocaZHpu4aw+4h8IlJuAhW/0FetuOoQ7wr3ctk55RgTWv9YbzLoPTswZ5Ug2EIAABM4mYBGcVKJwHxkZEz4wUBv47dMtga/Vq9z4RM9A+4OxydgMbeBsj1W+/yx8oyvIfdcR7gj38lk5JZkSONU9GQQRj709YJovmUEAAhAICVgEJ5Uo3MP2851uAjNzC3LhQ41y68vdzLrH6MqDdFslrY8DAQvf6CvW3XQId4R7HOyfOhRI4Esb6uSHW3ktSoH4OAwCEMhBwCI4QbjngMzu2BJ4rzPzA/lLNSOxrSMVgwAEyk/Awje6gtx3HeGOcC+/tVOiGYGbXuoKZt0XuN3LjCkZQQACHxCwCE7KJdz13nZ9OF340YfVsUCgGAJPHh8MfKw+zZwFAhCAQEjAwjf6inU3HcId4R7aIN8VSOBI83gQVDx/argCa0+VIQCBuBOwCE7KJdzjzpL6VR6BG1/qkq/xENjK6zhqDIESE7Dwja4g911HuCPcS2zaZF9qAp9aVyu/fqqt1MWQPwQgkEICFsEJwj2FhpOQJuutaNe/0JmQ1tAMCEDAioCFb/QV6246hDvC3cqGySciAlc/1x7MukdUPMVCAAIJJmARnCDcE2wgCW5afd+UfPT2annqxGCCW0nTIACBQghY+EZXkPuuI9wR7oXYK8fEiMCLNSOBcH+jcSxGtaIqEIBAEghYBCcI9yRYQvra8PypjG9t6p9OX+NpMQQgsCIBC9/oK9bddAh3hPuKhsnO+BOYm18IhDuX88W/r6ghBCqNgEVwgnCvtF6nvkrgjv09csH608CAAAQgcA4BC9/oCnLfdYQ7wv0cY2RD5RH4+fZW+fCa6sqrODWGAARiTcAiOEG4x7qLqdwyBC7d3iqX7+T5McvgYTMEUk3Awjf6inU3HcId4Z7qgZeUxm8/88qa97smk9Ik2gEBCMSAgEVwgnCPQUdShbwIdI3MymfXn5ZHjg7kdRyJIQCBdBCw8I2uIPddR7gj3NMxwhLeSn3H7IdWV8uaV3sS3lKaBwEIlJOARXCCcC9nj1GWBYH9daOBT32vkx/DLXiSBwSSRsDCN/qKdTcdwh3hnrSxlNr2/GBzk3z+Hu7HS60B0HAIlICARXCCcC9Bx5BlSQlsONQnH7u9pqRlkDkEIFC5BCx8oyvIfdcR7gj3yh011PwsAlWH+4IZgvahmbO28w8EIACBQglYBCcI90Lpc1xUBH7zdJtcsq05quIpFwIQiDkBC9/oK9bddAh3hHvMhwbV8yXQ2D8dCPeqI/2+h5AOAhCAwIoELIIThPuKiNkZMwLDk3Py1Y31cvdrvTGrGdWBAATiQsDCN7qC3Hcd4Y5wj8sYoB4GBL62sV6+s6nRICeygAAEICBiEZwg3LGkSiLwZst48CP4oaaxSqo2dYUABMpIwMI3+op1Nx3CHeFeRjOnqFITuP3VniDgGJueL3VR5A8BCKSAgEVwgnBPgaEkqImbj/YHfnRqdiFBraIpEICAJQEL3+gKct91hDvC3dKOyStiAsfbJ4KA48njgxHXhOIhAIEkELAIThDuSbCE9LTh2uc75XuPNKWnwbQUAhDIm4CFb/QV6246hDvCPW9j5YB4E7hg/Wm5ZFtLvCtJ7SAAgYogYBGcINwroquppIjMzUtwu9kt+7rgAQEIQGBZAha+0RXkvusId4T7skbJjsok8KcXOoNZ9yhrPzI1J3ce7A3q8fLp0SirQtkQgEARBCyCE4R7ER3AoWUl8H7nZOC3XqweKWu5FAYBCFQWAQvf6CvW3XQId4R7ZY0UapuTwIH60SDweKm2/IHH7pPDwWzFh1ZXB++U/+2z7fKxO2pE68QCAQhUHgGL4AThXnn9ntYa7zgxFPjP3rHZtCKg3RCAgAcBC9/oCnLfdYR7DIX7sWMnZNWqq4KPri9d3P27d+9dupv/IRAEHlc+014WEjU9U3L1c+1BmSrYV21vlaffHZLJ2QXRR/tcvrNNPrWuVg41jZelPhQCAQjYEbAIThDudv1BTqUlcNNLXcGr4EpbCrlDAAKVTsDCN/qKdTcdwj1mwn1yclLWr6+SxsZmGRwcktWr7w6+QwPX7Vu3bg/+1f033LAmSBvu57uyCYwf2yvNv/yQtN/4Hel96Pcy/OIDMv7ufpnty0+Eq1hWEV2qZX5+QfR98V+8ry4o5yv318sd+3vkZNfUOUVOzy3IL55sFb33Xl+zwwIBCFQOAYvgBOFeOf2d9pr+6NFmufb5jrRjoP0QgEAOAha+0RXkvusI95gJdxXmKtxVwOtSVbVFss266z5No/tVwLMkg8Dgs+sDwd65+mJp/f3npfG//9dZn+Zfflg6bv536d30Bxl+8UGZeO+gzPafG2Q89/5wIKithfL+ulH52RMti7Prv3qqVXadHJbZ+ZVfmzM+PS+XPtESXD7/TvtEMjqLVkAgBQQsghOEewoMJQFNbOyfDnybXi7PAgEIQGAlAha+0Vesu+kQ7jEU7uGMuhqMXgq/nHDX7cVeKj8+PiF84stgpLVOhk4clP5Xt0nPjjukc+NV0n7rRdJy9efOEvQq8Jt+eZ603fRd6ay6WlqfrQoCkOt3ni66f093jshf97TJ+etqgzy/9UCD3P5yh5xoGc4r766BMfnJo43BLP3RxqG8jsVG42uj9E15+malAKKU+yyCE4R7KXuIvK0IvHAq84N3Q9+5V45ZlUE+EIBAMghY+EZXkPuuI9xjKNx9ZtxVtOtl9OHMfDKGAa3Ih8Bsb6tMVh+R0dd3yOAzd0rvg1dL523/Ka1nRP0P/rpDPnHrO4HAb/7VR6Tjlv+Q3oevleGXNsnE+6/LbP/Kl98/9vagXLS5KRDrH7+jRq58pk32FPmk3b6xWdFLEb+6sV703ngWCEAg3gQsghOEe7z7mNplCKw90BNcFQYPCEAAArkIWPhGX7HupkO4x0y457rHXQ1JZ9kR7bmGVLr3q6jf9PyxQHS/9cQD0vvA76Tz1ouk9XefPWemvvnXH5GOv/2H9D1ynezfuVOufvR4cJzeI//9R5rkvjf6xHIGomtkVn6wuUm+9WCD1DOzkW5DpfWxJ2ARnCDcY9/NVFBEfr69RS57qg0WEIAABHISsPCNriD3XUe4x0y4q6XobPrSp8qHYr2rq1uuuebGxf2artjL5XNaJwkqkkDnyEwgwPV96u4SzNSfOiSjr22XgafXyemN18ntt90rX/3b/iD9P936llx+/X3y2G9+LM2//qh0/O370rf5ehne97BMnnxd9Phil9bBafn3hxuDV8c1D84Umx3HQwACJSJgEZwg3EvUOWRrRqB7ZFbOv7NWNr3Zb5YnGUEAAsklYOEbfcW6mw7hHkPhnlwzp2XlJnDhQ43yrxvqsha7r3ZULjvz9HmdXf/JYy3ywKuNcvroYRk5+IQM7FwrPVVXBcK95bcXnDNT33L5x6Xj1h9I3+Y/ysjLmzOX3+ch6vVBQN9+sEG+v7lJ2ocR71k7iY0QiJiARXCCcI+4Eyk+J4ED9WPBD9cnOjIPBs55AAkgAIFUE7Dwja4g911HuCPcUz3wkt74e17vDYKR7tHZoKn1fdOi9/F94d7Ma9z0dW43vNgpGrTkWvSe+MlTh86I+jukZ+OVwX3zLVd9Jouo/4R03HqR9G35k4y8skUm3n9NZntazimitmdKvl7VID/c2iQ9Z+p4TiI2QAACkRGwCE4Q7pF1HwV7Erj/UJ98al2tZ2qSQQACaSdg4Rt9xbqbDuGOcE/72Et0+/UBcDqb/vCbA/LTxz94jZvey/fo2wOil9NbLPpKusnqwzJ68AkZ3HmH9FZdKZ1/+760/jaLqP/NJ6Xjtv+Uvq1/luGXN8uxNw7Jl++tkf96rEUGxucsqkMeEICAEQGL4AThbtQZZFMyAlc+0y6XbDv3x+WSFUjGEIBARROw8I2uIPddR7gj3Ct64FD53AT+5czs+jeqGmT1K91yXzdpYwAAIABJREFUpHk890GGKWYHOoOn348ceFwGnrpdejZecWam/tOLM/V7fvVN+dwth+XiG5+SutX/HYj64PL79w7ITHeTyMLK74k3rC5ZQcCLwOjUvFe6Sk9kEZwg3CvdCpJd/9HpefnSfXWy/mBPshtK6yAAATMCFr7RV6y76RDuCHczIyajeBK4ZV+XPHViSPRVbHFb5ga6ZLLm78Hl9/sf3SSfWXNcfnrbLjl91T8vinp9R71+Wq74lHSuuVj6Hv2LjLz8iEy8u19muhplYSEdAipufZfW+pzunZJvVjXIx+6oCYL9t1rL+0NYublbBCcI93L3GuXlQ+Boy3hwZdrrjblvGcsnX9JCAALJJWDhG11B7ruOcEe4J3dU0bKKI6BXA5y/riZ4Jc9Uf6dM1mZE/cCO1dJz/2+k4+Z/l5Yrzz9X1F95vnSu+VEg6vXp9xMnXpWZrgZZmOfS+4ozgphX+D8eaQyCfL0FRT9fXubhjzFvhnf1LIIThLs3bhJGQGDLWwPBWJ6Y4UfgCPBTJAQqkoCFb/QV6246hDvCvSIHDJVOLoHXGkbl43fUiN5zuNwyN9QtU7Vvyoi+0u6pNdKz4XLpuOm7wax8OEMffrdc+U/SefuPM/fUv3RG1Hc2yMIcon45vmxfnoDOtKtg/8fbMsL9o7fXSJIDfovgBOG+vD2xJ3oC1z3fIf/xSFP0FaEGEIBAxRCw8I2uIPddR7gj3CtmkFDR9BB4tW5UPry6Wq5+bnnxvhyNuaEemTp9VEZeezK4p757w2XSfuN3pPk3nzx3pv6qT0vn7T+Rvi1/luGXNsn48VdkpqNehJn65fCmfru+iSGcbdfvf3uwIdFMLIIThHuiTaSiGze/sBCM4Vv2dVd0O6g8BCBQXgIWvtFXrLvpEO4I9/JaOqVBwJPA3pqRQCBd93yn5xG5k80N98rU6bdk9PUdMrDzjmCmvv3GC7OK+tbfXnBG1P9Jhvc+JOPHX5bpjjqRufg9KyB3y0lRLAGdVf/9rvbAJr/1QIN8Ym2NXLSlSRr6p4vNOtbHWwQnCPdYd3GqK3eyK/PmlT2nhlPNgcZDAAL5EbDwja4g911HuCPc87NUUkOgjASeP5UR73/eYyfel6v+3HCfTKqof+MpGdy5NnNP/Y3fkZYsM/Wtv1NR/2Pp2/xHGd77oIy/s0+m20+LzNm8Xm+5OrI9GgI9Y7Pyyx2tgWh/sWYkmkpEVKpFcIJwj6jzKDYnAX1wq1450z3KD7I5YZEAAhBYJGDhG33FupsO4Y5wXzRCViAQRwLPvJcJrG56qSuy6s2N9Mtk3dsy+sZOGXx6XfBKu/YbLpSWyz9xzuX3rb/7bPCgvN7N18vwiw/I+LGXZLq9VhZmkz0zG1nnlLjg+r4p+fFjzfKZu2rlUAqfOm0RnCDcS2ykZF8wAb1E/psPJPt2l4LhcCAEILAsAQvf6Apy33WEO8J9WaNkBwTiQmDHmVmRW1+O332I86MDGVF/aKcMPrNOejdeIcHl95d/fBlRf7H0PnKdDO+pkvFje2W6rUYWZqbigpp6OASOd0zIhQ81yFc31sv7nZPOnvSsWgQnCPf02EultfSHW5vlmt0dlVZt6gsBCERMwMI3+op1Nx3CHeEeselTPAT8CGw7Nhhc0njH/h6/A2KQSkX9VP0xGT38tAw+c6f0Vl0p7Td9R5ov+1gWUX+BdK7+ofQ9cp0M7dko42+/GNyPP930rky3VstMZ4PM9raKPlF/fmxQ5ifHZGGWS/NL1c36Tucv3VcXPG26ZTC9nC2CE4R7qayUfIsh0DQwHfiUJ48PFpMNx0IAAikkYOEbXUHuu45wR7incLjR5EolsOVofxBo3XWwcsT7cqxVfE/VvyOjh5+RwWfvkp6NVwavtGvOMlMfvtou6/fP/rc0/c//k+ZffzR4HZ5eqt96zb9I2x+/Ku1//bZ03Py94AeBrrWXSNddl0r3vb+SnqorpXfTH6Rvy5+kf9tN0v/kbTLw9DoZ3HWPDO2pkuF9j8jIgW3B/f6jR56TsbdelPETr8rEyddlsubvMtXwjky3nAzu65/taZbZ/g7RZwTMT4zI/PSkyHxlvw95T/WwfGpdrfz34y0yOJHu1wZaBCcI9+XOAmyPksCe6swzVOr7uI0pyn6gbAhUIgEL3+gr1t10CHeEeyWOF+qcYgKb/p4R7/e90ZdYCvNjQzLVcDwQyuMnXgmE89jfdwVCWgW1CmsV2EO77w0uzx94cnUgwFWIqyDvrbpKeu77tXSv/7l0rf0v6bzth4GA77jh36T9T1+Ttmu+KCrwm6/4lDT9+iPS9It/lMaf/cM5VwFk/aHgv/+XV7qmVf9Xmn91XvBwP31Cf+vvPy9t1/+rtP/lm8EPFJ23XiRdd/xEuvXHhHt+GTwMsPehq6Vv8/XS/9gNMrD91uDJ/4PP3S1DL2yUob2bZOTVrTL62nYZPfS0jB19Pnh+wMS7B2Sy+nDwYMGpxrOvTpgd7BK96mEhuDrBLzjX2Td9WNWVz7TL/EL0Jqa3UYwcfEJGXn8yklsqLIIThHv0dkQNziWw7kBPcFXNuXvYAgEIQGBlAha+0RXkvusId4T7ypbJXgjEkMDGQ32BuKo6nFzxHgX2hbk5mZ+akLmxIdFX5832t8tsd6PMtNXIdNN7MlX3diCSJ947EIjmsTd3y+ihnRlh+crW4LV5Q89vCK4gGNixRvofv1n6H/1LcPl/7wO/C16/1333/0j3nf8tXWt+JJ1/+37wPIC2P39DWq/9ciDuW676tOhVB02//JA0Xvp/vH4k8P6B4dL/E+Sr+Ws5+mNC63VfFi1fn0tw123rA7u6+vbtovXs2XC59DzwW+l9+Frp3/qXoD0DO1YHVyfojyb68MHw6oSR13fI2JHnglscxo+/IhPv69UJR4JbJaab3w8eUDjT3SRzwdUJvTI/PhywlvnlZ/T1doiW314gTb86T5ov/4S0/fnrMj81XlbTsAhOEO5l7TIK8yRw6RMt8uun2jxTkwwCEIDABwQsfKOvWHfTIdwR7h9YIWsQqCAC977eG4isTW/2V1CtqWohBPRefhWxc6MDMjfYFdzrP9NZL9Mtp4IrEyZr3zzr6gS9vF+F9Mj+x2Tk5UeCBwEO7boneCPAwJO3Sf+2G4NX+fU+9PvgDQF6+8AtazcH9vSX2x6S9pu+K+1//Za0/fEr0vqHLwTiufk3n5RmvTph1f9n+2PCz/5Bmv7nH6X5so8GV0DolRBt134xuNVBRXvDpZkrIfQKhpbfXSCjr+0oBGHBx1gEJwj3gvFzYIkI6CseP35HjTz0d/xHiRCTLQQSTcDCN7qC3Hcd4Y5wT/TAonHJJnDnwZ5AbG15ayDZDaV1JSVw80tdgR3plRy+y8LCfHA///z4iMyP9MvcQKfo/f4zHadFZ9j1oYT6PICJ91+T8eMvy9jRF2Ts8DPBpf56yf/wS5tk6Pn7ZfDZ9TLw1O3S/8Qt0vfoX4LZ/Z4HfifdGy6Xtj9+7YNbGH7+f6Xlqs8Ezx7wraNFOovgBOFu0RPkYUngYP1oMOaPt09YZkteEIBASghY+EZfse6mQ7gj3FMyxGhmUgncvj8j3h9/hycDJ7WPS9WuyZl5+cOujiCA17cWxG2ZHeiU5l9+WBov/Qdp+sX/k9arPyezQ+V9MKNFcIJwj5tlUZ+qw/3ymbtOAwICEIBAQQQsfKMryH3XEe4I94IMloMgECcCf3u5OxBfO07ET3zFiRN1+YBA79is/HJHq5y3plp2nxz+YEfM1mZ7W4JL/HVmfrav/PfjWgQnCPeYGRXVkSufaQveGgEKCEAAAoUQsPCNvmLdTYdwR7gXYq8cA4HYEbjxzOXOz74fXxEWO2gprVBj/7T86NFmuWD9aXmtYSylFPyabRGcINz9WJOqPATGpufln+8+LXceKO/VK+VpHaVAAALlIGDhG11B7ruOcEe4l8O+KQMCZSHwpz2dwcz786fSI967R2fl0bcH5JJtzfLdTY3SOTJbFtaVWsiJjgm58KFG+drGeuH+1ty9aBGcINxzcyZF+Qi81TYR+InXGkbLVyglQQACiSJg4Rt9xbqbDuGOcE/UQKIxELh2d+aeZX1g3amuyUQCaR+akYff7JeLtzYHAejn7j4tv3su0259B/njx3hYX7aOf6NxTL54X518/5Emaej3e697tnzStM0iOEG4p8li4t/WrW8NyEdvr5Hx6fn4V5YaQgACsSRg4RtdQe67jnBHuMdyQFApCBRD4Orn2gNBqyL2O5saRZ8a/mLNiAxMLP/O7GLKK8exTQPTUnWkPxCd2q4v3Hta9EeKp98bko7hmaAKOpt88damoO16//b07EI5qlYRZWj/f3Jdrfzs8RbpG69cOyg3bIvgBOFe7l6jvJUI/PGFTvnPLc0rJWEfBCAAgRUJWPhGX7HupkO4I9xXNEx2QqBSCbzVOiH3H+qTS59olY/dUROIWX1v76rtrVJ1uE/eqYDXANX2TIm+r15/fFCx/sV76+TPezpFbwVYSXyuPZB50r4ec6Cee7ifPDEU8NMHUvFjRn4j2iI4Qbjnx5zUpSOgP2V+vapebnqpq3SFkDMEIJB4Aha+0RXkvusId4R74gcXDYSAPkF8T/VIEKxd+FBDIOJU1H51Y71c/3xm1rp3LB6zsO93TYq+n/6bVfVBPb90X53cvK9L9tWOyvCkfx0P1I/KN87kccPeztQawaa/9wcc9fkHLPkTsAhOEO75c+eI0hA41T0ZnA9eODVSmgLIFQIQSAUBC9/oK9bddAh3hHsqBhiNhIBLoLp7SjYf7ZfLd7YFTxZXEa+fHz/WLHcd7JGjLeNu8pKv60PS1rzaLV+5PyPW9Xv1qz2yv25UJmYKvw9zem5Bwgf26Q8AJ7umSt6WOBVw18HeoF+VLUthBCyCE4R7Yew5yp7A0+9mrr7pGsncXmRfAjlCAAJpIGDhG11B7ruOcEe4p2F80UYILEtgbn4heCXYHft75KItmfvDVcR//p7TctUz7fLk8UHpPHMP+bKZFLDjaOu43LKvW/7l3rpAXOrs/9r9PXKoaUzmCtfqWWvyzHtDiz9Q6H3yaVhu3Jt5w4DeLsFSOAGL4AThXjh/jrQlcMu+ruCtEra5khsEIJA2Aha+0Vesu+kQ7gj3tI012guBFQnoTMzOd4fkD7s65EsbMjPgKuS/90iT3L6/R440Fz4bf6hpXG7Y2yWfvft0INa/XtUg6w70iN6PX+pFXxt32c62oNwfbW0SfTJ9Ehe9yiB8OOFjb/N0/WL72CI4QbgX2wscb0XgPx5pkmt2dVhlRz4QgEBKCVj4RleQ+64j3BHuKR1yNBsCfgT0nvOqw/3y020tct6azCX1n1pXG4jgx98ZlLYcAlgvd9f76M9fl3lA3jcfaAguxz/REc2r6vQ1cuGtAc++N+QHoUJS9Y3Nyi+ebA36adfJ4QqpdbyraRGcINzj3cdpqV3z4Exw7nvincG0NJl2QgACJSJg4Rt9xbqbDuGOcC+RSZMtBJJHQGdzVYjr5ZbfevCDh9ypGL/tlW55vTHzBPd9tSPy+13ti0+z/9YDDbL+tV7RByPFYdGn1f/o0cw74K9+rkNGp/wfeheH+merg74uT1+F99n1p3mSfjZABW6zCE4Q7gXC5zBTAvpKSP3Rsq43Xc/6MIVIZhCAQEDAwje6gtx3HeGOcGcIQgACBRLQy831Hnh9zdj5d9YuzmRrcPhvDzbIPa/3yukYB4l6mb7WVeteya+N0/fXf/vBBvn6xno51lb62w4KNJeKPMwiOEG4V2TXJ67S+rBKfZYICwQgAIFiCVj4Rl+x7qZDuCPci7VdjocABM4Q0KfD63vX6/umK4aJPiRPrxhQAa8P6Ku0Ra9y+MK9dcGDBZlJs+89i+AE4W7fL+SYP4FLHmuW3zzdlv+BHAEBCEBgCQEL3+gKct91hDvCfYkp8i8EIJBGAn95sSsQ73qlgP4AEfdFL/ff+taAfHxtjVy6vVX04Xss9gQsghOEu32/kGN+BHrHZuXDq6vlgSO8ZSI/cqSGAASyEbDwjb5i3U2HcEe4Z7NHtkEAAikkoPfmX7A+88T7B2P22jh9voDOrq890CMXbc68tu9jd9TIVc+2y3gR77pPYTfn1WSL4AThnhdyEpeAgJ479KoibqUpAVyyhEAKCVj4RleQ+64j3BHuKRxuNBkCEFiOgL5D/jdPtwdB7qrtrdLYH91l//oMgaffHZJrd+ur+TLvu//K/fXypxc6RZ8azyz7cr1ot90iOEG42/UHORVGoOpwn/zzPXWFHcxREIAABJYQsPCNvmLdTYdwR7gvMUX+hQAEICDBQ/d0hko/+gC+ci3vtE/IxsN9cukTrYtP5b94a3PwVP4jTWMyv1CumlCOErAIThDu2FLUBH79VKv8fHtL1NWgfAhAICEELHyjK8h91xHuCPeEDCGaAQEIWBPoH5+VnzyWeW2cznoPTNi/Nm5wYk721ozITS91yXc3NQY/FHzk9mq54ul22XZsUOr7eHWTdb/mk59FcIJwz4c4aa0JjM8syKfvqpV1B3qtsyY/CEAgpQQsfKOvWHfTIdwR7ikdcjQbAhDwJaD3u+vMuz69Xe+DL3YJHyynr9H7/D2ZS+C/vKFObt7XJS/VjMjwpP0PBMXWOa3HWwQnCPe0Wk882v1220Rw/jpQPxqPClELCECg4glY+EZXkPuuI9wR7hU/eGgABCBQegIN/dPBu9JVwN/6crfML/hfsx4+WG79wR65ZFuLfHhN5hL8/9zSFFwW/w7vXi99BxZYgkVwYiXcjx07IatWXRV8dH3psnv33sX9V1xxnTQ2Ni9Nwv8pJPDYsUH55LpaGZ3iB8EUdj9NhkBJCFj4Rl+x7qZDuCPcS2LQZAoBCCSTgL7rXcX79x5ulDdbxpdtpD5Y7rn3h+UvL3YuCv7z1lTL5Tvb5KkTg9I2NLPsseyIDwGL4MRCuE9OTsr69VWBGB8cHJLVq+8W/XYXFe7ZBL2bhvX0EfjDrg750aP8iJO+nqfFECgdAQvf6Apy33WEO8K9dFZNzhCAQCIJ6CuVPnd35rVx973xwX2j73ZOysNv9stlO9sWXyv3pfvq5MaXuuRg/ahMzfrP0icSXAU2yiI4sRDuOnuuwl0FvC5VVVvOEem6LZyR13UWCCiBL2+ol1v2dQEDAhCAgBkBC9/oK9bddAh3hLuZEZMRBCCQLgJ/eqEjmH3/2eMtcvHWzLvVdTZe37O+4VCfnOrOiKx0UUlWay2CEyvhvnXr9kW4K82uq7jXGfliZt8nJqaET+UzON4yHJyjdr7TT39i09hAAm1g0SmUecXCN7qC3Hcd4Y5wL7OpUxwEIJAkAq/WjQaBsc6y62vjesdmk9S81LfFIjixEu65ZtzdzlJhrx+WdBN4+r2h4PzUOcJ5Kd2WQOshYEvAwjf6inU3HcId4W5ryeQGAQhAAAKJIWARnFgI91z3uOv97ocOvRlwd9MmpiNoSEEE9DYdfR4HCwQgAAFLAha+0RXkvusId4S7pR2TFwQgAAEIJIiARXBiIdwVabanyuusul4W39XVLTt2PLd4jzuz7QkywiKacuFDjfLHFzqKyIFDIQABCJxLwMI3+op1Nx3CHeF+rjWyBQIQgAAEICAiFsGJlXCnQyCQD4GWwZngMvltxwbzOYy0EIAABHISsPCNriD3XUe4I9xzGicJIAABCEAgnQQsghOEezptJ+pW760ZCYR7bc9U1FWhfAhAIGEELHyjr1h30yHcEe4JG0o0BwIQgAAErAhYBCcId6veIJ98CKw90CPfqKrP5xDSQgACEPAiYOEbXUHuu45wR7h7GSiJIAABCEAgfQQsghOEe/rsJg4t1ldU/vbZ9jhUhTpAAAIJI2DhG33FupsO4Y5wT9hQojkQgAAEIGBFwCI4Qbhb9Qb5+BLoG5sNLpN/4Eif7yGkgwAEIOBNwMI3uoLcdx3hHkPhnu3JuUstSdPok3T1tTcsEIAABCAAgVIQsAhOEO6l6BnyXInA641jgXB/q3V8pWTsgwAEIFAQAQvf6CvW3XQI95gJd/f9s/peWhXn+u0uobBHuLtUWIcABCAAAWsCFsEJwt26V8gvF4GNh/rkX+49LQu5ErIfAhCAQAEELHyjK8h91xHuMRPujY3Nsn591eJMelXVluDdtUttSsW87mPGfSkZ/ocABCAAASsCFsEJwt2qN8jHl8Cq7S3yyx2tvslJBwEIQCAvAha+0Vesu+kQ7jEU7lu3bl80nt2795ZUuE9NTQsfGGAD2AA2EG8bWHQKZV6xCE4Q7mXutJQXNzEzL59YWyPrX+tNOQmaDwEIlIqAhW90BbnvOsI9hsK9nDPuCwsLwgcG2AA2gA3E2wZKFXzkytciOEG456LMfksCx9rGg/vb99ePWWZLXhCAAAQWCVj4Rl+x7qZDuMdMuPvc465Ww6Xyi2OHFQhAAAIQKBEBi+AE4V6iziHbrAQefXtA/unOWhmenMu6n40QgAAEiiVg4RtdQe67jnCPmXBXQwofPrdq1VWLl8nrJfPhw+j03nbdF340PQsEIAABCEDAmoBFcIJwt+4V8luJwFXPtssl25pXSsI+CEAAAkURsPCNvmLdTYdwj6FwL8qSOBgCEIAABCBgRMAiOEG4G3UG2XgR+Pw9dbL6lW6vtCSCAAQgUAgBC9/oCnLfdYQ7wr0Qe+UYCEAAAhBIAQGL4AThngJDiUkTa3qmgvvbd70/HJMaUQ0IQCCJBCx8o69Yd9Mh3BHuSRxPtAkCEIAABAwIWAQnCHeDjiALLwLPvDck562plvbhGa/0JIIABCBQCAEL3+gKct91hDvCvRB75RgIQAACEEgBAYvgBOGeAkOJSRP/tKdTLtrcFJPaUA0IQCCpBCx8o69Yd9Mh3BHuSR1TtAsCEIAABIokYBGcINyL7AQO9ybwzQca5K8vdnqnJyEEIACBQghY+EZXkPuuI9wR7oXYK8dAAAIQgEAKCFgEJwj3FBhKDJrYOjgd3N/+xDuDMagNVYAABJJMwMI3+op1Nx3CHeGe5HFF2yAAAQhAoAgCFsEJwr2IDuBQbwL7akcC4V7TPeV9DAkhAAEIFELAwje6gtx3HeGOcC/EXjkGAhCAAARSQMAiOEG4p8BQYtBEfQXcvz3YEIOaUAUIQCDpBCx8o69Yd9Mh3BHuSR9btA8CEIAABAokYBGcINwLhM9heRH4/iON8oddHXkdQ2IIQAAChRCw8I2uIPddR7gj3AuxV46BAAQgAIEUELAIThDuKTCUiJvYPz4XXCb/8Jv9EdeE4iEAgTQQsPCNvmLdTYdwR7inYXzRRghAAAIQKICARXCCcC8APIfkReCNxrFAuB9tGc/rOBJDAAIQKISAhW90BbnvOsId4V6IvXIMBCAAAQikgIBFcIJwT4GhRNzEe17vlX/dUCfzCwsR14TiIQCBNBCw8I2+Yt1Nh3BHuKdhfNFGCEAAAhAogIBFcIJwLwA8h+RF4KfbWuTynW15HUNiCEAAAoUSsPCNriD3XUe4I9wLtVmOgwAEIACBhBOwCE4Q7gk3koibNzkzL+etqZb73uiLuCYUDwEIpIWAhW/0FetuOoQ7wj0tY4x2QgACEIBAngQsghOEe57QSZ4XgePtE8H97fvrRvM6jsQQgAAECiVg4RtdQe67jnBHuBdqsxwHAQhAAAIJJ2ARnCDcE24kETdPnyT/2fWnZXhyPuKaUDwEIJAWAha+0Vesu+kQ7gj3tIwx2gkBCEAAAnkSsAhOEO55Qid5XgQu29kmlz7RktcxJIYABCBQDAEL3+gKct91hDvCvRi75VgIQAACEEgwAYvgBOGeYAOJQdMuWH9a1u7viUFNqAIEIJAWAha+0Vesu+kQ7gj3tIwx2gkBCEAAAnkSsAhOEO55Qie5N4Hanqng/vYXTo14H0NCCEAAAsUSsPCNriD3XUe4I9yLtV2OhwAEIACBhBKwCE4Q7gk1jhg068njg/LxO2qkbWgmBrWhChCAQFoIWPhGX7HupkO4I9zTMsZoJwQgAAEI5EnAIjhBuOcJneTeBK7d3SE/erTZOz0JIQABCFgQsPCNriD3XUe4I9wt7Jc8IAABCEAggQQsghOEewINIyZN+sr99XLzvq6Y1IZqQAACaSFg4Rt9xbqbDuGOcE/LGKOdEIAABCCQJwGL4AThnid0knsRaB+aCe5vf+rdIa/0JIIABCBgRcDCN7qC3Hcd4Y5wt7Jh8oEABCAAgYQRsAhOEO4JM4qYNGdP9Ugg3E91T8akRlQDAhBICwEL3+gr1t10CHeEe1rGGO2EAAQgAIE8CVgEJwj3PKGT3IvAzfu65d8fbvRKSyIIQAAClgQsfKMryH3XEe4Id0s7Ji8IQAACEEgQAYvgBOGeIIOIUVO+81CDXP9CZ4xqRFUgAIG0ELDwjb5i3U2HcEe4p2WM0U4IQAACEMiTgEVwgnDPEzrJcxIYGJ8NLpPf+vZAzrQkgAAEIGBNwMI3uoLcdx3hjnC3tmXygwAEIACBhBCwCE4Q7gkxhhg147WGsUC4v9U6HqNaURUIQCAtBCx8o69Yd9Mh3BHuaRljtBMCEIAABPIkYBGcINzzhE7ynATWHeiRr1fVy+z8Qs60JIAABCBgTcDCN7qC3Hcd4Y5wt7Zl8oMABCAAgYQQsAhOEO4JMYYYNeNHjzbLVc+2x6hGVAUCEEgTAQvf6CvW3XQId4R7msYZbYUABCAAgTwIWAQnCPc8gJM0J4Hp2YXgMvmqI/0505IAAhCAQCkIWPhGV5D7riPcEe6lsGfyhAAEIACBBBCwCE4Q7gkwhBg14e228UC4v9YwGqNaURUIQCBNBCx8o69Yd9Mh3BHuaRpntBUCEIAABPIgYBGcINzzAE7SnASqDvfJF+6tk8GJuZxpSQABCECgFAQsfKMryH3XEe4I91LYM3lCAAIO6NlPAAATJ0lEQVQQgEACCFgEJwj3BBhCjJrwiydb5Rc7WmNUI6oCAQikjYCFb/QV6246hDvCPW1jjfZCAAIQgIAnAYvgBOHuCZtkXgQ+ta5W1r/W65WWRBCAAARKQcDCN7qC3Hcd4Y5wL4U9kycEIAABCCSAgEVwgnBPgCHEpAk1PVPB/e17a0ZiUiOqAQEIpJGAhW/0FetuOoQ7wj2N4402QwACEICABwGL4ATh7gGaJF4Eth0bkPPvrJWWwWmv9CSCAAQgUAoCFr7RFeS+6wh3hHsp7Jk8IQABCEAgAQQsghOEewIMISZN+N1z7XLJtpaY1IZqQAACaSVg4Rt9xbqbDuGOcE/rmKPdEIAABCCQg4BFcIJwzwGZ3d4E/uXeOrntlW7v9CSEAAQgUAoCFr7RFeS+6wh3hHsp7Jk8IQABCEAgAQQsghOEewIMIQZNaB+aCe5vf+a9oRjUhipAAAJpJmDhG33FupsO4Y5wT/O4o+0QgAAEILACAYvgBOG+AmB2eRN47v1hOW9NtegD6lggAAEIREnAwje6gtx3HeGOcI/S7ikbAhCAgBmBxsZmWb++SiYnJ5fN0yeNe/CxYyekqmqLuylV6xbBCcI9VSZTssb+5cUu+cHmppLlT8YQSCoBH7/nk8blg288Lb5i2zIdwh3h7o5D1iEAAQhULAGfwMMnjQugkOBkcHAoEPsr/YDglhHndYR7nHsnXXX71oMNouKdBQIQyI+Aj9/zSeOWim9EuLv2kPe6RXBR7C8izCrk3W0cAAEIQMCMgE/g4ZPGrRDBSfHBCb7RtSjWCyEwNDkX3N+ur4NjgQAE8iPg4/d80ril4huL942F6E5m3Jlxd8ch6xCAAAQqgsDu3Xtl1aqrFj+rV98tNTWnz7pU3k2j67pocPLXv66WK664LjhWjwtnxvWS+DBPDUp08QlOwuOuueZG6erqFs1T89EytDz9hOWFl92HQZIeo2ndegQFx+SPxY/iCPeYdGYFV+OV06OBcD/ePlHBraDqECg9Adfvhb4F32jP3cI3ItwNRXghMAlO7AcGOUIAAhBYSkCFtgpgvSRdl61bty8K5PAedxXcoRjW9Lqu21Qw33DDmsVjNR8NdHT7qVO1QX66rnnqkku4hwJcy9D6NDW1BHmF9dBtWnZY11deObhY12z1CAqN0R+L4ATfGKMOrdCqrH6lW779YIPMzC9UaAuoNgRKTwDfWHrGYQkWvrEQrcmMu6HYJzgJzZlvCEAAAqUloII6nB0PBborolWMh7PsWpPwfzeNbtd8wnT6vTTPXMJd89A8dUY9Wz3CfWG++q15rlSP0pLLL3eL4ATfmB9zUp9L4KLNTfL7XR3n7mALBCBwFgF841k4SvaPhW9EuBuK8EJgEpyUbHyQMQQgAIGzCOzY8dziLHa4wxXDGryEQjrXjLum1U8o4N1jdT28vD0sx/0OZ9l1W5iHW4+lM+7hsW4a3aZl6PFxWyyCE3xj3Hq1suozPbcQXCa/6e/9lVVxaguBCAjgG8sD3cI3FqI1mXE3FPsEJ+UZLJQCAQhAQEWuO4sdXjqv94yHAtidQQ9FuQpm9x73UJQfOvTm4n3oKvh1uwp+Ta956ne2RS+vD+9T11l3zSf8ocCdXQ/vcddvzUs/4TZNF9YjWxlRbrMITvCNUfZg5Zd9pHksEO6Hm8YqvzG0AAIlJoBvLDHgM9lb+EaEu6EILwQmwYn/YBkfn5ChoRH/AxKQcmJC74EdTkBL/JtAm/1ZVXLKcvezzmKHwlq5Lf2/HCyLbbMK9/A++HLUt9AyLIITfGNh9Ht6+mV2drawgxN01O0vtciXN9TL4MRcglqVf1OwhwwzOCzPYakvXPp//lZX/iPS5BsL0ZrMuBuKfYIT/wGOcPdnVckpixU3ldh22lyeXlPhHs64h7PYpSx56SzG+vUPyKFDRxefSJ9v2WkKTvCN+VpHJj0CJcPhJ1vq5ddPtRYGMUFHYQ+MC9ecl7OHqH2j+uXwNjW3vr7rafKNCHdDEV4ITIIT32EpgnD3Z1XJKRGxldx7/nWnn/1ZVVpK9WuF+EP3GHxjYb2+XGBeWG6Ve9RHb6+Ruw92V24DjGqOPWRAwgEORkOqqGwsfKPrJ33XmXE3FPvlCk7016jw3sjwvs2irC+CgxHuEUCPoEgEXQTQIyiSfo4AepmKtAhO8I2FdRYCReT9zsng/vaXqjOvfiyMZDKOwh4y/QgHOMRhRFv4Rl+x7qZDuFegcNfLYPSyTX0Akt4jqUK+0haEe6X1WGH1RdAVxq3SjqKfK63H/OtrEZyUS7gnwTe6PYNAEdl8dEA+c1ettPRPuGhSuY49ZLodDnCIwwnAwje6gtx3HeFeYcJdHzSh947oty46417MrHtHR7fwgQE2gA1gA/G2gagCFYvgpBzC3do3Xry5TvjEg8GPN9cTpxCrYQPYQFYbqGTf6CvW3XQI9woU7jqroLPtuujMezHCvbOzRzo7u0W/NXDn/9LyyDBOF+/QrsLvNNibtjVN7U3r+SPTx+UZz5UcnJRLuCfNN4bnkPA7DefOtJ5LfGKv0A7Cb+yhPOfeuPIO6xV+p9UeKtk3uoLcdx3hXoHC3XLGPSqD51L5qMiXt1wuoS4v76hKo5+jIl/6ctM64156srlL4JLgDCM4wMEdLdgD9uDaQ1TrFr7RV6y76RDuFSbc1UCTcB8fwj2qU015y0XQlZd3VKXRz1GRL325FsFJOWbclUQSfKPbowiUDA04wIFx4RLAHs6lUf4tFr7RFeS+6wj3ChTuPFW+/APUokTEjQXF+OdBP8e/jyxqmJZ+tghOyiXck+AbXdtEsGZowAEOjAuXAPZwLo3yb7Hwjb5i3U2HcK9A4V5+87QvkRl3e6ZxzDEt4sZlT5tdGsldT0s/WwQn5RLuSbM2BGumR+EAB3dsYw/Yg2sPUa1b+EZXkPuuI9wR7pHYPMI9EuxlLzQt4sYFS5tdGsldT0s/WwQnCPfCxgECJcMNDnBwRxD2gD249hDVuoVv9BXrbjqEO8I9KpunXAhAAAIQiDkBi+AE4R7zTqZ6EIAABCCQFwEL3+gKct91hDvCPS9DJTEEIAABCKSHgEVwgnBPj73QUghAAAJpIGDhG33FupsO4Y5wT8P4oo0QgAAEIFAAAYvgBOFeAHgOgQAEIACB2BKw8I2uIPddR7gj3GM7KKgYBCAAAQhES8AiOEG4R9uHlA4BCEAAArYELHyjr1h30yHcEe62lkxuEIAABCCQGAIWwQnCPTHmQEMgAAEIQEBELHyjK8h91xHuCHcGIAQgAAEIQCArAYvgBOGeFS0bIQABCECgQglY+EZfse6mQ7gj3Ct0yFBtCEAAAhAoNQGL4AThXupeIn8IQAACECgnAQvf6Apy33WEO8K9nHZOWRCAAAQgUEEELIIThHsFdThVhQAEIACBnAQsfKOvWHfTIdwR7jmNkwQQgAAEIJBOAhbBCcI9nbZDqyEAAQgklYCFb3QFue86wh3hXtIxNTg4JDfcsEYaG5sXy9H1K664Tlatukp27967uL2qakuw7ZprbhQ9rhIXrbfWX9u2evXdMjk5GTQjLW12+y7JbQ5t89ixE0F/h/aa5Da7bXPHrrs9aeNZ+1nbpO3Vj/a3Lklvc2jf+m0RnCDcXaLnrquNqf9zl2z+cDm7c4+r1PVs40zbkmYOuc6nSbYH7XuNnzSOSjOH0P7V/7jxVbjd3ZZ0e4jbuc3CN/qKdTcdwh3hXrKxEIpYFel6QgkXPeFoAKwn5fXrq4J9ul/XdZvu0zSVtmjdtd6hiNP10OEktc3aR1u3bl/sX21vGtqs7Vab1aBC7dbt8yTadthe7eulS5Jt2z0v6fh+5ZWDQfOT3Oal/WsRnCDcl1L94H89X2pQrjYVLq7duf4wm92Fx1Tyt7Yx9Bvadv2xX8+paeOg7Q3Psdp+9S9p5BDastq7flzbyBYnJnVcKAc3vgq5pG1chO2O27eFb3QFue86wh3hXvKx4J54XGekBesJ2f3otqVpSl7BEhWg7dKAZGl73PbqepLaHDrQpLc5/NHp1KnaxR9rkt5mteVw5jn8lT/pbQ7HsHuKSHqb3bbqukVwgnBfSvXs/13BpntCH6Hrob01N7cuCrmlac7OrbL/0/aqH9FzbJo5uOIsjRzCH6zcH3XSxkHHgP54E/pdbf/SsZ+280Oczm4WvtFXrLvpEO4I95KPg6XCPXTKWnB4UtYTkq7roicrTaMnpEpd3ODDXdf2JLHN2n/qXMI+THqbwx8oXFtNepvdsaj9HY7RJI9nPXft2PHcYuCk9p2mftY+twhOEO7u6Dl3PZtwD8+l4TlGhXu2sXZubpW9xY0F3PW0cNDzi/4wqoJN26xL2ji4P1qE8VIaObgjWe0ivO00bfbgcojTuoVvdAW57zrCHeFe8nGwVLiHl39pwXoCcj+6TU9QbpqSV9C4APcEm609bnt1PVsa4yqVLTsNLENB5/ZhktocBlbhr+D6rYHWcjNiYduT1M9hYNXV1X3WWA3bGn5XeptDe9Z2pKXNS08WFsEJwn0p1bP/zybcdQzpEvrD5c4vZ+dU2f9pm3XMhUu280gaOGj71SbUr+h32jhoe13/qushA/3WJU3jIhwPOjbCHzLSzCHkEfW3hW/0FetuOoQ7wr3ktu8Kdy0sPPnor8lJusdd2+Y6WxdsktusM5LqRHVRpxIGXkluc9i34SxQ2P4kt1nv73ZngMLAIcltdu1Zx3Z4f2WS2xzadvhtEZwg3EOa2b/VttRPhotra64NZrO78JhK/9a26cdd0sZB2xs+R8MnPkqyPYR2EApV/T+N9qC34+kS/lCh32njENpC3L4tfKMryH3XEe4I95KNBT3hur+ahk5ZTzpJfaq8ttFtc3i5W1LbrMGF9nPYn+G9z2pUSW2zO2CWCvektnlpP4d2nYZ+Dse0+5DNpPaza9vhukVwgnAPaZ79reNKx1LoM1wbC+3O55x6dq6V9587nkIW6ld0SRMHFWX642DIIPxxNG0cXAt2hXvaOCy1h3BMpI2Daw9xWrfwjb5i3U2HcEe4x2kcUBcIQAACEIgRAYvgBOEeow6lKhCAAAQgUDQBC9/oCnLfdYQ7wr1o4yUDCEAAAhBIJgGL4AThnkzboFUQgAAE0krAwjf6inU3HcId4Z7WMUe7IQABCEAgBwGL4AThngMyuyEAAQhAoKIIWPhGV5D7riPcEe4VNVCoLAQgAAEIlI+ARXCCcC9ff1ESBCAAAQiUnoCFb/QV6246hDvCvfTWTQkQgAAEIFCRBCyCE4R7RXY9lYYABCAAgWUIWPhGV5D7riPcEe7LmCSbIQABCEAg7QQsghOEe9qtiPZDAAIQSBYBC9/oK9bddAh3hHuyRhKtgQAEIAABMwIWwQnC3aw7yAgCEIAABGJAwMI3uoLcdx3hjnCPgflTBQhAAAIQiCMBi+AE4R7HnqVOEIAABCBQKAEL3+gr1t10CHeEe6E2y3EQgAAEIJBwAhbBCcI94UZC8yAAAQikjICFb3QFue86wh3hnrKhRnMhAAEIQMCXgEVwgnD3pU06CEAAAhCoBAIWvtFXrLvpEO4I90oYH9QRAhCAAAQiIGARnCDcI+g4ioQABCAAgZIRsPCNriD3XUe4I9xLZtRkDAEIQAAClU3AIjhBuFe2DVB7CEAAAhA4m4CFb/QV6246hDvC/WxL5D8IQAACEIDAGQIWwQnCHXOCAAQgAIEkEbDwja4g911HuCPckzSOaAsEIAABCBgSsAhOEO6GHUJWEIAABCAQOQEL3+gr1t10CHeEe+TGTwUgAAEIQCCeBCyCE4R7PPuWWkEAAhCAQGEELHyjK8h91xHuCPfCLJajIAABCEAg8QQsghOEe+LNhAZCAAIQSBUBC9/oK9bddAh3Q+FeX18vc3NzqTJcGgsBCEAAAskkoP5M/ZobNBSyjm9Mpn3QKghAAAJpJGDlGwvxpwh3Q+He1dUl3d3diPc0jmLaDAEIQCBBBDQwUX+mfq2Q4MI9Bt+YIMOgKRCAAARSTMDSN7p+0ncd4W4o3BW6Big6u6CXUPCBATaADWAD2EAl2oD6MQvRHgYj+EbGQSWOA+qM3WID2IBrA9a+MfSRvt8Id2Ph7guedNNFz+LAEIbYADaADWAD2AA2gA1gA9gANpAGG0C4I9wR0NgANoANYAPYADaADWAD2AA2gA1gAzG2AYR7jDsnDb8c0UZ+IcUGsAFsABvABrABbAAbwAawAWxgZRtAuCPc+WUNG8AGsAFsABvABrABbAAbwAawAWwgxjaAcI9x5/Cr08q/OsEHPtgANoANYAPYADaADWAD2AA2kAYbQLgj3PllDRvABrABbAAbwAawAWwAG8AGsAFsIMY2gHCPceek4Zcj2sgvpNgANoANYAPYADaADWAD2AA2gA2sbAMId4Q7v6xhA9gANoANYAPYADaADWAD2AA2gA3E2AYQ7jHuHH51WvlXJ/jABxvABrABbAAbwAawAWwAG8AG0mADCHeEO7+sYQPYADaADWAD2AA2gA1gA9gANoANxNgG/n/ONIMRRUYsRwAAAABJRU5ErkJggg==)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Weights\n", + "\n", + "\n", + "\n", + "# ⚡ 💘 🏋️‍♀️ Supercharge your Training with PyTorch Lightning + Weights & Biases" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Weights" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "At Weights & Biases, we love anything\n", + "that makes training deep learning models easier.\n", + "That's why we worked with the folks at PyTorch Lightning to\n", + "[integrate our experiment tracking tool](https://docs.wandb.com/library/integrations/lightning)\n", + "directly into\n", + "[the Lightning library](https://pytorch-lightning.readthedocs.io/en/latest/common/loggers.html#weights-and-biases).\n", + "\n", + "[PyTorch Lightning](https://pytorch-lightning.readthedocs.io/en/stable/) is a lightweight wrapper for organizing your PyTorch code and easily adding advanced features such as distributed training and 16-bit precision.\n", + "It retains all the flexibility of PyTorch,\n", + "in case you need it,\n", + "but adds some useful abstractions\n", + "and builds in some best practices.\n", + "\n", + "## What this notebook covers:\n", + "\n", + "1. Differences between PyTorch and PyTorch Lightning, including how to set up `LightningModules` and `LightningDataModules`\n", + "2. How to get basic metric logging with the [`WandbLogger`](https://pytorch-lightning.readthedocs.io/en/latest/common/loggers.html#weights-and-biases)\n", + "3. How to log media with W&B and fully customize logging with Lightning `Callbacks`\n", + "\n", + "## The interactive dashboard in W&B will look like this:\n", + "\n", + "![](https://i.imgur.com/lIbMyFR.png)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Follow along with a [video tutorial](http://wandb.me/lit-video)!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🚀 Installing and importing" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`wandb` and `pytorch-lightning` are both easily installable via [`pip`](https://pip.pypa.io/en/stable/)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qqq wandb lightning torchmetrics onnx" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "PyTorch Lightning is built on top of PyTorch,\n", + "so we still need to import vanilla PyTorch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# numpy for non-GPU array math\n", + "import numpy as np\n", + "\n", + "# 🍦 Vanilla PyTorch\n", + "import torch\n", + "from torch.nn import functional as F\n", + "from torch import nn\n", + "from torch.utils.data import DataLoader, random_split\n", + "\n", + "# 👀 Torchvision for CV\n", + "from torchvision.datasets import MNIST\n", + "from torchvision import transforms\n", + "\n", + "# remove slow mirror from list of MNIST mirrors\n", + "MNIST.mirrors = [mirror for mirror in MNIST.mirrors\n", + " if not mirror.startswith(\"http://yann.lecun.com\")]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Much of Lightning is built on the [Modules](https://pytorch.org/docs/stable/generated/torch.nn.Module.html)\n", + "API from PyTorch,\n", + "but adds extra features\n", + "(like data loading and logging)\n", + "that are common to lots of PyTorch projects.\n", + "\n", + "Let's bring those in,\n", + "plus W&B and the integration.\n", + "\n", + "Lastly, we log in to the [Weights & Biases web service](https://wandb.ai).\n", + "If you've never used W&B,\n", + "you'll need to sign up first.\n", + "Accounts are free forever for academic and public projects." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# ⚡ PyTorch Lightning\n", + "import lightning.pytorch as pl\n", + "import torchmetrics\n", + "pl.seed_everything(hash(\"setting random seeds\") % 2**32 - 1)\n", + "\n", + "# 🏋️‍♀️ Weights & Biases\n", + "import wandb\n", + "\n", + "# ⚡ 🤝 🏋️‍♀️\n", + "from lightning.pytorch.loggers import WandbLogger\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> _Note_: If you're executing your training in a terminal, rather than a notebook, you don't need to include `wandb.login()` in your script.\n", + "Instead, call `wandb login` in the terminal and we'll keep you logged in for future runs." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🏗️ Building a Model with Lightning\n", + "\n", + "In PyTorch Lightning, models are built with `LightningModule` ([docs here](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html)), which has all the functionality of a vanilla `torch.nn.Module` (🍦) but with a few delicious cherries of added functionality on top (🍨).\n", + "These cherries are there to cut down on boilerplate and\n", + "help separate out the ML engineering code\n", + "from the actual machine learning.\n", + "\n", + "For example, the mechanics of iterating over batches\n", + "as part of an epoch are extracted away,\n", + "so long as you define what happens on the `training_step`.\n", + "\n", + "To make a working model out of a `LightningModule`,\n", + "we need to define a new `class` and add a few methods on top.\n", + "\n", + "We'll demonstrate this process with `LitMLP`,\n", + "which applies a two-layer perceptron\n", + "(aka two fully-connected layers and\n", + "a fully-connected softmax readout layer)\n", + "to input `Tensors`.\n", + "\n", + "> _Note_: It is common in the Lightning community to shorten \"Lightning\" to \"[Lit](https://www.urbandictionary.com/define.php?term=it%27s%20lit)\".\n", + "This sometimes it sound like\n", + "[your code was written by Travis Scott](https://www.youtube.com/watch?v=y3FCXV8oEZU).\n", + "We consider this a good thing." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🍦 `__init__` and `forward`\n", + "\n", + "First, we need to add two methods that\n", + "are part of any vanilla PyTorch model." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Those methods are:\n", + "* `__init__` to do any setup, just like any Python class\n", + "* `forward` for inference, just like a PyTorch Module\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `forward` pass method is standard,\n", + "and it'll be different for every project,\n", + "so we won't comment on it.\n", + "\n", + "The `__init__` method,\n", + "which `init`ializes new instances of the class,\n", + "is a good place to log hyperparameter information to `wandb`.\n", + "\n", + "This is done with the `save_hyperparameters` method,\n", + "which captures all of the arguments to the initializer\n", + "and adds them to a dictionary at `self.hparams` --\n", + "that all comes for free as part of the `LightningModule`.\n", + "\n", + "> _Note_: `hparams` is logged to `wandb` as the `config`,\n", + "so you'll never lose track of the arguments you used to run a model again!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class LitMLP(pl.LightningModule):\n", + "\n", + " def __init__(self, in_dims, n_classes=10,\n", + " n_layer_1=128, n_layer_2=256, lr=1e-4):\n", + " super().__init__()\n", + "\n", + " # we flatten the input Tensors and pass them through an MLP\n", + " self.layer_1 = nn.Linear(np.prod(in_dims), n_layer_1)\n", + " self.layer_2 = nn.Linear(n_layer_1, n_layer_2)\n", + " self.layer_3 = nn.Linear(n_layer_2, n_classes)\n", + "\n", + " # log hyperparameters\n", + " self.save_hyperparameters()\n", + "\n", + " # compute the accuracy -- no need to roll your own!\n", + " self.train_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=n_classes)\n", + " self.valid_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=n_classes)\n", + " self.test_acc = torchmetrics.Accuracy(task=\"multiclass\", num_classes=n_classes)\n", + "\n", + " def forward(self, x):\n", + " \"\"\"\n", + " Defines a forward pass using the Stem-Learner-Task\n", + " design pattern from Deep Learning Design Patterns:\n", + " https://www.manning.com/books/deep-learning-design-patterns\n", + " \"\"\"\n", + " batch_size, *dims = x.size()\n", + "\n", + " # stem: flatten\n", + " x = x.view(batch_size, -1)\n", + "\n", + " # learner: two fully-connected layers\n", + " x = F.relu(self.layer_1(x))\n", + " x = F.relu(self.layer_2(x))\n", + "\n", + " # task: compute class logits\n", + " x = self.layer_3(x)\n", + " x = F.log_softmax(x, dim=1)\n", + "\n", + " return x\n", + "\n", + " # convenient method to get the loss on a batch\n", + " def loss(self, xs, ys):\n", + " logits = self(xs) # this calls self.forward\n", + " loss = F.nll_loss(logits, ys)\n", + " return logits, loss" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> _Note_: for pedagogical purposes, we're splitting out\n", + "each stage of building the `LitMLP` into a different cell.\n", + "In a more typical workflow,\n", + "this would all happen in the `class` definition.\n", + "\n", + "> _Note_: if you're familiar with PyTorch,\n", + "you might be surprised to see we aren't taking care with `.device`s:\n", + "no `to_cuda` etc. PyTorch Lightning handles all that for you! 😎" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🍨 `training_step` and `configure_optimizers`\n", + "Now, we add some special methods so that our `LitMLP` can be trained\n", + "using PyTorch Lightning's training API.\n", + "\n", + "> _Note_: if you've used Keras, this might be familiar.\n", + "It's very similar to the `.fit` API in that library." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Those methods are\n", + "\n", + "* `training_step`, which takes a batch and computes the loss; backprop goes through it\n", + "* `configure_optimizers`, which returns the `torch.optim.Optimizer` to apply after the `training_step`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> _Note_: `training_step` is part of a rich system of callbacks in PyTorch Lightning.\n", + "These callbacks are methods that get called\n", + "at specific points during training\n", + "(e.g. when a validation epoch ends),\n", + "and they are a major part of what makes\n", + "PyTorch Lightning both useful and extensible." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here's where we add some more serious logging code.\n", + "`self.log` takes a name and value for a metric.\n", + "Under the hood, this will get passed to `wandb.log` if you're using W&B.\n", + "\n", + "The logging behavior of PyTorch Lightning is both intelligent and configurable.\n", + "For example, by passing the `on_epoch`\n", + "keyword argument here,\n", + "we'll get `_epoch`-wise averages\n", + "of the metrics logged on each `_step`,\n", + "and those metrics will be named differently\n", + "in the W&B interface.\n", + "When training in a distributed setting,\n", + "these averages will be automatically computed across nodes.\n", + "\n", + "Read more about the `log` method [in the docs](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html#log)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def training_step(self, batch, batch_idx):\n", + " xs, ys = batch\n", + " logits, loss = self.loss(xs, ys)\n", + " preds = torch.argmax(logits, 1)\n", + "\n", + " # logging metrics we calculated by hand\n", + " self.log('train/loss', loss, on_epoch=True)\n", + " # logging a pl.Metric\n", + " self.train_acc(preds, ys)\n", + " self.log('train/acc', self.train_acc, on_epoch=True)\n", + "\n", + " return loss\n", + "\n", + "def configure_optimizers(self):\n", + " return torch.optim.Adam(self.parameters(), lr=self.hparams[\"lr\"])\n", + "\n", + "LitMLP.training_step = training_step\n", + "LitMLP.configure_optimizers = configure_optimizers" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ➕ Optional methods for even better logging\n", + "\n", + "The code above will log our model's performance,\n", + "system metrics, and more to W&B.\n", + "\n", + "If we want to take our logging to the next level,\n", + "we need to make use of PyTorch Lightning's callback system.\n", + "\n", + "> _Note_: thanks to the clean design of PyTorch Lightning,\n", + "the training code below will run with or without any\n", + "of this extra logging code. Nice!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The other callbacks we'll make use of fall into two categories:\n", + "* methods that trigger on each batch for a dataset: `validation_step` and `test_step`\n", + "* methods that trigger at the end of an epoch,\n", + "or a full pass over a given dataset: `{training, validation, test}_epoch_end`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 💾 `test`ing and saving the model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use the test set to evaluate the performance of the final model,\n", + "so the `test` callbacks will be called at the end of the training pipeline.\n", + "\n", + "For performance on the `test` and `validation` sets,\n", + "we're typically less concerned about how\n", + "we do on intermediate steps and more\n", + "with how we did overall.\n", + "That's why below, we pass in\n", + "`on_step=False` and `on_epoch=True`\n", + "so that we log only `epoch`-wise metrics.\n", + "\n", + "> _Note_: That's actually the default behavior for `.log` when it's called inside of a `validation` or a `test` loop -- but not when it's called inside a `training` loop! Check out the table of default behaviors for `.log` [in the docs](https://pytorch-lightning.readthedocs.io/en/latest/lightning_module.html#log)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def test_step(self, batch, batch_idx):\n", + " xs, ys = batch\n", + " logits, loss = self.loss(xs, ys)\n", + " preds = torch.argmax(logits, 1)\n", + "\n", + " self.test_acc(preds, ys)\n", + " self.log(\"test/loss_epoch\", loss, on_step=False, on_epoch=True)\n", + " self.log(\"test/acc_epoch\", self.test_acc, on_step=False, on_epoch=True)\n", + "\n", + "LitMLP.test_step = test_step" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll also take the opportunity to save the model in the\n", + "[portable `ONNX` format](https://onnx.ai/).\n", + "\n", + "\n", + "Later,\n", + "we'll see that this allows us to use the\n", + "[Netron model viewer](https://github.com/lutzroeder/netron) in W&B." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def on_test_epoch_end(self): # args are defined as part of pl API\n", + " dummy_input = torch.zeros(self.hparams[\"in_dims\"], device=self.device)\n", + " model_filename = \"model_final.onnx\"\n", + " self.to_onnx(model_filename, dummy_input, export_params=True)\n", + " artifact = wandb.Artifact(name=\"model.ckpt\", type=\"model\")\n", + " artifact.add_file(model_filename)\n", + " wandb.log_artifact(artifact)\n", + "\n", + "LitMLP.on_test_epoch_end = on_test_epoch_end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 📊 Logging `Histograms`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "For the `validation_data`,\n", + "let's track not only the `acc`uracy and `loss`,\n", + "but also the `logits`:\n", + "the un-normalized class probabilities.\n", + "That way, we can track if our network\n", + "is becoming more or less confident over time.\n", + "\n", + "There's a problem though:\n", + "`.log` wants to average,\n", + "but we'd rather look at a distribution.\n", + "\n", + "So instead, on every `validation_step`,\n", + "we'll `return` the `logits`,\n", + "rather than `log`ging them.\n", + "\n", + "Then, when we reach the `end`\n", + "of the `validation_epoch`,\n", + "the `logits` are available as the\n", + "`validation_step_outputs` -- a list.\n", + "\n", + "So to log we'll take those `logits`,\n", + "concatenate them together,\n", + "and turn them into a histogram with [`wandb.Histogram`](https://docs.wandb.com/library/log#histograms).\n", + "\n", + "Because we're no longer using Lightning's `.log` interface and are instead using `wandb`,\n", + "we need to drop down a level and use\n", + "`self.experiment.logger.log`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def on_validation_epoch_start(self):\n", + " self.validation_step_outputs = []\n", + "\n", + "def validation_step(self, batch, batch_idx):\n", + " xs, ys = batch\n", + " logits, loss = self.loss(xs, ys)\n", + " preds = torch.argmax(logits, 1)\n", + " self.valid_acc(preds, ys)\n", + "\n", + " self.log(\"valid/loss_epoch\", loss) # default on val/test is on_epoch only\n", + " self.log('valid/acc_epoch', self.valid_acc)\n", + "\n", + " self.validation_step_outputs.append(logits)\n", + "\n", + " return logits\n", + "\n", + "def on_validation_epoch_end(self):\n", + "\n", + " validation_step_outputs = self.validation_step_outputs\n", + "\n", + " dummy_input = torch.zeros(self.hparams[\"in_dims\"], device=self.device)\n", + " model_filename = f\"model_{str(self.global_step).zfill(5)}.onnx\"\n", + " torch.onnx.export(self, dummy_input, model_filename, opset_version=11)\n", + " artifact = wandb.Artifact(name=\"model.ckpt\", type=\"model\")\n", + " artifact.add_file(model_filename)\n", + " self.logger.experiment.log_artifact(artifact)\n", + "\n", + " flattened_logits = torch.flatten(torch.cat(validation_step_outputs))\n", + " self.logger.experiment.log(\n", + " {\"valid/logits\": wandb.Histogram(flattened_logits.to(\"cpu\")),\n", + " \"global_step\": self.global_step})\n", + "\n", + "LitMLP.on_validation_epoch_start = on_validation_epoch_start\n", + "LitMLP.validation_step = validation_step\n", + "LitMLP.on_validation_epoch_end = on_validation_epoch_end" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that we're once again saving\n", + "the model in ONNX format.\n", + "That way, we can roll back our model to any given epoch --\n", + "useful in case the evaluation on the test set reveals we've overfit." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 📲 `Callback`s for extra-fancy logging" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "What we've done so far\n", + "will tell us how well our model\n", + "is using our system resources,\n", + "how well our model is training and generalizing,\n", + "and how confident it is.\n", + "\n", + "But DNNs often fail in pernicious and silent ways.\n", + "Often, the only way to notice these failures\n", + "is to look at how the model is doing\n", + "on specific examples.\n", + "\n", + "So let's additionally log some detailed information on some specific examples:\n", + "the inputs, outputs,\n", + "and `pred`ictions.\n", + "\n", + "We'll do this by writing our own `Callback` --\n", + "one that, after every `validation_epoch` ends,\n", + "logs input images and output predictions\n", + "using W&B's `Image` logger.\n", + "\n", + "> _Note_:\n", + "For more on the W&B media toolkit, read the [docs](https://docs.wandb.com/library/log#media)\n", + "or check out\n", + "[this Colab](http://wandb.me/media-colab)\n", + "to see everything it's capable of." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class ImagePredictionLogger(pl.Callback):\n", + " def __init__(self, val_samples, num_samples=32):\n", + " super().__init__()\n", + " self.val_imgs, self.val_labels = val_samples\n", + " self.val_imgs = self.val_imgs[:num_samples]\n", + " self.val_labels = self.val_labels[:num_samples]\n", + "\n", + " def on_validation_epoch_end(self, trainer, pl_module):\n", + " val_imgs = self.val_imgs.to(device=pl_module.device)\n", + "\n", + " logits = pl_module(val_imgs)\n", + " preds = torch.argmax(logits, 1)\n", + "\n", + " trainer.logger.experiment.log({\n", + " \"examples\": [wandb.Image(x, caption=f\"Pred:{pred}, Label:{y}\")\n", + " for x, pred, y in zip(val_imgs, preds, self.val_labels)],\n", + " \"global_step\": trainer.global_step\n", + " })" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🛒 Loading data\n", + "\n", + "Data pipelines can be created with:\n", + "* 🍦 Vanilla Pytorch `DataLoaders`\n", + "* ⚡ Pytorch Lightning `DataModules`\n", + "\n", + "`DataModules` are more structured definition, which allows for additional optimizations such as automated distribution of workload between CPU & GPU.\n", + "Using `DataModules` is recommended whenever possible!\n", + "\n", + "A `DataModule` is also defined by an interface:\n", + "* `prepare_data` (optional) which is called only once and on 1 GPU -- typically something like the data download step we have below\n", + "* `setup`, which is called on each GPU separately and accepts `stage` to define if we are at `fit` or `test` step\n", + "* `train_dataloader`, `val_dataloader` and `test_dataloader` to load each dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class MNISTDataModule(pl.LightningDataModule):\n", + "\n", + " def __init__(self, data_dir='./', batch_size=128):\n", + " super().__init__()\n", + " self.data_dir = data_dir\n", + " self.batch_size = batch_size\n", + " self.transform = transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize((0.1307,), (0.3081,))])\n", + "\n", + " def prepare_data(self):\n", + " # download data, train then test\n", + " MNIST(self.data_dir, train=True, download=True)\n", + " MNIST(self.data_dir, train=False, download=True)\n", + "\n", + " def setup(self, stage=None):\n", + "\n", + " # we set up only relevant datasets when stage is specified\n", + " if stage == 'fit' or stage is None:\n", + " mnist = MNIST(self.data_dir, train=True, transform=self.transform)\n", + " self.mnist_train, self.mnist_val = random_split(mnist, [55000, 5000])\n", + " if stage == 'test' or stage is None:\n", + " self.mnist_test = MNIST(self.data_dir, train=False, transform=self.transform)\n", + "\n", + " # we define a separate DataLoader for each of train/val/test\n", + " def train_dataloader(self):\n", + " mnist_train = DataLoader(self.mnist_train, batch_size=self.batch_size)\n", + " return mnist_train\n", + "\n", + " def val_dataloader(self):\n", + " mnist_val = DataLoader(self.mnist_val, batch_size=10 * self.batch_size)\n", + " return mnist_val\n", + "\n", + " def test_dataloader(self):\n", + " mnist_test = DataLoader(self.mnist_test, batch_size=10 * self.batch_size)\n", + " return mnist_test\n", + "\n", + "# setup data\n", + "mnist = MNISTDataModule()\n", + "mnist.prepare_data()\n", + "mnist.setup()\n", + "\n", + "# grab samples to log predictions on\n", + "samples = next(iter(mnist.val_dataloader()))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 👟 Making a `Trainer`\n", + "\n", + "The `DataLoader` and the `LightningModule`\n", + "are brought together by a `Trainer`,\n", + "which orchestrates data loading,\n", + "gradient calculation,\n", + "optimizer logic,\n", + "and logging.\n", + "\n", + "Luckily, we don't need to sub-class the `Trainer`,\n", + "we just need to configure it with keyword arguments." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And that is where we'll use the `pytorch_lightning.loggers.WandbLogger` to connect our logging to W&B." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb_logger = WandbLogger(project=\"lit-wandb\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> _Note_: Check out [the documentation](https://docs.wandb.com/library/integrations/lightning) for customization options. I like `group`s and `tag`s!.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can then set up our `Trainer` and customize several options, such as gradient accumulation, half precision training and distributed computing.\n", + "\n", + "We'll stick to the basics for this example,\n", + "but half-precision training and easy scaling to distributed settings are two of the major reasons why folks like PyTorch Lightning!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trainer = pl.Trainer(\n", + " logger=wandb_logger, # W&B integration\n", + " log_every_n_steps=50, # set the logging frequency\n", + " max_epochs=5, # number of epochs\n", + " deterministic=True, # keep it deterministic\n", + " callbacks=[ImagePredictionLogger(samples)] # see Callbacks section\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🏃‍♀️ Running our Model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, let's make it all happen:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# setup model\n", + "model = LitMLP(in_dims=(1, 28, 28))\n", + "\n", + "# fit the model\n", + "trainer.fit(model, mnist)\n", + "\n", + "# evaluate the model on a test set\n", + "trainer.test(datamodule=mnist,\n", + " ckpt_path=None) # uses last-saved model\n", + "\n", + "wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "> _Note_: In notebooks, we need to call `wandb.finish()` to indicate when we've finished our run. This isn't necessary in scripts." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Viewing the results on wandb.ai\n", + "\n", + "Among the outputs from W&B,\n", + "you will have noticed a few URLs.\n", + "One of these is the\n", + "[run page](https://docs.wandb.ai/ref/app/pages/run-page),\n", + "which has a dashboard with all of the information logged in this run, complete with smart default charts\n", + "and more.\n", + "The run page is printed both at the start and end of training, and ends with `lit-wandb/runs/{run_id}`.\n", + "\n", + ">_Note_: When visiting your run page, it is recommended to use `global_step` as x-axis to correctly superimpose metrics logged in different stages.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "![image.png](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAA+4AAAIrCAYAAABxtsMNAAAgAElEQVR4Aey9B7QVxbou6hvvnnPvePee8c4994R3zt77bPM2IxkWaZFBkiRFlCQIGBARRBQRMW1z1i2KAbaoYAAR4xFzQLe61U1WcgbJOf5vfL3456pVq3vO7jl7dvec8+sxenZ3hb+qvvq6qr6u6p7H/cdvfy/ciQE5QA6QA+QAOUAOkAPkADlADpAD5AA5kEwOHMeKSWbFsF5YL+QAOUAOkAPkADlADpAD5AA5QA6QA+AAhTtXHHDFBTlADpAD5AA5QA6QA+QAOUAOkAPkQII5QOGe4Mrh0zU+XSMHyAFygBwgB8gBcoAcIAfIAXKAHKBwp3DnkzVygBwgB8gBcoAcIAfIAXKAHCAHyIEEc4DCPcGVwydrfLJGDpAD5AA5QA6QA+QAOUAOkAPkADlA4U7hzidr5AA5QA6QA+QAOUAOkAPkADlADpADPjjw+5NOl9bnD5RLhj8sfW+eIUPvnytXPvaTXPX4fLnsvm+k97i3pMOQR6R+u4HynyedHhqmFO4+KodPuPiEixwgB8gBcoAcIAfIAXKAHCAHyIHS5cAZtZpJtyselBueWiCjn14o1z61SK59arFcO3GJXPvUz3LNxCUyfOJSGT5xubNf/fQKGfbUUmnZe6wcf8qZOQt4CncK95xJxAasdBsw1j3rnhwgB8gBcoAcIAfIAXKg2DnQZcCtMnLiArnu6UUy+pnFMmrSYke0j3xqqdw4ZZmMfGaZDH96uYx+dpWMfH6NjHpmtVz7zEoZdO+H0rBzP6ndrJmcfPbZOekuCncK95wIVOw3KcvHjogcIAfIAXKAHCAHyAFygBwoTQ7UrF8utzz2rtzy/AK5/plFMurpRTLm2cUy4cXljoi/7JY3pGGzblJe3k269bxCLhwwTkbc95Fc88T30vWq+6RB5y5Su7ypnN24gTTpVi6NOjWSE/5wSlb6Kyvh3rbpSbLmrZrOjvNciRy2vVzzw/ileWOy3lnv5AA5QA6QA+QAOUAOkAPkADkADjRq3klunvSt3PXSL3L71J/l9qlL5KnZq+WRmatk2F3vS5Pm3aWsbrm0LGshvbr0lKsGXi6X9Ogr1155g1xx9Q1Sv3lLadCuhdRt0VQatm8sLS9qKc0vaC4tezWXE08/NbCGzkq433ft6SJf13Z2nOdK7rDt5ZofxufNSg6QA+QAOUAOkAPkADlADpAD5EBpcuDc+s1l9JPfyejnl8qYycvkxj8vk4dnrJKZn66ToWMnS3n5+dK+WVtp3bhcLu7cQy7q0FW6n9dZTj3hVPnP3x4vDeqVSceOXaR153bStFMLadmrpTS7sFzKzm8kTS9oJg06lMnvTzo5kI7OSriHPUMetj3eYKV5g7HeWe+ZOHDW2edK74svcXacZwpPf3KKHCAHyAFygBwgB8iB0uPA1fe8J3e8vFLGvbBSxr6wSm56abXc89oqGTH+eSmr10x6tO8og3v3kf/813+Rf/s//yT/8o//r/yv//Hf5V/+8R/lf/73v5e//7v/Jv/wv/5B/s+//G9p1aOVM9PepEdTaXZBuZRf2FKadW8utcvrBxqLZiXc/ZKXgrz0SO6XGwxHbkTJgd/87ni5/Ior5aWXXpLp06c7O87hBr8o88K0yH1ygBwgB8gBcoAcIAeSy4Geg2+T+2ask0ff3CC3T1sn46etk1umr5ORD8yRdi3by7BLh8iogQNlePeu8uR1w+XGXj3khgt6yISLe8udF18iEy68UPo2bigX1q0rp/zrP8u5Dc+R5j2aS9PuTaVBpzJp0qmlNO/cRpp3ai1n1fU/kZRX4V5IS+Br1KwtY8feJM8995xMmzbN2SdPniwTJtwq9RqUcWDPj/iRAwXMgY6dusjUqVPloYcellZt2jk7zuEGP3aeye08WTesG3KAHCAHyAFygByIigN/OONcGT7hz3LvGxvk7hkb5fbXNsqtr66XCdPWSp9eA6VjeSvp27mz9CyrIw/06SnfPvaw/DDlOZl9910yccQ18qfLr5QH+w6QR/r2lTt7dJFr27aQs0/5nbTo2VzqtWss9Vo3lvKOraRV57bSqlsbad6tmZxwir+P1WUl3M8+8wSZcMVpzo5zLyD9zrj7teeVTi7umG3r13+ATJkyxRHpTZo2d2bg4I5zCHf4IUxUM3OnnXG2PP74E3Ld6Os9sc22zLDZt1//wHYhbp5//nmKnN/+3nmQc/fd9wjqKdt6YLxoO6Bbb7tdJj3zjNSp1yBVZziHG/xYH9HVBx6E4gFpNu1QnPXkpw1E+4q220/bUKg4xFkHTDu6+5RYE2tygBwoVQ607thbLht+m9z21Kfyxxkb5M6Zm+WO1zfK+EnfyIWde0jNU0+Vy9u3kGeHDZJZN42WqVcNlWeGXCqTrhgqL14/WiZeeaVMHj5CHu7bRyZ0ai/Xt20h55fVlIYdm0idVmXS6Lzm0qRjC2nbs4106t1eWvRoInXKa/kaiwYW7hDZq2fXTH2cDufpxHumSg/bXqb0bP9LB14mkyZNcmbgbD+9xgwdwiCsuuXzmC/hrgNFiM6g+fcatGKg6jZriUG5LknGMR8PITKVQXE08xFGXrzKnCk/bv6wFQc2bnlRN607W1gpnqYw8QqLMpk8w7l5jbRwbdaNpme7m2GyxQo2H330MfnDaWemuI9zuNn5UhzydVTMzHJByJbKyh5th7S+84Vz2Ha92kAzHfDTvD9MP/s8XzggD25tsp0+rzkoJwfIAXKAHCAHqnLg9yecIv2uuEn6Xj5Oho2+T0be/YbcMW2pPPjmRhl/3wypffpZctpvfyM3X9hNZtw0Ut67dazMvH6EzB5/gyx88QX5btLT8sn998m7t98pH/zxTnlyYH+5tVsHuaRxHSlr21jKzmsmzTo1lZY9mkuHS1pJh94tpFHn2tJjaFP53fGZX90MLNwx065flNcj3LKt+LDtBclHeYtW8vTTT0vnLl0z5h9hEBZxgqSRTVgVSNmKlGzSzBTHHrRqHiE+7EEiBuSmELHjZkpL/du2O0+GXzPC1+yVxjGPmkcTR+QF+U2KaIBoNPNn5j+b8xNP/oNcNexqh9PZrhBRYWkLWsXOFCYa1nRDvlEmMz7O0127ccSt/rLBBHHs9NUOVtRg1+tMx7DwNe8PpAm8bLdMecmnf673Xrq85UuwpkszDD83jtp2UY/2vWCH0euwcdD7xa1N1jR5rDpAIx7EgxwgB8gBcsDkQFnTNjLo6vEycNit0vfyW6TXwLEydMzjctMj78mY21+Q0084WWqdfIJc2a6FPDq4r7w6+ir58bEHZO4jD8rUUSPkgT4XyyP9+sr9l1wkN7RvK5e1aCbD2raWS1s2kbLzGkmzzk2krGUtaduzsXTs00I6XFIu3YY2lp5XNpGz6mbW0yUt3PFOO3azwtKdBw2fzlY6Px2AhSno0qXnx89r0OrlbtrU8gQVy7/7/YnS++I+WYt3TdfG0UvEmXmO6hx5sfOXa9pYAj7qutFZi3fUE8QHdogLzQ/y+eCDDzk7sIU7wj711FPObtYvwnoJda0XM7ymYR41XBj4hFnnYeBri3TgDBxxP5kYxHWe672XLt9hC9Z0aYXp56etA1fjEu5aVj/51LA8csBKDpAD5AA5QA5UcuD8ngNk6IhbZdCwW6T/lePlsmtuk0uvvFmuvPaPctmQG+X43/5G/vC730qn2mfLLT27yNyHb5dPJj4q13TrKk3POE06nHW6XNOysdzYvpVcUPN0OfPf/1n+8K//W2qfeLzUaV5PmnUsk7KW50rzjvXk/P6t5JJr2shFIxtKz2GNpUnHmhnHgIGFe5Cl7X7ecQ9iL0xiYZnsAw886Gu2XdPFrPvDDz8sZ5xVIyOwGiebowoWvHuLAb4uqbWFDq7VD0fTH+cYQA4ecrkTRkUYjqYQgqBRG+aAU/OgfhBsbu+4+xkkIj1bqPjFJRcBoWUwy4t0TREHP1zjiLLiHGEyYWuXxwxvr0DQNBVLsy7UDUfNp1ed+MUM4XIRl8od1DnOYQ9Y4vrq4ddUE+4oD3hmYqK4ap5NzOGGa5NvGs48etWfGUbP8dduDz/ySIrLJq72Ob4mP+DSQTndw7nia2KFMpj3kZuIB56Kl+ICzOGm5dO6gj0Vx+qn3FK8/BxzufdgX/OpedB7S/MGLgEH+Nv3DPCAmx1Xy4aHHCgv4iNczwt6OVi42UTZ3ezAFmyon50HO/9ebaCJpVlPbhiY9a44aL3ptV3Pmj/Fz0zP7dzkkps/3SoHaMSCWJAD5AA5QA6YHLjq6hvkhpvulxGj/ihDrr5Vrr7+Hhk0/DYZfsO9cvnVY+X3//H/yQn/9m/StWFteWhwX5k0fIh0OfsMuaThudL4hN9J4xP+XS6pd7ZMGjFcTv/nf5S/+7//Xs74j3+R/k1qSJO2daXdBeXSsEUNKe9QR7oOaioXXVsmPYc1kv5jWkmPoY0zjk0DC3cUDmLbz8fp/H5V3q89E9hczzFIwgAJR7+2sonj17YZTgeM5iDPHoxhsGcONHWgq4NAHZCagz21q4N4hDHTwDV2DacDSOQNccz0NL92vtQdR6SNQaeZB9Pf73m2AkLLoeVFejZO8EMezTDAwCyrHcfGzcYA1xAWyi2U38YSNhQjO223OvGLlRkuW3GJvCG/EEMQK8ARZcK5ikW4IS0Ni2uUU8uCo1nvODevtW6AvVleM/8aRm2afvY58oePSCLspQMHpd2feOKJKnmxbfm9zgVfu8zARjkC3tiz7yiX+isuJkfhb9q08R5707gUH/2WD+FyvffMOseDSNQTyoe82vk1y4d4ev8gDsqq94xbfMXEtmne27BjPnx0u9b7Vu1pnoAFMDYxd8PRrCeNo/lWm4qJlgP+eo74ateuQ1yb/hrOPtrlsv15zUEqOUAOkAPkADngzoFXZ86QL778Wn74bp7Mnv2+vPjiTLnv/qfl2jF3yCUXDZKzTjxB/ukf/qf0KKsrr46/XvrUqyHn/Ps/Sa0Tfi/n/uY3Uv/438rw5uWy6u3ZMrisgTQ+/j+ka83TpXe9GtK3T1tp262R1Kj7B6lddoZ07NtQ+l3fRIbe2louHlkuHfpk1qRZCXe/le1nxt2vrbDDYaCEQRmOfm1nE8evbTOcDvDMQZrtZg/qEN8cNGIwaA5i4W/bQBi3gajbwM/NDTa93M3yeKVjhsl0DgHRf8ClcuVVw+SEk/z9ZYKWF4N3c0d+ND0TM3ULiq3bgBpuSCcTPnbcMLDScuDYoGFjhxetWrdNldn0dztHHvTegFhHGYATdvUDtohrXiOcCh+ERdnUPs7Na3U368j2Vz/Y0vBex0w4m/G88mKG8XueLb4mH3Fulh3tjB/hbuJiYo+8w54pOv2Wxy1cNvceeGG3P2ob5YMfwqibySN106PNA7f4dhjEtW3a8YCRiSHiwA3x3Pjk5qZ51CPspcPd9Nf86MMwkwN2fcI+3PRBmqbndvSTT7d4dHMfxBEX4kIOkAPkQOlwYMTNV8szU/4sb856X+bO/VZ++OFHmT37bbnssstk6pD+8kDvHlL7+N/JwA5tZPKIK2VQ/VoytFEDGdSwnlxWVl9GNm8ij13SW1ZMniwzR10rY89rIxO6dZLx558nl/RuIc3a15J6Tc6U+i3OlnYX1pX2fevKRaMaS89hZdKiWx6WyhcLebNdKo/l9YibTxzcBqFIDwM77F7+GHDqYNk817y6xdPBoyke3OJ6DQa93DVNPWre9TroER8FxKA3yMcB3cprp2sOpOHnFcfExDzX8LYQwzXCmWHttHENXJAH08+tTkx/v+cnnnyqM0MOYYBzv/GQZxUfyNtN4252BAPq2vSDPfta6xnxcK5pqrte20fYNmdV4a/Y2vjYcXHtl4cImykvbvbd3HLBV+9TrWuzjHDLVbgjv6gb5SXO3crgxy2bew/lUQ7ZaWiZzTzZPEIc2ND846gYucV344pt04yn4U37eo542LWONP8mx+z4mje3ciu31b7iovmBOz58qu5ID+lrePOoYUxs1M0tn+rGY+kMOlnXrGtygBwgB7LnQNNO9aX74IEycORYue3eR+Xu+x6WkdeNlnbntZMp3dvKrW3LZESrRnLbpRfLq2OHywMXdZdHLxsgEy+7TB7vc4nc1qmDPD2on3x1110y9fKhMv68VvJAr+7ycL8LZHD/FtKqcx1nqXzDVudI087nSr22Z0rHQfWkU/860qjDWRnHalnNuIc9kx62Pb+EdT42d9M43//PHufH6XSgqANEN/FhDhrdBp62DRMn088coGoYNzf4eblrPD265Vf9Mh2zEQ6waZbJKw0TMw3jllcznI0twmu9qA09ZsInXVw/+dd07GO2ohJ2UD4VA8g/BDXyafu5XUOMQHRC7GschMO5eQ03c3crq5ubGcc8z4SzGTZTXsywXue54muKQptPiiHKpOmb/HPDBWF1tYPG0WMQbDSOHrO99+wyqT0cVbAijLqbnFN/vafs8qq/Gd8OA7umTbd0wQNNQ/OhRzfM3Nw0vB7NeoIb0tB7Cdemv5ZD7wszP+nqU9PyOvrJp1dculdtl4gH8SAHyAFyoLQ4UL91fbnw8j4yeOwYueDyK6TDBRdKu07nSeduHeWxoX3lznaN5aF+F8gtF3WWGeOullfGDJfnhw2RP/XpJTe3LpfR5Y3l6X695PVrrpBH+3SXG1o2lrEtG8oDF3WSwf3LpUm7c6Rm2elyToNTpU6L06Ws3dlS3rWWlHc5S2o2ybyiOCvhHva7637thX3zYFCa5L+Dswd89mDfXOaOwZo5Y4lBqxke2NmDW4QxB78YOGLXcDqgRFycm+lpXbgNEs3BKcLZedO4fo7ZCgfY1nJ4Dc4Rxs4r3ICJWVY7/za2dnjYaNGytSNKNA9mXXbo2Dn1ioZijjiatludqL+fYy6iUvOg+dX8a55wVD87rOYNmGKW0OaPXsMG/NWm2jExh5umna7+NE03HqqffbQxt/0zXYeBr3lvajkVH73Wciv/FHfbH/lFGFO4m++0q0A08c5URvjncu9pmlom2MM3E5B39TPzg3Mtn10W+IEviodbfDdMTJtI344Hf5tz9n1r5h/ndngbR+RRy6F5Qjpm+upv58fksMY101fs7DTta9OO7cfr0hqAsr5Z3+QAOUAOBONAWcu6csGA7nL56Guk9+A+0uvSLtJrQBfp0r2jXHPVEHni0t4ydfhgubVnB3lmWB+ZOmKg3HpeM7mpZQMZ3qiuDGvRTLqefopc07imDKhzjgxuWl+ualxT7uhaLn0uaiRl7c6R+uVnydn1TpFzy06ScxudKjUaniy1Gp0s9ZqfkZrQ8Kq3rIS7nxnyIF+L92PPqwC5ul868DKZNGmStGrTzhMs+D377HPOX2zlmp6f+Dpow6wlBooYtLoNGHVAC383IWSKA6SrdnUAbH6BGfF1QImwOqhU2whrLt+100Y4TQ+DX6ShcXHUwauf8muYxk3KHTsQEOoW5GiX1y2uOdA2/e3ymfnHuZZV42CwrHVlYgF/zYfigQ9kAU/4mfEwSE9XJ5pWuuN/Hn+SXDZ4iLNEHgIzXVgvP5TP5IIZzvazrxFWy2uKDpyb12a5gYsbv9WO8tXMh30eRKzYebFtpbsOC18v/mhZgavyBXWh70EDEzdcUH4V7rh3Idw1Po5qN13ZTL9c7z3YstsQ8B7iXd1RRk3T5hHqSPOPc+Rfy+AW3w0T26ZbPJuHZr1oeM2H3QZq3s2j2tO8mvUI27jW99TVPtzUBuKZ94KJQ6Z6NNPSPJvl0TR4DDaII17EixwgB8iB0uFAvabnSPf+rWXgsB4yeERnGT6us1w+uoMMvLKrXHFlfxnVq5v8qU8XueOCDvLM8P4yrn0jGdW0plzZ8FwZ2rS+jGnbXMa0biYjm9SXUU0byLWtmsg1zerJhE6NpccFdaRJ+xpyZp0T5fTaJ8jZdU+RU878nZxR90Tpf8l5clW/C1PjAS/OZSXcvYyZ7vjqvHxdu8oONzNMEs5/87vjpU/f/jJ58mSZMOFWadK0ubN0Hu44hxu+Vo1l8s88+6z0uujixJUhCTjmIw+nnnaG1KxdN3F4Y4Cc1AExeFu7boNA77Tno+6itgnBFMVX5UsF36Tee1HziumVzmCNdc26JgfIAXKAHDjtnFOlS5/G0uPSpjJ0dAcZMrq1XHFDaxl0dRvpN7iLtDm/THr3biz9OjSS+/qcL2PbQqDXkhtbN5by438j/RvWkFsuaivXd2oidw3qKp1rnCHXt2wkd/RsKX0GNZcWnWpJrcanSe3mp8tZ9U6SE07/jdRq9Qd55NarpXeXjhk1T8kLd71Jzzy7hiPOIcimTZvm7Di/7rrRUqNmbQdI/I87xTtvagh3rxlp5ROP0fIk6v9xZ/1GW7/Em3iTA+QAOUAOkAPkQL458LvfHy9d+zaSzhfXl15DmsnFVzSTfsObycVDm8qlw1vJkDFtZMj4VnLF4BZyW/c2MrZ1Hbm67Gy5tUNLGd+utYwqbyJXNKovl9WrKVfWqyM3tWkut5/fUm6+pLX0GthKWveoLc3OP1cad6ohdVufKTXLT5HmnWrKzEduk5NPyrxSNivh7ud/14Mslc93JYRpn+K9dBsNXQabaclqmHyjrdLlG+uedU8OkAPkADlADpAD5EC0HKjR4DQ57+K6ct7FdeT8S+tJ5371pNeQMrlsdAu56ubz5IpbOsiIEe1lbKcmMq51PbmxZV25sW0jmdC+ldx1fnu56/y2Mv68pnJ3t3Zy9wWd5NYebeXqAa3k/P6Npf3FdaTLgPrSqkcNqdvqdKnb5jTp07+V3HbVoIyz7eBBYOEeRJD7EfjIRJzvuGdzM3TqfL489tjjUrNW8pZxZ1Mexom2QSDexJscIAfIAXKAHCAHyAFygBxIHgd+d+Lx0qxrDUdkd+xbX9peVFs6D6wlPS+vK4NHtZPRt/SWm0b3kps7NZNxLevK2Db1ZGzbhnJ7t/ZyZ49Ocl/v7nLPhcf2i7rKmJ7tpUOvetJ1cJmc16e29B/WTPpe2UzKu5wr9dqcJreN6iNlderkR7jn4931uL4qz5sleTcL64R1Qg6QA+QAOUAOkAPkADlADpADcXGgVpMzpX2fetK2dx0p736OtO1zlrTsdYZ0699Iho26SMZf11fu79NFbmrXQG5sXlPGtqojY9s3ljsvOF/u73OB3HdhV7mza0e5tVs7qV/rBGnc5SzpOLC+dB/UQEbedJ4MG91O2veoKWWtzpRBF3TyJdqBReAZ93wI90KbcY+LREyXDRg5QA6QA+QAOUAOkAPkADlADpAD+eVA3VanSese50qzbmdJm4trSHmPM6V193OkV58WMunmq+Wxft3lri7lclPz2jK22dkyuvFZMrLxOTKqvI7c3K6RjGtfJv2anS3/evw/Sq1Wp0irS2pIzyGN5OrrO0nfoeXSvNPZUta8hm/RjvoOLNyDLJWnIM8voXjDEl9ygBwgB8gBcoAcIAfIAXKAHCAHwuXA8SefIGVtzpRWvWpKx0trStu+NaVVz3Pk7uv7yvznHpXnB18o93VtKRPaNpQbm50j45qeIzeX15Db29aWuzrXl7u6N5Le550jf6j9WynrdIa0719PLh5aLv2vbCkde9eWei1Pld+fdEJ+hTtI4ffddS6BD5dAvCGJJzlADpAD5AA5QA6QA+QAOUAOkAP558CJfzhJWvWoLR361ZF2F50rI4d3lLXTJ8rqKY/JG6MGyQPnt5Q/dmgi41rWkTFNa8iYJufIuFa15Y5OZTKhc5l0b3eWnNPoBCnvUkN6DiyXSy5rK136NJAmnc+U408+PpBoR30HnnEPQhLOuOefUEHqg2FZH+QAOUAOkAPkADlADpAD5AA5QA7448CpfzhVhvTpJPeP6Su7Zk+WjS8+Lgsfv01eGdZHHurWSu7u0FRuadtAxpTXkBvLz5XxrWvLHR0byq3nN5ELu9aR8q7nSOvuteSCAc3lwkGtpbxLLTnhlBMDi3bUV16FOwnhjxDEiTiRA+QAOUAOkAPkADlADpAD5AA5kEwOPDJqmKx78SFZ/OSd8t19N8iz/c+Xe85rIvd0KJO7O5TJLW3ryPg2teWP7evJoxc0lzt7NJU7bu4rlw5r6/w3fJdeTaR+41pZCXblBIX7b5NJDq0gHlk/5AA5QA6QA+QAOUAOkAPkADlADsTLgcY1a8kDV10qH942Qp7u01nu6dBY7unQyNnvbFdfbu9QX+7t3FDu79ZUJnRpIC89dZOMv7WPdOrWWE45/ZScRDvqnsKdwj1nErERibcRIf7EnxwgB8gBcoAcIAfIAXKAHIiGAyefeLK0r1dLLmlwrlzXsr5MaNdQ7mzfQO48r77c0amhjGnbQHrVO1vaNKkn/3lCsA/QpatDCncKdwp3coAcIAfIAXKAHCAHyAFygBwgB8iBBHOAwj3BlZPuiQv9onmiRpyJMzlADpAD5AA5QA6QA+QAOUAOxM0BCncKdz5ZIwfIAXKAHCAHyAFygBwgB8gBcoAcSDAHKNwTXDlxP9Vh+nyySA6QA+QAOUAOkAPkADlADpAD5ED8HKBwp3DnkzVygBwgB8gBcoAcIAfIAXKAHCAHyIEEc4DCPcGVwydb8T/ZYh2wDsgBcoAcIAfIAXKAHCAHyAFyIG4OULhTuPPJGjlADpAD5AA5QA6QA+QAOUAOkAPkQII5QOGe4MqJ+6kO0+eTRXKAHCAHyAFygBwgB8gBcoAcIAfi5wCFO4U7n6yRA+QAOUAOkAPkADlADpAD5AA5QA4kmAMU7gmuHD7Ziv/JFuuAdUAOkAPkADlADpAD5AA5QA6QA3Fz4Dgpku3nn3+WAwcOcCcG5AA5QA6QA+QAOUAOkAPkADlADpADRcUBCncSuqgIzYc3fHhFDpAD5AA5QA6QA+QAOUAOkAPFxgEKdwp3CndygBwgB8gBcoAcIAfIAXKAHCAHyIEEc4DCPcGVU3r9XqcAACAASURBVGxPiVgePvkkB8gBcoAcIAfIAXKAHCAHyAFyIDgHKNwp3PlkjRwgB8gBcoAcIAfIAXKAHCAHyAFyIMEcoHBPcOXwSVTwJ1HEjJiRA+QAOUAOkAPkADlADpAD5ECxcYDCncKdT9bIAXKAHCAHyAFygBwgB8gBcoAcIAcSzAEK9wRXTrE9JWJ5+OSTHCAHyAFygBwgB8gBcoAcIAfIgeAcoHCncOeTNXKAHCAHyAFygBwgB8gBcoAcIAfIgQRzgMI9wZXDJ1HBn0QRM2JGDpAD5AA5QA6QA+QAOUAOkAPFxgEKdwp3PlkjB8gBcoAcIAfIAXKAHCAHyAFygBxIMAco3BNcOcX2lIjl4ZNPcoAcIAfIAXKAHCAHyAFygBwgB4JzgMKdwp1P1sgBcoAcIAfIAXKAHCAHyAFygBwgBxLMAQr3BFcOn0QFfxJFzIgZOUAOkAPkADlADpAD5AA5QA4UGwco3Cnc+WSNHCAHyAFygBwgB8gBcoAcIAfIAXIgwRwIXbjv379fFi9eLO+9955MmzZNnn32WWfHOdzghzBhbz///DOJlmCiFdsTL5aHT3HJAXKAHCAHyAFygBwgB8gBciAqDoQm3A8fPizff/+9TJkyRT744AOBkN6+fbscPHjQ2XEON/ghDMIiTlhbUoT7vn37ZOfOnbJt2zbuxIAcIAfIAXKAHCAHipYDGO9g3BPVoJXpUCBFyYGvVm6Sps98In83YaYcN34Gd2LgcAGcADei5KKmFYpw3717t7z++uvy/vvvy44dOzJqcYRBWMRB3DC2JAh3dF4Q7FhRcPTo0TCKRRtEgAgQASJABIgAEUgcAhjnYLyDcQ/FOwW1CotiOUKYUbDzYYXXAxtwIw7xnrNwh/CeOnWq/Pjjj4E7FcRB3DDEexKEO5485+M1gMDAMgIRIAJEgAgQASJABCJAAOMejH+KRbCxHHwIAQ5gVtVLtNGdgh4cAEeibi9yEu5Y6o5Z82xEu/YliAsbuS6bT4Jwx1NnzrRrzfJIBIgAESACRIAIFDsCGPdg/BP1AJbpUWDnkwOcbac4z/SABhzJJwfdbOck3P/61786S97TdUp79+6VV155RZYuXeoZDMvm8c57LltShHsuZWBcIkAEiAARIAJEgAgUGgJBhPvWfQdl3s4j8sU2kc/yuMM+0kF6bgNgulH4p+NAJtFGfwp7cCAdh/Lhl7Vwx9IofGQu0zvt+CjdM888I7/88otnPwQbsJXLMnMKd0946UEEiAARIAJEgAgQgbwh4Fe4Q0TnW7DbDwOQHsU7RXpQEUVhTmHuhwNBeZVr+KyFO/7WDV+I97P5WT4OW7CZ7Ubhni1yjEcEiAARIAJEgAgQgewR8CvcMQNuC+sorpFurgNmxi8t8e9HtDEMxX3U7ULWwh3L2yGW021bt24VLKf/5ptv0gVz/GAL//Oe7Ubhni1yjEcEiAARIAJEgAgQgewR8Cvco55t14cCSDfqATbTK2yhT1FOUe6HA1Hf51kL92nTpjn/0+7VzOPd9q+//lpefvll+fLLL72CpdyxpB42s90o3LNFjvGIABEgAkSACBABIpA9An6FuwrpOI5RD7CZHoW7H+HHMIX9gCDq+zxr4f7ss8/KoUOHMrbyr776asaZeRiBLdjMdqNwzxY5xiMCRIAIEAEiQASIQPYIULgXtkiNWnxoepjkmz9/vnz00UfOEdfqF/eRgrqwBXVU9Rc1T3MS7gcPHkzbykOMP/3002ln5tUAbFG4Kxo8EgEiQASIABEgAkSgMBCgcE++cP/2229l7ty5sm/fPkcc//DDD/LJJ9H/D7UpdJCfGTNmON/MwvGrr76icB9PwRyV6A4jHZPPUZxnLdwzLZVHV7NhwwZ5/vnnff23OZfKF0bnzFwSASJABIgAESACRMBEgMI9+cId4+xZs2Y5r6/i+1NYEYtxehRiA2nggQG+fbVnzx4nTVy/8847smjRIud6yZIl8u6776YeLCAcwuuDhqjyqemEIepoo/gfQihfojpmLdzxITksT0+3zZs3T9566610QVJ+sMWP06Xg4AkRIAJEgAgQASJABAoCgVIU7hCUn376qfN3xpMnTxZ7x98cwz8u4ekmJFBPmNnG5Nv69esjE+342+fZs2fLiy++6KSNv4iGGx4krFq1ysnH2rVrnTBwhz/yiPAIg4cObuXJpxtFd/GL7jDqOJ8cdLOdtXD383dweGfliy++kE2bNjlfl0/X+/Dv4CrQOXz4sKxYuVLeevsdeW3GTGfHOdzgx40IEAEiQASIABEgAklCIB/C/d0V22TE+Dvlon4D5YXPfkj9jdw7y7bIVWPGy4V9BlRxz/TBO7dBcC5uW7ZskZkzZzpjXDc7GPtCfKYL4xYvn26YaX/llVec2fbPP/9conqn/OOPP5bPPvvMSW/FihUOLq+99ppgV1EOwQ6s4AbcEA4PPaAjoCfyiYub7TBEHW0Uv/h3404+3bIW7vv373eeMuJG89qw5AXvuOOGO3LkiFcw56kbnkzCZrZbMXycbt++/fLpZ5/L/AUL5aDx4T98K2DR4sXy2edfCMJwIwJEgAgQASJABIhAUhAIW7h/vPmg3PzIU/LIq+/IVTfeItff9aB8vOmgfLB+r9x476Py4MuzpN+QK+XJtz5OCfqohbufGXe8Qw6x/Pbbb8vu3bsjF5+mgFiwYEFqebwum8c75maYfJ1jcg44qH2dVd+1a1fKDX641tl4DYt4iK/XUR0puotfdIdRx1HxUdPJWrijs8DNhP9z99ogOHETZtqwRP7777/PFCytf6ELd8ymf/nVXFmzdq1nOeH36edf+Pqav6cRx2OBzJk+Xaan9lkyd0P6GHn13TBXZk2POQ9ZFHDBnOkyfc6CLGIyChEgAkSACBCB4kEgbOGO2fbp3y1xRDnEef/Lh8lbv2ySe6a8IpM/+c63WDfFvA58oz5iZv6NN94QHKNO20xv8+bNVfIA8RzVcnnMnmOmX5fFm/lKd47l83gXH/HThcuHXxiizr+Nv8mcfVXbgwXfUzj7xy8+rPLBvXQ2cxLuEJuvv/66/Pjjj1XZFuAKcWEj12XghS7cV61eLfPnL0j7Ib+jR4/KDz/+JAib/VYh2md9vbHSxMI5Mn36HIlNglK4V9YFz4gAESACRIAIFBgCYQt3U3C/uWi99B18pdz2p+flrmdekk9+PUzhfiD5H8Ozxcff/vY352N0+nE629++RjisVIBOsP2iuI5ONK6oGH9vXiGVacLtoMx5N3+CdOSygyJV0sxfWpXlymcaFQ8/on7gEQUXzTRyEu7oV7D0Z+rUqVmJd9yMiAsbuW6FLNwhyP/6w4+Cji/Thi9s/vWHH9IK/LQ24hbpbpmjcHdDhW5EgAgQASJABAoCgXwK9w837pdrJ9zlvO+OpfKmqA9ybg5+cz1fvny58wV0jGHtj9K5XeOd7bhn3HMtc7bxwQ2M0fGhPnyw2hTuWBaPGXVghiOuNR28ioAVuXjdFvH9ckzj53qMRmzOkEmbJRYBTeEezkOEXHkWNH7Owh09CoQ3Zs2xbD7dO+/a+yAMwiJOGKIddgtZuOOVgq+/+YvzEQ7FyOuIhgxhESerzRHJ02XOwgyxj4XT5fRVwlt+02fNFWP+XrCEHDP6zlJyLMfX5eR2vCruWCpvLuFPs3TesTNHFlj2quRRpDL9Y68EVPHHAwzku4qN6qsONn49y3iloKp/taXyzkORylcQqqSXAW56EwEiQASIABEoVAT8iqogQlvD4n330Xfe77zrPmfdnliFOz44h6+y4wNqmEH2U+6kLJUPKhByDY/x6ocffigvvfSS82V4CHcs11e7ugweH7tGWBwh3uGuYYAdPk6HL8vDDuwhrPrn8xiNcK+Ybc80Swxxv23Z3ypEPhoJnSl/d5OYU35V7RybyU81KjtlkvMf8dWX5cu+TTLy2P/HO4K+WhwXkfv9TnHi4ehsar9qush3CksnvztlUtp8z5DjLP8qNsZXPOyogsfufVVwQHbsOKk8HCtnWNf55KCb7VCEOwDCUne8p46PzOEjEhDS+PgFBCZ2nMMNfgiDsLkuj68gSsVvIQt34PDNX751vrZplsntHF8ARdhcsEsJaktwp9JzxKwpnCsEdYUQxbkpYKsvvVf7VYTrMYFcuUQfwn5Oxbv1KfFcadexkTZ/0yuE97FMVwjsyjzjujItkQr/SvuiIjuVxkaZO8t4yCDH4qT8RSriVNqoKtxNjJApG6cUujwhAkSACBABIlBUCPgRsBiEqhj3e/x061F5bOb7cvvEKdJ/6DB5/acVgW1oWm6D4CBu+J9xzJz/9NNPVcSj/nMSjrBnX5eqcEe58ZV4U6ybeH/99dfOLLzpBnEPd9NNz2EH9mBX3fJ5DEvYpbXjiN7MS+KdWXmMLM333h1xa8Y1HwJAnJt+x8S6Cv5j4jf1AMAU7YaIP87JnwpyS7yrYDdsHjfezAPCV6SbEtEqyI00Kh4UGHk9FqayrBU2UzY07zYe1dK28huyYNd6zScH3WyHJty1B8KX4fHUDMtb0MA9++yzzo5zuMEvl6/Hazr2sZCFO8ry409/kw0bzHlru4QV1wiDsDlvKlyd2ehKMQq7EKSm6IWbI3x1htxKvKqAPTbTbYW1w1QxUe1BQXWRnDG8VAhvO9+peHYaTvmrltspY0qoQ3hXPgiosFORhj6QqFIm234qYZ4QASJABIgAEShuBMIU7hDrH22qEPmv/7hcbps4Wd5buU0uv+5GeeLNDx3hjiXzj772TiqcivN0R7dBsF83fFQNH1dbtmxZFdGIFaT48Bx2zMTjY2846jUmrSjcK2fZTbwp3GdIWmFsCE235fQ6C68CEsd0y98dP0MwV7cJgWwIaCf9CuFdKaINMewi6l3TRzhNt9rDhurivnq+quPkGobCvbA6mUIX7uj08FX5dEvgMcs+9+tvfL0L77v2qs12H5t5Tn1xvnLpd2rJO4yn4h3zTwleN+EeUFTDvouwTpXJVSQfy7fbAwOjLCq63exXEe52+VxsVBHu+nDD5UFIKt88IQJEgAgQASJQhAiEKdzvfm6a9Bk4VCa9/6XzN3D433aI+TuefkEuGz5KJr79iYz+4wMCUZ9OqNt+pmgMco5XOvFutttH0pIo3PGQAf/65LdOgEU2cTJhqEvlX3zxxayXymOWHbPwWCoPO0W3VN4Rv7ZYNsTxMfFeXagem0F3a0tcZsArg1XOnlezqbPhlYFTZ36Fu2MzFcs48SHcK2b/rRl6fXhhCf5qeXfC2bP91XE0H3KEdZ7pPgjbP/QZd6OaIj0tdOGOD9QtXrLEmU13WwYPN3xRHrPtCBvuZi7zNs/dUlFhXzkbXUXwHpuxryLynWXj1WfxU9bdhHiWwj01464rClJC3iqXi/0q5XDxT+X32Ikt3NXfsQMBbzzMUD8eiQARIAJEgAgUGwJ+RaItpt2uX/p6vlw8cIhcesXwKkvjsUweX5cfMnKMvPZTMNGOdLIdQC9atMgR7hDwbjbclsZD5OvS+Shn3NetW+esDMCMP77Ibn4Izi3vcNu4caPzbnmQOF623NzBDfydG95Vtz9OB3e8145XaL0+TgfhjnB+OeaWh2zcwhJ2ae1UWxbuLjarC9XMIlVFdEp0WzPk1Wxa/mnzDaFcLbyH6FbxjaMlwCvSMONVXxbvhLFwqpZ3CvfC61IKXbgDcQjyn39ZKu/91wfOX77hfXbsa9aslTkffiTz5i+QTz79LO1/vWdXcxWiVkWvlyB1bLsI2iqC11W4u83CGzkNRbibwtxtht/0d5/Rr1oOK7yRXT1Ni5NbmTQij0SACBABIkAEiggBv6LKTahH5ZaNeEMcfEzZbbbdr71SF+6KEzD0+zABcUrl7+DcRWhVAe8Wxs0tJbbdBLIltKvHz/wwIGXfVbgfW6qvs+umYNdzt3xZS9yr56v6QwLXMJadKnnV9PNwVH5HdeSMe4j/hYmOK4wNlb98+Qr59rvvnR3ncMO2b9/+nMS7IzarfFzO5cNtx5aJq5B3El64oOJ/Jh3hXjnbXv0jbx4i3cXmgq+PfY3eTeS6PCBIYXvMljmr75QrNcNtL5vXVQLG1/Rd7FcV7oqLUVY8lFhY+W/3VYQ78pSa3Xd/MJDKP0+IABEgAkSACBQRAsUq3Hfu3Ol8EM3tL96CuEX5d3AbNmyQhQsXOv/y5FdMZBPHr22E05l184vxfuIjX5iJR3w/4cMME5Xw0w+6Vf1QXFUR7SpUj81Cmx9tO+77FRVfjrdmqCu/0l65VN5+5x3lddys99wnfW/+v7zxQMF6EFCBV0W+q5Tl3RUySf+P/li+TH+nbKbYt/N+TJCb5XTFw/4QXh5EuhsnwuScH1sU7gkU7pn6chXv69atzxTU8t8oGzeoIDXeXbeEvBNJxbG+2z1rlsw99hdyFeL/WHyIVYjglGj2EO4wWs0m/tJN3asKZLd30FOFcezMkrlfz/H8q7aqacF2hXj3/Y77scRSy96P4TBr1tyKBxjGO+14wLFx4QKZO8fE1CpPKvM8IQJEgAgQASJQXAjEKdw/+fWwPDz9LXlz0fq077z7GRTbYcKYLcd/vuNr6H4xsvNQDNf4Rym8d69lwbcB8J/tu3btSrnBD9dwh7+GRTzE1+uojm4iLX9u1d9ZxxL3ke9W/JWau1DVpedGW4IvyR/78nyFCD/mB2HsCOJK4Z56YOAEqXSvEg9/q7Zv07G/kTNEO0Sxq3BHmGPiPZWtg7Lg+2N/Cefk4aDMWaZ/IYdAlWmn8FWBf8yGKdoRxhMPJ0/HIlV519/Ke4iiPio+ajoU7gUo3EFJvPOe7kN2x2hbnAcV7hD93IgAESACRIAIEIFYEfArSvOxLB4frnv+o7/I0JE3pBXvOvANcoRwx2x5kNl1Oyw+qob35IOkGyQs3mvHX9R51QE+PJfOP0ha2Yb9+OOP5bPPPnNe/8TsOTB97bXXnB1f3oddHNUN/hDw+MAd3o1H/GzTzjZeSkSGKPJK3qYKd52BLwJss+VXtvEo3AtUuMfaQ8edOIV73DXA9IkAESACRIAIpBDwEo324NRLuN/y6CTp2uPCnPcR4++UDzfsc515t/NSDNcQ7VhK/s4777i+P44Pz0EMe/lHhQFm0GfPnu18GV5FOV5DwMfq8GAB+cARX4/X2XiE0y/Sq7iPKr9Ip+RFdj5ENYV7zg+gKNwp3FMdb8GcULgXTFUxo0SACBABIlD8COQq3L0EvR93nXG/YvRN8tYvm1xFO+xEKfqiSivTF+Ez+UeVT6SD2fOtW7dW+dL9e++9l1qNsGTJEnn33XedcAiPj9MhPOJFmU9Ni8I9D8vLKdxz5jKFO4V78Y8oWEIiQASIABEgAkQgbwjEKdzxjvsjr76TVrQXq3CHyMy0VD6TvwrVOI5z586VGTNmOO+w4zsAuI4jH25pUrjnQbjnYxY/Zptu3MmnG4U7hXveOnIaJgJEgAgQASJABIofgTiFu59Z+WIW7vkUCfm2jdn0pUuXyrx585xjXLPrbuWkcKdw98MBN+7k043CncK9+EcULCERIAJEgAgQASKQNwQo3A8kZqY4n6KhlGz7EW0MQ3Ef9T1B4U7hnreOnIaJABEgAkSACBCB4kfAr3D/Ypt4voPud+Y8m3BIN+oBNtMr7IcZFOUU5X44EPV9TuFO4V78IwqWkAgQASJABIgAEcgbAn6F+7ydR2IR7kg36gE206Nw9yP8GKawHxBEfZ9TuFO4560jp2EiQASIABEgAkSg+BHwK9y37jsoUc+6Iz2kG/UAm+lRuFOUF7Yo91N/Ud/nFO4U7sU/omAJiQARIAJEgAgQgbwh4Fe4Y5ALEY0Z8HwLeNhHOhTthS2goxZGmp4f0cYwxS/MM9Wx8iWq43GZMlRI/lGB5pUOOq6jR4/mrWOkYSJABIgAESACRIAIJAkBjHuCCHevMRTdKbCTxIG/mzBTCkkDMa/RP0QAR6LmLIV7iDPuO3fulP379yepP2VeiAARIAJEwEIAQoM7MSAHwuEA/sJrx44dzvgHY6Ck7FEPqJlecT14aPrMJxTuMf9HetIfRoAjUd/3FO4hCnd0XnjqjE4LAwJuRIAIEAEikAwEKNLCEWnEkTgqB44cOSIY92zdulX27t2bGMHu9eAg6gE20ytsIf/Vyk3CWffoZ7GTLtY1f+AGOBL1fU7hHqJwR+WhE8PMOwQ8d2JADpAD5EB8HICgyLRv2bJFuBMDciA7Dmzfvl327NnjjH0w/sl11/YyFztewl3dox5oM73CFfAQZphVpYCngDcFOzgRh2hHW0LhHrJwZwNduA006451Rw4UDwd0kG4evcQAZgu5EwNyIF4O4CHb559/4ew6ix+0TtzucbMNMM/Z3hdPe8+6ZF2WCgco3CncI1/mUSo3F8vJjoQciJ4D5sBcz83BvC0E9uzdK+Zu+/M6XjFH/EsD/wrR/qWsWLHS2T///MvUEnwvDpj3rZ7v3buvykM4897X9sA8so2Ovo0m5sScHMieAxTuFO4U7uQAOUAOFAUHzAE5znXQXjnw3+eIdL3GEl/uxIAciJcDeE3hs8++cAS7fhEDAh5u8AtaP6n721lJUynktT2w2wmKiOxFBLEjduRAtBygcOeAvSgG7Gw4om04iDfxThoH7MG4DtIrBvGVg3eIgN27d6f2Xbt2CXdiQA7Ew4HNmzdXE+22eEcYv/Vj3tsq+M02QNsFu71IWnvG/LCPJQfIATcOULhTuFO4kwPkADlQ8BwwB+IYnOusmx5VsKsAwEdEdcdfWXEnBuRAtBzYtGmTp2i3xTvCZqofvZ9x1PscQh73vrYDekQbYbYZbgNkulE4kQPkQNI4QOHOAXvBD9iTdlMxP2zoyYFoOWAOwG3RroIdg3kd+ONL2BVfr94uW7dtc3b9mnWmr9DTP/OX+okRMcrEgfXr18unn35eZXm8inX7iGXzCIs4bnZx71bexxX3Nu5xvd9x77sJeIr3aNtp9ovEmxzInQMU7hTuFO7kADlADhQsB2zRbgp3U7Rjxm7p0mWyYMFCmTdvPndiQA7ExIEffvhJPvzwY1+iXUU8xDviIK7f+xf3Ou553Ptu4h1tBcV77kKCYowYkgPRcYDCnQP2gh2ws6GIrqEg1sQ6qRywhbsuhTVF+8aNGx3BvmXLVjly5IhqAR6JABGICQEI5qBb0Di413HPQ8CjDTDFO75Cj7aCwp19W1L7NuaL3HTjAIU7hTuFOzlADpADBckBd9Fe8X47lsZioI6ltb/8stQZwAcVCgxPBIhA4SMA8Y42AG2BiveKB3wVbYUp3t0GynSjgCIHyIGkcIDCnQP2ghywJ+UGYj7YmJMD8XHAXbjvdT5GhQE63nP99ddfZf78BZxpL3z9xRIQgawQwMw72gC0BWgT0DaYH6wzhTvaFLbp8bXpxJ7YkwPpOUDhTuHOToocIAfIgYLkgAp3DLyx791bOduOD1Nhhg1LZPFOLDciQARKFwG0AWgL0CagbcCKHH2tRtsPbU8oHNILB+JDfMiB+DhA4c4Be0EO2NloxNdoEHtinxQO6EC7QrTvFby3ipk0/BUUBudYIosvUVO4l65gY8mJABBAG4C2AG0C2ga0EWgr3N51T0r7xnywryUHyAGbAxTuFO4U7uQAOUAOFCQHbOGOGTTMpGFQjiWxW7ZskbVr11G4U7sRgRJHAMIdbQHaBLQNKtzNWXdtT+yBMq8pnsgBciApHIhVuLd/4UtZtX2vs+P8uPEzctqTAirzwRucHCAHyIH8ckAH2TjqjLsKd7zDiv92xjutq1evoXAvcdHG4hMBCHe0BWgT0DZU/Ugdvy7P/iq//RXxJb5hcSBW4Q7R3uiZT5wd5xTuJHZYxKYdcokcKG4OqHDX91Mh2s2/gNu2bbts3rxZVq+hcKdsIwKljoAj3NescdoEtA0U7sXdP7D/Z/0WKwdiFe6rd1C4FyuxWC42muQAOZBPDpjC3RHtxvvtGJRv3bbNGaSvWr2aM+6lrtpY/pJHAMIdbQEe5qFtUOGuX5fHA0BtU3DMZ9tF2+wbyQFyIFsOxCrcP1q+WbbvO8il8ny/lp0kOUAOkAOBOKCDbF0mb3+YDoPzTZs2ycpVqyjcS162EYBSR0CFO9oEtA3mB+rwbxS6ckfblWwH1YxHQUYOkAP55EDowv3Eh96Xv6zdJtv2HZQbP5jvufx92Ns/ytzVW+W/TZjpGSbo0vl8AkXbvBHJAXKAHEgOB3SA7SXc8R5rhXDnjHupizaWnwhAuOMhHtoEtA2mcMdDPwr35LTt7GdZF+SANwdCF+5TflglN3wwX/7t3rdlzY59ctLD71cT5uc8MUc27t4vEPlBxXm68Kxo74omNsSGHCAHiokDFO4UY0SACPhFoEK4r/YU7njdBuJd25ViaitZFvb95EDxcCB04f7Cj1WF+zXv/CT/l/G1+P/n9lmyYNNO6f3qX0IV7RD0JGbxEJN1ybokB8iBdBzQAbbOuOvH6fQ/3Ldu3coZd7+qhuGIQJEjYAp3tA32jDuFO/ubdP0N/ciPpHAgdOH+5x9XyYZd+2Xr3oPy0Fe/yDdrtjo7zuG2ftc+mfrT6tBFO4U7b6qk3FTMB7lIDuSfAxTuRa60WDwiECICFO75b5PZ7xFjciD/HAhVuF/06l9k4eadgll1XdKO2XbMumPZPJbP4713zMqrf5hHEib/hCHGxJgcIAeSwAEK9xBVDU0RgSJHgMKd/VYS+i3mgTzMlQOhCfcTHnrPeW/93D99WE2U4z13U7jjPfgwBbvayhUMxucNRQ6QA+RAYXDAj3DfuHGjrFzFj9MVuSZj8YhARgRUuKNN4FL5wmjj2ReznsiB6hwIRbjjy/D4QvzVb//kKcgx044vzeOL82F/lI7CvXrFkuzEhBwgB4qZA36F+4qV/Du4jKqGAYhAkSNA4c7+sJj7Q5atd9iamgAAIABJREFUdPidk3Bv/8KXzn+w47/Y8Z/sKqDjOpK4pUNc1jXrmhwobQ5QuBe50mLxiECICFC4l3Z/wfEC679YOJCTcF+1fa80euYTZ1+9Yy+F+wHeGMVyY7Ac5DI5kGwOFLtwX758hQwYNFhw5EYEiEBuCFC4J7s9Z3/L+iEH/HEgNOEOER/XTLumy0r3V+nEiTiRA+RAoXMgauE+683Z0rlbzypCGn9FN+6WCc6Oc7dt2/btMmDQkLRh3OIhvXplTeS77//q5k03IkAEAiBA4c4+r9D7POafHAYHchLuulQeoh3nKqDjOpLUJDU5QA6QA6XBgaiFOzTCY0886eyqFyCqIcohzr02hIG4v2bkdVVEv1d4uhMBIhA+AhTupdEvsP9nPRc7B3IS7nEJdK90i72yWD42SOQAOUAOVHAgDuFuLl/X2XbMjKfbIPZfnv6KI945e54OKfoRgfwhQOHOvpN9JzlQDBygcOd76VIMRGYZ2CCTA6XFgTiEO2SFzrrrTLrXEnmExUy8zrRD4COubngIgKX3pvDHuc7gmw8JEEcfFGD5PHbM4qdLW9PhkQgQAREK99LqHzgeYH0XKwdyEu5Lft0VWn+w+NddOS+1L9ZKYrnYAJED5AA5UJUDcQl3Fdx+3j83l9Kb59pxmm62ULevIepNsY5rU/SrTR6JABGojgCFe9X2k/0J8SAHqnJgxYoV8u677wqOScYmJ+HutWQ9LvckA828Vb1BiAfxIAfIgVw4EJdw15lvnRmvLhEqXXR2Hi76kTqIdXPTMHpUP1u42/4ajkciQAQyI0Dhzv4ml/6GcYufP/PmzZPp06cLjkmubwp3LpUPTFAMXPfu3St79uyR3bt3p3Zcwx3+SSY981b8DTDruPjrOC7hDuGN5e/Y0814uwl1N/GtM/j2gwBbuKs9zPSnSzezhGEIIlB6CFC4F3+fwH6fdZwLByjcx8/Ieel70Jn7XCqMcTPf8BDktlg3hbt5jnAU8JkxJe+IETmQHQfiEO5o07BcHcLZXObuJoPgr++jm0dboGs4290W7pqGKeARlxsRIAKZEaBwz66dZf9E3EqFAxTuFO5FNeuMmXRTmPs9R7xSuelZTnZw5EB0HIhDuEOwq8BWEY9ZdLcN7uY76Qijs+squFWE49qejfcS7pqWHV7deSQCRKA6AhTu0bXN7AeJdSFygMKdwr1oBGu2ol3FPcU7G/FCbMSZ52TzNmrhbopslQUQ3PgyPES2ubmFhb8t9vEgQMW9Lept4f7c5CmpdNQOl8ybqPOcCHgjQOGe7Pac/S3rJ24OULiXuHB/+NHHqyyTfH3mGykh//U3f5H+AwfLpk2bUm5xE9YrfQwQVYDncoQdrzTozgabHCAHgnIgSuGuQllFtsoDdddZeHU3Z+bVTY/wg9h/9vnJ1US/+kG0m8IdDwI+mPOhE16X3XvN9Gs6PBIBIlCJAIU7+5igfQzDlxZnKNxLVLjv2LlTxo4b7+w4x42vbhDsuM6ncMcDAjw0CKvB8ftOeyZRDzth5Yl2SqsxZX2zvt04EKVwrxz+84wIEIFCRIDCnf2IWz9CN/JCOUDhXqLCHcIZwl1FuxLCPBaKcM802/7Ka6+nVhX0u3SQrF27Lu3sPGfd2UCa9wHPyYdcOEDhXojyiXkmAvEgQOHO/iaX/oZxi58/FO4FItwx+FuwYIEjOL1uTAjOJUuWyPbt29POGuvMurks3s2mCnccO3ft4Yhfe+k8ltHDTZdF2v6IiwcEH3/6qROm74BBMnrMjanwiKcPEGxbfmfk073bDtF+w003y5YtWxzsHnz4EcGebuad77oXf8Pnxne6sd7zwQEK93gEEFMlAoWIAIU7+6F89EO0WTy8onAvEOEOUf7OO+/Ixx9/7Cre4f/DDz/IG2+8IRs2bEgr3Jcs+dkR4hDV6W5m+JvCWgW/Cmpc33n3PQJ7sGP7w01taBxND9eZ3OCfKY+wF2SZ/PwFC+X2P96VEvJuAp7L5YungVO+8cg6jYsDFO6FKJ+YZyIQDwIU7uyr4uqrmG683MME46xZs2T69Omh7bAHu3HU7XFB/ys9yeGzBXDz5s3y9ttvVxPvKtpnzpwpP/9cIaLTpQGh3e/Sy1KC2yssRLM9g55pib3t72YD6dnCXUV/plUAbnl1E99ebpiBzzTjjrhu6dAt3kaN+BP/QuQAhXs8AoipEoFCRIDCnf1cIfZzzHPuvMWk4bfffitz5sxJu2MSF+Iex0xhYS+uyUgK9wMHBANAW7zDDTPtKtpxnekGCjLj7ke420vczTh+hTvyrPnCLL+fmXYtp5dIt93xbjvecf/iy6/SLpWncM+9AdK64ZFYljoH4hDueF3qm2++kbfefltmvfkmd2JADuSJA7jHcK/hngtjo3Bnn1nqfSbLn/4e4FL5Alkqr0Q2xftHH30UWLTDjgrtTLPbbqLbnlHHzLkptO049rWWw55xV3ccVcDjvXqcm35u536WymOpCN519zPbHtfTKbey0S19A0Z8iE/SORC1cIeA+K8PPpDVq1c7D3vDEBO0QQSIgDsCuL9xr+GeC0O8U7izT0t6n8b8xctRCvcCE+64YUzxHmSm3bzZbAFu+um5m+g240FU20vu7Tj2tdpOJ9wRJsjS+XQfp8PseRDRjvD8OF28jZJyhEfWQzFwIGrhjtm/tWvXuqsMuhIBIpAXBHDP4d7LdaNwZ79XDP0ey5A/HlO4F6Bwxw2BwSAE6bJly5zzoDeJCmP9ojviq5vOxLuJblu4Y1Yc4RBfZ8r9LJU37WjaEPNYDYBrXRWgtuHmteMdf3tZvF6raDe/LK9+XkfY80qL7t71QGyIDTlQnQNRC3cs3UU9cCMCRCA6BHDP4d7LdaNwr96Gsl8hJuRAJQco3AtUuCuJMSjU82yOutQdy92xQ1CvXLXKsZlJuCM9hNe4eAjw409/k+EjRqUEuJsNxFNhjrgQ+vPnL5AZhi3Ni98yeS2XN//DXfOJo9d77lwmX9k4+MWe4YgZOeDNgaiFO95p50YEiED0CIRx71G4e7el7GeIDTlwQCjcC1y4k8QVDVm6WXevmXU3d862s2PgPUUOhMkBCvfoBRRTJAJxIEDhzr4jzL6DtsgnNw4UjHCPoxHOR5p+/q7NraLolvkGzvSuu5tQN934bntmjMlDYkQOBOMAhXs+elLaJALJQ4DCPVjbyL6EeJEDwTlA4R5x20/hHpykQW7sbMU7RXt+6yVIHTIs66KYOEDhHnEny+SIQEwIULiz7yqmvotlSSafKdwjbuAp3PN/I2C5u9c77+YMO84Rjsvj818n7ACIcalygMI94k6WyRGBmBCgcGc/V6r9HMsdHffnz58vr7zyiuCYZNyPi6kdDj1ZCvfoyA1Bjpl0W8TjGu4U7NHVRZIbF+aNPMgnByjcQ+9GaZAIJBIBCnf2JfnsS2ib/AIHoF3wj1lJ1zAU7mn+Do03M29mcoAcIAeSyQEK90RqLGaKCISOAIV7Mttg9o2sF3Igeg5QuFO4J3pJCBuF6BsFYk7MC4EDFO6h6yMaJAKJRIDCnX1SIfRJzCN5GgUHKNwp3CncyQFygBwoOA5QuCdSYzFTRCB0BCjcKYiiEERMgzwrBA5QuHPAXnAD9kK4sZhHdgDkQH45QOEeuj6iQSKQSAQo3PPblrKvIr7kQOFwgMKdwp3CnRwgB8iBguMAhXsiNRYzRQRCR4DCvXBEBQUg64ocyC8HKNw5YC+4ATsbhfw2CsSX+BYCB4pRuH/3/V9lwKAhsm37dt/i57EnnhTsuWxh2MglfcYlAukQoHBnn1QIfRLzSJ5GwQEKdwp3CndygBwgBwqOAxTuFVInDNEdhg0VXrPenJ3zgwS1xSMRAAIU7hREUQgipkGeFQIHKNw5YC+4AXsh3FjMIzsAciC/HKBwrxB1YYjuMGyoxKRwVyRK87hw0WLpO2CQ6/7Jp59lBQqFe37bUvZVxJccKBwOULhTuFO4kwPkADlQcBwoROG+b98+GXfLBKlX1qTKDjf4uS2VX758hXTu1jMVHsLY3FR046h2cW5utg1NT8OoDb3OdMRSfizpN9NzK5uZDvKt4e3XAZC+Cn4NY8bNlB/6JwsBCHRbvE975bWsM0nhXjiiggKQdUUO5JcDFO4csBfcgJ2NQn4bBeJLfAuBA4Um3FXYqqjWa1OI28JdBTfcsalgNuPAHsSuHUbTQRyIYH1v3suGhvejrhDWDI9zTd/2gz3k1xTidjkRJ10Z/OSJYZKFgCnecxHtFfx5M+fCzZs3X1auWi0bN26UrVu3yo4dO2TXrl2yZ88e2bN3r+zdu9d5eKbtSiG0gcwj+2pyoPQ4ELpwf/nll1MNrJ/zVOAcT37++WcKUD6EIAfIAXKgRDigA2wIYAy6sWMQjsE4BuUYnGOQvmLlKsGgPdct11k/FcwqcJEfP4LWFMiI4yZ6M4Wxy26La/vaDm9euz1wMP1tWyj3NSOvEzyE0E1tKBZ2HISzy6lxeSwcBCDecxXtKG2u9x5sULiXnsChqGWdFyMHQhfuEydOTPUqfs5TgXM8oXDnDVqMNyjLRF6TA+4cKDThji7OFKgqXs3Zc1OsuvnDBgTwgEGDU0LYtKndKOxgeb0plnX23m0pupsNteV2NG2p+NZwti0zrKatR41rx4EttzJoGjyWFgIU7u5tIPsG4kIOlB4HKNxDnp3asGGDLF26VPAggTsxIAfIAXIgPxxYsmSJYF+8eLEsWrQodVywYIHMmzdPfvrpJ/n+++/liy+/SsSMO6SWLWIhWM0NYlXf/9YZelPYmzYyiV61ow8ATCFvz/S7CWczX17nWh7Ttm3LLJOXHTsOwvmJ52WP7sWFQFjC/cuv5jptAtoGtBFoK9B2LDLaEG1X2G7np90mrsS10DkAjQetF9dDEwr3EIU7KhJLMw8fPlxcvSZLQwSIABFIGAJHjx4V7EeOHHHaXLS7hw4dcjpTiNXdu3fLtm3bZO269YkQ7iqgVXC7wWmLVT+C1i2MKcxtm0jX9Me1mw23/Lm5abn0AYNtSx9ApCu3Hcctj25p0600EAhLuK9bv8FpE9A2gLcYeKPNOHT4sNOGoC3RdqU0kGUpiQARCIoAxhrQenGJ91CE+6pVq2Tq1KmCpfFBdsRB3DA2PMGJ6+mHpounMBTtYdQmbRABIkAE0iOgA+xCE+4qcN1KZ4tsndFW0asi2LQB0WvOeNtxENf0xzWWqpsfi3MTzm75gxsED8IjL9g0T5pH+6EAwsDNzAPcNDzOM5UBYbiVLgIU7qVb9yw5EUgiAtB60Hyq/6I8hiLcIcDXrVsXGFvEQdwwtiQId+SBGxEgAkSACOQfgUIT7kBERbW+461HFbHqb4pqddOwpmiHTYjel6e/UuVv5tzCaHyER3pmGggPfzueWy2u37BB3pz9lhNebZrxVMjDT5frw46moXHuvvf+lPhHnnCNPKm/adMtH3QrHQQo3EunrllSIlAoCMSlO0MR7uZH6IICnktcM624ADSfslC4mzXCcyJABIhA/hAoNOEOAY6vq+tMtSIDgWqKaHUvpSOEO3ZuRMANAQp3N1ToRgSIQJwIxKU7sxLu+Js3e0l8tuDZdsy/kAtiMy4AKdyD1BLDEgEiQATCQaDQhDtmue3l4vp+eJJEqz0zrjPgONr5D6cmc3vHPqw80E5yEaBwT27dMGdEoFQRiEt3ZiXc7UrKZdY8l7hmPuICkMLdrAWeEwEiQASiQaDQhDtQcRPFXBJO4R7NHVO4qVC4F27dMedEoFgRiEt3UriH+FV5VCI3IkAEiAARyD8ChSjc848KUyACxYcAhXvx1SlLRAQKHYGCEu5cKn/A9UuCFO6Ffhsy/0SACBQKAhTuhVJTzCcRyA0BCvfc8GNsIkAEwkegoIS7XfxclrvnEtfMR1wAcqm8WQs8JwJEgAhEgwCFezQ4MxUiEDcCFO5x1wDTJwJEwEYgLt3JpfJcKm9zkddEgAgQgcQjQOGe+CpiBolAKAhQuIcCI40QASIQIgIFJdy5VJ5L5UPkPk0RASJABAIjQOEeGDJGIAIFiQCFe0FWGzNNBIoagYIS7nZN5LLcPZe4Zj7iApBL5c1a4DkRIAJEIBoEKNyjwZmpEIG4EaBwj7sGmD4RIAI2AnHpTi6V51J5m4u8JgJEgAgkHgEK98RXETNIBEJBgMI9FBhphAgQgRARKCjhzqXyXCofIvdpiggQASIQGIGohftbb7/t/JtI4IwyAhEgAlkjgFWNuPdy3ebNmy/r1m+Qbdu2ye7du2Xfvn3O/Xzo0CE5dPiwHD58WI4cOSLaruSaHuMTASJQ3AgUlHC3qyKX5e65xDXzEReAXCpv1gLPiQARIALRIKADbAy2MejGjkE42mQMyjE4xyB97br1gkF7rts333wja9ety9UM4xMBIhAAgbVr1wruvVw3CvdcEWR8IkAETATi0p2hLJWfOnWqrMtiQIM4iBvGFheAFO5h1B5tEAEiQASCIRC1cN++fbv81wcfyJo1a2T//v3BMsvQRIAIBEJg3/79snr1aueew72X60bhniuCjE8EiICJQFy6MxThvmrVKkeAY/Y8yA7RjrhhbHEBSOEeRu3RBhEgAkQgGAJRC3fkDgICs39Yuov3brkTA3IgPxzAPYZ7LQzRjnuXwj1Y+8rQRIAIpEcgLt0ZinA3i2YuffdzbsbN5TwuACncc6k1xiUCRIAIZIdAHMI9u5wyFhEgAnEjQOEedw0wfSJQXAjEpTsp3PlV+eK6k1gaIkAESgIBCveSqGYWkgiEggCFeygw0ggRIALHEKBwz5EKcQHIGfccK47RiQARIAJZIEDhngVojEIEShQBCvcSrXgWmwjkCYG4dGfoM+74qzjd/Jxr2FyPcQFI4Z5rzTE+ESACRCA4AhTuwTFjDCJQqghQuJdqzbPcRCA/CMSlO0MX7vmBJ7PVuACkcM9cNwxBBIgAEQgbAQr3sBGlPSJQvAhQuBdv3bJkRCAOBOLSnRTufMc9Dr4zTSJABIhATghQuOcEHyMTgZJCgMK9pKqbhSUCeUeAwj1HiOMCkDPuOVYcoxMBIkAEskAgauE+683Z0rlbT1m+fEUqt/v27ZNxt0xwdpyb27bt22XAoCFSr6xJlR12uBEBIhAtAhTu0eLN1IhAsSMQl+7kjDtn3Iv93mL5iAARKEIEohbugPCxJ550doXzu+//6ohziHR7U+GOMLqpG8W7IsIjEYgGAQr3aHBmKkSgVBCgcM+xpuMCkDPuOVYcoxMBIkAEskAgDuGO2fYBgwY7s+462+4lwlWkm8IdxUR4zNLbM/RZQMAoRIAI+ESAwt0nUAxGBIiALwTi0p2cceeMuy+CMhARIAJEIEkIxCHcUX6ddYcgTyfA/Qp3PAzAEnxdUm/bRHoQ+9g1DNzMTdNSfz2a4XCu7nYapi2eE4FiRIDCvRhrlWUiAvEhQOGeI/ZxAcgZ9xwrjtGJABEgAlkgEJdwN4W2PZtuFkPFtBlG46qgRhiIaByxaRxzFl8Ft7qpDbVrx9Fr9Ydd2NA0cQ1bFO8O5PwpEQQo3EukollMIhARAnHpTs64c8Y9IoozGSJABIhAeAjEJdx1iTw+PKeC261UKqB1lluPmeLZItu+1vRNIa/L9zUfZhwI/WtGXlclr8gb3ODHjQiUAgIU7qVQyywjEYgOAQr3HLGOC0DOuOdYcYweKwIqLszZuGwzFKatbPMQNJ7OXqoIwtH+crhpE7OY6fzNsJlsmWF5HhyBuIQ7OADRi11545Z7vR905hthve4z5aGKe3M23BThSMcW7va1nS7SV7vm0S+P3cpGNyJQaAhQuLvXGNqXTA8T3WPSlQiUNgJx6U7OuHPGvbTvvAIvvT2oD1ocFQymUAhqQ8OrYPASJxouSUctvwow5B3iRsWWnVe4+xU8FO42euFexyHcTZEMLqQb8Or9oFzCtT3LrfZMToE35v1o3+MaRzkLVJGGKcpNP9teuLVAa0SgMBCgcHevJ7Qv6dox91h0JQJEgMI9Rw7EBSBn3HOsOEbPGgEdwCdFKKtQSUp+/ABrC/dMcSjcMyEUnX8cwh0iWAe5me4/vR9UuAMZW0TDT+0pcnaYTMLd7YGA2sIRHLeX0pv+PCcCpYAAhbt7LVO4u+NCVyKQCYG4dCdn3Dnjnomb9E8gAioa7Fk2HaSrIMBM3qLFS5wZPDOsigkVFyq2Vch+MOdDR1Agji0svOCwbSGcumnaECW62X6aB/jjXONkSl/tmLOUWg7Y0XM3e+qn+cLRnP208wJ7tr+Wxz662YKbWz4Q1/Qz09A8ajyE4yYStXBXnum9gzrAuVlXZr24hbfd7Pi4Rj2bXAbnsOum977ywLap4cwj4pv3EWwsWLjQDMJzIlDUCBSCcLfbem1r9B5/bvKUVL9stzvaLmg/YbYZqFi1Yftr2wDb6qdtS1ETgoUjAjkiQOFeoAByxj3HimP0rBHQjtrsoLXjNwfpcEOnjE3jqL925mrDjq/+fjpyDau27GsVJThqPlSg4BruiIO0zEEJBMbyFSvT4mTHgS21AT9cY9PyaR71Wstn27GvEU/tps3QMSFuhjVt2eVH/jBo0nyu37AhhQfqSvOLeF98+RX/A1yiFe52fWndq7veT+qOo/Jf61T9TB7ADXWrA2acI7zeF+qv9Y9rTRN2dFP+qB0cTe6pHdMfbQJscSMCpYBA0oW7thd6X+Oe1nZF/fRa2wD7WtsNu1/T+NqOID7C4Kjtj6aLo9otBV6wjEQgWwQo3LNF7li8uACkcM+x4hg9awS089bOGIbsDtvNODpmHdTbHbod3y0NN5tws22Z6cBfbengAvnWfJg2VYToQML08zrXfCMuNtjWdMw4dh40nqZl5tkuD+zAvluezTT03I8tCCnY1HyYdQk7ml8OpBTVymPUM+6VKSfrDPxx4zq4ZPMpWTlnbohAdAgkXbijvzDbee1/cH/ruXk/m/0LwmhfoogirNozw6q/Hs1wcIMtv32c2uCRCJQiAnHpTi6V51L5UrzfiqLMKurMzlwFIDpqc9OOX2fctGNWd7Vhx3dLw7Rrntu27AEBwppualvzhAGDbsi/umve1M/tqLYQVvNhYqADG7WpQscurznAUTtm+kEGNaYtOx2UwXYz86j5QzjNB/Ku9eaGQam5UbhX1Dh4pgN05YByxrwH1I9HIlCKCCRduKOf0f7JPKJf0PvZ7osQDv5mX6N1a7ohnt1GaDjbL0gfpzZ4JAKliACFe461HheAnHHPseIYPWsETLGqRmwxCHcVhDqINzt0e0Bgx3dLQ9Oyj7YtMx2EVVumKFUb9uBB3TXvOGbakB4GJ3g/3xS4sK3Xdh7s8pp5tsuD9IMMavzY0oGXWTY7T+pn513dS/VI4V5Z8+C4Odh341VlaJ4RgdJDIOnCXfsv9Dv25tYXmf2LWz9p9qlmWNu2GQ5+Qfo42xaviUApIRCX7uSMO2fcS+k+K6qyugk5N9FndtoaR4WsPSCw42t4dO6ZNtuWfW0PLqa98qozkwC7Zh4RDjs2Mz967ib8NSy+nn33vfenlg5r/jWObUOvkT42Mx+4RrkVK7drJ5LHj23LvHbLFx44YDNxQ7iXp7/iPPSAnz3I8ki6JJwp3EuimllIIhAKAkkX7ma7rwVev369c6p+2o/Z1279Cfot7de0n9N+HOFhA5vdp1C4K/o8EoH0CFC4p8cno29cAHLGPWPVMEAeEdAOGTNs6ID1WjtsJK2dPMKgM6/4YvxgJ6z6aYdux9cBgfpnKgrCIR1NX+3DzXTXdNQdRwwYND3TXdOGP9y9lvyZcTV95FfjIS4GPhDCOgAy4yAc4plC3fRHfMT1+9daWkYzv7CvZTPdFTf10/yZeYefmbdMdVHs/hTuxV7DLB8RCA+BpAt3lNTuL7W9V3f0PXBDX6B9hCJk91Xab6q/9kfax2j/g3B6jrDoczRdjcsjESAC1RGIS3dyxp0z7tXZSBcikFAEMPi4ZuR1qdmChGaT2YoAgTiE+/79B2TDho3OQ69ly5YLd28McK8CK2AWdCPO3rjanMsF56D1UsjhC0G4e+Grwt0W417h6U4EiED+EaBwzxHjuADkjHuOFcfoBYOADh70ib15jGJAoembs+lxgmfOnptYcLYimlqJWrhDTK5atVp27dolhw8fjqaQBZwKMAJWwCyIeCfOwSo9W5yDpVL4oSncC78OWQIikCQE4tKdnHHnjHuS7gPmhQh4IoD/L4d450YEgEDUwh2zx7t37yb4AREAZsDO70ac/SJVNVxQnKvGLv4rCvfir2OWkAhEiQCFe45oxwUgZ9xzrDhGJwJEgAhkgUDUwh1Lko8cOZJFTks7CjADdn434uwXqarhguJcNXbxXxWycC/+2mEJiUDhIRCX7uSMO2fcC+9uYY6JABEoeQSiFu54t5hbdggEwS5I2OxyU7yxiJ133VK4e2NDHyJABIIjQOEeHLMqMeICkDPuVaqBF0SACBCBSBCgcI8E5lASCSIog4QNJXNFZITYeVcmhbs3NvQhAkQgOAJx6U7OuHPGPThbGYMIEAEiEDMCFO4xV0CA5IMIyiBhA2ShJIISO+9qpnD3xoY+RIAIBEeAwj04ZlVixAUgZ9yrVAMviAARIAKRIEDhHgnMoSQSRFAGCRtK5orICLHzrkwKd29s6EMEiEBwBOLSnZxx54x7cLYyBhEgAkQgZgQo3GOugADJBxGUQcIGyEJJBCV23tVM4e6NDX2IABEIjgCFe3DMqsSIC0DOuFepBl4QASJABCJBgMI9EphDSSSIoAwSNpTMFZERYuddmRTu3tjQhwgQgeAIxKU7OePOGffgbGUMIkAEiEDMCFC4x1wBAZIPIiiDhA2QhZIISuy8q5nC3Rsb+hABIhAcAQr34JhViREXgJxxr1INvCACRIAIRIIAhXskMIeSSBBBGSRsKJkrIiPiKLxEAAAgAElEQVTEzrsyKdy9saEPESACwRGIS3dyxp0z7sHZyhiJQ2DX/iNy4NDRxOWLGfKPwNGD+wQ7N38IULj7wykJoYIIyiBhk1C2JOWB2HnXBoW7Nzb0IQKFiMDeg0fkx3V75ad1e2VfDONfCvccWRMXgJxxz7HiSjj6pLm/yj0fbswZgf2Hjspl01bJc19vydrW2h0H5YLJy6XJI0vkvo9yz1PWGSniiId3bJZ1d10kK0fUl22zn6hW0i3T7xLsXtv+ZT/Kr3++2cvbcf/1xdtk1XVNZP29l8iRfbvShi10Twr3wqnBIIIySNjCQSCanBI7b5wp3L2xoQ8RKDQEMFk19q11ct2sNXLD7LVy+39tkD0HjkRajLh0Z04z7hs3bpS1a9dGCpRXYnEBSOHuVSOF775j32FZve1g2n3t9oNZz3RDIE//YVuigEKeikm4Ywb78LYNGfcoZ7oh2t2EeyYi+BHusIFwGx4dQuG+e7ds27ZN1q5bLxi057rFIYoWLlwoM2fOdMqRa/7jjB8EuyBh4yxTEtMmdt61gjZg1eo1snnzZtm+fbvs2rVL9uzZI/v27ZN9+/fL/v37BeO5gwcPOvuhQ4eEOzEgB5LJgenfb5FOk5ZK2yd/kY5PL5WL/7xC3l6w3bsByINPXLozJ+G+ZcsWmTJliqxYsSIPkAQzGReAFO7B6qmQQl/12mqpff+ijPuMn4I3Fje+tc6Z3W722BJp/aef5Z2FOxxo8KDg0pdWOjPobZ78RYa+skrwZPHjX3ZJz+eXOWF7PLdM5m+oWFINP4TBTLmZD5yPemONDHhxpbR4/Gd5+NNNvqC3hTsW3//pi83SbuIvzo5zXZD/9oIdcuHk5TLo5ZWCPH2zco+Thpd7ugxc8dYV8g93/oOz4zysDQJ5+ZAzMu7ZCOltb/1Jdn05M5XVra/d54jmwzt+lU1Pj5TVN7SS1WPbyK4vXk+FwYmbcHdmykc2qibo96+YJ2tv7y5rbmonm54elXHGHfbdhPuhLetk/f39ZPWYFs4R19gOrFog6+/tIxv/NEzWTugie36Y47hjtn7zs9fLhscul3V39pQtr9wtRw8ddPy8fvJVh17plcKM+4cffiivvfaarFy50guGgnAPIiiDhC2IwkeYSWLnDTaFezIFWCEK46HTMUbjHicGnSctlboPVIzPGzy0WNpP/EVe/ynaibC4dGdOwh1NJJ5eJkG8xwUghbt3R1noPvd+uNERyhDWXvt5T/0iny/bnVVRIZJNsQ0jEO5dnlmasrlp1yHH9vZ9h1PLgJDemDfXpgQ0Ati2YBeiesuew4JVAVhKv2Fnha10mbWF+6dLd8nVr692VhXgHXqcww3btTPXyPuLdzrnyPe6HRXCzsvdK10IvuNuOa7KHpZ43zX3DUc8Q7B67hDXc9/wyp6n+4FVC2XzlHFy9PAhObJ7u2x+/gbnCIF7ZE/Fg5gje3Y6Iv7Q1vUpO27CHZ57vn+/inB34k4aJYe3bxI5elS2TPujYytlyOPEFu7IH0T4vkVznRg44hruyMvOT6c57kjv0ObVzjls4EGBI9aPHpUDqxc5efBIUvJZh15ploJw9yp7obkHEZRBwhYaDvnOL7HzRphL5b2xoY9/BNbvPCTNH18iL3y71X8khgwdga17DkuLJ36WsocXS7NHl8j5zy6T7XsPh55OOoNx6c6chTsKlQTxHheAFO7paE2/dAjYYhthIYAx049l+uYG0ayz6Fe8ulr6vbiiShjbFsLePWeDYwK2YBO2M222cMf1lL9UvjuPc7hhwwz7+c8sc94vWrix8qNqXu5eaWOm3RbucEv6dvTAPtk0aZQc3LhC9v7tU9n2xiMVWYbQXbNYtky7UzY/d4Osva2rMwuu5fEr3GHj1xduSQlmCHvM5GfabOGOVwU2Pn6F81ABcfGQAddwd2bi773EsYt4eECADYId79uv++MFsvvr2YKyptviqEMK93Q1kiy/IIIySNhklTL+3BA77zqgcPfGhj7+Efj4513OKkxd9eg/JkOGjQDE+5+/3SJTv9si2yIW7ShLXLozFOGOAsQt3uMCkMI97FuxdOzZYhsldxPuBw4fdcQxnvAePnLUmUHHzLcp7m1bEO4qsPMl3JFf5Oe71XtlwEsr5amvfk1Vnpd7KoBxEofoM5LP6XTXF6/J9vefFSyTxww8tt3fvSebJ98kR/ZVrMTAMnhHFB9LKYhwx4MBff8+H8LdydLRo3Jo0yrZ/PyNsuXlO1LiHX6Yhd/x0VRnOT0+rue1xVGHhSbcsez966+/FhynT5/u7Ljeu3evvPPOO861vSzefscd1wj7yy+/OEvoYacQ3oEPIiiDhHXjYzY4w45ZL8AVWGPDawqoF9SVbjhPIu65YqflK8YjhXsx1mr0ZZr45a9S78HF0SfMFBOHQFy6MzThDkTjFO9xAUjhnrh7qWAy9NAnm1Kz2XsOVnwN002428L7zfnbpc8LmWfcwxDuXkvlMS/7xt+2p5bvvzFvu+C9fS/3dJUSxzLrdPkJ4ndw00pZd09vZ8ZaBTaEOUQ2tsPbNsq6u3plJdwhmjEzfnDDMkdM52Op/N4FXwreycd2YOU82fTUCOdBwcENy2X/8r857kf275GNTw53VhE4Di4/cdRhIQp3UxBCGOLaFOsQjxDmEPPY3IQ74mgYFf2Il+QtiKAMEtatzCrAVXj7wRlCPJ0wN+sBQh6iPYnfHcgVOzc8i8WNwr1YajLecuBVwMHTVsWbCaaeCATi0p2hCncgGZd4jwtACvdE3D8FmYn56/dJ12eXOR99u/O/Njiz127CHYXDU178XVufqSsES+Xx0TkIevNv3PChO4hnbEFn3E07+NAd0oIbhLjbx+mwdP/Bjzc5ecdH9JCvFVsPOO/Cu7lnqiAIP8zaYsd5oWx4Txyi1vxIHT74hr99w4fd8HV3nGPG3fw7OPwlHNzh5gjjP10tq0Y2cv4qbuOfrnbcgAE+TrdmQmfn43RYuu737+BWXFVTVo9p7izhhx3Xj9MdPeq8279mXHvn/f91f7zQSQ/hD677xflLOXwXAB/G2zHnz1Vm4t3qJ+o6LEThbgpsfPEeAlAFJjA1BaKfa4SB4FQh71YvSXALIiiDhHUrGzAOirNtx02cq1092nGScJ0rdkkoQ77yQOGeL2RLxy7GXPiCOcZE3IhAXLozdOGOqlTxHuVfxcUFIIU7b95CQuDJLzZX+9Ceiv1CKkcp5xUPBtbc0qnKx/b0IUAp4ULhXlHbFO5VWW8Laz8PSNQC4mJFg+7mQxVdMp/EJfKafwp3RaL6kcK9OiZ0CYbAN6v2OO+3z11R8Q86wWIzdLEhEJfupHA/cMD5705TgGd7jkrkRgSIABEgAvlHgMK9AmMK96pcy0a4Q6BDrCMuNi+xjzAU7lXxLpQrCvdCqank5vPPf9niCHd8d4gbESga4a6z7VH/t3tcAJoin8KdNzIRIAJEIBoEKNwrcKZwr8q3oMJdvxMAHHWzhbt5bdvXOEk4csbduxYo3L2xoY8/BMa9vU56/3mFv8AMVfQIxKU7Q51xj0u0gx1xAUjhXvT3JgtIBIhAAhGgcK+oFAr3quS0hbUpujWk+S0BFe46267XmF3XpfImxrpkXv3UZhKOFO7etUDh7o0NfTIjcOjIUen5/DK559jf7GaOwRDFjkBcujM04R6naAc54gKQwr3Yb02WjwgQgSQiQOFeUSumqExiPSFPQQRlkLBu5Q0q3GFDxTjEOr7yj7/bwwf/IM6Br/nlf4R3c3PLS9RuuWIXdX6jTI/CPUq0iy+teev3Osvk/2vxzuIrHEuUFQJx6c5QhHvcoh2IxwUghXtWfGckIkAEiEBOCBSacM+psAUeOYigDBK2wGEJPfvEzhtSCndvbOiTGYHXftzuCPdfdx/KHJghSgKBuHRnzsI9CaIdDIkLQAr3krg/WUgiQAQShgCFe8IqJE12ggjKIGHTJFmSXsTOu9op3L2xoU9mBPCXvZ0mLc0ckCFKBoG4dGdOwn3Lli0yZcoUifpDdG6siAtACne32qAbESACRCC/CFC45xffMK0HEZRBwoaZx2KwRey8a5HC3Rsb+mRGoP+LK+Xmd9ZnDsgQJYNAXLozJ+G+ceNGifK/2tOxIS4AC124175/kXy6dFc6aOlHBIgAEUgcAhTuiasSzwwFEZRBwnomWKIexM674incvbGhT3oElm05IHUfWCQz/rY9fUD6lhQCcenOnIR7php6+eWXZeLEib52hM1liwvAQhfu17+5VsoeXix8bycX9jEuESACUSNA4R414tmnF0RQBgmbfY6KMyax865XCndvbOiTHoF3Fu5w3m9fvuVA+oD0LSkE4tKdeRXuUdZgXAAWunBf+usB6fn8chn08sooq4tpEQEiQARyQoDCPSf4Io0cRFAGCRtpIQogMWLnXUkU7t7Y0Cc9Ag9/sklaPvFz+kD0LTkE4tKdFO4HDogpvnM5RyUW4oa/t8CS+fs/2liI2WeeiQARKEEEKNwLp9KDCMogYQsHgWhySuy8caZw98aGPukRuPyVVTJi5pr0gehbcghQuOdY5XEBaAr9QhXugH7il7864h1LgrgRASJABJKOAIV70muoMn9BBGWQsJUp8AwIEDtvHlC4e2NDH28E/v/23sRNjuLK1/5/7p373O+OjXfGxh57bI93D/Z4bMbYHjzGGi9gwMZm8cZmkEACsQg1IJAEAiHEIoEQAiQBkoxASIDU3ep93/d9O99zspRNqFXdFVV1qjIr883nqa7szMhY3jiR5/wqcukamZHP3FUrD7/Zv3wi9qSSQFS6kxl3ZtwXB5ze764z7y2DM4vbWIEABCAQRwII9zj2SvY65SMo80mbvbT0boXd8n2PcF+eDXuWJ7C/fjSIi9/tnFw+EXtSSQDhXmS3RwUwKTPuij+83/17DzcW2RscDgEIQKC0BBDupeVrmXs+gjKftJZ1TEJesFu+FxHuy7Nhz/IENrzRK59YW7t8AvaklkBUupMZd2bczxp04f3uf97D+yrPAsM/EIBArAiUW7g3NjbJ/Px8rBhUQmWUmbLzXeDsS+rsdPlyPvvo5P+HcE9+H5eihVc83S6X8PDmUqCt+DwR7kV2YVQAkzTjHnZBeL/79ncGw018QwACEIgVgXIL966ubhkbG4sVg0qojDJTdr4LnH1JnZ0uX85nH538/xDuye9j6xaOTM3Lv26ol/Wv9VpnTX4JIBCV7mTGnRn3rMMnvN+9pmcq6342QgACEIiSQLmF+9TUtLS0tMro6JjMzc1F2fSKKFsZjY6OBsyUne8CZ19SmXSFcs6vlMpPjXCv/D4sdwvebBkP7m9/o5EfbMvNvhLKQ7gX2UtRAUzijLt2RffITPB+969urC+yZzgcAhCAgD2Bcgt3bYGKSp0R1su59X5iPsszUEbKKh/RHloJnJfnutTmiuEc8k7DN8I9Db1s28Ytbw0Ewn1ydsE2Y3JLBIGodCcz7sy4LzuAwvvddfadBQIQgECcCEQh3OPUfuoCAQj4E0C4+7MiZYbAtbs7hIc1Yw3LEUC4L0fGc3tUAJM64x5iD+933/R33mEZMuEbAhCIngDCPfo+oAYQqBQCoXAfGhoKnlUxOTkpGr/Nzs7K7NxccPuLPuAvPK9USruoZ2kIzM0vyHc2NcotL/s/n6M0NSHXuBKISncy486Me84xEd7vfrx9ImdaEkAAAhAoB4EwwNZgW+/z1Y8G4RqMa1CuD+vSIL2js0s0aGeBAATSSwDhnt6+L6Tl73dNBpfJ76keLuRwjkkBAYR7kZ0cFcCkz7hrt0zOzAf3u392/ekie4nDIQABCNgQQLjbcCQXCKSBAMI9Db1s18YdJwYD4d4zOmuXKTklikBUupMZd2bcvQbS/rrR4CT2P0+2eqUnEQQgAIFSEkC4l5IueUMgWQRc4T4+Ph5clcOl8snqY8vW3PxSl3yjqsEyS/JKGAGEe5EdGhXANMy4h10T3u/OOy1DInxDAAJREVgq3PU+VfdSeQ3OuVQ+qt6hXAjEiwDCPV79Effa/Ghrk+jD6VggsByBqHQnM+7MuC9nk1m3h/e7H2kez7qfjRCAAATKQQDhXg7KlAGBZBDIJdz1WRk8nC4ZfV1sKxr7p4MrTJ88MVRsVhyfYAII9yI7NyqAaZpxD7vowoca5aO314T/8g0BCECg7ARyCfexxRn3Th5OV/beoUAIxIuACvfOrq7gKhw9N7hPlZ+by4h2hHu8+iyq2uypHgmEe0P/dFRVoNwKIBCV7mTGnRn3vIfHsbaJ4KSmlxKxQAACEIiCwFLhrsF3eKn81NSUaHA+PDwcBOsatLNAAALpJRAKdz0n6LlBzxEzMzPBOQPhnl67yNbytQd65Av38DDmbGzY9gEBhPsHLApaiwpgGmfctYPC+93/9nJXQf3FQRCAAASKIeAK9/CVcCrcNRjXoHx8fCIQ7l3d3cy4FwOaYyGQAAIq3PVcoMJdzw1nC/e5sy6T13MLS3oJrNreKr9+igcxp9cC/Foele5kxp0Zdz8LzZIqvN/91brRLHvZBAEIQKB0BELhrt+hcNd3uX8g3MdlZGREurq65f33TwZpSlcbcoYABOJKQM8Peg7Qc4GeE/TBlaFw13OGfrhMPq69V956dY/OyifX1simv/eXt2BKqzgCCPciuywqgGmdcQ+7S+93/9Dqapma5RfqkAnfEIBA6QmsJNz1vDwxMSGjo6PS09MrtbWnpb9/oPSVogQIQCB2BHTs6zlAzwV6TtBzg54j9AodhHvsuivSCh2oz7z6+HjHRKT1oPD4E4hKdzLjzox7UaOjvm8qEO4XPsT7LosCycEQgEDeBELxHs64L30l3OjomPT19UlTU3Mw46YBvKZlgQAEkk9Ax7qOeZ1t13OAngv0nOA+mE7PGcy4J98WfFu48XCfnH9nrW9y0qWYAMK9yM6PCmDaZ9y12x59eyAQ79c/zzsvizRjDocABPIg4Ar3jHjPPKBOL5fX4HxsbEwGBwelq6tLGhoapLq6Jgji9X5XPjDABpJtAyrYdczr2NdzgJ4L9Jyg5wY9R7iz7Xr+CM8neZyCSJowAlc+3SaXbGtOWKtoTikIRKU7mXFnxt3EnsP73Z8/NWySH5lAAAIQyEUgDLT1OyPcM7Nnegms3sOql8SOjIxKb2+ftLe3S3Nzs9TV1Uvt6TqpqakNgnoN7E+dqpaTJ0/xycJgz54X5fjxE0V9NA/4Yl+ltAEdwzqW9aNjW8e4jnUd8zr29Ryg97frOUHPDctdJq/nEpZ0EhidmpMv3Fsndx3sSScAWp0XAYR7XrjOTRwVQGbcP+iL8H73wYm5DzayBgEIQKBEBLIL9/lgJi18SJ3OsA0NDQeBu866tbW1SXNLizQ1t0hjc7M0NjUFn4bGRsn2qW9okDR/Xnn1VamuqSnqo3mkmSFttxtD2caobgvHsY5pHds6xnWs65hX0a7nAD0XhA+lywj3zLmC2fYSnaArLNu3WjOvOn69YazCak51oyAQle5kxp0ZdzN77xubDS6Z//KGOrM8yQgCEIDASgSyi/e5YEbNFe8626aXyup9rvqQKn3CtH46u7qCT0dnp3R0dvFZwuCNQ4eluaW1qI/mAVtsq5Q2EI7jcFzrGNexrmNex/65ov3ce9uZbV/pTJv8fVveGpDz1lTL+AzPQUl+bxffQoR7kQyjAsiM+9kdt+vkcCDe9dJ5FghAAAKlJrBUuC+9ZD4U73qJrAbv+nAqfbK0vs85/AwNDUm2jwb9af+8/fbb0t3dXdRH80g7R9pvM5ayjdNwWzie9VvHuI51HfPh5fF6Llh6ibw7245wL/XZOt75X/9Cp/xgc1O8K0ntYkMgKt3JjDsz7uaDQB9Sp6+I23Fi0DxvMoQABCCwlMBS8R4+dEq/NVAPxbs+lEo/Gsjru5zHx1XMj8vYeOaj2zTQ5/MBgxMnTsjAwEBRH80Dph8whYUNi8wY1nHsjuHM2NYxHo73sy+P/2CmHdG+9Eya3v/nF0S++UCD3LyvK70QaHleBBDueeE6N3FUAJlxP7cvdEt4v3vH8Ez2BGyFAAQgYETAFe66ftas+5nXPYUCPjxnazAffsIAn+/MDxsuh/fee2/xygR3RjOfdc3DzZP1cznDpHgm4XjW73Cch7PszLQbnWwTms3JrslgwmlP9UhCW0izrAlEpTuZcY/hjPuxYydk1aqrgo+uL7dUVW0R/cRxmZlbCE6Cn76L92HGsX+oEwSSRiCbeP9AwGceQhXOwIciXoN6/YRBPt/T57A4efLkmcuO9dLjwj6aB2zPZQsTWybheM4m1ufmPngQHTPtSTv7F9+ene8OBTFr9wiTTcXTTEcOCPci+zkqgK7j1ToUu+iv7uvXV0ljY7MMDg7J6tV3B99L8929e2+QbuvW7Ut3xeb/1xrGghPhL3e0xqZOVAQCEEgugZXF+5lXxZ2ZgQ8vp1cRz2d5BtXV1WduK8hckuxenuy7rnnAeHnGsLFlszi2l4x1FexLRbueM1ggcMu+Lvn2gw2AgIA3gah0JzPuMZtxV8Guwl0FvC46o7501j1Mc+pUrcRZuGv9V7/SHYj3rW8PeA8GEkIAAhAolMBS8a7/hwG7fodB/dnf8zKr98PzOYeBvgpuXJ8JUMRH84At9lVKG8jMqJ87vt2xv/TcUOg5huOSR+CizU3yh10dyWsYLSoZgYoV7i0tLbJlyxa57777vD+aXo+zXKICaD3jrqLcFeM6s+4Kd3cWfmnaQniOj09KqT8XPlgfiPf324ZLXlap20L+pbcXGMPYwgbGxvTBc2d/RkfHJddnZGRM4vg5+4eGzNUD5dims+X6oK9iPppHOepKGeWzizixdsV5tvWlgl3/Z4FASKB5YDqIUbcf54HKIRO+cxOISncWPeOuIryjI79fqTS9Hme5RAWwFMJ9pRl39/738D74uN7n7vavPmX+w6ur3U2sQwACECgpgWwBe7Zt2YJ9tmUuK66pqSn6wXKaBzwzPOFQWg7ZxvfSbSU96ZB5xRHYU515jXF933TF1Z0KR0cgKt1ZtHDXmfZClkKPW66sqABaC3ffe9yVg8WM+3I8rbef6Mg8sfOSbbZXWljXk/wgAIFkElgavPP/gvgwqK2tXXz6vvvU7nzWNQ+fskjj1ydwyp9TMs9qtMqCwJ0He+XLG+ossiKPFBGISneaCvfm5mbZvHmz9yXzKt6tLpuPCqC1cFebd2fVw8vk9ZJ5fVBdeO+7pqsk4a713Xi4L7gcqepIX4qGNk2FAATiSADx4yd+EO5+nLCn+HGK43mHOsWPwH8/3iKX7WyLX8WoUawJRKU7TYW7ivaoLpuPCmAphHusLbXIyl2yrTlzv3tX5uF7RWbH4RCAAAQgUEICFr5V82CBAAQgEDcCPaOz8pHbq+VBJpTi1jWxr4+Fb3Q1pO+6qXAv9PL3Qo9zezUqgC5oghO3R5Zf1/vd9TM4Mbd8IvZAAAIQgEDkBCx8K74x8m6kAhCAQBYC4WuL32mfyLKXTRBYnoCFb3Q1pO86wj1mr4Nb3kSSsyd8gqeK96dO8BTP5PQsLYEABJJGwCI4QbgnzSpoDwSSQaDqcJ9csJ4rgpLRm+VthYVv9BXrbjqEO8K9vJZ+prSJmXn5yv11wcz7r5/i3qJIOoFCIQABCOQgYBGcINxzQGY3BCAQCYHLd7bJpU/w0ORI4Fd4oRa+0RXkvusId4R7pEPnjv09gXjX2fe63qlI60LhEIAABCBwNgGL4AThfjZT/oMABKInMD49H8y261PlWSCQLwEL3+gr1t10CHeEe762ap7+vc7Mq+JUvG88xBPnzQGTIQQgAIECCVgEJwj3AuFzGAQgUDICb7WOBxNHB+tHS1YGGSeXgIVvdAW57zrCHeEem1H1ix1twUn0B5sbZXp2ITb1oiIQgAAE0krAIjhBuKfVemg3BOJL4NG3B+Tjd9TI2PR8fCtJzWJLwMI3+op1Nx3CHeEeq0Hx9LtDi5fOH24ai1XdqAwEIACBtBGwCE4Q7mmzGtoLgfgTuPb5Drl4a3P8K0oNY0nAwje6gtx3HeGOcI/dgGgfnpEvb8g8uO6WfV2xqx8VggAEIJAWAhbBCcI9LdZCOyFQGQQWFkS+en+93PQSMWZl9Fj8amnhG33FupsO4Y5wj99oOFOjNa9mHlz3pfvqpHVwJrb1pGIQgAAEkkrAIjhBuCfVOmgXBCqTQHX3VHB15/OnhiuzAdQ6cgIWvtEV5L7rCHeEe+TGv1IFjjRnHh6iD6575r2hlZKyDwIQgAAEjAlYBCcId+NOITsIQKAoAuFtmV0js0Xlw8HpJWDhG33FupsucuE+OTkpGzZsKLrnowLowiQ4Kbobs2YwPDknq7a3Br+OXvlMu+glTiwQgAAEIFB6Aha+Fd9Y+n6iBAhAwJ/ALfu65bubGv0PICUElhCw8I2uhvRdL7twb25uls2bN8t99923+HniiSeW4Mj/36gAuqAJTvLvt3yOeOKdwUC8n7emWvQ1HiwQgAAEIFBaAha+Fd9Y2j4idwhAID8C//5wo1z3fGd+B5EaAg4BC9/oakjf9aKF+5YtW6SjoyNoiorxXIuK9q6uzMMgRkZGZOvWrdLYWPyvXlEBdEETnOTq/eL3v9sxIXrPu146X3Wkv/gMyQECEIAABJYlYOFb8Y3L4mUHBCBQZgLNA9NBDPn4O4NlLpnikkTAwje6GtJ3vWjh3tLSIirewxn0XJ0SivtQtL///vu5DvHaHxVAFzTBiVdXFZ1odn5BbnulOzjx/vixZuke4cF1RUMlAwhAAAJZCFj4VnxjFrBsggAEIiGwt2YkiB9P905FUj6FJoOAhW90NaTvetHC3cUfinJ329J1TWMt2rWMqAC6oAlOlvZ2af/ffXI4OPnq7Pue6pHSFkbuEIAABFJIwMK3WvrG3bv3SlXVlsMCyxsAACAASURBVKw9oftWrboq+FxxxXXS2Mg7mrOCYiMEUkzgroO98rWN9SkmQNMtCFj4RldD+q6bCnf3svlsUPRBdCrc9fJ4q5n2sJyoALqgLYOTsF18r0ygoX9aLj3z4LqbeR/nyrDYCwEIQCBPAha+1co3hsJ8JeF+7NiJPFtIcghAIE0Efvxos+iDjlkgUAwBC9/oakjfdVPhvvSyeRXp7kefHq8PorO4p30p7KgAuqCtgpOlbeP/3ATufb03mH3XX1Hf65zMfQApIAABCEAgJwEL32rpG3UWfevW7VnrrYI+nHFfTtxnPZCNEIBAKgj0jc0FsWLV4b5UtJdGlo6AhW90NaTvuqlwLx2e3DlHBdAFbRmc5G4xKZYS2F8/Jl+4N/PgukeODizdzf8QgAAEIJAnAQvfaukbVxLuYdP06r7Vq++WYmbfJyamhA8MsIFk2cAr1Zm3Ex2qH2Z8J+QcF573y/1t4RtdDem7jnCfnhZfWLnSWQYn5TbApJTXPTor1+zuCH5RvXxnmwyMzyalabQDAhCAQNkJWAQnlr7RR7grJL2sXj8sEIAABEICDxzpl8/fczr8l28IFEzAwjfm0pXZ9iPcEe4FG22cD9x8tD8Q759cWyMvnx6NpKqTM/PSMjgtb7VOBA/P28xVAJH0A4VCAAKFE7AITsoh3AcHh+TQoTeDhuqM+/r1VTycrvBu50gIJJLAL3a0yi+ebE1k22hUeQlY+MZswjzXNoQ7wr28ll7G0o62jMsPtzYHAn71q91mJY9MzUt937QcbhqTZ98bkgeO9Mkt+7qDh51cvLVZ/nVDvXzk9uqgXH3ivfs50jxmVg8yggAEIFBqAhbBiYVwDy9/D+9hD58ar7Pqell8V1e37Njx3OI97sy2l9oyyB8ClUVgfGZezl9XK/pUeRYIFEvAwjfmEunZ9iPcEe7F2m6sjx+bnpebXuoKxPNFm5ukpmf593b2j89KdfeUHKwfkx0nBmXDoT65YW+XXPZUm/xgc6N88b7M/fOuENf1T6ytka9vrJdLtjXLH3Z1yNr9PfLo2wOyr3ZE3u2YXHzP/D/dWSu/38WTTGNtMFQOAhA4i4BFcGIh3M+qFP9AAAIQyJPAsbaJIBY8UM8ESp7oSJ6FgIVvzCbMc21DuCPcs5hj8jY9dWJILlh/Ojhp3/1ar/x5T5foJVPfe7hR/vnuzPalgvxT62rlmw80yE+3tQT3za89kF2Q+9L6/a7Mvfe+6UkHAQhAIGoCFsEJwj3qXqR8CEDgsbcHghn3kak5YECgaAIWvjGXSM+2H+GOcC/aeCslg1Pdk3LpEy1y/p218u0HG+TSJ1rl2uc75M6DvaIn9KUz5Nbt2nVyOPjhoLZ3+Vl/6zLJDwIQgEAxBCyCE4R7MT3AsRCAgAWBq59rl/96rNkiK/KAgFj4xmzCPNc2hDvCneFXJgLDk5n3h972it399mWqOsVAAAIpJWARnCDcU2o8NBsCMSKgtzve+jLxV4y6pKKrYuEbc4n0bPsR7gj3ih44lVZ5fZqpzvizQAACEKgEAhbBCcK9EnqaOkIguQRquieDKx71ykcWCFgQsPCN2YR5rm0Id4S7hf2ShycBvSRf76XvHeP98p7ISAYBCERIwCI4QbhH2IEUDQEIyDPvDcuHV1dLxzCxF+ZgQ8DCN+YS6dn2I9wR7jYWTC5eBNqGZgLhXnWk3ys9iSAAAQhEScAiOEG4R9mDlA0BCPz1xS75/iNNgICAGQEL35hNmOfahnBHuJsZMRn5EdB3y3+9qt4vMakgAAEIREjAIjhBuEfYgRQNAQgEDyT+855OSEDAjICFb8wl0rPtR7gj3M2MmIz8COj74fVy+dn5Bb8DSAUBCEAgIgIWwQnCPaLOo1gIQEBaz1zpuO3tAWhAwIyAhW/MJsxzbUO4I9zNjJiM/Aic7Mo8JGXHiUG/A0gFAQhAICICFsEJwj2izqNYCEBAXqoZCSZLanp4FS/mYEfAwjfmEunZ9iPcEe52VkxO3gS++UCD/OhR3ifqDYyEEIBAJAQsghOEeyRdR6EQgICI3L6/R771YAMsIGBKwMI3ZhPmubYh3BHupoZMZn4EVr/SHfwC7JeaVBCAAASiIWARnCDco+k7SoUABEQu2twkV+/qAAUETAlY+MZcIj3bfoQ7wt3UkMnMj8CR5vFAuL9aN+p3AKkgAAEIREDAIjhBuEfQcRQJAQhI//hsEGs99Pc+aEDAlICFb8wmzHNtQ7gj3E0Nmcz8CXzu7tNy2c42/wNICQEIQKDMBCyCE4R7mTuN4iAAgYDAG41jgXB/q3UcIhAwJWDhG3OJ9Gz7Ee4Id1NDJjN/Atc/38nl8v64SAkBCERAwCI4QbhH0HEUCQEIyH1v9MqXNtQJL/HBGKwJWPjGbMI81zaEO8Ld2pbJz5PAi9XDgXA/3jHpeQTJIAABCJSXgEVwgnAvb59RGgQgkCFw6RMtXNmIMZSEgIVvzCXSs+1HuCPcS2LQZJqbwPTsgpy3plr+8mJn7sSkgAAEIBABAYvgBOEeQcdRJARSTmBydkE+dkeN3PN6b8pJ0PxSELDwjdmEea5tCHeEeynsmTw9Ceg97ireWSAAAQjEkYBFcIJwj2PPUicIJJvAO+0TwVWN+3kIcLI7OqLWWfjGXCI9236EO8I9IpOnWCXw5PHBwLE0D84ABAIQgEDsCFgEJwj32HUrFYJA4glsOTogF6w/LcOTc4lvKw0sPwEL35hNmOfahnBHuJff2ilxkUDPaOZVJetf41KuRSisQAACsSFgEZwg3GPTnVQEAqkhcMXTbfKzJ1pS014aWl4CFr4xl0jPth/hjnAvr6VT2jkE/uuxFvnCvXXnbGcDBCAAgagJWAQnCPeoe5HyIZA+AvrK3TWvdqev4bS4LAQsfGM2YZ5rG8Id4V4WA6eQ5Qk8eKQvuFyey7mWZ8QeCEAgGgIWwQnCPZq+o1QIpJVAbe9UEFftOjmcVgS0u8QELHxjLpGebT/CHeFeYtMm+1wE6s44mC1vDeRKyn4IQAACZSVgEZwg3MvaZRQGgdQT2PnuUPBE+fYhnh+UemMoEQAL35hNmOfahnBHuJfIpMk2HwLf2dQo393UmM8hpIUABCBQcgIWwQnCveTdRAEQgIBD4PoXOuWHW5udLaxCwJaAhW/MJdKz7Ue4I9xtLZncCiKw9kBPcFlXQQdzEAQgAIESEbAIThDuJeocsoUABLIS+NrGerlhb2fWfWyEgAUBC9+YTZjn2oZwR7hb2C95FEngrdbM+0afP8X9WEWi5HAIQMCQgEVwgnA37BCyggAEViTQNjQTTITo63ZZIFAqAha+MZdIz7Yf4Y5wL5VNk2+eBL50X51cyqtL8qRGcghAoJQELIIThHspe4i8IQABl8DempFAuFd3T7qbWYeAKQEL35hNmOfahnBHuJsaMpkVTuCvL3ZyuXzh+DgSAhAoAQGL4AThXoKOIUsIQCArgb+93C363CAWCJSSgIVvzCXSs+1HuCPcS2nX5J0HgVfrRgPhfqhxLI+jSAoBCECgdAQsghOEe+n6h5whAIGzCXzv4Ua5dnfH2Rv5DwLGBCx8YzZhnmsbwj2Gwv3YsROyatVVwUfXly7u/qqqLUt3838FE/jE2hr5/S4cTgV3IVWHQKIIWAQnCPdEmQSNgUBsCQxMzAUTIJuP9se2jlQsGQQsfGMukZ5tP8I9ZsJ9cnJS1q+vksbGZhkcHJLVq+8OvkMz120q1jWdmzbcz3dlE7jq2XYul6/sLqT2EEgUAYvgBOGeKJOgMRCILYE3GseCGOpo60Rs60jFkkHAwjdmE+a5tiHcYybcVbCrcFdRrouK9Gyz7rovm7BPxnBIbyuefncocDqnuqfSC4GWQwACsSFgEZwg3GPTnVQEAokmsP5gj3x1Y73MzS8kup00LnoCFr4xl0jPth/hHkPhvnXr9kWL3L177znCXUW9zsRfc82NZ83GLx6Ux8rIyJjwiQ+Djr7Mfe43vNBGv2Cb2AA2sGgDeZzWTZNaBCcId9MuITMIQGAZAv/1WLNc8XTbMnvZDAE7Aha+MZswz7UN4R5D4Z7PjLuK9+Vm5O3Mk5zKSUBfCXf+nbXlLJKyIAABCGQlYBGcINyzomUjBCBgSGBqdj64YvH+Q32GuZIVBLITsPCNuUR6tv0I95gJd/e+9WyXwuu2HTueW7SilS6lX0zESkUR0IeqfGh1tXQMz1ZUvaksBCCQPAIWwQnCPXl2QYsgEDcCx9sngtjpQP1o3KpGfRJIwMI3ZhPmubYh3GMm3NW23afGh7Ppesm8Xh7f1dUdCPfwqfM8VT55Z4O2oZnA+VQd5lfj5PUuLYJAZRGwCE4Q7pXV59QWApVI4KG/98vn76mTocm5Sqw+da4wAha+MZdIz7Yf4R5D4V5htkt1S0Dg+480ydc21pcgZ7KEAAQg4E/AIjhBuPvzJiUEIFAYgV/taJVV21sKO5ijIJAnAQvfmE2Y59qGcEe452mqJC8Hgbtf6w1m3cdneDJqOXhTBgQgkJ2ARXCCcM/Olq0QgIAdgX+6s1bWHeixy5CcILACAQvfmEukZ9uPcEe4r2CW7IqKwPtdk4Fwf/L4YFRVoFwIQAACYhGcINwxJAhAoJQETvdOBTHTnuqRUhZD3hBYJGDhG7MJ81zbEO4I90UjZCVeBPRdpBdvbY5XpagNBCCQKgIWwQnCPVUmQ2MhUHYCT7wzKJ9aVyutgzNlL5sC00nAwjfmEunZ9iPcEe7pHHEV0Opb9nUFvyBXQFWpIgQgkFACFsEJwj2hxkGzIBATAr/f1SE/fpSJjph0RyqqYeEbswnzXNsQ7gj3VAywSmzkoaaxQLjvq+XSr0rsP+oMgSQQsAhOEO5JsATaAIH4EvjShnrRyQ4WCJSLgIVvzCXSs+1HuCPcy2XjlFMAgc/cdVoue6q1gCM5BAIQgEDxBCyCE4R78f1ADhCAQHYC7cOZV+g+/e5Q9gRshUAJCFj4xmzCPNc2hDvCvQTmTJZWBP6wq4PL5a1gkg8EIJA3AYvgBOGeN3YOgAAEPAnsPjkcxEnV3VOeR5AMAsUTsPCNuUR6tv0Id4R78dZLDiUjoE9I/dDqajnaMl6yMsgYAhCAwHIELIIThPtydNkOAQgUS+CGvV3yvYcbi82G4yGQFwEL35hNmOfahnBHuOdlqCQuL4GZuYVAuP95T2d5C6Y0CEAAAiK8Dg4rgAAEYk3gwoca5U8vECPFupMSWDmEe5GdGhVA95cRZhWK7EQOz0rgFzta5cOrq7PuYyMEIACBUhKw8K34xlL2EHlDIL0EBifmgsmNx94eSC8EWh4JAQvf6GpI33Vm3Jlxj8TgKdSfwOPvDAaO6XQv92/5UyMlBCBgQcAiOEG4W/QEeUAAAksJ7K/PvH3nWNvE0l38D4GSErDwjb5i3U2HcEe4l9Swybx4An1js4Fwv+tgb/GZkQMEIACBPAhYBCcI9zyAkxQCEPAmcMf+bvlmVb3obYUsECgnAQvf6Apy33WEO8K9nHZOWQUSuHhrs3z+ntMFHs1hEIAABAojYBGcINwLY89REIDAygR+uKVJfvds+8qJ2AuBEhCw8I2+Yt1Nh3BHuJfAnMnSmsDGw33BrHvP6Kx11uQHAQhAYFkCFsEJwn1ZvOyAAAQKJDB95uG9Dx7pLzAHDoNA4QQsfKMryH3XEe4I98KtliPLRqC+bzoQ7o8cxUGVDToFQQACPFUeG4AABGJJQF+Tq6/LfaNxLJb1o1LJJoBwL7J/owLo/kLCrEKRncjhKxL41oMN8t1NvKt0RUjshAAETAlY+FZ8o2mXkBkEICAi9x/qky/eWycDE3PwgEDZCVj4RldD+q4z486Me9mNnQILI7D6le7g12W9PIwFAhCAQDkIWAQnCPdy9BRlQCBdBH6+vUV+taM1XY2mtbEhYOEbfcW6mw7hjnCPzSCgIisT0Ned6GVhz7w3tHJC9kIAAhAwImARnCDcjTqDbCAAgUUCH19bI3e/xtt2FoGwUlYCFr7RFeS+6wh3hHtZDZ3CiiPw+Xvq5GePtxSXCUdDAAIQ8CRgEZwg3D1hkwwCEPAicKp7MpjI2Fc76pWeRBCwJmDhG33FupsO4Y5wt7Zl8ishgT++0Bk4qxIWQdYQgAAEFglYBCcI90WcrEAAAgYEtr41IJ++q1bah2cMciMLCORPwMI3uoLcdx3hjnDP31o5IjIC++tGA+Gu3ywQgAAESk3AIjhBuJe6l8gfAukicOUz7fJTrj5MV6fHrLUWvtFXrLvpEO4I95gNBaqTi8BHb6+Rq5/ryJWM/RCAAASKJmARnCDci+4GMoAABBwCetvg6le7nS2sQqC8BCx8oyvIfdcR7gj38lo6pRVN4PKdbVwuXzRFMoAABHwIWAQnCHcf0qSBAAR8CLQMTgcx0K6Twz7JSQOBkhCw8I2+Yt1Nh3BHuJfEoMm0dAT0qfL6dPnj7ROlK4ScIQABCIiIRXCCcMeUIAABKwJPvzckH7m9Wur6pq2yJB8I5E3Awje6gtx3HeGOcM/bWDkgWgJj0/OBcL9lH5eJRdsTlA6B5BOwCE4Q7sm3E1oIgXIR+NOeTrloS1O5iqMcCGQlYOEbfcW6mw7hjnDPapBsjDeBS7Y1y/nrauNdSWoHAQhUPAGL4AThXvFmQAMgEBsC36hqkBv2dsWmPlQknQQsfKMryH3XEe4I93SOuApv9eajA8Gse1M/l4pVeFdSfQjEmoBFcFJO4X7s2AlZvfpumZycjDVXKgcBCORPoH98Loh9th8fzP9gjoCAIQEL3+gr1t10CHeEu6EZk1W5COi7S/U+9/ve6C1XkZQDAQikkIBFcFIu4a6ifdWqqxDuKbRTmpwOAvtqM6/EfbeTH+bS0ePxbaWFb3QFue86wh3hHt9RQc1WJPDdTY3y1Y31K6ZhJwQgAIFiCFgEJ+US7trOwcEhqarawox7MZ3OsWcRONoyLnziweDWl7vkwocaZHpu4aw+4h8IlJuAhW/0FetuOoQ7wr3ctk55RgTWv9YbzLoPTswZ5Ug2EIAABM4mYBGcVKJwHxkZEz4wUBv47dMtga/Vq9z4RM9A+4OxydgMbeBsj1W+/yx8oyvIfdcR7gj38lk5JZkSONU9GQQRj709YJovmUEAAhAICVgEJ5Uo3MP2851uAjNzC3LhQ41y68vdzLrH6MqDdFslrY8DAQvf6CvW3XQId4R7HOyfOhRI4Esb6uSHW3ktSoH4OAwCEMhBwCI4QbjngMzu2BJ4rzPzA/lLNSOxrSMVgwAEyk/Awje6gtx3HeGOcC+/tVOiGYGbXuoKZt0XuN3LjCkZQQACHxCwCE7KJdz13nZ9OF340YfVsUCgGAJPHh8MfKw+zZwFAhCAQEjAwjf6inU3HcId4R7aIN8VSOBI83gQVDx/argCa0+VIQCBuBOwCE7KJdzjzpL6VR6BG1/qkq/xENjK6zhqDIESE7Dwja4g911HuCPcS2zaZF9qAp9aVyu/fqqt1MWQPwQgkEICFsEJwj2FhpOQJuutaNe/0JmQ1tAMCEDAioCFb/QV6246hDvC3cqGySciAlc/1x7MukdUPMVCAAIJJmARnCDcE2wgCW5afd+UfPT2annqxGCCW0nTIACBQghY+EZXkPuuI9wR7oXYK8fEiMCLNSOBcH+jcSxGtaIqEIBAEghYBCcI9yRYQvra8PypjG9t6p9OX+NpMQQgsCIBC9/oK9bddAh3hPuKhsnO+BOYm18IhDuX88W/r6ghBCqNgEVwgnCvtF6nvkrgjv09csH608CAAAQgcA4BC9/oCnLfdYQ7wv0cY2RD5RH4+fZW+fCa6sqrODWGAARiTcAiOEG4x7qLqdwyBC7d3iqX7+T5McvgYTMEUk3Awjf6inU3HcId4Z7qgZeUxm8/88qa97smk9Ik2gEBCMSAgEVwgnCPQUdShbwIdI3MymfXn5ZHjg7kdRyJIQCBdBCw8I2uIPddR7gj3NMxwhLeSn3H7IdWV8uaV3sS3lKaBwEIlJOARXCCcC9nj1GWBYH9daOBT32vkx/DLXiSBwSSRsDCN/qKdTcdwh3hnrSxlNr2/GBzk3z+Hu7HS60B0HAIlICARXCCcC9Bx5BlSQlsONQnH7u9pqRlkDkEIFC5BCx8oyvIfdcR7gj3yh011PwsAlWH+4IZgvahmbO28w8EIACBQglYBCcI90Lpc1xUBH7zdJtcsq05quIpFwIQiDkBC9/oK9bddAh3hHvMhwbV8yXQ2D8dCPeqI/2+h5AOAhCAwIoELIIThPuKiNkZMwLDk3Py1Y31cvdrvTGrGdWBAATiQsDCN7qC3Hcd4Y5wj8sYoB4GBL62sV6+s6nRICeygAAEICBiEZwg3LGkSiLwZst48CP4oaaxSqo2dYUABMpIwMI3+op1Nx3CHeFeRjOnqFITuP3VniDgGJueL3VR5A8BCKSAgEVwgnBPgaEkqImbj/YHfnRqdiFBraIpEICAJQEL3+gKct91hDvC3dKOyStiAsfbJ4KA48njgxHXhOIhAIEkELAIThDuSbCE9LTh2uc75XuPNKWnwbQUAhDIm4CFb/QV6246hDvCPW9j5YB4E7hg/Wm5ZFtLvCtJ7SAAgYogYBGcINwroquppIjMzUtwu9kt+7rgAQEIQGBZAha+0RXkvusId4T7skbJjsok8KcXOoNZ9yhrPzI1J3ce7A3q8fLp0SirQtkQgEARBCyCE4R7ER3AoWUl8H7nZOC3XqweKWu5FAYBCFQWAQvf6CvW3XQId4R7ZY0UapuTwIH60SDweKm2/IHH7pPDwWzFh1ZXB++U/+2z7fKxO2pE68QCAQhUHgGL4AThXnn9ntYa7zgxFPjP3rHZtCKg3RCAgAcBC9/oCnLfdYR7DIX7sWMnZNWqq4KPri9d3P27d+9dupv/IRAEHlc+014WEjU9U3L1c+1BmSrYV21vlaffHZLJ2QXRR/tcvrNNPrWuVg41jZelPhQCAQjYEbAIThDudv1BTqUlcNNLXcGr4EpbCrlDAAKVTsDCN/qKdTcdwj1mwn1yclLWr6+SxsZmGRwcktWr7w6+QwPX7Vu3bg/+1f033LAmSBvu57uyCYwf2yvNv/yQtN/4Hel96Pcy/OIDMv7ufpnty0+Eq1hWEV2qZX5+QfR98V+8ry4o5yv318sd+3vkZNfUOUVOzy3IL55sFb33Xl+zwwIBCFQOAYvgBOFeOf2d9pr+6NFmufb5jrRjoP0QgEAOAha+0RXkvusI95gJdxXmKtxVwOtSVbVFss266z5No/tVwLMkg8Dgs+sDwd65+mJp/f3npfG//9dZn+Zfflg6bv536d30Bxl+8UGZeO+gzPafG2Q89/5wIKithfL+ulH52RMti7Prv3qqVXadHJbZ+ZVfmzM+PS+XPtESXD7/TvtEMjqLVkAgBQQsghOEewoMJQFNbOyfDnybXi7PAgEIQGAlAha+0Vesu+kQ7jEU7uGMuhqMXgq/nHDX7cVeKj8+PiF84stgpLVOhk4clP5Xt0nPjjukc+NV0n7rRdJy9efOEvQq8Jt+eZ603fRd6ay6WlqfrQoCkOt3ni66f093jshf97TJ+etqgzy/9UCD3P5yh5xoGc4r766BMfnJo43BLP3RxqG8jsVG42uj9E15+malAKKU+yyCE4R7KXuIvK0IvHAq84N3Q9+5V45ZlUE+EIBAMghY+EZXkPuuI9xjKNx9ZtxVtOtl9OHMfDKGAa3Ih8Bsb6tMVh+R0dd3yOAzd0rvg1dL523/Ka1nRP0P/rpDPnHrO4HAb/7VR6Tjlv+Q3oevleGXNsnE+6/LbP/Kl98/9vagXLS5KRDrH7+jRq58pk32FPmk3b6xWdFLEb+6sV703ngWCEAg3gQsghOEe7z7mNplCKw90BNcFQYPCEAAArkIWPhGX7HupkO4x0y457rHXQ1JZ9kR7bmGVLr3q6jf9PyxQHS/9cQD0vvA76Tz1ouk9XefPWemvvnXH5GOv/2H9D1ynezfuVOufvR4cJzeI//9R5rkvjf6xHIGomtkVn6wuUm+9WCD1DOzkW5DpfWxJ2ARnCDcY9/NVFBEfr69RS57qg0WEIAABHISsPCNriD3XUe4x0y4q6XobPrSp8qHYr2rq1uuuebGxf2artjL5XNaJwkqkkDnyEwgwPV96u4SzNSfOiSjr22XgafXyemN18ntt90rX/3b/iD9P936llx+/X3y2G9+LM2//qh0/O370rf5ehne97BMnnxd9Phil9bBafn3hxuDV8c1D84Umx3HQwACJSJgEZwg3EvUOWRrRqB7ZFbOv7NWNr3Zb5YnGUEAAsklYOEbfcW6mw7hHkPhnlwzp2XlJnDhQ43yrxvqsha7r3ZULjvz9HmdXf/JYy3ywKuNcvroYRk5+IQM7FwrPVVXBcK95bcXnDNT33L5x6Xj1h9I3+Y/ysjLmzOX3+ch6vVBQN9+sEG+v7lJ2ocR71k7iY0QiJiARXCCcI+4Eyk+J4ED9WPBD9cnOjIPBs55AAkgAIFUE7Dwja4g911HuCPcUz3wkt74e17vDYKR7tHZoKn1fdOi9/F94d7Ma9z0dW43vNgpGrTkWvSe+MlTh86I+jukZ+OVwX3zLVd9Jouo/4R03HqR9G35k4y8skUm3n9NZntazimitmdKvl7VID/c2iQ9Z+p4TiI2QAACkRGwCE4Q7pF1HwV7Erj/UJ98al2tZ2qSQQACaSdg4Rt9xbqbDuGOcE/72Et0+/UBcDqb/vCbA/LTxz94jZvey/fo2wOil9NbLPpKusnqwzJ68AkZ3HmH9FZdKZ1/+760/jaLqP/NJ6Xjtv+Uvq1/luGXN8uxNw7Jl++tkf96rEUGxucsqkMeEICAEQGL4AThbtQZZFMyAlc+0y6XbDv3x+WSFUjGEIBARROw8I2uIPddR7gj3Ct64FD53AT+5czs+jeqGmT1K91yXzdpYwAAIABJREFUpHk890GGKWYHOoOn348ceFwGnrpdejZecWam/tOLM/V7fvVN+dwth+XiG5+SutX/HYj64PL79w7ITHeTyMLK74k3rC5ZQcCLwOjUvFe6Sk9kEZwg3CvdCpJd/9HpefnSfXWy/mBPshtK6yAAATMCFr7RV6y76RDuCHczIyajeBK4ZV+XPHViSPRVbHFb5ga6ZLLm78Hl9/sf3SSfWXNcfnrbLjl91T8vinp9R71+Wq74lHSuuVj6Hv2LjLz8iEy8u19muhplYSEdAipufZfW+pzunZJvVjXIx+6oCYL9t1rL+0NYublbBCcI93L3GuXlQ+Boy3hwZdrrjblvGcsnX9JCAALJJWDhG11B7ruOcEe4J3dU0bKKI6BXA5y/riZ4Jc9Uf6dM1mZE/cCO1dJz/2+k4+Z/l5Yrzz9X1F95vnSu+VEg6vXp9xMnXpWZrgZZmOfS+4ozgphX+D8eaQyCfL0FRT9fXubhjzFvhnf1LIIThLs3bhJGQGDLWwPBWJ6Y4UfgCPBTJAQqkoCFb/QV6246hDvCvSIHDJVOLoHXGkbl43fUiN5zuNwyN9QtU7Vvyoi+0u6pNdKz4XLpuOm7wax8OEMffrdc+U/SefuPM/fUv3RG1Hc2yMIcon45vmxfnoDOtKtg/8fbMsL9o7fXSJIDfovgBOG+vD2xJ3oC1z3fIf/xSFP0FaEGEIBAxRCw8I2uIPddR7gj3CtmkFDR9BB4tW5UPry6Wq5+bnnxvhyNuaEemTp9VEZeezK4p757w2XSfuN3pPk3nzx3pv6qT0vn7T+Rvi1/luGXNsn48VdkpqNehJn65fCmfru+iSGcbdfvf3uwIdFMLIIThHuiTaSiGze/sBCM4Vv2dVd0O6g8BCBQXgIWvtFXrLvpEO4I9/JaOqVBwJPA3pqRQCBd93yn5xG5k80N98rU6bdk9PUdMrDzjmCmvv3GC7OK+tbfXnBG1P9Jhvc+JOPHX5bpjjqRufg9KyB3y0lRLAGdVf/9rvbAJr/1QIN8Ym2NXLSlSRr6p4vNOtbHWwQnCPdYd3GqK3eyK/PmlT2nhlPNgcZDAAL5EbDwja4g911HuCPc87NUUkOgjASeP5UR73/eYyfel6v+3HCfTKqof+MpGdy5NnNP/Y3fkZYsM/Wtv1NR/2Pp2/xHGd77oIy/s0+m20+LzNm8Xm+5OrI9GgI9Y7Pyyx2tgWh/sWYkmkpEVKpFcIJwj6jzKDYnAX1wq1450z3KD7I5YZEAAhBYJGDhG33FupsO4Y5wXzRCViAQRwLPvJcJrG56qSuy6s2N9Mtk3dsy+sZOGXx6XfBKu/YbLpSWyz9xzuX3rb/7bPCgvN7N18vwiw/I+LGXZLq9VhZmkz0zG1nnlLjg+r4p+fFjzfKZu2rlUAqfOm0RnCDcS2ykZF8wAb1E/psPJPt2l4LhcCAEILAsAQvf6Apy33WEO8J9WaNkBwTiQmDHmVmRW1+O332I86MDGVF/aKcMPrNOejdeIcHl95d/fBlRf7H0PnKdDO+pkvFje2W6rUYWZqbigpp6OASOd0zIhQ81yFc31sv7nZPOnvSsWgQnCPf02EultfSHW5vlmt0dlVZt6gsBCERMwMI3+op1Nx3CHeEeselTPAT8CGw7Nhhc0njH/h6/A2KQSkX9VP0xGT38tAw+c6f0Vl0p7Td9R5ov+1gWUX+BdK7+ofQ9cp0M7dko42+/GNyPP930rky3VstMZ4PM9raKPlF/fmxQ5ifHZGGWS/NL1c36Tucv3VcXPG26ZTC9nC2CE4R7qayUfIsh0DQwHfiUJ48PFpMNx0IAAikkYOEbXUHuu45wR7incLjR5EolsOVofxBo3XWwcsT7cqxVfE/VvyOjh5+RwWfvkp6NVwavtGvOMlMfvtou6/fP/rc0/c//k+ZffzR4HZ5eqt96zb9I2x+/Ku1//bZ03Py94AeBrrWXSNddl0r3vb+SnqorpXfTH6Rvy5+kf9tN0v/kbTLw9DoZ3HWPDO2pkuF9j8jIgW3B/f6jR56TsbdelPETr8rEyddlsubvMtXwjky3nAzu65/taZbZ/g7RZwTMT4zI/PSkyHxlvw95T/WwfGpdrfz34y0yOJHu1wZaBCcI9+XOAmyPksCe6swzVOr7uI0pyn6gbAhUIgEL3+gr1t10CHeEeyWOF+qcYgKb/p4R7/e90ZdYCvNjQzLVcDwQyuMnXgmE89jfdwVCWgW1CmsV2EO77w0uzx94cnUgwFWIqyDvrbpKeu77tXSv/7l0rf0v6bzth4GA77jh36T9T1+Ttmu+KCrwm6/4lDT9+iPS9It/lMaf/cM5VwFk/aHgv/+XV7qmVf9Xmn91XvBwP31Cf+vvPy9t1/+rtP/lm8EPFJ23XiRdd/xEuvXHhHt+GTwMsPehq6Vv8/XS/9gNMrD91uDJ/4PP3S1DL2yUob2bZOTVrTL62nYZPfS0jB19Pnh+wMS7B2Sy+nDwYMGpxrOvTpgd7BK96mEhuDrBLzjX2Td9WNWVz7TL/EL0Jqa3UYwcfEJGXn8yklsqLIIThHv0dkQNziWw7kBPcFXNuXvYAgEIQGBlAha+0RXkvusId4T7ypbJXgjEkMDGQ32BuKo6nFzxHgX2hbk5mZ+akLmxIdFX5832t8tsd6PMtNXIdNN7MlX3diCSJ947EIjmsTd3y+ihnRlh+crW4LV5Q89vCK4gGNixRvofv1n6H/1LcPl/7wO/C16/1333/0j3nf8tXWt+JJ1/+37wPIC2P39DWq/9ciDuW676tOhVB02//JA0Xvp/vH4k8P6B4dL/E+Sr+Ws5+mNC63VfFi1fn0tw123rA7u6+vbtovXs2XC59DzwW+l9+Frp3/qXoD0DO1YHVyfojyb68MHw6oSR13fI2JHnglscxo+/IhPv69UJR4JbJaab3w8eUDjT3SRzwdUJvTI/PhywlvnlZ/T1doiW314gTb86T5ov/4S0/fnrMj81XlbTsAhOEO5l7TIK8yRw6RMt8uun2jxTkwwCEIDABwQsfKOvWHfTIdwR7h9YIWsQqCAC977eG4isTW/2V1CtqWohBPRefhWxc6MDMjfYFdzrP9NZL9Mtp4IrEyZr3zzr6gS9vF+F9Mj+x2Tk5UeCBwEO7boneCPAwJO3Sf+2G4NX+fU+9PvgDQF6+8AtazcH9vSX2x6S9pu+K+1//Za0/fEr0vqHLwTiufk3n5RmvTph1f9n+2PCz/5Bmv7nH6X5so8GV0DolRBt134xuNVBRXvDpZkrIfQKhpbfXSCjr+0oBGHBx1gEJwj3gvFzYIkI6CseP35HjTz0d/xHiRCTLQQSTcDCN7qC3Hcd4Y5wT/TAonHJJnDnwZ5AbG15ayDZDaV1JSVw80tdgR3plRy+y8LCfHA///z4iMyP9MvcQKfo/f4zHadFZ9j1oYT6PICJ91+T8eMvy9jRF2Ts8DPBpf56yf/wS5tk6Pn7ZfDZ9TLw1O3S/8Qt0vfoX4LZ/Z4HfifdGy6Xtj9+7YNbGH7+f6Xlqs8Ezx7wraNFOovgBOFu0RPkYUngYP1oMOaPt09YZkteEIBASghY+EZfse6mQ7gj3FMyxGhmUgncvj8j3h9/hycDJ7WPS9WuyZl5+cOujiCA17cWxG2ZHeiU5l9+WBov/Qdp+sX/k9arPyezQ+V9MKNFcIJwj5tlUZ+qw/3ymbtOAwICEIBAQQQsfKMryH3XEe4I94IMloMgECcCf3u5OxBfO07ET3zFiRN1+YBA79is/HJHq5y3plp2nxz+YEfM1mZ7W4JL/HVmfrav/PfjWgQnCPeYGRXVkSufaQveGgEKCEAAAoUQsPCNvmLdTYdwR7gXYq8cA4HYEbjxzOXOz74fXxEWO2gprVBj/7T86NFmuWD9aXmtYSylFPyabRGcINz9WJOqPATGpufln+8+LXceKO/VK+VpHaVAAALlIGDhG11B7ruOcEe4l8O+KQMCZSHwpz2dwcz786fSI967R2fl0bcH5JJtzfLdTY3SOTJbFtaVWsiJjgm58KFG+drGeuH+1ty9aBGcINxzcyZF+Qi81TYR+InXGkbLVyglQQACiSJg4Rt9xbqbDuGOcE/UQKIxELh2d+aeZX1g3amuyUQCaR+akYff7JeLtzYHAejn7j4tv3su0259B/njx3hYX7aOf6NxTL54X518/5Emaej3e697tnzStM0iOEG4p8li4t/WrW8NyEdvr5Hx6fn4V5YaQgACsSRg4RtdQe67jnBHuMdyQFApCBRD4Orn2gNBqyL2O5saRZ8a/mLNiAxMLP/O7GLKK8exTQPTUnWkPxCd2q4v3Hta9EeKp98bko7hmaAKOpt88damoO16//b07EI5qlYRZWj/f3Jdrfzs8RbpG69cOyg3bIvgBOFe7l6jvJUI/PGFTvnPLc0rJWEfBCAAgRUJWPhGX7HupkO4I9xXNEx2QqBSCbzVOiH3H+qTS59olY/dUROIWX1v76rtrVJ1uE/eqYDXANX2TIm+r15/fFCx/sV76+TPezpFbwVYSXyuPZB50r4ec6Cee7ifPDEU8NMHUvFjRn4j2iI4Qbjnx5zUpSOgP2V+vapebnqpq3SFkDMEIJB4Aha+0RXkvusId4R74gcXDYSAPkF8T/VIEKxd+FBDIOJU1H51Y71c/3xm1rp3LB6zsO93TYq+n/6bVfVBPb90X53cvK9L9tWOyvCkfx0P1I/KN87kccPeztQawaa/9wcc9fkHLPkTsAhOEO75c+eI0hA41T0ZnA9eODVSmgLIFQIQSAUBC9/oK9bddAh3hHsqBhiNhIBLoLp7SjYf7ZfLd7YFTxZXEa+fHz/WLHcd7JGjLeNu8pKv60PS1rzaLV+5PyPW9Xv1qz2yv25UJmYKvw9zem5Bwgf26Q8AJ7umSt6WOBVw18HeoF+VLUthBCyCE4R7Yew5yp7A0+9mrr7pGsncXmRfAjlCAAJpIGDhG11B7ruOcEe4p2F80UYILEtgbn4heCXYHft75KItmfvDVcR//p7TctUz7fLk8UHpPHMP+bKZFLDjaOu43LKvW/7l3rpAXOrs/9r9PXKoaUzmCtfqWWvyzHtDiz9Q6H3yaVhu3Jt5w4DeLsFSOAGL4AThXjh/jrQlcMu+ruCtEra5khsEIJA2Aha+0Vesu+kQ7gj3tI012guBFQnoTMzOd4fkD7s65EsbMjPgKuS/90iT3L6/R440Fz4bf6hpXG7Y2yWfvft0INa/XtUg6w70iN6PX+pFXxt32c62oNwfbW0SfTJ9Ehe9yiB8OOFjb/N0/WL72CI4QbgX2wscb0XgPx5pkmt2dVhlRz4QgEBKCVj4RleQ+64j3BHuKR1yNBsCfgT0nvOqw/3y020tct6azCX1n1pXG4jgx98ZlLYcAlgvd9f76M9fl3lA3jcfaAguxz/REc2r6vQ1cuGtAc++N+QHoUJS9Y3Nyi+ebA36adfJ4QqpdbyraRGcINzj3cdpqV3z4Exw7nvincG0NJl2QgACJSJg4Rt9xbqbDuGOcC+RSZMtBJJHQGdzVYjr5ZbfevCDh9ypGL/tlW55vTHzBPd9tSPy+13ti0+z/9YDDbL+tV7RByPFYdGn1f/o0cw74K9+rkNGp/wfeheH+merg74uT1+F99n1p3mSfjZABW6zCE4Q7gXC5zBTAvpKSP3Rsq43Xc/6MIVIZhCAQEDAwje6gtx3HeGOcGcIQgACBRLQy831Hnh9zdj5d9YuzmRrcPhvDzbIPa/3yukYB4l6mb7WVeteya+N0/fXf/vBBvn6xno51lb62w4KNJeKPMwiOEG4V2TXJ67S+rBKfZYICwQgAIFiCVj4Rl+x7qZDuCPci7VdjocABM4Q0KfD63vX6/umK4aJPiRPrxhQAa8P6Ku0Ra9y+MK9dcGDBZlJs+89i+AE4W7fL+SYP4FLHmuW3zzdlv+BHAEBCEBgCQEL3+gKct91hDvCfYkp8i8EIJBGAn95sSsQ73qlgP4AEfdFL/ff+taAfHxtjVy6vVX04Xss9gQsghOEu32/kGN+BHrHZuXDq6vlgSO8ZSI/cqSGAASyEbDwjb5i3U2HcEe4Z7NHtkEAAikkoPfmX7A+88T7B2P22jh9voDOrq890CMXbc68tu9jd9TIVc+2y3gR77pPYTfn1WSL4AThnhdyEpeAgJ479KoibqUpAVyyhEAKCVj4RleQ+64j3BHuKRxuNBkCEFiOgL5D/jdPtwdB7qrtrdLYH91l//oMgaffHZJrd+ur+TLvu//K/fXypxc6RZ8azyz7cr1ot90iOEG42/UHORVGoOpwn/zzPXWFHcxREIAABJYQsPCNvmLdTYdwR7gvMUX+hQAEICDBQ/d0hko/+gC+ci3vtE/IxsN9cukTrYtP5b94a3PwVP4jTWMyv1CumlCOErAIThDu2FLUBH79VKv8fHtL1NWgfAhAICEELHyjK8h91xHuCPeEDCGaAQEIWBPoH5+VnzyWeW2cznoPTNi/Nm5wYk721ozITS91yXc3NQY/FHzk9mq54ul22XZsUOr7eHWTdb/mk59FcIJwz4c4aa0JjM8syKfvqpV1B3qtsyY/CEAgpQQsfKOvWHfTIdwR7ikdcjQbAhDwJaD3u+vMuz69Xe+DL3YJHyynr9H7/D2ZS+C/vKFObt7XJS/VjMjwpP0PBMXWOa3HWwQnCPe0Wk882v1220Rw/jpQPxqPClELCECg4glY+EZXkPuuI9wR7hU/eGgABCBQegIN/dPBu9JVwN/6crfML/hfsx4+WG79wR65ZFuLfHhN5hL8/9zSFFwW/w7vXi99BxZYgkVwYiXcjx07IatWXRV8dH3psnv33sX9V1xxnTQ2Ni9Nwv8pJPDYsUH55LpaGZ3iB8EUdj9NhkBJCFj4Rl+x7qZDuCPcS2LQZAoBCCSTgL7rXcX79x5ulDdbxpdtpD5Y7rn3h+UvL3YuCv7z1lTL5Tvb5KkTg9I2NLPsseyIDwGL4MRCuE9OTsr69VWBGB8cHJLVq+8W/XYXFe7ZBL2bhvX0EfjDrg750aP8iJO+nqfFECgdAQvf6Apy33WEO8K9dFZNzhCAQCIJ6CuVPnd35rVx973xwX2j73ZOysNv9stlO9sWXyv3pfvq5MaXuuRg/ahMzfrP0icSXAU2yiI4sRDuOnuuwl0FvC5VVVvOEem6LZyR13UWCCiBL2+ol1v2dQEDAhCAgBkBC9/oK9bddAh3hLuZEZMRBCCQLgJ/eqEjmH3/2eMtcvHWzLvVdTZe37O+4VCfnOrOiKx0UUlWay2CEyvhvnXr9kW4K82uq7jXGfliZt8nJqaET+UzON4yHJyjdr7TT39i09hAAm1g0SmUecXCN7qC3Hcd4Y5wL7OpUxwEIJAkAq/WjQaBsc6y62vjesdmk9S81LfFIjixEu65ZtzdzlJhrx+WdBN4+r2h4PzUOcJ5Kd2WQOshYEvAwjf6inU3HcId4W5ryeQGAQhAAAKJIWARnFgI91z3uOv97ocOvRlwd9MmpiNoSEEE9DYdfR4HCwQgAAFLAha+0RXkvusId4S7pR2TFwQgAAEIJIiARXBiIdwVabanyuusul4W39XVLTt2PLd4jzuz7QkywiKacuFDjfLHFzqKyIFDIQABCJxLwMI3+op1Nx3CHeF+rjWyBQIQgAAEICAiFsGJlXCnQyCQD4GWwZngMvltxwbzOYy0EIAABHISsPCNriD3XUe4I9xzGicJIAABCEAgnQQsghOEezptJ+pW760ZCYR7bc9U1FWhfAhAIGEELHyjr1h30yHcEe4JG0o0BwIQgAAErAhYBCcId6veIJ98CKw90CPfqKrP5xDSQgACEPAiYOEbXUHuu45wR7h7GSiJIAABCEAgfQQsghOEe/rsJg4t1ldU/vbZ9jhUhTpAAAIJI2DhG33FupsO4Y5wT9hQojkQgAAEIGBFwCI4Qbhb9Qb5+BLoG5sNLpN/4Eif7yGkgwAEIOBNwMI3uoLcdx3hHkPhnu3JuUstSdPok3T1tTcsEIAABCAAgVIQsAhOEO6l6BnyXInA641jgXB/q3V8pWTsgwAEIFAQAQvf6CvW3XQI95gJd/f9s/peWhXn+u0uobBHuLtUWIcABCAAAWsCFsEJwt26V8gvF4GNh/rkX+49LQu5ErIfAhCAQAEELHyjK8h91xHuMRPujY3Nsn591eJMelXVluDdtUttSsW87mPGfSkZ/ocABCAAASsCFsEJwt2qN8jHl8Cq7S3yyx2tvslJBwEIQCAvAha+0Vesu+kQ7jEU7lu3bl80nt2795ZUuE9NTQsfGGAD2AA2EG8bWHQKZV6xCE4Q7mXutJQXNzEzL59YWyPrX+tNOQmaDwEIlIqAhW90BbnvOsI9hsK9nDPuCwsLwgcG2AA2gA3E2wZKFXzkytciOEG456LMfksCx9rGg/vb99ePWWZLXhCAAAQWCVj4Rl+x7qZDuMdMuPvc465Ww6Xyi2OHFQhAAAIQKBEBi+AE4V6iziHbrAQefXtA/unOWhmenMu6n40QgAAEiiVg4RtdQe67jnCPmXBXQwofPrdq1VWLl8nrJfPhw+j03nbdF340PQsEIAABCEDAmoBFcIJwt+4V8luJwFXPtssl25pXSsI+CEAAAkURsPCNvmLdTYdwj6FwL8qSOBgCEIAABCBgRMAiOEG4G3UG2XgR+Pw9dbL6lW6vtCSCAAQgUAgBC9/oCnLfdYQ7wr0Qe+UYCEAAAhBIAQGL4AThngJDiUkTa3qmgvvbd70/HJMaUQ0IQCCJBCx8o69Yd9Mh3BHuSRxPtAkCEIAABAwIWAQnCHeDjiALLwLPvDck562plvbhGa/0JIIABCBQCAEL3+gKct91hDvCvRB75RgIQAACEEgBAYvgBOGeAkOJSRP/tKdTLtrcFJPaUA0IQCCpBCx8o69Yd9Mh3BHuSR1TtAsCEIAABIokYBGcINyL7AQO9ybwzQca5K8vdnqnJyEEIACBQghY+EZXkPuuI9wR7oXYK8dAAAIQgEAKCFgEJwj3FBhKDJrYOjgd3N/+xDuDMagNVYAABJJMwMI3+op1Nx3CHeGe5HFF2yAAAQhAoAgCFsEJwr2IDuBQbwL7akcC4V7TPeV9DAkhAAEIFELAwje6gtx3HeGOcC/EXjkGAhCAAARSQMAiOEG4p8BQYtBEfQXcvz3YEIOaUAUIQCDpBCx8o69Yd9Mh3BHuSR9btA8CEIAABAokYBGcINwLhM9heRH4/iON8oddHXkdQ2IIQAAChRCw8I2uIPddR7gj3AuxV46BAAQgAIEUELAIThDuKTCUiJvYPz4XXCb/8Jv9EdeE4iEAgTQQsPCNvmLdTYdwR7inYXzRRghAAAIQKICARXCCcC8APIfkReCNxrFAuB9tGc/rOBJDAAIQKISAhW90BbnvOsId4V6IvXIMBCAAAQikgIBFcIJwT4GhRNzEe17vlX/dUCfzCwsR14TiIQCBNBCw8I2+Yt1Nh3BHuKdhfNFGCEAAAhAogIBFcIJwLwA8h+RF4KfbWuTynW15HUNiCEAAAoUSsPCNriD3XUe4I9wLtVmOgwAEIACBhBOwCE4Q7gk3koibNzkzL+etqZb73uiLuCYUDwEIpIWAhW/0FetuOoQ7wj0tY4x2QgACEIBAngQsghOEe57QSZ4XgePtE8H97fvrRvM6jsQQgAAECiVg4RtdQe67jnBHuBdqsxwHAQhAAAIJJ2ARnCDcE24kETdPnyT/2fWnZXhyPuKaUDwEIJAWAha+0Vesu+kQ7gj3tIwx2gkBCEAAAnkSsAhOEO55Qid5XgQu29kmlz7RktcxJIYABCBQDAEL3+gKct91hDvCvRi75VgIQAACEEgwAYvgBOGeYAOJQdMuWH9a1u7viUFNqAIEIJAWAha+0Vesu+kQ7gj3tIwx2gkBCEAAAnkSsAhOEO55Qie5N4Hanqng/vYXTo14H0NCCEAAAsUSsPCNriD3XUe4I9yLtV2OhwAEIACBhBKwCE4Q7gk1jhg068njg/LxO2qkbWgmBrWhChCAQFoIWPhGX7HupkO4I9zTMsZoJwQgAAEI5EnAIjhBuOcJneTeBK7d3SE/erTZOz0JIQABCFgQsPCNriD3XUe4I9wt7Jc8IAABCEAggQQsghOEewINIyZN+sr99XLzvq6Y1IZqQAACaSFg4Rt9xbqbDuGOcE/LGKOdEIAABCCQJwGL4AThnid0knsRaB+aCe5vf+rdIa/0JIIABCBgRcDCN7qC3Hcd4Y5wt7Jh8oEABCAAgYQRsAhOEO4JM4qYNGdP9Ugg3E91T8akRlQDAhBICwEL3+gr1t10CHeEe1rGGO2EAAQgAIE8CVgEJwj3PKGT3IvAzfu65d8fbvRKSyIIQAAClgQsfKMryH3XEe4Id0s7Ji8IQAACEEgQAYvgBOGeIIOIUVO+81CDXP9CZ4xqRFUgAIG0ELDwjb5i3U2HcEe4p2WM0U4IQAACEMiTgEVwgnDPEzrJcxIYGJ8NLpPf+vZAzrQkgAAEIGBNwMI3uoLcdx3hjnC3tmXygwAEIACBhBCwCE4Q7gkxhhg147WGsUC4v9U6HqNaURUIQCAtBCx8o69Yd9Mh3BHuaRljtBMCEIAABPIkYBGcINzzhE7ynATWHeiRr1fVy+z8Qs60JIAABCBgTcDCN7qC3Hcd4Y5wt7Zl8oMABCAAgYQQsAhOEO4JMYYYNeNHjzbLVc+2x6hGVAUCEEgTAQvf6CvW3XQId4R7msYZbYUABCAAgTwIWAQnCPc8gJM0J4Hp2YXgMvmqI/0505IAAhCAQCkIWPhGV5D7riPcEe6lsGfyhAAEIACBBBCwCE4Q7gkwhBg14e228UC4v9YwGqNaURUIQCBNBCx8o69Yd9Mh3BHuaRpntBUCEIAABPIgYBGcINzzAE7SnASqDvfJF+6tk8GJuZxpSQABCECgFAQsfKMryH3XEe4I91LYM3lCAAIO6NlPAAATJ0lEQVQQgEACCFgEJwj3BBhCjJrwiydb5Rc7WmNUI6oCAQikjYCFb/QV6246hDvCPW1jjfZCAAIQgIAnAYvgBOHuCZtkXgQ+ta5W1r/W65WWRBCAAARKQcDCN7qC3Hcd4Y5wL4U9kycEIAABCCSAgEVwgnBPgCHEpAk1PVPB/e17a0ZiUiOqAQEIpJGAhW/0FetuOoQ7wj2N4402QwACEICABwGL4ATh7gGaJF4Eth0bkPPvrJWWwWmv9CSCAAQgUAoCFr7RFeS+6wh3hHsp7Jk8IQABCEAgAQQsghOEewIMISZN+N1z7XLJtpaY1IZqQAACaSVg4Rt9xbqbDuGOcE/rmKPdEIAABCCQg4BFcIJwzwGZ3d4E/uXeOrntlW7v9CSEAAQgUAoCFr7RFeS+6wh3hHsp7Jk8IQABCEAgAQQsghOEewIMIQZNaB+aCe5vf+a9oRjUhipAAAJpJmDhG33FupsO4Y5wT/O4o+0QgAAEILACAYvgBOG+AmB2eRN47v1hOW9NtegD6lggAAEIREnAwje6gtx3HeGOcI/S7ikbAhCAgBmBxsZmWb++SiYnJ5fN0yeNe/CxYyekqmqLuylV6xbBCcI9VSZTssb+5cUu+cHmppLlT8YQSCoBH7/nk8blg288Lb5i2zIdwh3h7o5D1iEAAQhULAGfwMMnjQugkOBkcHAoEPsr/YDglhHndYR7nHsnXXX71oMNouKdBQIQyI+Aj9/zSeOWim9EuLv2kPe6RXBR7C8izCrk3W0cAAEIQMCMgE/g4ZPGrRDBSfHBCb7RtSjWCyEwNDkX3N+ur4NjgQAE8iPg4/d80ril4huL942F6E5m3Jlxd8ch6xCAAAQqgsDu3Xtl1aqrFj+rV98tNTWnz7pU3k2j67pocPLXv66WK664LjhWjwtnxvWS+DBPDUp08QlOwuOuueZG6erqFs1T89EytDz9hOWFl92HQZIeo2ndegQFx+SPxY/iCPeYdGYFV+OV06OBcD/ePlHBraDqECg9Adfvhb4F32jP3cI3ItwNRXghMAlO7AcGOUIAAhBYSkCFtgpgvSRdl61bty8K5PAedxXcoRjW9Lqu21Qw33DDmsVjNR8NdHT7qVO1QX66rnnqkku4hwJcy9D6NDW1BHmF9dBtWnZY11deObhY12z1CAqN0R+L4ATfGKMOrdCqrH6lW779YIPMzC9UaAuoNgRKTwDfWHrGYQkWvrEQrcmMu6HYJzgJzZlvCEAAAqUloII6nB0PBborolWMh7PsWpPwfzeNbtd8wnT6vTTPXMJd89A8dUY9Wz3CfWG++q15rlSP0pLLL3eL4ATfmB9zUp9L4KLNTfL7XR3n7mALBCBwFgF841k4SvaPhW9EuBuK8EJgEpyUbHyQMQQgAIGzCOzY8dziLHa4wxXDGryEQjrXjLum1U8o4N1jdT28vD0sx/0OZ9l1W5iHW4+lM+7hsW4a3aZl6PFxWyyCE3xj3Hq1suozPbcQXCa/6e/9lVVxaguBCAjgG8sD3cI3FqI1mXE3FPsEJ+UZLJQCAQhAQEWuO4sdXjqv94yHAtidQQ9FuQpm9x73UJQfOvTm4n3oKvh1uwp+Ta956ne2RS+vD+9T11l3zSf8ocCdXQ/vcddvzUs/4TZNF9YjWxlRbrMITvCNUfZg5Zd9pHksEO6Hm8YqvzG0AAIlJoBvLDHgM9lb+EaEu6EILwQmwYn/YBkfn5ChoRH/AxKQcmJC74EdTkBL/JtAm/1ZVXLKcvezzmKHwlq5Lf2/HCyLbbMK9/A++HLUt9AyLIITfGNh9Ht6+mV2drawgxN01O0vtciXN9TL4MRcglqVf1OwhwwzOCzPYakvXPp//lZX/iPS5BsL0ZrMuBuKfYIT/wGOcPdnVckpixU3ldh22lyeXlPhHs64h7PYpSx56SzG+vUPyKFDRxefSJ9v2WkKTvCN+VpHJj0CJcPhJ1vq5ddPtRYGMUFHYQ+MC9ecl7OHqH2j+uXwNjW3vr7rafKNCHdDEV4ITIIT32EpgnD3Z1XJKRGxldx7/nWnn/1ZVVpK9WuF+EP3GHxjYb2+XGBeWG6Ve9RHb6+Ruw92V24DjGqOPWRAwgEORkOqqGwsfKPrJ33XmXE3FPvlCk7016jw3sjwvs2irC+CgxHuEUCPoEgEXQTQIyiSfo4AepmKtAhO8I2FdRYCReT9zsng/vaXqjOvfiyMZDKOwh4y/QgHOMRhRFv4Rl+x7qZDuFegcNfLYPSyTX0Akt4jqUK+0haEe6X1WGH1RdAVxq3SjqKfK63H/OtrEZyUS7gnwTe6PYNAEdl8dEA+c1ettPRPuGhSuY49ZLodDnCIwwnAwje6gtx3HeFeYcJdHzSh947oty46417MrHtHR7fwgQE2gA1gA/G2gagCFYvgpBzC3do3Xry5TvjEg8GPN9cTpxCrYQPYQFYbqGTf6CvW3XQI9woU7jqroLPtuujMezHCvbOzRzo7u0W/NXDn/9LyyDBOF+/QrsLvNNibtjVN7U3r+SPTx+UZz5UcnJRLuCfNN4bnkPA7DefOtJ5LfGKv0A7Cb+yhPOfeuPIO6xV+p9UeKtk3uoLcdx3hXoHC3XLGPSqD51L5qMiXt1wuoS4v76hKo5+jIl/6ctM64156srlL4JLgDCM4wMEdLdgD9uDaQ1TrFr7RV6y76RDuFSbc1UCTcB8fwj2qU015y0XQlZd3VKXRz1GRL325FsFJOWbclUQSfKPbowiUDA04wIFx4RLAHs6lUf4tFr7RFeS+6wj3ChTuPFW+/APUokTEjQXF+OdBP8e/jyxqmJZ+tghOyiXck+AbXdtEsGZowAEOjAuXAPZwLo3yb7Hwjb5i3U2HcK9A4V5+87QvkRl3e6ZxzDEt4sZlT5tdGsldT0s/WwQn5RLuSbM2BGumR+EAB3dsYw/Yg2sPUa1b+EZXkPuuI9wR7pHYPMI9EuxlLzQt4sYFS5tdGsldT0s/WwQnCPfCxgECJcMNDnBwRxD2gD249hDVuoVv9BXrbjqEO8I9KpunXAhAAAIQiDkBi+AE4R7zTqZ6EIAABCCQFwEL3+gKct91hDvCPS9DJTEEIAABCKSHgEVwgnBPj73QUghAAAJpIGDhG33FupsO4Y5wT8P4oo0QgAAEIFAAAYvgBOFeAHgOgQAEIACB2BKw8I2uIPddR7gj3GM7KKgYBCAAAQhES8AiOEG4R9uHlA4BCEAAArYELHyjr1h30yHcEe62lkxuEIAABCCQGAIWwQnCPTHmQEMgAAEIQEBELHyjK8h91xHuCHcGIAQgAAEIQCArAYvgBOGeFS0bIQABCECgQglY+EZfse6mQ7gj3Ct0yFBtCEAAAhAoNQGL4AThXupeIn8IQAACECgnAQvf6Apy33WEO8K9nHZOWRCAAAQgUEEELIIThHsFdThVhQAEIACBnAQsfKOvWHfTIdwR7jmNkwQQgAAEIJBOAhbBCcI9nbZDqyEAAQgklYCFb3QFue86wh3hXtIxNTg4JDfcsEYaG5sXy9H1K664Tlatukp27967uL2qakuw7ZprbhQ9rhIXrbfWX9u2evXdMjk5GTQjLW12+y7JbQ5t89ixE0F/h/aa5Da7bXPHrrs9aeNZ+1nbpO3Vj/a3Lklvc2jf+m0RnCDcXaLnrquNqf9zl2z+cDm7c4+r1PVs40zbkmYOuc6nSbYH7XuNnzSOSjOH0P7V/7jxVbjd3ZZ0e4jbuc3CN/qKdTcdwh3hXrKxEIpYFel6QgkXPeFoAKwn5fXrq4J9ul/XdZvu0zSVtmjdtd6hiNP10OEktc3aR1u3bl/sX21vGtqs7Vab1aBC7dbt8yTadthe7eulS5Jt2z0v6fh+5ZWDQfOT3Oal/WsRnCDcl1L94H89X2pQrjYVLq7duf4wm92Fx1Tyt7Yx9Bvadv2xX8+paeOg7Q3Psdp+9S9p5BDastq7flzbyBYnJnVcKAc3vgq5pG1chO2O27eFb3QFue86wh3hXvKx4J54XGekBesJ2f3otqVpSl7BEhWg7dKAZGl73PbqepLaHDrQpLc5/NHp1KnaxR9rkt5mteVw5jn8lT/pbQ7HsHuKSHqb3bbqukVwgnBfSvXs/13BpntCH6Hrob01N7cuCrmlac7OrbL/0/aqH9FzbJo5uOIsjRzCH6zcH3XSxkHHgP54E/pdbf/SsZ+280Oczm4WvtFXrLvpEO4I95KPg6XCPXTKWnB4UtYTkq7roicrTaMnpEpd3ODDXdf2JLHN2n/qXMI+THqbwx8oXFtNepvdsaj9HY7RJI9nPXft2PHcYuCk9p2mftY+twhOEO7u6Dl3PZtwD8+l4TlGhXu2sXZubpW9xY0F3PW0cNDzi/4wqoJN26xL2ji4P1qE8VIaObgjWe0ivO00bfbgcojTuoVvdAW57zrCHeFe8nGwVLiHl39pwXoCcj+6TU9QbpqSV9C4APcEm609bnt1PVsa4yqVLTsNLENB5/ZhktocBlbhr+D6rYHWcjNiYduT1M9hYNXV1X3WWA3bGn5XeptDe9Z2pKXNS08WFsEJwn0p1bP/zybcdQzpEvrD5c4vZ+dU2f9pm3XMhUu280gaOGj71SbUr+h32jhoe13/qushA/3WJU3jIhwPOjbCHzLSzCHkEfW3hW/0FetuOoQ7wr3ktu8Kdy0sPPnor8lJusdd2+Y6WxdsktusM5LqRHVRpxIGXkluc9i34SxQ2P4kt1nv73ZngMLAIcltdu1Zx3Z4f2WS2xzadvhtEZwg3EOa2b/VttRPhotra64NZrO78JhK/9a26cdd0sZB2xs+R8MnPkqyPYR2EApV/T+N9qC34+kS/lCh32njENpC3L4tfKMryH3XEe4I95KNBT3hur+ahk5ZTzpJfaq8ttFtc3i5W1LbrMGF9nPYn+G9z2pUSW2zO2CWCvektnlpP4d2nYZ+Dse0+5DNpPaza9vhukVwgnAPaZ79reNKx1LoM1wbC+3O55x6dq6V9587nkIW6ld0SRMHFWX642DIIPxxNG0cXAt2hXvaOCy1h3BMpI2Daw9xWrfwjb5i3U2HcEe4x2kcUBcIQAACEIgRAYvgBOEeow6lKhCAAAQgUDQBC9/oCnLfdYQ7wr1o4yUDCEAAAhBIJgGL4AThnkzboFUQgAAE0krAwjf6inU3HcId4Z7WMUe7IQABCEAgBwGL4AThngMyuyEAAQhAoKIIWPhGV5D7riPcEe4VNVCoLAQgAAEIlI+ARXCCcC9ff1ESBCAAAQiUnoCFb/QV6246hDvCvfTWTQkQgAAEIFCRBCyCE4R7RXY9lYYABCAAgWUIWPhGV5D7riPcEe7LmCSbIQABCEAg7QQsghOEe9qtiPZDAAIQSBYBC9/oK9bddAh3hHuyRhKtgQAEIAABMwIWwQnC3aw7yAgCEIAABGJAwMI3uoLcdx3hjnCPgflTBQhAAAIQiCMBi+AE4R7HnqVOEIAABCBQKAEL3+gr1t10CHeEe6E2y3EQgAAEIJBwAhbBCcI94UZC8yAAAQikjICFb3QFue86wh3hnrKhRnMhAAEIQMCXgEVwgnD3pU06CEAAAhCoBAIWvtFXrLvpEO4I90oYH9QRAhCAAAQiIGARnCDcI+g4ioQABCAAgZIRsPCNriD3XUe4I9xLZtRkDAEIQAAClU3AIjhBuFe2DVB7CEAAAhA4m4CFb/QV6246hDvC/WxL5D8IQAACEIDAGQIWwQnCHXOCAAQgAIEkEbDwja4g911HuCPckzSOaAsEIAABCBgSsAhOEO6GHUJWEIAABCAQOQEL3+gr1t10CHeEe+TGTwUgAAEIQCCeBCyCE4R7PPuWWkEAAhCAQGEELHyjK8h91xHuCPfCLJajIAABCEAg8QQsghOEe+LNhAZCAAIQSBUBC9/oK9bddAh3Q+FeX18vc3NzqTJcGgsBCEAAAskkoP5M/ZobNBSyjm9Mpn3QKghAAAJpJGDlGwvxpwh3Q+He1dUl3d3diPc0jmLaDAEIQCBBBDQwUX+mfq2Q4MI9Bt+YIMOgKRCAAARSTMDSN7p+0ncd4W4o3BW6Big6u6CXUPCBATaADWAD2EAl2oD6MQvRHgYj+EbGQSWOA+qM3WID2IBrA9a+MfSRvt8Id2Ph7guedNNFz+LAEIbYADaADWAD2AA2gA1gA9gANpAGG0C4I9wR0NgANoANYAPYADaADWAD2AA2gA1gAzG2AYR7jDsnDb8c0UZ+IcUGsAFsABvABrABbAAbwAawAWxgZRtAuCPc+WUNG8AGsAFsABvABrABbAAbwAawAWwgxjaAcI9x5/Cr08q/OsEHPtgANoANYAPYADaADWAD2AA2kAYbQLgj3PllDRvABrABbAAbwAawAWwAG8AGsAFsIMY2gHCPceek4Zcj2sgvpNgANoANYAPYADaADWAD2AA2gA2sbAMId4Q7v6xhA9gANoANYAPYADaADWAD2AA2gA3E2AYQ7jHuHH51WvlXJ/jABxvABrABbAAbwAawAWwAG8AG0mADCHeEO7+sYQPYADaADWAD2AA2gA1gA9gANoANxNgG/n/ONIMRRUYsRwAAAABJRU5ErkJggg==)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/colabs/pytorch-lightning/Wandb_End_to_End_with_PyTorch_Lightning.ipynb b/colabs/pytorch-lightning/Wandb_End_to_End_with_PyTorch_Lightning.ipynb index 32a5df77..0d2f738c 100644 --- a/colabs/pytorch-lightning/Wandb_End_to_End_with_PyTorch_Lightning.ipynb +++ b/colabs/pytorch-lightning/Wandb_End_to_End_with_PyTorch_Lightning.ipynb @@ -1,737 +1,682 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "Ws0nlkuGOpDy" - }, - "source": [ - "\"Open\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aU1q92uCOpD1" - }, - "source": [ - "\"Weights\n", - "\n", - "\n", - "\n", - "# W&B Tutorial with Pytorch Lightning" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mHCKzEzSOpD1" - }, - "source": [ - "## 🛠️ Install `wandb` and `pytorch-lightning`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "RnyGWvwDOpD1" - }, - "outputs": [], - "source": [ - "!pip install -q lightning wandb torchvision" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "G5wpAEAoOpD2" - }, - "source": [ - "## Login to W&B either through Python or CLI\n", - "If you are using the public W&B cloud, you don't need to specify the `WANDB_HOST`.\n", - "\n", - "You can set environment variables `WANDB_API_KEY` and `WANDB_HOST` and pass them in as:\n", - "```\n", - "import os\n", - "import wandb\n", - "\n", - "wandb.login(host=os.getenv(\"WANDB_HOST\"), key=os.getenv(\"WANDB_API_KEY\"))\n", - "```\n", - "You can also login via the CLI with:\n", - "```\n", - "wandb login --host \n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YjXGiBLQOpD2" - }, - "outputs": [], - "source": [ - "import wandb" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "r5HflvclOpD2" - }, - "outputs": [], - "source": [ - "wandb.login()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zNuxn8BjOpD2" - }, - "source": [ - "## ⚱ Logging the Raw Training Data as an Artifact" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "X9mkr942OpD2" - }, - "outputs": [], - "source": [ - "#@title Enter your W&B project and entity\n", - "\n", - "# FORM VARIABLES\n", - "PROJECT_NAME = \"pytorch-lightning-e2e\" #@param {type:\"string\"}\n", - "ENTITY = \"wandb\"#@param {type:\"string\"}\n", - "\n", - "# set SIZE to \"TINY\", \"SMALL\", \"MEDIUM\", or \"LARGE\"\n", - "# to select one of these three datasets\n", - "# TINY dataset: 100 images, 30MB\n", - "# SMALL dataset: 1000 images, 312MB\n", - "# MEDIUM dataset: 5000 images, 1.5GB\n", - "# LARGE dataset: 12,000 images, 3.6GB\n", - "\n", - "SIZE = \"TINY\"\n", - "\n", - "if SIZE == \"TINY\":\n", - " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_100.zip\"\n", - " src_zip = \"nature_100.zip\"\n", - " DATA_SRC = \"nature_100\"\n", - " IMAGES_PER_LABEL = 10\n", - " BALANCED_SPLITS = {\"train\" : 8, \"val\" : 1, \"test\": 1}\n", - "elif SIZE == \"SMALL\":\n", - " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_1K.zip\"\n", - " src_zip = \"nature_1K.zip\"\n", - " DATA_SRC = \"nature_1K\"\n", - " IMAGES_PER_LABEL = 100\n", - " BALANCED_SPLITS = {\"train\" : 80, \"val\" : 10, \"test\": 10}\n", - "elif SIZE == \"MEDIUM\":\n", - " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_12K.zip\"\n", - " src_zip = \"nature_12K.zip\"\n", - " DATA_SRC = \"inaturalist_12K/train\" # (technically a subset of only 10K images)\n", - " IMAGES_PER_LABEL = 500\n", - " BALANCED_SPLITS = {\"train\" : 400, \"val\" : 50, \"test\": 50}\n", - "elif SIZE == \"LARGE\":\n", - " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_12K.zip\"\n", - " src_zip = \"nature_12K.zip\"\n", - " DATA_SRC = \"inaturalist_12K/train\" # (technically a subset of only 10K images)\n", - " IMAGES_PER_LABEL = 1000\n", - " BALANCED_SPLITS = {\"train\" : 800, \"val\" : 100, \"test\": 100}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "o8nApIhdOpD3" - }, - "outputs": [], - "source": [ - "%%capture\n", - "!curl -SL $src_url > $src_zip\n", - "!unzip $src_zip" - ] - }, - { - "cell_type": "code", - "source": [ - "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" - ], - "metadata": { - "id": "ALmdQ7wISLaA" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "XQ_Kwsg9OpD3" - }, - "outputs": [], - "source": [ - "import wandb\n", - "import pandas as pd\n", - "import os\n", - "\n", - "with wandb.init(project=PROJECT_NAME, entity=ENTITY, job_type='log_datasets') as run:\n", - " img_paths = []\n", - " for root, dirs, files in os.walk('nature_100', topdown=False):\n", - " for name in files:\n", - " img_path = os.path.join(root, name)\n", - " label = img_path.split('/')[1]\n", - " img_paths.append([img_path, label])\n", - "\n", - " index_df = pd.DataFrame(columns=['image_path', 'label'], data=img_paths)\n", - " index_df.to_csv('index.csv', index=False)\n", - "\n", - " train_art = wandb.Artifact(name='Nature_100', type='raw_images', description='nature image dataset with 10 classes, 10 images per class')\n", - " train_art.add_dir('nature_100')\n", - "\n", - " # Also adding a csv indicating the labels of each image\n", - " train_art.add_file('index.csv')\n", - " wandb.log_artifact(train_art)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1eJpOk_VOpD3" - }, - "source": [ - "## Using Artifacts in Pytorch Lightning `DataModule`'s and Pytorch `Dataset`'s\n", - "- Makes it easy to interopt your DataLoaders with new versions of datasets\n", - "- Just indicate the `name:alias` as an argument to your `Dataset` or `DataModule`\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Z2g9JRrwOpD3" - }, - "outputs": [], - "source": [ - "from torchvision import transforms\n", - "import lightning.pytorch as pl\n", - "import torch\n", - "from torch.utils.data import Dataset, DataLoader, random_split\n", - "from skimage import io, transform\n", - "from torchvision import transforms, utils, models\n", - "import math\n", - "\n", - "class NatureDataset(Dataset):\n", - " def __init__(self,\n", - " wandb_run,\n", - " artifact_name_alias=\"Nature_100:latest\",\n", - " local_target_dir=\"Nature_100:latest\",\n", - " transform=None):\n", - " self.local_target_dir = local_target_dir\n", - " self.transform = transform\n", - "\n", - " # Pull down the artifact locally to load it into memory\n", - " art = wandb_run.use_artifact(artifact_name_alias)\n", - " path_at = art.download(root=self.local_target_dir)\n", - "\n", - " self.ref_df = pd.read_csv(os.path.join(self.local_target_dir, 'index.csv'))\n", - " self.class_names = self.ref_df.iloc[:, 1].unique().tolist()\n", - " self.idx_to_class = {k: v for k, v in enumerate(self.class_names)}\n", - " self.class_to_idx = {v: k for k, v in enumerate(self.class_names)}\n", - "\n", - " def __len__(self):\n", - " return len(self.ref_df)\n", - "\n", - " def __getitem__(self, idx):\n", - " if torch.is_tensor(idx):\n", - " idx = idx.tolist()\n", - "\n", - " img_path = self.ref_df.iloc[idx, 0]\n", - "\n", - " image = io.imread(img_path)\n", - " label = self.ref_df.iloc[idx, 1]\n", - " label = torch.tensor(self.class_to_idx[label], dtype=torch.long)\n", - "\n", - " if self.transform:\n", - " image = self.transform(image)\n", - "\n", - " return image, label\n", - "\n", - "\n", - "class NatureDatasetModule(pl.LightningDataModule):\n", - " def __init__(self,\n", - " wandb_run,\n", - " artifact_name_alias: str = \"Nature_100:latest\",\n", - " local_target_dir: str = \"Nature_100:latest\",\n", - " batch_size: int = 16,\n", - " input_size: int = 224,\n", - " seed: int = 42):\n", - " super().__init__()\n", - " self.wandb_run = wandb_run\n", - " self.artifact_name_alias = artifact_name_alias\n", - " self.local_target_dir = local_target_dir\n", - " self.batch_size = batch_size\n", - " self.input_size = input_size\n", - " self.seed = seed\n", - "\n", - " def setup(self, stage=None):\n", - " self.nature_dataset = NatureDataset(wandb_run=self.wandb_run,\n", - " artifact_name_alias=self.artifact_name_alias,\n", - " local_target_dir=self.local_target_dir,\n", - " transform=transforms.Compose([transforms.ToTensor(),\n", - " transforms.CenterCrop(self.input_size),\n", - " transforms.Normalize((0.485, 0.456, 0.406),\n", - " (0.229, 0.224, 0.225))]))\n", - "\n", - " nature_length = len(self.nature_dataset)\n", - " train_size = math.floor(0.8 * nature_length)\n", - " val_size = math.floor(0.2 * nature_length)\n", - " self.nature_train, self.nature_val = random_split(self.nature_dataset,\n", - " [train_size, val_size],\n", - " generator=torch.Generator().manual_seed(self.seed))\n", - " return self\n", - "\n", - " def train_dataloader(self):\n", - " return DataLoader(self.nature_train, batch_size=self.batch_size)\n", - "\n", - " def val_dataloader(self):\n", - " return DataLoader(self.nature_val, batch_size=self.batch_size)\n", - "\n", - " def predict_dataloader(self):\n", - " pass\n", - "\n", - " def teardown(self, stage: str):\n", - " pass" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "NLhjdvF-OpD3" - }, - "source": [ - "##How Logging in your Pytorch `LightningModule`works:\n", - "When you train the model using `Trainer`, ensure you have a `WandbLogger` instantiated and passed in as a `logger`.\n", - "\n", - "```\n", - "wandb_logger = WandbLogger(project=\"my_project\", entity=\"machine-learning\")\n", - "trainer = Trainer(logger=wandb_logger)\n", - "```\n", - "\n", - "\n", - "You can always use `wandb.log` as normal throughout the module. When the `WandbLogger` is used, `self.log` will also log metrics to W&B.\n", - "- To access the current run from within the `LightningModule`, you can access `Trainer.logger.experiment`, which is a `wandb.Run` object" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Oi1NpQs7OpD4" - }, - "source": [ - "### Some helper functions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "HLgYji2oOpD4" - }, - "outputs": [], - "source": [ - "# Some helper functions\n", - "\n", - "def set_parameter_requires_grad(model, feature_extracting):\n", - " if feature_extracting:\n", - " for param in model.parameters():\n", - " param.requires_grad = False\n", - "\n", - "def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):\n", - " # Initialize these variables which will be set in this if statement. Each of these\n", - " # variables is model specific.\n", - " model_ft = None\n", - " input_size = 0\n", - "\n", - " if model_name == \"resnet\":\n", - " \"\"\" Resnet18\n", - " \"\"\"\n", - " model_ft = models.resnet18(pretrained=use_pretrained)\n", - " set_parameter_requires_grad(model_ft, feature_extract)\n", - " num_ftrs = model_ft.fc.in_features\n", - " model_ft.fc = torch.nn.Linear(num_ftrs, num_classes)\n", - " input_size = 224\n", - "\n", - " elif model_name == \"squeezenet\":\n", - " \"\"\" Squeezenet\n", - " \"\"\"\n", - " model_ft = models.squeezenet1_0(pretrained=use_pretrained)\n", - " set_parameter_requires_grad(model_ft, feature_extract)\n", - " model_ft.classifier[1] = torch.nn.Conv2d(512, num_classes, kernel_size=(1, 1), stride=(1, 1))\n", - " model_ft.num_classes = num_classes\n", - " input_size = 224\n", - "\n", - " elif model_name == \"densenet\":\n", - " \"\"\" Densenet\n", - " \"\"\"\n", - " model_ft = models.densenet121(pretrained=use_pretrained)\n", - " set_parameter_requires_grad(model_ft, feature_extract)\n", - " num_ftrs = model_ft.classifier.in_features\n", - " model_ft.classifier = torch.nn.Linear(num_ftrs, num_classes)\n", - " input_size = 224\n", - "\n", - " else:\n", - " print(\"Invalid model name, exiting...\")\n", - " exit()\n", - "\n", - " return model_ft, input_size" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "LLRK0S6oOpD4" - }, - "source": [ - "### Writing the `LightningModule`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "k0tTtK5zOpD4" - }, - "outputs": [], - "source": [ - "import torch\n", - "from torch.nn import Linear, CrossEntropyLoss, functional as F\n", - "from torch.optim import Adam\n", - "from torchmetrics.functional import accuracy\n", - "from lightning.pytorch import LightningModule\n", - "from torchvision import models\n", - "\n", - "class NatureLitModule(LightningModule):\n", - " def __init__(self,\n", - " model_name,\n", - " num_classes=10,\n", - " feature_extract=True,\n", - " lr=0.01):\n", - " '''method used to define our model parameters'''\n", - " super().__init__()\n", - "\n", - " self.model_name = model_name\n", - " self.num_classes = num_classes\n", - " self.feature_extract = feature_extract\n", - " self.model, self.input_size = initialize_model(model_name=self.model_name,\n", - " num_classes=self.num_classes,\n", - " feature_extract=True)\n", - "\n", - " # loss\n", - " self.loss = CrossEntropyLoss()\n", - "\n", - " # optimizer parameters\n", - " self.lr = lr\n", - "\n", - " # save hyper-parameters to self.hparams (auto-logged by W&B)\n", - " self.save_hyperparameters()\n", - "\n", - " # Record the gradients of all the layers\n", - " wandb.watch(self.model)\n", - "\n", - " def forward(self, x):\n", - " '''method used for inference input -> output'''\n", - " x = self.model(x)\n", - "\n", - " return x\n", - "\n", - " def training_step(self, batch, batch_idx):\n", - " '''needs to return a loss from a single batch'''\n", - " preds, y, loss, acc = self._get_preds_loss_accuracy(batch)\n", - "\n", - " # Log loss and metric\n", - " self.log('train/loss', loss)\n", - " self.log('train/accuracy', acc)\n", - "\n", - " return loss\n", - "\n", - " def validation_step(self, batch, batch_idx):\n", - " '''used for logging metrics'''\n", - " preds, y, loss, acc = self._get_preds_loss_accuracy(batch)\n", - "\n", - " # Log loss and metric\n", - " self.log('validation/loss', loss)\n", - " self.log('validation/accuracy', acc)\n", - "\n", - " # Let's return preds to use it in a custom callback\n", - " return preds, y\n", - "\n", - " def test_step(self, batch, batch_idx):\n", - " '''used for logging metrics'''\n", - " preds, y, loss, acc = self._get_preds_loss_accuracy(batch)\n", - "\n", - " # Log loss and metric\n", - " self.log('test/loss', loss)\n", - " self.log('test/accuracy', acc)\n", - "\n", - " def configure_optimizers(self):\n", - " '''defines model optimizer'''\n", - " return Adam(self.parameters(), lr=self.lr)\n", - "\n", - "\n", - " def _get_preds_loss_accuracy(self, batch):\n", - " '''convenience function since train/valid/test steps are similar'''\n", - " x, y = batch\n", - " logits = self(x)\n", - " preds = torch.argmax(logits, dim=1)\n", - " loss = self.loss(logits, y)\n", - " acc = accuracy(preds, y, task=\"multiclass\", num_classes=10)\n", - " return preds, y, loss, acc" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bvZHZuZzOpD4" - }, - "source": [ - "### Instrument Callbacks to log additional things at certain points in your code" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "j9pHqaw1OpD5" - }, - "outputs": [], - "source": [ - "from lightning.pytorch.callbacks import Callback\n", - "\n", - "class LogPredictionsCallback(Callback):\n", - "\n", - " def __init__(self):\n", - " super().__init__()\n", - "\n", - "\n", - " def on_validation_epoch_start(self, trainer, pl_module):\n", - " self.batch_dfs = []\n", - " self.image_list = []\n", - " self.val_table = wandb.Table(columns=['image', 'ground_truth', 'prediction'])\n", - "\n", - "\n", - " def on_validation_batch_end(\n", - " self, trainer, pl_module, outputs, batch, batch_idx):\n", - " \"\"\"Called when the validation batch ends.\"\"\"\n", - "\n", - " # Append validation predictions and ground truth to log in confusion matrix\n", - " x, y = batch\n", - " preds, y = outputs\n", - " self.batch_dfs.append(pd.DataFrame({\"Ground Truth\": y.cpu().numpy(), \"Predictions\": preds.cpu().numpy()}))\n", - "\n", - " # Add wandb.Image to a table to log at the end of validation\n", - " x = x.cpu().numpy().transpose(0, 2, 3, 1)\n", - " for x_i, y_i, y_pred in list(zip(x, y, preds)):\n", - " self.image_list.append(wandb.Image(x_i, caption=f'Ground Truth: {y_i} - Prediction: {y_pred}'))\n", - " self.val_table.add_data(wandb.Image(x_i), y_i, y_pred)\n", - "\n", - "\n", - " def on_validation_epoch_end(self, trainer, pl_module):\n", - " # Collect statistics for whole validation set and log\n", - " class_names = trainer.datamodule.nature_dataset.class_names\n", - " val_df = pd.concat(self.batch_dfs)\n", - " wandb.log({\"validation_table\": self.val_table,\n", - " \"images_over_time\": self.image_list,\n", - " \"validation_conf_matrix\": wandb.plot.confusion_matrix(y_true = val_df[\"Ground Truth\"].tolist(),\n", - " preds=val_df[\"Predictions\"].tolist(),\n", - " class_names=class_names)}, step=trainer.global_step)\n", - "\n", - " del self.batch_dfs\n", - " del self.val_table\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jAiH-kb1OpD5" - }, - "source": [ - "## 🏋️‍ Main Training Loop" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "FrYTQ7uaOpD5" - }, - "outputs": [], - "source": [ - "from lightning.pytorch.callbacks import ModelCheckpoint\n", - "from lightning.pytorch.loggers import WandbLogger\n", - "from lightning.pytorch import Trainer\n", - "\n", - "wandb.init(project=PROJECT_NAME,\n", - " entity=ENTITY,\n", - " job_type='training',\n", - " config={\n", - " \"model_name\": \"squeezenet\",\n", - " \"batch_size\": 16\n", - " })\n", - "\n", - "wandb_logger = WandbLogger(log_model='all', checkpoint_name=f'nature-{wandb.run.id}')\n", - "\n", - "log_predictions_callback = LogPredictionsCallback()\n", - "checkpoint_callback = ModelCheckpoint(every_n_epochs=1)\n", - "\n", - "model = NatureLitModule(model_name=wandb.config['model_name']) # Access hyperparameters downstream to instantiate models/datasets\n", - "\n", - "nature_module = NatureDatasetModule(wandb_run = wandb_logger.experiment,\n", - " artifact_name_alias = \"Nature_100:latest\",\n", - " local_target_dir = \"Nature_100:latest\",\n", - " batch_size=wandb.config['batch_size'],\n", - " input_size=model.input_size)\n", - "nature_module.setup()\n", - "\n", - "trainer = Trainer(logger=wandb_logger, # W&B integration\n", - " callbacks=[log_predictions_callback, checkpoint_callback],\n", - " max_epochs=5,\n", - " log_every_n_steps=5)\n", - "trainer.fit(model, datamodule=nature_module)\n", - "\n", - "wandb.finish()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uEII8J7UOpD5" - }, - "source": [ - "### Syncing with W&B Offline\n", - "If for some reason, network communication is lost during the course of training, you can always sync progress with `wandb sync`\n", - "\n", - "The W&B sdk caches all logged data in a local directory `wandb` and when you call `wandb sync`, this syncs the your local state with the web app." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "xi-EhHsxOpD5" - }, - "source": [ - "## Retrieve a model checkpoint artifact and resume training\n", - "- Artifacts make it easy to track state of your training remotely and then resume training from a checkpoint" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jiE1Bk7fOpD5" - }, - "outputs": [], - "source": [ - "#@title Enter which checkpoint you want to resume training from:\n", - "\n", - "# FORM VARIABLES\n", - "ARTIFACT_NAME_ALIAS = \"nature-oyxk79m1:v4\" #@param {type:\"string\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jZZXWRatOpD5" - }, - "outputs": [], - "source": [ - "wandb.init(project=PROJECT_NAME,\n", - " entity=ENTITY,\n", - " job_type='resume_training')\n", - "\n", - "# Retrieve model checkpoint artifact and restore previous hyperparameters\n", - "model_chkpt_art = wandb.use_artifact(f'{ENTITY}/{PROJECT_NAME}/{ARTIFACT_NAME_ALIAS}')\n", - "model_chkpt_art.download() # Can change download directory by adding `root`, defaults to \"./artifacts\"\n", - "logging_run = model_chkpt_art.logged_by()\n", - "wandb.config = logging_run.config\n", - "\n", - "# Can create a new artifact name or continue logging to the old one\n", - "artifact_name = ARTIFACT_NAME_ALIAS.split(\":\")[0]\n", - "wandb_logger = WandbLogger(log_model='all', checkpoint_name=artifact_name)\n", - "\n", - "log_predictions_callback = LogPredictionsCallback()\n", - "checkpoint_callback = ModelCheckpoint(every_n_epochs=1)\n", - "\n", - "model = NatureLitModule.load_from_checkpoint(f'./artifacts/{ARTIFACT_NAME_ALIAS}/model.ckpt')\n", - "\n", - "nature_module = NatureDatasetModule(wandb_run = wandb_logger.experiment,\n", - " artifact_name_alias = \"Nature_100:latest\",\n", - " local_target_dir = \"Nature_100:latest\",\n", - " batch_size=wandb.config['batch_size'],\n", - " input_size=model.input_size)\n", - "nature_module.setup()\n", - "\n", - "\n", - "\n", - "trainer = Trainer(logger=wandb_logger, # W&B integration\n", - " callbacks=[log_predictions_callback, checkpoint_callback],\n", - " max_epochs=10,\n", - " log_every_n_steps=5)\n", - "trainer.fit(model, datamodule=nature_module)\n", - "\n", - "wandb.finish()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KTforR6EOpD5" - }, - "source": [ - "## Model Registry\n", - "After logging a bunch of checkpoints across multiple runs during experimentation, now comes time to hand-off the best checkpoint to the next stage of the workflow (e.g. testing, deployment).\n", - "\n", - "The model registry offers a centralized place to house the best checkpoints for all your model tasks. Any `model` artifact you log can be \"linked\" to a Registered Model. Here are the steps to start using the model registry for more organized model management:\n", - "1. Access your team's model registry by going the team page and selecting `Model Registry`\n", - "![model registry](https://drive.google.com/uc?export=view&id=1ZtJwBsFWPTm4Sg5w8vHhRpvDSeQPwsKw)\n", - "\n", - "2. Create a new Registered Model.\n", - "![model registry](https://drive.google.com/uc?export=view&id=1RuayTZHNE0LJCxt1t0l6-2zjwiV4aDXe)\n", - "\n", - "3. Go to the artifacts tab of the project that holds all your model checkpoints\n", - "![model registry](https://drive.google.com/uc?export=view&id=1r_jlhhtcU3as8VwQ-4oAntd8YtTwElFB)\n", - "\n", - "4. Click \"Link to Registry\" for the model artifact version you want. (Alternatively you can [link a model via api](https://docs.wandb.ai/guides/models) with `wandb.run.link_artifact`)\n", - "\n", - "**A note on linking:** The process of linking a model checkpoint is akin to \"bookmarking\" it. Each time you link a new model artifact to a Registered Model, this increments the version of the Registered Model. This helps delineate the model development side of the workflow from the model deployment/consumption side. The globally understood version/alias of a model should be unpolluted from all the experimental versions being generated in R&D and thus the versioning of a Registered Model increments according to new \"bookmarked\" models as opposed to model checkpoint logging.\n", - "\n", - "\n", - "### Create a Centralized Hub for all your models\n", - "- Add a model card, tags, slack notifactions to your Registered Model\n", - "- Change aliases to reflect when models move through different phases\n", - "- Embed the model registry in reports for model documentation and regression reports. See this report as an [example](https://api.wandb.ai/links/wandb-smle/r82bj9at)\n", - "![model registry](https://drive.google.com/uc?export=view&id=1lKPgaw-Ak4WK_91aBMcLvUMJL6pDQpgO)\n" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Weights\n", + "\n", + "\n", + "\n", + "# W&B Tutorial with Pytorch Lightning" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🛠️ Install `wandb` and `pytorch-lightning`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -q lightning wandb torchvision" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Login to W&B either through Python or CLI\n", + "If you are using the public W&B cloud, you don't need to specify the `WANDB_HOST`.\n", + "\n", + "You can set environment variables `WANDB_API_KEY` and `WANDB_HOST` and pass them in as:\n", + "```\n", + "import os\n", + "import wandb\n", + "\n", + "wandb.login(host=os.getenv(\"WANDB_HOST\"), key=os.getenv(\"WANDB_API_KEY\"))\n", + "```\n", + "You can also login via the CLI with:\n", + "```\n", + "wandb login --host \n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import wandb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ⚱ Logging the Raw Training Data as an Artifact" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#@title Enter your W&B project and entity\n", + "\n", + "# FORM VARIABLES\n", + "PROJECT_NAME = \"pytorch-lightning-e2e\" #@param {type:\"string\"}\n", + "ENTITY = \"wandb\"#@param {type:\"string\"}\n", + "\n", + "# set SIZE to \"TINY\", \"SMALL\", \"MEDIUM\", or \"LARGE\"\n", + "# to select one of these three datasets\n", + "# TINY dataset: 100 images, 30MB\n", + "# SMALL dataset: 1000 images, 312MB\n", + "# MEDIUM dataset: 5000 images, 1.5GB\n", + "# LARGE dataset: 12,000 images, 3.6GB\n", + "\n", + "SIZE = \"TINY\"\n", + "\n", + "if SIZE == \"TINY\":\n", + " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_100.zip\"\n", + " src_zip = \"nature_100.zip\"\n", + " DATA_SRC = \"nature_100\"\n", + " IMAGES_PER_LABEL = 10\n", + " BALANCED_SPLITS = {\"train\" : 8, \"val\" : 1, \"test\": 1}\n", + "elif SIZE == \"SMALL\":\n", + " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_1K.zip\"\n", + " src_zip = \"nature_1K.zip\"\n", + " DATA_SRC = \"nature_1K\"\n", + " IMAGES_PER_LABEL = 100\n", + " BALANCED_SPLITS = {\"train\" : 80, \"val\" : 10, \"test\": 10}\n", + "elif SIZE == \"MEDIUM\":\n", + " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_12K.zip\"\n", + " src_zip = \"nature_12K.zip\"\n", + " DATA_SRC = \"inaturalist_12K/train\" # (technically a subset of only 10K images)\n", + " IMAGES_PER_LABEL = 500\n", + " BALANCED_SPLITS = {\"train\" : 400, \"val\" : 50, \"test\": 50}\n", + "elif SIZE == \"LARGE\":\n", + " src_url = \"https://storage.googleapis.com/wandb_datasets/nature_12K.zip\"\n", + " src_zip = \"nature_12K.zip\"\n", + " DATA_SRC = \"inaturalist_12K/train\" # (technically a subset of only 10K images)\n", + " IMAGES_PER_LABEL = 1000\n", + " BALANCED_SPLITS = {\"train\" : 800, \"val\" : 100, \"test\": 100}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "!curl -SL $src_url > $src_zip\n", + "!unzip $src_zip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "import pandas as pd\n", + "import os\n", + "\n", + "with wandb.init(project=PROJECT_NAME, entity=ENTITY, job_type='log_datasets') as run:\n", + " img_paths = []\n", + " for root, dirs, files in os.walk('nature_100', topdown=False):\n", + " for name in files:\n", + " img_path = os.path.join(root, name)\n", + " label = img_path.split('/')[1]\n", + " img_paths.append([img_path, label])\n", + "\n", + " index_df = pd.DataFrame(columns=['image_path', 'label'], data=img_paths)\n", + " index_df.to_csv('index.csv', index=False)\n", + "\n", + " train_art = wandb.Artifact(name='Nature_100', type='raw_images', description='nature image dataset with 10 classes, 10 images per class')\n", + " train_art.add_dir('nature_100')\n", + "\n", + " # Also adding a csv indicating the labels of each image\n", + " train_art.add_file('index.csv')\n", + " wandb.log_artifact(train_art)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Using Artifacts in Pytorch Lightning `DataModule`'s and Pytorch `Dataset`'s\n", + "- Makes it easy to interopt your DataLoaders with new versions of datasets\n", + "- Just indicate the `name:alias` as an argument to your `Dataset` or `DataModule`\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision import transforms\n", + "import lightning.pytorch as pl\n", + "import torch\n", + "from torch.utils.data import Dataset, DataLoader, random_split\n", + "from skimage import io, transform\n", + "from torchvision import transforms, utils, models\n", + "import math\n", + "\n", + "class NatureDataset(Dataset):\n", + " def __init__(self,\n", + " wandb_run,\n", + " artifact_name_alias=\"Nature_100:latest\",\n", + " local_target_dir=\"Nature_100:latest\",\n", + " transform=None):\n", + " self.local_target_dir = local_target_dir\n", + " self.transform = transform\n", + "\n", + " # Pull down the artifact locally to load it into memory\n", + " art = wandb_run.use_artifact(artifact_name_alias)\n", + " path_at = art.download(root=self.local_target_dir)\n", + "\n", + " self.ref_df = pd.read_csv(os.path.join(self.local_target_dir, 'index.csv'))\n", + " self.class_names = self.ref_df.iloc[:, 1].unique().tolist()\n", + " self.idx_to_class = {k: v for k, v in enumerate(self.class_names)}\n", + " self.class_to_idx = {v: k for k, v in enumerate(self.class_names)}\n", + "\n", + " def __len__(self):\n", + " return len(self.ref_df)\n", + "\n", + " def __getitem__(self, idx):\n", + " if torch.is_tensor(idx):\n", + " idx = idx.tolist()\n", + "\n", + " img_path = self.ref_df.iloc[idx, 0]\n", + "\n", + " image = io.imread(img_path)\n", + " label = self.ref_df.iloc[idx, 1]\n", + " label = torch.tensor(self.class_to_idx[label], dtype=torch.long)\n", + "\n", + " if self.transform:\n", + " image = self.transform(image)\n", + "\n", + " return image, label\n", + "\n", + "\n", + "class NatureDatasetModule(pl.LightningDataModule):\n", + " def __init__(self,\n", + " wandb_run,\n", + " artifact_name_alias: str = \"Nature_100:latest\",\n", + " local_target_dir: str = \"Nature_100:latest\",\n", + " batch_size: int = 16,\n", + " input_size: int = 224,\n", + " seed: int = 42):\n", + " super().__init__()\n", + " self.wandb_run = wandb_run\n", + " self.artifact_name_alias = artifact_name_alias\n", + " self.local_target_dir = local_target_dir\n", + " self.batch_size = batch_size\n", + " self.input_size = input_size\n", + " self.seed = seed\n", + "\n", + " def setup(self, stage=None):\n", + " self.nature_dataset = NatureDataset(wandb_run=self.wandb_run,\n", + " artifact_name_alias=self.artifact_name_alias,\n", + " local_target_dir=self.local_target_dir,\n", + " transform=transforms.Compose([transforms.ToTensor(),\n", + " transforms.CenterCrop(self.input_size),\n", + " transforms.Normalize((0.485, 0.456, 0.406),\n", + " (0.229, 0.224, 0.225))]))\n", + "\n", + " nature_length = len(self.nature_dataset)\n", + " train_size = math.floor(0.8 * nature_length)\n", + " val_size = math.floor(0.2 * nature_length)\n", + " self.nature_train, self.nature_val = random_split(self.nature_dataset,\n", + " [train_size, val_size],\n", + " generator=torch.Generator().manual_seed(self.seed))\n", + " return self\n", + "\n", + " def train_dataloader(self):\n", + " return DataLoader(self.nature_train, batch_size=self.batch_size)\n", + "\n", + " def val_dataloader(self):\n", + " return DataLoader(self.nature_val, batch_size=self.batch_size)\n", + "\n", + " def predict_dataloader(self):\n", + " pass\n", + "\n", + " def teardown(self, stage: str):\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "##How Logging in your Pytorch `LightningModule`works:\n", + "When you train the model using `Trainer`, ensure you have a `WandbLogger` instantiated and passed in as a `logger`.\n", + "\n", + "```\n", + "wandb_logger = WandbLogger(project=\"my_project\", entity=\"machine-learning\")\n", + "trainer = Trainer(logger=wandb_logger)\n", + "```\n", + "\n", + "\n", + "You can always use `wandb.log` as normal throughout the module. When the `WandbLogger` is used, `self.log` will also log metrics to W&B.\n", + "- To access the current run from within the `LightningModule`, you can access `Trainer.logger.experiment`, which is a `wandb.Run` object" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Some helper functions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Some helper functions\n", + "\n", + "def set_parameter_requires_grad(model, feature_extracting):\n", + " if feature_extracting:\n", + " for param in model.parameters():\n", + " param.requires_grad = False\n", + "\n", + "def initialize_model(model_name, num_classes, feature_extract, use_pretrained=True):\n", + " # Initialize these variables which will be set in this if statement. Each of these\n", + " # variables is model specific.\n", + " model_ft = None\n", + " input_size = 0\n", + "\n", + " if model_name == \"resnet\":\n", + " \"\"\" Resnet18\n", + " \"\"\"\n", + " model_ft = models.resnet18(pretrained=use_pretrained)\n", + " set_parameter_requires_grad(model_ft, feature_extract)\n", + " num_ftrs = model_ft.fc.in_features\n", + " model_ft.fc = torch.nn.Linear(num_ftrs, num_classes)\n", + " input_size = 224\n", + "\n", + " elif model_name == \"squeezenet\":\n", + " \"\"\" Squeezenet\n", + " \"\"\"\n", + " model_ft = models.squeezenet1_0(pretrained=use_pretrained)\n", + " set_parameter_requires_grad(model_ft, feature_extract)\n", + " model_ft.classifier[1] = torch.nn.Conv2d(512, num_classes, kernel_size=(1, 1), stride=(1, 1))\n", + " model_ft.num_classes = num_classes\n", + " input_size = 224\n", + "\n", + " elif model_name == \"densenet\":\n", + " \"\"\" Densenet\n", + " \"\"\"\n", + " model_ft = models.densenet121(pretrained=use_pretrained)\n", + " set_parameter_requires_grad(model_ft, feature_extract)\n", + " num_ftrs = model_ft.classifier.in_features\n", + " model_ft.classifier = torch.nn.Linear(num_ftrs, num_classes)\n", + " input_size = 224\n", + "\n", + " else:\n", + " print(\"Invalid model name, exiting...\")\n", + " exit()\n", + "\n", + " return model_ft, input_size" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Writing the `LightningModule`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torch.nn import Linear, CrossEntropyLoss, functional as F\n", + "from torch.optim import Adam\n", + "from torchmetrics.functional import accuracy\n", + "from lightning.pytorch import LightningModule\n", + "from torchvision import models\n", + "\n", + "class NatureLitModule(LightningModule):\n", + " def __init__(self,\n", + " model_name,\n", + " num_classes=10,\n", + " feature_extract=True,\n", + " lr=0.01):\n", + " '''method used to define our model parameters'''\n", + " super().__init__()\n", + "\n", + " self.model_name = model_name\n", + " self.num_classes = num_classes\n", + " self.feature_extract = feature_extract\n", + " self.model, self.input_size = initialize_model(model_name=self.model_name,\n", + " num_classes=self.num_classes,\n", + " feature_extract=True)\n", + "\n", + " # loss\n", + " self.loss = CrossEntropyLoss()\n", + "\n", + " # optimizer parameters\n", + " self.lr = lr\n", + "\n", + " # save hyper-parameters to self.hparams (auto-logged by W&B)\n", + " self.save_hyperparameters()\n", + "\n", + " # Record the gradients of all the layers\n", + " wandb.watch(self.model)\n", + "\n", + " def forward(self, x):\n", + " '''method used for inference input -> output'''\n", + " x = self.model(x)\n", + "\n", + " return x\n", + "\n", + " def training_step(self, batch, batch_idx):\n", + " '''needs to return a loss from a single batch'''\n", + " preds, y, loss, acc = self._get_preds_loss_accuracy(batch)\n", + "\n", + " # Log loss and metric\n", + " self.log('train/loss', loss)\n", + " self.log('train/accuracy', acc)\n", + "\n", + " return loss\n", + "\n", + " def validation_step(self, batch, batch_idx):\n", + " '''used for logging metrics'''\n", + " preds, y, loss, acc = self._get_preds_loss_accuracy(batch)\n", + "\n", + " # Log loss and metric\n", + " self.log('validation/loss', loss)\n", + " self.log('validation/accuracy', acc)\n", + "\n", + " # Let's return preds to use it in a custom callback\n", + " return preds, y\n", + "\n", + " def test_step(self, batch, batch_idx):\n", + " '''used for logging metrics'''\n", + " preds, y, loss, acc = self._get_preds_loss_accuracy(batch)\n", + "\n", + " # Log loss and metric\n", + " self.log('test/loss', loss)\n", + " self.log('test/accuracy', acc)\n", + "\n", + " def configure_optimizers(self):\n", + " '''defines model optimizer'''\n", + " return Adam(self.parameters(), lr=self.lr)\n", + "\n", + "\n", + " def _get_preds_loss_accuracy(self, batch):\n", + " '''convenience function since train/valid/test steps are similar'''\n", + " x, y = batch\n", + " logits = self(x)\n", + " preds = torch.argmax(logits, dim=1)\n", + " loss = self.loss(logits, y)\n", + " acc = accuracy(preds, y, task=\"multiclass\", num_classes=10)\n", + " return preds, y, loss, acc" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Instrument Callbacks to log additional things at certain points in your code" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from lightning.pytorch.callbacks import Callback\n", + "\n", + "class LogPredictionsCallback(Callback):\n", + "\n", + " def __init__(self):\n", + " super().__init__()\n", + "\n", + "\n", + " def on_validation_epoch_start(self, trainer, pl_module):\n", + " self.batch_dfs = []\n", + " self.image_list = []\n", + " self.val_table = wandb.Table(columns=['image', 'ground_truth', 'prediction'])\n", + "\n", + "\n", + " def on_validation_batch_end(\n", + " self, trainer, pl_module, outputs, batch, batch_idx):\n", + " \"\"\"Called when the validation batch ends.\"\"\"\n", + "\n", + " # Append validation predictions and ground truth to log in confusion matrix\n", + " x, y = batch\n", + " preds, y = outputs\n", + " self.batch_dfs.append(pd.DataFrame({\"Ground Truth\": y.cpu().numpy(), \"Predictions\": preds.cpu().numpy()}))\n", + "\n", + " # Add wandb.Image to a table to log at the end of validation\n", + " x = x.cpu().numpy().transpose(0, 2, 3, 1)\n", + " for x_i, y_i, y_pred in list(zip(x, y, preds)):\n", + " self.image_list.append(wandb.Image(x_i, caption=f'Ground Truth: {y_i} - Prediction: {y_pred}'))\n", + " self.val_table.add_data(wandb.Image(x_i), y_i, y_pred)\n", + "\n", + "\n", + " def on_validation_epoch_end(self, trainer, pl_module):\n", + " # Collect statistics for whole validation set and log\n", + " class_names = trainer.datamodule.nature_dataset.class_names\n", + " val_df = pd.concat(self.batch_dfs)\n", + " wandb.log({\"validation_table\": self.val_table,\n", + " \"images_over_time\": self.image_list,\n", + " \"validation_conf_matrix\": wandb.plot.confusion_matrix(y_true = val_df[\"Ground Truth\"].tolist(),\n", + " preds=val_df[\"Predictions\"].tolist(),\n", + " class_names=class_names)}, step=trainer.global_step)\n", + "\n", + " del self.batch_dfs\n", + " del self.val_table\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🏋️‍ Main Training Loop" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from lightning.pytorch.callbacks import ModelCheckpoint\n", + "from lightning.pytorch.loggers import WandbLogger\n", + "from lightning.pytorch import Trainer\n", + "\n", + "wandb.init(project=PROJECT_NAME,\n", + " entity=ENTITY,\n", + " job_type='training',\n", + " config={\n", + " \"model_name\": \"squeezenet\",\n", + " \"batch_size\": 16\n", + " })\n", + "\n", + "wandb_logger = WandbLogger(log_model='all', checkpoint_name=f'nature-{wandb.run.id}')\n", + "\n", + "log_predictions_callback = LogPredictionsCallback()\n", + "checkpoint_callback = ModelCheckpoint(every_n_epochs=1)\n", + "\n", + "model = NatureLitModule(model_name=wandb.config['model_name']) # Access hyperparameters downstream to instantiate models/datasets\n", + "\n", + "nature_module = NatureDatasetModule(wandb_run = wandb_logger.experiment,\n", + " artifact_name_alias = \"Nature_100:latest\",\n", + " local_target_dir = \"Nature_100:latest\",\n", + " batch_size=wandb.config['batch_size'],\n", + " input_size=model.input_size)\n", + "nature_module.setup()\n", + "\n", + "trainer = Trainer(logger=wandb_logger, # W&B integration\n", + " callbacks=[log_predictions_callback, checkpoint_callback],\n", + " max_epochs=5,\n", + " log_every_n_steps=5)\n", + "trainer.fit(model, datamodule=nature_module)\n", + "\n", + "wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Syncing with W&B Offline\n", + "If for some reason, network communication is lost during the course of training, you can always sync progress with `wandb sync`\n", + "\n", + "The W&B sdk caches all logged data in a local directory `wandb` and when you call `wandb sync`, this syncs the your local state with the web app." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Retrieve a model checkpoint artifact and resume training\n", + "- Artifacts make it easy to track state of your training remotely and then resume training from a checkpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#@title Enter which checkpoint you want to resume training from:\n", + "\n", + "# FORM VARIABLES\n", + "ARTIFACT_NAME_ALIAS = \"nature-oyxk79m1:v4\" #@param {type:\"string\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.init(project=PROJECT_NAME,\n", + " entity=ENTITY,\n", + " job_type='resume_training')\n", + "\n", + "# Retrieve model checkpoint artifact and restore previous hyperparameters\n", + "model_chkpt_art = wandb.use_artifact(f'{ENTITY}/{PROJECT_NAME}/{ARTIFACT_NAME_ALIAS}')\n", + "model_chkpt_art.download() # Can change download directory by adding `root`, defaults to \"./artifacts\"\n", + "logging_run = model_chkpt_art.logged_by()\n", + "wandb.config = logging_run.config\n", + "\n", + "# Can create a new artifact name or continue logging to the old one\n", + "artifact_name = ARTIFACT_NAME_ALIAS.split(\":\")[0]\n", + "wandb_logger = WandbLogger(log_model='all', checkpoint_name=artifact_name)\n", + "\n", + "log_predictions_callback = LogPredictionsCallback()\n", + "checkpoint_callback = ModelCheckpoint(every_n_epochs=1)\n", + "\n", + "model = NatureLitModule.load_from_checkpoint(f'./artifacts/{ARTIFACT_NAME_ALIAS}/model.ckpt')\n", + "\n", + "nature_module = NatureDatasetModule(wandb_run = wandb_logger.experiment,\n", + " artifact_name_alias = \"Nature_100:latest\",\n", + " local_target_dir = \"Nature_100:latest\",\n", + " batch_size=wandb.config['batch_size'],\n", + " input_size=model.input_size)\n", + "nature_module.setup()\n", + "\n", + "\n", + "\n", + "trainer = Trainer(logger=wandb_logger, # W&B integration\n", + " callbacks=[log_predictions_callback, checkpoint_callback],\n", + " max_epochs=10,\n", + " log_every_n_steps=5)\n", + "trainer.fit(model, datamodule=nature_module)\n", + "\n", + "wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model Registry\n", + "After logging a bunch of checkpoints across multiple runs during experimentation, now comes time to hand-off the best checkpoint to the next stage of the workflow (e.g. testing, deployment).\n", + "\n", + "The model registry offers a centralized place to house the best checkpoints for all your model tasks. Any `model` artifact you log can be \"linked\" to a Registered Model. Here are the steps to start using the model registry for more organized model management:\n", + "1. Access your team's model registry by going the team page and selecting `Model Registry`\n", + "![model registry](https://drive.google.com/uc?export=view&id=1ZtJwBsFWPTm4Sg5w8vHhRpvDSeQPwsKw)\n", + "\n", + "2. Create a new Registered Model.\n", + "![model registry](https://drive.google.com/uc?export=view&id=1RuayTZHNE0LJCxt1t0l6-2zjwiV4aDXe)\n", + "\n", + "3. Go to the artifacts tab of the project that holds all your model checkpoints\n", + "![model registry](https://drive.google.com/uc?export=view&id=1r_jlhhtcU3as8VwQ-4oAntd8YtTwElFB)\n", + "\n", + "4. Click \"Link to Registry\" for the model artifact version you want. (Alternatively you can [link a model via api](https://docs.wandb.ai/guides/models) with `wandb.run.link_artifact`)\n", + "\n", + "**A note on linking:** The process of linking a model checkpoint is akin to \"bookmarking\" it. Each time you link a new model artifact to a Registered Model, this increments the version of the Registered Model. This helps delineate the model development side of the workflow from the model deployment/consumption side. The globally understood version/alias of a model should be unpolluted from all the experimental versions being generated in R&D and thus the versioning of a Registered Model increments according to new \"bookmarked\" models as opposed to model checkpoint logging.\n", + "\n", + "\n", + "### Create a Centralized Hub for all your models\n", + "- Add a model card, tags, slack notifactions to your Registered Model\n", + "- Change aliases to reflect when models move through different phases\n", + "- Embed the model registry in reports for model documentation and regression reports. See this report as an [example](https://api.wandb.ai/links/wandb-smle/r82bj9at)\n", + "![model registry](https://drive.google.com/uc?export=view&id=1lKPgaw-Ak4WK_91aBMcLvUMJL6pDQpgO)\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W&B.ipynb b/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W&B.ipynb index 1ef18a83..e14e81e5 100644 --- a/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W&B.ipynb +++ b/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W&B.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/pytorch/Simple_PyTorch_Integration.ipynb b/colabs/pytorch/Simple_PyTorch_Integration.ipynb index 9369972e..bf1a6054 100644 --- a/colabs/pytorch/Simple_PyTorch_Integration.ipynb +++ b/colabs/pytorch/Simple_PyTorch_Integration.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/raytune/RayTune_with_wandb.ipynb b/colabs/raytune/RayTune_with_wandb.ipynb index 00dc6caf..78f411e3 100644 --- a/colabs/raytune/RayTune_with_wandb.ipynb +++ b/colabs/raytune/RayTune_with_wandb.ipynb @@ -1,409 +1,350 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "CHj4COZvzWta" - }, - "source": [ - "\"Open\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "f5SyUYlHzWtd" - }, - "source": [ - "\"Weights\n", - "\n", - "\n", - "\n", - "Use Weights & Biases for machine learning experiment tracking, dataset versioning, and project collaboration.\n", - "\n", - "
\n", - "\n", - "\"Weights\n", - "\n", - "
" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "oQYw5uVFzWtd" - }, - "source": [ - "\n", - "# 🌞 Ray/Tune and 🏋️‍♀️ Weights & Biases\n", - "\n", - "Both Weights and Biases and Ray/Tune are built for scale and handle millions of models every month for teams doing some of the most cutting-edge deep learning research.\n", - "\n", - "[W&B](https://wandb.com) is a toolkit with everything you need to track, reproduce, and gain insights from your models easily; [Ray/Tune](https://docs.ray.io/en/latest/tune/) provides a simple interface for scaling and running distributed experiments.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "SFI8z2gWzWte" - }, - "source": [ - "\n", - "### 🤝 They're a natural match! 🤝\n", - "\n", - "Here's just a few reasons why our community likes Ray/Tune –\n", - "\n", - "* **Simple distributed execution**: Ray/Tune makes it easy to scale all the way from a single node on a laptop, through to multiple GPUs, and up to multiple nodes on multiple machines.\n", - "* **State-of-the-art algorithms**: Ray/Tune has tested implementations of a huge number of potent scheduling algorithms including\n", - "[Population-Based Training](https://docs.ray.io/en/latest/tune/tutorials/tune-advanced-tutorial.html),\n", - "[ASHA](https://docs.ray.io/en/master/tune/tutorials/tune-tutorial.html#early-stopping-with-asha),\n", - "and\n", - "[HyperBand](https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#hyperband-tune-schedulers-hyperbandscheduler)\n", - "* **Method agnostic**: Ray/Tune works across deep learning frameworks (including PyTorch, Keras, Tensorflow, and PyTorchLightning) and with other ML methods like gradient-boosted trees (XGBoost, LightGBM)\n", - "* **Fault-tolerance**: Ray/Tune is built on top of Ray, providing tolerance for failed runs out of the box.\n", - "\n", - "This Colab demonstrates how this integration works for a simple grid search over two hyperparameters. If you've got any questions about the details,\n", - "check out\n", - "[our documentation](https://docs.wandb.com/library/integrations/ray-tune)\n", - "or the\n", - "[documentation for Ray/Tune](https://docs.ray.io/en/master/tune/api_docs/integration.html#weights-and-biases-tune-integration-wandb).\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Hw1mGxhlzWte" - }, - "source": [ - "W&B integrates with `ray.tune` by offering two lightweight standalone integrations:\n", - "\n", - "1. For simple cases, `WandbLoggerCallback` automatically logs metrics reported to Tune to W&B, along with the configuration of the experiment, using Tune's [`logger` interface](https://docs.ray.io/en/latest/tune/api_docs/logging.html).\n", - "2. The `@wandb_mixin` decorator gives you greater control over logging by letting you call `wandb.log` inside the decorated function, allowing you to [log custom metrics, plots, and other outputs, like media](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-log/Log_(Almost)_Anything_with_W%26B_Media.ipynb).\n", - "\n", - "These methods can be used together or independently.\n", - "\n", - "The example below demonstrates how they can be used together." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "qRrP4G23zWte" - }, - "source": [ - "# 🧹 Running a hyperparameter sweep with W&B and Ray/Tune" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QfDVmu9szWte" - }, - "source": [ - "## 📥 Install, `import`, and set seeds" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "XRjOn9QRzWte" - }, - "source": [ - "\n", - "Let's start by installing the libraries and importing everything we need.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LCVSvebNzWtf" - }, - "outputs": [], - "source": [ - "!pip install -Uq ray[tune] wandb" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qBCk5SvQ1q_n" - }, - "outputs": [], - "source": [ - "import os\n", - "os.environ[\"WANDB_API_KEY\"] = \"654d20637b9232f03417d6ae6e620f3b7cb70125\" #@param {type:\"string\"}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "dfHgJfVszWtf" - }, - "outputs": [], - "source": [ - "import random\n", - "import numpy as np\n", - "from filelock import FileLock\n", - "import tempfile\n", - "from ray import train, tune\n", - "from ray.air.integrations.wandb import WandbLoggerCallback, setup_wandb\n", - "from ray.tune.schedulers import AsyncHyperBandScheduler\n", - "from ray.train import Checkpoint\n", - "import torch\n", - "import torch.optim as optim\n", - "import wandb" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Y1ysNX6pzWtf", - "outputId": "f6a36dad-1d03-4f13-d552-e00896d919db" - }, - "outputs": [], - "source": [ - "wandb.login()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mYPSEuIgzWtg" - }, - "source": [ - "We'll make use of Ray's handy [`mnist_pytorch` example code](https://github.com/ray-project/ray/blob/master/python/ray/tune/examples/mnist_pytorch.py)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "8b-t9a7pzWtg" - }, - "outputs": [], - "source": [ - "from ray.tune.examples.mnist_pytorch import ConvNet, get_data_loaders, test_func, train_func" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BJ-zAJWpzWtg" - }, - "source": [ - "In order to make this experiment reproducible, we'll set the seeds for random number generators of various libraries used in this experiment." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GbXqaFYizWtg" - }, - "outputs": [], - "source": [ - "torch.backends.cudnn.deterministic = True\n", - "random.seed(2022)\n", - "np.random.seed(2022)\n", - "torch.manual_seed(2022)\n", - "torch.cuda.manual_seed_all(2022)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nAmSUMP4zWtg" - }, - "source": [ - "## 🤝 Integrating W&B with Ray/Tune" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OVmU1CRgzWtg" - }, - "source": [ - "Now, we define our training process, decorated with `@wandb_mixin` so we can call `wandb.log` to log our custom metric\n", - "(here, just the error rate; you might also [log media](https://docs.wandb.com/library/log#media), e.g. images from the validation set, captioned by the model predictions).\n", - "\n", - "When we execute our hyperparameter sweep below,\n", - "this function will be called with a `config`uration dictionary\n", - "that contains values for any hyperparameters.\n", - "For simplicity, we only have two hyperparameters here:\n", - "the learning rate and momentum value for accelerated SGD." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "nf2FKgaM21Em" - }, - "outputs": [], - "source": [ - "def train_mnist(config):\n", - " # Setup wandb\n", - " wandb = setup_wandb(config)\n", - " should_checkpoint = config.get(\"should_checkpoint\", False)\n", - " use_cuda = torch.cuda.is_available()\n", - " device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n", - " train_loader, test_loader = get_data_loaders()\n", - " model = ConvNet().to(device)\n", - "\n", - " optimizer = optim.SGD(\n", - " model.parameters(), lr=config[\"lr\"], momentum=config[\"momentum\"]\n", - " )\n", - " while True:\n", - " train_func(model, optimizer, train_loader, device)\n", - " acc = test_func(model, test_loader, device)\n", - " metrics = {\"mean_accuracy\": acc}\n", - "\n", - " # Report metrics (and possibly a checkpoint)\n", - " if should_checkpoint:\n", - " with tempfile.TemporaryDirectory() as tempdir:\n", - " torch.save(model.state_dict(), os.path.join(tempdir, \"model.pt\"))\n", - " train.report(metrics, checkpoint=Checkpoint.from_directory(tempdir))\n", - " else:\n", - " train.report(metrics)\n", - " # enables logging custom metrics using wandb.log()\n", - " error_rate = 100 * (1 - acc)\n", - " wandb.log({\"error_rate\": error_rate})" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "FjSalAQkzWth" - }, - "source": [ - "## 🚀 Launching a Sweep with W&B and Ray/Tune" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "lBS6iyXAzWth" - }, - "source": [ - "We're now almost ready to call `tune.run` to launch our hyperparameter sweep!\n", - "We just need to do three things:\n", - "1. set up a `wandb.Run`,\n", - "2. give the `WandbLoggerCallback` to `tune.run` so we can capture the output of `tune.report`, and\n", - "3. set up our hyperparameter sweep.\n", - "\n", - "A `wandb.Run` is normally created by calling `wandb.init`.\n", - "`tune` will handle that for you, you just need to pass\n", - "the arguments as a dictionary\n", - "(see [our documentation](https://docs.wandb.com/library/init) for details on `wandb.init`).\n", - "At the bare minimum, you need to pass in a `project` name --\n", - "sort of like a `git` repo name, but for your ML projects.\n", - "\n", - "In addition to holding arguments for `wandb.init`,\n", - "that dictionary also has a few special keys, described in\n", - "[the documentation for the `WandbLoggerCallback`](https://docs.ray.io/en/master/tune/tutorials/tune-wandb.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "E6HRghT9zWth" - }, - "source": [ - "We handle steps 2 and 3 when we invoke `tune.run`.\n", - "\n", - "Step 2 is handled by passing in the `WandbLoggerCallback` class in a list\n", - "to the `loggers` argument of `tune.run`.\n", - "\n", - "The setup of the hyperparameter sweep is handled by the\n", - "`config` argument of `tune.run`.\n", - "For the purposes of the integration,\n", - "the most important part is that this is where we pass in the `wandb_init`\n", - "dictionary.\n", - "\n", - "This is also where we configure the \"meat\" of the hyperparameter sweep:\n", - "what are the hyperparameters we're sweeping over,\n", - "and how do we choose their values.\n", - "\n", - "Here, we do a simple grid search, but\n", - "[Ray/Tune provides lots of sophisticated options](https://docs.ray.io/en/latest/tune/api_docs/suggestion.html)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "tw4eawRD0u8y", - "outputId": "e4ddf59b-5445-4d55-8f69-84a9fcb35024" - }, - "outputs": [], - "source": [ - "# for early stopping\n", - "sched = AsyncHyperBandScheduler()\n", - "\n", - "resources_per_trial = {\"gpu\": 1} # set this for GPUs\n", - "tuner = tune.Tuner(\n", - " tune.with_resources(train_mnist, resources=resources_per_trial),\n", - " tune_config=tune.TuneConfig(\n", - " metric=\"mean_accuracy\",\n", - " mode=\"max\",\n", - " scheduler=sched,\n", - " num_samples=50,\n", - " ),\n", - " run_config=train.RunConfig(\n", - " name=\"exp\",\n", - " stop={\n", - " \"mean_accuracy\": 0.98,\n", - " \"training_iteration\": 5,\n", - " },\n", - " callbacks=[WandbLoggerCallback(project=\"raytune-colab\")]\n", - " ),\n", - " param_space={\n", - " \"lr\": tune.loguniform(1e-4, 1e-2),\n", - " \"momentum\": tune.uniform(0.1, 0.9),\n", - " },\n", - ")\n", - "results = tuner.fit()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "1f8oI7tYzWth", - "outputId": "9fece67a-47e7-443a-ab8f-971f3d8dfd94" - }, - "outputs": [], - "source": [ - "print(\"Best config is:\", results.get_best_result().config)" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n", + "" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Weights\n", + "\n", + "\n", + "\n", + "Use Weights & Biases for machine learning experiment tracking, dataset versioning, and project collaboration.\n", + "\n", + "
\n", + "\n", + "\"Weights\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "# 🌞 Ray/Tune and 🏋️‍♀️ Weights & Biases\n", + "\n", + "Both Weights and Biases and Ray/Tune are built for scale and handle millions of models every month for teams doing some of the most cutting-edge deep learning research.\n", + "\n", + "[W&B](https://wandb.com) is a toolkit with everything you need to track, reproduce, and gain insights from your models easily; [Ray/Tune](https://docs.ray.io/en/latest/tune/) provides a simple interface for scaling and running distributed experiments.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "### 🤝 They're a natural match! 🤝\n", + "\n", + "Here's just a few reasons why our community likes Ray/Tune –\n", + "\n", + "* **Simple distributed execution**: Ray/Tune makes it easy to scale all the way from a single node on a laptop, through to multiple GPUs, and up to multiple nodes on multiple machines.\n", + "* **State-of-the-art algorithms**: Ray/Tune has tested implementations of a huge number of potent scheduling algorithms including\n", + "[Population-Based Training](https://docs.ray.io/en/latest/tune/tutorials/tune-advanced-tutorial.html),\n", + "[ASHA](https://docs.ray.io/en/master/tune/tutorials/tune-tutorial.html#early-stopping-with-asha),\n", + "and\n", + "[HyperBand](https://docs.ray.io/en/latest/tune/api_docs/schedulers.html#hyperband-tune-schedulers-hyperbandscheduler)\n", + "* **Method agnostic**: Ray/Tune works across deep learning frameworks (including PyTorch, Keras, Tensorflow, and PyTorchLightning) and with other ML methods like gradient-boosted trees (XGBoost, LightGBM)\n", + "* **Fault-tolerance**: Ray/Tune is built on top of Ray, providing tolerance for failed runs out of the box.\n", + "\n", + "This Colab demonstrates how this integration works for a simple grid search over two hyperparameters. If you've got any questions about the details,\n", + "check out\n", + "[our documentation](https://docs.wandb.com/library/integrations/ray-tune)\n", + "or the\n", + "[documentation for Ray/Tune](https://docs.ray.io/en/master/tune/api_docs/integration.html#weights-and-biases-tune-integration-wandb).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "W&B integrates with `ray.tune` by offering two lightweight standalone integrations:\n", + "\n", + "1. For simple cases, `WandbLoggerCallback` automatically logs metrics reported to Tune to W&B, along with the configuration of the experiment, using Tune's [`logger` interface](https://docs.ray.io/en/latest/tune/api_docs/logging.html).\n", + "2. The `@wandb_mixin` decorator gives you greater control over logging by letting you call `wandb.log` inside the decorated function, allowing you to [log custom metrics, plots, and other outputs, like media](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/wandb-log/Log_(Almost)_Anything_with_W%26B_Media.ipynb).\n", + "\n", + "These methods can be used together or independently.\n", + "\n", + "The example below demonstrates how they can be used together." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🧹 Running a hyperparameter sweep with W&B and Ray/Tune" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 📥 Install, `import`, and set seeds" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "Let's start by installing the libraries and importing everything we need.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -Uq ray[tune] wandb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"WANDB_API_KEY\"] = \"654d20637b9232f03417d6ae6e620f3b7cb70125\" #@param {type:\"string\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import numpy as np\n", + "from filelock import FileLock\n", + "import tempfile\n", + "from ray import train, tune\n", + "from ray.air.integrations.wandb import WandbLoggerCallback, setup_wandb\n", + "from ray.tune.schedulers import AsyncHyperBandScheduler\n", + "from ray.train import Checkpoint\n", + "import torch\n", + "import torch.optim as optim\n", + "import wandb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll make use of Ray's handy [`mnist_pytorch` example code](https://github.com/ray-project/ray/blob/master/python/ray/tune/examples/mnist_pytorch.py)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ray.tune.examples.mnist_pytorch import ConvNet, get_data_loaders, test_func, train_func" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to make this experiment reproducible, we'll set the seeds for random number generators of various libraries used in this experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "torch.backends.cudnn.deterministic = True\n", + "random.seed(2022)\n", + "np.random.seed(2022)\n", + "torch.manual_seed(2022)\n", + "torch.cuda.manual_seed_all(2022)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🤝 Integrating W&B with Ray/Tune" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now, we define our training process, decorated with `@wandb_mixin` so we can call `wandb.log` to log our custom metric\n", + "(here, just the error rate; you might also [log media](https://docs.wandb.com/library/log#media), e.g. images from the validation set, captioned by the model predictions).\n", + "\n", + "When we execute our hyperparameter sweep below,\n", + "this function will be called with a `config`uration dictionary\n", + "that contains values for any hyperparameters.\n", + "For simplicity, we only have two hyperparameters here:\n", + "the learning rate and momentum value for accelerated SGD." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train_mnist(config):\n", + " # Setup wandb\n", + " wandb = setup_wandb(config)\n", + " should_checkpoint = config.get(\"should_checkpoint\", False)\n", + " use_cuda = torch.cuda.is_available()\n", + " device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n", + " train_loader, test_loader = get_data_loaders()\n", + " model = ConvNet().to(device)\n", + "\n", + " optimizer = optim.SGD(\n", + " model.parameters(), lr=config[\"lr\"], momentum=config[\"momentum\"]\n", + " )\n", + " while True:\n", + " train_func(model, optimizer, train_loader, device)\n", + " acc = test_func(model, test_loader, device)\n", + " metrics = {\"mean_accuracy\": acc}\n", + "\n", + " # Report metrics (and possibly a checkpoint)\n", + " if should_checkpoint:\n", + " with tempfile.TemporaryDirectory() as tempdir:\n", + " torch.save(model.state_dict(), os.path.join(tempdir, \"model.pt\"))\n", + " train.report(metrics, checkpoint=Checkpoint.from_directory(tempdir))\n", + " else:\n", + " train.report(metrics)\n", + " # enables logging custom metrics using wandb.log()\n", + " error_rate = 100 * (1 - acc)\n", + " wandb.log({\"error_rate\": error_rate})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 🚀 Launching a Sweep with W&B and Ray/Tune" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We're now almost ready to call `tune.run` to launch our hyperparameter sweep!\n", + "We just need to do three things:\n", + "1. set up a `wandb.Run`,\n", + "2. give the `WandbLoggerCallback` to `tune.run` so we can capture the output of `tune.report`, and\n", + "3. set up our hyperparameter sweep.\n", + "\n", + "A `wandb.Run` is normally created by calling `wandb.init`.\n", + "`tune` will handle that for you, you just need to pass\n", + "the arguments as a dictionary\n", + "(see [our documentation](https://docs.wandb.com/library/init) for details on `wandb.init`).\n", + "At the bare minimum, you need to pass in a `project` name --\n", + "sort of like a `git` repo name, but for your ML projects.\n", + "\n", + "In addition to holding arguments for `wandb.init`,\n", + "that dictionary also has a few special keys, described in\n", + "[the documentation for the `WandbLoggerCallback`](https://docs.ray.io/en/master/tune/tutorials/tune-wandb.html)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We handle steps 2 and 3 when we invoke `tune.run`.\n", + "\n", + "Step 2 is handled by passing in the `WandbLoggerCallback` class in a list\n", + "to the `loggers` argument of `tune.run`.\n", + "\n", + "The setup of the hyperparameter sweep is handled by the\n", + "`config` argument of `tune.run`.\n", + "For the purposes of the integration,\n", + "the most important part is that this is where we pass in the `wandb_init`\n", + "dictionary.\n", + "\n", + "This is also where we configure the \"meat\" of the hyperparameter sweep:\n", + "what are the hyperparameters we're sweeping over,\n", + "and how do we choose their values.\n", + "\n", + "Here, we do a simple grid search, but\n", + "[Ray/Tune provides lots of sophisticated options](https://docs.ray.io/en/latest/tune/api_docs/suggestion.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# for early stopping\n", + "sched = AsyncHyperBandScheduler()\n", + "\n", + "resources_per_trial = {\"gpu\": 1} # set this for GPUs\n", + "tuner = tune.Tuner(\n", + " tune.with_resources(train_mnist, resources=resources_per_trial),\n", + " tune_config=tune.TuneConfig(\n", + " metric=\"mean_accuracy\",\n", + " mode=\"max\",\n", + " scheduler=sched,\n", + " num_samples=50,\n", + " ),\n", + " run_config=train.RunConfig(\n", + " name=\"exp\",\n", + " stop={\n", + " \"mean_accuracy\": 0.98,\n", + " \"training_iteration\": 5,\n", + " },\n", + " callbacks=[WandbLoggerCallback(project=\"raytune-colab\")]\n", + " ),\n", + " param_space={\n", + " \"lr\": tune.loguniform(1e-4, 1e-2),\n", + " \"momentum\": tune.uniform(0.1, 0.9),\n", + " },\n", + ")\n", + "results = tuner.fit()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Best config is:\", results.get_best_result().config)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/colabs/raytune/tune-wandb.ipynb b/colabs/raytune/tune-wandb.ipynb index f35865f7..ddb2baca 100644 --- a/colabs/raytune/tune-wandb.ipynb +++ b/colabs/raytune/tune-wandb.ipynb @@ -2,7 +2,6 @@ "cells": [ { "cell_type": "markdown", - "id": "91d48998", "metadata": {}, "source": [ "\"Open\n" diff --git a/colabs/scikit/wandb_decision_tree.ipynb b/colabs/scikit/wandb_decision_tree.ipynb index d2d34873..b4520093 100644 --- a/colabs/scikit/wandb_decision_tree.ipynb +++ b/colabs/scikit/wandb_decision_tree.ipynb @@ -1,247 +1,213 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "e6qvOYJV8BQw" - }, - "source": [ - "## Author: [@SauravMaheshkar](https://twitter.com/MaheshkarSaurav)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BL5XadIr0MbM" - }, - "source": [ - "# Packages 📦 and Basic Setup\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "fZH0PUfw0OBV" - }, - "source": [ - "## Install Packages" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_uk-bdosSzwk" - }, - "outputs": [], - "source": [ - "%%capture\n", - "## Install Sklearn\n", - "!pip install -U scikit-learn\n", - "## Install the latest version of wandb client 🔥🔥\n", - "!pip install -q --upgrade wandb" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UgQLiLtz0STo" - }, - "source": [ - "## Project Configuration using **`wandb.config`**" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "OvABqBlB0VhB" - }, - "outputs": [], - "source": [ - "import wandb\n", - "\n", - "## Importing Libraries\n", - "from sklearn.datasets import load_iris\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "wandb.login()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Initialize the run\n", - "run = wandb.init(project='simple-scikit')\n", - "\n", - "# Feel free to change these and experiment !!\n", - "config = wandb.config\n", - "config.max_depth = 5\n", - "config.min_samples_split = 2\n", - "config.clf_criterion = \"gini\"\n", - "config.reg_criterion = \"mse\"\n", - "config.splitter = \"best\"\n", - "config.dataset = \"iris\"\n", - "config.test_size = 0.2\n", - "config.random_state = 42\n", - "config.labels =['setosa', 'versicolor', 'virginica']\n", - "\n", - "# Update the config\n", - "wandb.config.update(config)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OGV1c4VYJMGa" - }, - "source": [ - "# 💿 Dataset\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lWnhwnPI2NWD" - }, - "outputs": [], - "source": [ - "## Loading the Dataset\n", - "iris = load_iris(return_X_y = True, as_frame= True)\n", - "dataset = iris[0]\n", - "target = iris[1]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "kycxKC7ER7lW" - }, - "source": [ - "# ✍️ Model Architecture\n", - "---" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "4wWuB4NZl3h_" - }, - "source": [ - "## Classification" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vElZpyEQtXzX" - }, - "outputs": [], - "source": [ - "X, y = load_iris(return_X_y=True)\n", - "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = config.test_size, random_state = config.random_state)\n", - "\n", - "clf = DecisionTreeClassifier(\n", - " max_depth=config.max_depth,\n", - " min_samples_split=config.min_samples_split,\n", - " criterion=config.clf_criterion,\n", - " splitter=config.splitter\n", - ")\n", - "clf = clf.fit(x_train,y_train)\n", - "\n", - "y_pred = clf.predict(x_test)\n", - "\n", - "# Visualize Confustion Matrix\n", - "wandb.sklearn.plot_confusion_matrix(y_test, y_pred, config.labels)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "BX31c0s8MVu5" - }, - "source": [ - "## Regression" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "1kBLTtH4MbsY" - }, - "outputs": [], - "source": [ - "X, y = load_iris(return_X_y=True)\n", - "\n", - "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = config.test_size, random_state = config.random_state)\n", - "\n", - "reg = DecisionTreeRegressor(\n", - " max_depth=config.max_depth,\n", - " min_samples_split=config.min_samples_split,\n", - " criterion=config.reg_criterion,\n", - " splitter=config.splitter\n", - ")\n", - "\n", - "reg = reg.fit(x_train,y_train)\n", - "\n", - "# All regression plots\n", - "wandb.sklearn.plot_regressor(reg, x_train, x_test, y_train, y_test, model_name='DecisionTreeRegressor')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Finish the W&B Process\n", - "wandb.finish()" - ] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "bE1OZxeIwPqg", - "_N6qEF3cvIpN", - "_vdIezY_zb_9", - "OGV1c4VYJMGa", - "H8IIKXd0PFL8", - "kycxKC7ER7lW", - "4wWuB4NZl3h_", - "BX31c0s8MVu5", - "c6PoNscLNuog", - "3Cf152r2NK1M", - "a_HO0jHMyyyP", - "ahE27LgSzUVx", - "FaENg6O44dso", - "t3K1L5UWSZCL" - ], - "name": "Decision Trees", - "provenance": [], - "toc_visible": true - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 0 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Author: [@SauravMaheshkar](https://twitter.com/MaheshkarSaurav)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Packages 📦 and Basic Setup\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%capture\n", + "## Install Sklearn\n", + "!pip install -U scikit-learn\n", + "## Install the latest version of wandb client 🔥🔥\n", + "!pip install -q --upgrade wandb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Project Configuration using **`wandb.config`**" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import wandb\n", + "\n", + "## Importing Libraries\n", + "from sklearn.datasets import load_iris\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the run\n", + "run = wandb.init(project='simple-scikit')\n", + "\n", + "# Feel free to change these and experiment !!\n", + "config = wandb.config\n", + "config.max_depth = 5\n", + "config.min_samples_split = 2\n", + "config.clf_criterion = \"gini\"\n", + "config.reg_criterion = \"mse\"\n", + "config.splitter = \"best\"\n", + "config.dataset = \"iris\"\n", + "config.test_size = 0.2\n", + "config.random_state = 42\n", + "config.labels =['setosa', 'versicolor', 'virginica']\n", + "\n", + "# Update the config\n", + "wandb.config.update(config)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 💿 Dataset\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Loading the Dataset\n", + "iris = load_iris(return_X_y = True, as_frame= True)\n", + "dataset = iris[0]\n", + "target = iris[1]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# ✍️ Model Architecture\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Classification" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X, y = load_iris(return_X_y=True)\n", + "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = config.test_size, random_state = config.random_state)\n", + "\n", + "clf = DecisionTreeClassifier(\n", + " max_depth=config.max_depth,\n", + " min_samples_split=config.min_samples_split,\n", + " criterion=config.clf_criterion,\n", + " splitter=config.splitter\n", + ")\n", + "clf = clf.fit(x_train,y_train)\n", + "\n", + "y_pred = clf.predict(x_test)\n", + "\n", + "# Visualize Confustion Matrix\n", + "wandb.sklearn.plot_confusion_matrix(y_test, y_pred, config.labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Regression" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "X, y = load_iris(return_X_y=True)\n", + "\n", + "x_train, x_test, y_train, y_test = train_test_split(X, y, test_size = config.test_size, random_state = config.random_state)\n", + "\n", + "reg = DecisionTreeRegressor(\n", + " max_depth=config.max_depth,\n", + " min_samples_split=config.min_samples_split,\n", + " criterion=config.reg_criterion,\n", + " splitter=config.splitter\n", + ")\n", + "\n", + "reg = reg.fit(x_train,y_train)\n", + "\n", + "# All regression plots\n", + "wandb.sklearn.plot_regressor(reg, x_train, x_test, y_train, y_test, model_name='DecisionTreeRegressor')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Finish the W&B Process\n", + "wandb.finish()" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/colabs/super-gradients/yolo_nas.ipynb b/colabs/super-gradients/yolo_nas.ipynb index 2c953fdf..14af9eb9 100644 --- a/colabs/super-gradients/yolo_nas.ipynb +++ b/colabs/super-gradients/yolo_nas.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -359,7 +366,18 @@ ] } ], - "metadata": {}, + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, "nbformat": 4, "nbformat_minor": 2 } diff --git a/colabs/super-gradients/yolo_nas_data_analysis.ipynb b/colabs/super-gradients/yolo_nas_data_analysis.ipynb index e98425cf..d512547c 100644 --- a/colabs/super-gradients/yolo_nas_data_analysis.ipynb +++ b/colabs/super-gradients/yolo_nas_data_analysis.ipynb @@ -1,438 +1,425 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] }, - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "SJenmP6BkY2V" - }, - "outputs": [], - "source": [ - "!sudo apt install libcairo2-dev pkg-config python3-dev -qq\n", - "!pip install roboflow pycairo wandb sweeps -qqq\n", - "!pip install super_gradients" - ] - }, - { - "cell_type": "code", - "source": [ - "import os\n", - "import glob\n", - "import torch\n", - "import wandb\n", - "import warnings\n", - "import numpy as np\n", - "import pandas as pd\n", - "import matplotlib.pyplot as plt\n", - "\n", - "from matplotlib import patches\n", - "from google.colab import userdata\n", - "from torchvision.io import read_image\n", - "from torch.utils.data import DataLoader\n", - "from super_gradients.training import models, dataloaders\n", - "from super_gradients.training.dataloaders.dataloaders import (\n", - " coco_detection_yolo_format_train, coco_detection_yolo_format_val\n", - ")\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "os.environ[\"WANDB_API_KEY\"] = userdata.get('wandb')\n", - "os.environ[\"ROBOFLOW_API_KEY\"] = userdata.get('roboflow')" - ], - "metadata": { - "id": "zxmwpTBzpd-d" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "from roboflow import Roboflow\n", - "rf = Roboflow(api_key=os.getenv(\"ROBOFLOW_API_KEY\"))\n", - "project = rf.workspace(\"easyhyeon\").project(\"trash-sea\")\n", - "dataset = project.version(10).download(\"yolov5\")" - ], - "metadata": { - "id": "77XdRx8rpfqQ" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "DATASET_PATH = \"/content/trash-sea-10\"\n", - "WANDB_PROJECT_NAME = \"fconn-yolo-nas\"\n", - "ENTITY = \"ml-colabs\"" - ], - "metadata": { - "id": "8shwiGscpglK" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "dataset_params = {\n", - " 'data_dir':DATASET_PATH,\n", - " 'train_images_dir':'train/images',\n", - " 'train_labels_dir':'train/labels',\n", - " 'val_images_dir':'valid/images',\n", - " 'val_labels_dir':'valid/labels',\n", - " 'test_images_dir':'test/images',\n", - " 'test_labels_dir':'test/labels',\n", - " 'classes': [\"Buoy\", \"Can\", \"Paper\", \"Plastic Bag\", \"Plastic Bottle\"]\n", - "}" - ], - "metadata": { - "id": "13DJyeaIvKOf" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "from IPython.display import clear_output\n", - "\n", - "train_data = coco_detection_yolo_format_train(\n", - " dataset_params={\n", - " 'data_dir': dataset_params['data_dir'],\n", - " 'images_dir': dataset_params['train_images_dir'],\n", - " 'labels_dir': dataset_params['train_labels_dir'],\n", - " 'classes': dataset_params['classes'],\n", - " },\n", - " dataloader_params={\n", - " 'batch_size':16,\n", - " 'num_workers':2\n", - " }\n", - ")\n", - "\n", - "val_data = coco_detection_yolo_format_val(\n", - " dataset_params={\n", - " 'data_dir': dataset_params['data_dir'],\n", - " 'images_dir': dataset_params['val_images_dir'],\n", - " 'labels_dir': dataset_params['val_labels_dir'],\n", - " 'classes': dataset_params['classes'],\n", - " },\n", - " dataloader_params={\n", - " 'batch_size':16,\n", - " 'num_workers':2\n", - " }\n", - ")\n", - "\n", - "test_data = coco_detection_yolo_format_val(\n", - " dataset_params={\n", - " 'data_dir': dataset_params['data_dir'],\n", - " 'images_dir': dataset_params['test_images_dir'],\n", - " 'labels_dir': dataset_params['test_labels_dir'],\n", - " 'classes': dataset_params['classes'],\n", - " },\n", - " dataloader_params={\n", - " 'batch_size':16,\n", - " 'num_workers':2\n", - " }\n", - ")\n", - "\n", - "train_data.dataset.transforms = train_data.dataset.transforms[5:]" - ], - "metadata": { - "id": "7EAeEm3xvS82" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "colors = {\n", - " 0: 'red',\n", - " 1: 'green',\n", - " 2: 'blue',\n", - " 3: 'yellow',\n", - " 4: 'black'\n", - "}\n", - "classes = {\n", - " 0:\"Buoy\",\n", - " 1:\"Can\",\n", - " 2:\"Paper\",\n", - " 3:\"Plastic Bag\",\n", - " 4:\"Plastic Bottle\"\n", - "}" - ], - "metadata": { - "id": "0XKew_u_vRxs" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "def process_bounding_boxes_list(annots):\n", - " result = []\n", - "\n", - " class_count = {i:0 for i in range(0, 5)}\n", - "\n", - " for annot_idx, annotation in enumerate(annots):\n", - " class_count[int(annotation[1])] += 1\n", - " result.append({\n", - " \"position\":{\n", - " \"middle\":[float(annotation[2]), float(annotation[3])],\n", - " \"width\":float(annotation[4]),\n", - " \"height\":float(annotation[5])\n", - " },\n", - " \"domain\":\"pixel\",\n", - " \"class_id\":int(annotation[1]),\n", - " \"box_caption\":classes[int(annotation[1])]\n", - " })\n", - "\n", - " return result, class_count\n", - "\n", - "def populate_wandb_image_samples(train_data):\n", - " wandb.init(\n", - " project=WANDB_PROJECT_NAME,\n", - " entity=ENTITY,\n", - " id='add-image-samples',\n", - " job_type=\"add-tables\",\n", - " resume='allow'\n", - " )\n", - "\n", - " class_set = wandb.Classes(\n", - " [\n", - " {\"name\": \"Buoy\", \"id\": 0},\n", - " {\"name\": \"Can\", \"id\": 1},\n", - " {\"name\": \"Paper\", \"id\": 2},\n", - " {\"name\": \"Plastic Bag\", \"id\": 3},\n", - " {\"name\": \"Plastic Bottle\", \"id\": 4},\n", - " ]\n", - " )\n", - "\n", - " table = wandb.Table(\n", - " columns=[\n", - " \"Annotated-Image\", \"Number-of-objects\",\n", - " \"Number-Buoy\", \"Number-Can\", \"Number-Paper\",\n", - " \"Number-Plastic-Bag\", \"Number-Plastic-Bottle\"\n", - " ]\n", - " )\n", - "\n", - " img_count = 0\n", - "\n", - " for batch_idx, batch_sample in enumerate(train_data):\n", - " batch_images = batch_sample[0]\n", - " batch_annotations = batch_sample[1]\n", - "\n", - " annots_dict = {i:[] for i in range(0, batch_images.shape[0])}\n", - " for annot in batch_annotations:\n", - " annots_dict[int(annot[0])].append(annot)\n", - "\n", - " for idx, image in enumerate(batch_images):\n", - "\n", - " bbox, class_count = process_bounding_boxes_list(annots_dict[idx])\n", - "\n", - " image = image.flip(0)\n", - "\n", - " img = wandb.Image(\n", - " image,\n", - " boxes={\n", - " \"ground_truth\":{\n", - " \"box_data\":bbox,\n", - " \"class_labels\": classes,\n", - " }\n", - " },\n", - " classes=class_set,\n", - " )\n", - "\n", - " table.add_data(img, len(bbox), class_count[0],\n", - " class_count[1], class_count[2],\n", - " class_count[3], class_count[4])\n", - " img_count += 1\n", - "\n", - " print(f\"{img_count}/{len(train_data)*16} completed\")\n", - "\n", - " wandb.log({\"ground_truth_dataset\": table})\n", - " wandb.finish()\n", - "\n", - "populate_wandb_image_samples(train_data)" - ], - "metadata": { - "id": "ncYSU2svvd-B" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "def process_bounding_boxes_list(annots):\n", - " result = []\n", - "\n", - " class_count = {i:0 for i in range(0, 5)}\n", - "\n", - " for annot_idx, annotation in enumerate(annots):\n", - " class_count[int(annotation[1])] += 1\n", - " result.append({\n", - " \"position\":{\n", - " \"middle\":[float(annotation[2]), float(annotation[3])],\n", - " \"width\":float(annotation[4]),\n", - " \"height\":float(annotation[5])\n", - " },\n", - " \"domain\":\"pixel\",\n", - " \"class_id\":int(annotation[1]),\n", - " \"box_caption\":classes[int(annotation[1])]\n", - " })\n", - "\n", - " return result, class_count\n", - "\n", - "def populate_wandb_bbox(train_data):\n", - " wandb.init(\n", - " project=WANDB_PROJECT_NAME,\n", - " entity=ENTITY,\n", - " id='add-bbox-data',\n", - " job_type=\"add-tables\",\n", - " resume='allow'\n", - " )\n", - "\n", - " class_set = wandb.Classes(\n", - " [\n", - " {\"name\": \"Buoy\", \"id\": 0},\n", - " {\"name\": \"Can\", \"id\": 1},\n", - " {\"name\": \"Paper\", \"id\": 2},\n", - " {\"name\": \"Plastic Bag\", \"id\": 3},\n", - " {\"name\": \"Plastic Bottle\", \"id\": 4},\n", - " ]\n", - " )\n", - "\n", - " table = wandb.Table(\n", - " columns=[\n", - " \"Image-Id\",\n", - " \"BBox-Height\",\n", - " \"BBox-Width\",\n", - " \"Class-Id\"\n", - " ]\n", - " )\n", - "\n", - " img_count = 0\n", - "\n", - " for batch_idx, batch_sample in enumerate(train_data):\n", - " batch_images = batch_sample[0]\n", - " batch_annotations = batch_sample[1]\n", - "\n", - " annots_dict = {i:[] for i in range(0, batch_images.shape[0])}\n", - " for annot in batch_annotations:\n", - " annots_dict[int(annot[0])].append(annot)\n", - "\n", - " for idx, image in enumerate(batch_images):\n", - "\n", - " result, class_count = process_bounding_boxes_list(annots_dict[idx])\n", - "\n", - " for bbox in result:\n", - " height = bbox[\"position\"][\"height\"]\n", - " width = bbox[\"position\"][\"width\"]\n", - " class_id = bbox[\"class_id\"]\n", - " table.add_data(img_count, height, width, classes[class_id])\n", - "\n", - " img_count += 1\n", - "\n", - " print(f\"{img_count}/{len(train_data)*16} completed\")\n", - "\n", - " wandb.log({\"bounding_box_information\": table})\n", - " wandb.finish()\n", - "\n", - "populate_wandb_bbox(train_data)" - ], - "metadata": { - "id": "-8W-zcs9vh93" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "def populate_wandb_spatial_heatmaps(train_data):\n", - " wandb.init(\n", - " project=WANDB_PROJECT_NAME,\n", - " entity=ENTITY,\n", - " id='add-heatmap',\n", - " job_type=\"add-tables\",\n", - " resume='allow'\n", - " )\n", - "\n", - " class_set = wandb.Classes(\n", - " [\n", - " {\"name\": \"Buoy\", \"id\": 0},\n", - " {\"name\": \"Can\", \"id\": 1},\n", - " {\"name\": \"Paper\", \"id\": 2},\n", - " {\"name\": \"Plastic Bag\", \"id\": 3},\n", - " {\"name\": \"Plastic Bottle\", \"id\": 4},\n", - " ]\n", - " )\n", - " heatmaps = [np.zeros((224, 224, 1), dtype=np.float32) for _ in classes]\n", - " annotation_counts = {i:0 for i in range(len(classes))}\n", - "\n", - " table = wandb.Table(columns=[\"Class-Id\", \"Class-Name\", \"Spatial-Heatmap\",\n", - " \"Num-Total-Objects\"])\n", - "\n", - " for batch_idx, batch_sample in enumerate(train_data):\n", - " batch_images = batch_sample[0]\n", - " batch_annotations = batch_sample[1]\n", - "\n", - " annots_dict = {i:[] for i in range(0, batch_images.shape[0])}\n", - "\n", - " for annot in batch_annotations:\n", - " class_idx = int(annot[1])\n", - "\n", - " midpoint_x = int(annot[2])\n", - " midpoint_y = int(annot[3])\n", - " width = int(annot[4])\n", - " height = int(annot[5])\n", - "\n", - " x_min = midpoint_x - (width//2)\n", - " x_max = midpoint_x + (width//2)\n", - "\n", - " y_min = midpoint_y - (height//2)\n", - " y_max = midpoint_y + (height//2)\n", - "\n", - " heatmaps[class_idx][y_min:y_max, x_min:x_max] += 1\n", - "\n", - " annotation_counts[class_idx] += 1\n", - "\n", - " print(f\"{batch_idx+1}/{len(train_data)} batches completed\")\n", - "\n", - " for class_idx in range(len(classes)):\n", - " heatmap = wandb.Image(\n", - " heatmaps[class_idx],\n", - " caption=classes[class_idx]\n", - " )\n", - " table.add_data(class_idx, classes[class_idx], heatmap, annotation_counts[class_idx])\n", - "\n", - " wandb.log({\"spatial_heatmap_information\": table})\n", - " wandb.finish()\n", - "\n", - "populate_wandb_spatial_heatmaps(train_data)" - ], - "metadata": { - "id": "AaN1pUvJvqjA" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!sudo apt install libcairo2-dev pkg-config python3-dev -qq\n", + "!pip install roboflow pycairo wandb sweeps -qqq\n", + "!pip install super_gradients" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import glob\n", + "import torch\n", + "import wandb\n", + "import warnings\n", + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "\n", + "from matplotlib import patches\n", + "from google.colab import userdata\n", + "from torchvision.io import read_image\n", + "from torch.utils.data import DataLoader\n", + "from super_gradients.training import models, dataloaders\n", + "from super_gradients.training.dataloaders.dataloaders import (\n", + " coco_detection_yolo_format_train, coco_detection_yolo_format_val\n", + ")\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "os.environ[\"WANDB_API_KEY\"] = userdata.get('wandb')\n", + "os.environ[\"ROBOFLOW_API_KEY\"] = userdata.get('roboflow')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from roboflow import Roboflow\n", + "rf = Roboflow(api_key=os.getenv(\"ROBOFLOW_API_KEY\"))\n", + "project = rf.workspace(\"easyhyeon\").project(\"trash-sea\")\n", + "dataset = project.version(10).download(\"yolov5\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "DATASET_PATH = \"/content/trash-sea-10\"\n", + "WANDB_PROJECT_NAME = \"fconn-yolo-nas\"\n", + "ENTITY = \"ml-colabs\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_params = {\n", + " 'data_dir':DATASET_PATH,\n", + " 'train_images_dir':'train/images',\n", + " 'train_labels_dir':'train/labels',\n", + " 'val_images_dir':'valid/images',\n", + " 'val_labels_dir':'valid/labels',\n", + " 'test_images_dir':'test/images',\n", + " 'test_labels_dir':'test/labels',\n", + " 'classes': [\"Buoy\", \"Can\", \"Paper\", \"Plastic Bag\", \"Plastic Bottle\"]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import clear_output\n", + "\n", + "train_data = coco_detection_yolo_format_train(\n", + " dataset_params={\n", + " 'data_dir': dataset_params['data_dir'],\n", + " 'images_dir': dataset_params['train_images_dir'],\n", + " 'labels_dir': dataset_params['train_labels_dir'],\n", + " 'classes': dataset_params['classes'],\n", + " },\n", + " dataloader_params={\n", + " 'batch_size':16,\n", + " 'num_workers':2\n", + " }\n", + ")\n", + "\n", + "val_data = coco_detection_yolo_format_val(\n", + " dataset_params={\n", + " 'data_dir': dataset_params['data_dir'],\n", + " 'images_dir': dataset_params['val_images_dir'],\n", + " 'labels_dir': dataset_params['val_labels_dir'],\n", + " 'classes': dataset_params['classes'],\n", + " },\n", + " dataloader_params={\n", + " 'batch_size':16,\n", + " 'num_workers':2\n", + " }\n", + ")\n", + "\n", + "test_data = coco_detection_yolo_format_val(\n", + " dataset_params={\n", + " 'data_dir': dataset_params['data_dir'],\n", + " 'images_dir': dataset_params['test_images_dir'],\n", + " 'labels_dir': dataset_params['test_labels_dir'],\n", + " 'classes': dataset_params['classes'],\n", + " },\n", + " dataloader_params={\n", + " 'batch_size':16,\n", + " 'num_workers':2\n", + " }\n", + ")\n", + "\n", + "train_data.dataset.transforms = train_data.dataset.transforms[5:]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "colors = {\n", + " 0: 'red',\n", + " 1: 'green',\n", + " 2: 'blue',\n", + " 3: 'yellow',\n", + " 4: 'black'\n", + "}\n", + "classes = {\n", + " 0:\"Buoy\",\n", + " 1:\"Can\",\n", + " 2:\"Paper\",\n", + " 3:\"Plastic Bag\",\n", + " 4:\"Plastic Bottle\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_bounding_boxes_list(annots):\n", + " result = []\n", + "\n", + " class_count = {i:0 for i in range(0, 5)}\n", + "\n", + " for annot_idx, annotation in enumerate(annots):\n", + " class_count[int(annotation[1])] += 1\n", + " result.append({\n", + " \"position\":{\n", + " \"middle\":[float(annotation[2]), float(annotation[3])],\n", + " \"width\":float(annotation[4]),\n", + " \"height\":float(annotation[5])\n", + " },\n", + " \"domain\":\"pixel\",\n", + " \"class_id\":int(annotation[1]),\n", + " \"box_caption\":classes[int(annotation[1])]\n", + " })\n", + "\n", + " return result, class_count\n", + "\n", + "def populate_wandb_image_samples(train_data):\n", + " wandb.init(\n", + " project=WANDB_PROJECT_NAME,\n", + " entity=ENTITY,\n", + " id='add-image-samples',\n", + " job_type=\"add-tables\",\n", + " resume='allow'\n", + " )\n", + "\n", + " class_set = wandb.Classes(\n", + " [\n", + " {\"name\": \"Buoy\", \"id\": 0},\n", + " {\"name\": \"Can\", \"id\": 1},\n", + " {\"name\": \"Paper\", \"id\": 2},\n", + " {\"name\": \"Plastic Bag\", \"id\": 3},\n", + " {\"name\": \"Plastic Bottle\", \"id\": 4},\n", + " ]\n", + " )\n", + "\n", + " table = wandb.Table(\n", + " columns=[\n", + " \"Annotated-Image\", \"Number-of-objects\",\n", + " \"Number-Buoy\", \"Number-Can\", \"Number-Paper\",\n", + " \"Number-Plastic-Bag\", \"Number-Plastic-Bottle\"\n", + " ]\n", + " )\n", + "\n", + " img_count = 0\n", + "\n", + " for batch_idx, batch_sample in enumerate(train_data):\n", + " batch_images = batch_sample[0]\n", + " batch_annotations = batch_sample[1]\n", + "\n", + " annots_dict = {i:[] for i in range(0, batch_images.shape[0])}\n", + " for annot in batch_annotations:\n", + " annots_dict[int(annot[0])].append(annot)\n", + "\n", + " for idx, image in enumerate(batch_images):\n", + "\n", + " bbox, class_count = process_bounding_boxes_list(annots_dict[idx])\n", + "\n", + " image = image.flip(0)\n", + "\n", + " img = wandb.Image(\n", + " image,\n", + " boxes={\n", + " \"ground_truth\":{\n", + " \"box_data\":bbox,\n", + " \"class_labels\": classes,\n", + " }\n", + " },\n", + " classes=class_set,\n", + " )\n", + "\n", + " table.add_data(img, len(bbox), class_count[0],\n", + " class_count[1], class_count[2],\n", + " class_count[3], class_count[4])\n", + " img_count += 1\n", + "\n", + " print(f\"{img_count}/{len(train_data)*16} completed\")\n", + "\n", + " wandb.log({\"ground_truth_dataset\": table})\n", + " wandb.finish()\n", + "\n", + "populate_wandb_image_samples(train_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def process_bounding_boxes_list(annots):\n", + " result = []\n", + "\n", + " class_count = {i:0 for i in range(0, 5)}\n", + "\n", + " for annot_idx, annotation in enumerate(annots):\n", + " class_count[int(annotation[1])] += 1\n", + " result.append({\n", + " \"position\":{\n", + " \"middle\":[float(annotation[2]), float(annotation[3])],\n", + " \"width\":float(annotation[4]),\n", + " \"height\":float(annotation[5])\n", + " },\n", + " \"domain\":\"pixel\",\n", + " \"class_id\":int(annotation[1]),\n", + " \"box_caption\":classes[int(annotation[1])]\n", + " })\n", + "\n", + " return result, class_count\n", + "\n", + "def populate_wandb_bbox(train_data):\n", + " wandb.init(\n", + " project=WANDB_PROJECT_NAME,\n", + " entity=ENTITY,\n", + " id='add-bbox-data',\n", + " job_type=\"add-tables\",\n", + " resume='allow'\n", + " )\n", + "\n", + " class_set = wandb.Classes(\n", + " [\n", + " {\"name\": \"Buoy\", \"id\": 0},\n", + " {\"name\": \"Can\", \"id\": 1},\n", + " {\"name\": \"Paper\", \"id\": 2},\n", + " {\"name\": \"Plastic Bag\", \"id\": 3},\n", + " {\"name\": \"Plastic Bottle\", \"id\": 4},\n", + " ]\n", + " )\n", + "\n", + " table = wandb.Table(\n", + " columns=[\n", + " \"Image-Id\",\n", + " \"BBox-Height\",\n", + " \"BBox-Width\",\n", + " \"Class-Id\"\n", + " ]\n", + " )\n", + "\n", + " img_count = 0\n", + "\n", + " for batch_idx, batch_sample in enumerate(train_data):\n", + " batch_images = batch_sample[0]\n", + " batch_annotations = batch_sample[1]\n", + "\n", + " annots_dict = {i:[] for i in range(0, batch_images.shape[0])}\n", + " for annot in batch_annotations:\n", + " annots_dict[int(annot[0])].append(annot)\n", + "\n", + " for idx, image in enumerate(batch_images):\n", + "\n", + " result, class_count = process_bounding_boxes_list(annots_dict[idx])\n", + "\n", + " for bbox in result:\n", + " height = bbox[\"position\"][\"height\"]\n", + " width = bbox[\"position\"][\"width\"]\n", + " class_id = bbox[\"class_id\"]\n", + " table.add_data(img_count, height, width, classes[class_id])\n", + "\n", + " img_count += 1\n", + "\n", + " print(f\"{img_count}/{len(train_data)*16} completed\")\n", + "\n", + " wandb.log({\"bounding_box_information\": table})\n", + " wandb.finish()\n", + "\n", + "populate_wandb_bbox(train_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def populate_wandb_spatial_heatmaps(train_data):\n", + " wandb.init(\n", + " project=WANDB_PROJECT_NAME,\n", + " entity=ENTITY,\n", + " id='add-heatmap',\n", + " job_type=\"add-tables\",\n", + " resume='allow'\n", + " )\n", + "\n", + " class_set = wandb.Classes(\n", + " [\n", + " {\"name\": \"Buoy\", \"id\": 0},\n", + " {\"name\": \"Can\", \"id\": 1},\n", + " {\"name\": \"Paper\", \"id\": 2},\n", + " {\"name\": \"Plastic Bag\", \"id\": 3},\n", + " {\"name\": \"Plastic Bottle\", \"id\": 4},\n", + " ]\n", + " )\n", + " heatmaps = [np.zeros((224, 224, 1), dtype=np.float32) for _ in classes]\n", + " annotation_counts = {i:0 for i in range(len(classes))}\n", + "\n", + " table = wandb.Table(columns=[\"Class-Id\", \"Class-Name\", \"Spatial-Heatmap\",\n", + " \"Num-Total-Objects\"])\n", + "\n", + " for batch_idx, batch_sample in enumerate(train_data):\n", + " batch_images = batch_sample[0]\n", + " batch_annotations = batch_sample[1]\n", + "\n", + " annots_dict = {i:[] for i in range(0, batch_images.shape[0])}\n", + "\n", + " for annot in batch_annotations:\n", + " class_idx = int(annot[1])\n", + "\n", + " midpoint_x = int(annot[2])\n", + " midpoint_y = int(annot[3])\n", + " width = int(annot[4])\n", + " height = int(annot[5])\n", + "\n", + " x_min = midpoint_x - (width//2)\n", + " x_max = midpoint_x + (width//2)\n", + "\n", + " y_min = midpoint_y - (height//2)\n", + " y_max = midpoint_y + (height//2)\n", + "\n", + " heatmaps[class_idx][y_min:y_max, x_min:x_max] += 1\n", + "\n", + " annotation_counts[class_idx] += 1\n", + "\n", + " print(f\"{batch_idx+1}/{len(train_data)} batches completed\")\n", + "\n", + " for class_idx in range(len(classes)):\n", + " heatmap = wandb.Image(\n", + " heatmaps[class_idx],\n", + " caption=classes[class_idx]\n", + " )\n", + " table.add_data(class_idx, classes[class_idx], heatmap, annotation_counts[class_idx])\n", + "\n", + " wandb.log({\"spatial_heatmap_information\": table})\n", + " wandb.finish()\n", + "\n", + "populate_wandb_spatial_heatmaps(train_data)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/colabs/super-gradients/yolo_nas_sweep_run.ipynb b/colabs/super-gradients/yolo_nas_sweep_run.ipynb index 93ab971c..8b1ff62b 100644 --- a/colabs/super-gradients/yolo_nas_sweep_run.ipynb +++ b/colabs/super-gradients/yolo_nas_sweep_run.ipynb @@ -1,345 +1,326 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] }, - "cells": [ - { - "cell_type": "markdown", - "source": [ - "## Installation and Imports" - ], - "metadata": { - "id": "Fe3Dp6NDzPyr" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "KXwTjgsFkZnm" - }, - "outputs": [], - "source": [ - "!sudo apt install libcairo2-dev pkg-config python3-dev -qq\n", - "!pip install roboflow pycairo wandb sweeps -qqq\n", - "!pip install super_gradients" - ] - }, - { - "cell_type": "code", - "source": [ - "import os\n", - "import glob\n", - "import torch\n", - "import wandb\n", - "import warnings\n", - "import pandas as pd\n", - "\n", - "from google.colab import userdata\n", - "from torchvision.io import read_image\n", - "from torch.utils.data import DataLoader\n", - "from IPython.display import clear_output\n", - "\n", - "from super_gradients.training import models, Trainer, dataloaders\n", - "from super_gradients.training.losses import PPYoloELoss\n", - "from super_gradients.training.metrics import DetectionMetrics_050\n", - "from super_gradients.training.models.detection_models.pp_yolo_e import PPYoloEPostPredictionCallback\n", - "from super_gradients.training.dataloaders.dataloaders import coco_detection_yolo_format_train, coco_detection_yolo_format_val\n", - "\n", - "os.environ[\"WANDB_API_KEY\"] = userdata.get('wandb')\n", - "os.environ[\"ROBOFLOW_API_KEY\"] = userdata.get('roboflow')" - ], - "metadata": { - "id": "PpsF3k8awWbP" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Definitions" - ], - "metadata": { - "id": "qRjY7eYPzScK" - } - }, - { - "cell_type": "code", - "source": [ - "seed = 42\n", - "torch.manual_seed(seed)\n", - "\n", - "if torch.cuda.is_available():\n", - " torch.cuda.manual_seed(seed)\n", - " torch.cuda.manual_seed_all(seed)\n", - "\n", - "torch.backends.cudnn.deterministic = True\n", - "torch.backends.cudnn.benchmark = False" - ], - "metadata": { - "id": "nVJIsdFOwXX2" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Download and Register dataset" - ], - "metadata": { - "id": "FhcF4opozT8c" - } - }, - { - "cell_type": "code", - "source": [ - "from roboflow import Roboflow\n", - "rf = Roboflow(api_key=os.getenv(\"ROBOFLOW_API_KEY\"))\n", - "project = rf.workspace(\"easyhyeon\").project(\"trash-sea\")\n", - "dataset = project.version(10).download(\"yolov5\")" - ], - "metadata": { - "id": "BcSFy0iewaJM" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "ENTITY = \"ml-colabs\"\n", - "SWEEP_NUM_RUNS = 100\n", - "WANDB_PROJECT_NAME = \"fconn-yolo-nas\"\n", - "DATASET_PATH = \"/content/trash-sea-10\"" - ], - "metadata": { - "id": "QIjvNsTowbKO" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Define Sweep Configuration and functions" - ], - "metadata": { - "id": "VO0p4iM0zW1N" - } - }, - { - "cell_type": "code", - "source": [ - "sweep_configuration = {\n", - " \"name\": WANDB_EXP_NAME,\n", - " \"metric\": {\"name\": \"Valid_mAP@0.50\", \"goal\": \"maximize\"},\n", - " \"method\": \"bayes\",\n", - " \"parameters\": {\n", - " \"batch_size\": {\"values\": [16, 24, 32]},\n", - " \"optimizer\": {\"values\": [\"Adam\", \"SGD\", \"RMSProp\", \"AdamW\"]},\n", - " \"ema_decay\": {\"min\":0.5, \"max\":0.9},\n", - " \"ema_decay_type\": {\"values\": [\"constant\", \"threshold\"]},\n", - " \"cosine_lr_ratio\": {\"min\": 0.01, \"max\": 0.4},\n", - " \"iou_loss_weight\": {\"min\": 0.25, \"max\": 2.0},\n", - " \"dfl_loss_weight\": {\"min\": 0.25, \"max\": 2.0},\n", - " \"classification_loss_weight\": {\"min\": 0.25, \"max\": 2.0},\n", - " \"model_flavor\": {\"values\": [\"yolo_nas_s\", \"yolo_nas_m\", \"yolo_nas_l\"]},\n", - " \"weight_decay\": {\"min\": 0.0001, \"max\": 0.01},\n", - " },\n", - "}" - ], - "metadata": { - "id": "8kMUNQ6HwoFa" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "def main_call():\n", - "\n", - " CHECKPOINT_DIR = 'checkpoints'\n", - "\n", - " wandb.init(\n", - " project=WANDB_PROJECT_NAME,\n", - " entity=ENTITY,\n", - " resume=\"allow\",\n", - " save_code=True,\n", - " id=WANDB_EXP_NAME\n", - " )\n", - "\n", - " config = wandb.config\n", - "\n", - " dataset_params = {\n", - " 'data_dir':DATASET_PATH,\n", - " 'train_images_dir':'train/images',\n", - " 'train_labels_dir':'train/labels',\n", - " 'val_images_dir':'valid/images',\n", - " 'val_labels_dir':'valid/labels',\n", - " 'test_images_dir':'test/images',\n", - " 'test_labels_dir':'test/labels',\n", - " 'classes': [\"Buoy\", \"Can\", \"Paper\", \"Plastic Bag\", \"Plastic Bottle\"]\n", - " }\n", - "\n", - " train_data = coco_detection_yolo_format_train(\n", - " dataset_params={\n", - " 'data_dir': dataset_params['data_dir'],\n", - " 'images_dir': dataset_params['train_images_dir'],\n", - " 'labels_dir': dataset_params['train_labels_dir'],\n", - " 'classes': dataset_params['classes'],\n", - " },\n", - " dataloader_params={\n", - " 'batch_size':config[\"batch_size\"],\n", - " 'num_workers':4\n", - " }\n", - " )\n", - "\n", - " val_data = coco_detection_yolo_format_val(\n", - " dataset_params={\n", - " 'data_dir': dataset_params['data_dir'],\n", - " 'images_dir': dataset_params['val_images_dir'],\n", - " 'labels_dir': dataset_params['val_labels_dir'],\n", - " 'classes': dataset_params['classes'],\n", - " },\n", - " dataloader_params={\n", - " 'batch_size':config[\"batch_size\"],\n", - " 'num_workers':4\n", - " }\n", - " )\n", - "\n", - " test_data = coco_detection_yolo_format_val(\n", - " dataset_params={\n", - " 'data_dir': dataset_params['data_dir'],\n", - " 'images_dir': dataset_params['test_images_dir'],\n", - " 'labels_dir': dataset_params['test_labels_dir'],\n", - " 'classes': dataset_params['classes'],\n", - " },\n", - " dataloader_params={\n", - " 'batch_size':config[\"batch_size\"],\n", - " 'num_workers':4\n", - " }\n", - " )\n", - "\n", - " train_data.dataset.transforms = train_data.dataset.transforms[1:]\n", - "\n", - " model = models.get(\n", - " config[\"model_flavor\"],\n", - " num_classes=len(dataset_params['classes']),\n", - " pretrained_weights=\"coco\"\n", - " )\n", - "\n", - " train_params = {\n", - " 'silent_mode': False,\n", - " \"average_best_models\":True,\n", - " \"warmup_mode\": \"linear_epoch_step\",\n", - " \"warmup_initial_lr\": 1e-6,\n", - " \"lr_warmup_epochs\": 3,\n", - " \"initial_lr\": 1e-3,\n", - " \"lr_mode\": \"cosine\",\n", - " \"cosine_final_lr_ratio\": config[\"cosine_lr_ratio\"],\n", - " \"optimizer\": config[\"optimizer\"],\n", - " \"optimizer_params\": {\n", - " \"weight_decay\": config[\"weight_decay\"]\n", - " },\n", - " \"zero_weight_decay_on_bias_and_bn\": True,\n", - " \"ema\": True,\n", - " \"ema_params\": {\n", - " \"decay\": config[\"ema_decay\"],\n", - " \"decay_type\": config[\"ema_decay_type\"]\n", - " },\n", - " \"max_epochs\": 5,\n", - " \"mixed_precision\": False,\n", - " \"loss\": PPYoloELoss(\n", - " use_static_assigner=False,\n", - " num_classes=len(dataset_params['classes']),\n", - " reg_max=16,\n", - " iou_loss_weight=config[\"iou_loss_weight\"],\n", - " dfl_loss_weight=config[\"dfl_loss_weight\"],\n", - " classification_loss_weight=config[\"classification_loss_weight\"]\n", - " ),\n", - " \"valid_metrics_list\": [\n", - " DetectionMetrics_050(\n", - " score_thres=0.1,\n", - " top_k_predictions=300,\n", - " num_cls=len(dataset_params['classes']),\n", - " normalize_targets=True,\n", - " post_prediction_callback=PPYoloEPostPredictionCallback(\n", - " score_threshold=0.01,\n", - " nms_top_k=1000,\n", - " max_predictions=300,\n", - " nms_threshold=0.7\n", - " )\n", - " )\n", - " ],\n", - " \"metric_to_watch\": 'mAP@0.50',\n", - " \"sg_logger\": \"wandb_sg_logger\",\n", - " \"sg_logger_params\": {\n", - " \"project_name\": WANDB_PROJECT_NAME,\n", - " \"save_checkpoints_remote\": True,\n", - " \"save_tensorboard_remote\": True,\n", - " \"save_logs_remote\": True,\n", - " \"entity\": ENTITY\n", - " }\n", - " }\n", - "\n", - " trainer = Trainer(\n", - " experiment_name=WANDB_EXP_NAME,\n", - " ckpt_root_dir=CHECKPOINT_DIR\n", - " )\n", - "\n", - " trainer.train(\n", - " model=model,\n", - " training_params=train_params,\n", - " train_loader=train_data,\n", - " valid_loader=val_data\n", - " )\n", - "\n", - " wandb.finish()" - ], - "metadata": { - "id": "nnZzxVmEwb-U" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Execute Sweep" - ], - "metadata": { - "id": "3Xp0UU73zajw" - } - }, - { - "cell_type": "code", - "source": [ - "sweep_id = wandb.sweep(\n", - " sweep=sweep_configuration,\n", - " project=\"yolo-nas-sweep\"\n", - ")\n", - "\n", - "wandb.agent(sweep_id, function=main_call, count=SWEEP_NUM_RUNS)" - ], - "metadata": { - "id": "9r6LrkhQwxdA" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Installation and Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!sudo apt install libcairo2-dev pkg-config python3-dev -qq\n", + "!pip install roboflow pycairo wandb sweeps -qqq\n", + "!pip install super_gradients" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import glob\n", + "import torch\n", + "import wandb\n", + "import warnings\n", + "import pandas as pd\n", + "\n", + "from google.colab import userdata\n", + "from torchvision.io import read_image\n", + "from torch.utils.data import DataLoader\n", + "from IPython.display import clear_output\n", + "\n", + "from super_gradients.training import models, Trainer, dataloaders\n", + "from super_gradients.training.losses import PPYoloELoss\n", + "from super_gradients.training.metrics import DetectionMetrics_050\n", + "from super_gradients.training.models.detection_models.pp_yolo_e import PPYoloEPostPredictionCallback\n", + "from super_gradients.training.dataloaders.dataloaders import coco_detection_yolo_format_train, coco_detection_yolo_format_val\n", + "\n", + "os.environ[\"WANDB_API_KEY\"] = userdata.get('wandb')\n", + "os.environ[\"ROBOFLOW_API_KEY\"] = userdata.get('roboflow')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Definitions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "seed = 42\n", + "torch.manual_seed(seed)\n", + "\n", + "if torch.cuda.is_available():\n", + " torch.cuda.manual_seed(seed)\n", + " torch.cuda.manual_seed_all(seed)\n", + "\n", + "torch.backends.cudnn.deterministic = True\n", + "torch.backends.cudnn.benchmark = False" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Download and Register dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from roboflow import Roboflow\n", + "rf = Roboflow(api_key=os.getenv(\"ROBOFLOW_API_KEY\"))\n", + "project = rf.workspace(\"easyhyeon\").project(\"trash-sea\")\n", + "dataset = project.version(10).download(\"yolov5\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ENTITY = \"ml-colabs\"\n", + "SWEEP_NUM_RUNS = 100\n", + "WANDB_PROJECT_NAME = \"fconn-yolo-nas\"\n", + "DATASET_PATH = \"/content/trash-sea-10\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Define Sweep Configuration and functions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sweep_configuration = {\n", + " \"name\": WANDB_EXP_NAME,\n", + " \"metric\": {\"name\": \"Valid_mAP@0.50\", \"goal\": \"maximize\"},\n", + " \"method\": \"bayes\",\n", + " \"parameters\": {\n", + " \"batch_size\": {\"values\": [16, 24, 32]},\n", + " \"optimizer\": {\"values\": [\"Adam\", \"SGD\", \"RMSProp\", \"AdamW\"]},\n", + " \"ema_decay\": {\"min\":0.5, \"max\":0.9},\n", + " \"ema_decay_type\": {\"values\": [\"constant\", \"threshold\"]},\n", + " \"cosine_lr_ratio\": {\"min\": 0.01, \"max\": 0.4},\n", + " \"iou_loss_weight\": {\"min\": 0.25, \"max\": 2.0},\n", + " \"dfl_loss_weight\": {\"min\": 0.25, \"max\": 2.0},\n", + " \"classification_loss_weight\": {\"min\": 0.25, \"max\": 2.0},\n", + " \"model_flavor\": {\"values\": [\"yolo_nas_s\", \"yolo_nas_m\", \"yolo_nas_l\"]},\n", + " \"weight_decay\": {\"min\": 0.0001, \"max\": 0.01},\n", + " },\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def main_call():\n", + "\n", + " CHECKPOINT_DIR = 'checkpoints'\n", + "\n", + " wandb.init(\n", + " project=WANDB_PROJECT_NAME,\n", + " entity=ENTITY,\n", + " resume=\"allow\",\n", + " save_code=True,\n", + " id=WANDB_EXP_NAME\n", + " )\n", + "\n", + " config = wandb.config\n", + "\n", + " dataset_params = {\n", + " 'data_dir':DATASET_PATH,\n", + " 'train_images_dir':'train/images',\n", + " 'train_labels_dir':'train/labels',\n", + " 'val_images_dir':'valid/images',\n", + " 'val_labels_dir':'valid/labels',\n", + " 'test_images_dir':'test/images',\n", + " 'test_labels_dir':'test/labels',\n", + " 'classes': [\"Buoy\", \"Can\", \"Paper\", \"Plastic Bag\", \"Plastic Bottle\"]\n", + " }\n", + "\n", + " train_data = coco_detection_yolo_format_train(\n", + " dataset_params={\n", + " 'data_dir': dataset_params['data_dir'],\n", + " 'images_dir': dataset_params['train_images_dir'],\n", + " 'labels_dir': dataset_params['train_labels_dir'],\n", + " 'classes': dataset_params['classes'],\n", + " },\n", + " dataloader_params={\n", + " 'batch_size':config[\"batch_size\"],\n", + " 'num_workers':4\n", + " }\n", + " )\n", + "\n", + " val_data = coco_detection_yolo_format_val(\n", + " dataset_params={\n", + " 'data_dir': dataset_params['data_dir'],\n", + " 'images_dir': dataset_params['val_images_dir'],\n", + " 'labels_dir': dataset_params['val_labels_dir'],\n", + " 'classes': dataset_params['classes'],\n", + " },\n", + " dataloader_params={\n", + " 'batch_size':config[\"batch_size\"],\n", + " 'num_workers':4\n", + " }\n", + " )\n", + "\n", + " test_data = coco_detection_yolo_format_val(\n", + " dataset_params={\n", + " 'data_dir': dataset_params['data_dir'],\n", + " 'images_dir': dataset_params['test_images_dir'],\n", + " 'labels_dir': dataset_params['test_labels_dir'],\n", + " 'classes': dataset_params['classes'],\n", + " },\n", + " dataloader_params={\n", + " 'batch_size':config[\"batch_size\"],\n", + " 'num_workers':4\n", + " }\n", + " )\n", + "\n", + " train_data.dataset.transforms = train_data.dataset.transforms[1:]\n", + "\n", + " model = models.get(\n", + " config[\"model_flavor\"],\n", + " num_classes=len(dataset_params['classes']),\n", + " pretrained_weights=\"coco\"\n", + " )\n", + "\n", + " train_params = {\n", + " 'silent_mode': False,\n", + " \"average_best_models\":True,\n", + " \"warmup_mode\": \"linear_epoch_step\",\n", + " \"warmup_initial_lr\": 1e-6,\n", + " \"lr_warmup_epochs\": 3,\n", + " \"initial_lr\": 1e-3,\n", + " \"lr_mode\": \"cosine\",\n", + " \"cosine_final_lr_ratio\": config[\"cosine_lr_ratio\"],\n", + " \"optimizer\": config[\"optimizer\"],\n", + " \"optimizer_params\": {\n", + " \"weight_decay\": config[\"weight_decay\"]\n", + " },\n", + " \"zero_weight_decay_on_bias_and_bn\": True,\n", + " \"ema\": True,\n", + " \"ema_params\": {\n", + " \"decay\": config[\"ema_decay\"],\n", + " \"decay_type\": config[\"ema_decay_type\"]\n", + " },\n", + " \"max_epochs\": 5,\n", + " \"mixed_precision\": False,\n", + " \"loss\": PPYoloELoss(\n", + " use_static_assigner=False,\n", + " num_classes=len(dataset_params['classes']),\n", + " reg_max=16,\n", + " iou_loss_weight=config[\"iou_loss_weight\"],\n", + " dfl_loss_weight=config[\"dfl_loss_weight\"],\n", + " classification_loss_weight=config[\"classification_loss_weight\"]\n", + " ),\n", + " \"valid_metrics_list\": [\n", + " DetectionMetrics_050(\n", + " score_thres=0.1,\n", + " top_k_predictions=300,\n", + " num_cls=len(dataset_params['classes']),\n", + " normalize_targets=True,\n", + " post_prediction_callback=PPYoloEPostPredictionCallback(\n", + " score_threshold=0.01,\n", + " nms_top_k=1000,\n", + " max_predictions=300,\n", + " nms_threshold=0.7\n", + " )\n", + " )\n", + " ],\n", + " \"metric_to_watch\": 'mAP@0.50',\n", + " \"sg_logger\": \"wandb_sg_logger\",\n", + " \"sg_logger_params\": {\n", + " \"project_name\": WANDB_PROJECT_NAME,\n", + " \"save_checkpoints_remote\": True,\n", + " \"save_tensorboard_remote\": True,\n", + " \"save_logs_remote\": True,\n", + " \"entity\": ENTITY\n", + " }\n", + " }\n", + "\n", + " trainer = Trainer(\n", + " experiment_name=WANDB_EXP_NAME,\n", + " ckpt_root_dir=CHECKPOINT_DIR\n", + " )\n", + "\n", + " trainer.train(\n", + " model=model,\n", + " training_params=train_params,\n", + " train_loader=train_data,\n", + " valid_loader=val_data\n", + " )\n", + "\n", + " wandb.finish()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Execute Sweep" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sweep_id = wandb.sweep(\n", + " sweep=sweep_configuration,\n", + " project=\"yolo-nas-sweep\"\n", + ")\n", + "\n", + "wandb.agent(sweep_id, function=main_call, count=SWEEP_NUM_RUNS)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/colabs/tables/W&B_Tables_Quickstart.ipynb b/colabs/tables/W&B_Tables_Quickstart.ipynb index 185c4766..577562d9 100644 --- a/colabs/tables/W&B_Tables_Quickstart.ipynb +++ b/colabs/tables/W&B_Tables_Quickstart.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb b/colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb index d45de2a0..94ee7a9c 100644 --- a/colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb +++ b/colabs/tensorflow/Hyperparameter_Optimization_in_TensorFlow_using_W&B_Sweeps.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/tensorflow/Simple_TensorFlow_Integration.ipynb b/colabs/tensorflow/Simple_TensorFlow_Integration.ipynb index a42a5a0c..bd6ed614 100644 --- a/colabs/tensorflow/Simple_TensorFlow_Integration.ipynb +++ b/colabs/tensorflow/Simple_TensorFlow_Integration.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/ultralytics/00_inference.ipynb b/colabs/ultralytics/00_inference.ipynb index 684762db..48b1724f 100644 --- a/colabs/ultralytics/00_inference.ipynb +++ b/colabs/ultralytics/00_inference.ipynb @@ -4,13 +4,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 🔥🔥 Explore Predictions from Ultralytics models using Weights & Biases 🪄🐝\n", - "\n", - "\n", - "\n", - "This notebook demonstrates a typical workflow of using an [Ultralytics](https://docs.ultralytics.com/modes/predict/) model for inference and visualizing the results using [Weights & Biases](https://wandb.ai/site).\n", - "\n", - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://wandb.me/ultralytics-inference)" + "\"Open\n", + "" ] }, { @@ -147,8 +142,15 @@ } ], "metadata": { - "language_info": { - "name": "python" + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } }, "nbformat": 4, diff --git a/colabs/ultralytics/01_train_val.ipynb b/colabs/ultralytics/01_train_val.ipynb index 463d9559..a80d7804 100644 --- a/colabs/ultralytics/01_train_val.ipynb +++ b/colabs/ultralytics/01_train_val.ipynb @@ -4,13 +4,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 🔥🔥 Explore Predictions from Ultralytics models using Weights & Biases 🪄🐝\n", - "\n", - "\n", - "\n", - "This notebook demonstrates a typical workflow of using an [Ultralytics](https://docs.ultralytics.com/modes/predict/) model for training, fine-tuning, and validation and performing experiment tracking, model-checkpointing, and visualization of the model's performance using [Weights & Biases](https://wandb.ai/site).\n", - "\n", - "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](http://wandb.me/ultralytics-train)" + "\"Open\n", + "" ] }, { @@ -136,8 +131,15 @@ } ], "metadata": { - "language_info": { - "name": "python" + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" } }, "nbformat": 4, diff --git a/colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb b/colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb index af130144..bac864d6 100644 --- a/colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb +++ b/colabs/wandb-artifacts/Pipeline_Versioning_with_W&B_Artifacts.ipynb @@ -1,7 +1,6 @@ { "cells": [ { - "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ diff --git a/colabs/wandb-model-registry/New_Model_Logging_in_W&B.ipynb b/colabs/wandb-model-registry/New_Model_Logging_in_W&B.ipynb index 449e9d7e..fd6920a8 100644 --- a/colabs/wandb-model-registry/New_Model_Logging_in_W&B.ipynb +++ b/colabs/wandb-model-registry/New_Model_Logging_in_W&B.ipynb @@ -1,416 +1,381 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "gpuType": "T4", - "authorship_tag": "ABX9TyOlvkZjseUluPYEdqGH1TjK", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Open\n" + ] }, - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "view-in-github", - "colab_type": "text" - }, - "source": [ - "\"Open" - ] - }, - { - "cell_type": "markdown", - "source": [ - "# Logging and Registering Models in W&B\n", - "It's never been easier to log your model checkpoints, keep track of the best ones, and maintain lineage of runs and results!\n", - "\n", - "W&B is introducing a few convenience methods to make logging models and linking them to the registry simple:\n", - "- `log_model`\n", - "- `use_model`\n", - "- `link_model`" - ], - "metadata": { - "id": "saeQIrcF155i" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Imports" - ], - "metadata": { - "id": "_ewwM5T_OWTr" - } - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "BbkuyVKRwCbi" - }, - "outputs": [], - "source": [ - "!pip install -qqq wandb einops" - ] - }, - { - "cell_type": "code", - "source": [ - "import torch\n", - "from torch import nn\n", - "from einops import rearrange, repeat\n", - "from einops.layers.torch import Rearrange\n", - "import torch\n", - "from torch.utils.data import DataLoader, Dataset\n", - "from torchvision import transforms\n", - "import torch\n", - "from torch import nn\n", - "from torch.optim import Adam\n", - "from torch.utils.data import DataLoader\n", - "from torchvision import datasets, transforms\n", - "\n", - "import wandb" - ], - "metadata": { - "id": "6OsMa64TOacf" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "## Log in to W&B\n", - "- You can explicitly login using `wandb login` or `wandb.login()` (See below)\n", - "- Alternatively you can set environment variables. There are several env variables which you can set to change the behavior of W&B logging. The most important are:\n", - " - `WANDB_API_KEY` - find this in your \"Settings\" section under your profile\n", - " - `WANDB_BASE_URL` - this is the url of the W&B server\n", - "- Find your API Token in \"Profile\" -> \"Setttings\" in the W&B App\n", - "\n", - "![api_token](https://drive.google.com/uc?export=view&id=1Xn7hnn0rfPu_EW0A_-32oCXqDmpA0-kx)" - ], - "metadata": { - "id": "m3Q_QG14Ovvz" - } - }, - { - "cell_type": "code", - "source": [ - "wandb.login()" - ], - "metadata": { - "id": "5-r3O3gFwfcr" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Define the Model and Dataset\n", - "This is a simple implementation of a Vision Transformer (ViT) and utilizes a random dataset for training.\n", - "- Credit to https://github.com/lucidrains/vit-pytorch" - ], - "metadata": { - "id": "ZuL3yZ17qIGE" - } - }, - { - "cell_type": "markdown", - "source": [ - "## Define some config for the model and dataset" - ], - "metadata": { - "id": "WE4j_CI7Tt9O" - } - }, - { - "cell_type": "code", - "source": [ - "# Define the number of samples, classes, and image size\n", - "num_samples = 100\n", - "num_classes = 10\n", - "image_size = 256\n", - "batch_size = 32" - ], - "metadata": { - "id": "D7sVekyVTlxN" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "source": [ - "# helpers\n", - "def pair(t):\n", - " return t if isinstance(t, tuple) else (t, t)\n", - "\n", - "# classes\n", - "class FeedForward(nn.Module):\n", - " def __init__(self, dim, hidden_dim, dropout = 0.):\n", - " super().__init__()\n", - " self.net = nn.Sequential(\n", - " nn.LayerNorm(dim),\n", - " nn.Linear(dim, hidden_dim),\n", - " nn.GELU(),\n", - " nn.Dropout(dropout),\n", - " nn.Linear(hidden_dim, dim),\n", - " nn.Dropout(dropout)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " return self.net(x)\n", - "\n", - "class Attention(nn.Module):\n", - " def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):\n", - " super().__init__()\n", - " inner_dim = dim_head * heads\n", - " project_out = not (heads == 1 and dim_head == dim)\n", - "\n", - " self.heads = heads\n", - " self.scale = dim_head ** -0.5\n", - "\n", - " self.norm = nn.LayerNorm(dim)\n", - "\n", - " self.attend = nn.Softmax(dim = -1)\n", - " self.dropout = nn.Dropout(dropout)\n", - "\n", - " self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n", - "\n", - " self.to_out = nn.Sequential(\n", - " nn.Linear(inner_dim, dim),\n", - " nn.Dropout(dropout)\n", - " ) if project_out else nn.Identity()\n", - "\n", - " def forward(self, x):\n", - " x = self.norm(x)\n", - "\n", - " qkv = self.to_qkv(x).chunk(3, dim = -1)\n", - " q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)\n", - "\n", - " dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n", - "\n", - " attn = self.attend(dots)\n", - " attn = self.dropout(attn)\n", - "\n", - " out = torch.matmul(attn, v)\n", - " out = rearrange(out, 'b h n d -> b n (h d)')\n", - " return self.to_out(out)\n", - "\n", - "class Transformer(nn.Module):\n", - " def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):\n", - " super().__init__()\n", - " self.norm = nn.LayerNorm(dim)\n", - " self.layers = nn.ModuleList([])\n", - " for _ in range(depth):\n", - " self.layers.append(nn.ModuleList([\n", - " Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout),\n", - " FeedForward(dim, mlp_dim, dropout = dropout)\n", - " ]))\n", - "\n", - " def forward(self, x):\n", - " for attn, ff in self.layers:\n", - " x = attn(x) + x\n", - " x = ff(x) + x\n", - "\n", - " return self.norm(x)\n", - "\n", - "class ViT(nn.Module):\n", - " def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.):\n", - " super().__init__()\n", - " image_height, image_width = pair(image_size)\n", - " patch_height, patch_width = pair(patch_size)\n", - "\n", - " assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'\n", - "\n", - " num_patches = (image_height // patch_height) * (image_width // patch_width)\n", - " patch_dim = channels * patch_height * patch_width\n", - " assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'\n", - "\n", - " self.to_patch_embedding = nn.Sequential(\n", - " Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width),\n", - " nn.LayerNorm(patch_dim),\n", - " nn.Linear(patch_dim, dim),\n", - " nn.LayerNorm(dim),\n", - " )\n", - "\n", - " self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n", - " self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n", - " self.dropout = nn.Dropout(emb_dropout)\n", - "\n", - " self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)\n", - "\n", - " self.pool = pool\n", - " self.to_latent = nn.Identity()\n", - "\n", - " self.mlp_head = nn.Linear(dim, num_classes)\n", - "\n", - " def forward(self, img):\n", - " x = self.to_patch_embedding(img)\n", - " b, n, _ = x.shape\n", - "\n", - " cls_tokens = repeat(self.cls_token, '1 1 d -> b 1 d', b = b)\n", - " x = torch.cat((cls_tokens, x), dim=1)\n", - " x += self.pos_embedding[:, :(n + 1)]\n", - " x = self.dropout(x)\n", - "\n", - " x = self.transformer(x)\n", - "\n", - " x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]\n", - "\n", - " x = self.to_latent(x)\n", - " return self.mlp_head(x)\n", - "\n", - "\n", - "# Define a custom dataset\n", - "class RandomImageDataset(Dataset):\n", - " def __init__(self, num_samples, num_classes, image_size):\n", - " self.num_samples = num_samples\n", - " self.num_classes = num_classes\n", - " self.image_size = image_size\n", - "\n", - " def __len__(self):\n", - " return self.num_samples\n", - "\n", - " def __getitem__(self, idx):\n", - " # Generate a random image tensor\n", - " image = torch.randn(3, self.image_size, self.image_size) # 3 channels, image_size x image_size\n", - " # Generate a random label\n", - " label = torch.randint(0, self.num_classes, (1,)).item()\n", - " return image, label\n", - "\n", - "\n", - "\n", - "# Create the dataset\n", - "dataset = RandomImageDataset(num_samples=num_samples, num_classes=num_classes, image_size=image_size)\n", - "\n", - "# Create a DataLoader\n", - "dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)" - ], - "metadata": { - "id": "TntUlCT2ppV2" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Log Model Checkpoints to W&B with 1 Line!\n", - "\n", - "Use the `log_model` method to log a model artifact containing the contents inside the ‘path’ to an a run. It also marks it as an output to the run. You can see the full lineage graph of the model artifact by accessing the [lineage](https://docs.wandb.ai/guides/artifacts/explore-and-traverse-an-artifact-graph#docusaurus_skipToContent_fallback) tab inside the Artifacts view.\n", - "\n", - "`log_model()` takes as input:\n", - "\n", - "- `path`: A path to the model file(s), which can be a local file (of the form `/local/directory/file.txt`), directory (of the form `/local/directory`), or reference path to S3 (`s3://bucket/path`).\n", - "- `name`: An optional name for the model artifact the files will be logged to. Note that if no name is specified, This will default to the basename of the input path prepended with the run ID.\n", - "- `aliases`: An optional list of aliases, which can be thought of as semantic ‘nicknames’ or identifiers for a model version. For example, if this model yielded the best accuracy, you might add the alias ‘highest-accuracy’ or ‘best’." - ], - "metadata": { - "id": "ZkomIMW6VNky" - } - }, - { - "cell_type": "code", - "source": [ - "run = wandb.init(project=\"new_model_logging\",\n", - " job_type=\"training\")\n", - "\n", - "v = ViT(\n", - " image_size = image_size,\n", - " patch_size = 32,\n", - " num_classes = num_classes,\n", - " dim = 128,\n", - " depth = 3,\n", - " heads = 2,\n", - " mlp_dim = 256,\n", - " dropout = 0.1,\n", - " emb_dropout = 0.1\n", - ")\n", - "\n", - "# Define the loss function and optimizer\n", - "criterion = nn.CrossEntropyLoss()\n", - "optimizer = Adam(v.parameters(), lr=0.003)\n", - "\n", - "# Training loop\n", - "best_accuracy = 0\n", - "for epoch in range(5): # number of epochs\n", - " for images, labels in dataloader:\n", - " # Forward pass\n", - " preds = v(images)\n", - " loss = criterion(preds, labels)\n", - "\n", - " # Backward pass and optimization\n", - " optimizer.zero_grad()\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " wandb.log({\"train/loss\": loss})\n", - "\n", - " # Model evaluation after each epoch (using a validation set)\n", - " # Here you would write your validation loop and calculate accuracy\n", - " val_accuracy = 0.5 # Assume this is the validation accuracy you compute\n", - " model_path = 'model_vit.pth'\n", - " torch.save(v.state_dict(), model_path)\n", - "\n", - " # Check if this is the best model so far\n", - " if val_accuracy > best_accuracy:\n", - " best_accuracy = val_accuracy\n", - " # Log the model to your W&B run\n", - " wandb.log_model(name=f\"model_vit-{wandb.run.id}\", path=model_path, aliases=[\"best\", f\"epoch_{epoch}\"])\n", - " else:\n", - " wandb.log_model(name=f\"model_vit-{wandb.run.id}\", path=model_path, aliases=[f\"epoch_{epoch}\"])" - ], - "metadata": { - "id": "48khrxaKt-rm" - }, - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "source": [ - "# Link Your Best Models to the Model Registry\n", - "You can bookmark your best model checkpoints and centralize them across your team. The Model Registry allows you can organize your best models by task, manage model lifecycle, facilitate easy tracking and auditing throughout the ML lifecyle, and automate downstream actions with webhooks or jobs. You can this via api through `link_model()`, which takes as input:\n", - "\n", - "- `path`: A path to the model file(s), which can be a local file (of the form `/local/directory/file.txt`), directory (of the form `/local/directory`), or reference path to S3 (`s3://bucket/path`).\n", - "- `registered_model_name`: the name of the Registered Model - a collection of linked model versions in the Model Registry, typically representing a team’s ML task - that the model should be linked to. If no Registered Model with the given name exists, a new one will be created with this name.\n", - "- `name`: An **optional** name for the model artifact the files will be logged to. Note that if no name is specified, This will default to the basename of the input path prepended with the run ID.\n", - "- `aliases`: An **optional** list of aliases, which can be thought of as semantic ‘nicknames’ or identifiers for a linked model version. For example, since this model is being linked, or published, to the Model Registry, you might add an alias “staging” or “QA”." - ], - "metadata": { - "id": "km2nU7IUVg63" - } - }, - { - "cell_type": "code", - "source": [ - "# Link the best model to the W&B Model Registry (after all epochs are finished)\n", - "artifact_name = f\"model_vit-{wandb.run.id}:best\"\n", - "best_model_path = wandb.use_model(artifact_name)\n", - "\n", - "# Link the best model to the registry\n", - "wandb.link_model(path=best_model_path,\n", - " registered_model_name=\"Industrial ViT\",\n", - " aliases=[\"staging\"])\n", - "run.finish()" - ], - "metadata": { - "id": "ckH-yexDVgJI" - }, - "execution_count": null, - "outputs": [] - } - ] -} \ No newline at end of file + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Logging and Registering Models in W&B\n", + "It's never been easier to log your model checkpoints, keep track of the best ones, and maintain lineage of runs and results!\n", + "\n", + "W&B is introducing a few convenience methods to make logging models and linking them to the registry simple:\n", + "- `log_model`\n", + "- `use_model`\n", + "- `link_model`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Imports" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install -qqq wandb einops" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from torch import nn\n", + "from einops import rearrange, repeat\n", + "from einops.layers.torch import Rearrange\n", + "import torch\n", + "from torch.utils.data import DataLoader, Dataset\n", + "from torchvision import transforms\n", + "import torch\n", + "from torch import nn\n", + "from torch.optim import Adam\n", + "from torch.utils.data import DataLoader\n", + "from torchvision import datasets, transforms\n", + "\n", + "import wandb" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Log in to W&B\n", + "- You can explicitly login using `wandb login` or `wandb.login()` (See below)\n", + "- Alternatively you can set environment variables. There are several env variables which you can set to change the behavior of W&B logging. The most important are:\n", + " - `WANDB_API_KEY` - find this in your \"Settings\" section under your profile\n", + " - `WANDB_BASE_URL` - this is the url of the W&B server\n", + "- Find your API Token in \"Profile\" -> \"Setttings\" in the W&B App\n", + "\n", + "![api_token](https://drive.google.com/uc?export=view&id=1Xn7hnn0rfPu_EW0A_-32oCXqDmpA0-kx)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "wandb.login()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Define the Model and Dataset\n", + "This is a simple implementation of a Vision Transformer (ViT) and utilizes a random dataset for training.\n", + "- Credit to https://github.com/lucidrains/vit-pytorch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Define some config for the model and dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define the number of samples, classes, and image size\n", + "num_samples = 100\n", + "num_classes = 10\n", + "image_size = 256\n", + "batch_size = 32" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# helpers\n", + "def pair(t):\n", + " return t if isinstance(t, tuple) else (t, t)\n", + "\n", + "# classes\n", + "class FeedForward(nn.Module):\n", + " def __init__(self, dim, hidden_dim, dropout = 0.):\n", + " super().__init__()\n", + " self.net = nn.Sequential(\n", + " nn.LayerNorm(dim),\n", + " nn.Linear(dim, hidden_dim),\n", + " nn.GELU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(hidden_dim, dim),\n", + " nn.Dropout(dropout)\n", + " )\n", + "\n", + " def forward(self, x):\n", + " return self.net(x)\n", + "\n", + "class Attention(nn.Module):\n", + " def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):\n", + " super().__init__()\n", + " inner_dim = dim_head * heads\n", + " project_out = not (heads == 1 and dim_head == dim)\n", + "\n", + " self.heads = heads\n", + " self.scale = dim_head ** -0.5\n", + "\n", + " self.norm = nn.LayerNorm(dim)\n", + "\n", + " self.attend = nn.Softmax(dim = -1)\n", + " self.dropout = nn.Dropout(dropout)\n", + "\n", + " self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)\n", + "\n", + " self.to_out = nn.Sequential(\n", + " nn.Linear(inner_dim, dim),\n", + " nn.Dropout(dropout)\n", + " ) if project_out else nn.Identity()\n", + "\n", + " def forward(self, x):\n", + " x = self.norm(x)\n", + "\n", + " qkv = self.to_qkv(x).chunk(3, dim = -1)\n", + " q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = self.heads), qkv)\n", + "\n", + " dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale\n", + "\n", + " attn = self.attend(dots)\n", + " attn = self.dropout(attn)\n", + "\n", + " out = torch.matmul(attn, v)\n", + " out = rearrange(out, 'b h n d -> b n (h d)')\n", + " return self.to_out(out)\n", + "\n", + "class Transformer(nn.Module):\n", + " def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):\n", + " super().__init__()\n", + " self.norm = nn.LayerNorm(dim)\n", + " self.layers = nn.ModuleList([])\n", + " for _ in range(depth):\n", + " self.layers.append(nn.ModuleList([\n", + " Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout),\n", + " FeedForward(dim, mlp_dim, dropout = dropout)\n", + " ]))\n", + "\n", + " def forward(self, x):\n", + " for attn, ff in self.layers:\n", + " x = attn(x) + x\n", + " x = ff(x) + x\n", + "\n", + " return self.norm(x)\n", + "\n", + "class ViT(nn.Module):\n", + " def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads, mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.):\n", + " super().__init__()\n", + " image_height, image_width = pair(image_size)\n", + " patch_height, patch_width = pair(patch_size)\n", + "\n", + " assert image_height % patch_height == 0 and image_width % patch_width == 0, 'Image dimensions must be divisible by the patch size.'\n", + "\n", + " num_patches = (image_height // patch_height) * (image_width // patch_width)\n", + " patch_dim = channels * patch_height * patch_width\n", + " assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'\n", + "\n", + " self.to_patch_embedding = nn.Sequential(\n", + " Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_height, p2 = patch_width),\n", + " nn.LayerNorm(patch_dim),\n", + " nn.Linear(patch_dim, dim),\n", + " nn.LayerNorm(dim),\n", + " )\n", + "\n", + " self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim))\n", + " self.cls_token = nn.Parameter(torch.randn(1, 1, dim))\n", + " self.dropout = nn.Dropout(emb_dropout)\n", + "\n", + " self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)\n", + "\n", + " self.pool = pool\n", + " self.to_latent = nn.Identity()\n", + "\n", + " self.mlp_head = nn.Linear(dim, num_classes)\n", + "\n", + " def forward(self, img):\n", + " x = self.to_patch_embedding(img)\n", + " b, n, _ = x.shape\n", + "\n", + " cls_tokens = repeat(self.cls_token, '1 1 d -> b 1 d', b = b)\n", + " x = torch.cat((cls_tokens, x), dim=1)\n", + " x += self.pos_embedding[:, :(n + 1)]\n", + " x = self.dropout(x)\n", + "\n", + " x = self.transformer(x)\n", + "\n", + " x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]\n", + "\n", + " x = self.to_latent(x)\n", + " return self.mlp_head(x)\n", + "\n", + "\n", + "# Define a custom dataset\n", + "class RandomImageDataset(Dataset):\n", + " def __init__(self, num_samples, num_classes, image_size):\n", + " self.num_samples = num_samples\n", + " self.num_classes = num_classes\n", + " self.image_size = image_size\n", + "\n", + " def __len__(self):\n", + " return self.num_samples\n", + "\n", + " def __getitem__(self, idx):\n", + " # Generate a random image tensor\n", + " image = torch.randn(3, self.image_size, self.image_size) # 3 channels, image_size x image_size\n", + " # Generate a random label\n", + " label = torch.randint(0, self.num_classes, (1,)).item()\n", + " return image, label\n", + "\n", + "\n", + "\n", + "# Create the dataset\n", + "dataset = RandomImageDataset(num_samples=num_samples, num_classes=num_classes, image_size=image_size)\n", + "\n", + "# Create a DataLoader\n", + "dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Log Model Checkpoints to W&B with 1 Line!\n", + "\n", + "Use the `log_model` method to log a model artifact containing the contents inside the ‘path’ to an a run. It also marks it as an output to the run. You can see the full lineage graph of the model artifact by accessing the [lineage](https://docs.wandb.ai/guides/artifacts/explore-and-traverse-an-artifact-graph#docusaurus_skipToContent_fallback) tab inside the Artifacts view.\n", + "\n", + "`log_model()` takes as input:\n", + "\n", + "- `path`: A path to the model file(s), which can be a local file (of the form `/local/directory/file.txt`), directory (of the form `/local/directory`), or reference path to S3 (`s3://bucket/path`).\n", + "- `name`: An optional name for the model artifact the files will be logged to. Note that if no name is specified, This will default to the basename of the input path prepended with the run ID.\n", + "- `aliases`: An optional list of aliases, which can be thought of as semantic ‘nicknames’ or identifiers for a model version. For example, if this model yielded the best accuracy, you might add the alias ‘highest-accuracy’ or ‘best’." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run = wandb.init(project=\"new_model_logging\",\n", + " job_type=\"training\")\n", + "\n", + "v = ViT(\n", + " image_size = image_size,\n", + " patch_size = 32,\n", + " num_classes = num_classes,\n", + " dim = 128,\n", + " depth = 3,\n", + " heads = 2,\n", + " mlp_dim = 256,\n", + " dropout = 0.1,\n", + " emb_dropout = 0.1\n", + ")\n", + "\n", + "# Define the loss function and optimizer\n", + "criterion = nn.CrossEntropyLoss()\n", + "optimizer = Adam(v.parameters(), lr=0.003)\n", + "\n", + "# Training loop\n", + "best_accuracy = 0\n", + "for epoch in range(5): # number of epochs\n", + " for images, labels in dataloader:\n", + " # Forward pass\n", + " preds = v(images)\n", + " loss = criterion(preds, labels)\n", + "\n", + " # Backward pass and optimization\n", + " optimizer.zero_grad()\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " wandb.log({\"train/loss\": loss})\n", + "\n", + " # Model evaluation after each epoch (using a validation set)\n", + " # Here you would write your validation loop and calculate accuracy\n", + " val_accuracy = 0.5 # Assume this is the validation accuracy you compute\n", + " model_path = 'model_vit.pth'\n", + " torch.save(v.state_dict(), model_path)\n", + "\n", + " # Check if this is the best model so far\n", + " if val_accuracy > best_accuracy:\n", + " best_accuracy = val_accuracy\n", + " # Log the model to your W&B run\n", + " wandb.log_model(name=f\"model_vit-{wandb.run.id}\", path=model_path, aliases=[\"best\", f\"epoch_{epoch}\"])\n", + " else:\n", + " wandb.log_model(name=f\"model_vit-{wandb.run.id}\", path=model_path, aliases=[f\"epoch_{epoch}\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Link Your Best Models to the Model Registry\n", + "You can bookmark your best model checkpoints and centralize them across your team. The Model Registry allows you can organize your best models by task, manage model lifecycle, facilitate easy tracking and auditing throughout the ML lifecyle, and automate downstream actions with webhooks or jobs. You can this via api through `link_model()`, which takes as input:\n", + "\n", + "- `path`: A path to the model file(s), which can be a local file (of the form `/local/directory/file.txt`), directory (of the form `/local/directory`), or reference path to S3 (`s3://bucket/path`).\n", + "- `registered_model_name`: the name of the Registered Model - a collection of linked model versions in the Model Registry, typically representing a team’s ML task - that the model should be linked to. If no Registered Model with the given name exists, a new one will be created with this name.\n", + "- `name`: An **optional** name for the model artifact the files will be logged to. Note that if no name is specified, This will default to the basename of the input path prepended with the run ID.\n", + "- `aliases`: An **optional** list of aliases, which can be thought of as semantic ‘nicknames’ or identifiers for a linked model version. For example, since this model is being linked, or published, to the Model Registry, you might add an alias “staging” or “QA”." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Link the best model to the W&B Model Registry (after all epochs are finished)\n", + "artifact_name = f\"model_vit-{wandb.run.id}:best\"\n", + "best_model_path = wandb.use_model(artifact_name)\n", + "\n", + "# Link the best model to the registry\n", + "wandb.link_model(path=best_model_path,\n", + " registered_model_name=\"Industrial ViT\",\n", + " aliases=[\"staging\"])\n", + "run.finish()" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "include_colab_link": true, + "provenance": [], + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}