From 5b9a39ea6cbdda8e6465b0f1ed502a05b55287fa Mon Sep 17 00:00:00 2001 From: zhoudw-zdw Date: Mon, 13 Mar 2023 13:37:56 +0800 Subject: [PATCH] first commit --- README.md | 82 ++ convs/__init__.py | 0 convs/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 132 bytes convs/__pycache__/bamboo_vit.cpython-39.pyc | Bin 0 -> 1756 bytes convs/__pycache__/cifar_resnet.cpython-39.pyc | Bin 0 -> 6554 bytes convs/__pycache__/clip.cpython-39.pyc | Bin 0 -> 8773 bytes convs/__pycache__/clipmodel.cpython-39.pyc | Bin 0 -> 15111 bytes convs/__pycache__/linears.cpython-39.pyc | Bin 0 -> 3837 bytes convs/__pycache__/mae.cpython-39.pyc | Bin 0 -> 2199 bytes .../mae_simple_tokenizer.cpython-39.pyc | Bin 0 -> 5744 bytes .../modified_represnet.cpython-39.pyc | Bin 0 -> 5682 bytes convs/__pycache__/resnet.cpython-39.pyc | Bin 0 -> 11605 bytes convs/__pycache__/resnet_cbam.cpython-39.pyc | Bin 0 -> 8300 bytes convs/__pycache__/resnet_scale.cpython-39.pyc | Bin 0 -> 12425 bytes .../ucir_cifar_resnet.cpython-39.pyc | Bin 0 -> 6680 bytes convs/__pycache__/ucir_resnet.cpython-39.pyc | Bin 0 -> 8627 bytes .../vision_transformer.cpython-39.pyc | Bin 0 -> 32550 bytes .../vision_transformer_adapter.cpython-39.pyc | Bin 0 -> 10979 bytes .../vision_transformer_ssf.cpython-39.pyc | Bin 0 -> 32530 bytes convs/__pycache__/vpt.cpython-39.pyc | Bin 0 -> 3880 bytes convs/cifar_resnet.py | 198 ++++ convs/linears.py | 167 +++ convs/modified_represnet.py | 177 +++ convs/original_resnet.py | 1004 +++++++++++++++++ convs/resnet.py | 384 +++++++ convs/resnet_cbam.py | 267 +++++ convs/resnet_scale.py | 410 +++++++ convs/ucir_cifar_resnet.py | 204 ++++ convs/ucir_resnet.py | 299 +++++ convs/vision_transformer_adapter.py | 468 ++++++++ convs/vision_transformer_ssf.py | 872 ++++++++++++++ convs/vpt.py | 144 +++ exps/adam_adapter.json | 25 + exps/adam_adapter_cub.json | 25 + exps/adam_ssf.json | 25 + exps/adam_ssf_cub.json | 25 + exps/adam_vpt_deep.json | 24 + exps/adam_vpt_deep_cub.json | 24 + exps/adam_vpt_shallow.json | 24 + exps/adam_vpt_shallow_cub.json | 24 + exps/simplecil.json | 24 + exps/simplecil_cub.json | 24 + imgs/adam.png | Bin 0 -> 130335 bytes main.py | 24 + models/__init__.py | 0 models/__pycache__/L2P.cpython-39.pyc | Bin 0 -> 15997 bytes .../__pycache__/PretrainedCNN.cpython-39.pyc | Bin 0 -> 6136 bytes .../PretrainedCNNfc.cpython-39.pyc | Bin 0 -> 6196 bytes .../__pycache__/PretrainedVit.cpython-39.pyc | Bin 0 -> 6156 bytes models/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 133 bytes .../__pycache__/adam_adapter.cpython-39.pyc | Bin 0 -> 5538 bytes .../__pycache__/adam_finetune.cpython-39.pyc | Bin 0 -> 5090 bytes models/__pycache__/adam_ssf.cpython-39.pyc | Bin 0 -> 5847 bytes models/__pycache__/adam_vpt.cpython-39.pyc | Bin 0 -> 5658 bytes models/__pycache__/base.cpython-39.pyc | Bin 0 -> 10276 bytes models/__pycache__/bic.cpython-39.pyc | Bin 0 -> 5505 bytes models/__pycache__/coil.cpython-39.pyc | Bin 0 -> 8233 bytes models/__pycache__/der.cpython-39.pyc | Bin 0 -> 6170 bytes models/__pycache__/ewc.cpython-39.pyc | Bin 0 -> 6594 bytes models/__pycache__/fetril.cpython-39.pyc | Bin 0 -> 8496 bytes models/__pycache__/finetune.cpython-39.pyc | Bin 0 -> 4748 bytes .../__pycache__/finetune_vit.cpython-39.pyc | Bin 0 -> 6455 bytes .../finetune_vit_adapter.cpython-39.pyc | Bin 0 -> 6410 bytes ...netune_vit_adapter_ablation.cpython-39.pyc | Bin 0 -> 6551 bytes .../finetune_vit_lwf.cpython-39.pyc | Bin 0 -> 6921 bytes models/__pycache__/foster.cpython-39.pyc | Bin 0 -> 11175 bytes models/__pycache__/gem.cpython-39.pyc | Bin 0 -> 6824 bytes models/__pycache__/icarl.cpython-39.pyc | Bin 0 -> 5289 bytes models/__pycache__/il2a.cpython-39.pyc | Bin 0 -> 9044 bytes models/__pycache__/lwf.cpython-39.pyc | Bin 0 -> 5224 bytes models/__pycache__/pa2s.cpython-39.pyc | Bin 0 -> 7903 bytes models/__pycache__/podnet.cpython-39.pyc | Bin 0 -> 7116 bytes models/__pycache__/replay.cpython-39.pyc | Bin 0 -> 4841 bytes models/__pycache__/rmm.cpython-39.pyc | Bin 0 -> 6805 bytes models/__pycache__/simplecil.cpython-39.pyc | Bin 0 -> 2991 bytes models/__pycache__/ssre.cpython-39.pyc | Bin 0 -> 8445 bytes models/__pycache__/tunedshot.cpython-39.pyc | Bin 0 -> 6821 bytes .../tunedshot_ablation.cpython-39.pyc | Bin 0 -> 6812 bytes .../tunedshot_ablation_1plus1.cpython-39.pyc | Bin 0 -> 6685 bytes .../tunedshot_adapter.cpython-39.pyc | Bin 0 -> 7584 bytes .../__pycache__/tunedshot_bn.cpython-39.pyc | Bin 0 -> 7576 bytes .../__pycache__/tunedshot_cam.cpython-39.pyc | Bin 0 -> 10716 bytes .../__pycache__/tunedshot_cam2.cpython-39.pyc | Bin 0 -> 10052 bytes .../tunedshot_scale.cpython-39.pyc | Bin 0 -> 7880 bytes .../tunedshot_scale_pca.cpython-39.pyc | Bin 0 -> 10327 bytes .../tunedshot_scale_randomdrop.cpython-39.pyc | Bin 0 -> 10262 bytes .../tunedshot_scale_tsne.cpython-39.pyc | Bin 0 -> 13990 bytes .../__pycache__/tunedshot_vit.cpython-39.pyc | Bin 0 -> 6644 bytes .../__pycache__/tunedshot_vpt.cpython-39.pyc | Bin 0 -> 7630 bytes models/__pycache__/wa.cpython-39.pyc | Bin 0 -> 5394 bytes models/adam_adapter.py | 162 +++ models/adam_finetune.py | 157 +++ models/adam_ssf.py | 180 +++ models/adam_vpt.py | 178 +++ models/base.py | 401 +++++++ models/simplecil.py | 84 ++ trainer.py | 128 +++ utils/__init__.py | 0 utils/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 132 bytes utils/__pycache__/autoaugment.cpython-39.pyc | Bin 0 -> 6427 bytes utils/__pycache__/data.cpython-39.pyc | Bin 0 -> 7019 bytes utils/__pycache__/data_manager.cpython-39.pyc | Bin 0 -> 8004 bytes utils/__pycache__/factory.cpython-39.pyc | Bin 0 -> 624 bytes utils/__pycache__/inc_net.cpython-39.pyc | Bin 0 -> 22898 bytes utils/__pycache__/l2putils.cpython-39.pyc | Bin 0 -> 7595 bytes utils/__pycache__/ops.cpython-39.pyc | Bin 0 -> 5755 bytes utils/__pycache__/toolkit.cpython-39.pyc | Bin 0 -> 2221 bytes utils/data.py | 305 +++++ utils/data_manager.py | 280 +++++ utils/factory.py | 19 + utils/inc_net.py | 733 ++++++++++++ utils/ops.py | 121 ++ utils/rl_utils/ddpg.py | 206 ++++ utils/rl_utils/rl_utils.py | 20 + utils/toolkit.py | 73 ++ 115 files changed, 7991 insertions(+) create mode 100644 README.md create mode 100644 convs/__init__.py create mode 100644 convs/__pycache__/__init__.cpython-39.pyc create mode 100644 convs/__pycache__/bamboo_vit.cpython-39.pyc create mode 100644 convs/__pycache__/cifar_resnet.cpython-39.pyc create mode 100644 convs/__pycache__/clip.cpython-39.pyc create mode 100644 convs/__pycache__/clipmodel.cpython-39.pyc create mode 100644 convs/__pycache__/linears.cpython-39.pyc create mode 100644 convs/__pycache__/mae.cpython-39.pyc create mode 100644 convs/__pycache__/mae_simple_tokenizer.cpython-39.pyc create mode 100644 convs/__pycache__/modified_represnet.cpython-39.pyc create mode 100644 convs/__pycache__/resnet.cpython-39.pyc create mode 100644 convs/__pycache__/resnet_cbam.cpython-39.pyc create mode 100644 convs/__pycache__/resnet_scale.cpython-39.pyc create mode 100644 convs/__pycache__/ucir_cifar_resnet.cpython-39.pyc create mode 100644 convs/__pycache__/ucir_resnet.cpython-39.pyc create mode 100644 convs/__pycache__/vision_transformer.cpython-39.pyc create mode 100644 convs/__pycache__/vision_transformer_adapter.cpython-39.pyc create mode 100644 convs/__pycache__/vision_transformer_ssf.cpython-39.pyc create mode 100644 convs/__pycache__/vpt.cpython-39.pyc create mode 100644 convs/cifar_resnet.py create mode 100644 convs/linears.py create mode 100644 convs/modified_represnet.py create mode 100644 convs/original_resnet.py create mode 100644 convs/resnet.py create mode 100644 convs/resnet_cbam.py create mode 100644 convs/resnet_scale.py create mode 100644 convs/ucir_cifar_resnet.py create mode 100644 convs/ucir_resnet.py create mode 100644 convs/vision_transformer_adapter.py create mode 100644 convs/vision_transformer_ssf.py create mode 100644 convs/vpt.py create mode 100644 exps/adam_adapter.json create mode 100644 exps/adam_adapter_cub.json create mode 100644 exps/adam_ssf.json create mode 100644 exps/adam_ssf_cub.json create mode 100644 exps/adam_vpt_deep.json create mode 100644 exps/adam_vpt_deep_cub.json create mode 100644 exps/adam_vpt_shallow.json create mode 100644 exps/adam_vpt_shallow_cub.json create mode 100644 exps/simplecil.json create mode 100644 exps/simplecil_cub.json create mode 100644 imgs/adam.png create mode 100644 main.py create mode 100644 models/__init__.py create mode 100644 models/__pycache__/L2P.cpython-39.pyc create mode 100644 models/__pycache__/PretrainedCNN.cpython-39.pyc create mode 100644 models/__pycache__/PretrainedCNNfc.cpython-39.pyc create mode 100644 models/__pycache__/PretrainedVit.cpython-39.pyc create mode 100644 models/__pycache__/__init__.cpython-39.pyc create mode 100644 models/__pycache__/adam_adapter.cpython-39.pyc create mode 100644 models/__pycache__/adam_finetune.cpython-39.pyc create mode 100644 models/__pycache__/adam_ssf.cpython-39.pyc create mode 100644 models/__pycache__/adam_vpt.cpython-39.pyc create mode 100644 models/__pycache__/base.cpython-39.pyc create mode 100644 models/__pycache__/bic.cpython-39.pyc create mode 100644 models/__pycache__/coil.cpython-39.pyc create mode 100644 models/__pycache__/der.cpython-39.pyc create mode 100644 models/__pycache__/ewc.cpython-39.pyc create mode 100644 models/__pycache__/fetril.cpython-39.pyc create mode 100644 models/__pycache__/finetune.cpython-39.pyc create mode 100644 models/__pycache__/finetune_vit.cpython-39.pyc create mode 100644 models/__pycache__/finetune_vit_adapter.cpython-39.pyc create mode 100644 models/__pycache__/finetune_vit_adapter_ablation.cpython-39.pyc create mode 100644 models/__pycache__/finetune_vit_lwf.cpython-39.pyc create mode 100644 models/__pycache__/foster.cpython-39.pyc create mode 100644 models/__pycache__/gem.cpython-39.pyc create mode 100644 models/__pycache__/icarl.cpython-39.pyc create mode 100644 models/__pycache__/il2a.cpython-39.pyc create mode 100644 models/__pycache__/lwf.cpython-39.pyc create mode 100644 models/__pycache__/pa2s.cpython-39.pyc create mode 100644 models/__pycache__/podnet.cpython-39.pyc create mode 100644 models/__pycache__/replay.cpython-39.pyc create mode 100644 models/__pycache__/rmm.cpython-39.pyc create mode 100644 models/__pycache__/simplecil.cpython-39.pyc create mode 100644 models/__pycache__/ssre.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_ablation.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_ablation_1plus1.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_adapter.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_bn.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_cam.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_cam2.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_scale.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_scale_pca.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_scale_randomdrop.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_scale_tsne.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_vit.cpython-39.pyc create mode 100644 models/__pycache__/tunedshot_vpt.cpython-39.pyc create mode 100644 models/__pycache__/wa.cpython-39.pyc create mode 100644 models/adam_adapter.py create mode 100644 models/adam_finetune.py create mode 100644 models/adam_ssf.py create mode 100644 models/adam_vpt.py create mode 100644 models/base.py create mode 100644 models/simplecil.py create mode 100644 trainer.py create mode 100644 utils/__init__.py create mode 100644 utils/__pycache__/__init__.cpython-39.pyc create mode 100644 utils/__pycache__/autoaugment.cpython-39.pyc create mode 100644 utils/__pycache__/data.cpython-39.pyc create mode 100644 utils/__pycache__/data_manager.cpython-39.pyc create mode 100644 utils/__pycache__/factory.cpython-39.pyc create mode 100644 utils/__pycache__/inc_net.cpython-39.pyc create mode 100644 utils/__pycache__/l2putils.cpython-39.pyc create mode 100644 utils/__pycache__/ops.cpython-39.pyc create mode 100644 utils/__pycache__/toolkit.cpython-39.pyc create mode 100644 utils/data.py create mode 100644 utils/data_manager.py create mode 100644 utils/factory.py create mode 100644 utils/inc_net.py create mode 100644 utils/ops.py create mode 100644 utils/rl_utils/ddpg.py create mode 100644 utils/rl_utils/rl_utils.py create mode 100644 utils/toolkit.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..1e3b2f4 --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +# Revisiting Class-Incremental Learning with Pre-Trained Models: Generalizability and Adaptivity are All You Need +
+ +
+ Da-Wei Zhou1  + Han-Jia Ye1  + De-Chuan Zhan1  + Ziwei Liu 2 +
+
+1State Key Laboratory for Novel Software Technology, Nanjing University  + +2S-Lab, Nanyang Technological University  +
+
+ +The code repository for "Revisiting Class-Incremental Learning with Pre-Trained Models: Generalizability and Adaptivity are All You Need" in PyTorch. + + + + + +## Updates +[03/2023] Code has been released. + +## Introduction +Class-incremental learning (CIL) aims to adapt to emerging new classes without forgetting old ones. Traditional CIL models are trained from scratch to continually acquire knowledge as data evolves. Recently, pre-training has achieved substantial progress, making vast pre-trained models (PTMs) accessible for CIL. Contrary to traditional methods, PTMs possess generalizable embeddings, which can be easily transferred for CIL. In this work, we revisit CIL with PTMs and argue that the core factors in CIL are adaptivity for model updating and generalizability for knowledge transferring. 1) We first reveal that frozen PTM can already provide generalizable +embeddings for CIL. Surprisingly, a simple baseline (**SimpleCIL**) which continually sets the classifiers of PTM to prototype features can beat state-of-the-art even without training on the downstream task. 2) Due to the distribution gap between pre-trained and downstream datasets, PTM can be further cultivated with adaptivity via model adaptation. We propose **ADapt And Merge** (ADAM), which aggregates the embeddings of PTM and adapted models for classifier construction. ADAM is a general framework that can be orthogonally combined with any parameter-efficient tuning method, which holds the advantages of PTM’s generalizability and adapted model’s adaptivity. 3) Additionally, we find previous benchmarks are unsuitable in the era of PTM due to data overlapping and propose four new benchmarks for assessment, namely ImageNet-A, ObjectNet, OmniBenchmark, and VTAB. Extensive experiments validate the effectiveness of ADAM with a unified and concise framework. + +
+ + +

TL;DR

+ +A simple baseline (**SimpleCIL**) beats SOTA even without training on the downstream task. **ADapt And Merge** (ADAM) extends SimpleCIL with better adaptivity and generalizability. Four new benchmarks are proposed for assessment. +
+ + + +## Requirements +### Environment +1. [torch 1.11.0](https://github.com/pytorch/pytorch) +2. [torchvision 0.12.0](https://github.com/pytorch/vision) +3. [timm 0.6.12](https://github.com/huggingface/pytorch-image-models) + + +### Dataset +We provide the processed datasets as follows: +- **CIFAR100**: will be automatically downloaded by the code. +- **CUB200**: Google Drive: [link](https://drive.google.com/file/d/1XbUpnWpJPnItt5zQ6sHJnsjPncnNLvWb/view?usp=sharing) or Onedrive: [link](https://entuedu-my.sharepoint.com/:u:/g/personal/n2207876b_e_ntu_edu_sg/EVV4pT9VJ9pBrVs2x0lcwd0BlVQCtSrdbLVfhuajMry-lA?e=L6Wjsc) +- **ImageNet-R**: Google Drive: [link](https://drive.google.com/file/d/1SG4TbiL8_DooekztyCVK8mPmfhMo8fkR/view?usp=sharing) or Onedrive: [link](https://entuedu-my.sharepoint.com/:u:/g/personal/n2207876b_e_ntu_edu_sg/EU4jyLL29CtBsZkB6y-JSbgBzWF5YHhBAUz1Qw8qM2954A?e=hlWpNW) +- **ImageNet-A**:Google Drive: [link](https://drive.google.com/file/d/19l52ua_vvTtttgVRziCZJjal0TPE9f2p/view?usp=sharing) or Onedrive: [link](https://entuedu-my.sharepoint.com/:u:/g/personal/n2207876b_e_ntu_edu_sg/ERYi36eg9b1KkfEplgFTW3gBg1otwWwkQPSml0igWBC46A?e=NiTUkL) +- **OmniBenchmark**: Google Drive: [link](https://drive.google.com/file/d/1AbCP3zBMtv_TDXJypOCnOgX8hJmvJm3u/view?usp=sharing) or Onedrive: [link](https://entuedu-my.sharepoint.com/:u:/g/personal/n2207876b_e_ntu_edu_sg/EcoUATKl24JFo3jBMnTV2WcBwkuyBH0TmCAy6Lml1gOHJA?e=eCNcoA) +- **VTAB**: Google Drive: [link](https://drive.google.com/file/d/1xUiwlnx4k0oDhYi26KL5KwrCAya-mvJ_/view?usp=sharing) or Onedrive: [link](https://entuedu-my.sharepoint.com/:u:/g/personal/n2207876b_e_ntu_edu_sg/EQyTP1nOIH5PrfhXtpPgKQ8BlEFW2Erda1t7Kdi3Al-ePw?e=Yt4RnV) +- **ObjectNet**: Onedrive: [link](https://entuedu-my.sharepoint.com/:u:/g/personal/n2207876b_e_ntu_edu_sg/EZFv9uaaO1hBj7Y40KoCvYkBnuUZHnHnjMda6obiDpiIWw?e=4n8Kpy) You can also refer to the [filelist](https://drive.google.com/file/d/147Mta-HcENF6IhZ8dvPnZ93Romcie7T6/view?usp=sharing) if the file is too large to download. + +These subsets are sampled from the original datasets. Please note that I do not have the right to distribute these datasets. If the distribution violates the license, I shall provide the filenames instead. + +You need to modify the path of the datasets in `./utils/data.py` according to your own path. + +## Running scripts +Please follow the settings in the `exps` folder to prepare your json files, and then run: + +``` +python main.py --config ./exps/[configname].json +``` + + +## Acknolegment +This repo is based on [CIL_Survey](https://github.com/zhoudw-zdw/CIL_Survey) and [PyCIL](https://github.com/G-U-N/PyCIL). + +The implemenations of parameter-efficient tuning methods are based on [VPT](https://github.com/sagizty/VPT), [AdaptFormer](https://github.com/ShoufaChen/AdaptFormer), and [SSF](https://github.com/dongzelian/SSF). + +## Correspondence +If you have any questions, please contact me via [email](mailto:zhoudw@lamda.nju.edu.cn) or open an [issue](https://github.com/zhoudw-zdw/RevisitingCIL/issues/new). + + +
+ +![visitors](https://visitor-badge.glitch.me/badge?page_id=zhoudw-zdw.RevisitingCIL&left_color=green&right_color=red) + +
\ No newline at end of file diff --git a/convs/__init__.py b/convs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/convs/__pycache__/__init__.cpython-39.pyc b/convs/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..735671c7138108f94dc5666e8ede2497738d8a72 GIT binary patch literal 132 zcmYe~<>g`kf;KJg6cGIwL?8o3AjbiSi&=m~3PUi1CZpdFhAPN&7twtoLDT@M-i zn}ipe2jNR}onsIzVuA}N=MlHF6FGKvBiGK}*pGb9ge$x=76n$dgF?T7;egH9538q~jQq^b`LE7>Z=XG7J;!)yIeBP8A?1Wt&^Y~149}i& z6O2ny=Tecme?Qz97u7~y>q^ND@nCvb)*F*F%Qw=pIMEx!WIQa(_#~_PFxT47GhL@5KkQiqpI-r0#F$Wt!xA2aDy&o@<<(1(VmH&=x|j*04^|uQVb9HGK<%WFP?p0YVF^C;YP}Xb^}766!r-9*cYc z+cf^(!+Y7?i~s)QY;Y%C+8{f$ewLzj2o;+H-*Zr)0$++Dnfo)feVgq;E#pa2rH8ja zi#InvMY>)|4<$~NSy9o>p=lTO7%*FE^#bGTdLoqx@Q=uLUp1z^<8vJ^%*+?jx$hO;L-jBEJw}_b_x`MoHI~Fb#USxOlECVst1V@ z@d($bmDD{~(Q`5UM-wh}i{OYG3)(a;Kl%t7ZBxqM;?vF|#>+j}Aw~`E0776_Bx4Ee zS}`F7dSbm2$M@?bU+mahtM|!s=Mt1u=J^sB>-*+10QD+X^e(OQkgs01nk$T@FDSI{ z=os0}*eNhMVk4|(><|762fMhHI7z^U55R|@o(N>rS|srp@Equ9Rmf3N=T*FrUxp+A z8$hrv@@Dhqczlhk9^@-wQKD$yCW{_YWoznVeu`k8I8-}PE&X#3!7{2*r;a6jFfeal zCGgfJ5g4T0ynWo`QPZ-MsD43LzaU2!e_=o(2rlDSrAiNthms(mv)T8vadIF{P}bF? zu3o-VtCAW|rp|KlIV?QmBkr|H7LnF5@jC8xOe2m(nIc^`u%r4kPy{BlUi(F%R!B#M z#k8&NB-5zjCfHjPNs7_JWo@euNVHjhKG>2KZZ8P5L;qa1ID2mBggoRcJaj{T?PeH+ He#rj^5u=CZa&kv>6^{;o>9wyVsRT3dq~NGlHzGl(nCrPlnl>;k{wd=pyYTS zl>Cq~2g)2TfKnJz3j7Q|`%J5y;pg}~?q~UVzJU8Vet|FIKF=@mCEU;RWqt|w1%8=d z!Tkc)Ki5jH^&8pG{JJmv(DOgYZpU#q`k+#21o3untL$~!m7VRL5N^BaKKA+hcPc^X z3BT=!vD>=Y@?8-IVWYCu>TFfqUfjVd9qx;$(#EY-5q=c<@mjUq{VIC|L28>G_0yLz ze2Lz~gReorT222(t1Cf2OV^AlR74^El5HwPz9VZBb*m0yDhB9;SapTiYr&ZjkZf*9jj-6)&i} z!bxNaNs2Uf8^)#}K?ss)$rL%4x6gNIr45#k_acn+XWCs;qX33;)xZx8eX z;M7SG3JpSlaw&B8?~mJTvxA$trg_sc`D2-;|B@q+*#{&JPk5$Q-1l@aAtt8C+RmEFg7C#Y;HMx3Oo z;}%Pv_ZhQxY{x_3Usugb&6Q?Qm zHtnB;nB&&C@p9GD!OOG)K?Bxk3!=QBS_K>$b{*X){Bwq|pHfKxWl@=$ndDA2OC|g=%QzmASoyM65Xt>zl%p!V|4P`@Yxfuou0x# zW?;g1XH^DaZ5}Bk(77HYNXRATOb0xM1Zk6tc3eSVfKLw{cSzb&Z(B0U$C*yaN+NAK z2)ive^do5}cNH4M3N3$vTPLR!5yXHZeTn5&>n|S{-pO#HQ&3w+Q}P?IU1Uqcyunaj z-hh`kV!dgQv*Y@6eV6FwK+*AJjh>n|*LqBBf@Z7OpsQKP!7i!{`gxLbe1YsJrOxR2 zrIfgVt;J1rl0A^7@LN4Wuo7e)>d?qcC<76TG$xHKDd5N%YT2y`Fir?H4>3BT)I?*Z zPVg>JuUdcU1>p>1Pf1TO2DX+lQ(U6%GCG;@pLX3a3OZq;Vv0lJ3U!mcm2y;aLQwbt znzF7m7%1cf(N4FHyZCp;I`T8tF~@D5dd6xNPxB0Nd)rU(EYCgD_E;^gY$gMnDahP9 zrIN|<`ps++SMpNqbmO3nG?y1~)?&Qv7uC?>h8w$)AKxgtQ8DWJNJRsT2|_h@R4usT zY4Et*5siww6;)PmuU5+|YisY_E_Zod4HWOv=?F=qixQWyWpbpMvzAHrwxn@=5p@(Y z_P0JpCAG)CR@oELFKN4`vP^?K1L`PLQam%sr?@rDrz}`z8i>mb7`Fx{PaP;*9a6HO zFh$7?DY-Ev4-22;d9_Y~=LW2KW&!chIScx^(L0|zpz2{Bv(F!B8`>{^yKBV@1LSl~ zb5H*@6NQ1*yugiRP2E$=8rHCyiwoMWjr+ys(*80x;!FFNd0}8*)(+T#&a_A04%e{N z8t)B9UIF(ERgBeq4fC#2MKd0GZci8AV%OLAi!}PhamqWLN#fKId36=6`q#PNjkk*t zN{$vUZu!Mdh*mFFS8f#7syB+aZrv!}zH_6vy1H_tpiG)nR*-nNqaxmq?ovnUJj$YL zpwV@=^hVApb@Z?% zXfv5Yq18prCEinSonGwlpe^km2O-K+sn@5 z>JvX`Y{$~%K)pok z;3HIFWU1IBEhT4}9_>}jtJ68^-$=5yWZ_Bf+rdT=`9QdDW8}VVHm`npRDKJ39u?pv zc117XPPJeGnctM1k3GB6+Dv#PjgcWL_x?2=xVB41!jP+88E8#yfU>a37PK8hti%BN zgfcMqj9sfiFt+y%BuUAC=D^xB75IiTVH{eF6$i$i4)a*tO-+yAL)ilme;Hd%tn(`> z!S$s0f%pgtN~uYuZM)sBA9864H*EMxsgwQOf7HXbh`??cf%z@c31~#{T zPf^p}{6ef@LuJ!P|Hf6)YgWDOb|aat`)=GrTr6e9T^d7qouKkm+@p@T6&uw3kh%xx zr1?1Tp8)X)2typJ@LDsTI;+%GsavbrWOqInTQqSJ=#)e$0{#*$T0o~Ez%HmTOCfgN zNHhIk{qxf@9Y$D>J~)gnde+gDJSl}Cka4!GHBp-oP+ZUulNH~T6rfa57?@>6X7Hf} zR*qEbcNi4WlA7MXFunbF1ePEVhZ{vJi0oLPPdN_nc=+f*wj&5O$tQ_)S9I`EDE>|FY+&IJAAp^Yy({y6!fBTk;jxL=KvRcxT~(T!yt

=_MN}Z@D%;; zJiU7oo{&DB$P{7Ybfz*1pfLGwGhFrXR$3Hwn*1qsRLID*`ZQY(Llqlq=SQ#QdN1^1 zlqPOVW+%@o4oLeG^|4ij*yKavJG5Yly6e;_XF)bE$hH)i1))cfx&+Bb5Ia*ws?3#Q mfJ}dwBwinpD~qn7GtIP)j{yh)^SSvv+T8rr`3Grx^7{|eNlOm^ literal 0 HcmV?d00001 diff --git a/convs/__pycache__/clip.cpython-39.pyc b/convs/__pycache__/clip.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0fa2fcfe6303b10a2ae2ec382a79a49cd269d3c5 GIT binary patch literal 8773 zcmb_hU5p#ob)FgibGfV4-)eU)+hf^rL|fV&{*$t5NtW$csk~7oIj(7|^l z{juW)D$3otynl1gx#ymHzwca{)6-c6pW*pGv;OuOMfoQx?EO&dc|6`vR7GJ5Q*EUg zeXC7XmbIoPzhli9zBN1EPc#$#WHZ@MHB&0}joImbrkRm>)y_7vsEgZE{aiCA>k{^K zf2KLpKhQkTpKZ?e4>k|>4>b?TwxoTyf4F%V`4mgrNBT#bM`b=^&-ITr=T&7ihtaYa zVWGL8DtuvWvAOuB;>^bt{@D88sS5q^9M!(1IGUmG%-Zqs<2<|e#FiT6+0>SLQxTu= z*!W47+fr4ftFY<6P@6iR-&C4Uu^D!N&9Z~+5If9{u%m2_9s4lTJi(vlPqXw>~Z!4zMthO{`7*vp8QZ{I(rH|z6z-)$N4Sg165pMCmfAE&7NUu zCx(%dc(QBHb+iSAKaX+0@?o4k%f5Oq(LBXY$yPa z>NO>Q`XR01NHPXO|#o`x+o+r_02BNC&JW4w?A+_ z9wx7HuQ%dh_9AzDE-ngp5T>uY*SX`lBFwC~qHo&hfS2R`E!KYs&rnd9Y7cuhYxP~m zZ4?h&>-7gVzwX}T4tfhz9lS&X4qV&xq0j5+I7$?jYDy>J)ZMJJ=vs`55 zf>mi0Dn=1%@N%V8w@Pi+=JmRP=9*#Fik*_tHdv$FG24|!1A+@ifh4a#qU0K{6sv7k zC{=j5QL=c)sMV^B7i+B2DHp4i60f$)R=o|A7#$Sb?LwheD;2AyqEYEECZ~M2Orl4R zD7sQC)#~lCQR|e-rAnD&zU8XX;pGD9P%SZ5X_p#SrED>y-C@Ocu~w+o%(7`zxz#Yu zNOYn4f05nh1*_ApR9LNIHmcQXq0#8Hi$$Yq6^v@FTw*3>Tdvlv1}hbzAUBv{R>~!_ z(I_-n!-`~A<-EW3i1RKRtW>Sk3RvM1#y1=FO07{U6x%hUXx58Xq2A`jLc_9(W~~JC zSY<3xr&8x_vr#e3&hj^V*H6E6x>TfzKdSIb9oDPWELJx`*WBYNJ%clGouC+-k!-^*Za63)PA6$m)+Oyj{1jg*tF=lN(l}(Kc(m-l!Bz)+y9j zu?n>y9J;{*)lSVSa)VoDp;oGyMyXz^%9&q2U65w~&Lf(=WMCsVirBzL16!t8?ld}v zQ7n|oC>Zr>rP#4*+%zgiwOr!$cBx*mip;doR4#Ok;*RjsrBZcp7sk(LnsM6k&4k>3 z%_J4>mYXU0U8rtns^w8yF4=OaILQ^NQSS0`p*)(QR?3~1Bj>d+HWYRk7q07D8lB_Y zpA5bUjxXZzet<*QSJu=(Wh&F&(^M*H<5-|e}rLy>fvJsOli9ijsEo~z% zOUWR2sqwtO5TrL%D&6|!n;;i) z{B6cm#W_>}ta@{X1bkI_3*Wz?z4}|)zqz;lOXnQ}fPc?Vvjb!)vOs=z!&F=zX zTkg$I)#Zo({n=-qjh@nFm8Y8yBMfs~UpIkd-Jxv?z0*6Irw;xu#bI0SjXarsI#+MV`j=AsLnb^OzpxzqJm zA6|jOG_~GKJ=jVC(kFb^H|;RtSgsA&96CKe%u?F&EE5{#WS;eI_W<}kMOT#3%LLZ8 zzG`}Un{x;1M6x;i?HbdZrPleyP;pg>gc8>>3UZkXhw-S+C1IKTO{- zg#!yg80Eqw_7=UC68zSXI;2oRuZkS36vlcEqg8jfJg|E9vgJCrJ;_&=2J2y_H5vVRyycxlqNp=! zPMc9@w0S%^^|%_>X0=82gqBs~$Y<4o_Jo?�B)Vh_g*Tl#k(&2Me7rI6lDiw}?q< zD`D*F*It5ir$_@Xh(IQK#2KZ+QF0zHg-2W4JzBE4<-0A>ZT~KsJb8?%;uYk-AUTan zDfwAse8pD-#n+hnp625)+f@CypV(CVB-2=IQ|l%+Nvrq@w3x78HsOMmn8bKUR!>aoll zfBB1Z`IJaOv>-*o)JvBxzJW*3X)6^LWDzliB(FB(PHS-Y^Apdud|^6X2gLXr=;qBJ zQB+OMs!61AHT_@WHTim&6x<&Q$D$56a*va%dAi%t zci69f@$#iBdPHGCi{myqzsc9VJ^GT5iSv{kMDiA%Xl*oE4&&C4 znPJQt3}4O1!?@g(NycyY;I^?fY|ZJ3+qEP-;XK*-Oe04|qq%sW!l$r~%56nF;{!|r zWdVROc*57#Vt^+2)+V6HIVsBtf(lU$6x7VoN(TynDCVa&@%>R;!V$oU#Q`FNjl}2~ zKeLuyn+g(vCe|Jxs$&V3d{08tMv|pAQi%V~#NjjRKaJu)^!u?tJ)U6^#AQALG$M-3 zZfF<@<3#Uoq@oBBu#wtDYs99Pa;W(S#$SfObaPl|M4IT z2~V)2fbl0udo~A>FcnOV^)2Q5+Ij%1K!=A`Xz>mPI)!>vQv)8$U&((G2qo z<}b13F9V!(bB}{&vpPOkOaU*(*7aoZ^v*{X$>;PlI?g{?q!%tk?ybuv@^3<&b^Tom z-j;SY!gd&VGXL%=iW><;30`#Hg${CmL})qyxcEAbD&Oo8GRU5ukK?_?dX^vQ82LQ( zkdgD-(B>2dLV&sywcOZI?mFab_l6{#>vBB8WoRkIMk8YPV7RvH+B>hlxyqp>RCRUg zFJE*Az&M1)UE$HhoucpfAF91iWAo>FelZ6Bij&=c`uM%w}(5WL*t=`3+A^$v`|T^2E-%U3i5nB+M|Tjr#=j7W0a2kl zgPl_t(=z1}6JCLP=@?0uprm8%$sa(Ez-^*_zJHceHS{QWSCxOyoJl-4IR*FRQQaKM8ILI|jn!@W>LftaF1i#%?qCV7b$Vfw6X_S?*S@rS7YcRaXxBd=XL zOB1oZ7m76fji`Om93>GJ8!K=k@& z&Rle1Mbq&gf~|kHPfx-&q6E^CAJAZngkucIC5{9Ly}GKSb(IYjXfp@2(bkO!ZKy6n z64_rv|E~usg=TB<)FW@{Kr&)qR zHMC`D^$_^Hb8CgceEH-`nCJ>~016H60eZ^{;$0{jCT^Sdkc-!m&u1l7Hho`+Hp(JT z9g^k%l$Qx|oiNwwIqZUMufUVNaE7|}5Qgv-;u>KLXJ43wjj)sM^g!2VEO)SeVbaNi z8ztf8*y3SQD&t8xAP*Q0i#8PSA)Bm(onZrzUG40b#xwGuf4*-r>0c}-63A#F~f z0$b2tZ{|3naE61DBC6)pcFE^={m8foFdaraQ;_+jjS-*vyq}*I6l_Gtv{d^lX^@qO z(;}b`rzoyPKq|xMglrJsqE6G>dvevaF|Zg=T`pRasH!;?mE<(UU6Mn41H0!67=G3G%hagnbj$^rUT3kJZK&YrC)j4eHv^pD`R~NP1Ocp^> zHa3F=okwVzo0Vk^K`M>;P#$0eQ!ockCgNhvGSU zsLuO0By|;$X}uO>DsgL>!*3|xSMDp|*&4&SdoLX*#J;iH;%kX<(oe17 z0xHVWX^dZs)=-!6vq5YVS8&KrAp(iX_+@{6E~@vZsV8>pyYFZl2}BCnAW3iDjEkQk zI{9gUxFt}(3q3ZH4-hf1^jZ9d<zm!C3!gqYW!Uw4QCiQC5~w>#*OW6%KD z<)C4iL(jn0f#gV?K&I1Hy0S0l&>E_krn(U2L*hJS?^9Au7 zz9Wz>P>%2xnynAu(}I>>&=D?(i-*&Ln_UU1bc^aqYech;A-oghBh5MKtx-6*v=a_` zyZ3w1wO*FSq#HeX>D$cEMM4CRg!0WJeUrC5`Rnzr2Z;mJlx%ZzBcUGKe;cD>q_ z&%Ki4Wd*b{#I(53@Fpop%m6iy@@z|oLfdI6OoxFPm?<+C7z$(t{1-F4DbL^UJNMq* zmF1L`?ziWD=Y7t1zUSFyxm+;t`NePliPpc z-zn|dD`jAEo(qf{V=BPpy#g?W7&8S-(JKK{0%jVurhzGY6<{iHnHgZFylG&jV@ws8 z8LtXVHO9;WGwaO(GZ$m#fH~sL12Z3Ej_}(%`jY7#^NwG2R_6Vq-U8lEc#C*D=Go5} z^*e5IrRru>{8r`p%J-gVx5By=WzKJHcKs;xj$XUpTSmUs>$O;P{KJLiCpZJ(8!M(~ ztXQ78VtelM#;WPro^v(7;`o_P*2|#dtd~Q_x#tbXSaH#LJ}Nxk55unC^II2hvbCGo zA)}SFi!_QFMhTCNCwLIqp|NZ1nfut1(CXMjbBG>2%d@XLL!;q@)?Q|4?VCtvk)QJ% zFLRx5u9UV=IxFRK*E7K8$I2Hb$`^;WXI^yF03}L8d*AG#y}k0jfm)TMt)bDGngcyn12#>I(i}nwi>OQnFA0G8EV%_tq^Eh`s`_arvaxzKI~=Wm!%uHD=0He3EpE-SV>x7qZ( zc5hAP@jkHbKOLD50-X&))dpD*8=0r;$D&NIwdt$KZ8X}wcGzgB470N5`n~NlUX*F! z`*c)%ycxFEm-}i1IQv5HbmY9X)4J3dW}MNeGg1Dr?X{=-{VvM8Z}$9N z*bc8mx#srTCXi9ye{Qqc3qXmw?nA8mkgoesRPg$jdqHyp^s1&X8kOac3eWhLw%Aa! zt1KYuPUHlBcQtZl>ODV*vRDFqOC_vHzR_rPn?cZM+%Ud!?*kjX@PY0Y7QlbNyF6I$ zZ#}TtYIh&t41)(+-S*~2-}Admn^#m7&6n{6_W&4X!JacqX2qHpxM(eyRlH@*taS{q zWGOlH_6W-&((Q+@8msW%T(_MHqUm@vOTn3V@TOTTnjwnhPhlXs$=L~%@fQK zoFG^rI8Jbq;A;qK1Yb*V7eL)ncQbuI09HQPS-Za#s(YCs;ML5{t@hRBrt*jh*oR>{ zSi^ad$eL_jigF~-Urf@|S*;=_D? zCb?tIxAmWAz2u}^vb+L5MPag%5$>49xpkwd$HI^#-0$hLu?!+E2Aj<}>weP0WLK9Pq+U$KJ?BM|ohR^5$%MacMA9)F}x&a+YtB#AVO^4hej|{Aj(bW{{AL z(JUqZbW>q3Ltm+HKn@Ojv*-1qxy^pi=89kiWb))%QTC~J&u^+Id$F;p`kg3yDgNG$ zzgzOHz8NjmGnyp7iT%BffLQe~f$VZhk}?`*1{DAtZ1a>^k-w@rICfxf(o;NffVtF1 z@SMXF{0YF&?1;R$Zth{JDIxdm9aD4nj)Qkl!F3$#H7vcCyKe26kPb#C<3U;Z9*dCQ ztM+kYFN?Nv`{q#t_1zARF!G%5v5un;{&fnTi?q=x;wg2?fE9sLow5hnNp=DJwBQM+ zI@4=l71BrEWA2QUpXp3{kd>V(RhjFv@4}eCD$KoE52Zn;#bxz+%&ZO$8iY+Hbf+lC zWsA?>up*nZn!D)xn@HR+)tixd^MCPqvOXJSz^2ue$iC=bi89;G?v@`_Q?hP^{YGmY z+JS6RRMz|2xa_yr*2Ac*w>y2+yPdpN+CACzNlEU z*o*R>>Tlv4H8!Iu`2txb4V6=bv~+AM@Ed`@33?6vi5}fhH6bQJprxLVbE=0O)aStL zy6DF5LGb!&V+&L+L~gg5VdQN1&7OJ;YZO^wKgxLFl}$e~pNtkawz^?k4mk&@IYT4n zf{@JXO*AIN`~)j!b$5!ir`|w7F|M8@corba1z382;MeUjCT)&$Jz;ocjBGGi0x-ak z7jX_8oC7dF6Jia2mbrjO|LJ$fwEm$mJ2-mql%za+xt>#08`VPuUk?!3?Ov!k0J~JL zAvlZ!zZMxhxB$3LXYcAHwu z*9El;TVV~VLkM~ZP};Ap_PgEwWza^gwchNl`N8X?-TQ0j*P({jns|Es%e9AUcLkxp zaThCZ_q$tUeL<}m)cTv?gucg~PTyC%+z!`k&0ej!?W^XRU!&SoLsuaLLao1AYt}ZJ z&k1%Z?&fJHiP6bmbVIdA zYNto@s;#!cw`CaYtJ}QdD>l`vz3naUtfeG9G4RPbNn1C+j19ec7NC{LSfoW!qM~c# z3Eqy48i_ZEEu&!#X(2$&ft#AcjAt#xn!DHtS>$CQ0&_Y?M4pSXd0H)EK@?b1=ccv6 zzdcthiQ<8}rHQ)biMqn~<9ZccPx_mhs56b68Bg0X)gi5wS*U7rNSR*#xFKTr2>3EE zo;3%q??EbvRX|}oALkS$2T~!;IU47bB*z@fIfj1@`5ph@;J11xo!(0 zyerF!#+Wet*s6d^a?w{kzuO4f;Gd#_QyUg;b6!zZ0)y!jlcGgDKyQ=6RB&%{<8Iaf z>LjB0M28;CLyWkefOiW!;9 zZ7*Ec>c$~T#wyAL#8b%t1GN{;X)xH60vBjUm?b+4J7R%<(1Zs^CmBq`w-w?i;u z_#%8l;NFWTARQU71h^J+)?ne_{6~9tivmW5AWUHe31R&(p0B#1#2DAG79AMZuzxV~ zp}C9iYqXA*MaW)mf$O9nrb*i*)Zl3VG{%RGf*-{u8ZkB= z2M^Bp7&Z$`4v#y5>&soAr#LD3I%guVR`B!3*ivP+VJT_aOy6j5MnkoNEpH7?S1# zNX3gy{y`*zZvy* zk$|K#82UD>037Uuma=-w#1s=9?0u-NRa0oJ{bu;knF)(N-&gImhUllZ^23An6Pwt> zBc`$e#x`dTX407~CA#3j-BG(quj(6_l9Ak28@!4lA7&?4`+amRlebSa2nsYdrbDA-( zxxciy;(hafJ_>3S%{~{j*EafX?}jzfwKc&D!%GSN=h-a#FtAkVFp{NO1_wQZyphr@ zt&c6!h<%RQy@nb;hez6VO~v89_l zm&Z!N{|g^)9;;c5=AkLKJ$PVJTi|hW{^hzI^-O=x$2(8TCQDCr9o3S>QSWu?G(sY4Dfu#6GivRRM#fuHeGY zSjd2V-+A~3jC6)uSO^J_(-b0q1PN#ia6G_Uc#KRc|!Yw1SBVlST^kYoDK=6J7p=SXitVk)^ z0{$d&$dHAP{SGo?b~s0Nc=9&GO?V@GuZ4q0{vtadSJCI51fa+8%8c>@DEEta zV#nn|dD>CL-NFjEAp8VWzf)G=3M)vMf%8htU@X8>JP*HJKuZ&%?S`XvmBDs^{^iHz2WE2BBZ4pLEG`GMA&6DQ`VXu6!5 zKfJ`6uO=XQd>)TzF0p-K4(1LU+2PWTTXBvJ6aeIO;1nd(Z>paL_O|5yMb_i8L!8Zy zcR5Di`Tq}nzcOk61UmQ}yCUhh=3-V6)50_HAxq4ZjG-2SI`CG&m!Yu`kMMP5;MmW?v2X8F z6tYY)gn$!qSq>`@b{;d*D0il%g|E{y?*!&D2{ECejP zQ$hn#F&r_VD94%pw zG8hr7!NM&hlJbDVPa4-pPTJoik(@NjA4;Q+<-u1Y-}L6#j2lp!2m5p2C1MnT{wbrA z#g3UBcc1T2aCAM7;u@4AEx>3+bRcs*18QKT0WtA3(ojrli{vv9fyQZ%d%_4jiC183 z<=fyZP^q9QAXX87GgcujAi^Lj#LJ}Y&eXI>RxoK$5fuROT&Sm*dOg9961)YVJ}o9? zlmqt@7xx---a{bd&}54Gt>S^#YDR{h{6L8c@ffi`Pw)|f8w9^V@P2}OQy|=ueCDt4 z1oRzoUl-tSI16n`>s1wNQDTvU-d0 z$iu}IuZ-`KSMjFsUG}EE8GKhn;7?&MtC4f=sW(3jo#k~H5?ps1@J5fh?xM2xh-)t! zI7R#_?y+blej72+s6wZ?|6JI>C6TpoJt}O&<#0imYcw-R~?<0u^st&XF_s~5DA zh2!Ft@Q@7)3w{}iF&0ebYlSeALs%B8@QbOlaEyY*dSIwKMd%#7fKmioM(9qMa;|q0 zX`Sl=4<8!J6mck=>ufC-=7;XEFoYXtudolMEZ0_wL&WAArLeqLLCO_dW(sf9;mlqY zhs7dJSoUl#-M5v+JYvICBDEEdODnsHPiL%&K z7=J>+nkCO&*lKsZu@DyRJ$*QTligA4iSln?Xj`JYQ(^z2-y4N{RVoUijK8rNUK!*4 zE;zhI8KNm68}fO-#i&W-^m*yS>TO1Lx4#w@0dRvwxO;Sjw^nminTyt-%p;eImI)suB?^~Qq{?T&X5s3k z>1K4q^Wj<7E!?9@a2dp$H3p>Gf$ji;|97V1t0EuO=}w_j6wL#AbKDwAAeIdC??8*h zmv4iw+y-A5;ibPRfs9b+Ix_;Rs}3C0_rZZ{?al5ZIQi^JM5S%GvhBS&FSBo|FL7jh zM~03#t-o~Xi>No>Im%IcoZEco7!EPs5bdDMjPrJ~aqdx_yMtqiyyNKMT{~_(Teq{& zIWf#m%#^rAj2r&Fu@!YYcXUnyo4WyU0@O z1R@`Qi>cox=n~LDE8Y!J>{k6|xCQB}o*1O za#viQ`34qdMdCA(%C5Hf9c85aL81QzwD@WqjyS?SXI2qrC?WWAO#fLmqc&C-FWc=rzO+^DM%7r8TS&!i9JDOz35MpbwoALw4ZY z-|nr7k_0E9C=#DWE5d@J%xW9=mGDI$EqJhMZ#Cdyp>~Vw1mvy<)SOQt=S`e3C7)q4 z*aU_3xXk4+?mRIS)Pop_r$X_GDF`Fz;I^0!N;sO9=8&n0%wy`i2@Z^e8BcR0VkKIG zf{a1#Jlx``NjVugsMe458qzD1bZzUd-^ae#c(M)|(M1k)AC>%G3m(XZWS&pQ0q3iX zV?1|aIMf!>;e2m8oPDb5+)!ST5jLB=L!BnO!UofTgSw7f1bpI>2k`J^Otuu}5C*cq zhM~qm51{%Eb?v%IwGRCip-iK75K30Zp6`!44f!jf&`6K%&K#b zHJBb>T^4O|L6Z-OGF{vYc+q<;5euFgE*jM;o3@SE!`2C2TXcrb1O{>Ij=ksXlgHyy zBpkiC; z>7yt@|C`*$buE0rtf4*OJO2CU>DRb-xUPkPy0$yzPAk@dJk3ZrVuIUQ$l(ajvBh}+ zF56Wauf`tOG+DIVq<03dK8HIx7;_DA?`|7r3SxB9UTvgOGJMCl-9#5$1=9O7?BeqP z47!KV)9~1MoK;&)58MN!bBih?|7)7g_2Bseu%wmehCw zEJ@ceQR4;CRNTpMIMa79@>!;Tf$5SG#fh7zFTtPo^u?6NS!A7P$7_bo$XRc8SM>$F4-uPN!)>xKR8eld z88CsXy=GU!M7YL^$Of+@)4_CL!qKtY&E&m2ev7EVD|%X8DVcYH<+`|it^?<|H*)&G zI(+pN;*UQvr06S{`O6&9KOQoysRQkPCr88WHf*=5XzkoEqXLXXT#@wf6O#%0pFuWF zavrnbIn?yvi{BRikh*XU#`bZBpWsJg6kGc#a4!PFvI&zy{>pZ-~+qrdPTNEFX}KCw^d5&qCd7(PTt4b!pVaSa=i*H3il{( zag0h`w1O{ui}5}ax%f@T4(@Sy)6xR&0~Hj}isQ@~p_eJkQ^Hhn7%OWqvZ4CT-a zGy11J*WwvtoE119^7?)jW>tWTp)@QHE4ziADeoxG>-^62aGI+Ko8s|n#&G)eNL_k7 z%nhe@sb5ZYT=a$?2GL#_PVME%{p$@iq|jFIb9e9)~yE^d7=$M9TlDY>AltrrAdNH8iC<3ar)+P$Iqfckr+ z4;&0dPVgdzra8f#T8kC-JEHN?85c+6;6yUlLoStTe`>qA0QFvVuYtFcHh0ZrwiB%+EA|jup*4Uw)Tv~>t{E?a9b&C`TSHcj)csEA+?OD) zu}jW?{slNymc*YGnM>j_UB>eK8`^@)8uBMp|Bf_Hoq0ho6@3p;{^2$J+UmJY^~hej zR`sv4R!eCeY>5p=9L!EXM0kumS^W#ae-VgtgWHfcjR-b%IqlG;r69rc!Q%MnPcJ>V zG)`WFQ*kayf|u)-4!iQNo4mv(KTGf!z+nD>qbMfW!BpZDN+`I8(zomfxmCdD9sume z$z(rXaVL(9xfhqx_?-F=w4paeeT7Nx#X*YwAK-pD-G!5f??SR&(8(0r_zi;J1mN$v zh?A$=hs+Gd|kgzTz(C|0@DF4dcmS>3yp`t3gbMynmKm4F&&=z7~A+b!6yhrax60? zs-sYiP-Jmz-am?w20U_x=~}#EK96SselFf-V9LRrNEhBR{fB6a=yZ^7|LtT&#zE3m zOod-(8ujB*v4u<4+WQyCT?2M%SR40DVqWMO1Y x+=s!q@av^VFb^ZR3jp2-Obu5$a1pfnVD(h>(Q3Y$sUE8qt9Mtw%P37B`#=0Sf#CoE literal 0 HcmV?d00001 diff --git a/convs/__pycache__/linears.cpython-39.pyc b/convs/__pycache__/linears.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb879e317dbb20d434c16359c08e78e951d28e34 GIT binary patch literal 3837 zcmaJ^TW{OQ6`mU@in_#cy=jx}wn;DPszt1Ix^1_MqFunPix#P|h>fDqLkXHgS=3Ub zc4ibOg8Z@$kf;2EK1h%GMFr-yPkHNefucQUC`+=}*)isDZlq?;`Mz(C<5nvqP+tAC zA^-UvA^*b8W)aZ&45}``NYW#cN|w_ebx+n~@Z>og@gAo*Cvq|JdOjt8B+`@qOCtT0 z4{H~^7r;nO1~3vVN9r(Amm!RV%aKqvWb-BIH6%MCo!0a`e46%Cl@>|*X*e)utUv8` zPct)^oa`p$s5{Vui8e`h|F54v-TQU)ry^15C@oB!M^DmN6n6nL~rp6c%UWcan?2R4YJtjJ6h*+y1G{WLZcm1^58CuY@C3|H1ak5xQMA*$AMC$;`@ zTIf<)|18Z;2S$0gY+I$L8OA;uuhwtDCsVT?&>dlgPV>GB;1yLzgD=0KwjM3f@@ z-n~3=LUHY~U>hB>4^=+|W5|#W!Tm!H?qsjQoj5WVFycv$?saYzht{vc41jGGWl?o9 z^FrY#JAUOnfljttU&Pr6C=yMItY4~8q*}1X*5NqX#D$5R7dlkoDb%-CW(Qlhz`PGt zI~37vI{o0vv99|#!KhspY$Mv-@$NuFQbXnxQC1yt!82N+#Lr70JcfuKC{QTkHd*bA zxt&=rE5;LZZG*N3M>>m@{64&+-H+4h&Xp}L7j)=xCqU#CfG0p+7)2Fhpxun3FDG%n zoT){TER!g5@pT?hAHdgbYl)I1&2yMtd+Ijs*SQP5)WDq}!~!ZryAcS7`FCr{E-YPX zN$7M06a;jHOJCMNCWQ=S9iE=k5g#1Zu+9BagN9T(@)m=BaGap8FAKJ@I;M}I@eZc* z9bjxEW*n?H1E58Y^`<@#AxdJ!{l?5we?vXM841jH+;2MWw+VoA+R?gD0VPfX^W?`pSX7k>LG6s85WCH_On!s77ke5l>*GHe);FoJlcbP7l1p zx9H)@3Cz;54FKFDR5Cbsr6aY%Lt>2iUzy zE|3gZHBuv>?N;DBf+WzrQczd0xJzOs7PXC}gD;Z~;Y7j%kqGSVqj8>@&Aq!Fe}hnb z1;6nwq4+kLiKSSeOnAlzA!joIfu|Q9jMUr+-xvu%G!zPkY0R4yFJN5@Lf#s-=eH{C zI|+{vX6%lqxX?bI>2bGuTGU%=k3DMuAt^q}SaWDm{b8->Fu!O!pmkI~xE@3}Rw+Pz=B0r3+5 z-~br!&{52J{tZX)!eA#0pWTb{&5{T#b%-g(vi zO`rD!xOvan$LeETv?hhqCuHDLs7rq$_wI4WLEeV+iv4$b_v`+=q$8`q4RO7Rjri$z z6A`h6I<7GWNJNDrSCSc|C`QRes1FU4yf2u79TJ|@pH@dWCUJv0l+9( zcA{+vcd*NWA?lMQ@+rZ`XyG5ZmPN1w?|1a*J7?&|sse)s@1JoKTu7!ZH!EdS@?zfl$ WxjhIGz%H7!$+y3={b2h+(EdL;o<3y& literal 0 HcmV?d00001 diff --git a/convs/__pycache__/mae.cpython-39.pyc b/convs/__pycache__/mae.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..39e9b9dee3ce3010369e5e0ca11ae96a6b908f07 GIT binary patch literal 2199 zcmb7F&5ImG6tAlO?wO67K%xl}hA7CO?x1K;331Vo9F##Yco>kfx2tBSXZvfbdUlhg zbJ|Ew@t_wE9yZ`*FOrM@f`3F^J;)q93m&5H)l7CbA4065Ue&yM_1>@FS0yVeF@f<* z^JDr$M9A;>=j3qV&pJ#!0wRc@IVtFfQrxqgjhJ1X!X3Fwo;UJu5a9^-E)lNur@=Iw zM*Gev5}xqyl2I%I5yILLk%(dK3ic)Gcjlm<^r;EzL^WBG4^w2E90&!6eiw)&BPs}( zrz1y1_sE!9lhMdsn(>xq{CmVDqX10A=JZ#Y&Z_dJN=iMhR3X*gdT+t&onZEMvu4uM zv%0QS(*wcYW?pS1c`wCoLuBXhRS`ZHw#MyzVuK{UDLtC=Xrff_y z(z2iFci_Ahc-(2p)KTj#e8YKi$AOi>>b6en?&G=Jl5adg0J$?XaS3>Np4^toM6%eB zf{U!^`^E#DQYi;)nUM3W%o@&3XMva_QWK0*j@ONgwC@>L%Y3Xtyy9=|CTdeF1g#ML zAvF=_X`X1!`62oH^+8cKgM6kNB?n@6Z&J+$b(-aav?_P>ph)CeeOm=^^DIn#3J9U+ zm`{Bg!Y^if7oPF@+GCtg_|Au$h~QJ0`XP{(P9fpIC#)q0w51Q|9q`MP9avup_Q1L0 zH15;`kI^y1_}^zBfinKf1Z59`qg&xM@^u%y>$Y?s^lX0tQjz+;g)9=}TfB|b?#Iq~ zGL5Z1Yw@6CwRq(P(AZsSe0&cq146I$AWq&;QNTU_2gV$Ckk1Z=X$8VGNor|2b<(60 zZUbndG}pYTwq$91d8BKl`(wL~O^ind!tx#2M{EfUyxAY=-ZXuIWLD>?8 zRowR#=AEIp4E@B!N7}V5FIcjQ;;WG1kOYE{Wzx)))IH$pSAh^3 zLjYIl8N1@%r7n}(pen7!-bl9ir67c%WeuBCP z`=@aROZD%=nYKko_s;((h(5jE53wT3q>xY%G3P}kW;yP=oNv#Pe0dUZE~=Dso5U7{ zdJRmNm1P;EGSA`k*lQ~Y4qa!f?hL}gS_-MlPIbbwmYV|OXdu`w%+Fxzw}3Qc44Ex3 z>mX}|*U*Md9pQ`MJ`>?jK})btU@n7mqTw8kW$3**s+nW4-u&bKpT8db_1nX%P_d4z z^IQT&Il#jOylVP$Ee z&#~DIp<^5}g=UQh4wU%`M=a832RhD1q9wQ1-?{uD$S(VUOS`oD$Yt!2@2HP}|H5ve zFW?r{2aOr_+xaW7Q*Q#fsnEabZ6pIEm@Vo9B*y`?xKAVQ9Gc$|@O6-S9`Gx`e_`PE z{aMBn9y*9;^i(|K;T*eUhRtr?gPr}3|A~GT(NAVa;NQCeik=7lW8g#oiN>=sZQwP~ zHo^)WW{hQFhWO;!|ZHpqHy$3B(3IesgRi|58)2$(a^N-Q!6Zo c`*F(&uPkcuD(00%-G#RTrgQF%FbIQx0at+uLjV8( literal 0 HcmV?d00001 diff --git a/convs/__pycache__/mae_simple_tokenizer.cpython-39.pyc b/convs/__pycache__/mae_simple_tokenizer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7e147d9db5ed90ad18d4ce229c58690765416b85 GIT binary patch literal 5744 zcmbtY&2JmW6`$E%l1oaK6vwe_!woiR5{I!PIp0aEs`+%#I#t@(Nf5>kvE&S?wU)co z>`=1QWr|iVnoEEbJr&4-IvOa7{sk@2{uguY$>*XMU((;3B_&aCiWH^2vpe5!-uu1Z zd!z8gi5kQ2v)BLFUOdUzKdEx`=b`dCo-9WpnBXyM6mQ<(w(c|>yq(zXc@2*jeWgap z_LUnI>FqJ$3hxfv;|<@|N}_CQRZ$WC9oDFcs;J>TBWA=b-m~vAaYD@9VPa06*mE0m zf?s3x`FoInt<|}y#%j=PhwXLQrlzC63My~m$sR?a*)2YDZ0Xw4)1`gB&jsIiu%NTH zd%o3|%`EEmW7*tF+hMEu+{^OiC4E~jb$9D!GOfve@2TPY28Wb`&qS(_|}K?u7@t>U2XU(XLcW z{T(wSWG5WNx(Tt!ESZCXIcf&w)KSSQWjyw%f5C@lnrlb!Ju00&F5E@7hxfSMv+bqB z_QlDJvhArH?HRJ1319fXbgNTKgU>-3d%nbzJ|lq0{lLZdZ3alh@(sgX(t%y&a+3R5a!aZ#YQ-{u9x+?UEZ9!f zMvx}K;K8$^Ea=G)LhPi2Ad!-`jYHLy!S}65<5}(XP=#%c<%1}pu_~~z38Idjxg91t z&}k5ErI83eT)R3J>*IB4hhI=QlMb@9C-r)9m<_Z9&s}~K(}09PggVTmwyQ~;e$cn# z(AZF=1GKz!HRz-&5V9??Sv$pH!>*hh9|ytzFvy~A(uvw6PhTn;ia~S~D|C{oUKn2r zvT)~8@WS&~t)lEn#_LRupw~lvDcF{QJz1Ql8xTl`&}|m9q~4Yg`Ose2m_&Y%ir*kh zl2)MTu-~WiKu_E2k%nc%wj|HcDvg69D%LRO#DYO9RGkbKECjsIA2a;(dmPaJH#y1Q z_lwt`ir1g*>mPqxQ#9fpL)oUU$&aRx4_(8PokGzC=5NjdrG!xaduP+TR=6_g#%-^w zx?`NB%Z4Y1POiv9O>Ko>+fIA^)yGl)7aq3z1c5he>a;o5#nMU)c8Rl9d`yOeUOcLp zk_wZqG;XSdDML3fyYWQS*2dk9`gPAZt;{&>Or53;dF!|c3&(ex$2fZ`HnYZOxrfK+ zH9lt_MPg1VgA`H1^A=5e8HHwhaD0#e2LU*E!=;G~Kmeyf0DQ!ACwDuJb$1u8el91M z_I7_&xN<-M2Gp{hUJDlOhf(1#A$g%F4EQm=2&qc;;oE>Gxm*_L0jT(JZWcUUEJA1_ z%xr}!BJ@2Q%cQH&g&yr}N>-Fc6NvLF+(+q+iXTBhcZHX=mipLp7>{Hobk|?k-=e zmyFlZot=i?iEcNwyseG5u6wcZ2Z_wuVPBdOkz4;RHy(8#mvg2SW>LG@j=@~dW7mvA z0)v$oxVnt5ZVP3_cu`qK`n|cxTHlC^9L1satC8KPj;YfFnb-45d{~DqR_YP$eo+u$J zD2uYF;EkL?RPpvj4WiDN`D-?QU!Me0c&txhox*dQ9DVX53mpVaq6ETu&YD&Wcko5o z!ah*G2irAW?Ttz~Y&I(Ivr$DL)Qx=M!G=!m2&6DJJUT@RlRGxZIl4+~j%xc@gF+<| z8zeZnOTCK>Ef{z6)X>Y_(Tu2#W^=4ZUbMM|mJ<{(yVk4poSx5Jy`Ue;-8~Qz1x?6w z5^0fRB|UjW(ueJ6$BxFhgVCpGJu&w+EZlrVpB^oWd42lUnSH9C6=dgywJC_WP|yJ@ z64^vh%FVeO&d?q@=YKflMT>#Eu6hK+4Bs$(>o_WEE5u=z8Q;Jlbq2-mrDMigYBr-J z(#_@(hCG;dzhpy;;Wg{i(KmlRhH0`cS=G`wt#4_3p`h^*Co3GKa>tY`zrd~fair@` zA$_SQ5$KPJuNJY~+jzda+y~4)rBssfK&eMVO%A{>vO>N(zwfOqZ?9S*#&W6GX!mJX zvr=7|NM+dl`IS7=q0(uG2qIs-e_|y^{FqdyB>A7st6q741V2FA)rkZU^-cPw(6RgJ z%8jGK+{jmRv>Zr7^$S^kLFdSV@(WvddijMo&M%1k(;NL?zBjyi(SEEAZ|0wVb|bTG zC{VvxKWF2XnWb3QlnEtu%9M~j^s;dy5+d8^brCgrKDuj0222AZI z@2ogaqS)Bf{2LfGZ}Xjh1W*e0xoNmY{sR2OuJ3n+2C4H_yu*^%M1gNp62He z%jfX#Bx)z|K7+P1XxlyWZH_fXft0fNcnAvF7AP?1U$9$ji>Xue&Jpmh;U+=Q6r2bS zv;%rPixglHybKC+H0ZA&x9ke~(2g0Py5FJS-t&A>`qBZ#d05fIO=0~jzWG`1fkjHg za+eJ&qCE6-AGZOlv%$VXc-Va^cShyBl>7V6u$os!6(ksbUL^@^l2IM{AG72M4%Wtf zW85#;r2iAFviagCEO9O~nt7Jxr40uwpP%d_DrcD8>pb;~$|A#<@_0;TM z5AHAi=56fb7pqwrFso>Xek#yFC9gmm_7tLGSGrCo-Mli<(B>6U9aVD>I)#ivzblXr z#9Ldt_YHpPb=YCbM<K+lAdJaXsrhY-+ zKca$iBIEX=#0sT;M%7Z32zgt*NBsw$hFK&VsF(t_dXhRR&1#feQCEdM90lp3HsuWd zoQQ^>$nB=h-gspAt>R2KXqG*g3#myB8SxP1J~jR*_Z^S>?kRlZjmLwhIPenknlj2+ z=Xtc$9Lk8PA6cD4Gi!YJ;Rk4C%7uu_>8}jdg%W%K()b$#ISs{Lipz*U?ug(|`2$7# zD??WxkFYL6NsW(aU!h!r@&dpEdBMoHy(Q}Xf&n&^Z`4G2G6p%q;}$xVHAUfc3N>UB zlII0H85tL08&R-;+*$S!oMzLZh_CACtaC+yZ4zc4!?&^uwH#!%T3zYNMBeVJ)gkp- zyws0treMV*keSR;Kg6W}Pr$W90UMTV3$#p&sn#*-^ND~3CTDE2Y13a*iqmbW#?;&a z7{fSL*J=Ei@{2HH@!6$j3J`U#@SSCRH)_Q5cI!;FXoOYK0s7Pi+IH|1cLX450qA;^ zKgD-XKUiB+P$H9&Bo<0!5`csq0m%d&`1^}(^(=G%C<59on}mwWm$+O|MhxZz%UEuC zFhB^nmR2aUvx(B&Aw-U*NnE{;5fgaWV;|EyqIBs|v{A)bpTh1B&^aD&@=wYp1rEpGvHf%Oct=a6QVnBZ}G_%d-<{*rV9{Wc= zT(q=OAJK*ukc6*7u*3Y|z)RIj=q1Qz6w-Zu>LI`G`?j?{uimAp7pWk4nQCW{v~`-s znIg7O&roAY$u0<*au>V3&9N+%TL-ePWXfslmTYUgZlQ)lSu9!jE5(UcDR^a+Hu+#x TA1sT~^?1$0bK0Nv%YOMkv%x|f literal 0 HcmV?d00001 diff --git a/convs/__pycache__/modified_represnet.cpython-39.pyc b/convs/__pycache__/modified_represnet.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7885bb16e330fd992ebae8fd02319676bdfe51c GIT binary patch literal 5682 zcmc&&-ESmE5$~R_on5cjzV*l1&IOi$LKYGmpAT|`C?t2$g$v1&+#M2HK*MBuY|naj zcCTml?6cMqiX$Q+If{}O9_=VXGJgX91`-eb%9G!CMo7o6dS<x0NoPYq=x ziE?gZ!0Y1FROKk~`mvvcsjBn_S<#kd40+0A9k+jMtaX!YEgoRNa82y)Z}$dk{UD0h zXjSQ2wd;`to@nw zz4b{w6uUMhsZp<2PEo5$WQGVWJ|}Z%OAN?5ql?-!Jee(h6(nOB z%zkJb8z=DV%pAh}L+j8SGS3)Am$ZL4WeSxd8 z=LNB!rWi_VmXs=2KuC$HMmewH&*3iLpBrmob+m$jP%yfvP1?|Ar%}l49b?EkX2%j* z(MKj>oQ&U6rAT`C(Xlpxba``npUiJ_mI9C=eo&-|u3Vf08U!B;`q*S0C38Hk7Rp|}Ar{~ca zvSUE{gdfnp0BzXuIoOvS8^pcJz&GwQiMYs`rhEpsT;sG>FlY&i?k z@eN3%1OUb>_^TQF^@)WRLf9>7x>+J;Nc?W-d8*=h@PR>0?W*TJ9QbkGqb(|%ke4rl zsOiG$K^Vs)%WLyDNJRSp`56IM!`wA$u4`)F3&6W@odfGTV4nkv15PCkF2D-l0&@tje=X}plSs=q!go5n|4V0Pq#syiYgPhvItk1ee4X46X4D(Wp>9UZ#$x= zGVlkB3{8!{%7omM#$t77Luz_Rqtvu#nym+?T$Q2(cnQMhj9kX-a-9hIbPhqAOtPLk z=hQ7f3$|PE$?HPdEwRl>PZo| z<=1JZbEL{w=&?cMB23AgDX|zpI?JmXtX1w94&XZ0?-;I=u00A`+O>a!F^1Zy3-?B# zI_5(Yj!rEouxOd*Ed;Zi*Rcf~aQQam>;f84`pfLXnSrOFmVYl+Lh^LooX4kG+GET`JapXetkj*BjgwwIjv?%zC^Q9 zLXc}TT{+zA`$>v?oEw^yE5AoXEBz8e$RP+)IYPK))T-J^0`M+e=Loq4_BldkCeUGu z*`lJcP>+;F1r>=crbHEpam2K!;SMw*9l20qU6K*c1W4;bA|l98$Tsmy|Bhgcx`TO0 zMj65r(veB&XlRkG(#G5DLyoqwjDY4@0zeYhGJC{iMi2pnN&>*>FajthPU&Nb3rRFR%%sGEV^UW0e1@LDl__GCn z5qx`se+Bc^#Z|0${s>z4=ENfCJZ4-tVQ;cu|Kh;T7Kb*ZN{0qC^k=lLc9w>AXIU(X zWwAmDGFdHTTrKk%$sDg}8RrD$^ZD~LqE64B@`s=4(Dh8~>~$DF4>EQTZTeD`grE6o zm`QRE?fQ2l)n*Umb$SK$H>`oB-)Wg<5~l--w6lBx2${@#1NUW}+a<#qj?(2M=Yp;V5(trz#}#eJi=-%#Z@ z9&Huxl{P~^8%R{tR40q5t4etaMdci&QIckU5`^-3n)=&BUZDBQ2>#TQdAd9uMQ>Xd zh{ZIino0)Uyt+aiZuxR6RlNJOApR4Knce|msIzJa;uVzDD}4TmV<7z_tXrs7bSdyp z*B!SVcb+flzgg~>6#5sn^!`kKcsURHUoBb5d~_PPjbHi)5Wx;?pp@^J=`}()5+?8P zdA4K9*D^$o%;*3OV?H7aVuvt~Y-9j?iyb&ahhl#r!#WNUs(pxFM4VzQ;DBW%ylEa9 zK*GvF>GC^=$S0gVl$gyNI-lfaqF((2xqwwR-Q00rLtCd_`3|+nMU> z6+&EBgl!U{gsL2Ep)@~P=$!gXV)JMs!t3(qXJDuFTLGcmvVRv)+ zHd6JZ`{mW?g({LzKtHW;@+JfXrUSy#uZ#oZz{H6GXM#gRx=^C&84q0zXrrY!gcIpB zm2#pe@KROU_ESI0WZqpQO+t{eNKiSs8(-QN4v;u$2HPJ(Fh9UyQ%IB3#WwQZ3pG6@ zlk=66_n)tP1uI9vfR!7?%8L`vc&gTkimHancWUX9UAJ&ZD|lAX;>Y$0GUzf2`poH+ zuu1ZFmwO~kh};Lki47=SNYvd!Z~cDNb2^M7~R!(he0@ zuNBH07ehaWWNUPs;4;7?mUpo<$^$fkLT(d@i2UdBh~hfMG<}G~0ph!rnTq9Fx^}s9 z!NbT<Eo*p|AKEC5zO`_|3nw2@=YCzbkU*^n$eQuj#FDz#;aG(4$hKDIkq0A zy)f?b#_zOiNpE-J`LaKm*aD@8ltL`C7ZC)>kI1Ja_5FYCxN+luVn+uXmE@dfxC>R5Hb0 zVXr<=s?)5<&a&4r&I~Ito4o;UmX+C?>@9FvR$*_mcfcKE7uh-XF1R^%iJfOTaL3sa z`yTr~xD)I$yTInbon%+o0$T)kifNxJxgTtOZr$PyF1TCgSFAPP?|D}i7n?1At-rcZ z?{*e@_x-M@uPxqfd9ALyxZ3WnE_NKx=VDP-MCF|><85zIaL?ubLhrtHKx$aYsllms z*I_l!cYI!Bt-4=ph;FCW7ww!Cn76pM!u=o>_A4yKT&Wy$m3)wnYV(C5TPX(9vUJn0 zV^EpI^zpHf9*xOmdf*`QmJu?*UK;3wUVDNT&^$iQst8DdgTQsoG;8*SZTG) zmKrbIw^E#6EffnJ1}OW!|J?T-sbv0Aoi)mprEuogUtolx4(FFvoyx^Vqw@Td7}GH+ zeg1N#;nbaS_!!SMiLnu+mCToloEHj}i!_N`S{N{Gksx9cNfJpBNfV)%BPNJsKn@Ui zlsYwz58|oK2ROY~KzwCg9Vi3Dq=7at208;}`fdvTX>6S-ZIIBb=LN=U%khE~YqcF3YF`U9 z*A2`MFnW>Y5<$A<)_QHn1g2;iHT6iG0avF7h!5gPmQpZut8u|F4I)WMkxUmh z3*#aYlW4n~)5SFY!`U7q?l=faPZ585SVfbUPas|e7tRg?zKsh{15q^b8fcx^@k7oW z1zsM6D_BC5s>akR)YVk021|dXG*ns`9;B-YS{x9*&94+gsU|UNE3j@jUaNki-K}rP z2?rL19M9?W+I;2UeGFPB^NA1Q$<9b9-q#?$;=@XIv_0)1)}}#0p zQngykZTYpDI6;>aHyppdw$c@yA`7g~oOYl8P>8Mwj;(b4k4V-X?)sd`LTQ3Qg6y*p z=&NoaFmCZrei|5pxBFpvk;;qWb&M@7E0+!okGC5^N=}Dd#SC30G(6dmd?m0(&no>U zuw2Zl)^_f5fyYVLYW23`d6>AgNHRu2MmUetBi~T8DK({<>I^7;xU$qKH3Mp@TgOJi zS%@W2m*z`7am2PGo#{&y4AgCCbPxXz)oo~TPaWtH2aAX}eaImWg=OmrSehz6MvsK- zi`E0}k|M{5d+MJm`|3*06z6d#kt6bbA{U6z6G;i~8)5-`pl(8Z40>q!_X8b~CKSO? z4neZf755y${tT@>(yO8xnuRB~h{`jC|E=tp6rwJe1BHVz+YGPwQ)DWx)8VyRVAX0^ z+WT$dGqu{CzSE9s(tNY$xCC{ANoc6+0=T$7X^C_ru7Jo2QeB<5+p+rO+)LC%Dw0f- zq~OsKkWvlR)Zny_rn1s#UpTf{GPVG6%Gd(fc__9_AhvXUzs+4<-#8$}hxQuE!N&EW z$U~^f6^S`zJMEi@B54^Pl86s!8ADQjdO$yYkQgL) z5{MGkuxjEpt2GS*~etT9%am?(~)ynkD~ zf&bhw@fQBX4I)xCWG3PrkU(#_zKr2x)<_ODW{qT6p@Wym!As(MR4+|-AD)|1lR*Wm70ED!>o;_jk?DdbXPB0l2qKSaG)CL$$zkvJ)qk!-G@ zjPx3YyKxtP+|ftU`6G@lesK6sBc~)=xka}-;*_(}iXL-Hf+BQyH$jFA{>+#w>TH&B zKx9(}<$-X@1m%L{oIJ^<*z^PVOf@AzKpHD%R*=sf{0HdrXzg67lSjtK1Q^1S)Y4NB z(Vyj=@FJy#wI38KKWGvwKH-9E$W--j)a|67f;Xzn04n&xPqW0HHAp^GB{m=*ofw#) znL(PA?)e`p%#=7mU!P=H@7KdD7vP3jyVP=u?zoON;HTlUGnD;G)G&*yEHfYvjZN+n zN;n1+nhQ}an>wvLq_X4aO=Z&qjcKP9hS^aao7tt=oQP^?QJactsa05eGOEp@)`)7U zRakq<*Y{4dV-IO&udwVuU4L~>0e=Rwc-NA} ze}gh~c48NM0d6^JW1#lU22c=Men>6%-jXe~hnlJkuis`XUOEjrtR{5xxpzHihOu$FeBS z5oaQBbR0j-WMu3;H>)XtNZ!X3=YqxDeH zLv(9uCYHaHo&vHpRqc_y)@Dw;g6kPHC6tg(d{Atq^8=jTzk(p5DBC7ewv*sg%A>@&0iZMTapN)!w1%8~+dwu>+0`|) zL0*K&qR%Oar_49@59mk-!-Z4oju=JPYlw(p{z^~kuN-zvXCPf1*Y&9 zR4PG>#5H7$Vdi9g%J1|ElSB2C00*Yi>v5Nj#t9O_ahp7{C&8&J;iABlmgR-`=o2a? zKqS}w>TFN&yRB~D8^$KlpgwAp_l(d0S;fEL^eAaiU<^RbIHz!)!kJYw-x>)6YEHpO zNd1%;Y2RALRyJ1p$U_h1LbRR|ES7rT1cN9Y=0VU2@*sja#6brUrw=(J0zm|pv@j#u z3&9fBM`+bR8P={n*bX?;Ph*7zUM9^$r#g^qq6v*N?f10b8f!-j5{-_7Etyn3pCfy5}a5|^k>->e~@-Nc3`NT~`^ zMwW`$!u6u zz8LRt&8Qh|YbKiTut6>GG1d0L^MxJyQE?&$<65yDIuxJFz4G{+`v+Cvngc?dFwz>>S^ zwDa!;(??zk0Pl91yB*65YOm;fmI4wBKH zf130`U`avrFeq9ea#*VPdz38SK4h*mO%rEaWyU*Z!#EG&$<(-6j@!Y&5)S zUyXOh9kJQEyFhOqdaSWns1%9|`BG^Ko8PQ)?aP`R{5oWCMRIhjk&x{3tMKsL75k%x zeOvSa?;yXvZ~^C{`JwH6BvI_qg%iQsm$9)ztvx;xYk9z$_i3nDwb;rNcDG^sYut|C zoe*a9e1@$#Ii}nvhY5TPaULQ2npA?^4cXTP8LCvW_$6^8dzPCi6uRmCUATFioP!$W z(BG8;OXZge3}1d7RbT>=-#28wAwB?qKrRwq{sVDe5jmnQ>hszWZBf~GrL9W-rM8}| zEvcMeLDaE#-5~WmNk`Q3{1F{d+5byNZ(zVUB<2e*mGopKQ8InVrDPIHiZ%l9s_V(>B2&Wg+GswC_jOCT#474z+p{%dcj6$W{$eRvL!%vToj zmkMQkpfXAnZUdQziAf8)xwyc$2HbYM=+-vhtn;2P{hw-Ahi_PH>X4vvXdwgplN27!}`liK` zgGMvFVDp9M0!H`za7gMpl$MRu+%D$J_!g^BoF^t6ESl#9`IP}W#1b6Jc-XIyFP3G; z1=;gYD4n9*G_>Xk;+`F?s0rbma;GbG6?4LOECe?dkMcdkAXnGut%;h|Qlg$>M2T#;T;Hrl<(`A#V$b56{R9; zfH`yebWhLB>GPdSZ!k7iv~c|UcYj=)dCju^la1lSN8>G=@h?E6B}LO(Pp_h7uiGN! zops0bxh>DOgw^ub{j4X~vW+CaUJ%x%CGDSx^`dm7D?RDUoXr1|w>~0&M;7JCGqFDU zO-qi-v1gVX3+=|(p0i$pq$I~78P6o+kW9!)NG3DM1SC^(8j|TuG6~6yoP}gIlT67k z$%}Fh&zP29mhz9cV$o0YH1m*o|37vyX56auC?yt$Pmo%r^`f^0vDn(aVd z?|hWBRc-5fTWv11+A?g$3;hRPTdgfEeyt|M>z!olv)sCJCM`$aAozTx%KvEc;UsRYpZhQ;SG6f<1^p8x_s^7((+O$m!XnmtK#W=Uk!b3bMae?5iwVO8`DACS-CgL^ z>dghdRo^O%8Nq0Dj#9ky3~r5yTFV|4?x*S!xEhFTlybc8-hxB zRsf!X&}Y`V`?2L(>mF2+(-U{Mf(Re_c9Mh`fA5Q(m5lCVs(C?moU?0LU zB+h{=#UUhB54@l8xx{JYdjhLqUxJG!UA#-s05gqp^+GOoS2+Xz5IpU7wt}aoQ_b>rXQQ^H-4Da}zo$JFHoHrdEbH9;`ev(LmpT`x43^xD z!{&yf(P3x+{T&His8(yuAdc}iTyr%~as`C18kayUQ4*tK)SeSXF=y}0pIw9N{m-e{ z9Gx8=t_7UN&ff#mvkrJd^9MFgr-xNY|906P?sJFwoSxltE?T=-jRUXe9>OIa33x+A z!%?r2e3_(7@+FejK`NfA;G)G-oue*c;5tXWk#RuMk3IQfo!e-uCxMcjH6}hSaT$Mk zu`~DF_hv(}xLo8Tp9>lVt*~0v#cCBUv)d#;TCG0n2F8a*$5wphJw($z z@z;cnvu&%dN;k5lw`&6`E@G<(K-rA2`(7Vz1}HfLl)_rYRlScanG zNM|r=Kd}UJ|8;c6CM^*qdqPau7sSr|S+%{`?rHmXIR)+B_;-2?xV3!;%CQ=@@gS!C zXZEhM>q2JieAC7_ci-M6N(>4N2Sw8E`@ZD?e7Pg*EuGsA!wwi&;@9ycqzIk~-P&!1 zL8KiRymo3qqJ9s(!?;lVK(#=!2%<#>{(%-~sW)+ET9^di2ZRL#2a~)ZICv&FFka?$@lFueYj>LM z+O~1QYc;Mz{~*sN8g_XTARb_py$6p~ zEe?$uNk$>%h_R%!$9OT`WBi!talT1o@Xp!;_}g4PLe#B=m8sNbX(y@867cCtnXoW& zCrD~rYi-p+KyV&LOX?~HXqP@-8>?QXcsY$Jb}caa)OH-`B6GPoXmy%l8qYItm^aU6 z#P`$4Z+g`SeDFCaxC432cVPrz3`pZZQRHo3>`V{TeLfQU+BQh|OU%*}nWu*}??Nw# z1y066^BL!yaBiP#c+%<$^`40UyceT;_D##o(ZAEVZz-A=B7?E^Z-Q6fB6*jDNHiaM zh6xOznIm-)h!+7ZM0sjJW7B>y}DY*I6ztQGRUUh;VgaRT)Vl~YeZ{Nq5 zJaeyX3^0|A@n@KO(JykIVmd}bYLLhbY}AC43}=`XeFk;o$%H}Oq(R-3LEUr~X3R%1 zUKp3pc#+1EadLkI@<6=IoS77c`QcWhU;z6JfGsd|{bs-xUkunk#_Ov;Au-wGn^_IZ?0}d(kbNILIm!+N`td@txGxSFyb94zq9|=_iKRVB+jLlof`hI?9 zPVALnUS@sg&WYKJ=pV<&PwYE~1bQL5b7l_i%>0J;I&Tix+6dDsIEBf1%e?t0zK60Y zI*NJh-^gj>SjIB>4CLcO@=_+Bh1?sGkK=h0@&evydY^%RQqJ~7V`_$Bej3l6Ikeuf z{#@;P$!yPPch82N{;lUVF7RE<70<_buW=E0@o}AN%pY8mlgUd5m*vHt{}S|mh`p@n z^=yuund6gesA?hsRr$vidfzsk5b>51g$o-VYyj zS=l2Z&$7$-+^ggm)OBG{1&>r)!FE_}1|Nk=`{uHgU6-@#N_M@W3xm=+w-E+OSB2^U zC(Lo@!=%z_*@nH4dR&jbh0g2O-u)evKXqRf#xd`c5(5xci4w5a@ z*N;@9y^T5wJ5!iwZzojkY9*IequxU%r`cP@dL-($(&8H#nakzUqNPQ2UU2vuylu=d zWg!TZ5D-SiB{75WK;9J>OlDI0bv`%XeSZd_gMSnDD8fhycU(43aA*8Ovy*rdKum}{ zl`b>npOo+ajE1y!QGQFY?ZlVqClJ&{L(Ev)j(R_V=Sge?@gX^o=$cUOq1Wq=rj_q{ zo5X;R1OVR4%;fgGJ(Tva@N8Bf%Sv*X!`ZWetqZ%k;qiM&&4u--hi5uH|A*Xmano&= zd#CI&s|}60akrxc6&P=qk) zVhHiiaAq;wh2mW-j{hk#xBqrME({l8#@?CgtAAj^r^;^^v#?&d%h@6jI2nX-1X3sr zS#zI*vND<3O*vIHiUaQu;bGoF#pyS4aB0k|P;DYN5__CU9AL-mz>`oscEM0{vS6>X z&h*!gKgD6JLNP5-1h>v-(1|OBG-PkEi@;V}By|pQS-_`?ZkYDtdibO=lysX?RlD5X zI;%k@-2aL*D>{mHKi#I@@+d%eN}1*c5&KUuehmfi7_PwC>5MydEnFD0_449en|^2y zXByb;(rd*D6NB&7_~reBYr?f*G7_`4U@p7~E&oJp*mbPXPsI;SjicT%kN^M>idgFa z0*Bnga$)pHn`r0=x6nK}Xx!O(ij7(9vvz9=hGVOv z;KYM^6w31J524J;?ee=Dkr08^1Jl$_$9&2g|(qWKTGHU zj`Btcrs~iIGgWn`-PU>jPEhTtrY@#Ju<)Glnn?(6T1&Lw?Z82Xdfbd0bR2oQICydG zZ)495ZqGB*=-EM6b#RX^MeQd;?Xypv>B}7eI z!iSnTd+d`lP5N*3wEDt3 zJ;^5|-zWJ2$$6@zU-&Pa=c$r?zcp2UbgmOwT>AWV2`@4*!HZOsX^xQl2a@yjMpOL- z&hzxfzAspBtINMNy&cot9$y7X^)^cGg?eKd$n5$&y|J$!4h4RI8v378-$19%r$6sT zQ5vOV%4Sm;erIgP*9VT?GXh0KtWWoEyS~wlYW(pnfd3!j#XX&?Rs-{+hCvQDG`0VM z`8~w!Ql{G>cOf=aa~fReq7@yEB2<)dU;Kz6e`P!ub1a?zI$jPxuY7}HIer5qkEDvJ f!IYi%Y?1x_$}bfCsp+YysaK|6o%(}fDgFB&&ZS8k literal 0 HcmV?d00001 diff --git a/convs/__pycache__/resnet_scale.cpython-39.pyc b/convs/__pycache__/resnet_scale.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..025e8fea92d5f37fd503f80733c8a94e6b94c495 GIT binary patch literal 12425 zcmeHN+jHAiddHn4bf;xozQjQi$7?3mg`{XJUdOQ=uhOI&O=^4Bsk$u!@qiR15@ZfQ zOJ<;Ur}A`nI^%72dzopQPNmFtTEFyf*=gVK*6GXYy)&KZL)_nY0A3`^iM^YboiRB0 zcyMkwIN$B}eSkAEVoEq_tN&Sk=b9w_H(d-K)Oro4|EerWOj?$h%#^ykEQ^1op)RYk z)EUQ>TGtx-vMx(^C2vZW=qGB8xb_bvPnnad64Snvms3n<21~It%P^A-v0*mCM!(1` zr`bFkXA=+P>>& z*&FOll#a3QvI}e)rQ_@hn_;skO|iGx9LuA0f?Z_=Rz&F}Q*KMScRF{?TW-bWu2*(1 zo2x<4@-NTMR%^j(du68FY|OUq2TfjHoxNA{YfW!$W4)Hj;Ut^2dw z^*uM3Y27#Xh%HMwIhv|B9cKH16Sy|3l>@uNn+?0o>p3$rZn^%F8${`NTz;WjD$aLH zrMW1Rw9e&w#Zn;}6SbQ`Y0fSdHs@J1o>b)L@>H|HqOpw{bM1IEx)q}FfePwY=)Bc^ zWY*m9Xr)-Yc&?cyug>m!s{F!qshpp?SZ3}_D_DJ=p~d;>VzF4c}g_C*Zvn3z8OR;l8Yo%#4No@o&SGYBb}E*4xjpD$gcMdUJEgJ^S| zh(RPpBuykkglrHWA~FoJ2SX&4sdM+BGWB^Er+*S8kk;f-3Spx{CDcNdfim@B8v3bq zP8TEIS^NZW095>CNG0&Re4!snV(1%ncT&L z%(6}{14i8ViKI!(DXiLz%o~nhE8nO$%j;snkx7QfcN(p_yR>%=la@({x(Ag>+Gtb! z&p`qyfUImQJIXFRR+wf|TjHk!bxqq!hdPtjRsTY0FlAlk^MTIPZTZjTFok-J-wX_< z(RGjtNz&A9HI(dhm=2X~%xPyZi%Cm=NLNGXrJzMYdAQeS1oA1_qalqPV-g-IwIrT{ zTUcDD*nc{!>DBLBe!JCb^1!!Z34A+mIUciC9N)E}WWnm&7w0?YZzl3+UA6L-*9^L& zTxQigt3TzPM{<;^d63Yun;VaGOzv_{0u`lgyXMsb+vdmUa_EK=lvkIUyis70`9r7P zcE87YlSfCEn!)?D*A3STTqY`o2u3=|XFgI_ynLkHa=-tBNaJq39oHAAzQA9`+(NQ) znMm{9dL>GW<&eS{vEsynCpwbGL}ve4h4w_IhgI2i=f2DFI2qe6*B#%-!i7YVFme*Y zX`DXkhNO(jY1xp+LGiZ5s~0U8nM}ZGYN!>Pbt+-E27?0iI|C9BO)i zFJ3@gBHr-JAfka-S9a_5F8jpVi_}Fhl0=hs!6!$MmNnT>ptO%ivqET}+qRg(wg89< z+X67U&$bM~wlssF?s{%{eUBI)%WJ$37OwYM7V`EVhNOc`NIx~y!_>A8D`EE9hA<+7d8D;r%tCf$gzQRkO*Ur~ z7HCYEc)A)Fwq`=unyj!j+ENt@#qrU#JNy;==Z^5#@W*cu5v(CG;je>4YRwCT4Ihw3 zQm6rGB*BUmyhsXOB5ZS(}n?hRoEH#nQKcL|}Pj%#{cmX8Rg6UK21V!J-G8UPQj)CP zqT3x%$_XJwWrAzngLH$?+PgUY4?qIg%Rt$I-Q1;3k&|oqL~*SEs_1Lb3yL-YR1j+> zH43h^hYCD4Ja2^}FFm$R)z2pX{k{L{{LMQ#CDIzM<3*|;u$;`_$L(^d*qvoQayU#D zVy;LR!AYbw$>-=Dmp1|H$n`}jax6~W&Z&{gY7MwYf?oJrtyP{dt&_rA`hFz_XtwA? z;^dQEkl<~`l`Kwv8&^+uBJEzy-QediJ+R{?y2~FB zA;CEyzfp>|(P7R{(I$k7wr>(HdQv{C@Vlrj{qdh{ znZc`}Nl&}2d@l39MCrA)v!S{6Iy*{fAu$wU*H+rs6&z8#bPP z2R=ef>y}8-#9^l}Db_h_1#u*|AVxyea_ZvZA|J(N=l8fP;5qU=6oLZe2dl{X#PKE+ zui35x{=3!^?3V={b*$pF8>4Y6XakpDu&M|EKHKV?x!-Je2ebgQEQeQpD`;AAI%khm zXb&=skWU*3`6g&lM?eyQFc=PsHnJFdKL*Ju!jngaSf0BFWG2-l%!mfUOo9M|I-X6~ zl>C~o(MWVf!yclWn&&zk*flpDgAh_H;pij1#R+`$vB;3!NBuF9Aixl*0Bgd=LKHJf!rHmQcQ*+MZIvK!928%Gt9A%cs1@?S{)i^>0d zG;y6dt)O<#y?(FSoyfUYrHQyeL@A2>;X;J*kB0hQBA;B4N3rv`y+F@M79ck$AadE| zOY~r4>rSl!;SrAwpA;DzZmqf+M49&}6G4Gll&XN7(U8|}#94kbamU5SaATLb@B7 z=nM?Cv7O!}myyg1)2F}%Ag0__VEoQ)nfu0e$I%MmQXU#K_R}4hAwA6ejIs%8jEbPr zSv=qYYz=W(p;`g91s*4HjUkeG+x@gnu_V|}iJQo9S}oUO{dpptJ6_dIWQuRlP{Q+( zA*9QXas7{|x zYA=(JWE6(gapV+~uR4=mh71nf#}e{8WbOq^2x4Ii1xs#AaYQ8Zhp2@o>8J(g%V;D*vyB}fzUv# z2QUi@XF%Q+dY?sWtU|y5M_sF;8Un-bK2NEDc3uFLz5wA!&SbPJxD>ti%Ns&K$)w zRjz(wi^;EJ4=#%m-KxJy*7+6acaNb+_y%_2jj#w} zZyaFP?r+PkS1@7MX3phb@au4XQPjRq(L|U_UX1S$IZV;w4?O~U6E9fz;Q`kD5(9rL zO&ergsqlhzhqI0#6#O0WYLCi$0>C{R>q>=~bsryK-M@Y<>j>%6HxW#^A|dsN{O=V) zF1kQ2#v#Zh{;0DI9etT$Uh^JAwuT$soE*JP_Pn1(G(>{ zhGp%sM&N~pKV36_1NM@W6-E~9g*_ntF0dE;^`8Z={u{ATWL5r!N`k)>y7DiH><~G` z`iifaT2J9G4Sza+$v$+|7iy~twZ-R@PXrvga|tojUF3^N9ieGZ-RVQ@aJNSscRYhe z&Xs27F6HO(*;YToxPuo!aYR!?I+@-E;mw-m_{ps;ceCua0-^uZy3%{&Xd&yk;)<}K zyy|%P+R7J!z{Y9~@0Dn>jau2Y>Imb|3m>ky_gsG863L|*i*Ebm1k?@5R`kcWM`AXL zw39Erd@*UV!V8u=Q=P%=eh^PdLwmbrA*QzqbMyE@F<+RbLOfZr&I{tJ8w}{~U~i9m z9uQI{=}=3x}1p#fV(E#HbPFC36%*r^x7j6i)r z-ChXu(~=5(VI~3n2GdMk9v&*p)Pnccz1LXrYdSgpQx@;!zUe#_rEHru%lNQ~l15QF z{wCVd)Zo)VeQ{Vau7gH$EY!X&mu=)Gz3HDEE}4=RXwhjzo?m ze)V&k+U^ojiBK9#WIs4X(tMW)A$?Am@vskV2jX)Psh>i>D18O1!xvsxu}zY{2%>4E t^a{P1k+VwLP*C3Y`(I7neONfYUeIn?`&m?P%+{{wN@B}@PS literal 0 HcmV?d00001 diff --git a/convs/__pycache__/ucir_cifar_resnet.cpython-39.pyc b/convs/__pycache__/ucir_cifar_resnet.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3a1c9a549b9646e44a10a2da0064ce86941f50a5 GIT binary patch literal 6680 zcmb_g&2JmW72nw}eu$E7O15PuY1t%oO2-x@N;ZpW(B74)?SCJimbZIewAP<6h*K_+{K@`78WY z+~>IdsaAfi|Ihpbf87^;==tyE8*$u?-m6qLg1FIJTktyV%4VY{gxhYqTRwmHW+ezc z;kW%Tc3an5zAM5Y+^DRzI%^fT7k98qhx;O`v~g=ygdc@|yj)%AK0R*fM%3shHJOFJJ;EZuga*cTNqSa-$J;^K9$o`Fe#Ev8NFqFAlTN|HrI;|>~hP$<~Z%Q-r zTkFyY?}gF~f6|MEjamCl<{U@K&T;m&e^)E*Fs`(q0^wKq<9?<8~4fRq;CER#fSE zfk@>z5GuqevRG#tS40bHY=#wBe{M`J3q!^{;&b?;K22D~6_wB=+OC$cJ+{qawyP&B zVY?909$VMBzDKw;YJ#L8NUUX3OyMT2xFfuVG(0z!M!VPA*9Ea6*%MKuv8tIRN@l$y z9=n1QB~qGVdDfpD=W{S(-zHgxZrg_}^Ns@?^;*;~IL@P<+e+W4V~b0)^z=Z2p5JOY z&e+Chi3QqSX{a#VIm-%`qMSAaZ@&OR(v5=PO>`z9NU(#HbC%z+tkp;B1ePlRX_(YGzmg- z>b4tujao;vLDJVkK~fP)P=~rwli_(sh?<{3)a{cIMQ2rrqUj?wi1RDu^Mpl zUE269YA#YU2^Xc<*FYbDI)4N#w-*Lr5tk^9zyGfNR zuVX~7liD|NA9|nTU_h!DB`i_4Q?*`2f23CNCOs+{o}@t0=s% z-o*DnDzkLglayAbU5~Tx)t9S`j`+rKzET-cPFrPnPFC4%YC$)lowR1fJ4Jb@xvT|GsOO^$^=YqJq$cT+$VQmeiO$`L?Fn>@ASiYs^r*B(0VfV8Hb)?}U^WMXAOHzn`=ELYM|LFUO>=^aZ%fb zK%1GUsn#$DYnX|tvXX~LqB82>o)0@ClnF8x$?i&1_^qB;B+?Q!hmt5;N@9YHLS~Vn z1{|q{%dM;aX@X~%@6`dG=3PyP89L^p_5Wn+NKaz?y@1;ne= zP@I!F|4G*kqo5O}aG=aae2bb93!~shu=zeNWsq6$F;fskJK3Nhy7EOhz0 z8Yr&Q>4xqB{YhJ%L1sE>%VeSR8rLtN>_eu1fDdM(kct zBQkqB(+;06a)X{9eWiR(F%3coR;u4H*S<76csIATJuYt5Z8LCx_GU$T)kFW zS-Doaar0VfX=(95L5*at0Gs-(gA$$(K0x!Z%w(P%kJD{Hxd-G^k>R_cLpJj+o=`KH zRB*~d*a=%fh}toTpO)uSuSY)TsMpIYueanZKUp4mz9q3jOHVm%chh%T?o(gLOcc8t zzOyv!REM2qkym`6-s+ROsqvdJ@>E}$E3I3e<4 zx8+Ftu^((SVwnR^G%|L?8(3`UW-S$9q^0C3vtGjuQ5;6Hpcr)2+eotZaqgC5o#=Hm zaJR+a?$y$$S;C)Zj%~ zlg(+HhFFZ@okx_!+%dN84Q)GUn#h^h>@dp6JJbW^Fvgu>?7ygma;|KfzAMp&JwxJg44EtQrU=)eUWR@_aBnUMdU z`r3gA2~8kEQq4qfEUE}q)U;KsgbUiuB+$^dl~}u3Tsas&&HIdkBE#y!a7~d50P@VR4U|A5-%YH6K$`LnF5EA+YWXuv3nDalfJ`z5Yu4vmH&I;TP@h0&s! zW%_^mv*R)y#7~btKn#D>r>_`Fo&|IWyPRFpny6LEdQG^XmGzWwN&x^Q#p@(1GN&9F zd7oVjifBnq@1Gyv{%{1AAg72`DHPp>%#MWdjN|Z*hp!Oi5(VKW{YH`PiVi*%#ZN_( zXeS1C@{B*n714nZ*sYmM`C|5*&IG+7Ngn>_{KSPiKly74fRr1-q9F^)jrwn`k^>Vx zFOEv?AW~niE|$nOw^X41xOVT;hYJU!26ZM>=CqT_CXiTJYUB~^$!XvPUlFT|?J$Vq z-(q}P^63R89^vV9_*|~Oz^Ax1PtnqdhhP72iKPt7epHioXi-dnM!JZpv_2i8sIJ>+)sn232}{hQW85+ zSOcDxtACu}DLy5hR#wJTc*xTMrK&HL)5^-9CV1L9C7y2FJeDWr_g*Ha8#n(v!PArf zpQpEv!xK`jW0_L?oXAuz1r#R#b%Lutp33LNCT@ZZTV~Z~+J!JwL9#wOd|X)Xg$F1u8#5<}S#r6pjS}N06ojiAE47WA>>mmHfZV i{vb`q-X|{>mC%?LzWBiT7pIB^T+_ujiXUd}(f@ywKY*10 literal 0 HcmV?d00001 diff --git a/convs/__pycache__/ucir_resnet.cpython-39.pyc b/convs/__pycache__/ucir_resnet.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0f2194e4f0a49b283f5c1236b5d35fe2b0c00ab GIT binary patch literal 8627 zcmeHMTXP)8b)Ngq&f>ZNFOVo|MC!tgiOXUEl42x=q-fZ3WRxN?wu8hT4W?&-1$Jkb zJ+mZ;3@R#a+M&UBwX{hjah_@kqag6rS@;^)DabBgjG^e}u;?RDJozawGFn!;43HPtm${%b9L zO;?rO2|Vddqh+p{s`7m$I;|@7lciQx`d3P%Ehx;`P}d5~WELwhn-!VEN^FFUvaw$k z*KBs4O|Z#_>ROS#&dO{GIp{ZB_U~e*)y@uQ*dyAc826APleXLY2?0(<*D6EG(jDqm0vza8F z`0D)pMkCqmuFnPS)_muF(&oYD{M|;}Xh-ww&G!0y%a4( zBD;EJb~&gnTnbn?*GV>Cq-$+ywpOdvFNeX>i*{XJn617YTnTH-mu1(hFEHSR>g+PB ztuL`F^%w5Dyg0jFU95&|5h}rKuRr&Dcar4=ueP|e#AYuCtiH5-B^&vT7m!e0sFH{) zZ1&2!zkI1)Uw-~U+-tDV>n*UiIQ#Z;-4FbwY#Prsh#?6ykj2^BVi;Dd%a>>nm156) zs#kis+7j1cbfTEO7M=Rp@mSTM9mC)Xzm>4RMV>ndtg1{u(&M(Rn zPP~UZehEpUY^kY|0wYo_HBy}+W%|C2-Dd2bsqNI88@r*f;)Fw&qL8-|)&_5PJFzg<8-6To)@b@9>aixYC=%9tkiN(& zrYJTduhaCSFcwa`o8);uj+gLBN~VxJR`?A5{^#O+D@x{@UA!!uXZLnD+uiw2&}h!n zVB`62(BS!MVrz;5m0~Jq&^5M@DB6f>KjkkY7tkzv7v&4MrFN!~8P*Qdpchl>WxTpm z9o7pULEDvz&d2bd4Y^D?5`ofFlt10Cq8Z3(a9V(DbP#wB53zKfJzNNq6ZGo`K)T0bnVnS|ItRqHdwP}T|?li1Dd@ZGBG zN6cOK^AY zFBZ1vHKIo1dHe)DmTvk?qOjcsi~glU?h1?@8#u)|xvf?T{4KnXU!deqD4|L5it_dcFRL5$va^vgcn0r zn?0{Qw#kYjzZH6(a6Au8qT8hWi09qu`pvwi818iZi1>{d#fr5fu$3sGRim}ySCPmH zTFM}7Hhaq{hhL*6a$(78NhL82##Rm0(y*(aj!j8>4p4aH`a1L}N%R3zl;{Jtcm#b$ zfIjUcX@*f4Y#)*tWLnP@K|A*-_LNXUNUh_Je}m+!uqQEr78X-qR;Cr8i8-z8If(_F zaU_BifFO=U89OPa4t{!SriBM45X9-%mC(b19A#?+QV4lQ33>8oLZvZa)VTBl^fV#S zXi}n4S)$RDL?dH$1H;Aj;q}|x#eZd*6SVW2l*kn#fZ(qq5qcv^B-jrbCwZkIOR?#UgVoDz6(k;PV&B8T03(L+ zz>YsXiU)id1E~~;lzt2mzKMFiLP;MR_$7LjIvOa7_Sz=y3>=3s;8{p8P}SEV0sqsZ zFB|MyvZ9aZjmLIvS=!g{-?g{Ur_UKChM0oRCM8#p41$%gYfXysX!nK%Hpa#uVo$Bv zl1CKbZOg*?D2!Lb)n>k`=_scNAt(9V?*3o>A^9 z{GSv3(I__X7z2se0vW+Z)+ngmw@@R?4ph0p5#;Wx2ShR^qJLzgJqcf^{BMvSn^BOC z&nVj(N?tl96(KPs|6Zw(PxX^{b9rAsAUZP<-8?4aYex1g8L;khM&a4ij3UQAg;`SB zbc%V-C~OiMMIAe_Px_k9YfqwfGOwjpS?!6u_7rN3yp~#(*y(-J^GS^O6q`Aq^*YT? zrRvtqw1PZokl@&e;yWaga4 z#*Gn?G(j^%ceNdbZkxNlTbm6t*5@W&@V|@h28ib0?(Lqv-|qH0bl|gHpKru&(sr}J z5a8da`&<;5pMbR{D-~lX&X~YhI0YM-vjJ~u`5W-E2~k4O(V!VqXcchDHxJ2+A7VFB zSTYD!q!5Y~4?m?UEn}FLoK1KLR-iN#bV48LZ3usaL;Tb0Y+^|0KxiZrP$NI`h%ynR zRy1MId59Q^K*k$U=yOo2N+t7%-=+gKnZa2ZfOJ2+04l}|lEXjMT`Tt`6U(5fO#N-X; zcap~4@W$PZynn&JyU~FtE*)}#LR8jh3F`wnE1}f`QOF1z0$7|rNmdqw!k}dkES`*o#F?)$VM!8%LAEH=*0UJGk1?t^L89nc zRA6bu32fa)L}01fQ~9h-;OQlG`YA;VWzfu%M3dFAFYUr0ZxyByQ&?!zOKs#w(34ol z82V8Cv}vIPWw>|8GcPn?9X*0d10r)I5A+`HcnS6kL{s+QQxK@)z+5PJ=65PkyJ^l1eihb6`%6ZCMXG&K;wQjjX!JWvm`J$nPRGj)i6 z|K@bZM>;n~HbKVEo2-3=bn>g@$I#W-BbOI&c+$$ZC(d$nmEii50A{lpn z^A&A$X6(Z^N-UMKrQ8e8BlipSj*=K#X6k%Sfwh^1D8L(e0CSW--maL>DweboX8HqY zJ|mlQ6Vs;2cKKgZj(DZ8{7xr~*g%RfxgTwWm643dM%0pstgxiH#{=}?x2aCjSR!a* zvcto>jdnNgBQS4Mxtd3^1IG446#pyk_&O2=W{S86_ZaTexXbFu6T>uMVm8coM3zvn zt37da`Zu09#+VF$wI?ISZn-lPF!mPJ^oVeNb&`-L+fQp?l z-5+59+@s_ZN_$nd7M$AyBtv z3mWg|al#;(g1&nA6c?xHp<;*vd?WP9Geyggg3cT&YBttmdMiqQN=HCnqEUWK3GCE% zC-MRJ8{h!peY6?o-m9x{&V+g^D;0;ogA9L{l53Qlr-Te&>MA=RusggBPmtW_pHamP zN(Lhu#2e(JKgAsr$FS*yKvkZM-Ev;D%eJnmoa&F9F!xOU&#BWA5~vU-es!&(q~<@U zpTm$-oi8M60(uBC4anUmgSH+4P$fWGFlLRu#n2cxV81aCh~C4=8iXT1u+S%&Sd1V0 zT2ryfV+EUBVuyqP2hfmkA6}aAzo-7zHcoEf)J!QyID>CJLkgf15>d#{kPVncJcbfQ zIxwICu)$IJPf@aR`-r`g@zA-2oIRmjf-h@{iZDj|HMV@J)=lU^BR$ZT^<9Y0mUp&# z#g!v+>5G%&;`-#=tuXAksFz;c{Q&eS0*P>8y6*eL^H3MGH$X6O7)70ttM2=C_crf>rb2&R z9)XtT<3rooT%$N*%?5&aa7b3RYX4UfYsBEp_eoT5X>k(E-FDqgHbb}fIgPk)oG=_r zN|~Gx6ejQ?{*EP&HMt1##Okrm_o)RfGXI>Cfj!FuFA_6KOGyi)fN^L~rRYD@0@jxP zxPaOB(BlheLf?-K88`9wkUyjq$R2-2xt~*VY_+HlzD*xlEh_s%t=96wA6M(yt3?>b ze}M#El^`w)M4q)q)bjk%HOk%4ARvUhBfnJ9R&Mw2DL|{o+5t|c_fRM7?CW_HNs733 zdgxRAoW#U<4&NV}v2*}y#jG_x`I8u59&rDJ#wkxu33P?^U3qjaaUmPN^dE9I1OuEb zNq&AEGCOI268PBsEQO9!3@;AzJ<-b!* KyXcgh(*FY83N6|I literal 0 HcmV?d00001 diff --git a/convs/__pycache__/vision_transformer.cpython-39.pyc b/convs/__pycache__/vision_transformer.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b70bcbb92b5c7ea77dd3d035ea95a0316fe2c245 GIT binary patch literal 32550 zcmdsg3v^t^dEVZ;ci#&T1VNAlDe6k1L@rGL1Sr!oO-mw0Qld;jtVqiaB6+pky8ss0 z2RQeFkM+WKLdK-r(oqt}P7^5s9OuDPr%BtSCw1F2iF2GZjg$0on)s%t&BLw3JWi52 z=d?#rq<-H&cX#jNA&atf5_f?+JMTL)|3Cly^Z)1lnC0ey6ErV%RC0313#>F42B&x~Dr1;HBYBD8d z($&mlCWy;cGiJnyRC^|SgP0AHD_-fVZk*g$?Vs$gZkpUw9he-D^h9NIb#QX9x@B@p zb?fBT>bA*k)uG8DNlR9?S9eVAsNOMoM|J1q&gz|$cS?M!Vps2)yi5G)%H7p_ChsvK zC39(4>E2~y@;;~HWSyR?#^mnOZl~ATaKpfNpR>{FcQ)OK!aoqiZ4Si^260!k_5ofov z=W4{+Q_9Tdu0$vIA|~hTMaor6~+lVi@ioCokdjvViH4k5>(AjbixN?+-I=RGf) z&SB@hS7Vd!bZR=TbkKR|#i;YJa|E&Pa_XU+N1aCy_kc5xJjY)&oDHQ=kcqtE5_t|9OXRWOaQ8fohO~sXw7>a*Lli0gO~?P_c)$&)_MO` z134c;Xu)~f`GABTM(Bd`LFd~fbOfP`&ZP705;}^|CFdFESqVLY&}HXCPF_OC5NbHj zIRyzFN9e=OIj1P069_dO$09OCY?Ug^4jIMOL+SMHBQ^VEwJ=?>PrCL8>dKq3#}3+;=pf{{hr%y+Z8Rdt z{#2=0pRScXwy)sXXNrYNX?NPjXV{+cym|M~=x9M*Dqk3>tLf3gId^n?|JeSKv9WiL zzdN0N?y>qs+pF82DxfQu>kA6)J^PS-tl$;)*+&+p*^2@z-)9$Uj{Q_=dZAKK<%Yzu z`>Th>eeU&NKeqqi$k_hzckNH7&(4%EL{sHrxlplZ3zzI--6`2+*H)!Uso-L8_|n3K zLb*~nS1GZBtv*ecy_tn`BgJ}kbh=)ju9SvV$t@KCiO~xJtvveFf);>NoU7F@R!WZa z$UT(aYxCvvg?YfTu;9(ql^#|7=5lRXw}tP&SSnA?c=h+`CZrq&-!>gyvN* z+r>)3buor>rJ4ZVJ|_(aI8M)$EA|HpwdvjNTFtI36w40B-9?MN;rYuR@yH$-8L^8q zrQ)1j2L3E~X~NpAS4(!OT&vH!qxfm@27Ktyt3IAuR&}PqW>g{n*0v zG{$PGfb`JpKn9H4*;37|tEmc5j8&!6j}?k@XpvW+E_pK;cQkF(exyEsS#f~v@%`ib z_t{U?F`OS@M72;$-{NXGku&_nd_j3Y6L>REDhF+Hj+KiZg2^Z6iJ(Zb{LLqyIP&Q6 ziQ{MU$Bv&k^8Qn2^G_T`FgqL<=xVGG(UMv)#pzGMt+Z{7HY+O4aBBU$@@K(QgNZy4kq%zISeU! zpYrPYac=2O3C+;Mi z@SJV*0k?`X`BD`$ zZDd~6XMJm?RB&<`Kf0hQ7^8)1Ugk>4_0xKE+;XGjr{?SRN?yFlBD$ZSFM56t#-ya? z>lL9hexJ;?U`!>eRV@{2e$@3GKRs1eZZOnoc|&=sTv9nxCD`619QcRu<;UkP7SyzR zGotQ8@Z|%e)tWb2S#Uj78g(uuZLe)9LfvT zw*tX#pRl}c!@__ZdlQ7EHD0$rACJBsx$7FQ+iTZYnBct$ynlP!)f%ri1I7o|C0}pZ zaLu>SKEzzBivcS(8ZfAO!5J17$Q=zBb$8UMU#wN?1!wdEHW$NK;ASt4&R)uMrNf3{ z8qCju@jUuJ{O)t_9^Zei@W9BtH}fU{bn12yXH2h@W8Yw<{ALm7?QK^JajGo{z3q$z z8HVzi_qTZ`wVexZ?Y7^4=Jnteravn=8#drgyzfm<;5YfcHw|~;$Ts-{oFQJhb~!Kn z^VmUhYiq;;1N_a*>taJt(U~!$n0bxpwQ>hlugr~T=9YGF>)KWI>ew~%hTGh!A*go~ z+uVtLpwr@jEO6Hl?nrlZyCb3Ka7VhMnR{rh0maqfzVq$cC;z7uL0G?6=!rBN0wUbfDbj*@98`Jx(sra?XlgSc*u-R!Nt0ob4468 zarSGiMq=CKP8GQ?YJF1h#PD#J=l%>FFLEWaWGq8C+>FlmHI2n+)12RmZ__iI5b-U> z@Eyl@(u>YoO{0le$JiEe%p1{WBp+*9SFpRx$GzxMqM2mMvVpM9Z8oFG9c|}M1-TF- zxzf!9@@AR|$C@+M=e=yxykb0KBCZEdFT&9vyaC}pCxX$6O(<@4ZyLUJu3T_$F`yoT zliTRW+yzK^{A3>R0y5@RKjM9}R`cVh$~DN0{ODA1%#RkZ@l0?B4ylVDbKpVbRB>GU z$Nh}AP=l~Tc5eQF)P;e}yA-+Q-I?-~=Vv+*V?luN`Bp%(q9(~EDdV}6Z@Hz)l%FZr z@>8XPw}361zhR~fnPn&P0}} zIA3f%di>P;1r+KzhI;}?6-yxR0kou1lStKYdoO$+$K%G~Bn`{lF`yW?iUM&i^W(CD zHU&xvy@@$-(3>dbJfJt0(3_aho754{gSZzmk6R=TC)9&*Wu%J<_*x&PwTZ<~A`l`L zAOwRDD$SYh&c{Ivt{C4*(h$iL56q#+Vgi#ox{0JA>VXg<7CNf|5E&;Xgd^@*AQ)Ca zEG%LR;LaLSk`fCDL>w`hX55Jb%RUE!AhoO@63{OQ(dZC~geDS=T~4`byO8!&ARaB* zPMMM(%9kOKq?n3wD|cG=_xJyuh$Ym+2nGa25$6SReyW`vRECpVV5?3bgC9LVcR`(` zFIDioTAuULw?s0QqAL-CylR3Kb`ZCXskx*^(kL3N%B(7hO7rJH;Z%iXYIJ1UC)wyc z^9jBX@SVv{lI~IhMpPbvxGHI+G1;?5uaT9hK43KZ+t6Qo!c$R6(948}RuPkZHu6k# z(Oir!TJ0(S6sB%uDay&pNslLXC5ovIn(rWG7Qm%D6j_cfC1o8n9bn{gY%x|FX_b6q z`J1t&lmk>j8{&(J#bh(yOf02|*UM%fP;53c5Lrw$Q?psFtv3w#BTGHa6i5_eo5{Wi za$_A@Hud{}cA7->t{_J*2jmEdG}{-Dqm3PMluK)Jq{UaHi?S4{qww7#iM;jdYp=a_ z3(spe4OK@(t{>PkL)8O8r=F!VN2iSfegcbtMKoPO4TU!S z3JL{lMHUEj*c<^hB>JSyAx#nCHa2z;s6`iJY6e9ELei!tH{+_n7M`O++SQ?gYKlQY zUU53KkJIs|@wgn(q|rN|7_*AAN_GWQ9SNmI3c1(j>iNKV>L}LrkAw=&B-Llo7^-eFpoKHG|fmr z_+5d*L@xuW(*=cv1JZ6kCJG?mg0R>1W7CCdwUDzk*oZY`5Wv1%C{z|ouDZ_p)|l#F zLANQD z^6DSKt3E`h1Ge1^J_qN&<8k?h2nbB5B`pLo)yI)`LS2O$D&GP7cOl_-@wC7mQ;aF% z>H4ICkb4-O_JRjn2>_kSjc5sMDxR1Vo6^f56dmiZ-La#UdU5V+y2ieTHTLQ$UZnb@ z^`V~xk%Q;{C7cjSVCb;sfoHiwAJ#gqe;|GFW~`ZUj2m3PV$Fnl*aH`YHPO6b*5Y1b zDY?wG5yZ)f&RJ^GOE;5Oj8ICZU1!$oX&N3#RWmve!CO*Nvvnhe@*7a|53X5rUr_VU zuBv%sGlP=-Xk83z>YsU=*mhYz2N1tG6dpu)i!j~siCj#UB{h%kuUL^l>uxZX(;nH* z(hqS$M16qH2k8j)6DECYXMPBU^kdX~jmaEWSD2Sz*E1%05{$7|_AdL;iaVxyk>qY~l ze}0)w`<{DG=t0<_+YLrto4aR%O90HrU!x%D#S}b_4pyT>;H-lPa?*1e^oV&t$ zJDjte!Ar6p0|5ip8##b{vxzIwWvtQ5#+C@!a^?fGy~GtVEqS2V9ZAh28!s(zR&!DlnUJZorhhKe)jzC(L!|!FVy&S3^Wbhx;`2jkgrqfyA)}ubJ zAe$>w7K6Lb7%^?bHX1#8KwBgzwQ?~!&dN)0{FG3v2|)6VtWe*LB7ca-B~eHMq}f+{ zw<^XK@pOHdgCF-OoYx~)Q^^p|F1lID2%MCY#y8Te$Ycti$v z^Y5ci5RMxs1)IAtN)gTctSqyE(MK+`f)J=5ViBQt-6TN0$OIv7FVVNgginBLcKeYS z;e3YIZ1nm)F3Ch|tHu`=o?7 z3R0%6D}plyr$u~?_awayAW{%!HX(5U#{@AWwuM&kHve_s_Sfsb?fgC=Tj>ek;+n6i z&JxDRH!H6C9(XimAo-Hj3+!_@AC%kJAzyx$5(U^g2~;%0lYTVR%%+rZ9b`bbf)XQK3R};fUs*gmDtGCseRa_HsQdDcen!>5w-eq#YX9 zP2PMXzZQkNm&koL>JbVD%7#^t`$P~1Os`kcar!yYhF)n5cHv%!P>T{SpyW5#O!d#%qA1Wt{R{XytyDjY;9)dR{Tw`C)IY(D zF)RedIrUHIkHXqeeICBhE>G%^PG<9=9vmY0Wi2r7!(U@){m!mh3}SeLke0*n5X@ab z1_`+Wa?t{Au%8bhKr0O(wN`_a6(7Lk4#ELIds_f~8f?dTv%M_partM+v|^b2)L~j# zU7A-99%&6$Be!<I>|{zoE0j?q(e+7n(AJz;)+90~WZC8Vp#C}dKv_HItbu#t-?rWNsYeG)-PJt;q zTYY$&7D;kiO`O$&{MOEcO`$)IhfsBY9ZqLKr^FN5JpQ9OOVn8z|q1hkzftKnOn~`2?1E>H)!+KnnCf;jeRwvaOgOF|ahqg04)lQ{gw2_LX0D`aFxeH` z(Kzfq%dj98$-_Wr?bUk0WVt%6^QA>2ul=cM6wzL>WR)a{=@L`WwL)v6gy$D-I{pZ( zN5>si#fYl!qw^DVo}e>9r?ahXZ(##Lzxoo>e-)0Ogf_4Udvn)MiAk-9_T?pnl&)=! zEyj~XNE z>Ps}30~gn{eh7xdtr+;Nmtt3B8$ysHUrh2r0@lulm`*+r3{e|UOE_z(k3j%wIf-Rs ziJWd?*_3#w$$VMo&mEb0dOTjZNWl7t0QgZR}CPMGJEE zs*4ymwULg1R&YW9_+iEgPCQFr2PXu^f-xQJh-+3&_=`nm|0o>ac+59G5F$;xnerT* zDI&oTJP{~(_85Dx(TE%U;Jo_qz17Sj+~^O1zeO+;LOIkgqo7uLeG$HZa^w;#sj0(k zvh#V4z9~BQwH?7tY0)ns?i5=@gre56cZ*_N*P2KgL=HfZ?QwX>=6SNGqCD6kM!Duhf}O+&SFBjeUC?=JQaB!=qut$XMz0tn+SoTnp?1! zmvq=ogwki2zC>*YpXf5xt7bUNavsKr%GnmJT)?|KTC2t}9M6EQ38w^T8xCPd z89&a5s~kTqyX(U$pT)Dw6jfOfsa+rPbo{t~45w>drTCv)Ws#gw3&_o|JXvgpaqzIj z)izU#IlWVZvIcK(`a~_Tmq!3`TrJ^jat0`CD`CZ2%HY7;JDknVpd1m)GHy2GY;m?; z1$R2x6NL9UL(X=|(Hq1JJ3E{^BxXYpv(vd#%Jkvb-$uuN(cnQs9INZcF~ePW-zI0Y zG~nEe`1_pQh~Mn&adL=9E3L@n7GU*$f1^Ho6*_m*xVgr8th6u%-KW!dFzt~2T>H=& zj$E}4on3lUN6z$_uQzz?jM-L_n$3V6HDEYKrM?LNEy7-85ofh5LJcs2WVN^jPV2J` zo*q1$O0Ey2=`b-gNxA@*UnWx4y$@k?F6!RLFeU)x5+D*ijz31VgUs0=PtRVBpr$QoLtito9Oahj`Dl~+nyt;jY3P{-BF|^g zmcBXj1w$S7wxJb}p$I!Qgk0NED|#xP@;3i_#}J3v$^F3!~Mv0v|SiCYw1p5+-%0HPp^uI8?M zR+nofU>Vk@32}a@s1I1usH3a3)0O%;EJ8dpW$ytWiw1a?L7DM-on$i!I{`T&I#tH8 zI2&inAQmc?>^)9ts<2S;4%xe9qV3Lg)!pVZTTPS*ZYy9SgM*+v4b@e^DZz%jm01V4 z5OmHWcI9rH%Ct&!Mow608Xkf7g6U0Hv9{i=RZNs{>@ZSnZ6?R|@fd4YgS%>Jna$XH z&S88oYNeW-C_M*L8W}2Nu**|8#JJFn9wD9kA$t!FGZs9QuFzg!SS)@3b{y2yTKTS5 z3Rv^{U})>DFyag!812eGrH}jBds^oiznx8e7H`A3!QrVQ7G71g3sW!^5FqaZV8ODs z0IzZ4z3;Vm7cNZiX8Z&U2|_Suup8i_v&3*BSF2-;f~BF&*sj)5?kE;ZRDy;IbU?Rm z(Y9)IMN!zWW3sqjxro!y#H3X()fxyf+iYRGj?#nf;s9l}yO|n?1w8hiT5Uv&PC16c zz;_TtvUK&3!>cv60Cc#hX8c-bo>v!a+hVRLCx*L4uY5tzh!|2`dutu z;xuyacRAlq1LBxRuugS&Alv3qQ+>AEzRx~G!_kom^?NAwwHtr_b4&l{OF#1S+*gU*-f$e!c3;X?^yzKaPr5li9c89Xb5pG%SDNJ#jJvR2tE zRl!PAuo#kj3M(gE90iNNh6Py!i;a<0%u0LL8IqTEyS36DDtM?DzH_Y8c+%o^xq&)pqSPjpW=X3XV5#>~g!R>JTOlbAc ziiVcGu`~}x3s|OvV7Y$(w1Pch=Y%q_($DC%QYMl5_pG+FU!s5jD#Qxpc;w+kH{sNH zy4Tfbg0(4Rl3@}AKZ#Y!ou?g?jPDA`$WOEOpJJVH8B(_+NrB8oQ%W(jRAc}}_mg$aDpLG5Zj^Eo&#QaQ%H2Ikj)zN|= z!J9u~X?D)ffbQp8t(Jk4?eQNlgTR>E=z^Ti*)T?5+gUqj60ku>Z;N#poh2Pe?Qbe5dJ%8lqUGW`#N zMQ=p2P+awLz>UVv)kFNcs&39n8D`exVW7GBXsaDSPB({(%dt7HTqS*#h8oosO@q03 zxWL!gNWLRtHtt;Aw646XBoJ*B=+kXO>i{xHRKo4+$@6qyJ{ZK-0B-0I2%a07@#ElC za9Dq3Yd1*Y&iw~}8fx!dtJ}LGch@3?lA;!r(WD4_1{P00N@1Qk@gOl8R$d^Wnhb@I zLl&kVL;9+=)#^dF@=mE(xcnEd-?kOGPjt;Ff?&mPY5~=>2&F_A)v!O*CX>xJd2EJg zYtC%+en^wOXGZosh<8jW=>Nc6b>^fGzQ2y`W$k zEL|3dM8@x#R`rGXyofSO>g((#$B$PS;1YvZ*Fv1G603UJMrzJ9N@Vk4lu>_**mjdN zzrC_agJ5Cvt5=$rLmq|fIYX$@`t$2pdzhg0Aj!0;MFGJb5)lQp8qhO~jnzRd`{daw*&Lfo zG1*qfg@i^Yl#jZE&x7c)v&0DnJi(+d)A>F+Tp)w_8p^o>H~JKA&k6V31aJfSB~=Dl ztU-;tSA$|@u~j(2u@@lF3Z93> zV$4s{5!ZSEC34WC95t%kwT7r){TcnT!h8ZgKZ?z{ABQd&hwZe0|1X(A)*<4&XcE;g z(APtUQ#7OlqI5GP?_Wd7iTh6U5v2BkuQZ_{a!y(CcY!(E)sJ7&X`+>r@A}G}~xS z+R@~g>NNE%h#?|#q#=y|Jq|!;kU34UR}mKNPndKUov+ggqppaIX>7KVhGbhkSpAXRVbwTwoQ#atRTH-3m__VZjDzR6;kjKHF4-Y|mCYYk*O7F?X>+oCI=; z%xr91J%@tkfP7HI+5S3_GtTC4cEC81+q$Oi<(H6K>`S#4W|_el+d?eWNcfBNr5Kl? zvw_Y=I{kD$!_-amu^%`&33-F=&2vndqSK)h9gWWKAmNiJA!Y#3=p+IwpI3SxPbZTW zs0ECd(|EFRJmT+5CO7uQ6-&ZqCJw#}C*)u;g`*FvAkbbwA4hKRN@B@)#Z>nPj3+2$ zoAHc-gq(n-V;slEz=%>*SnKzo7=(VqR3`wxlvtKViF%FWdNZx{d*q*BDi!SO_t|ya zLM8GtGF}iR&O^cm(BL>NPHQ>1tm|C40;}_!t=fF)gx+|hX}`exunLf}X)g2x$4=PO z&(U{;&S&BHaZxGd{RAW**gPt2HaJ8ebcR~?1Yp1-dx$;4e45+=P=l8?6jOvlh7e&p z{X6J@zmBXwtx1kTV|fWWB4~(~4RsKs$Afk-n&aq6h+~;-nzSN4{{@MGSX-wv#W6AJ zUx2cg8-y9ALx{_G*%H*U?HZE;kE6kRGl0lX!psgeiRisu(~A84INA_#;>i64z^_-} zM?GGm@y^$SpYSylo;-f!gZ7>imBJ<5&qpW<8JWd9=(IasD4VMbgX{kJp0|P;-dA;5E4@@Tut#I>YU=C{2*A3U;AdKDSKE1y>|o)jskhUP_j)DB>TNKBdumTJKMGOr zN|3JJysG3p#7ZMbScffm*4=<~4r zarZg#>n1v}&q>J50yrw}B(In+fYt-CaAa%If&~&3Mpw*iWHAP7pQoEv&1%BReG&FV zL}3{ErnHZy&XD^>uA^p{m1JH5GXk4I2ri)Ji_d15x1E~RsqM6ED6J=y7JI?O<&Hgg zV=t9nFBs3BddB2wQ^X}_Hw1A95tmv_%X?|7l-@$amJKjf*1o=%;)@v%*V>9H^k)~d z-o|Fy>2GFr`b!Xyv|E=!$##mf=_QN3U^}JEz`A8Nw^E$J*DcfPDc{p6v!zq!1>^ic zGd2a2^W2vp7@nLFbWSP2cRciQ+L!H#~Up@cZZFp?g@rymRU4SQHzX`&m zZiohXz~obnDmid!sYhU_!>u`we*Z?jj{w9T_= zs3J}Ezr4XF&Z*391{5OD)u*pe2o{~)a>YVk#G@s>K}$O0!FJr;)sD<;xQ+L8(mbko zw|PUe+kyX4^=XD?vls!&vNcBUVs6CTOFNKuo-kJ5*QrmJ5A|9~2bje1+px4_8JHzF z{R4v28h>{;6KHkcOV;9s<_2dEEN)>cKYK?TKQ|yX7kUrVT76>L-mbKrvv#OH)pKz69%sBYZ@Ok7dvc(eg}Y(~t|}4kyDkx)OV;@xK<{?7+T3Kl_jlSO zY2g~)*-3jLdj4~A3GurcPur&;d~01qYfq_q)#m*{`l495K~!ro+n4D+DtF@AN9A5$ z@O@tS*>lc*PH93flPDh;BX951jNNrr+!Mhwj5~L)86iy1T@<_pi z0K{gl@(~-w2d5!q$mk&C1ut=g)D??&XolmdP z*xn^R?P$DMva!8B?~Q`*gg9xmt)U#%CaaB1C=vQ4nB|{?B3JPMj_^H=KHgTESM|Er zD$+Au(_v9iH?pnZIf%IZmWC8*FFeW8*1W2TX8hus>)@UpR_yFQUKg59b{D_)U1+5jdjSybr2PkbaV| zCP`i7CCoW8R5#2eaDO=Q5*$YXs^5?5PN)Z3I062SSnL~TZ)ydtZ`80?97VdHuFe#A z6EUUq!z|FoV>t@4L^NiDAn&_t9i;6XPa#7H;}V6l*mI@JeoV|D*6-WCR^N7gqrL?S zTT316^o+teNJlQ=OC4g%<=9Gh^FJGEej0`hEqEsB=fKO<_sywdW8)K*`JrcCyTg_9QT0NUSHyu&bGOPku=h5T7#+&G97X@wS5vUx>r%32=^8RK|l z^Ua+V6{Xy$&6*Ho%jTKiTrhDz4h5l@!W5o5*jj55Dq?V^q9sr@vBSZx26-Pqi5kw* zwDLRAPX29BelZ^n6qjN+OAT4dGS6xx;D<@0`=UWbGFR#;w zs|Ow5pG42V_DT}#e+w1p0>W4KE8fPq|u88_` zJ${0s@oC({VwGzY$@0CQp+gCR+C`j>Rxo{p{#-_!Z=B$rEUk;mOP!gNOF|JU>lbA1 z@|~}hF3PIq$ERHe{A3M=67)^BvLGxI1VaIxSlN-RbNdI&3%yC|dmB-0ml^ zHNXW=1TZ%0&+*j9ERF<$$3qARj!ilY@y%x}BBflzWI; zEQu0{SymJ zY8Uw11fntx8U1OQn#KhDcmQr*(^<+>m+g1hus6Y7e=w%2&l0~_i4eHPmXPXIZmu8E z6=}9oFiCVfWI8lXQVjufgcwbrT=hYOG`h$dP-Qle7z(CHQ363dN{93r_r+1p=#_A& zr|3x|d@fKr_#G_&op5~PN#8iBev{$fqSK8B{y<3iK2@#&4`F!pMIy#dD%;?S~Z4h;-|&A+8ZFZ(4npt2YbKRS;~7CC)Hk0C>d--Jk9zz?F}+5}Or1)Z4{m zaa~H_5zvu=^deKjDpG;!#9qWi6EHx;3mOjTei;@X5(ALQq#B}-cK9srq2({zc<_ym z9n{vMavtc`FQQiU6*{DHjjFzot{BvzN|1S_e?j5AL&vW55QZ2vcNa{H#b3 z^ZMr))*hlYuW^-4#vgNt;;24iZ6$@fh-%;B;kb9r;jkaJal`B-JR^Gl{LAbwcbA=! zQ7(h;U_vku;`bQ4KxZW?tm-GK*LRBk)rx*zLh-k@pD*8#exBi#!t#qo!)MxHpLp`@ zar@-%3nePRcwd?P+70H+g>oIj`Q364@$TSA91P1voyH#_>ilAkHd~d0LGp%RqG$rm z=wl_=w#sktgvK++u_LD4B8>HT5~o|^DgJfGbDb-G1#wBx(cDm8Vi#WvV8vk`Snzz#p5QHvr0CsN`IY>5sP4MCfcV<)4CCSy&sv%>A7-}M=K{v-tgF5M~;ORc^HY4$yqw;WmWg z!j@cl>ml^J0--hey!OKUEexQwc&$Sr0EvHH0Nq6Zg??5e{L2p-LXWP15D%xnVNU!F zaa@JXs8WC=OxN49+37q~L}*8Wuw4cn(y!1lXt2exmBcPP-h zQegXbvJcV`y2Teh2nS*dF~K#p z6;*fCQRd?Aoff%ozlc+!xO`J? z?epU}&`ljDg*sXVos^5gM`|@ORGi$@3XVu!C|RgLP0f16uf7eP^apjW1+QGek52Lj zWyMYaSI5J=b;Q9B{3=TPlx(soJd$6^MCD8t`s0qwlq$G+-qkX%H1lNWP#7kr?UViC z+;YIxl?$1*>8lt+ZbatEq1E}?mG?8lKsbZ`6;pLtQ6cK5AdNdEl6h6&i=RgCcj75nIEC^<8*$B&IBE?!~Y%l_%otP8`a*+y6=$}(4j3H zFMg3DDq0yNl3WT;MW`jRCq==G2on^y;)**##Vv>;=c~wYDX!^?%Z4I-QAER#B#2lg z2Lb)$VGKP?qaQ>D=wabnoB8W3yhSl9nlFg>&8-mcZ)Hp(9ZmN@Hu#zqg=<=WYgylP zNY`apuS%Vrnv;}4nmTr^?_Y?Y>d)aX-M_2<4t$R!)5&!IMaG}mQCYH;mHKwtuVhs&;JAL;VJU~ literal 0 HcmV?d00001 diff --git a/convs/__pycache__/vision_transformer_adapter.cpython-39.pyc b/convs/__pycache__/vision_transformer_adapter.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..91a9d65f22980d74fd35df9b0e86c9b5050ae8c9 GIT binary patch literal 10979 zcmeHNYm6kDIaGT3*35KI zcXiLLn%Uh+4M^TKGY(m22?z+XLO0=&m4Xz&M9B{n3qt;aB7u_2D-e*gNFb#A02U?7 zch2qZdD;2JFGRQM)P2{jd(ZjK<5sO$^bP#B?$3*lpEZo{P-gefLFQ3h@qdB{Lzpe2 zs_$mimIC@w3X+GVTY)dD;+J;axR^7uwD7D|U4PB{|`v#2=JQq9iX0C@Bm|rchE8 zC6tr~C5Ob6D2wr{W_4N|5))z)sl&oNZ&apl;_Z$3imALOWvBaGEm=l7+pWo@QEMTY zf4>x=48@a;dV;Kv7WL()+6$pTt|Y^yMx2B)Xm>=|s?%7zf7wAMz7L7epmnWUBEMxU zn!**?tGTKz3j981)!8x}qng2-iYohDCe4CkS3$FBR+uYR zJdeAzV#ya0ua`@WrrkHW))h8NFQL>=^1Xs^`X<*DBR?sn?iFfPMC};$!IkOLQ_CnB z@6-4bgSrflNp+K{13HyvL&Ejjd}8T~AB?uvEYl#k;n(|zMpRnthk=HccM^eZ=y zCNsS|M7D1V@0z7+ddE;R**w}ecje2?JNM)#b^b%(Y6t%-;O}VSlyTjR-}NBYi~b+M zB@FCCZeHfGq{{BBCDn2(tjVah&k21)RQS*sjZnRu-m zO63JXBWff;pt3@ntQn_=~VICb+?zx(*wL>tE$|N0GUsg^h3S-O&vkp~evDK+1 z%01VJ@IFdEyem6Rj-p&WW)RbHtz+Kp%_Sc@8%Wjj#r#jvrooG7Oa5mQdQDlvlsRR)mTQ*HlI59GmTyg%zBz#s&vcRU%^B2{&5iP7dN1dOEO4Pt zh&%idG7!T&fh+zb$UrI!sYx=qNsaXK@Ph(z8=Y-lmf{%}moJ z2?8~mklAbYCe?dfpGis1a#O}8-GSN|ah*rZpm!#<L$>=(0A(hMO?xRmslJ&r8Be7F!BOE!hO|d4e3;~S}#^T)+0IWVnO8|OA-hvcH}0#^jUuOx{KuSH{?Us z9mI#bbq7!x{FYiX;0}F9iJiK_oFpW765>4YCU_TIhm!1p67N7sF3I$K;!U*ABh4eQ zjwo;q_!2cmMqqn>Nl*0KU#*z(ok&+oavV3_1(l1|+QD*I6S4B>7KlchIgQy--ie0F zUDfdAt`F`j12`jfPjHrFASQ5A4g+0|Xg~!+hP7$jr)V^9b!9zR09whPqFKC;2(hB% zEg!KdPp^jZnB>9$YZ5*XV@{a(TLrUZZcL0eXM_B>4+P$hn( zQHkkz_OKo?`=;@Ny=iS~*wRo3umaQ($N~5SSg>#R9Dc*h6kri=>Go|frEZn|p{-!r zU@U;#Ta1!{L6Sqg+w+Ab>^{~F8B>^iIBIl6j9Jv1qf&Cat%c%xu@NaWMzLrc9+6WuDDO`teP2)NYS%G8*M;O90 z^8gZN()Z&vcEKjS-qNpshJIgpPbH%bhB<0;lE{NZra&r3cMxCx2ntk&%->VIPs$<2 zlLRo@1o55`r0|lTzQY%&p_G0~%0y(YhT0N1^(x_g$)n(LIxy8ju z8-c8!VRIl4Rih4%DqtncPM}r9AFygK-^<`)RbmxCFG( ziCxe8Jc$dngjh9kBH1ZlT7vcg%Nn0VC zL>fehF{K275e#n8{)xOurK?0pLCT0on@9&l`5&yc)4#6c!_TMhp zdWf=wXTObWVEoR&BPqe$J!VeBA1UzPw+cva6!n%3aWuCdHQ8i$o_^}w2jpQInNjqI z!zEc`pQMMA@OpSi>2~R`q|yU`Nuo=?_~$`(022WZf=rmP6aWz({s`wFfMX#Lf`8~B zok#d4zc|G(S`uS`(Tq4Y2;jJ3w&^u<9K->3RegkZ4gp~E;!ZIw4kNHr5J$vOloZ8` zxC8eRL+==%_qZDWAiH}PcD%ifM`v}}?5@Jui;ZM?7CwBpBa>OI;OtVXvrucz*49H= zTMB2poerGjC4Uybvp2tuSG-v!VuLik3wo2HF*ga)xhxn3z5>y=4?ZNoO{o2VVf$gdu(!IS9O6FW8O zA&iqfY>awN;(1fsa7V7uIEQFwuNoMCTKMT0EG7F%?AOZ{S{>dpw#+SS%ieOfGF$Fe zcFWuH7l{jUSH0v2=`3pd3O(`Ywz~}hMQSF^BOFvs3kcR!(>zKtn9bqj4pG4U*tV1A z`_`f*ipbsBHx{iW>k7ml?T#ZfmFvB&Pvaejmh@K~jFfV(ToW#G--3=DwwUENy^EeB zzcgz7g+XfPJ4|g zs4pWPqH+jTh3r-V1b+-%T8F_t`jOTp>?d0_CV5{1GJmuHjD|zfH-6DtbNhr4u*OdQA3z5`%r`@$c#8LInhE%JJG-6P_O|V9a~o>ssDYgWhd>n>*YbMu6%!WmJi2KM<`Shy zC(Fx3*ac)g@Nvp+5Me!#B83GbT&zG5;ik+6%vNflN^B~#aRMbDDqD0Z4oAic<>hrW zlKdndb!0lJVuD~ALTZEX7@M=K5`GCizJ)6$xN%+Uzb(?@(Avscth44huqLFgc}?y{ zg4g62tx3;JEclMbHbTSA%(g+8FkA`dp^B@$i6iKyE$&R%?k1PzXHcNB>rh)TltwQd zAjR+V0JYycW5hxX`O7G;ymQCEQ<<(L)18OQlY7sBJj5>xlvxaG$r`*C9u(FOijvL> zOnylas;9=f>_wVRUh_2bu;RhE0Nr;@pF@x;S8v67DCG;V(;%ILQ%vF7Pz-ED%F|XU z5AYv%YjwEH=SOCN%0RCyh00wZ13y*{nP-wjM4O$?g<7<%{H_dxR%fY^#Bzn^=}_!K zUt(q^%bPh`#ok9fl!?Qy_XQQZ%OgRU-U|w~`P2 zX2du;^hV~)>$u`7h~fA)1!&2lEg=NsTbAkFh6Q6yo5x{1m8>J?tn1$Lv%X#6b|usQ zK_TPYCpdR^h66a3gHfhoZlTR>=eBG6=ElrhW7Hlinb4B(=?Pph1s+M+zX-574@Vyd zQ1b04gS?QaAWDw&r2Paqln#F!2Xg25(=uwJ`N;T#+J$>u6N5^&G#5wETS{e@bNEG)qfiqH(0L2${e!r1tkYE@y2F z?7v;+-`w5~by}9;kJ2R3PvD9_2tv+I019Wuk;f8~;#N5Lg7*xY!lVcsd*pC(_&DwP zLCg1!TH;0bw9T?!N6e14iq&_;)6z*6J0pn(RRRymuimE=y+0O&p_YZ2rF`F7lu3kl zuf_ze2JwLbMxTY^*f?=;=k|Bn!`Rf>#lG(au-i$!E|6{)#!IwAeiI?)mtfTrU9>SbHlr+;Ag0TKhkrt zvK!(OLWQjr`BkdQB6j3JM2M**&`5qyTxd&ZHetg96=md-++c*zH+QWYqp@S0~ z=_^}r#!vf)+`~^KVHsS^Pm4%kg~3xgV%o$Gu9@<~D1*<t>$QbU-Gs=6%z`J`&$G zZTT$mL(lCS@w4Re6L4tPU)Xm5Kq897VhkRa@DGPxc(cJKokt4c|39Y!1{RjHhbem$ zMA=Q8auymf4ufR_XSC7MQ1ocAm3s+67o0fml|FXfPt)?1(TJB1+NTUHMB&ekqXz|V z=)9ZqWpsd&;9Gg0E3dLaAUbuNLD9NtNB&PJMN^f30ixW-5{!{hevL~0jL6rC@caF6 zpZ^Nl{V}e12E;QR3z`vs_xq)%d>c_C+jD)_n~~J+>>f-f4P3xQc5>XqYbVAcfPn)5 z90EvSZZ>TS1YEO(_bP<`x6D_teY9~13&2qmd7NhAiv`+7oONbXhXR=l2;bPuh(elK zH{_R5iU2+~5~r6CXzL{{egIricI>?W4J~zsj zDZhG7jPrBOjh=&$|L8e+q=(NbAU%9eG0hLg85a{gE&z+4GcG1MUmWG9^mF#S-~D*M zL-brYl@E-bTuMuWUI=zjuj}UO1F6--vncEVARR+gYm7n%snsj?4M6AJfX?#fI06A< z%@U9IX^~0CgR#gJN9c|*)>DW_WoBN!gw2v$M7BY0lB&BPKSB9l0J)LX7UOj)_zIDy zh+HDVjuDx6YHA(dJS^1WkixI^v_r%yk`!ga%9^A`tB_{2c4R=efnOl?k^5jRPZ z+$!oLyFuBlqCT-3W%WVcFiR(La|<|;yOkew)0sghof&lDjElhN++zKfLwA;^vv7!A z2rItuxlv|5-seszCuy|Xs!*5kKSKQ>2f~3cghPmV9UnE23#y!z@G@Yv3-tC0gq9X@F^P5y8KPWjS2ZJ>gBHB1tK)Qs!rU#&uKfp-;k2L?ckVgI`Hgy-$$iG5f&AcgFHKNm}RR*uN~XijW#c>FcjToiGbqdGy5G{;B~&#`MA>tnF*q6Y+*zQjTJKu1PY zzTSb8&fn(59FV9U-`#)xz(-JQNmQqHH`3oa%F7ZM4Ra9@j3oe;vO2(x_wlzfd|bxo zavb~8P)fooIoPwaw49>YmGU0JE8sJmGc;&?KM2{6z=Nzl%@~|%;HZI8o-^^-!~+wj MCVmdzA-(&50NtsAuK)l5 literal 0 HcmV?d00001 diff --git a/convs/__pycache__/vision_transformer_ssf.cpython-39.pyc b/convs/__pycache__/vision_transformer_ssf.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9cf16c1c7940022697a5e0e566f59ebda66742e9 GIT binary patch literal 32530 zcmdsg3v^t^dEVZ;ci#&T1VNAlDQYEAB332<0+eZqrX`UgDN&{%R-|PIk-S>$T>uO0 zi*qmdSTAfRWK7B}9VKyWHaUMK%nzT)NQnyW$ILArTI7uI;iEnz^Jls0W<0Pqb zPJ0wZ>i7L~clRzHvM5U@aTmC=^S(3l|MSm3|NqZyzNaS{!sq62+P?kMq0k>O5c~`x z@Gu@ng+n1bG!?Q9+bkPXhJ2e-CceYvaK)Ok45nG-NF_QI6@R20tHh_`;y25Qsf3hC zR#H6z;FV>U^yXt}Sld1`Z|f2zN-Wok=hU}`|pW96-t!KuN@wyABE z?Ni$;JEnG2hNgxjEnePP*)_GRa@*8xmEBXjE4NSGF7b(SrgF#B9pX=x@2uQ4b(aw; zn#+5Ncdr;z_t<4SZTDO;ruG*1+P(Is>ju92?9Fz+z2$ls{sBL3YanjWkJ}cA+wRBh z2*eHfaXSNXyZpG6|9x8^ZMPqHdmt`j-(lZ*-9$_7YWw!IeRpf0eb0+qriLF4*?aBb zDAtoK6oWGHEzGlz8~Kc$nk)E2ssYGmbQGbB_A~ag5_$xoOZErtoP>@c)Ucnk z^Ab9a(1+}4yC9(x2sLfnE=uT8gqG|Xdsae^A@pH;&MrylBtjpt=j{avJ#L$4LfMbz zKZJ>wdAj73YSqkHm9IK8HB~99%<$8tv)N3knmK*xY)uvBlF1`2Z>ChKmy4BR)y>pq z)+y^`@=nGsIt5jlF50N$P%^pqNHue^lAkSRPCA+Q*OWV#89$i0NCzR?ITU=klR+c= z>`xU7wb^RPW&84O=1d`9F78cc@EOU>xo+J#G&Yu37fa_yYif2ZKkbZ7>>uAhIzIlu z!~@CXbC1<7WZYWDRe5yfQf*P8y=NcF9Lu};eVHSRv+PBlmG8^st9It8;_PBMuSyMx zWB1n%jq}{=zkYoG!O`*k6YtueOrD)9Vu)r+g;Ktpna^L$6l!)cQ*tt@ST5!r3=UtK zKc6p^^V8)bJJ{;eY{{KloE|OIDr2*?+HAQvqKZy24@iui7ii_srxvsToWer2cA;Ff zrAN-8cbyq>QEmia|@uBP;;>Nl6Fv$`#O|Ak^{cFuM7W!$-Zb-}pR6>zef0EM)P1z5TtH%`sEXA>QIChT3YLl!0QbDw%vNUsynk+}do>ox`}JX=9m3YV}Ks1Du)I zKe2yb=BXNn^ZksdTn#6)h8L^nl?yb1H}#~l(I)#?so)|Qf3i*lMUv%hJ^93u zM~_b)Kbt#t{KS#>ojRL);`otCZ(vRInX|{dZSB~TlaCxfefH#&lREq0IteI{jd%mo zi>0!is{l>QxeGwSTw!L`OI7ogqMggv>*Y&cU*68w-CU_!UvzVYTJ^j)2+XXN&lhub zRg{T_ye!r$gITYYs&39Ho<1fMaN6((Q!(RqL--GYUP}G;{|j-S1-6;55}aZ z>b0`a8Lv-fn?I(K)v6TpRWIzgwwIhKDaRk`q`aXtQ!1*gsbXwz91i?L`0}C)7xHS> zxe-$LAo%jT#wt~Jti0&BsyJp}Xw20X#{j*`7{SI_Nva>DBRR*J8LeOPEW&RLRXBZc zLZ)d%@n;#<-&j%eWi^br#t*hAm4kT=!al0WRK8wvglb{VYbZHm`NdfPZtQF+H-2!W zG%>y~a$x`dVtu5na{EX3?;jsQ2Ee{r9JydK=D>(uXyg%OzyzMhQnJX#TozB2l($@yZoVjQ|)>kbo@IVueY;Zt?~NwE#ueG*TXL? z5T(V+TY+G=PFP;IVL?ESzX`(98n0WQk4ImR+;xrDt+i`BNbueS-oL%=YK_;M0ptA} zlCQUHxauvm4=~v3g3peP`An)_a7Kg;a>jgS-5IlM7pmo2-X1%T4aNu-xcQ4?^A~em z>9A>-1p{6b+pZSkR9X;v z+ZhWo45c&gYx7TPCl}u8Ex-TF>%l8Xe^zrgY`~j%-rn?-9rqW4@g(W{{u*>zn>OhxUuoq(_` zb~R*$rsCLtCB5(y<$B}M%oC?hQ)skEk;uA2mwn7K@&tqu5NBZT4Vs*Hkr|tU3pKUi zh)8Dg?AKe3#Ma4~DR6Jp`Xu0q;Necs`58EF=yGV;Sb>1J8Lsy=jiqqYtZ&D+>6%T5 z`IaL1j^aD+hUcxO(L}6m>8zki!`mv*kR_PZg@G?j5B4$Kv?HCn_=V*w{s`_ zT!@ie$z}|BQ_YxdEtu*HZn|k+Hl8sN*Mp}Q;jkaxgm9l7!e~V%6}P)L49}V_<(-=h zsE6QWH+vCh5t1G+o&&sqj5*bhc+ad>z38b@6>=jlJX09=!g*{yW88&9^5R8ocn~>L zn9%+SFXb*)A@Go$n>QeJVIXr3#cnxgt~BF%sZPYWA7Fg06_Bi`NwP`GL^k1BPO&`W zrApP@Ofl~+V$0@jnkzwW8Hh-uQJv%j+KP4cQcVANU?!ePcrH>oyX`rPKQzt^$+MgMduWp8>TvgFVDEB2c|tbUJODv zhzjhNkn9Y=;Rq#-K9jERGaD(6jGq%A1aeUjl*@U^91&y@D3SBhx!k$MeA!Ql<#Kkd z07x9?i>*hGpL(BwLOsWDj}NIr5#-&6mNaS#sTywYf$w8@oG6^QVVS!I6yw%VAkJle z99Gb#KnbBY5jzTc6Q-;O^u`i;6A^loIO4hx`9kh-lf>bKdJwLRbRh;`>%+7*vG_>@ z0>lD@U=V_(1=HF6IEcYz3LOJ4rITTuoVN!>;kTir{5JJR4Z#4iSV@HH=L|qF6 z!}5uRMQj1wSwlinB0hnLA|}<0+EHNH=RpvpmQ_RoItC#c9U>9aM53|BE>$uPBtB(` zNQ)V}L}?FY%n(XaR7Kg9Gpqai`~ObF66#?DeS)Hh^8z_9(asJk!^thMRVR?a3!hs! zuTIjJ$h&Se$9d^lA{|T6l?XvzHOUG)h}*{0Y+NI07!6h>RuxC3xoJ>1Rc4tg9hvrV zHaf?Af-eMoXR?!|JCuSEl?NcMiW^Bx_O#J!q-ClP7>)im^w*#8R8|u7GU1_F#AKfj zJriCsm%>X{d&)nBsT*1jbFy;M5{1}i zyf1{@Scg_j{XU?bCQ-es$kEF_IRYZh_xa>#bB7#dlbRf9@fGQ!EJf-ld^brVZ@&84 zYp>nJ^V$tV)ew>G2e!;n{Xo#EXXz}^X`_G_!y+I#1nMzltt`4lFS=NrE>&$vekt^? z*B}Tc?e$_#Z3Z&)iyA8`tgDSgf+X3bV3Nu)w{g@nCOzX3d8ucd^Na=0IPY0R(^b?^ zXw$EtkiS->fk21NQBXsoPtqLH6cKJ?a|eN1bTOjlP}ChJo3@i_+{-&|a8Emd#!Ieea96(>xRpf#^mT&j0 z%3?XE{t>+DgLFDz+soi{aQ-_Uhi?dhzyw;-LJ(7Z3~49S6}W-&9k7275`Gs?3+xfa zm;#=zPr?s5hv8{2c(9iM(5c`E7s00DiP(`Dy$nLxu>sp1Ia;n27QU`)?0Z;aub$!s zs!&=V`biKuc+OwK37`aq4r?BGmdo^Et>gL!(id$;nkn13&h;zOjH!oRa6woT&Ff}0 z>c*DiD_k2vob2#|rKa3uGk)0!q@>z)rrn;V;eu2(!vi6_B_TCi*CQyu2{r%Fx;6Lt zHUI3Inm0F7DA|wJMX;v+nY)E;m-TZ1@mmAoL4>yn(;c16Mr2u1b##B#iUeABow1zu z$aa=~h!aBU{d7J+N2s4L>DxQ=Lnx#dq4sNB=D51dyac%`d$?ILp%%$!UI7i{U9=V!{2Y7bT&r(KUC+sA?k!FRa65!oaUizt`bGqiZ(l_-2 zs`cqsf%wt-&>zDS#^Z2^10;)kZmeQ}u1}~yrbtYzHpz*Ac$gqd4}!FGlcfa6iQxjy z9b#mMuLI|D>2~QJzjSj=>0aQ-CdwLws92LXtaXWs5E5%WLXzzZkPjg`v*??~CP5_9 z2+f)Q0DXdR+&C%N-33vKXy&D5ne`1nvZ+;sKz)`)gx+j^!Hp0jGg<=6-mDte{Ii(GubmAW!HV7%ZQNbqH6hJ5dc(f%aw8rYVNd z385Gpk%-&cttZVj8UtOeZ_$Y4hBQL`Fkg0z4hKV+o{gE-CWP zESHOAO*a(Yq>0C*tlM4uP7e|{;R)k$(&$Bic(hqy;a?(ThQ$Y-hQhjIp=Z4%E+yFX zAr0ihHo$Ly!!3(UgJYOI9zi&)s}bc5{b(d1*=%VDBVQ(F{-W36UqAf_0A<;U3NL zteF|O5o`iyu_=V{Q4yQM$soRzdC)K^QQ-UP(sj=R_NNr7_rrdmWM|+XYsE5B=?J z*J%W<0qd3xSrozDU!jE1JO*g481usEUk5|;d@xtU&tkqyg^c?pR+|_ppE($@O4_Leip&QXrB5xc)+NC zoEaln2nq}8pU@wM#i9BFe1Tn_)FGWr=K?)AMDWX6VBCYh#?Z!{U9%WO@CG3*hv6ZZ zyMPQ5av9{J1zdkWA3%Us8bE5T1t}{&fX5kx1Az9n0Qxl8j&o*vS=8h5&yZ==FnNi? zw6eNbR}UU(4OSz&e)j@!Y?WC9yjyF~zg*iO^{)|kwAB>Qlz+uM1lY!~inn0(P#aF> zF6=heOhq50Yhc7yKZ3{E2gik&f7x8YG!b8T1;$yjNMkZWX5cT?XN0TMJV#U`R~Wbs zF3W-DM3rE_S)GMZphH``0{rTW?8CpIv(ch`yxn}zm899&+6{-+s@?lWR&9A3abEpf zI9{|g3o2gJ?|+WbGj!S$UK2fGetsMYcd;d;tMM>oQm^)IjYqJNizucQ@N|7*en{j; zodq4T_#VM$a`lNR%f>hVix}WF*DL;EAeF z%rtNI;cZ$Z$!RrtRtxf5I}bL6{wN+o)%kTeodul|PiXfHd_#!8gb;Xn=m2Fu0Oih( zP}8J41h&mKB?NZ}3p8?&0~Kwcd?OwLe&_-r{0QY@Smvn*1Y-g%AS7m}NADC;P0${s zVi||PBe9$o2BJrFfxTF&sSErvmT;s$!?*O|*^Dy5xPHWKk>%Le2Z|6jYkH}LqN>7d zSL{clu=y;(idZBM1D&;3>jmSb%B;?p6pg(0CuUJZd&Qbnk|3sw%|O=*t%(wzU%2V$ zBd{QyuvG;ks(ygZPtbXS&Lo}Awzj>6^#%Rv%S``OI9?puzyfT}9WNnfwIbS=mk?6A zwsp1`PZA+B$k9^k(Kz)^NEYyo4acA#w~;4Wg_}`2Pft2>dNA@P`5z zd=yt-qQL^VxTf`47!$W*;J03iT$XJJL5_S8$p;BoJ0D^?`9Ls4Z9pyIyrn)00i7*BWGD) z8v`e4vJD%4p2CcjVTGmUVgKLGv~!HfyzP``|VTIuyA_!o1Mz?&<_r(=fEFA@jbH!Jz(0NT4;r6*2o~XY zAgu0$1Fnr<=pE;-dY;b5>HI@DUZhb}HAj7p{usG?*yCfVtue#AWDvD~pxe@8&hiF( zoP;NYQSUSR@yP5iu&1pL{V_b!YZo3)v8FkX3FY)X9)j{zS|G8Bb$r-hP+ziW;#%36Y z4~txFQ^kngJ2NP2@Fu%Y)B^YN7$A&8O^b z_Vz2_PN#bO@IHIU-YGeH{g@GZmwlVWZ1Q7v+qX-ZJ{-urja1Y+M z#U3jT*mooT9(yn1x7x#Y7V&7M6`I-xtlsZ!*5|JRXKxxe);W`v6sDm2gc=X09m+h{ zK6r*>SFM9*7vI#eGkxys4IVvXw$-F&D_}-h@FTsD4uoqdxSuKlD1B@V9Eo_6+ z`s{$G2M?!`;{j`hSh(wQVEka3wH0V95X0vB$ z+qHQ8DOc?UfK}+J-jtHDfDbZke7BH>t1L-W;5To@pTT zdo74_`yug>kc%Zx_Zb!+7eE%qB}Kac+1i54WG zc&GNORr0PTrrtsrIsPkF>pjiQD}a-YZFhD{{>#{#3px{pj^H@Yp>~KdE!HhF480G| z?8(unLp~+bi#s{Mt}-1aZ&fKt$gIiV;>#p=_7Dw5reS!838e`pnKrPR+Ukyt;)g;V z{{Yn5a;*d`!}>%a&M+19Axj!{bhUQ2T${!s#B)=bVeqkNfO`p)8L!t#875&TAjd>! zN;n#q!TBMCFtVZ+_Z ztOHyKI_D9)a<@%oS|vIoCoD7#kHCAu^rov=TkqB?CQ3MV7%3TTCdc;iC~H@PyJ~5f z&18nBF+LczVpUF+PQ#Q&h6))nr5PM#TqXuc~D7GcXkpAnyZU z!LqghuW{l%@5$`VpP${!_(>QN1Ypi!H^4<_iQz=9R>v3xOT!FfyIM!NqgX6a2^uKS z0o{g0+p5u3MPb8^$>O-B0!~B|lh(XcYaqmIvxV(CN)Nh&LzI>7W@;Q3@MMOo)ln@v zYh)5yNh;e0y{h+`hXI@RW(>hs;1dopKeI66A1eh;O-e*MpXZu$Rw`DY%) z3D7Xk{t2fmoZJb>^1jS^f0fQ}(D@1-*>n6hd?;bmyO?kju@ruu!Lv&Exg1&v1%#g{ zYn9w$1*|j$ivh`}uyVr1QLy-HSdc}q*ce^Ith9HX0eM+R?kNjBO1D339Mun7@Hi82 zL^6Btl5x$v7P*$V7Q2?b7QJR&Gp=D9xg&&yCv*)e23Q$~uBUXpE&pjqPfIQ|y7VeTfPmif0P<>{}&YIvbk&)(gUUwDza!n&t8QhGZh%BdoQJK=b# z!0MqD4J~_PX&#IguuKWTa=iXo1$)BIDP>@#m(pvcOd|E~S#4*(L;(R*h-Jv}$is+L3DUMfJEyi|bd zXhD$R%^$HeJLjc9_j9dQ%fQL@_z##tV9agw4-&=?!xP#?tx2C5$DRkfc8F}E2=_vO z1IWJ?H{<3W*fI2)+mS*IOTW3tioTMJZ4YlZ?&RKnXyduoCDJ9&%=!KZ9><2m?KC*% zG&y8Qbl_Kaa^npy4TfN;5I9p9e8NZo5*lu-WSdQ4AC=Ij-ROKwWHQ`vNt!E$0T3ZV zU`JVyY&T<*IPD(BjSpTNPOe&>n}#V_tx(eh(PD%)Jtmb>|B=prg99rE3u2f#nDW$S z1)&w`f%*$Ne@;iX@SmgaKhxoS@dj|;fVTVB5Oa7W4LeZMIKI}#DReQNB`2_YBf5f2 z|AS!B8__frSG^o?qp^GK5WlXfn{!fznKii>Xl_2*Y6pmSAblcE6fD96qVEcMqx7=Z5CI zD0med*k9e+O;WgX2g08Q+Iz>^_O8m^wMe0)s0C#-DZ-wE#nX>cm}gEtNQ{P+7YL{( zLmDpxLu6t4F{H0)TeaqQD`yuA`AdKC`fXd4`$X4_A_!IurxsAnicm^~QH{(8+hnrY zCXdYkZOxjE-VbWB_sr<#yDE-LC40IOs9}BSh-*_WeW1AOZ_}CIH z)ba|Z!O~@MNMyX8SyfxC=R}lQRNr7XIexst0GAlNx*p+u_C|Amtcczt9>iA*KkHMA=Q!G?5a&Kg3MHlAO{+QS5`2T5jzS`-l6ArVnfs{uW;*jOFZvQM6^ zlFhNX9Fc8xR7hxeQhBIL_&kU%J4>8Uz!OaR3Y{OI!v)fxuYsJaaHCJ*2AyEfO#s)I zUs7d|#v0VPb1f)V7h8iP9D4x*&2X@v2cERg2yqDRcTG7T1Zh8nnEoL$+gt$?2g?kI z@2-pFpJvqnVQd@3zoH3GH( z0Fg+70E3L+1~>M#R)BZ&p;FnjSk>*-KfF_S*O~t zrP)Dq(vBv_RHvzDK@1U@BMo8v?{NS+gUo4)y^64Cf5N0Y=zN1t5OqalOk=avG$h^X zDVOn`94Mh`K-l{+IP0y{!2)X#mP?2r>~?s92n#kyqY}EI_1UHxWP7&LSqF@wi@Adh z;v|q$WM*T>+BxJm2jqhy&i2=dtZ_DrvjfJ7?DlnaFTaG`VqdDYFe?m3*cM``M#5jD zFTuDJolSH$)9I)4X{K(WkNv>ONyr;?Z=Pey44n?0=xB6)2MM1*2{8kJMknT5`MlEm zcrqThKrLXroWzrk;t_veJifUvs#p>>Gg0teI3Wj%DI9%R1A+De`Z#ifR}xFc%ci=| zXFNe6+l*%zB;*7v9iuol21b;k!fL+@#US(>raA%mCB(8cOw?-}*PCgr-y{D7Qz?I6 zzb{k6O;jQ;BjW{8VjU7TfCk5Daazm9g?0q zICjFGexAM~bUp{ii;7Aq=fxoTz~)hDv%w((p)=I7CjbK$8YcD#^J#JiKn-5fP)rdH z8A61e^zWhr{yMbrv?e(UjO9h>h@c@_G1Nhf9uL~VXpW;NA&zCdY0`@H+!rMVVr`wy z6x+n8e-X-FZV={}4k0e%WlK;^w`+_GJdXPBO#vc52{Sv?B%=3rO{?AN6>##yejRe!|yKc=Gs>4`hZ_a`I+V8vywq-{3k-jhMUTMoHeJj^r=@Z@4h#gT)I@G{kTX-6F)Ya?Rf9w&_bo&a2I$ zpZIKlSnDgfD9U2u&&AacRj(ogc;#~n#*-rD)X==Zaxx?1Z7|+RXdPjKK)nq{a8K=N z=7k~ZT@BK;n>Tjn7-8Cv>Il=gc`vLl+EaL#uX-6tP(aJXMc(#@{x%UDMgXu30hTbd zg4KnPKbTcv7mm1$i>cv)ya{7yO-iLI%j#c2NgNIjek4k%q&xXB-*gz#bxIIgS>ErsoaqzzXLDA2-s z9eo~_Kh8codd);9_SrGHSpY}H?f7N$1<-mR7LIH!S+GEY!sxP@4lPAs?elcgs#;B0 zxi7(jmT4Q_q+@ZHl<~{3bu{AmS2BNqH}Am6F?N*s=+x%G%fWQgkWh;%Zwl zh5p=9+TGkt+WpP6PJamkl6LD-DA`W2x4dMr7i_1L8Q8GQ)>eu=__}3UJ>`2kWwv$7 zykMLgXeQ_Th01@8K0VGdb}Y3$Fg{G}j4V@ktVs>Naj27e?rZ0MyA6+>I)16Ax(Dzi z?6*Le)D2NT514$aQ6&dXE%gWtb-48=!hcuiO~^eEfbVTgv7v##F1XQdL#7kI)Z6Tp zHErvB5~@g3{V#X0iE}FRTLFa-boI&06oQ54w_Uc77x8FGuiui+c(5IJcC{llA8g}Y zoivv!-W~4H{7&G1Sbd72`7}m=vTTjfdzc$Bche4}ohOLZ_jKyhwf>ylqUsZTU_+8K`SZ zt3{G$f2T#5gX1y>=k*+%zssIz&6}=S$etW%rs1xdfonsm_=aO~qGw9u(R-2ow z_r6YhBrRCOJ3DDFgwK6mE+T$c-xkXfKG20jFJ}UR( zW{%3;zT|~uVU>%CMH?lWFIhn+&vx+?t+j_@M`sxhPz7WhyFf70*!=+okU^labGsZWSBgQmAoADYY=ILXUg;QJ)7y!W)`TGnJpmqQ6y0eI2V`qwkc?`zikMX}sP#y+ zM^Jx)VQ%aWW`}_TWcf<2$<=Lc+_qu5G)~0+(k&7DG*Xlww7Rupow8cD*SP%&m{61+ zL*`>O_q1#c-6O4=Q*z^Djm@WM9WcqA!G6CfeZeTcy@39yisD2-+JE87x;RN_QVzFnOy`dGfo>9eKaSZ8R zvND(FZN!w)kFY=+kL4)HGSQd~g1qmJ4Uo2TJcSG)j7t>GVlNagc@Z&#*tl=|T7BE| z&HCmmY%O)L(=!U^ARW1gFLj75mt!m4&Hr4W`AHZuwBQ-1p93#b-!o?ljm=M#>qkXh zdtmGstXqn?Amk?2LT*ATpj75=95N?C@GNR7^nOo-g!|bnnVJXaYo)^n&qjC{NQrzfKYzu=X z*nYIMtiQ&}Wm7AP(Y8R?ix+SuIqm|l%IQvhOl^UgN#)c4n@gPO9I(cz+Itz>X7zDp zR$Z}&UV^HJT&-HZq&xfjC@5!!a2m~+SL4iMPS-s{d1*a~g+^U~2D#PH+yyVKCzOwi z0!M);X{zEe$!7>mf;(iz&19NVCRh(cFG^X+K4TaDqR^2J7`qX3AO0xt+l{|FjR7b% zcE4spO3E;9w1SFb5aG0xx{q#TJ7VJ!-es7tB~9(_LH@KcVH`w?v_gw2*}MWbWvvf| zj8QzY`R2}wic)UW=1mB)W%JB$E||C=PAKk8-`4d{{4dXx^O6S zH?P!(s|OwT- zIwI=N_INRh#%FO4i&d&pB+K`HnhqrhY7cQbT*mYf`f~|!o^gVAva~KMFLq{9HV#Fo ztY471%Xh9?ydbNV7oBw&@Zwb%O3*je%7U;=5DW!$VkKL$)-Y#fYKS9K<0P8$Q=A$Z zG$ItHVZVjbc1d$LG}AO1=)?CeJon&H%-zM|B~XaSD~S6AcuJuqvn62w(qY3vK+)m{ z=XO7itpP50B7m_`e~za<=5ZtlJRU+oaBR|Hh;Ke)5h>*wCJVZn2rO7}@gI=Ksy+y$ z;GH6$WJ#2W&BF>;eG)=iYJ6hY?#U&H*if;yBsNr5H3^$nzVC<6?c#)Jh7X6}qYyCB zcpSe*P(9DzClHls!01oQ)HEjG2Ly2Qn$A+1xs-WF2KFYn>kr0swRz$fD-ijocyA zg3w9s^TWZPeh8mP0i&=Uq5Y8J8If+?HN-W-g}fKv&S^L}Hb8y@mkB`f1ipg@1|W`M zJUMa1i*^o#h|)#q>%vVC8jzK+9o+%hlpT8!^xTg%)nO(KM@3D6^a9LP|d$)0%m~PD^|q(n`l&> zL%m&07T2X19swO0NH;VitRfYtcH~7&GywxdyrAKb?w4TUAu#}%OsXLYX-CfD9$Nmo zjSJt{_(5$gD(8W2{vv8sU!_B;iYq#PmHw~L`D;2QI``74qjdlU{op=(I0kGmiZDe+ z#!HJ7F{gi$Vcj9xzV0=yxa9GVIY3cV9sr=$yW%6q`m@nr`H3;VS${oae{o`;jDi>86e}t&>i#gh4RSpHo z8~jP4i8G^*6=ByZzrhn2%`8WbSay>T)}u*$ZjGk+Hyq6!>tFAS;e4e%oZ$L^G#`4) zhf`f5cm*Zeolp(N2}+CwIwAgB>4c9HQHa$w=Ic&0Z>19;1YKkRY~C6&5O4gN;KWf} z1FccwC{aSgtgRl^zne7Nx(R%vADGJOS#pv`DkqEX$oN7>eub5I1QxHP@Dydq<;<8~ zxKjBd9vHUc$EXT8i0S`^Rp2`MZ~l8v!O8h0j=B&n0m4K82;$!m2r@6%+yxCp>R$ze zrMDgizuAF-P|3dwNce>AZG*&t03^hJ3y|<(13`0%Yj0^6cDFdKeLAaRO0JR(h<`&+2zgz52|8xQ-c_*puK!PA@s>}Ek6@tGN9hQG zx<3F5@!tw8{CP3H3KrjeE3kND%;HyNUaV#oSJ8Xix8ElCZz*XfbAnTEb2ziA=_4#@ zc>bznXZX&J@unOm6P~166DuB=IQqY6m>C8$Gonr5vidYvfD2iVr+Xj(8u4!kn%(Pi zuCm6Fvpd{|PgGcu%Wpk=epleLE@#(Xj=zn;vmSqSC;%SuZwQ{-*Jl#j@Og9miK7G_Ssdi#Sw{)C9Srrg%pdgdk_x96Jly>YHO+P zsH4oqeL5{~-^l-amCYg)6MHerka<=vf!}DwH5fJA0f@aPj(`?zq$Q%EGgiiygZEDNYJI#(>?hIvQJwvx<~qC=sVn6OXv2Xo6o zR!1&g)+VlE^tc(Br-s(%YggXS3zyiu|r3qouf-D=ry|)I|{u12P|Cl^po< z;)gNxFoAv$8K7^4Yi;GPu<*9Tv}m{>$~U(|jK7^Rv1B;e0~z3JRv4~n{jFtv%OPEt zVZADKc504O)@bV3^}c^0da6H*zhwWO{@d_98c)WP{T~VK97qoK4{Yj7M2*4I$-($w Xa+7JKQCm8RCxIvBf49Q?4m|%4Q9&aE literal 0 HcmV?d00001 diff --git a/convs/__pycache__/vpt.cpython-39.pyc b/convs/__pycache__/vpt.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f00a1a87f385b9b46206ef6740d215f95bb6b56 GIT binary patch literal 3880 zcmZ`+TW=f36`t7}cbAkz$%-sFjzc$Y8m6g~xUGw#Y2enbdVwJru4wc$ammH}7!Mwt9BQ=G13(ou1oqdtS#wI~1O%z2u$R zubJ>g@REr@y61Ms7yJop*DoNC?X)eu{4CAWezvRPEZ^^|o>V$~92d#quX=k@=+%?7 zi1y-KMg!{n>|S*D?vE}=Y3)3ea*)_0X!-vDPy~BuET+ng^JTm3LL)qSoM2@BHHag zmsykzd)j~YcsD9u4CJVOC}XjCD$|3*VicIhUK;1y?Yee~wAa&(M9H|2QLis#SJ!Ca zvvgO7c|pA*O$u!f`UAbx?Z+aT>vfC1N)EM^W!nF(kXexyFSJ8DXm49dIg;%_2S#ib z<3j3mlmOETW8OZzi=d>^L7P-$oIFj>k&+{#xH!Kf5>(;0_4Jw>B2`Ep-pCPCb*SvmjEbN-@Ids~bBQd+_Lq z!353DjS76UI#Y0dLPB0LduSrv<^%lm-+%z!Y{DM0KZa*qz&|2>>*18;k7 z9H+qeN55xTgCoLo_9e@#Pm%3_Y?A>d|jOp z$f3DcPN%M8#!vCjDQ7U=;B*HT1vT8tXmNJLDm4Gk;mG05;V2-!^>C!DwX+LDe~u~Y z=S03BVhns~W&deb?eP$UIbA9#a|wdk@c&nV&I$_x0TX&7xc*q_FYRoZsN zn=DCYnxa52S7ClGU0v03p}$uke@^oO-DplBlmW5MH=aFuXJ)TA=DmSgGTFD7_$wNt zwMy#}h}b|SLy-@6zFOTm;-gLR5qBScO3~6W(Go~GaE2%1IRn{JD3{A&Tm zv!Qpd3>~B=GMf%+6mr!)npk)jmNYvdWRg7N@al)?)pjx%Y72FT35}kRCut%90`g_` zi`lxXS6w%iOQquuCRO1IuCWHkSVW#*u~+!2<=Z}-rO8LvW>-JA?v0D5giByC{&Ejt zg8T)6iUXubmSed4kzGLmAh6G@!Z~u0K>5C@5l|pv9$hkK*RInQ6Xy#T*Z((g0an`^ z?9490qKY$Zc?}oh%?Ko%u1m_yZAOZklOB& zguQM*d5#KTnB~Vqs#Uc^NxfY&Skn9}^@xO1Qq(-H;22UlVKm6ELad2zj?-9)Xdkuo z5RqG8VSWdMp_*I+zM2+Ql`B@$`VM+Nv>WysUjcW^3i)V#hPbO@-hdtDAQyiFJCZ#I zyP3hK?|r)PS;RC(Zgce|mP`YrdItXLA!^gzUtuh#z{af5S+hq=SG0pD$}{Z+DrTs( zM==vdRH_fV#5bboco=u95k*nbR0`^6#8H+~_lf+H2nAlFnCgJj3)3!Iz}eaH9wHU9%s7$Yz^w2W+4K`KxS#GB~ho6ncc zEHm+?bBjKPHcj1@Z=RTs$p~Lqv#&`-#;?von^%SRW#XI5exFx%_=1p~(-)=>{Tg)t G>;DC(SdM7` literal 0 HcmV?d00001 diff --git a/convs/cifar_resnet.py b/convs/cifar_resnet.py new file mode 100644 index 0000000..8ce7d4d --- /dev/null +++ b/convs/cifar_resnet.py @@ -0,0 +1,198 @@ +''' +Reference: +https://github.com/khurramjaved96/incremental-learning/blob/autoencoders/model/resnet32.py +''' +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class DownsampleA(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleA, self).__init__() + assert stride == 2 + self.avg = nn.AvgPool2d(kernel_size=1, stride=stride) + + def forward(self, x): + x = self.avg(x) + return torch.cat((x, x.mul(0)), 1) + + +class DownsampleB(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleB, self).__init__() + self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=stride, padding=0, bias=False) + self.bn = nn.BatchNorm2d(nOut) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class DownsampleC(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleC, self).__init__() + assert stride != 1 or nIn != nOut + self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=stride, padding=0, bias=False) + + def forward(self, x): + x = self.conv(x) + return x + + +class DownsampleD(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleD, self).__init__() + assert stride == 2 + self.conv = nn.Conv2d(nIn, nOut, kernel_size=2, stride=stride, padding=0, bias=False) + self.bn = nn.BatchNorm2d(nOut) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class ResNetBasicblock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(ResNetBasicblock, self).__init__() + + self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn_a = nn.BatchNorm2d(planes) + + self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn_b = nn.BatchNorm2d(planes) + + self.downsample = downsample + + def forward(self, x): + residual = x + + basicblock = self.conv_a(x) + basicblock = self.bn_a(basicblock) + basicblock = F.relu(basicblock, inplace=True) + + basicblock = self.conv_b(basicblock) + basicblock = self.bn_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(x) + + return F.relu(residual + basicblock, inplace=True) + + +class CifarResNet(nn.Module): + """ + ResNet optimized for the Cifar Dataset, as specified in + https://arxiv.org/abs/1512.03385.pdf + """ + + def __init__(self, block, depth, channels=3): + super(CifarResNet, self).__init__() + + # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + + self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False) + self.bn_1 = nn.BatchNorm2d(16) + + self.inplanes = 16 + self.stage_1 = self._make_layer(block, 16, layer_blocks, 1) + self.stage_2 = self._make_layer(block, 32, layer_blocks, 2) + self.stage_3 = self._make_layer(block, 64, layer_blocks, 2) + self.avgpool = nn.AvgPool2d(8) + self.out_dim = 64 * block.expansion + self.fc = nn.Linear(64*block.expansion, 10) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + # m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + nn.init.kaiming_normal_(m.weight) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = DownsampleA(self.inplanes, planes * block.expansion, stride) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv_1_3x3(x) # [bs, 16, 32, 32] + x = F.relu(self.bn_1(x), inplace=True) + + x_1 = self.stage_1(x) # [bs, 16, 32, 32] + x_2 = self.stage_2(x_1) # [bs, 32, 16, 16] + x_3 = self.stage_3(x_2) # [bs, 64, 8, 8] + + pooled = self.avgpool(x_3) # [bs, 64, 1, 1] + features = pooled.view(pooled.size(0), -1) # [bs, 64] + + return { + 'fmaps': [x_1, x_2, x_3], + 'features': features + } + + @property + def last_conv(self): + return self.stage_3[-1].conv_b + + +def resnet20mnist(): + """Constructs a ResNet-20 model for MNIST.""" + model = CifarResNet(ResNetBasicblock, 20, 1) + return model + + +def resnet32mnist(): + """Constructs a ResNet-32 model for MNIST.""" + model = CifarResNet(ResNetBasicblock, 32, 1) + return model + + +def resnet20(): + """Constructs a ResNet-20 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 20) + return model + + +def resnet32(): + """Constructs a ResNet-32 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 32) + return model + + +def resnet44(): + """Constructs a ResNet-44 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 44) + return model + + +def resnet56(): + """Constructs a ResNet-56 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 56) + return model + + +def resnet110(): + """Constructs a ResNet-110 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 110) + return model diff --git a/convs/linears.py b/convs/linears.py new file mode 100644 index 0000000..f2eb0a3 --- /dev/null +++ b/convs/linears.py @@ -0,0 +1,167 @@ +''' +Reference: +https://github.com/hshustc/CVPR19_Incremental_Learning/blob/master/cifar100-class-incremental/modified_linear.py +''' +import math +import torch +from torch import nn +from torch.nn import functional as F + + +class SimpleLinear(nn.Module): + ''' + Reference: + https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/linear.py + ''' + def __init__(self, in_features, out_features, bias=True): + super(SimpleLinear, self).__init__() + self.in_features = in_features + self.out_features = out_features + self.weight = nn.Parameter(torch.Tensor(out_features, in_features)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_features)) + else: + self.register_parameter('bias', None) + self.reset_parameters() + + def reset_parameters(self): + nn.init.kaiming_uniform_(self.weight, nonlinearity='linear') + nn.init.constant_(self.bias, 0) + + def forward(self, input): + return {'logits': F.linear(input, self.weight, self.bias)} + + +class CosineLinear(nn.Module): + def __init__(self, in_features, out_features, nb_proxy=1, to_reduce=False, sigma=True): + super(CosineLinear, self).__init__() + self.in_features = in_features + self.out_features = out_features * nb_proxy + self.nb_proxy = nb_proxy + self.to_reduce = to_reduce + self.weight = nn.Parameter(torch.Tensor(self.out_features, in_features)) + if sigma: + self.sigma = nn.Parameter(torch.Tensor(1)) + else: + self.register_parameter('sigma', None) + self.reset_parameters() + + def reset_parameters(self): + stdv = 1. / math.sqrt(self.weight.size(1)) + self.weight.data.uniform_(-stdv, stdv) + if self.sigma is not None: + self.sigma.data.fill_(1) + + def forward(self, input): + out = F.linear(F.normalize(input, p=2, dim=1), F.normalize(self.weight, p=2, dim=1)) + + if self.to_reduce: + # Reduce_proxy + out = reduce_proxies(out, self.nb_proxy) + + if self.sigma is not None: + out = self.sigma * out + + return {'logits': out} + + +class SplitCosineLinear(nn.Module): + def __init__(self, in_features, out_features1, out_features2, nb_proxy=1, sigma=True): + super(SplitCosineLinear, self).__init__() + self.in_features = in_features + self.out_features = (out_features1 + out_features2) * nb_proxy + self.nb_proxy = nb_proxy + self.fc1 = CosineLinear(in_features, out_features1, nb_proxy, False, False) + self.fc2 = CosineLinear(in_features, out_features2, nb_proxy, False, False) + if sigma: + self.sigma = nn.Parameter(torch.Tensor(1)) + self.sigma.data.fill_(1) + else: + self.register_parameter('sigma', None) + + def forward(self, x): + out1 = self.fc1(x) + out2 = self.fc2(x) + + out = torch.cat((out1['logits'], out2['logits']), dim=1) # concatenate along the channel + + # Reduce_proxy + out = reduce_proxies(out, self.nb_proxy) + + if self.sigma is not None: + out = self.sigma * out + + return { + 'old_scores': reduce_proxies(out1['logits'], self.nb_proxy), + 'new_scores': reduce_proxies(out2['logits'], self.nb_proxy), + 'logits': out + } + + +def reduce_proxies(out, nb_proxy): + if nb_proxy == 1: + return out + bs = out.shape[0] + nb_classes = out.shape[1] / nb_proxy + assert nb_classes.is_integer(), 'Shape error' + nb_classes = int(nb_classes) + + simi_per_class = out.view(bs, nb_classes, nb_proxy) + attentions = F.softmax(simi_per_class, dim=-1) + + return (attentions * simi_per_class).sum(-1) + + +''' +class CosineLinear(nn.Module): + def __init__(self, in_features, out_features, sigma=True): + super(CosineLinear, self).__init__() + self.in_features = in_features + self.out_features = out_features + self.weight = nn.Parameter(torch.Tensor(out_features, in_features)) + if sigma: + self.sigma = nn.Parameter(torch.Tensor(1)) + else: + self.register_parameter('sigma', None) + self.reset_parameters() + + def reset_parameters(self): + stdv = 1. / math.sqrt(self.weight.size(1)) + self.weight.data.uniform_(-stdv, stdv) + if self.sigma is not None: + self.sigma.data.fill_(1) + + def forward(self, input): + out = F.linear(F.normalize(input, p=2, dim=1), F.normalize(self.weight, p=2, dim=1)) + if self.sigma is not None: + out = self.sigma * out + return {'logits': out} + + +class SplitCosineLinear(nn.Module): + def __init__(self, in_features, out_features1, out_features2, sigma=True): + super(SplitCosineLinear, self).__init__() + self.in_features = in_features + self.out_features = out_features1 + out_features2 + self.fc1 = CosineLinear(in_features, out_features1, False) + self.fc2 = CosineLinear(in_features, out_features2, False) + if sigma: + self.sigma = nn.Parameter(torch.Tensor(1)) + self.sigma.data.fill_(1) + else: + self.register_parameter('sigma', None) + + def forward(self, x): + out1 = self.fc1(x) + out2 = self.fc2(x) + + out = torch.cat((out1['logits'], out2['logits']), dim=1) # concatenate along the channel + if self.sigma is not None: + out = self.sigma * out + + return { + 'old_scores': out1['logits'], + 'new_scores': out2['logits'], + 'logits': out + } +''' diff --git a/convs/modified_represnet.py b/convs/modified_represnet.py new file mode 100644 index 0000000..99d2854 --- /dev/null +++ b/convs/modified_represnet.py @@ -0,0 +1,177 @@ +import torch +import torch.nn as nn +import math +import torch.utils.model_zoo as model_zoo +import torch.nn.functional as F + +__all__ = ['ResNet', 'resnet18_rep', 'resnet34_rep' ] + + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=True) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=True) + +class conv_block(nn.Module): + + def __init__(self, in_planes, planes, mode, stride=1): + super(conv_block, self).__init__() + self.conv = conv3x3(in_planes, planes, stride) + self.mode = mode + if mode == 'parallel_adapters': + self.adapter = conv1x1(in_planes, planes, stride) + + + def re_init_conv(self): + nn.init.kaiming_normal_(self.adapter.weight, mode='fan_out', nonlinearity='relu') + return + def forward(self, x): + y = self.conv(x) + if self.mode == 'parallel_adapters': + y = y + self.adapter(x) + + return y + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, mode, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv_block(inplanes, planes, mode, stride) + self.norm1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv_block(planes, planes, mode) + self.norm2 = nn.BatchNorm2d(planes) + self.mode = mode + + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.norm2(out) + if self.downsample is not None: + residual = self.downsample(x) + out += residual + out = self.relu(out) + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=100, args = None): + self.inplanes = 64 + super(ResNet, self).__init__() + assert args is not None + self.mode = args["mode"] + + if 'cifar' in args["dataset"]: + self.conv1 = nn.Sequential(nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(self.inplanes), nn.ReLU(inplace=True)) + print("use cifar") + elif 'imagenet' in args["dataset"]: + if args["init_cls"] == args["increment"]: + self.conv1 = nn.Sequential( + nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False), + nn.BatchNorm2d(self.inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + ) + else: + # Following PODNET implmentation + self.conv1 = nn.Sequential( + nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(self.inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + ) + + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + self.feature = nn.AvgPool2d(4, stride=1) + self.out_dim = 512 + + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=True), + ) + layers = [] + layers.append(block(self.inplanes, planes, self.mode, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes, self.mode)) + + return nn.Sequential(*layers) + + def switch(self, mode='normal'): + for name, module in self.named_modules(): + if hasattr(module, 'mode'): + module.mode = mode + def re_init_params(self): + for name, module in self.named_modules(): + if hasattr(module, 're_init_conv'): + module.re_init_conv() + def forward(self, x): + x = self.conv1(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + dim = x.size()[-1] + pool = nn.AvgPool2d(dim, stride=1) + x = pool(x) + x = x.view(x.size(0), -1) + return {"features": x} + + +def resnet18_rep(pretrained=False, **kwargs): + """Constructs a ResNet-18 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet18']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model + + +def resnet34_rep(pretrained=False, **kwargs): + """Constructs a ResNet-34 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet34']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model \ No newline at end of file diff --git a/convs/original_resnet.py b/convs/original_resnet.py new file mode 100644 index 0000000..3f08eca --- /dev/null +++ b/convs/original_resnet.py @@ -0,0 +1,1004 @@ +from functools import partial +from typing import Any, Callable, List, Optional, Type, Union + +import torch +import torch.nn as nn +from torch import Tensor + +from ..transforms._presets import ImageClassification +from ..utils import _log_api_usage_once +from ._api import register_model, Weights, WeightsEnum +from ._meta import _IMAGENET_CATEGORIES +from ._utils import _ovewrite_named_param, handle_legacy_interface + + +__all__ = [ + "ResNet", + "ResNet18_Weights", + "ResNet34_Weights", + "ResNet50_Weights", + "ResNet101_Weights", + "ResNet152_Weights", + "ResNeXt50_32X4D_Weights", + "ResNeXt101_32X8D_Weights", + "ResNeXt101_64X4D_Weights", + "Wide_ResNet50_2_Weights", + "Wide_ResNet101_2_Weights", + "resnet18", + "resnet34", + "resnet50", + "resnet101", + "resnet152", + "resnext50_32x4d", + "resnext101_32x8d", + "resnext101_64x4d", + "wide_resnet50_2", + "wide_resnet101_2", +] + + +def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> nn.Conv2d: + """3x3 convolution with padding""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + groups=groups, + bias=False, + dilation=dilation, + ) + + +def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> nn.Conv2d: + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion: int = 1 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError("BasicBlock only supports groups=1 and base_width=64") + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion: int = 4 + + def __init__( + self, + inplanes: int, + planes: int, + stride: int = 1, + downsample: Optional[nn.Module] = None, + groups: int = 1, + base_width: int = 64, + dilation: int = 1, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super().__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.0)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x: Tensor) -> Tensor: + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + def __init__( + self, + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + num_classes: int = 1000, + zero_init_residual: bool = False, + groups: int = 1, + width_per_group: int = 64, + replace_stride_with_dilation: Optional[List[bool]] = None, + norm_layer: Optional[Callable[..., nn.Module]] = None, + ) -> None: + super().__init__() + _log_api_usage_once(self) + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError( + "replace_stride_with_dilation should be None " + f"or a 3-element tuple, got {replace_stride_with_dilation}" + ) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu") + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck) and m.bn3.weight is not None: + nn.init.constant_(m.bn3.weight, 0) # type: ignore[arg-type] + elif isinstance(m, BasicBlock) and m.bn2.weight is not None: + nn.init.constant_(m.bn2.weight, 0) # type: ignore[arg-type] + + def _make_layer( + self, + block: Type[Union[BasicBlock, Bottleneck]], + planes: int, + blocks: int, + stride: int = 1, + dilate: bool = False, + ) -> nn.Sequential: + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append( + block( + self.inplanes, planes, stride, downsample, self.groups, self.base_width, previous_dilation, norm_layer + ) + ) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block( + self.inplanes, + planes, + groups=self.groups, + base_width=self.base_width, + dilation=self.dilation, + norm_layer=norm_layer, + ) + ) + + return nn.Sequential(*layers) + + def _forward_impl(self, x: Tensor) -> Tensor: + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.fc(x) + + return x + + def forward(self, x: Tensor) -> Tensor: + return self._forward_impl(x) + + +def _resnet( + block: Type[Union[BasicBlock, Bottleneck]], + layers: List[int], + weights: Optional[WeightsEnum], + progress: bool, + **kwargs: Any, +) -> ResNet: + if weights is not None: + _ovewrite_named_param(kwargs, "num_classes", len(weights.meta["categories"])) + + model = ResNet(block, layers, **kwargs) + + if weights is not None: + model.load_state_dict(weights.get_state_dict(progress=progress)) + + return model + + +_COMMON_META = { + "min_size": (1, 1), + "categories": _IMAGENET_CATEGORIES, +} + + +class ResNet18_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnet18-f37072fd.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 11689512, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnet", + "_metrics": { + "ImageNet-1K": { + "acc@1": 69.758, + "acc@5": 89.078, + } + }, + "_ops": 1.814, + "_weight_size": 44.661, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + DEFAULT = IMAGENET1K_V1 + + +class ResNet34_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnet34-b627a593.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 21797672, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnet", + "_metrics": { + "ImageNet-1K": { + "acc@1": 73.314, + "acc@5": 91.420, + } + }, + "_ops": 3.664, + "_weight_size": 83.275, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + DEFAULT = IMAGENET1K_V1 + + +class ResNet50_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnet50-0676ba61.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 25557032, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnet", + "_metrics": { + "ImageNet-1K": { + "acc@1": 76.130, + "acc@5": 92.862, + } + }, + "_ops": 4.089, + "_weight_size": 97.781, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/resnet50-11ad3fa6.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 25557032, + "recipe": "https://github.com/pytorch/vision/issues/3995#issuecomment-1013906621", + "_metrics": { + "ImageNet-1K": { + "acc@1": 80.858, + "acc@5": 95.434, + } + }, + "_ops": 4.089, + "_weight_size": 97.79, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +class ResNet101_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnet101-63fe2227.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 44549160, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnet", + "_metrics": { + "ImageNet-1K": { + "acc@1": 77.374, + "acc@5": 93.546, + } + }, + "_ops": 7.801, + "_weight_size": 170.511, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/resnet101-cd907fc2.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 44549160, + "recipe": "https://github.com/pytorch/vision/issues/3995#new-recipe", + "_metrics": { + "ImageNet-1K": { + "acc@1": 81.886, + "acc@5": 95.780, + } + }, + "_ops": 7.801, + "_weight_size": 170.53, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +class ResNet152_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnet152-394f9c45.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 60192808, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnet", + "_metrics": { + "ImageNet-1K": { + "acc@1": 78.312, + "acc@5": 94.046, + } + }, + "_ops": 11.514, + "_weight_size": 230.434, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/resnet152-f82ba261.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 60192808, + "recipe": "https://github.com/pytorch/vision/issues/3995#new-recipe", + "_metrics": { + "ImageNet-1K": { + "acc@1": 82.284, + "acc@5": 96.002, + } + }, + "_ops": 11.514, + "_weight_size": 230.474, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +class ResNeXt50_32X4D_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 25028904, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnext", + "_metrics": { + "ImageNet-1K": { + "acc@1": 77.618, + "acc@5": 93.698, + } + }, + "_ops": 4.23, + "_weight_size": 95.789, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/resnext50_32x4d-1a0047aa.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 25028904, + "recipe": "https://github.com/pytorch/vision/issues/3995#new-recipe", + "_metrics": { + "ImageNet-1K": { + "acc@1": 81.198, + "acc@5": 95.340, + } + }, + "_ops": 4.23, + "_weight_size": 95.833, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +class ResNeXt101_32X8D_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 88791336, + "recipe": "https://github.com/pytorch/vision/tree/main/references/classification#resnext", + "_metrics": { + "ImageNet-1K": { + "acc@1": 79.312, + "acc@5": 94.526, + } + }, + "_ops": 16.414, + "_weight_size": 339.586, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/resnext101_32x8d-110c445d.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 88791336, + "recipe": "https://github.com/pytorch/vision/issues/3995#new-recipe-with-fixres", + "_metrics": { + "ImageNet-1K": { + "acc@1": 82.834, + "acc@5": 96.228, + } + }, + "_ops": 16.414, + "_weight_size": 339.673, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +class ResNeXt101_64X4D_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/resnext101_64x4d-173b62eb.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 83455272, + "recipe": "https://github.com/pytorch/vision/pull/5935", + "_metrics": { + "ImageNet-1K": { + "acc@1": 83.246, + "acc@5": 96.454, + } + }, + "_ops": 15.46, + "_weight_size": 319.318, + "_docs": """ + These weights were trained from scratch by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V1 + + +class Wide_ResNet50_2_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 68883240, + "recipe": "https://github.com/pytorch/vision/pull/912#issue-445437439", + "_metrics": { + "ImageNet-1K": { + "acc@1": 78.468, + "acc@5": 94.086, + } + }, + "_ops": 11.398, + "_weight_size": 131.82, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/wide_resnet50_2-9ba9bcbe.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 68883240, + "recipe": "https://github.com/pytorch/vision/issues/3995#new-recipe-with-fixres", + "_metrics": { + "ImageNet-1K": { + "acc@1": 81.602, + "acc@5": 95.758, + } + }, + "_ops": 11.398, + "_weight_size": 263.124, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +class Wide_ResNet101_2_Weights(WeightsEnum): + IMAGENET1K_V1 = Weights( + url="https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth", + transforms=partial(ImageClassification, crop_size=224), + meta={ + **_COMMON_META, + "num_params": 126886696, + "recipe": "https://github.com/pytorch/vision/pull/912#issue-445437439", + "_metrics": { + "ImageNet-1K": { + "acc@1": 78.848, + "acc@5": 94.284, + } + }, + "_ops": 22.753, + "_weight_size": 242.896, + "_docs": """These weights reproduce closely the results of the paper using a simple training recipe.""", + }, + ) + IMAGENET1K_V2 = Weights( + url="https://download.pytorch.org/models/wide_resnet101_2-d733dc28.pth", + transforms=partial(ImageClassification, crop_size=224, resize_size=232), + meta={ + **_COMMON_META, + "num_params": 126886696, + "recipe": "https://github.com/pytorch/vision/issues/3995#new-recipe", + "_metrics": { + "ImageNet-1K": { + "acc@1": 82.510, + "acc@5": 96.020, + } + }, + "_ops": 22.753, + "_weight_size": 484.747, + "_docs": """ + These weights improve upon the results of the original paper by using TorchVision's `new training recipe + `_. + """, + }, + ) + DEFAULT = IMAGENET1K_V2 + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNet18_Weights.IMAGENET1K_V1)) +def resnet18(*, weights: Optional[ResNet18_Weights] = None, progress: bool = True, **kwargs: Any) -> ResNet: + """ResNet-18 from `Deep Residual Learning for Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.ResNet18_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet18_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet18_Weights + :members: + """ + weights = ResNet18_Weights.verify(weights) + + return _resnet(BasicBlock, [2, 2, 2, 2], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNet34_Weights.IMAGENET1K_V1)) +def resnet34(*, weights: Optional[ResNet34_Weights] = None, progress: bool = True, **kwargs: Any) -> ResNet: + """ResNet-34 from `Deep Residual Learning for Image Recognition `__. + + Args: + weights (:class:`~torchvision.models.ResNet34_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet34_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet34_Weights + :members: + """ + weights = ResNet34_Weights.verify(weights) + + return _resnet(BasicBlock, [3, 4, 6, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNet50_Weights.IMAGENET1K_V1)) +def resnet50(*, weights: Optional[ResNet50_Weights] = None, progress: bool = True, **kwargs: Any) -> ResNet: + """ResNet-50 from `Deep Residual Learning for Image Recognition `__. + + .. note:: + The bottleneck of TorchVision places the stride for downsampling to the second 3x3 + convolution while the original paper places it to the first 1x1 convolution. + This variant improves the accuracy and is known as `ResNet V1.5 + `_. + + Args: + weights (:class:`~torchvision.models.ResNet50_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet50_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet50_Weights + :members: + """ + weights = ResNet50_Weights.verify(weights) + + return _resnet(Bottleneck, [3, 4, 6, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNet101_Weights.IMAGENET1K_V1)) +def resnet101(*, weights: Optional[ResNet101_Weights] = None, progress: bool = True, **kwargs: Any) -> ResNet: + """ResNet-101 from `Deep Residual Learning for Image Recognition `__. + + .. note:: + The bottleneck of TorchVision places the stride for downsampling to the second 3x3 + convolution while the original paper places it to the first 1x1 convolution. + This variant improves the accuracy and is known as `ResNet V1.5 + `_. + + Args: + weights (:class:`~torchvision.models.ResNet101_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet101_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet101_Weights + :members: + """ + weights = ResNet101_Weights.verify(weights) + + return _resnet(Bottleneck, [3, 4, 23, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNet152_Weights.IMAGENET1K_V1)) +def resnet152(*, weights: Optional[ResNet152_Weights] = None, progress: bool = True, **kwargs: Any) -> ResNet: + """ResNet-152 from `Deep Residual Learning for Image Recognition `__. + + .. note:: + The bottleneck of TorchVision places the stride for downsampling to the second 3x3 + convolution while the original paper places it to the first 1x1 convolution. + This variant improves the accuracy and is known as `ResNet V1.5 + `_. + + Args: + weights (:class:`~torchvision.models.ResNet152_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNet152_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + + .. autoclass:: torchvision.models.ResNet152_Weights + :members: + """ + weights = ResNet152_Weights.verify(weights) + + return _resnet(Bottleneck, [3, 8, 36, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNeXt50_32X4D_Weights.IMAGENET1K_V1)) +def resnext50_32x4d( + *, weights: Optional[ResNeXt50_32X4D_Weights] = None, progress: bool = True, **kwargs: Any +) -> ResNet: + """ResNeXt-50 32x4d model from + `Aggregated Residual Transformation for Deep Neural Networks `_. + + Args: + weights (:class:`~torchvision.models.ResNeXt50_32X4D_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNext50_32X4D_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + .. autoclass:: torchvision.models.ResNeXt50_32X4D_Weights + :members: + """ + weights = ResNeXt50_32X4D_Weights.verify(weights) + + _ovewrite_named_param(kwargs, "groups", 32) + _ovewrite_named_param(kwargs, "width_per_group", 4) + return _resnet(Bottleneck, [3, 4, 6, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNeXt101_32X8D_Weights.IMAGENET1K_V1)) +def resnext101_32x8d( + *, weights: Optional[ResNeXt101_32X8D_Weights] = None, progress: bool = True, **kwargs: Any +) -> ResNet: + """ResNeXt-101 32x8d model from + `Aggregated Residual Transformation for Deep Neural Networks `_. + + Args: + weights (:class:`~torchvision.models.ResNeXt101_32X8D_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNeXt101_32X8D_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + .. autoclass:: torchvision.models.ResNeXt101_32X8D_Weights + :members: + """ + weights = ResNeXt101_32X8D_Weights.verify(weights) + + _ovewrite_named_param(kwargs, "groups", 32) + _ovewrite_named_param(kwargs, "width_per_group", 8) + return _resnet(Bottleneck, [3, 4, 23, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", ResNeXt101_64X4D_Weights.IMAGENET1K_V1)) +def resnext101_64x4d( + *, weights: Optional[ResNeXt101_64X4D_Weights] = None, progress: bool = True, **kwargs: Any +) -> ResNet: + """ResNeXt-101 64x4d model from + `Aggregated Residual Transformation for Deep Neural Networks `_. + + Args: + weights (:class:`~torchvision.models.ResNeXt101_64X4D_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.ResNeXt101_64X4D_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + .. autoclass:: torchvision.models.ResNeXt101_64X4D_Weights + :members: + """ + weights = ResNeXt101_64X4D_Weights.verify(weights) + + _ovewrite_named_param(kwargs, "groups", 64) + _ovewrite_named_param(kwargs, "width_per_group", 4) + return _resnet(Bottleneck, [3, 4, 23, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", Wide_ResNet50_2_Weights.IMAGENET1K_V1)) +def wide_resnet50_2( + *, weights: Optional[Wide_ResNet50_2_Weights] = None, progress: bool = True, **kwargs: Any +) -> ResNet: + """Wide ResNet-50-2 model from + `Wide Residual Networks `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + + Args: + weights (:class:`~torchvision.models.Wide_ResNet50_2_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.Wide_ResNet50_2_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + .. autoclass:: torchvision.models.Wide_ResNet50_2_Weights + :members: + """ + weights = Wide_ResNet50_2_Weights.verify(weights) + + _ovewrite_named_param(kwargs, "width_per_group", 64 * 2) + return _resnet(Bottleneck, [3, 4, 6, 3], weights, progress, **kwargs) + + +@register_model() +@handle_legacy_interface(weights=("pretrained", Wide_ResNet101_2_Weights.IMAGENET1K_V1)) +def wide_resnet101_2( + *, weights: Optional[Wide_ResNet101_2_Weights] = None, progress: bool = True, **kwargs: Any +) -> ResNet: + """Wide ResNet-101-2 model from + `Wide Residual Networks `_. + + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-101 has 2048-512-2048 + channels, and in Wide ResNet-101-2 has 2048-1024-2048. + + Args: + weights (:class:`~torchvision.models.Wide_ResNet101_2_Weights`, optional): The + pretrained weights to use. See + :class:`~torchvision.models.Wide_ResNet101_2_Weights` below for + more details, and possible values. By default, no pre-trained + weights are used. + progress (bool, optional): If True, displays a progress bar of the + download to stderr. Default is True. + **kwargs: parameters passed to the ``torchvision.models.resnet.ResNet`` + base class. Please refer to the `source code + `_ + for more details about this class. + .. autoclass:: torchvision.models.Wide_ResNet101_2_Weights + :members: + """ + weights = Wide_ResNet101_2_Weights.verify(weights) + + _ovewrite_named_param(kwargs, "width_per_group", 64 * 2) + return _resnet(Bottleneck, [3, 4, 23, 3], weights, progress, **kwargs) + + +# The dictionary below is internal implementation detail and will be removed in v0.15 +from ._utils import _ModelURLs + + +model_urls = _ModelURLs( + { + "resnet18": ResNet18_Weights.IMAGENET1K_V1.url, + "resnet34": ResNet34_Weights.IMAGENET1K_V1.url, + "resnet50": ResNet50_Weights.IMAGENET1K_V1.url, + "resnet101": ResNet101_Weights.IMAGENET1K_V1.url, + "resnet152": ResNet152_Weights.IMAGENET1K_V1.url, + "resnext50_32x4d": ResNeXt50_32X4D_Weights.IMAGENET1K_V1.url, + "resnext101_32x8d": ResNeXt101_32X8D_Weights.IMAGENET1K_V1.url, + "wide_resnet50_2": Wide_ResNet50_2_Weights.IMAGENET1K_V1.url, + "wide_resnet101_2": Wide_ResNet101_2_Weights.IMAGENET1K_V1.url, + } +) \ No newline at end of file diff --git a/convs/resnet.py b/convs/resnet.py new file mode 100644 index 0000000..94c1ea4 --- /dev/null +++ b/convs/resnet.py @@ -0,0 +1,384 @@ +''' +Reference: +https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py +''' +import torch +import torch.nn as nn +try: + from torchvision.models.utils import load_state_dict_from_url +except: + from torch.hub import load_state_dict_from_url + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + __constants__ = ['downsample'] + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + __constants__ = ['downsample'] + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + + + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None,args=None): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + + assert args is not None, "you should pass args to resnet" + # if 'cifar' in args["dataset"]: + # pass + # # Do nothing + # self.conv1 = nn.Sequential(nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + # nn.BatchNorm2d(self.inplanes), nn.ReLU(inplace=True)) + # elif 'imagenet' in args["dataset"]: + # if args["init_cls"] == args["increment"]: + # self.conv1 = nn.Sequential( + # nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False), + # nn.BatchNorm2d(self.inplanes), + # nn.ReLU(inplace=True), + # nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + # ) + # else: + # self.conv1 = nn.Sequential( + # nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + # nn.BatchNorm2d(self.inplanes), + # nn.ReLU(inplace=True), + # nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + # ) + + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.out_dim = 512 * block.expansion + # self.fc = nn.Linear(512 * block.expansion, num_classes) # Removed in _forward_impl + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) # [bs, 64, 32, 32] + + # added. + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) # [bs, 64, 16, 16] + + x_1 = self.layer1(x) # [bs, 128, 32, 32] + x_2 = self.layer2(x_1) # [bs, 256, 16, 16] + x_3 = self.layer3(x_2) # [bs, 512, 8, 8] + x_4 = self.layer4(x_3) # [bs, 512, 4, 4] + + pooled = self.avgpool(x_4) # [bs, 512, 1, 1] + features = torch.flatten(pooled, 1) # [bs, 512] + # x = self.fc(x) + + return { + 'fmaps': [x_1, x_2, x_3, x_4], + 'features': features + } + + def forward(self, x): + return self._forward_impl(x) + + @property + def last_conv(self): + if hasattr(self.layer4[-1], 'conv3'): + return self.layer4[-1].conv3 + else: + return self.layer4[-1].conv2 + + +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18(pretrained=False, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) diff --git a/convs/resnet_cbam.py b/convs/resnet_cbam.py new file mode 100644 index 0000000..dbf59f6 --- /dev/null +++ b/convs/resnet_cbam.py @@ -0,0 +1,267 @@ +import torch +import torch.nn as nn +import math +import torch.utils.model_zoo as model_zoo +import torch.nn.functional as F + +__all__ = ['ResNet', 'resnet18_cbam', 'resnet34_cbam', 'resnet50_cbam', 'resnet101_cbam', + 'resnet152_cbam'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1): + "3x3 convolution with padding" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +class ChannelAttention(nn.Module): + def __init__(self, in_planes, ratio=16): + super(ChannelAttention, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.max_pool = nn.AdaptiveMaxPool2d(1) + + self.fc1 = nn.Conv2d(in_planes, in_planes // 16, 1, bias=False) + self.relu1 = nn.ReLU() + self.fc2 = nn.Conv2d(in_planes // 16, in_planes, 1, bias=False) + + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = self.fc2(self.relu1(self.fc1(self.avg_pool(x)))) + max_out = self.fc2(self.relu1(self.fc1(self.max_pool(x)))) + out = avg_out + max_out + return self.sigmoid(out) + + +class SpatialAttention(nn.Module): + def __init__(self, kernel_size=7): + super(SpatialAttention, self).__init__() + + assert kernel_size in (3, 7), 'kernel size must be 3 or 7' + padding = 3 if kernel_size == 7 else 1 + + self.conv1 = nn.Conv2d(2, 1, kernel_size, padding=padding, bias=False) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + avg_out = torch.mean(x, dim=1, keepdim=True) + max_out, _ = torch.max(x, dim=1, keepdim=True) + x = torch.cat([avg_out, max_out], dim=1) + x = self.conv1(x) + return self.sigmoid(x) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + + self.ca = ChannelAttention(planes) + self.sa = SpatialAttention() + + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + if self.downsample is not None: + residual = self.downsample(x) + out += residual + out = self.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * 4) + self.relu = nn.ReLU(inplace=True) + self.ca = ChannelAttention(planes * 4) + self.sa = SpatialAttention() + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + out = self.conv3(out) + out = self.bn3(out) + out = self.ca(out) * out + out = self.sa(out) * out + if self.downsample is not None: + residual = self.downsample(x) + out += residual + out = self.relu(out) + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=100, args=None): + self.inplanes = 64 + super(ResNet, self).__init__() + assert args is not None, "you should pass args to resnet" + if 'cifar' in args["dataset"]: + self.conv1 = nn.Sequential(nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(self.inplanes), nn.ReLU(inplace=True)) + elif 'imagenet' in args["dataset"]: + if args["init_cls"] == args["increment"]: + self.conv1 = nn.Sequential( + nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False), + nn.BatchNorm2d(self.inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + ) + else: + self.conv1 = nn.Sequential( + nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(self.inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + ) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + self.feature = nn.AvgPool2d(4, stride=1) + # self.fc = nn.Linear(512 * block.expansion, num_classes) + self.out_dim = 512 * block.expansion + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(0, math.sqrt(2. / n)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1) + m.bias.data.zero_() + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv1(x) + + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + dim = x.size()[-1] + pool = nn.AvgPool2d(dim, stride=1) + x = pool(x) + x = x.view(x.size(0), -1) + return {"features": x} + +def resnet18_cbam(pretrained=False, **kwargs): + """Constructs a ResNet-18 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet18']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model + + +def resnet34_cbam(pretrained=False, **kwargs): + """Constructs a ResNet-34 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet34']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model + + +def resnet50_cbam(pretrained=False, **kwargs): + """Constructs a ResNet-50 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet50']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model + + +def resnet101_cbam(pretrained=False, **kwargs): + """Constructs a ResNet-101 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet101']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model + + +def resnet152_cbam(pretrained=False, **kwargs): + """Constructs a ResNet-152 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) + if pretrained: + pretrained_state_dict = model_zoo.load_url(model_urls['resnet152']) + now_state_dict = model.state_dict() + now_state_dict.update(pretrained_state_dict) + model.load_state_dict(now_state_dict) + return model \ No newline at end of file diff --git a/convs/resnet_scale.py b/convs/resnet_scale.py new file mode 100644 index 0000000..60cbed4 --- /dev/null +++ b/convs/resnet_scale.py @@ -0,0 +1,410 @@ +''' +Reference: +https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py +''' +import torch +import torch.nn as nn +try: + from torchvision.models.utils import load_state_dict_from_url +except: + from torch.hub import load_state_dict_from_url + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152', 'resnext50_32x4d', 'resnext101_32x8d', + 'wide_resnet50_2', 'wide_resnet101_2'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + __constants__ = ['downsample'] + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + __constants__ = ['downsample'] + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +def init_ssf_scale_shift(dim): + scale = nn.Parameter(torch.ones(dim)) + shift = nn.Parameter(torch.zeros(dim)) + + nn.init.normal_(scale, mean=1, std=.02) + nn.init.normal_(shift, std=.02) + + return scale, shift + + +def ssf_ada(x, scale, shift): + assert scale.shape == shift.shape + if x.shape[-1] == scale.shape[0]: + return x * scale + shift + elif x.shape[1] == scale.shape[0]: + return x * scale.view(1, -1, 1, 1) + shift.view(1, -1, 1, 1) + else: + raise ValueError('the input tensor shape does not match the shape of the scale factor.') + + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None,args=None): + super(ResNet, self).__init__() + + print('Using ResNet with scale & bias') + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + + + + assert args is not None, "you should pass args to resnet" + # if 'cifar' in args["dataset"]: + # pass + # # Do nothing + # self.conv1 = nn.Sequential(nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + # nn.BatchNorm2d(self.inplanes), nn.ReLU(inplace=True)) + # elif 'imagenet' in args["dataset"]: + # if args["init_cls"] == args["increment"]: + # self.conv1 = nn.Sequential( + # nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False), + # nn.BatchNorm2d(self.inplanes), + # nn.ReLU(inplace=True), + # nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + # ) + # else: + # self.conv1 = nn.Sequential( + # nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + # nn.BatchNorm2d(self.inplanes), + # nn.ReLU(inplace=True), + # nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + # ) + + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.out_dim = 512 * block.expansion + # self.fc = nn.Linear(512 * block.expansion, num_classes) # Removed in _forward_impl + + # scale layer + self.ssf_scale_1, self.ssf_shift_1 = init_ssf_scale_shift(self.out_dim) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) # [bs, 64, 32, 32] + + # added. + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) # [bs, 64, 16, 16] + + x_1 = self.layer1(x) # [bs, 128, 32, 32] + x_2 = self.layer2(x_1) # [bs, 256, 16, 16] + x_3 = self.layer3(x_2) # [bs, 512, 8, 8] + x_4 = self.layer4(x_3) # [bs, 512, 4, 4] + + pooled = self.avgpool(x_4) # [bs, 512, 1, 1] + features = torch.flatten(pooled, 1) # [bs, 512] + # x = self.fc(x) + + features= ssf_ada(features, self.ssf_scale_1, self.ssf_shift_1) + return { + 'fmaps': [x_1, x_2, x_3, x_4], + 'features': features + } + + def forward(self, x): + return self._forward_impl(x) + + @property + def last_conv(self): + if hasattr(self.layer4[-1], 'conv3'): + return self.layer4[-1].conv3 + else: + return self.layer4[-1].conv2 + + +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18_scale(pretrained=False, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34_scale(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50_scale(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet101_scale(pretrained=False, progress=True, **kwargs): + r"""ResNet-101 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet101', Bottleneck, [3, 4, 23, 3], pretrained, progress, + **kwargs) + + +def resnet152_scale(pretrained=False, progress=True, **kwargs): + r"""ResNet-152 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet152', Bottleneck, [3, 8, 36, 3], pretrained, progress, + **kwargs) + + +def resnext50_32x4d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-50 32x4d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 4 + return _resnet('resnext50_32x4d', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def resnext101_32x8d(pretrained=False, progress=True, **kwargs): + r"""ResNeXt-101 32x8d model from + `"Aggregated Residual Transformation for Deep Neural Networks" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + return _resnet('resnext101_32x8d', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) + + +def wide_resnet50_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-50-2 model from + `"Wide Residual Networks" `_ + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet50_2', Bottleneck, [3, 4, 6, 3], + pretrained, progress, **kwargs) + + +def wide_resnet101_2(pretrained=False, progress=True, **kwargs): + r"""Wide ResNet-101-2 model from + `"Wide Residual Networks" `_ + The model is the same as ResNet except for the bottleneck number of channels + which is twice larger in every block. The number of channels in outer 1x1 + convolutions is the same, e.g. last block in ResNet-50 has 2048-512-2048 + channels, and in Wide ResNet-50-2 has 2048-1024-2048. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + kwargs['width_per_group'] = 64 * 2 + return _resnet('wide_resnet101_2', Bottleneck, [3, 4, 23, 3], + pretrained, progress, **kwargs) diff --git a/convs/ucir_cifar_resnet.py b/convs/ucir_cifar_resnet.py new file mode 100644 index 0000000..9e71b74 --- /dev/null +++ b/convs/ucir_cifar_resnet.py @@ -0,0 +1,204 @@ +''' +Reference: +https://github.com/khurramjaved96/incremental-learning/blob/autoencoders/model/resnet32.py +https://github.com/hshustc/CVPR19_Incremental_Learning/blob/master/cifar100-class-incremental/modified_resnet_cifar.py +''' +import torch +import torch.nn as nn +import torch.nn.functional as F +# from convs.modified_linear import CosineLinear + + +class DownsampleA(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleA, self).__init__() + assert stride == 2 + self.avg = nn.AvgPool2d(kernel_size=1, stride=stride) + + def forward(self, x): + x = self.avg(x) + return torch.cat((x, x.mul(0)), 1) + + +class DownsampleB(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleB, self).__init__() + self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=stride, padding=0, bias=False) + self.bn = nn.BatchNorm2d(nOut) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class DownsampleC(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleC, self).__init__() + assert stride != 1 or nIn != nOut + self.conv = nn.Conv2d(nIn, nOut, kernel_size=1, stride=stride, padding=0, bias=False) + + def forward(self, x): + x = self.conv(x) + return x + + +class DownsampleD(nn.Module): + def __init__(self, nIn, nOut, stride): + super(DownsampleD, self).__init__() + assert stride == 2 + self.conv = nn.Conv2d(nIn, nOut, kernel_size=2, stride=stride, padding=0, bias=False) + self.bn = nn.BatchNorm2d(nOut) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + return x + + +class ResNetBasicblock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, last=False): + super(ResNetBasicblock, self).__init__() + + self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn_a = nn.BatchNorm2d(planes) + + self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn_b = nn.BatchNorm2d(planes) + + self.downsample = downsample + self.last = last + + def forward(self, x): + residual = x + + basicblock = self.conv_a(x) + basicblock = self.bn_a(basicblock) + basicblock = F.relu(basicblock, inplace=True) + + basicblock = self.conv_b(basicblock) + basicblock = self.bn_b(basicblock) + + if self.downsample is not None: + residual = self.downsample(x) + + out = residual + basicblock + if not self.last: + out = F.relu(out, inplace=True) + + return out + + +class CifarResNet(nn.Module): + """ + ResNet optimized for the Cifar Dataset, as specified in + https://arxiv.org/abs/1512.03385.pdf + """ + + def __init__(self, block, depth, channels=3): + super(CifarResNet, self).__init__() + + # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model + assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110' + layer_blocks = (depth - 2) // 6 + + self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False) + self.bn_1 = nn.BatchNorm2d(16) + + self.inplanes = 16 + self.stage_1 = self._make_layer(block, 16, layer_blocks, 1) + self.stage_2 = self._make_layer(block, 32, layer_blocks, 2) + self.stage_3 = self._make_layer(block, 64, layer_blocks, 2, last_phase=True) + self.avgpool = nn.AvgPool2d(8) + self.out_dim = 64 * block.expansion + # self.fc = CosineLinear(64*block.expansion, 10) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, planes, blocks, stride=1, last_phase=False): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = DownsampleB(self.inplanes, planes * block.expansion, stride) # DownsampleA => DownsampleB + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + if last_phase: + for i in range(1, blocks-1): + layers.append(block(self.inplanes, planes)) + layers.append(block(self.inplanes, planes, last=True)) + else: + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + x = self.conv_1_3x3(x) # [bs, 16, 32, 32] + x = F.relu(self.bn_1(x), inplace=True) + + x_1 = self.stage_1(x) # [bs, 16, 32, 32] + x_2 = self.stage_2(x_1) # [bs, 32, 16, 16] + x_3 = self.stage_3(x_2) # [bs, 64, 8, 8] + + pooled = self.avgpool(x_3) # [bs, 64, 1, 1] + features = pooled.view(pooled.size(0), -1) # [bs, 64] + # out = self.fc(vector) + + return { + 'fmaps': [x_1, x_2, x_3], + 'features': features + } + + @property + def last_conv(self): + return self.stage_3[-1].conv_b + + +def resnet20mnist(): + """Constructs a ResNet-20 model for MNIST.""" + model = CifarResNet(ResNetBasicblock, 20, 1) + return model + + +def resnet32mnist(): + """Constructs a ResNet-32 model for MNIST.""" + model = CifarResNet(ResNetBasicblock, 32, 1) + return model + + +def resnet20(): + """Constructs a ResNet-20 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 20) + return model + + +def resnet32(): + """Constructs a ResNet-32 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 32) + return model + + +def resnet44(): + """Constructs a ResNet-44 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 44) + return model + + +def resnet56(): + """Constructs a ResNet-56 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 56) + return model + + +def resnet110(): + """Constructs a ResNet-110 model for CIFAR-10.""" + model = CifarResNet(ResNetBasicblock, 110) + return model diff --git a/convs/ucir_resnet.py b/convs/ucir_resnet.py new file mode 100644 index 0000000..14d4245 --- /dev/null +++ b/convs/ucir_resnet.py @@ -0,0 +1,299 @@ +''' +Reference: +https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py +''' +import torch +import torch.nn as nn +try: + from torchvision.models.utils import load_state_dict_from_url +except: + from torch.hub import load_state_dict_from_url + +__all__ = ['resnet50'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', + 'wide_resnet50_2': 'https://download.pytorch.org/models/wide_resnet50_2-95faca4d.pth', + 'wide_resnet101_2': 'https://download.pytorch.org/models/wide_resnet101_2-32ee1156.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + __constants__ = ['downsample'] + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, last=False): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + self.last = last + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + if not self.last: + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + __constants__ = ['downsample'] + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None, last=False): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.last = last + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + if not self.last: + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None, args=None): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + + assert args is not None, "you should pass args to resnet" + if 'cifar' in args["dataset"]: + self.conv1 = nn.Sequential(nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(self.inplanes), nn.ReLU(inplace=True)) + elif 'imagenet' in args["dataset"]: + if args["init_cls"] == args["increment"]: + self.conv1 = nn.Sequential( + nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False), + nn.BatchNorm2d(self.inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + ) + else: + self.conv1 = nn.Sequential( + nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False), + nn.BatchNorm2d(self.inplanes), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2, padding=1), + ) + + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2], last_phase=True) + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.out_dim = 512 * block.expansion + self.fc = nn.Linear(512 * block.expansion, num_classes) # Removed in _forward_impl + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False, last_phase=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + if last_phase: + for _ in range(1, blocks-1): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer, last=True)) + else: + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) # [bs, 64, 32, 32] + + x_1 = self.layer1(x) # [bs, 128, 32, 32] + x_2 = self.layer2(x_1) # [bs, 256, 16, 16] + x_3 = self.layer3(x_2) # [bs, 512, 8, 8] + x_4 = self.layer4(x_3) # [bs, 512, 4, 4] + + pooled = self.avgpool(x_4) # [bs, 512, 1, 1] + features = torch.flatten(pooled, 1) # [bs, 512] + # x = self.fc(x) + + return { + 'fmaps': [x_1, x_2, x_3, x_4], + 'features': features + } + + def forward(self, x): + return self._forward_impl(x) + + @property + def last_conv(self): + if hasattr(self.layer4[-1], 'conv3'): + return self.layer4[-1].conv3 + else: + return self.layer4[-1].conv2 + + +def _resnet(arch, block, layers, pretrained, progress, **kwargs): + model = ResNet(block, layers, **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + model.load_state_dict(state_dict) + return model + + +def resnet18(pretrained=False, progress=True, **kwargs): + r"""ResNet-18 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet18', BasicBlock, [2, 2, 2, 2], pretrained, progress, + **kwargs) + + +def resnet34(pretrained=False, progress=True, **kwargs): + r"""ResNet-34 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet34', BasicBlock, [3, 4, 6, 3], pretrained, progress, + **kwargs) + + +def resnet50(pretrained=False, progress=True, **kwargs): + r"""ResNet-50 model from + `"Deep Residual Learning for Image Recognition" `_ + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _resnet('resnet50', Bottleneck, [3, 4, 6, 3], pretrained, progress, + **kwargs) diff --git a/convs/vision_transformer_adapter.py b/convs/vision_transformer_adapter.py new file mode 100644 index 0000000..03204f5 --- /dev/null +++ b/convs/vision_transformer_adapter.py @@ -0,0 +1,468 @@ +# -------------------------------------------------------- +# References: +# https://github.com/jxhe/unify-parameter-efficient-tuning +# -------------------------------------------------------- + +import math +import torch +import torch.nn as nn +from timm.models.layers import DropPath +# -------------------------------------------------------- +# References: +# timm: https://github.com/rwightman/pytorch-image-models/tree/master/timm +# DeiT: https://github.com/facebookresearch/deit +# MAE: https://github.com/facebookresearch/mae +# -------------------------------------------------------- +import timm +from functools import partial +from collections import OrderedDict +import torch +import torch.nn as nn +from timm.models.vision_transformer import PatchEmbed +from timm.models.registry import register_model + +import logging +import os +from collections import OrderedDict +import torch + + + +class Adapter(nn.Module): + def __init__(self, + config=None, + d_model=None, + bottleneck=None, + dropout=0.0, + init_option="bert", + adapter_scalar="1.0", + adapter_layernorm_option="in"): + super().__init__() + self.n_embd = config.d_model if d_model is None else d_model + self.down_size = config.attn_bn if bottleneck is None else bottleneck + + #_before + self.adapter_layernorm_option = adapter_layernorm_option + + self.adapter_layer_norm_before = None + if adapter_layernorm_option == "in" or adapter_layernorm_option == "out": + self.adapter_layer_norm_before = nn.LayerNorm(self.n_embd) + + if adapter_scalar == "learnable_scalar": + self.scale = nn.Parameter(torch.ones(1)) + else: + self.scale = float(adapter_scalar) + + self.down_proj = nn.Linear(self.n_embd, self.down_size) + self.non_linear_func = nn.ReLU() + self.up_proj = nn.Linear(self.down_size, self.n_embd) + + self.dropout = dropout + if init_option == "bert": + raise NotImplementedError + elif init_option == "lora": + with torch.no_grad(): + nn.init.kaiming_uniform_(self.down_proj.weight, a=math.sqrt(5)) + nn.init.zeros_(self.up_proj.weight) + nn.init.zeros_(self.down_proj.bias) + nn.init.zeros_(self.up_proj.bias) + + def forward(self, x, add_residual=True, residual=None): + residual = x if residual is None else residual + if self.adapter_layernorm_option == 'in': + x = self.adapter_layer_norm_before(x) + + down = self.down_proj(x) + down = self.non_linear_func(down) + down = nn.functional.dropout(down, p=self.dropout, training=self.training) + up = self.up_proj(down) + + up = up * self.scale + + if self.adapter_layernorm_option == 'out': + up = self.adapter_layer_norm_before(up) + + if add_residual: + output = up + residual + else: + output = up + + return output + + + + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0.,): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.q_proj = nn.Linear(dim, dim, bias=qkv_bias) + self.v_proj = nn.Linear(dim, dim, bias=qkv_bias) + self.k_proj = nn.Linear(dim, dim, bias=qkv_bias) + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward(self, x): + B, N, C = x.shape + + q = self.q_proj(x) + k = self._shape(self.k_proj(x), -1, B).view(B * self.num_heads, -1, self.head_dim) + v = self._shape(self.v_proj(x), -1, B).view(B * self.num_heads, -1, self.head_dim) + q = self._shape(q, N, B).view(B * self.num_heads, -1, self.head_dim) + + # attn = (q @ k.transpose(-2, -1)) * self.scale + attn_weights = torch.bmm(q, k.transpose(1, 2)) * self.scale + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + attn_probs = self.attn_drop(attn_weights) + attn_output = torch.bmm(attn_probs, v) + + attn_output = attn_output.view(B, self.num_heads, N, self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(B, N, C) + + x = self.proj(attn_output) + x = self.proj_drop(x) + + return x + + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, config=None, layer_id=None): + super().__init__() + self.config = config + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + + self.fc1 = nn.Linear(dim, mlp_hidden_dim) + self.fc2 = nn.Linear(mlp_hidden_dim, dim) + self.act = act_layer() + self.mlp_drop = nn.Dropout(drop) + + if config.ffn_adapt: + self.adaptmlp = Adapter(self.config, dropout=0.1, bottleneck=config.ffn_num, + init_option=config.ffn_adapter_init_option, + adapter_scalar=config.ffn_adapter_scalar, + adapter_layernorm_option=config.ffn_adapter_layernorm_option, + ) + + def forward(self, x): + x = x + self.drop_path(self.attn(self.norm1(x))) + if self.config.ffn_adapt and self.config.ffn_option == 'parallel': + adapt_x = self.adaptmlp(x, add_residual=False) + + residual = x + x = self.mlp_drop(self.act(self.fc1(self.norm2(x)))) + x = self.drop_path(self.mlp_drop(self.fc2(x))) + + if self.config.ffn_adapt: + if self.config.ffn_option == 'sequential': + x = self.adaptmlp(x) + elif self.config.ffn_option == 'parallel': + x = x + adapt_x + else: + raise ValueError(self.config.ffn_adapt) + + x = residual + x + return x + + + + + +class VisionTransformer(nn.Module): + """ Vision Transformer with support for global average pooling + """ + def __init__(self, global_pool=False, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, + num_heads=12, mlp_ratio=4., qkv_bias=True, representation_size=None, distilled=False, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0., embed_layer=PatchEmbed, norm_layer=None, + act_layer=None, weight_init='', tuning_config=None): + super().__init__() + + + print("I'm using ViT with adapters.") + self.tuning_config = tuning_config + self.num_classes = num_classes + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 2 if distilled else 1 + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + act_layer = act_layer or nn.GELU + + self.patch_embed = embed_layer( + img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + self.dist_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if distilled else None + self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + self.blocks = nn.Sequential(*[ + Block( + dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, drop=drop_rate, + attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, + config=tuning_config, layer_id=i, + ) + for i in range(depth)]) + self.norm = norm_layer(embed_dim) + + # Representation layer + if representation_size and not distilled: + self.num_features = representation_size + self.pre_logits = nn.Sequential(OrderedDict([ + ('fc', nn.Linear(embed_dim, representation_size)), + ('act', nn.Tanh()) + ])) + else: + self.pre_logits = nn.Identity() + + # Classifier head(s) + self.head = nn.Linear(self.num_features, num_classes) if num_classes > 0 else nn.Identity() + self.head_dist = None + if distilled: + self.head_dist = nn.Linear(self.embed_dim, self.num_classes) if num_classes > 0 else nn.Identity() + + # self.init_weights(weight_init) + + ######### MAE begins ############ + self.global_pool = global_pool + if self.global_pool: + self.fc_norm = norm_layer(embed_dim) + + del self.norm # remove the original norm + + ######## Adapter begins ######### + if tuning_config.vpt_on: + assert tuning_config.vpt_num > 0, tuning_config.vpt_num + # properly registered + self.embeddings = nn.ParameterList( # batch, num_prompt, embed_dim + [nn.Parameter(torch.empty(1, self.tuning_config.vpt_num, embed_dim)) for _ in + range(depth)]) + for eee in self.embeddings: + torch.nn.init.xavier_uniform_(eee.data) + + def init_weights(self, mode=''): + raise NotImplementedError() + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token', 'dist_token'} + + def get_classifier(self): + if self.dist_token is None: + return self.head + else: + return self.head, self.head_dist + + def reset_classifier(self, num_classes, global_pool=''): + self.num_classes = num_classes + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + if self.num_tokens == 2: + self.head_dist = nn.Linear(self.embed_dim, self.num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + B = x.shape[0] + x = self.patch_embed(x) + + cls_tokens = self.cls_token.expand(B, -1, -1) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + x = x + self.pos_embed + x = self.pos_drop(x) + + for idx, blk in enumerate(self.blocks): + if self.tuning_config.vpt_on: + eee = self.embeddings[idx].expand(B, -1, -1) + x = torch.cat([eee, x], dim=1) + x = blk(x) + if self.tuning_config.vpt_on: + x = x[:, self.tuning_config.vpt_num:, :] + + if self.global_pool: + x = x[:, 1:, :].mean(dim=1) # global pool without cls token + outcome = self.fc_norm(x) + else: + x = self.norm(x) + outcome = x[:, 0] + + return outcome + + def forward(self, x): + x = self.forward_features(x,) + if self.head_dist is not None: + x, x_dist = self.head(x[0]), self.head_dist(x[1]) # x must be a tuple + if self.training and not torch.jit.is_scripting(): + # during inference, return the average of both classifier predictions + return x, x_dist + else: + return (x + x_dist) / 2 + else: + x = self.head(x) + return x + + +# def vit_base_patch16(**kwargs): +# model = VisionTransformer( +# patch_size=16, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True, +# norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) +# return model + + +# def vit_large_patch16(**kwargs): +# model = VisionTransformer( +# patch_size=16, embed_dim=1024, depth=24, num_heads=16, mlp_ratio=4, qkv_bias=True, +# norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) +# return model + + +# def vit_huge_patch14(**kwargs): +# model = VisionTransformer( +# patch_size=14, embed_dim=1280, depth=32, num_heads=16, mlp_ratio=4, qkv_bias=True, +# norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) +# return model + + +# def _create_vision_transformer(variant, pretrained=False, **kwargs): +# if kwargs.get('features_only', None): +# raise RuntimeError('features_only not implemented for Vision Transformer models.') + +# pretrained_cfg = resolve_pretrained_cfg(variant, pretrained_cfg=kwargs.pop('pretrained_cfg', None)) +# model = build_model_with_cfg( +# VisionTransformer, variant, pretrained, +# pretrained_cfg=pretrained_cfg, +# pretrained_filter_fn=checkpoint_filter_fn, +# pretrained_custom_load='npz' in pretrained_cfg['url'], +# **kwargs) +# return model + + + + +def vit_base_patch16_224_adapter(pretrained=False, **kwargs): + + model = VisionTransformer(patch_size=16, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + + # checkpoint_model = torch.load('./pretrained_models/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npz') + checkpoint_model=timm.create_model("vit_base_patch16_224", pretrained=True, num_classes=0) + state_dict = checkpoint_model.state_dict() + # modify the checkpoint state dict to match the model + # first, split qkv weight into q, k, v + for key in list(state_dict.keys()): + if 'qkv.weight' in key: + qkv_weight = state_dict.pop(key) + q_weight = qkv_weight[:768] + k_weight = qkv_weight[768:768*2] + v_weight = qkv_weight[768*2:] + state_dict[key.replace('qkv.weight', 'q_proj.weight')] = q_weight + state_dict[key.replace('qkv.weight', 'k_proj.weight')] = k_weight + state_dict[key.replace('qkv.weight', 'v_proj.weight')] = v_weight + elif 'qkv.bias' in key: + qkv_bias = state_dict.pop(key) + q_bias = qkv_bias[:768] + k_bias = qkv_bias[768:768*2] + v_bias = qkv_bias[768*2:] + state_dict[key.replace('qkv.bias', 'q_proj.bias')] = q_bias + state_dict[key.replace('qkv.bias', 'k_proj.bias')] = k_bias + state_dict[key.replace('qkv.bias', 'v_proj.bias')] = v_bias + # second, modify the mlp.fc.weight to match fc.weight + for key in list(state_dict.keys()): + if 'mlp.fc' in key: + fc_weight = state_dict.pop(key) + state_dict[key.replace('mlp.', '')] = fc_weight + + msg = model.load_state_dict(state_dict, strict=False) + print(msg) + + # s=model.state_dict() + # # print the keys in s + # for key in s.keys(): + # print(key) + # # print the keys in checkpoint_model + # for key in state_dict.keys(): + # if key in s.keys(): + # print(key, 'yes') + # else: + # print(key, 'NOOOOOOOOOOOOOOOOOOO') + + # freeze all but the adapter + for name, p in model.named_parameters(): + if name in msg.missing_keys: + p.requires_grad = True + else: + p.requires_grad = False + return model + + + +def vit_base_patch16_224_in21k_adapter(pretrained=False, **kwargs): + + model = VisionTransformer(patch_size=16, embed_dim=768, depth=12, num_heads=12, mlp_ratio=4, qkv_bias=True, + norm_layer=partial(nn.LayerNorm, eps=1e-6), **kwargs) + + # checkpoint_model = torch.load('./pretrained_models/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npz') + checkpoint_model=timm.create_model("vit_base_patch16_224_in21k", pretrained=True, num_classes=0) + state_dict = checkpoint_model.state_dict() + # modify the checkpoint state dict to match the model + # first, split qkv weight into q, k, v + for key in list(state_dict.keys()): + if 'qkv.weight' in key: + qkv_weight = state_dict.pop(key) + q_weight = qkv_weight[:768] + k_weight = qkv_weight[768:768*2] + v_weight = qkv_weight[768*2:] + state_dict[key.replace('qkv.weight', 'q_proj.weight')] = q_weight + state_dict[key.replace('qkv.weight', 'k_proj.weight')] = k_weight + state_dict[key.replace('qkv.weight', 'v_proj.weight')] = v_weight + elif 'qkv.bias' in key: + qkv_bias = state_dict.pop(key) + q_bias = qkv_bias[:768] + k_bias = qkv_bias[768:768*2] + v_bias = qkv_bias[768*2:] + state_dict[key.replace('qkv.bias', 'q_proj.bias')] = q_bias + state_dict[key.replace('qkv.bias', 'k_proj.bias')] = k_bias + state_dict[key.replace('qkv.bias', 'v_proj.bias')] = v_bias + # second, modify the mlp.fc.weight to match fc.weight + for key in list(state_dict.keys()): + if 'mlp.fc' in key: + fc_weight = state_dict.pop(key) + state_dict[key.replace('mlp.', '')] = fc_weight + + msg = model.load_state_dict(state_dict, strict=False) + print(msg) + + # s=model.state_dict() + # # print the keys in s + # for key in s.keys(): + # print(key) + # # print the keys in checkpoint_model + # for key in state_dict.keys(): + # if key in s.keys(): + # print(key, 'yes') + # else: + # print(key, 'NOOOOOOOOOOOOOOOOOOO') + + # freeze all but the adapter + for name, p in model.named_parameters(): + if name in msg.missing_keys: + p.requires_grad = True + else: + p.requires_grad = False + return model + diff --git a/convs/vision_transformer_ssf.py b/convs/vision_transformer_ssf.py new file mode 100644 index 0000000..f271de3 --- /dev/null +++ b/convs/vision_transformer_ssf.py @@ -0,0 +1,872 @@ +""" Vision Transformer (ViT) in PyTorch + +A PyTorch implement of Vision Transformers as described in: + +'An Image Is Worth 16 x 16 Words: Transformers for Image Recognition at Scale' + - https://arxiv.org/abs/2010.11929 + +`How to train your ViT? Data, Augmentation, and Regularization in Vision Transformers` + - https://arxiv.org/abs/2106.10270 + +The official jax code is released and available at https://github.com/google-research/vision_transformer + +Acknowledgments: +* The paper authors for releasing code and weights, thanks! +* I fixed my class token impl based on Phil Wang's https://github.com/lucidrains/vit-pytorch ... check it out +for some einops/einsum fun +* Simple transformer style inspired by Andrej Karpathy's https://github.com/karpathy/minGPT +* Bert reference code checks against Huggingface Transformers and Tensorflow Bert + +Hacked together by / Copyright 2020, Ross Wightman +""" +import math +import logging +from functools import partial +from collections import OrderedDict +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint + +from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD +from timm.models.helpers import build_model_with_cfg, named_apply, adapt_input_conv, resolve_pretrained_cfg, checkpoint_seq +from timm.models.layers import DropPath, trunc_normal_, lecun_normal_, _assert +from timm.models.layers.helpers import to_2tuple +from timm.models.registry import register_model + + + +# import ipdb + + +_logger = logging.getLogger(__name__) + + +def _cfg(url='', **kwargs): + return { + 'url': url, + 'num_classes': 1000, 'input_size': (3, 224, 224), 'pool_size': None, + 'crop_pct': .9, 'interpolation': 'bicubic', 'fixed_input_size': True, + 'mean': IMAGENET_INCEPTION_MEAN, 'std': IMAGENET_INCEPTION_STD, + 'first_conv': 'patch_embed.proj', 'classifier': 'head', + **kwargs + } + + +default_cfgs = { + # patch models (weights from official Google JAX impl) + 'vit_tiny_patch16_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz'), + 'vit_tiny_patch16_384': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz', + input_size=(3, 384, 384), crop_pct=1.0), + 'vit_small_patch32_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz'), + 'vit_small_patch32_384': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'S_32-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz', + input_size=(3, 384, 384), crop_pct=1.0), + 'vit_small_patch16_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz'), + 'vit_small_patch16_384': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz', + input_size=(3, 384, 384), crop_pct=1.0), + 'vit_base_patch32_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'B_32-i21k-300ep-lr_0.001-aug_medium1-wd_0.03-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_224.npz'), + 'vit_base_patch32_384': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'B_32-i21k-300ep-lr_0.001-aug_light1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.03-res_384.npz', + input_size=(3, 384, 384), crop_pct=1.0), + 'vit_base_patch16_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npz'), + 'vit_base_patch16_384': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_384.npz', + input_size=(3, 384, 384), crop_pct=1.0), + 'vit_base_patch8_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'B_8-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0--imagenet2012-steps_20k-lr_0.01-res_224.npz'), + 'vit_large_patch32_224': _cfg( + url='', # no official model weights for this combo, only for in21k + ), + 'vit_large_patch32_384': _cfg( + url='https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-vitjx/jx_vit_large_p32_384-9b920ba8.pth', + input_size=(3, 384, 384), crop_pct=1.0), + 'vit_large_patch16_224': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_224.npz'), + 'vit_large_patch16_384': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/' + 'L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1--imagenet2012-steps_20k-lr_0.01-res_384.npz', + input_size=(3, 384, 384), crop_pct=1.0), + + + + # patch models, imagenet21k (weights from official Google JAX impl) + 'vit_tiny_patch16_224_in21k': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/Ti_16-i21k-300ep-lr_0.001-aug_none-wd_0.03-do_0.0-sd_0.0.npz', + num_classes=21843), + 'vit_small_patch16_224_in21k': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/S_16-i21k-300ep-lr_0.001-aug_light1-wd_0.03-do_0.0-sd_0.0.npz', + num_classes=21843), + 'vit_base_patch16_224_in21k': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/B_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.0-sd_0.0.npz', + num_classes=21843), + 'vit_large_patch16_224_in21k': _cfg( + url='https://storage.googleapis.com/vit_models/augreg/L_16-i21k-300ep-lr_0.001-aug_medium1-wd_0.1-do_0.1-sd_0.1.npz', + num_classes=21843), + + +} + + + + +class Mlp(nn.Module): + """ MLP as used in Vision Transformer, MLP-Mixer and related networks + """ + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, bias=True, drop=0., tuning_mode='ssf'): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + bias = to_2tuple(bias) + drop_probs = to_2tuple(drop) + + self.fc1 = nn.Linear(in_features, hidden_features, bias=bias[0]) + self.act = act_layer() + self.drop1 = nn.Dropout(drop_probs[0]) + self.fc2 = nn.Linear(hidden_features, out_features, bias=bias[1]) + self.drop2 = nn.Dropout(drop_probs[1]) + + + self.tuning_mode = tuning_mode + if tuning_mode == 'ssf': + self.ssf_scale_1, self.ssf_shift_1 = init_ssf_scale_shift(hidden_features) + self.ssf_scale_2, self.ssf_shift_2 = init_ssf_scale_shift(out_features) + + + def forward(self, x): + x = self.fc1(x) + if self.tuning_mode == 'ssf': + x = ssf_ada(x, self.ssf_scale_1, self.ssf_shift_1) + + x = self.act(x) + x = self.drop1(x) + x = self.fc2(x) + if self.tuning_mode == 'ssf': + x = ssf_ada(x, self.ssf_scale_2, self.ssf_shift_2) + + x = self.drop2(x) + + return x + + + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, attn_drop=0., proj_drop=0., tuning_mode='ssf'): + super().__init__() + assert dim % num_heads == 0, 'dim should be divisible by num_heads' + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + + self.tuning_mode = tuning_mode + if tuning_mode == 'ssf': + self.ssf_scale_1, self.ssf_shift_1 = init_ssf_scale_shift(dim * 3) + self.ssf_scale_2, self.ssf_shift_2 = init_ssf_scale_shift(dim) + + + + def forward(self, x): + B, N, C = x.shape + if self.tuning_mode == 'ssf': + qkv = (ssf_ada(self.qkv(x), self.ssf_scale_1, self.ssf_shift_1)).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + else: + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + if self.tuning_mode == 'ssf': + x = ssf_ada(x, self.ssf_scale_2, self.ssf_shift_2) + x = self.proj_drop(x) + return x + + +class LayerScale(nn.Module): + def __init__(self, dim, init_values=1e-5, inplace=False): + super().__init__() + self.inplace = inplace + self.gamma = nn.Parameter(init_values * torch.ones(dim)) + + def forward(self, x): + return x.mul_(self.gamma) if self.inplace else x * self.gamma + + +class Block(nn.Module): + + def __init__( + self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., init_values=None, + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, tuning_mode='ssf'): + super().__init__() + self.dim = dim + self.norm1 = norm_layer(dim) + self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop, tuning_mode=tuning_mode) + self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity() + + self.norm2 = norm_layer(dim) + self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), act_layer=act_layer, drop=drop, tuning_mode=tuning_mode) + self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity() + self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity() + + + self.tuning_mode = tuning_mode + if tuning_mode == 'ssf': + self.ssf_scale_1, self.ssf_shift_1 = init_ssf_scale_shift(dim) + self.ssf_scale_2, self.ssf_shift_2 = init_ssf_scale_shift(dim) + + + + def forward(self, x): + if self.tuning_mode == 'ssf': + x = x + self.drop_path1(self.ls1(self.attn(ssf_ada(self.norm1(x), self.ssf_scale_1, self.ssf_shift_1)))) + x = x + self.drop_path2(self.ls2(self.mlp(ssf_ada(self.norm2(x), self.ssf_scale_2, self.ssf_shift_2)))) + else: + x = x + self.drop_path1(self.ls1(self.attn(self.norm1(x)))) + x = x + self.drop_path2(self.ls2(self.mlp(self.norm2(x)))) + return x + + +class ResPostBlock(nn.Module): + def __init__( + self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., init_values=None, + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.init_values = init_values + + self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) + self.norm1 = norm_layer(dim) + self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity() + + self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), act_layer=act_layer, drop=drop) + self.norm2 = norm_layer(dim) + self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity() + + self.init_weights() + + def init_weights(self): + # NOTE this init overrides that base model init with specific changes for the block type + if self.init_values is not None: + nn.init.constant_(self.norm1.weight, self.init_values) + nn.init.constant_(self.norm2.weight, self.init_values) + + def forward(self, x): + x = x + self.drop_path1(self.norm1(self.attn(x))) + x = x + self.drop_path2(self.norm2(self.mlp(x))) + return x + + +class ParallelBlock(nn.Module): + + def __init__( + self, dim, num_heads, num_parallel=2, mlp_ratio=4., qkv_bias=False, init_values=None, + drop=0., attn_drop=0., drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.num_parallel = num_parallel + self.attns = nn.ModuleList() + self.ffns = nn.ModuleList() + for _ in range(num_parallel): + self.attns.append(nn.Sequential(OrderedDict([ + ('norm', norm_layer(dim)), + ('attn', Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop)), + ('ls', LayerScale(dim, init_values=init_values) if init_values else nn.Identity()), + ('drop_path', DropPath(drop_path) if drop_path > 0. else nn.Identity()) + ]))) + self.ffns.append(nn.Sequential(OrderedDict([ + ('norm', norm_layer(dim)), + ('mlp', Mlp(dim, hidden_features=int(dim * mlp_ratio), act_layer=act_layer, drop=drop)), + ('ls', LayerScale(dim, init_values=init_values) if init_values else nn.Identity()), + ('drop_path', DropPath(drop_path) if drop_path > 0. else nn.Identity()) + ]))) + + def _forward_jit(self, x): + x = x + torch.stack([attn(x) for attn in self.attns]).sum(dim=0) + x = x + torch.stack([ffn(x) for ffn in self.ffns]).sum(dim=0) + return x + + @torch.jit.ignore + def _forward(self, x): + x = x + sum(attn(x) for attn in self.attns) + x = x + sum(ffn(x) for ffn in self.ffns) + return x + + def forward(self, x): + if torch.jit.is_scripting() or torch.jit.is_tracing(): + return self._forward_jit(x) + else: + return self._forward(x) + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768, norm_layer=None, flatten=True, tuning_mode='ssf'): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + self.norm_layer = norm_layer + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + self.tuning_mode = tuning_mode + if tuning_mode == 'ssf': + self.ssf_scale_1, self.ssf_shift_1 = init_ssf_scale_shift(embed_dim) + + if norm_layer: + self.ssf_scale_2, self.ssf_shift_2 = init_ssf_scale_shift(embed_dim) + + + + def forward(self, x): + B, C, H, W = x.shape + _assert(H == self.img_size[0], f"Input image height ({H}) doesn't match model ({self.img_size[0]}).") + _assert(W == self.img_size[1], f"Input image width ({W}) doesn't match model ({self.img_size[1]}).") + + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + if self.tuning_mode == 'ssf': + x = ssf_ada(x, self.ssf_scale_1, self.ssf_shift_1) + if self.norm_layer: + x = ssf_ada(self.norm(x), self.ssf_scale_2, self.ssf_shift_2) + else: + x = self.norm(x) + else: + x = self.norm(x) + return x + + + +def init_ssf_scale_shift(dim): + scale = nn.Parameter(torch.ones(dim)) + shift = nn.Parameter(torch.zeros(dim)) + + nn.init.normal_(scale, mean=1, std=.02) + nn.init.normal_(shift, std=.02) + + return scale, shift + + +def ssf_ada(x, scale, shift): + assert scale.shape == shift.shape + if x.shape[-1] == scale.shape[0]: + return x * scale + shift + elif x.shape[1] == scale.shape[0]: + return x * scale.view(1, -1, 1, 1) + shift.view(1, -1, 1, 1) + else: + raise ValueError('the input tensor shape does not match the shape of the scale factor.') + + +class VisionTransformer(nn.Module): + """ Vision Transformer + + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` + - https://arxiv.org/abs/2010.11929 + """ + + def __init__( + self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, global_pool='token', + embed_dim=768, depth=12, num_heads=12, mlp_ratio=4., qkv_bias=True, init_values=None, + class_token=True, fc_norm=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., weight_init='', + embed_layer=PatchEmbed, norm_layer=None, act_layer=None, block_fn=Block, tuning_mode='ssf'): + """ + Args: + img_size (int, tuple): input image size + patch_size (int, tuple): patch size + in_chans (int): number of input channels + num_classes (int): number of classes for classification head + global_pool (str): type of global pooling for final sequence (default: 'token') + embed_dim (int): embedding dimension + depth (int): depth of transformer + num_heads (int): number of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + init_values: (float): layer-scale init values + class_token (bool): use class token + fc_norm (Optional[bool]): pre-fc norm after pool, set if global_pool == 'avg' if None (default: None) + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + weight_init (str): weight init scheme + embed_layer (nn.Module): patch embedding layer + norm_layer: (nn.Module): normalization layer + act_layer: (nn.Module): MLP activation layer + """ + super().__init__() + assert global_pool in ('', 'avg', 'token') + assert class_token or global_pool != 'token' + + print('Using Pre-trained ViT with Scale & Shift.') + use_fc_norm = global_pool == 'avg' if fc_norm is None else fc_norm + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + act_layer = act_layer or nn.GELU + + self.num_classes = num_classes + self.global_pool = global_pool + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + self.num_tokens = 1 if class_token else 0 + self.grad_checkpointing = False + + self.patch_embed = embed_layer( + img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim, tuning_mode=tuning_mode) + num_patches = self.patch_embed.num_patches + + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) if self.num_tokens > 0 else None + self.pos_embed = nn.Parameter(torch.randn(1, num_patches + self.num_tokens, embed_dim) * .02) + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule + + self.tuning_mode = tuning_mode + tuning_mode_list = [tuning_mode] * depth + if tuning_mode == 'ssf': + self.ssf_scale_1, self.ssf_shift_1 = init_ssf_scale_shift(self.num_features) + + self.blocks = nn.Sequential(*[ + block_fn( + dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, init_values=init_values, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer, act_layer=act_layer, tuning_mode=tuning_mode_list[i]) + for i in range(depth)]) + + + self.norm = norm_layer(embed_dim) if not use_fc_norm else nn.Identity() + + # Classifier Head + self.fc_norm = norm_layer(embed_dim) if use_fc_norm else nn.Identity() + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + if weight_init != 'skip': + self.init_weights(weight_init) + + def init_weights(self, mode=''): + assert mode in ('jax', 'jax_nlhb', 'moco', '') + head_bias = -math.log(self.num_classes) if 'nlhb' in mode else 0. + trunc_normal_(self.pos_embed, std=.02) + if self.cls_token is not None: + nn.init.normal_(self.cls_token, std=1e-6) + named_apply(get_init_weights_vit(mode, head_bias), self) + + def _init_weights(self, m): + # this fn left here for compat with downstream users + init_weights_vit_timm(m) + + @torch.jit.ignore() + def load_pretrained(self, checkpoint_path, prefix=''): + _load_weights(self, checkpoint_path, prefix) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token', 'dist_token'} + + @torch.jit.ignore + def group_matcher(self, coarse=False): + return dict( + stem=r'^cls_token|pos_embed|patch_embed', # stem and embed + blocks=[(r'^blocks\.(\d+)', None), (r'^norm', (99999,))] + ) + + @torch.jit.ignore + def set_grad_checkpointing(self, enable=True): + self.grad_checkpointing = enable + + @torch.jit.ignore + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes: int, global_pool=None): + self.num_classes = num_classes + if global_pool is not None: + assert global_pool in ('', 'avg', 'token') + self.global_pool = global_pool + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + + def forward_features(self, x): + x = self.patch_embed(x) + if self.cls_token is not None: + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + x = self.pos_drop(x + self.pos_embed) + + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint_seq(self.blocks, x) + else: + x = self.blocks(x) + + x = self.norm(x) + if self.tuning_mode == 'ssf': + x = ssf_ada(x, self.ssf_scale_1, self.ssf_shift_1) + + return x + + def forward_head(self, x, pre_logits: bool = False): + if self.global_pool: + x = x[:, self.num_tokens:].mean(dim=1) if self.global_pool == 'avg' else x[:, 0] + x = self.fc_norm(x) + return x if pre_logits else self.head(x) + + def forward(self, x): + x = self.forward_features(x) + x = self.forward_head(x) + + return x + + +def init_weights_vit_timm(module: nn.Module, name: str = ''): + """ ViT weight initialization, original timm impl (for reproducibility) """ + if isinstance(module, nn.Linear): + trunc_normal_(module.weight, std=.02) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif hasattr(module, 'init_weights'): + module.init_weights() + + +def init_weights_vit_jax(module: nn.Module, name: str = '', head_bias: float = 0.): + """ ViT weight initialization, matching JAX (Flax) impl """ + if isinstance(module, nn.Linear): + if name.startswith('head'): + nn.init.zeros_(module.weight) + nn.init.constant_(module.bias, head_bias) + else: + nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.normal_(module.bias, std=1e-6) if 'mlp' in name else nn.init.zeros_(module.bias) + elif isinstance(module, nn.Conv2d): + lecun_normal_(module.weight) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif hasattr(module, 'init_weights'): + module.init_weights() + + +def init_weights_vit_moco(module: nn.Module, name: str = ''): + """ ViT weight initialization, matching moco-v3 impl minus fixed PatchEmbed """ + if isinstance(module, nn.Linear): + if 'qkv' in name: + # treat the weights of Q, K, V separately + val = math.sqrt(6. / float(module.weight.shape[0] // 3 + module.weight.shape[1])) + nn.init.uniform_(module.weight, -val, val) + else: + nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.zeros_(module.bias) + elif hasattr(module, 'init_weights'): + module.init_weights() + + +def get_init_weights_vit(mode='jax', head_bias: float = 0.): + if 'jax' in mode: + return partial(init_weights_vit_jax, head_bias=head_bias) + elif 'moco' in mode: + return init_weights_vit_moco + else: + return init_weights_vit_timm + + +@torch.no_grad() +def _load_weights(model: VisionTransformer, checkpoint_path: str, prefix: str = ''): + """ Load weights from .npz checkpoints for official Google Brain Flax implementation + """ + import numpy as np + + def _n2p(w, t=True): + if w.ndim == 4 and w.shape[0] == w.shape[1] == w.shape[2] == 1: + w = w.flatten() + if t: + if w.ndim == 4: + w = w.transpose([3, 2, 0, 1]) + elif w.ndim == 3: + w = w.transpose([2, 0, 1]) + elif w.ndim == 2: + w = w.transpose([1, 0]) + return torch.from_numpy(w) + + w = np.load(checkpoint_path) + if not prefix and 'opt/target/embedding/kernel' in w: + prefix = 'opt/target/' + + if hasattr(model.patch_embed, 'backbone'): + # hybrid + backbone = model.patch_embed.backbone + stem_only = not hasattr(backbone, 'stem') + stem = backbone if stem_only else backbone.stem + stem.conv.weight.copy_(adapt_input_conv(stem.conv.weight.shape[1], _n2p(w[f'{prefix}conv_root/kernel']))) + stem.norm.weight.copy_(_n2p(w[f'{prefix}gn_root/scale'])) + stem.norm.bias.copy_(_n2p(w[f'{prefix}gn_root/bias'])) + if not stem_only: + for i, stage in enumerate(backbone.stages): + for j, block in enumerate(stage.blocks): + bp = f'{prefix}block{i + 1}/unit{j + 1}/' + for r in range(3): + getattr(block, f'conv{r + 1}').weight.copy_(_n2p(w[f'{bp}conv{r + 1}/kernel'])) + getattr(block, f'norm{r + 1}').weight.copy_(_n2p(w[f'{bp}gn{r + 1}/scale'])) + getattr(block, f'norm{r + 1}').bias.copy_(_n2p(w[f'{bp}gn{r + 1}/bias'])) + if block.downsample is not None: + block.downsample.conv.weight.copy_(_n2p(w[f'{bp}conv_proj/kernel'])) + block.downsample.norm.weight.copy_(_n2p(w[f'{bp}gn_proj/scale'])) + block.downsample.norm.bias.copy_(_n2p(w[f'{bp}gn_proj/bias'])) + embed_conv_w = _n2p(w[f'{prefix}embedding/kernel']) + else: + embed_conv_w = adapt_input_conv( + model.patch_embed.proj.weight.shape[1], _n2p(w[f'{prefix}embedding/kernel'])) + model.patch_embed.proj.weight.copy_(embed_conv_w) + model.patch_embed.proj.bias.copy_(_n2p(w[f'{prefix}embedding/bias'])) + model.cls_token.copy_(_n2p(w[f'{prefix}cls'], t=False)) + pos_embed_w = _n2p(w[f'{prefix}Transformer/posembed_input/pos_embedding'], t=False) + if pos_embed_w.shape != model.pos_embed.shape: + pos_embed_w = resize_pos_embed( # resize pos embedding when different size from pretrained weights + pos_embed_w, model.pos_embed, getattr(model, 'num_tokens', 1), model.patch_embed.grid_size) + model.pos_embed.copy_(pos_embed_w) + model.norm.weight.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/scale'])) + model.norm.bias.copy_(_n2p(w[f'{prefix}Transformer/encoder_norm/bias'])) + if isinstance(model.head, nn.Linear) and model.head.bias.shape[0] == w[f'{prefix}head/bias'].shape[-1]: + model.head.weight.copy_(_n2p(w[f'{prefix}head/kernel'])) + model.head.bias.copy_(_n2p(w[f'{prefix}head/bias'])) + # NOTE representation layer has been removed, not used in latest 21k/1k pretrained weights + # if isinstance(getattr(model.pre_logits, 'fc', None), nn.Linear) and f'{prefix}pre_logits/bias' in w: + # model.pre_logits.fc.weight.copy_(_n2p(w[f'{prefix}pre_logits/kernel'])) + # model.pre_logits.fc.bias.copy_(_n2p(w[f'{prefix}pre_logits/bias'])) + for i, block in enumerate(model.blocks.children()): + block_prefix = f'{prefix}Transformer/encoderblock_{i}/' + mha_prefix = block_prefix + 'MultiHeadDotProductAttention_1/' + block.norm1.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/scale'])) + block.norm1.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_0/bias'])) + block.attn.qkv.weight.copy_(torch.cat([ + _n2p(w[f'{mha_prefix}{n}/kernel'], t=False).flatten(1).T for n in ('query', 'key', 'value')])) + block.attn.qkv.bias.copy_(torch.cat([ + _n2p(w[f'{mha_prefix}{n}/bias'], t=False).reshape(-1) for n in ('query', 'key', 'value')])) + block.attn.proj.weight.copy_(_n2p(w[f'{mha_prefix}out/kernel']).flatten(1)) + block.attn.proj.bias.copy_(_n2p(w[f'{mha_prefix}out/bias'])) + for r in range(2): + getattr(block.mlp, f'fc{r + 1}').weight.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/kernel'])) + getattr(block.mlp, f'fc{r + 1}').bias.copy_(_n2p(w[f'{block_prefix}MlpBlock_3/Dense_{r}/bias'])) + block.norm2.weight.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/scale'])) + block.norm2.bias.copy_(_n2p(w[f'{block_prefix}LayerNorm_2/bias'])) + + +def resize_pos_embed(posemb, posemb_new, num_tokens=1, gs_new=()): + # Rescale the grid of position embeddings when loading from state_dict. Adapted from + # https://github.com/google-research/vision_transformer/blob/00883dd691c63a6830751563748663526e811cee/vit_jax/checkpoint.py#L224 + _logger.info('Resized position embedding: %s to %s', posemb.shape, posemb_new.shape) + ntok_new = posemb_new.shape[1] + if num_tokens: + posemb_tok, posemb_grid = posemb[:, :num_tokens], posemb[0, num_tokens:] + ntok_new -= num_tokens + else: + posemb_tok, posemb_grid = posemb[:, :0], posemb[0] + gs_old = int(math.sqrt(len(posemb_grid))) + if not len(gs_new): # backwards compatibility + gs_new = [int(math.sqrt(ntok_new))] * 2 + assert len(gs_new) >= 2 + _logger.info('Position embedding grid-size from %s to %s', [gs_old, gs_old], gs_new) + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate(posemb_grid, size=gs_new, mode='bicubic', align_corners=False) + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_new[0] * gs_new[1], -1) + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + return posemb + + +def checkpoint_filter_fn(state_dict, model): + """ convert patch embedding weight from manual patchify + linear proj to conv""" + out_dict = {} + if 'model' in state_dict: + # For deit models + state_dict = state_dict['model'] + for k, v in state_dict.items(): + if 'patch_embed.proj.weight' in k and len(v.shape) < 4: + # For old models that I trained prior to conv based patchification + O, I, H, W = model.patch_embed.proj.weight.shape + v = v.reshape(O, -1, H, W) + elif k == 'pos_embed' and v.shape != model.pos_embed.shape: + # To resize pos embedding when using model at different size from pretrained weights + v = resize_pos_embed( + v, model.pos_embed, getattr(model, 'num_tokens', 1), model.patch_embed.grid_size) + elif 'pre_logits' in k: + # NOTE representation layer removed as not used in latest 21k/1k pretrained weights + continue + out_dict[k] = v + return out_dict + + +def _create_vision_transformer(variant, pretrained=False, **kwargs): + if kwargs.get('features_only', None): + raise RuntimeError('features_only not implemented for Vision Transformer models.') + + pretrained_cfg = resolve_pretrained_cfg(variant, pretrained_cfg=kwargs.pop('pretrained_cfg', None)) + model = build_model_with_cfg( + VisionTransformer, variant, pretrained, + pretrained_cfg=pretrained_cfg, + pretrained_filter_fn=checkpoint_filter_fn, + pretrained_custom_load='npz' in pretrained_cfg['url'], + **kwargs) + return model + + + +@register_model +def vit_tiny_patch16_224_ssf(pretrained=False, **kwargs): + """ ViT-Tiny (Vit-Ti/16) + """ + model_kwargs = dict(patch_size=16, embed_dim=192, depth=12, num_heads=3, **kwargs) + model = _create_vision_transformer('vit_tiny_patch16_224', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_tiny_patch16_384_ssf(pretrained=False, **kwargs): + """ ViT-Tiny (Vit-Ti/16) @ 384x384. + """ + model_kwargs = dict(patch_size=16, embed_dim=192, depth=12, num_heads=3, **kwargs) + model = _create_vision_transformer('vit_tiny_patch16_384', pretrained=pretrained, **model_kwargs) + return model + + + + +@register_model +def vit_small_patch16_224_ssf(pretrained=False, **kwargs): + """ ViT-Small (ViT-S/16) + NOTE I've replaced my previous 'small' model definition and weights with the small variant from the DeiT paper + """ + model_kwargs = dict(patch_size=16, embed_dim=384, depth=12, num_heads=6, **kwargs) + model = _create_vision_transformer('vit_small_patch16_224', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_small_patch16_384_ssf(pretrained=False, **kwargs): + """ ViT-Small (ViT-S/16) + NOTE I've replaced my previous 'small' model definition and weights with the small variant from the DeiT paper + """ + model_kwargs = dict(patch_size=16, embed_dim=384, depth=12, num_heads=6, **kwargs) + model = _create_vision_transformer('vit_small_patch16_384', pretrained=pretrained, **model_kwargs) + return model + + + + +@register_model +def vit_base_patch16_224_ssf(pretrained=False, **kwargs): + """ ViT-Base (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). + ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer. + """ + model_kwargs = dict(patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) + model = _create_vision_transformer('vit_base_patch16_224', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_base_patch16_384_ssf(pretrained=False, **kwargs): + """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). + ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer. + """ + model_kwargs = dict(patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) + model = _create_vision_transformer('vit_base_patch16_384', pretrained=pretrained, **model_kwargs) + return model + + + +@register_model +def vit_large_patch16_224_ssf(pretrained=False, **kwargs): + """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). + ImageNet-1k weights fine-tuned from in21k @ 224x224, source https://github.com/google-research/vision_transformer. + """ + model_kwargs = dict(patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) + model = _create_vision_transformer('vit_large_patch16_224', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_large_patch16_384_ssf(pretrained=False, **kwargs): + """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). + ImageNet-1k weights fine-tuned from in21k @ 384x384, source https://github.com/google-research/vision_transformer. + """ + model_kwargs = dict(patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) + model = _create_vision_transformer('vit_large_patch16_384', pretrained=pretrained, **model_kwargs) + return model + + + +@register_model +def vit_tiny_patch16_224_in21k_ssf(pretrained=False, **kwargs): + """ ViT-Tiny (Vit-Ti/16). + ImageNet-21k weights @ 224x224, source https://github.com/google-research/vision_transformer. + NOTE: this model has valid 21k classifier head and no representation (pre-logits) layer + """ + model_kwargs = dict(patch_size=16, embed_dim=192, depth=12, num_heads=3, **kwargs) + model = _create_vision_transformer('vit_tiny_patch16_224_in21k', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_small_patch16_224_in21k_ssf(pretrained=False, **kwargs): + """ ViT-Small (ViT-S/16) + ImageNet-21k weights @ 224x224, source https://github.com/google-research/vision_transformer. + NOTE: this model has valid 21k classifier head and no representation (pre-logits) layer + """ + model_kwargs = dict(patch_size=16, embed_dim=384, depth=12, num_heads=6, **kwargs) + model = _create_vision_transformer('vit_small_patch16_224_in21k', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_base_patch16_224_in21k_ssf(pretrained=False, **kwargs): + """ ViT-Base model (ViT-B/16) from original paper (https://arxiv.org/abs/2010.11929). + ImageNet-21k weights @ 224x224, source https://github.com/google-research/vision_transformer. + NOTE: this model has valid 21k classifier head and no representation (pre-logits) layer + """ + model_kwargs = dict(patch_size=16, embed_dim=768, depth=12, num_heads=12, **kwargs) + model = _create_vision_transformer('vit_base_patch16_224_in21k', pretrained=pretrained, **model_kwargs) + return model + + +@register_model +def vit_large_patch16_224_in21k_ssf(pretrained=False, **kwargs): + """ ViT-Large model (ViT-L/16) from original paper (https://arxiv.org/abs/2010.11929). + ImageNet-21k weights @ 224x224, source https://github.com/google-research/vision_transformer. + NOTE: this model has valid 21k classifier head and no representation (pre-logits) layer + """ + model_kwargs = dict(patch_size=16, embed_dim=1024, depth=24, num_heads=16, **kwargs) + model = _create_vision_transformer('vit_large_patch16_224_in21k', pretrained=pretrained, **model_kwargs) + return model + + + diff --git a/convs/vpt.py b/convs/vpt.py new file mode 100644 index 0000000..9acde98 --- /dev/null +++ b/convs/vpt.py @@ -0,0 +1,144 @@ +import timm +import torch +import torch.nn as nn +from timm.models.vision_transformer import VisionTransformer, PatchEmbed + +def build_promptmodel(modelname='vit_base_patch16_224', Prompt_Token_num=10, VPT_type="Deep"): + + # VPT_type = "Deep" / "Shallow" + edge_size=224 + patch_size=16 + num_classes=1000 if modelname == 'vit_base_patch16_224' else 21843 + basic_model = timm.create_model(modelname, pretrained=True) + model = VPT_ViT(Prompt_Token_num=Prompt_Token_num,VPT_type=VPT_type) + # model.New_CLS_head(num_classes) + + # drop head.weight and head.bias + basicmodeldict=basic_model.state_dict() + basicmodeldict.pop('head.weight') + basicmodeldict.pop('head.bias') + + model.load_state_dict(basicmodeldict, False) + + model.head = torch.nn.Identity() + + model.Freeze() + + return model + + +class VPT_ViT(VisionTransformer): + def __init__(self, img_size=224, patch_size=16, in_chans=3, num_classes=1000, embed_dim=768, depth=12, + num_heads=12, mlp_ratio=4., qkv_bias=True, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., + embed_layer=PatchEmbed, norm_layer=None, act_layer=None, Prompt_Token_num=1, + VPT_type="Shallow", basic_state_dict=None): + + # Recreate ViT + super().__init__(img_size=img_size, patch_size=patch_size, in_chans=in_chans, num_classes=num_classes, + embed_dim=embed_dim, depth=depth, num_heads=num_heads, mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, drop_rate=drop_rate, attn_drop_rate=attn_drop_rate, + drop_path_rate=drop_path_rate, embed_layer=embed_layer, + norm_layer=norm_layer, act_layer=act_layer) + + print('Using VPT model') + # load basic state_dict + if basic_state_dict is not None: + self.load_state_dict(basic_state_dict, False) + + self.VPT_type = VPT_type + if VPT_type == "Deep": + self.Prompt_Tokens = nn.Parameter(torch.zeros(depth, Prompt_Token_num, embed_dim)) + else: # "Shallow" + self.Prompt_Tokens = nn.Parameter(torch.zeros(1, Prompt_Token_num, embed_dim)) + + def New_CLS_head(self, new_classes=15): + self.head = nn.Linear(self.embed_dim, new_classes) + + def Freeze(self): + for param in self.parameters(): + param.requires_grad = False + + self.Prompt_Tokens.requires_grad = True + try: + for param in self.head.parameters(): + param.requires_grad = True + except: + pass + + def UnFreeze(self): + for param in self.parameters(): + param.requires_grad = True + + def obtain_prompt(self): + prompt_state_dict = {'head': self.head.state_dict(), + 'Prompt_Tokens': self.Prompt_Tokens} + # print(prompt_state_dict) + return prompt_state_dict + + def load_prompt(self, prompt_state_dict): + try: + self.head.load_state_dict(prompt_state_dict['head'], False) + except: + print('head not match, so skip head') + else: + print('prompt head match') + + if self.Prompt_Tokens.shape == prompt_state_dict['Prompt_Tokens'].shape: + + # device check + Prompt_Tokens = nn.Parameter(prompt_state_dict['Prompt_Tokens'].cpu()) + Prompt_Tokens.to(torch.device(self.Prompt_Tokens.device)) + + self.Prompt_Tokens = Prompt_Tokens + + else: + print('\n !!! cannot load prompt') + print('shape of model req prompt', self.Prompt_Tokens.shape) + print('shape of model given prompt', prompt_state_dict['Prompt_Tokens'].shape) + print('') + + def forward_features(self, x): + x = self.patch_embed(x) + # print(x.shape,self.pos_embed.shape) + cls_token = self.cls_token.expand(x.shape[0], -1, -1) + + # concatenate CLS token + x = torch.cat((cls_token, x), dim=1) + x = self.pos_drop(x + self.pos_embed) + + if self.VPT_type == "Deep": + + Prompt_Token_num = self.Prompt_Tokens.shape[1] + + for i in range(len(self.blocks)): + # concatenate Prompt_Tokens + Prompt_Tokens = self.Prompt_Tokens[i].unsqueeze(0) + # firstly concatenate + x = torch.cat((x, Prompt_Tokens.expand(x.shape[0], -1, -1)), dim=1) + num_tokens = x.shape[1] + # lastly remove, a genius trick + x = self.blocks[i](x)[:, :num_tokens - Prompt_Token_num] + + else: # self.VPT_type == "Shallow" + Prompt_Token_num = self.Prompt_Tokens.shape[1] + + # concatenate Prompt_Tokens + Prompt_Tokens = self.Prompt_Tokens.expand(x.shape[0], -1, -1) + x = torch.cat((x, Prompt_Tokens), dim=1) + num_tokens = x.shape[1] + # Sequntially procees + x = self.blocks(x)[:, :num_tokens - Prompt_Token_num] + + x = self.norm(x) + return x + + def forward(self, x): + + x = self.forward_features(x) + + # use cls token for cls head + # x = self.pre_logits(x[:, 0, :]) + x=x[:, 0, :] + + # x = self.head(x) + return x \ No newline at end of file diff --git a/exps/adam_adapter.json b/exps/adam_adapter.json new file mode 100644 index 0000000..a385272 --- /dev/null +++ b/exps/adam_adapter.json @@ -0,0 +1,25 @@ +{ + "prefix": " ", + "dataset": "cifar224", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_adapter", + "convnet_type": "pretrained_vit_b16_224_in21k_adapter", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "ffn_num":64, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":5 +} + diff --git a/exps/adam_adapter_cub.json b/exps/adam_adapter_cub.json new file mode 100644 index 0000000..7364019 --- /dev/null +++ b/exps/adam_adapter_cub.json @@ -0,0 +1,25 @@ +{ + "prefix": " ", + "dataset": "cub", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_adapter", + "convnet_type": "pretrained_vit_b16_224_adapter", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "ffn_num":64, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":5 +} + diff --git a/exps/adam_ssf.json b/exps/adam_ssf.json new file mode 100644 index 0000000..98f76c8 --- /dev/null +++ b/exps/adam_ssf.json @@ -0,0 +1,25 @@ +{ + "prefix": " ", + "dataset": "cifar224", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_ssf", + "convnet_type": "pretrained_vit_b16_224_in21k_ssf", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "ffn_num":64, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":5 +} + diff --git a/exps/adam_ssf_cub.json b/exps/adam_ssf_cub.json new file mode 100644 index 0000000..ba44930 --- /dev/null +++ b/exps/adam_ssf_cub.json @@ -0,0 +1,25 @@ +{ + "prefix": " ", + "dataset": "cub", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_ssf", + "convnet_type": "pretrained_vit_b16_224_ssf", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "ffn_num":64, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":5 +} + diff --git a/exps/adam_vpt_deep.json b/exps/adam_vpt_deep.json new file mode 100644 index 0000000..dce8e85 --- /dev/null +++ b/exps/adam_vpt_deep.json @@ -0,0 +1,24 @@ +{ + "prefix": " ", + "dataset": "cifar224", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_vpt", + "convnet_type": "pretrained_vit_b16_224_in21k_vpt", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "optimizer":"sgd", + "vpt_type":"deep", + "prompt_token_num":5 +} + diff --git a/exps/adam_vpt_deep_cub.json b/exps/adam_vpt_deep_cub.json new file mode 100644 index 0000000..dfca22e --- /dev/null +++ b/exps/adam_vpt_deep_cub.json @@ -0,0 +1,24 @@ +{ + "prefix": " ", + "dataset": "cub", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_vpt", + "convnet_type": "pretrained_vit_b16_224_vpt", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "optimizer":"sgd", + "vpt_type":"deep", + "prompt_token_num":5 +} + diff --git a/exps/adam_vpt_shallow.json b/exps/adam_vpt_shallow.json new file mode 100644 index 0000000..2f17600 --- /dev/null +++ b/exps/adam_vpt_shallow.json @@ -0,0 +1,24 @@ +{ + "prefix": " ", + "dataset": "cifar224", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_vpt", + "convnet_type": "pretrained_vit_b16_224_in21k_vpt", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":50, + "init_lr":0.01, + "batch_size":24, + "weight_decay":0.0005, + "min_lr":0, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":5 +} + diff --git a/exps/adam_vpt_shallow_cub.json b/exps/adam_vpt_shallow_cub.json new file mode 100644 index 0000000..71098ee --- /dev/null +++ b/exps/adam_vpt_shallow_cub.json @@ -0,0 +1,24 @@ +{ + "prefix": " ", + "dataset": "cub", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "adam_vpt", + "convnet_type": "pretrained_vit_b16_224_vpt", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":20, + "init_lr":0.01, + "batch_size":48, + "weight_decay":0.0005, + "min_lr":0, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":5 +} + diff --git a/exps/simplecil.json b/exps/simplecil.json new file mode 100644 index 0000000..12024e2 --- /dev/null +++ b/exps/simplecil.json @@ -0,0 +1,24 @@ +{ + "prefix": " ", + "dataset": "cifar224", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "simplecil", + "convnet_type": "pretrained_vit_b16_224_in21k", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":0, + "init_lr":0.01, + "batch_size":256, + "weight_decay":0.05, + "min_lr":1e-8, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":3 +} + diff --git a/exps/simplecil_cub.json b/exps/simplecil_cub.json new file mode 100644 index 0000000..49a0351 --- /dev/null +++ b/exps/simplecil_cub.json @@ -0,0 +1,24 @@ +{ + "prefix": " ", + "dataset": "cub", + "memory_size": 0, + "memory_per_class": 0, + "fixed_memory": false, + "shuffle": true, + "init_cls": 10, + "increment": 10, + "model_name": "simplecil", + "convnet_type": "pretrained_vit_b16_224", + "device": ["0"], + "seed": [1993], + + "tuned_epoch":0, + "init_lr":0.01, + "batch_size":256, + "weight_decay":0.05, + "min_lr":1e-8, + "optimizer":"sgd", + "vpt_type":"shallow", + "prompt_token_num":3 +} + diff --git a/imgs/adam.png b/imgs/adam.png new file mode 100644 index 0000000000000000000000000000000000000000..866a2cd8c282011ec878e9ba8405f73534562bb7 GIT binary patch literal 130335 zcmYhi19V*98#X#|8mF<1v2EM78mqCbMw7-?W82BZww>Jh{lEL&b=R6%XJ*ft zbI#uT_4B-uit>`k2>1v9003F)rVoLTVKYzHSYkxVZTB^>uA+Eww_f?PVAER=;wmr>Fn^{kyeg=;QPHa4}|5 zTvD>p4gU%Y zo6OTLdDKLoG z-zX_LUi`0j26A(Abs3lD=IZr$!@$#I%gD$uHN6MlfFR@T{e7uQ3*+n8n-iZBMVg0; z4FW8zh532B+>FM?#`l-|wOSL&_O~Z^F+TFNF&7sX8ENU+xw*%)mAT=Q%gb)p<5}?Z z(w2faQy1x~3z2KWJQ;duaC1g|2e=lFRHds$N>Xf`VkOGF2RtMj_;5p|%UJPLsgefG zXJeR0vvd<`|NOCBs8B<;fDZ`xw3;i0fra(<@o6?0LYs;eDd2WLTW+wLTWzO+7t={h zO-)Tt_q^Kb4T}y;NJud7yI-h8LXVG*e%*;?!hh)9+zbG9ctJowoG1D`pFDrOUKSM= zf@k>odf6S92Z;aC@eRDT?(Xi!#>PczP2i=I%i`=TF6)h!wqXD&S(?&S>z1f({Q_3E zg?gTox}juEn$5K`T{3b9$*6KTH)=-P=!5joYKUxk(}vj1WzvSOB)Sx7F0t%+1ZuH*lZ}W34#d_SaioYI;Jn_4GQ8`oh9KZo#`29v0Sh zzo^6Xx=0j$a$*f$-8v!+j9Xe66v@w#H0)FJ!F-$#O*|8l8Le|ufVR}BGnF-H_NQ627L>X^Cm_jF)=p& z{l_0lkp?pa4pm-SdbQSMsKe`awVe;XUq-kI`&y>FTAe{5sN*gE)y}fpg^?iuo}r4!NRZCH>7%trUc+?I&j9`Xe|Qu z+6{Yy(+ckJVV*j^1*i^{6^vy<|L>OUQFhmlkMXgwc72bP7XGhQD(WfHHwi!qeyqrf zQrX1Mo#V&Gs3>WzFS82~>V?xk$G$z+_`v#$>UFxGub!_pQ|P^$=PS~1+AbSDJ%I(G z*~9`ZA}zMF^Zns`&EsrY*k8BH_Zigbv-ZY_?AF-b9pLW%0G2u40$|tyF)ab7ts;#g z**9pB_>7E|Qn@VfhDiF$N=t(c2t6I0b>*p0$PzyF75D+bQ?(cv9}er=654ECxKYl~ zUT=CAc7xA<+?DfS47U6;{zZAe>GZ({_? zH?yru>h~^wkfpS?_UaHxfGliybHQa><1F|l&Q{x}7w{qtIE1j9R4(~5f3R8&n;R<&Wz3t){|Yj(KIp0xg2pqQSK z@dXfe-=3b13;5gAbfFr$juxSQ!%9qAjR1d{+=ic~yLw`AaIRV@A=|XL(SUQBCmBXTQ*Ne$6F3e)iv1c%m_MM_B&ORIvot(M%W(GAyo1d zU!A1%FTIE>ML+#)(fgo6;m@8*3O*l;YZ~Rd*gx&qAB6zh5Prm|G z-auXGO_+()BrAgvp|u1}*)J|q?-R6`2%$G0f(3O>HMT;ZkD!&Q@a0v;NE|4D>LAl~ zQA5Oe1PO}$s(bw|&H}&7f$Ut)2-pHtR93E)2*LMPnu4u^%@UAu4H5gh=e1FdA`RFE zS~&8$IqM+=!zAN+S8-pr;qQJc{@M&gyG zw~+j6f>EB-8o%QbY;^icZKf0~{Bkk1Tjkfepx*D@{7D)~dWw#QambnzLci6(pIzP^ zlf#kA+;&57{(e6lKQN@HXL?PQ!2Gxxgb9DwsaK^#XW(#j{dA~ zt(bt;G?ksmR&fBNrVdRPN3vY%efnR_2Qjg#2}YCB%tG2J!OPsiTH~x;CnkvoUC`dn zB?j0c@GA%=g%4IVy(Bu_@H=WBWNBBiHkdHDS-(498^bIou5tm418bbmDG;OJD@JEjaRF50ENd3U5p%Evt! z&@Zl^pPyr3V2nK6?2qm4?C4*``@hhqh^ej7dMwYXVun;>5&ck6`fY%M!ssqB<&Unb zuIo5O<>*MVi;beV|51B5waVxaf8IxPjz@ep=~E!+;cAVARlO1qRzz#B75XEEA!w!= zXg(Dr029+FUE__T-M;eV?(nF~rDhXZS9emoN{BFk!t4&^`#A-Gj)O%sr{ZD@&HmoCp9~dK zWR2}M{B@@vVlZlJP1gAk3}9=uy`9}7r*547F4QIDoVn-h-$QD$uI_vqB4Bp5O_LLM?} z0@PrIu>w|=dPN%F3mFbyL;JO+>bV0A3SN%cy*%|xoFE#C_PsC4$Szr4^4)Yq++@?w zSPW${Y>$3*yj;H8%y}JTj97z=c=rS16x}Outnk645p8FPh_)Zk&KBN8f#O$#rg~EA zb)*mx#sU0G%m=gNo+hkH9%S0g{tiurrn|{b)%-WjB<79w6Ytf{Al787hL1H_Rr#gLmKi z`73vf7xDcEZxBYCwO0n#Bkiq0bzukF64X#z7tspV-6`uhHEWnW0@L~;+d6Qgp zTINfng3o=pW|#(aP=IHLnNBXt0ks>C0ma}oTdrUCp{kgbQ~6?Q*=AA4YHFwK{W$|I zJhKxupge^kJ~K(x)tzwPnl>!o&88g#G9@6Z)V7|fAEHImP^5$y0ZR7n379_}9B<(c zDT$?^dt+cB?_qv}1b8jpsA@V_P8l*gGuX{8vm@@xaX4R`M}&v}I=N#3BK}kTmMkMs zC)R6`ct49l(jd+j97DQ(KzU&{5}`L}Ba1w$F3>wL+Zsu%_hm(c~e*gdik~N#*;*`r}B9s6L;< zF}16%_vJM8r7r&4V^aok7~7M5ZxZ><&DYZfL8>FRs+#kcU&$>@C{!^z80%Y?y)w76 z9TE~PC!5yVUd3xI=vcb}Qjr-3pBFRa=;-K&Q+YyOH>S$UvtZ9|Pz)GVDQ?u%TF4Mb z0R(<~?WWLD#~_AT+%eN=7W=U8@YZvln60Fz5M)5@814gD1N}Si6gC&aZLsMs@?OZa zVE-p&H^P_C+-l#32IpY?T0L}cKkMQvAGz>T)Pb{f=?&N$PTIP2QYuvhDrx9>n)Vn+ zhH6ak*L9UP-EWI~Nd_07_m|mbQS%hLZY!if8OC#Xp#!h8y?rx;?SyI(+CVJYu@E@_ zb5)=Bp9u7xA2X2*<0crRMri{p^%g>}Fpf_D zNJjd|`<@gJ?GT+Pt-MmW8_p6^Q%x-*@Zslie$d4T`CpU{bGgoja=`9*sy5Mce-ulU=E+5XIWJhGV7c9cX=Q}7`>C>jW23XP zeVX&>^>r$XtH$L*4fNIXy7he_BsFf5ALNU=m~z^3>}6hjHn{&J%KS_47R1A#cpv=T=0s!|ii$cP`l6 z8@C~?fhJziXtwCBQJr|i?mUw(YeK)(netgRGAb&Q-D=jQ=79fL;^$rarLU6SJD>G!<&%hZ2|HDN2e(Ye@=go~32>G959!G2&2cJIB<~AIo#B1C z*xP}J;Q@JfWTsVFO>!#LSA2^j6c0-s1J?6cZ=&Vfb+bZq-M>xu!@$x)Qj)vUwwNce z(5Au$5h%juFZ3X%-oSBV08`P9Uii%`%6*glj81g)$6BL$>s$OVEXp|-Muv6_1`)67 z@^4VR$xz&(SH4WrWA5dLlw!TS2~>+JZnwh-JVK}l3vrv&9-o_7O;|l4W1ClC!>atI zv+3F)+zDG<@xOp(+2+P~e*G9Iu+RK!3P0dJoSLP~Iu?B>$Zo6xS!|1My`rnfuIqc- zzX)xaJRiHl?Sn{sxmKM-t=A`dh8yoa+t81%pOL0M>-&?yeESS6TNW5D$<@!jXz*e_ zM&{<6cB|}les53Lhb_TIWX{L4C3Y86-|6V^`8^#~Rp(EEm7CuZ6BA3pQ7JfM<6IAT z;wWR?4sY)xe|y?9eM$b#%(_*1UHqvV5qcev)2>Axkt+av9_H4~ycsJTC?~Tig*i6! zHT(DA_A1uo=ck)l6b&B@jYxF@X%Bv|hm0Ol;;^E^|v-)!VW8LNUVe zn62Ic0H=^l`W@-ETpJ`^_xvL^8(N~h1cjsR51xOT{!@RfkknUhla{^N$cbz9Xf8Ym zz!m?d0mw#VzCV56e%pX4GLxq#G-Ml9MPVSMMB}a6fS%0w==c>sXHPHF`rZueu&_?l`#%_h5>Kpg7# zjD1V9dyUZnUwAL-3XR)dkmpFNTe#ueV7xYNh665ZWBQQoSF3dvaH!*X9W?}91>tF;B;%!y|v{= z+5hi}k|Nwb_z2O03zlo)p*U~F-MXmQ9$4M35z^zKM2B7vJpw<>3br`k%ReJQmA9h4}?y>Gg5VFzkHU4k0t0_vad&jt;uT81NdY4~($AqW8@ptZc6aL$Tz0B9r`beIn_1xd&8VaVS-&$>Pu&}Vu(XV~U zzA7XxBE+8Y0H4gsm?mg{NeZ{N>@6;&D~~x!s>r6M;pV9k^H&H)Wv#i5uI@bwwt0X2 zt1TJpVi$09jH<6{+0x&NIQm4U2WHmq^yf%{ASNOYsZKgENL7|N=&P~ttt=FNOp8)! z>x2a(on*ouR&Bpt$Gdt=HKV9;``0r{=T240c6I z&-P4Ot}fr7ivl`NdFy?|!kX(ROp5BQ3f<-iI5dcmgMPR&kB^U&Q&5aRVw9JcPbGi1 zqdDYr0CjMpqJp~o_{uB&d|ECH1W1lgP(<3Uw$~fmRW9~^H|`@3T=CL+8m&ux#H7gM zvTK!b5|X#wY|LKLl{)AII2}lUGfbsk^etj zS?KrQ;2#8pd{qd0o;sTnv(`QyFaCv)0|GbWK-nh*gdMeF&HNvT@dc-3FaJ8tF{Vps(87)z;tRrX64DDc z(}WC0BdG^6J6l;I$W2MYJyJFEZ4Lq?Nrr+8IN-F}2>$3uG#6dEi(J&7pwgC+sbl*s zN)Y5}hb!dIztZA%y2zGe8FBpu1;u`i5v-JWA`|iUaT0B9yVGmcV;>8*BM;zI-@sSp zZVAurI&$JxUBSIfnFbuctO}6JL;nD(EQK8AJsCgzn@wN~K4bRl(dd&B5_11|x@ze1 zwz1OBj98CUp^f2D;`3Vop+Uee?^Aa@qYKKp zn513kyu&N5Td!Fp51V)b(m&ujN|{B^wyg&sVh;JvD3$^&?yv52ygGZaB`ekq4*4=+ zeXsPelRutE_3|DDsbGT`4=3NHIV8FJb~ zK`}7c6k{4N`Rf^Z9mkV!$LR!=!=~As0vWE^JUj?*lN`5ZV@+&NPiOg79htZ8aRrth zh8xHx|9ec_!PL^X+4x%#uJb7TxsCaa8SsnU_PvD5#o$zeo$rJfGL6ZAiU3@xp7Lg7 zOaYfYcq?i;qy4LUsX#(0 zksNVuG28bt`nt!s@c)TA<6BFObWO?i8wNr107p8nPE*}_DO1j|5hQ?Ay3>Z^wH?M` zp{8_cYPeX_j#pSHF5hW-We%Lc)Oj!Fg|^1(u!;gQUi~|Mhj3+CZd6d@nN0YBi~Xi1 zw+9nb&4?P&5ZgJRfQV~Y23eri@{vy5V@%QgCyzoz>l+Qa<8((6M?8-8kSK*`3WV?Xp4@0u8atzoUmK{}46ZxqP&Wn0PwPbD|cSNeDQh z3KjI?{LiMtcB3}wPlkg3NGuiVa1a4=pi0y~ex2N%M;NVDTL6=z5A&9_*}dlG55A@& zZ5tK8grfD7{7Vv|RMvA{A$N2N)meMQn@B&_sc5bc;|f^y7oVOIfeikb%wAufn#%f~ zgEO9nZFU%JIKF2DR_h;T|3EoTWG)`RDd&clo#9=ZEh3ar{Ms?%pS)g7VA@y?d%gty zo@wy87yrlRZ{TsVOA=tE*qM|(t?yz-xzuDgOwwuo)l2wY+w<1t+cQ_(Cc%tyuQD{` zOf56K39D0E3a+v5zkmN$J7kp0^|Ro=%dNfbMBofpG8xS_TT*BZyX6Zp${LOtIe&KkuD36{ruSaIt z1{cFXAW${`%#Uaf5puX|<(Zv{UqeNN&|=j6ikIxHtT%WHYB+hzgm&xQ*V)46`Wy(T zP)fRD3h|ANmoYBoa`X_~57#Hr4?+5Gl8Z_z@~-b^qM%q&T2gJH`2drT_T7-DbpJ?g zlJyeofzQ9ScYOq}Wk>gHt>2J_$k${2nl(^#ZuFC4YczpD6r!2*`r4OzTtx3ymVNbi zUwIh|Uy}o}%{x>aW|`2~{VR|U-$#1s$kttVb)qx_BM~E zX*4tlB|Ce0s`*<4a$`R=YF!@yfJ06AM$YIM>%Wqj9RZ_%(Iior1(*8fkoWgGl`#D@ z@CzC8<7dwk*d8GO#LE2n43?d3fekcSkD5R6mwb#>(*eqy)w*u|;{`#e9sjefOZ>Z; z?PMHGTwIS@lXG)7u$DjfNcE{K&tb1m$aqbGjk=7~1f``@JJh|G=LFlfBp-bta-y-c zmrDu`E!-b2I(`Q!G>nCO3lF!REPWh~rtqHfYnkacW3H$)6O)1YN9kuOHviV207z?~ z_Yl%=-k*`1OGr3Zb{m~-u;L`L^3NHvm5Vt0anX)oS0k#P9Tdp|CwNT%=aJynCpP^A zG1w~0BvHj*i+Q~|;~CKfQKva0NAH`R?bd3W&lhMm3;DCTJBD7D)r0zb0chP82k6jq zVkacPj|TrgEDP>Zz9AKV8fk^>1lWDwfh+zBj6zXh^w8t$*RR8a1ob9tkTcp>Zx{w2 zmi$#pFDZi1RU;odU>OJpRul10N~KakM(DJ^DCln$(Oj6IZ&>6Vz0ZWt--(i=*&$pT z(0hB_0J;P@Rzs`!qoNsYmo=N720hsB#ujG_&vb9jgD+l12mWeJe77)}48nkv!f7M+ z-?HciA_b58-FB_xXOYha)L@rn$89wG8Avk#@TwPS0&E4ho@0Wh;^z#ww4Du`w!mcA{I?bxpb70DOT{VJT0`YW6?Tz* zpT=x?uv@z`@$n_FmE*_`ag!a8R{D11HTd*IQtIufLvS4mhDIU5;Ng@vf>zhs=4SVV zO$Y@V5gvHulYPVp0cZ8=m~~vQNIZG!gG_ilt}0+4oiN%_=t2B0)Q12LS|0uv(!(MK z!NFr=>fmQC1lbJBW;Oin79?Q#j`#xVB=@!1mvcb_AfmwppV1a@Z=o`>aOV>#`4vJ8 z{bGn5EI^lu5$Smw#1qx#{uCn6S*m?pH7~9=d|koj80lK}mrSd^dza+27p17>Ch);& z@406_2S;%cj@<(QKydz%ni7=n%ao^1AZ_R?Xo)H3^wq$?Gh2w%c;ZP8UGcJr>Jiy) zBEeA8B|hMR!viteeA8VUF_bjI967YH8hKKfG#83$VmNB81@HymEh0_2z6iL1ju2Y> zN=Zd^U}lQig`09MBY`4->AVff+`^$l4UvlVilV^7 zt;dEwiJn}6BGP$ksUdVB8yW!t0WmQOATlztdL>}&szJBI^LlLAe`GqZFAKYyZ!g14 zXvfTDXXhTZkV_OUhjA0W|5}DWj5u@k-=KE2xE>*aK&AVG0h_?8mGNK+?PjP0*XxDX z(mL|ooy+)4%|tu=fBXP_j(AG#Zg||K$D*dC1wap~b_3%X$Ls<;bRlg{3mq2Zj1%c& zzJDBp5Pt7pM4i@(3z6WK97xrHH1{FybIUSK&O6@V{}=-YGfR+u4|?qX@eUwJyYqaF zp3i=!25^rB+~0JwUeM=?+D2fwi`BpNX?@I$B@HMDbwGITzl-vz|8EG<+@Q*Gpfk(8 z1^98XUG)d?F{|}g)85XMp2T}9sn2+6w_DOBI#K%2*J=YKj7^xl4$Z%7h|@ruizBx$ z|4|ku3dg1RvsmnIA1^&5_raAH=$)0GN(}dlb<&3D)mF(2*D6+X?zb$pH-6S663j+U z6Y&5*rM-If6!TDt7mkp&TN2KTkePA}zxj8$SaLvtVoW8UW*F#QBbjPe4v+qAv&id+Op0A^sPD3^{6-@~nd9M-_| zz9E?-XIazBN8#?~8}{WQ{qhVvfO&{)GroEgQR~|9(*QN|Y~|_~%IT=I7&-H)-IxvV z@y^}^ht?zhdgjv#oL5c#J}Lw_(8ru}yH&w8?_8uF^&NL^WG3xLpg`w>v5vv3iOp~i zNgj13Kn2-#{DeCaeyY@zMCU&i zF)_E}jh*G*8n=8?&}8szPeVOiVUMRdwYN5VB#j>0d{{}H6hBNd(+ehUU&Y{+MR*1E z>7e)g(m;6p%=un-RcQe8dsAU-5;DG~qx!e*>6W4W#KmfmC+F`5SS6|B6q%)|+^A?i zzk>3GwV`O~qLTHFh55am)q(l(Vl4~+68_C%Nwh{L$NkB!*Q>gDJ=JEHpLKI6A&w zRZ-E_(a^C?bAG< z-eQXQ=F>o;zzP~N(C*L_CZ(3*GyF1ldIH*00IJQyXRmIs1g1Fvh0`f%ZlkJ(vu{V!JK2fhtl3hKwmpJ@oCEU#;ir0d#PHSaN^K}xTG){c#} zCLs-^=ZAi9O<7a2@i#kA085!m{grDyLNP|I(;+PC&lw;2DtJJ?q3N`V5?Os;%>=AU z9raHW3XmahV}pEGLYvQPt~pj?Py_5s0r~`jAr!;0HNa9z1L}v22$440!Z)Wfrr!bL z4kR_Ld4Bw+Px)b8PGXW0<*NE`kF%2w0n!de#R5jYrDf&~u8SD5fa#y|T*V3S4khJA z)S@6_{EyytOEhw{2$X0=n)MDZHh?XtgNK#%E0{3B1mV2|(O2YmX(!FYlC|!J>$^mf z-JQ<|ca>S(7f@Bd7ra%(%x<&^3#gG z%jfk`x?bx3dO3{Z@ne_E@k65$h{bfn=F?GJ-*k78LL?c*PvuK%VD+7;rPcUBW3zZZ zP<;AD{6o)Mj+^_h4F;U+7?HZ_=2Vc8OPjNDeE0;9^A`Z*=$PY{sN25_5{ZvQ*y(`B zLt8-1CD*fX%*67QP!DOr_0IckcV-UT-pmaI?rEfNf-qF5PxwA1PG^9EOFK=#K3#lRtYi%K_kIlMN zJFo<5^Izl`i@ikl@&hxLXJ*SoBR+G*)G{p@vfObwUVW+fM6_nQFGb^VWCTj6ayh>X zo&DPpEqI(Gr57aF85;HGkT zH(Mw@@3qcZ1z}#zpsU{-3}zZgfXz-@xjExf;oLqdNd7Jme)A#eW;X0z-eW2`9hG3c zaYc?q`|N6Cty|-!s&X!V?`}1~_JY|@fK4~so1x2RdD}{l;9Jn{?9FUdnAS?v$tw@X z6#8h~kB(ATiJJQi6({g=;0eAwc`Iab`c3!rf;znoI=wj@r+}2FmwS4HTq!g|X5zR^QusF*_V92M!}odF zZ#C!QOpGfF0rjq`N|~cTlQR0IQ$iUUKl=d(=NWspR@q(&NMD`m=5d$Qv}D%O%$&7U zCC)M)G{Z`hiIS#AqKd!Bb$|(dy7L*E*oc;xnfZe&Q8Lo-HL`*I5cs#{@3)Wf&5Yv= zMIm^L0Q^=*Elg(v3bcs0kX|#nV8W${v@e5hXU{;29|lRI1oQ!L(`D>%F@Mq~ysrw^ zd_D>lR`>hiv`(%~#*_`05kz_H#hv_l=JSGaHbA#aA-4g3t9Ew{4zO@T1Dr8z)Cp91 z-M(L{sE&nZ+mVk^0f0q6Z|Ag)?`U!ti`C*eYYlzHf4Ly*^+{sg{w1l@1>WVE_;$}2 z><+w|jA%ODM77m@P52|M&Z)1Kt8cYBH-cox7IgH@mC>qNl`fu;w~O4H<9TLi$!Dc* zYLl2FYJw9okfiIA>|{;>8BUThV|QjLWWo@Qo2nzJ-ERdpqsn( zV+ulB1n0VbsMwv(}*c17T@s zs{9a*Z&wP-(xk$^VBP0}Xvx3vaKK-3Z#VV9&KfSipkH#5qN>U-*YjSm3insP3kF>$ zTcZ!ZKtlDDI7iAdQp04RSr|9M3Km*B zGm`z{NS8z)<=KnTZBC3VV7b#z!A@=?xa3w5k`s*FlCrz06vfzx>4ZclDcA-X8N711 zQhVggZ$>hWeLmhnSrS5&d;NAl<=2d@SN!(w|AD!T61)mp%uMdX!7_nG4Sowy5JKy) zi`wjLF*wGMNH!N<`aFLzt~OxnO+;2)8lDFrLw6LOsbi0)Pgk}|Z!mb^VwQtrz8Rkr zMQX;Qi=+=1`IUlhX^|oCt9yAX75@8qPTYUQT1 zQL<&MXWy~ztoPL9dzQKb#Tv zwLn-SOD79f&an{__xeeeS5)ujhp%&NFin4F#5_%#Ns-urQK z!fZvI&htW}bO{TT%veoja>dUi2O*qOpe|K3%s9o2TC)S*xX}j&iN`=DPNg+oz|cJeCWfo_ftwX1=0huwq3rH*oXWX{gbpStBo+`=7Yh7F>?ag@$VC8Lbcl3-{XnTI1&Q66@L=+$S{E)04%h$FX zM})t1VJg=PA1Uj6slQ$45dWJB2j^Ar9d>_T@DCQw+_7wMeuVTWQjTtyd0PA@T1@aR z{UD4-Gq!Wndw*IevXQA*mz2#1T8M&}Wc=gA*I?;GBYHc`KO8p`9S*4spRj6pzi(WrPd_c% z%zfnq6G^%9pIDEi zT6O~6FYOLLaI{t$zUajg|f4`QUO$cyw zs(W?XEewZQc)YNvoPYFomSwa^#dp0=b}DM;AvkrcRhJvKHZ5;PBwn96J-HBu461?2 z8k<(HbOZpurf0ZGB$v1h2*fE%@5svR~PG2RT?PDylh-P6DQK&X<-9 z%D{wdI>3|rt9uCW;$+8}ewP@d2y0(R^(SR}z#CZYgg$7DK~?aaS;A#UxcSm@a@rl9 zPVeU(-Ve@OI`}r_s;$y4pZ>i|d>`NBKKy)MQE*e!(z5&vhK<01Ms|JuWfE0T&c<6! zw~LGG(3k9dH0Yp8ct|)(U(#x+34RQus(Q^Y)}*cgetgesw&075}obH3FG7bNivonvOZn z4S160Zl{V_7acQydTm!}o`}o!vj;m;&@~BPlTMPRP0X6#d~Zl<03pxKOalf9#Qy?d z2jtwxeaPrk{a2k6<_t@@wi*cL-1@A!z3sVfg41D`X$y57q>WCSK_m-`X?nYOr}&4J zLWH(f)El?Aj;>p9e}RGp-jNob2hOCHNK^NR^~#^-j!Fa@ZK> z?rvKy$<{gQYSfAIsu7<&_@bHKkLSF;6^LjpVil=TDXg&2wKKzN1732n_nw>tJvcE11UVY!4wtQwDX%qel> z%24)r|A-wMX)U(;(UKGfC!tFbswy+&L=*(Y>`0_mSt3n$a#6NfdwXa2m0z-hPfyg( z@iT5w%Gk_UrLP~?HWjJn3g7(p8I*BK{s7Svb+n+)sa`b5uYQGeeWFB{a@w^jG8p9Z zcqOJbcdQAx(eBf`YR%d5?ItWbMYABVV>a9383GX)j=fkH!Bc zVNEB$g$v`Udvbo4JaF%{^$z(G2K|hFq8&3?W?WTwQ|B=#MF_N1=?`yS)bumk%f`Q|H}bwn7^=X z9a|A6RdmV>AX0Q{ilZ8ucZN-2T(a7uld2ZeqO6=&-m|5jyB?#6uik(HgrgFVP&4hy%tM29kOm!n@r91O!T3_F}UXUt@paQF7-MWP@ue=8F zQxIHvg=+LS+T7nSM>%f#K_t8tau)~E%26rgp%oICva#e%EtivdBLzb5{U0iOD=gPXh z+Fm%!z=$D*L<}y!=T!n9?VQ9wvMERg zAy^@uAW`zQ^jj)b4v)uOPi|2_LU_gsAHK`7FK2Wgdm%()UB&n9Fu{25ek2b9I0{|~ z$_92i`9l^td%dM4qmS#dylT6uAdy~XMVjLM0Uo|)2TD2pi^+{Px7XLR##632=UbV! z7H$iylZ`}yPP?{O?ONQ?%oFX5#6Kytm-~A-sSAL>%B)xw(paNkE4<7rw!ytyiYyc; zITOF}Gc*hnW9qVX9<9D(2LkH~>nfglOP=a|i8t4qWW>vr$*U?Szn6|$=9iAKtC3;6 zz{C>51Hjq5BnnYLs1Q#fi*$+eVY5)ec~SszV|@cz19ZW{F}pTlI8^sBF-MCi^J0f=#bv!>vz;*E8VA-VN?pp_jif=%ejeF&%GXfp4R7Pwo5=Dn9XJ)0qw=J zmi4#Q6(6c-R(Y0nDmNZ3@PgPL8gXtry-pqoOmLyCddp6W73d;fascK_#(A?J{~@)uSw{eukxsim)>9;}UYL`Q+by||b$ku2%5rC2gOWtr;q zW22@6FpU;bqlp{{Qn9RvL$_9*FiG;Y&MYW79N8TQ7w13m$#P@CO+EOK9_U0-;BK~_ zo*%_PaNTliAl!fI>llIqi-hDee`^aq zp7$qaCMISk3^b!S()%|$8PB;P;ex)JqvXpsd_35=I+L&rs^!ZkQiwxT*_}uL`a+HH zIG>^Z`Mrhry9n4fzn$-R2!g(1S6jlu+Bw)u$Me=D+9e(SY{!;od4R2j`l0a*HZ^HC zYwMR(2BGhI4;igIpRaoD!}Pw~`z8nw=Ba}ua<~UZ7JJj3Lwsy$iKR1ru$tS}L>rc`&^_SRO4t)7OJsLT&s>Ez1hA_h1Hb&ERQ zI?i5E0r~A|1e%J)ZpJF5B|zB(xBavCV*a3{(+1Q-AFqeM%ti#axZkN#qzk}Az>>kr zKhml>S>I>_I|xvjEB(8!8Of5R%_JlS!Cg#KxqNT{a3{>uW%sB4(6A#II5(%M zxFq6rivJXzmlVj_U?B)P;_rB#ed&DUP8Iw*_mx$vLQ85 z2_0z9_#nDTG=N4wKby(80aexZ093QE*qEV5qa&FbE8n_~ffuRzUD--#?=3B8^|DtF zqZ%U(QCL`f<-J~;xszSVbuYZ(A_zZ13v6y` zdyVS8+u2r)vnAFfm39T8Oy-)$HMy%bqH3w~iVhUdtEW4kpW(VH>5F^0xtW>e$X2r) z)PLt@pI%H&OYv#-i0yW6mM!PxM4$fS?u#9wUue2^!!xK(_5GD`gB?SN_;pAj0HWty zCnF>n%3#|e-m^a_FMyI@EX}WE?!;9)R-#(DVu4r0g_kjook8;|qn1U3E(z?~TPsE@ zl8asGl1Sx>Bs)zGEh`twg9_Q2v14#~*g45E*h9nP9?lnHM~ZFST2gyhi1y_dJ6jOH z0H!AvLaoLZk6gLsXA}eb3c=iCgwU6nwSj8tEpCaj`T2!WGi3^t)6M9#pBNLpVktvr zTC?Sl!4-wrr8uLMH&Oq{jb82_B~X|-S;!rcLm_d)Ag0r0vxk}*w_`?HbaS0QI<$+9 z^)AjhvMr4IN_O!hizVX5#`VsLO@#n~Mco7BRZ+<>_JqLuv;`R_XOkV%)8PMUmknH$LsJOf zZG4E)@bLiG8_>Xl!~5)wu<~R&sT%_jPGih)HKFFK?xbbva$MFt4^g>xkty4r+xPq381T|_j->A$fp_D;p5t&zo|<{ zk~n!+z7eBKQKG33#KuU=3NWag!@^0y!a9}-XouCD{=*$^XMSo*onUi3h>dz9^D}Ae zM>ww*a`fy#%wSn+x?2L8_VUp|J#0mbG|om+c*tS}`N-7u^1SjREM{q@HY&(2kat#zLUXny`MimjXxf zfS&PjXs>}RO8Q;K)-E>VdFD%yK3$|NS$QEEFG5n0n##lZQsqSgg=C<*&&S&U{gO&f zo9q4YVZG}y`Rq5Kxf2iWFN3~wr#pM*&DM* z$VLU6RmWOdkUGJ2)s83LpprvkGge}k##5xhjg7g=iZP++EVZBULy0W;e(*@j<<1zpc>)uN}lg&V@a!EI8kSDx=|mw+&P@PP3|4quw9 zu0gl|dx^9S7(Bzl@c_5&g1u9klV7sal_{9eITTOwnW-o)-yML2=;j!T(5PTXs%qgIi|J@s}OHUmI*lgsN(24p#b-L;@?r zHI{m|QJNri8D!}He3O&+Yoo=YQ}VL2N@3^Wcz;=t!J~~Vjx_&;4qXk@R?`EjjyT1L z&<`&}AEHvp7tfSqlJ=*Rn4v1u2BxSjCYFBW>M>Gpq%I`q@p-)b8<5eCRrdVUul+*^ zh+#O=4T|ZjWoNlCHai^Nc5=Cm`jDGH4ZuE%Kqw?{#2hnC)vu@r&PKV69pig>yC-N@ zJ7Hh(d_UO&$#3K^u(T+nC54n1^&qLJ-{SsGPNYm2`$L`~|4nUIxmDVD zzi`Tj2MoH|$(>{PS60QeCnOy{obqrFJBW4Xh72JQdC&@8j|g3YEAl2`afmkr=zCm+ zX)@hM3f-h;5A<+p0xi!|ag_dhr0lVppF`))x+$-xzW*)l+~4~&>(v|kx^*^s3PQOJ zBDtaO%Ng1q4Z@b1VEW(*k#1QZ4Q?2^bc$wma&gI#*;W6S%UkY_)zOoV4oHG_MzGg6 zbI}4*wgFqayOl~dm6a9ngf!6d1@!~?qa@Tj02GWj%m2#-$Pbs~YG!;;MG&uC{DXOt znaQi9wK6j2&DA1X2@nx2Kx5}slMGa*W}u%z1q?B6;3CmQ3-9BzxZ$gLi9h!Ri_Dtt z7&8uGam&$D{8>QoQmclfrr|_sQYoD}njH|XR6A|J4<>EEM+VMq|3VpZf*f+=Ci`2f z*=gQb-J=dv#NHU2U>sLgGpWnuRm)i&47;6tr5HZl>`Hk~&O4PjdDQWU$K$mhpLW`G z*m$4cv5+pg882@EFQX&7TYNVEW6xB#pjS49K4Mf<)bL=jRFbTm+*&hRL@BtbB`LOa zO{^J9q=1>1cXeY!fR65j^4Pjx1>Bf}s?~_n`iOoWry56hL(a>~bk>;qCI0&hnTMiS zKWdrzIpptsJ<#JC2Mcl^n7H@0|Ces7Et zxtrO#jHzpGrUaB3#spDFW_My3Xe$3dY`tT2UETLKe1b-e8ry1Y+qT&_jcpr^ZQHh; zG`5pAPGkGs_wPTRZ|@%G(-~)s?CibfTyxEN!MRft9Au_GLVr=?{k^nER9#1{IKEtn z((j2VtP6Z2bdbsHWRW{NB~KZ)TD?@+LTaH;NC+#Wq<=}xbQmo#qM?a4|jXHKEglTY*xx8~YzivFe~%d>`g z$Mi_W@U)7GjEpzw42;q=M@CYpYT1%T4zsiku5m6~4m)$_`(qmuTPtHbks~e4%}K_> zaC}-V?QC&%Es1Ww+`*>7W6lv}QTmr3Gbw{0MLoTZgh9qoJtuHj_t!}iw=Fb8C-}oq+cx+`jT^zg?8WvfiT>mud`K?Lk+bB}UX z>SsSZUP5r@E1xxS3zw2wnpQjwRSsn7f{y=gO^$5_512V(sp8a>%r?mqy!*E33v8^f zv79(lB#I4MK*mfaJ33VB$7nx}{Q7lLVoe}VuBuu*p>1Ssjg8E>xUv!^Obnwd0Xk;I z`6XSf2=IQSks;-gLBZ;kF3t`30H?j*z%M>LSzVWXBW?HEA2F?^Mt*W>COqO(&rA63 zzw$-zTle}^{M+RtY>?vrjWhNQLC9_|!kZ^wfkL_WMkw6E82%MLGc6A*Y% zfh<2ie=roGvANkM@r+}A1!pelY^4@=Sq@J8t)Ay;H61i%$1ytn=v=2;E3@** zf841*|IzANiyPDd&h4c^f0>l=Yu3w)t$Q6$txIWpy9+5yo#l)pvw-3Y0v}mo% zd532GDsRa$7G)%p?oUh6_{ybL+a2O+aXjwK2#4RR(#4T|-&$Ezb!r$=A(^qc;PyjZ8~q3E{18)dd67F8xr7bzC}UMghj{h8 z3?jjbs#;iP9TjV|vZnI6bxd;9#MwTMl2|lT>e|$}YJR?DfYA^K`#0x8=5wox+PY_T zez5n04(g832`0CKd6x%u3RO5sbZUob5gm-wwI$6-jE#7Ly-dJ6+dG+9bjdPs$riF1k8+gJ&^I#PANjG53NN6#4UunPheVxNpk5ZxN5$>p zP|uIqw>}_CU!yWvT%-5z*VGg~9^EHbHPZI9k7DrOHo+fqbKf2C5m`euYbfLnOGf-r z1qbaj;xP($7^vEu`J!1Jo<`8{)#x;$*tns>qzjLon_u#Ii27o!hK%P=SaC|!D_L>q zVpNjg5Pq`Nn-~f*V&UYWLVn?BVLu_<=k-~4rw=Qhs{pN>G-<;n3l%QF1_~o{g5@oo zl%c_e8ba3Kvso_28~Z6f;mEMWL@Gj{iA!~Z7vQcJH+hhpoD8VAz=ix-P!O=fUcXxNWzAxh&8=c)=kIMFLZjXwxbs>wtH?7$XG7!GgV;A2rgd% zhI5uy-o*R;4Y@tmEvtr!iz(aM@me7?2-$zKa*~kjBhmFpD3#dtCD&gr`KRTSoOP%3 zrJnx@Yjd&ldLhUFXj4|ISSJ4%I{XHSt`Q;D{=(=eVNOY{*o*`9RYRGv&z?^4Za!p^ zBW1D~k)0Q=tdYe|3%2TnmekOe2~^g+e)C2h>94P;n}>Sa)zx*eH8<%FB~+$%v&miG z9RD44!BSP0OAe8fg3(wCo%^e?t7}B$WZtzL?Q@_lZ%(y#G4*#^&ormeaN#{{8ZK{C zy5s`#(;~PpvC~!YW!GWJ6uia*`Vz=5mYU4zU2BA=<4hf;3OU)W_O68t#;2s`nF%== z@#(%tOHc3)bHjE8pZA*l2leQ9`a6>E!U0{ZrLz;0+YQ_6n?%;OV|(bWv3Qy+k4I<7 zoR|9@VnJ1z^vHXrTGw0z0v~)XpxWk2iS(0l(m-f?xmC`5a3T;4xO0)z_tXh;v}|=2 z`zt{i9Ukgv5|k4aBp4zX@YQ;>0lMAggx%N;X7`Va3V2ON16O?*Q-ABKNX1(e> zZtV)+ka@5C1(g(O5r$deiZEq}oD_8`oYv!Pt1S7bRQW(T27CfkK!&(+9(oh!s>MHzd{83(fpr$!t z%KIxsqkR%J>Ez-V2G8jB+U6}EAE5@zr>&e`u}lB6%wmV^A%RllBFuXOH`41BmRG!{wPr!eVDOP?dgnhuwRAemj_-h&F08m%;nI=gwEY zAqZHXy>wbhO~M0!k^Qi&lFOh^YJzJ-+s{pJq(!cFtetUGcrIW8DN2qYBG(EE|o@CVX} zNoQ4;U`B4>;QHJ(Z0Qu9ln1YE*hsB~8n*IFYo-z;HFXyDAuUD}C^TB|7tYo-YVY_y zT;Ue>Y|Dm?`aJC&N`fvyCK08CXydj1wA}GDme|}Nk!W1K4sIHnD8%oNs6Z8jgNv)8qEcS)2ng7`jm49Gj(v{>j<*Q7 z?7&t7-WSV-sp}Y1l3NF8Jrxz%Dz5^;0_#Vrv9YSHv%cQAR%2aehws}xtL^9Ty0Ftg zkd8Jk5uvUHOGYN%4rUQ0zn_6G-9C@_b|s-c1uyi!RUcJft|j76ba!2P=&CbNQ9JhB zQe;eZV@3q_MbMfeS(0e?MssT742s2w{H}UE<@U?b*C^^=;fnivLRzZYUy-dNR(19d zr_i5dHpAcv@D4hN0*TrN4O=&02PTFjQH1z5pOPYmQITS>%<5LHwCRMwtjc0D)&6>` z8wXJiz3?+8vCJn1khnIE=QV0(!R{gcB0nUi3Sz#AkJaq5bc2D7TGa!l?+7t*f5nCZULu*r%MgT?kI`7Dm){ve z>B3={puw@-C)+e3y&o_we0BkC=q4g>5Pxe^S*Wxvp7%!leNyipk$yH#qCuXc>GJ`R z^sGMJVAG_KAN~tJwA1Px%$}em3RptGhwq+&yq@YQt$L=Ix|$IyR+YNd^f=x`Fe;}I z@{*bwdu4iE2A(=;43AoCzlr*poMpFEB}duD1|3C|q^O)3SmNM-QXQ6D8g@i%xQoQo z(G2I{uxsa*S^c^-J4V#-f#Vl&{@5eycj999M@WtWgoCxap}>QhOTGXoM9s; zWcE&kUe993=LU{#uSbYxZ3HsugtBpyjRud`>FgbxJE+VplV+{%?m)6*WLa{fQ?Ox0 z_6d`E4{obeY&e_@`Z87OWa*_9{n^c*(lRoc85#5jJ+e|bAWqLas_M#DbbzY>(1}!E zJKo;jH1s`+Yxz3#fVpB?P2Xi0pwwk%YD%Zy)mDoS#D+#D_}v zSYc}PqlA!;*$PK#sK5(5;TL<{_tn4%q#Aeo^xGy2T8n}%{&yVu|P z?aynYl+d@G`kiX(B!mtuA~s#xnRJ92M3HA zz9<{)Mqn4Q9mEE};ota)=K9-tmG*ja;vI=3cFZyBK%Vpu?uG%=o0 z*-OLC2ov$~+drL@5eZ+#CK^1ivCS&~8mZBMSlL<--EI%H?u_{P&7Ni>N6aqnGfW~{ zk^I5?k$Hq*G{gM|g^-6#n6wUsz<+&|RH8 zg(@Z*oSK{p0!eY>sY_h62acNUOqz3pT9Iy?N^90iIHE+MT=Y6iQzhKPd$&y@u=q=Q zDquj!D4cjHgGP^zm`rp}@lNfjkrQQ&suiwTWH!J8wk3s~pBd$#BC0FtzyMWD;_#&f zxF%LKbU+>jEKY9eq5bQ_cb$*56n{NYibcq}}$_><)r=`gZFc2~xB`ro`5Ew0wO66RO@k*|i zOcXCI?XM6fe@bFS_5dS)PMafoj6E3hvADW&jHTqsRCgnep5-+SJkl{DJEX;5oN2#> z428_lg3dHd8Z>E%(&2{?i{ido{~9*q`G;7t_D`_gI&nOoDFG&rj=Fo%j3xb-zY!wJ zrs$uR2Pi2}s5{PY8?$d3RZKiUCrH|A-wPMgdWOQIa(LIfkx%M4&0+2qqsV0=#oGC9 zEWNSRZA-v>`(IAcJ&O!;hfRtr?(TE&7jV{P_Vm0=$gr4@Wj+L9e_+7_8r0?(V@ z*9MQ(?)SY3_&kJeUuvqGy$~nNr{BAKUMSYn+OaSNP@qvL@u(v6V1yI=LBi>(6%W~+ zy*9Qsx!fLE{Z*-wT|Pcp_j(wO>5vUIJs($g=9l=G{d+`3Q|SeMaukmaZjGGQ6j2^e zv-yR_6uV?zkLn;$R$VCKz3Oshq3~e8rN-&q=^6UE|AzghVG9c9#gBrtuuiYhl1}Q7 zlC6TdYc!IYIIEjFEqwJb2x#GIRXje&o2P372Rv^dAEjS&%Qd1(LnaN-kdV!+G>-B) z*(~N1QNVOlJa_~vSv_nC=a?zLa1@O@MC(X48!LCV+vJZu=Vg`Fk_|2k$lZLZq9*!! zL_QoFpRb{m^#=R<)-RomPH>Imo09r*q;4+0!t>-!gjzAo*&l7sUfioNZ*$jAo&QN@3EQX!V<*!LQ79=4OUH+cH6Eqb0ailWbd;(b4=vngKH z?8W@o-~;x@@tX~gK*l$Ja1ioK7ZPQOjiTU;UBzd=2{Rzdla-h#LXJB+GD5H2R92f^ zaFXNi?_X6#ThaTn-fZ<*K92)x)4kZ@>0MCLuGMNRLE_8L$F_Kjw$o!}ypDM7-rwWOs^4pJq?Js-z9a=k0W$$b8dp}|@l>bA}h+}Dy!o%gS+$nbX;u@j*dnqog;5U?2 zUiJ++ZYE!AvEoMXhs|=7dGXWgXj-K6@M9r@99!+E!O_ujR%`XFev>OmJZQsZY*n&d zvXi88m*y2lNJFB&y^fP9M2>ii{W`{pA`7K|+wW8TB9w3wCKS8ZMcev0@(MK=sVwPf ziyVD)M5t~|StU&Ua+M;Rgi?vfE%6PPh;J5#VEhOAjb3Ky`kA$Vrj z>(zj7jvUrOA$+}py+w?@g^@{u&o~}^Fp2jj@f>LD>;C{}0*P}=zA5K4s$?t{j!u-d zu5|I+32B#R7ZytR)>|33cK7(adpE@XJHY`|WTi%AYPU~k{`A80xKBXDJ|$V}zaH%2 zK!8B2It-r=bj`c%JtRju2eXU}{CB)C`Dby)rwZq9Pf3Bh&+9Uz$@ZX%`Qtk%(olcJ zClN;X3{PQGE}~U%~z7iu?D4*)BDz20X3}!fh3g3>Gm=gbEo(Z2Er2MV!yj z)jy$>PDm};e+@f$&}T1`~0}pX!OOM-jle2LxEBk%$ zWZ|evJ!;U_kTck5QDovClSsLFp5DZ(Q4-AM9Tzf@P>ZqkjHrlUm-OqQ>(S6zMHO$_UPXedKO;Ej_Mgx(yLigwCOaY}^6YC1PFQfeLO9u)Zg!JB0qn+DrPz2%8cuyN? z2M%bJ__o)gqsgYn?K$T>EV9hbP%z}WR)6B%{q**g`f(G#W8JoO0e%=vsDb#lr>NGDTLCif=-^>vtvZ#iD#1q0?)l=YE^4ML%$>mR;e| zg4IZ@UY)TmamJD>jI|5#dmQhAF!pboW2 zBQ~5rc=65SOB~gP%s#wZ79H)L{j+1YNI83^oO%`+bVx7)EE2T8Oe$Qo>j02c?RjE0 zV0aAOc4eUg6M_rGPU0)#;-}J*Ga-wZ)7XpAK^XjE8;WNOwe_4N)t?Z7RNKw|(ReW4gHPcw+vjBv zUE}keOOn}oRfg@_>*H~cwpZXCEgFj}X??6$vzqn9J`4in4+(u?pURzVZ>)kE7p`7h zUS0l4|2m6laKV>)se98Qwr$j+rGy5kuHbVfIhE(7**HC**HV4&uy4P!d}hgx5B>v9a*^ zLfFwzv24a(i8fLqt|nsaWSppL$BNU0Q>|{rjFUz?a<^ZON;y)`g~x9|hSnAlZ0n>< zcz0#rsBLq^ki~)J(uUAv>0;bi&{Cpbxuk{PCS~5Znw??3m`0FVxNpyCSL^Yf%~r{M z;SFq(Kx*7x$cx0r^l8|&kPymYHJ-SrI)|v{;4GGTU61M!*0>W{;`~xcmcY-c;N2NsS)R`cz(QQ}O)}~GzFFVLKGB*C~5193X z(xrDBjznL4$QKEFS;oe}1Y}6`d%W8_JH1p@RY%_eDM6S(>&>=!TVo*+C+Mfl=Hajl=ig*0fOJJfqf$t774VnJfp6x z5EGt;SfB_)!aY1AFQehb9~wTA!|8ZbM+%EtlP=@WaR`U{DNzxmk{AQG_9T zG0D*e{wPEto?p%}Zv0}1hk%J&l)^(kwdv9LdFKG&7<-Dv8i!D>i(XMEwo>hHOYhjJ z1X%Y@reGE9E{wj7bZe_$SDcw51=UT)8GFiUzh!rn@?*=^GJUW8NV~B_I<$?5=nPvQ z8>b<_!#?ech8!nB#_Pu;C4o{YVNcE&Mu{g|J$DTe&R?+f00#>-+Rnoo ziyO6dw>6CX5P=sb1??Z@;uCUw6JiwfpBfMqHvAzd=;1*QN>Ww*@~UusWm3izHD{qg zz2WV%g%?ccI8@hm&9u$WAIqB%%7jehfJX_Yh$gbRMczm#t<`Jzp0^A1T7F|xY8Nj2 zY=-IB zP02ZytIF1z^7Rq@(m=w4OKY?~KFR%|qg6csW7u0MU2s3~QEk`TMwpXx4rC3tdKG(= z>X6sg8V^HM-uJGi-#pjPN_wNe>f2j{98TVnmscTc5j}Rxj*t!Io8zyZMQY#-t;=a3 zF59;xok0c7_8X^u-R%ZUAJ4HR2-v&}M?(;|+$Q;`|J9oGq0y!KQpg3Mm-VjPo@bM( zlE(PT7;oP0{rjg>KX9!3w;sA%9Clv)%$LcDCxHhm@&AVfK+H&=HRvvr8`p$nV9;)- zaUpT}Vzu=wz1McOggmmoN%&CZ<*|T`mGN*GM#Eyncb0W>wmoyncN6~B9j3o2L1(j< z-F~^)KxKLnqaeV^=lgf%HYiEMZ}nw)l-$Y9hfm-i`xKU<+A@Gmkch#{e<&&~-O14b z1EozJ%B0dC?C;}oUcLT_|Brgi<7#bWL_&9Bms4+?9S{t_u)7|#AIn8p-Qbg@@B5(% zon^tH{eS&}FIJYC(gYI(l-$kC4uM&c?U}@{(9lC$Nz7NTA%)07c73g$o5|P}-KU4I z`M4|o$a5lL+}6{}o4t6PE88X{uyvfY@b%GXtxJJX<@KlWJLt#Qxp-G`wv2 zQKN>ppzM=Bxpwb?!+V}zf*IG8;nSk;8@vi6R7$pXOrD>;pq{)wuD19EV4=kn{S?gm zu6C~*p*)s+Q2+im8#pAYc9j@IHqWht3z@#?!zT-6!XlF@z=8tFVoaJ0IV91=zj$~= zb`A1+1j&}!@$mE+9ngsBgI6LjWhT=?1yP#Z@XFzF$gV(p)(Pa$=08T-5&vS}pz*LhsI3ij-RVm5)#x3$K0$Bg(oj*!B|}hjVF4J#Pg2@@ZYZiQX;~=H~#p^ zz%5|&JE-zj^}h%y3`4|!`FYzQQ5o2YSKIE$EU_o2_Zhq0oZ~5QzSQVY$-sem8O7hN z4v;bJ=FdWgUvmiW!<@pPUaysi_S-Le-+qcL&ilObOI7jf6@1O3@^FG>><7%2{Vzf9m6mm;J)uYq;8u<;>w_1hdcG!*kRY1n?f+wx9vAxD#XS19` z!-lW2Bd3So(l&|fHTm&T%r}d&GX*G-v2e&#b!iuIR7(v-kqhZBWgE8@9a}cyd~1OY z@n{VG(f7*@ifRB3M}&LuqgeCSwz z1aQ3=(|R>r_!Bk#W)@f_@G<6mHQ&nrC?SCzEU% zLJ+}hmaP25IW;}EvC-Gh0p+QeE9CYjB>Ek=e=Kd%m`1}DPno$?H>#<6M-ahgktw1X zH)`^4Z!0MoQ!Ug|Iw6QBYL}{3WB_vwR-B+U>-w~4UHX}Y!9*g-YW1D%5?fpKrY0&@ zOFE%Aqk4y%H_r1~MC1+cPeLZvG(Hs3IH)4dO0u#Eb7G{jkHZ3+FW&4Zvu~QY%x_av zSP0XO2f_a<0>uWo&(uIfzna6gb$oOgKci9(33GyL51+(_N`2gmwj&WUKBuDo@*HWS zR*V>ZQC#_is9@3A(V5BM=2M0YuBcDrYqiR0>G`+sJG>8V9Hb=d_b@rUgW&rF=5YUE z_qZ{ilZ&$7r5oNn9oN>_Zm)23Twkw6di9s(DV_ymDmClb$RK7D>ZM`foFIn^q-o{1 z`X+DlXOLy^xo_9)!hy(P@o%E5zV#Z503;jx3cI@(PV?~?xhW83fFfOWS=rdc#Cn~f2w*V* z6e33e)86L!=Ah}S4VywCN7kmPx%s}F6apt*_E6i}8wegQE-pYB6Bq{I$6~XX{r)o? z1<35MFf)UG00R3mDYPfJZa*!cK%d?}5kR-|*)D0L3I`f$Y{W&&3tMhial}!WQ(l(a ztP`rvr`q?!+>Ki1PRB|q?>SHP>9yVHC`@pmU!Fdl0~T(pvKTC(KxY#^w%y!lh|&E1 zUuLB`3c!%H&+WIiw6nW?xcB=UMEaiCpm%gY32Zbm!LekRnCm-sO3TdbuJb65O5mmYU6r_^;@V z^c8s5kmZUMtV~wV^kov3Z=TWZ%CUK7{?O2;m~eQEa89ZTA{f!Nb=w9`Kb#abb##nW-7anK_NFee;t??8hAh4m-Hbv6iyM<7T(^_FOEJisza&Xu4*j-kEGsmYAHHT(P8Gt-r9iNpiQVp|K3&>lJZA zy>joWs;bJ$%E~;Mk{OHjOGvaAUEZAE+??0#x_nkr)L;2$UAcC(Q8PGg0qDMseUMZj zXO!`DTg7^6hI0_Y7+*)cbCD9_VXi($Df+o?W!+zm6<`w!gGA(1?kLi$UdFmg7$(F} zM_u$HW6LH#iJ?2bH+=+IZ$BEwxRqUAQg8DxkD-#^X(*g)4DoZ)BNq>>L?ZnBT-T>= zvpb*0MD5Z9xtb2L{rK`Lm$T`NJE^zQVv)gNuM0M^R(~U>R;8s~0f!>^?BKvJd>mQ{ z5qTmMW!GakI<{#0YrZAzawuX{H)XILP>0@P$P5gMJU#3jk=i1C`SQiobQ&E@A%|DW z###A`AEC6D^+pQ}_@|I^ed^Gp-B=Pfl1d3f@7v8SUFJE}>0e^UG#A%QJ}wU^Q|j_Ib+uAP zY-|e1-!Q})nT19pPad5Rf`!73=Y3L(h7PFp2$7z8DmH1YL>_@Go?08BXVvOaTbAVh zqF8tsnNz!nHyD#R8<5^L8hJ{!sknF1coaLyE3JJgwUM zE!LZ5^*xKhL5Fmbh2(NO1AF#4Uf16(y-scz7!2CyDSAT_*x|oKguf32LVr)(HhFlA z2LLkuc_6F_Gj3dxG#iSjtzaS;VG^u>7ThUy=zx4I5E^=NyfQ5mPl#}ajhnnjsxcpj}ljGRELB7L427|VO zlML{I3kyng_wcdzBTlUt4pUc@Bc`Scj=u>hFKW6m)wYdz+2@&;4pUIIpLu@IzPd6Y zL_2^B-eTxpy3Uy9tJq{fTt-=RDf3#&;rCG6J5`VuYX|eU=yp7<2)qecPk(aQ?n%Bi z|J6&s8$H#GfXqcw)oBU?W17gF2G?HrHijK_h@@uyvVX6@0QVitPIWPsx^(Lz7@4Bvn!}sj4rF*%hr4@Sg)J(d= z_Jt9C&f#ZsYRMYg^)QTn5`iv`moHxrfkVKbX=garKwQncx@qR@bP_KG7zB#TLG>$h}q@mj|5<`Cws!#!Z;8QAeWjxdyZn{SGCYw~HFrXzyKCZX9REaI`0ACj6f7$I`#H zYbVkZxg$&TrM7wfiii+P)6_EkDW)a{u&O_X1G5%NWySe2hZ!^F#5AJHE9nfHlX7ii1tV50=sZBz@O@^cXVWq*Yt0JgarkLZk_J^W{sv}~e zU_dGH(9>Y+Zv61FrbYRiz47@tU1sFP`%VGumgFXV$nwaG^5!J!xqGcE$OSXKqK4n= z!!l|BTk|3>IAVF=eJ48^Rdo}$Ly8m{RJWYnOXYatL2Zv<{`ftN>kID=k8>GEGO@y# z-&Wg5*Xa_puwv;TKJ{{0ucc)WVU9fVl}{t&)3)2 z%#4F3-`Ab5vB8){00$>y+bB=c3Nh?m6FxeP|=y%i6$v;}OGT z5;b&C(2b84$;Y22(Fs-M70;T8e2#TVNJ$hdc+s_*+lRez9UV4pyo0hO=_v6KTzJ=L z@mi*)W7gp9RSLVAR!&xKF4rfz>4^8~;!OUhyKG$AJTa%yU3<>j?t4N@ZMypg9vtY}v!E{iprwX|aJb6FLHAra%%-Ax`^1GO7I+2{ltiZb{rYe-fBbLt3Z> z6{>vMFken3bzQFSOrBRn0_;(fdosM<+2^A+?3G(F5D`wRX_8|YSF1|#{Ym9bRkS=_ zb~-vdnZp=Y&nWd9|DXx;q?@P2tz?{dgm!@fPTNIpF1aA1B4cD^z2MK=%?(NCAaRN( zoTyQ@zTd~dno6m>KBpgG&WH4NGsy(hI%dtFQwvMWf(vF~csAmsu$g`&%L1&a3l9qG zmrP2I;*gmg6&a8}_2X93e?6wQj_)=TZ+uJ0c&|T5M%60?M}G;+ILZ$wFpw8jO|wA7 zcDgYiTbWdqU02N$-Vh%hJb7);EPXn#(d|7IYrXPE(EA#`VEG9M?37`;C81o`c0H>m zX{6T5s$F!(me_Q_0cFiMupVFE|2`rg@0qKKNV1pTm{7a8EDi#zs;;}*ST38-M5BOlF7)zPm{BX^{~MLyq4p47Q!G%- z^7csRy@h>b?&;|PD*m(gGgrZ4Wu^H)TW&k&+6HyDTOEK({$LoA+xjKkna@=}w6iv+ zTrW8TL#*_u`^WQ<-H9k*UpYl+YGyW1niX2IA|#pcZ+W)t8JeES!2Uz|^zOMPHFhS0 z^>rIEUvzVQKE*(_lkO~U+^keKmbP2qN0vbBa-bG#RFu(}5Uwow1PAD=!)$WEm!{w3D67k$b{pH zZZYYj`pte=V?Y=)Ga?54WW-&h;4_WH!{&e9Zd+&$*8IIZVz#!jbFKpH0^eZVDH72K88LAGUVD%Q z!UPNVDE1jS+4F#rO_&i^EVD<87cQLGLxTb(hD8m!?Blt808obBYwLa3_SRNR2Y+cp z12YK;IH;kap#zH07DdBSMB#J(l3O?Sp#3OE=l|U5mhZ8y@#XPcA&1JX0OR=d zbPkI-A_$m)+$EX$-D0+nfBM!4Of{HJE>~&YuGqfJ_~UE?qMw-by2q9-d}X!*ip=S> z{~>n(U&PU0*iM~##rf@LPdo-vZU-yR1{7wxch3oU)YdSdPCwrJm!q8zPi}>+hykGKdn_%Z}jpw9=qGexd{iV zTwAhsRQ<~pcsu^Qq0w=HM+#jP@bRrkE#(1GtuG4bTD)B}lxWKk1tV~zPkG}SvVg|_ zbby_JHiL1SU~q|l!<96+Ujg1GJ?p-pu>RVs>+<5ir@6U3= zq@K8urX~fnW%TUgXzVl!sYcA}zXgNAgyz$txh(5#;Hf97Nq{2qsRBSzQ|Mo_>d*cZ zOhli#Y147TTf+r2%wu1J0EtT1IMWxP0u@yTiO9Fc3)GfXBWRcqVJj zej8qP-c4>R{2;2@vhTHJVE&W@GO!N_cXxA<&%e7W*+6*~O6Tu)$J+|A;-X=}!=l0B z(XOfP1RuT(F4^+~k-#U;-Vf=!0_`8|?joD&X|rFs-gk49gg$7vJg&=C(4|w74_BKX zcZpF{3*`$QAY|lSnr~5oP|rxPNGYvU_nxF6RrRHh$En95BM!HSjqt14Az0&?L(GYc z1CZiIQI(A{c;BjhE%AYNxuQ7}6W6>MN4(-(mBdCH-sBZwYZf?_OZQX}w#v5Fs{uDs zm*1bWqsGgltxvt444h!hX*7LKh&>UJpdgh2KcUwW5&LejH}~n}YHkv+2gnmz zh6i0Je~-+!kw%x_WgjqO*FgqHM(KwI2VKZWS5?~NWdx|u8Z@*1ZKveKo5uIXjQ<|8Y0Zj&oi5h#ck-B(gBlF=e4r3vR->nN5>oBfD9P5 z%FtnM3OtV%mCe3=DJyvarX7Yx^BgnW8L^Kpyp(`X1=WCTt8xg&jG*B4LO2B*=3T% zW;ApSXSScL0P9}IsXBkdO25pyTM@Pn8$=5N<$VG9m&AJm>Ua;Q;+hUYY9U@}-W*in9_#Oxo|Y^xtb;v&k`{*l{;FVq;N8NWhv3+?&&(Ozw@|V z0P9`Cjx8eSFc%^4P7_D=b_FYZK7#~ErX%OMx|}Lk3jw66!1>R7JFv;?GFLR}v@?8P zOX_i3;H1?+-t?SrH{&4R{^ZowVSKly&Dn*+{KSj*3<}d3_(~Svxj710-Se1y+$TN3 zInxC|7>A1sbXFd$^*j{BQqj=d-raRUtnJe4w$}pve!XUi(mYnZei^6MxcK;bn~j#k zL$lKZpmYbi#(FH$E-o$$C2|n{k&!!voCE{}O*JL??OX7=K;ID$(R|(qHt|Bn{k-~9 zRb!c2X9MnQjWaKi_bPI|=Fktu-)1tDiGKdIsj--`Gc4G0Y--H_cOemWkP&ZG=b%~+ zpEY4j4vU*Cn7dL3HFeTqw@tjdnzr)Py|XNBPM3bvggI08P`>2KP^7DiS6obN{-DZ@ z6c3ToLtXi%swlT~EGX#=reT4;*kQwb z80m!ll=}_tjKDyMcHSEK$|FZEd^sT5$0%Th6~KUmk%s_MOihj@=<0U@eAL>23A3bs zXvauQ4sRnFgb*@CV*biWrU_kk7Mjc^3d$%FHjZm*3b7(FEvRkWmzO%ZqPiO4n2?5* zl_PrGq{I8!QCfN;yQ^HfxPU(&n2Ho){3+lBj0=GSjkL7%Xe@7c7TeTTpePBLKaGa( z=g-p&u~W&`9gB)HSPSREo&b0~3!hA^+4=PlO_CC3jYRYNCx$~P3Sb$<*5=M^diZU} znoPYc@ull~S*;l`&Y75-`vK))jlDYXcQfsJJc3Hf%^IJhIku0n{%Hd&o&<84j(07Q z+f;|8Kifohfrl3bO-*-*s##dV2WHb4V)W zCxk<+AV!zd#b&D&z%=xH5J{kG>0~P~5Zu!E>1waFpABd=O!-&d6aWNMhkA#@xG1LuD;MQx9Iru zvPi4r+Q{No2(--kJnw97kCEB}U9v06?NFq>Hupe02vbvl`KySH_w5u%tLKGzt`8Il z7!2S2VnmRu6_K$5PKaRArl#c0|7lG34c`q>y@2;FIeo|u~p=oY-_KiKu%UUXh6 z(#Dm>jP}=uvZ!0dg5<5&t3fO^2owBIjmjH+h|fIS~-1K*goAti$7p zl${@S7gCWBeE?2Vqe@d~MemWs>1etbZ5p`!k~xcy`&vJ-7_7M!c?4%#3>hV*IpBNO z{bMCicyzR!y!^*Pqb!ArzP|o{Ybt3O2w6~EE&z4E+S5T9*J-iNXE&S9TBFHPRaeKr z#C)76=1$1Y-a1<;O5^oley^?iy#38G({||J(AdZmng;<6?&##CcYdw6e{e9B&I0{k z%awas0(pd5o<7X}2pu_=Qw zucG#?DxFr}{DidshXshyu09>PlS+2+_U2s@^O(-!0_2sm9Su~JmF07Ixd7L9dGmvV z10_dOU{C43cTlHJPRcpavmIso{$d!yC8trO{1dPy6&4l-0!71ROO1<=$Mu&bE&(p? z>jgi#oROxa?J}^e)hOx51YrfBydEuwDiI zM{ElS2*BmA{nwII-q7F(%xUQLwxog=fJa2)yqFOhYUc0w7YBINv+`cRE&mtN_g>s6 zxVT=4U)|<5QGme>;8HB`ezyQfc(nqU_an;LO|uj_9pL?>W@UM^Tw8)>Et`GV=){VL zl?}W5`ucz&K+Yz6<`lraE>tF?I(JCCyf8a{sx{tffHzQ(veRqFF6qASCPsepmAfydFUg)RO!Y>0BUnk?`~P)YmhSl0txPZh_;S zF4nhi{eQ2KAsyV@Vk09be*Z3Vz6Z`WF?d{#JVu@XS#fb>xFJKo_i1O;x>bTAp zdrD7KP_Qq9&1!vPqp_iZyZEm6^`aBVKWyo;CnqQWx7*Ds{g+t+WCR5T<>jY=ml{jx z_ED9^?Q%L4foga4N(zwT>FF<4>y1@0n*az_Q4!h8C>4TS>(%pHjZNewFEGc`X|?tF z6G7?d=vYQ?iXQ=lwt_-$HKB<{`#lmRL)T61#**(p1{JLxQJdM2Wj8kEa#ptgu4Fg^ zm9B?%E1zQ&gek$Glx0>{RtOyW)Xb#+F6Z0Q;IOa-?TpYvh5C*2_lh8m|6^%d)po0E zX<>i+CUpL9o%hXkm%6n_Q+sQ1{o?Y#xAW3n22E{A1=l&B4FO11I|1@Ar9koqt`k#P ziiMB&R{*|ual*;wI;zbrm(88LPE~2s#p2fGaRWD577eCvp{J$wyzO-v4djIO$&;u5 zkEX8-i>mF~-gK9AcQ+E!NJ>hlbc29M4;=ymf;7_IA>AbsQbTulcjve7=R4l{Ilt!E zd(FDmb*?iC9~P23N5Qa1kCUZ&2dE24svjDA1-N(l?*kz&9c-y;-;=vq{A%AamTCr; zC`iErX$D6Pjr1cKyI}Mz2wSI%`9i=pXZ$EBG^=@Bz#&OZ3#zOKD@3l2cWA^4t)Bc1$U3IWPNQEYY~; zT!TNuumu8=5BJdsKF~coAD;)xI?Mm-r}TCGucdn)loWv_u!p(J`%I~?KRI><4xj~v z2m0O3k&m|&O~{Og)QPl&4l=1B%`&z8pehF8duMWc>xhn?-G zr@7x}*qF&0J_U7KcD5gHA^X!xff8SWy+zN~cso?bbU7Dm{0wH{gjOPx=E0f)?Q2~( z{(k@U*7~NN_8D{bV8*ikhp3{xCZ6d1kEYd3!J->w<1>XQ2Pomip^qoDaIaJGfWklKn6*?R%b zy_}+tgFmo_UQSGIJdnI=jG8s4Dn6VHq}M7_`iSGM1f!qVdi6P%8jr ze;#a3vgV#<$@YIeEO9o6h+w)=^37n4mNc7DU?DFT#5-kZ7&qdC4o>vcj3w(Z$hvsv z6vlAWAqF@srvDf4a;>=vk(>l6@0T|QPNFGsumI>|R8+vH{2G=tldWc7NkH)Gx;C(4 z_}HS)`0-ar_JC98mMh;6a>~v1Z6iZ94Lzgfe|Tb_KX^*pSbDiN7G}SP5vl-D(V)uN z^jMyXiu!f~4ap67L|kF7Cs&=xHUwL9#=jub?2un7rI7OLUB7vj?}Lm^!NVWLUw}!q zF3TY6$ILyEgD0Ohxfqi56XyDNM|^zO-w>7)C<-`zmXwb|O_+f6Q9(n`>i&y-#tEx+ zWLTxXdWfE*qd>hs^R29xmp939!w~(n<#dh8jI;BmtTZNJ1pL%c>EMBsIJvuCWBF=> z<#U5|l7-0aM!YM0TBwzTnz3mSOncgSODmgf*FA}=Cy2h;%QZ8udHnxW9nhhtx82y;fkYIC_7f30iMk8!W zZcTY&v$M17$1HHspdcdU`?__OqyC*pMolmBDes4Y@WBDt3S}Y*lvA)7 z6BXp&fO0vQH`(p&zWb4iw?5DHhXeWlP%h&OLoVkqgS)$}Y>5X-*;^G@{IUQA5@9o= zje%-fDH##W@7x@k0~6Yx5B6TsIN9VCz26B5wb=iQS14d%Wfk_@NEUc~B&ZL2GdC}H zGiDwaYPH5?Jv#EeFR;I&%2An!Wz=^D@6u1`bUI@jkl@|&uhVF=*!|FHbRelns1K$Y z?X`HMwnoCRRDf*Io0Fy4dLv#l&w-z&)LPs1`1JLL(2HxgM&UF5g2FP}FX401dHO%J zs`%V_ybv!?knFhEWZO&Pzq7BwMEu6qteW=V`Ch;jCZ`|e^J8J`-xlE;ej87^to|g0 zrHIQ*MNa${rT<>T(O8xI+d?+OSV6O`-kcGGLemD0Z`%U2I3ray{`&^in@dG|@1u`vYq zRxhp(oyM^F9(owI`z`m!6o^wNDtJRYz#~uWf1k^H`qqw>`;EAQsh6q!OY!-q_5T>i zJCk|f0ksRt3qkYkl$4YX_x#tdF55F4uJ`@WYAcc0>k?axPIuP=g>Oz@Dyjw5S2Ax=Ax1z z(%?NSV0V2=;;9s6Y>}?>lW^6PK8CW{eShPW*kD~PlcHO)8jG}d?cn=PsCvZ^-8pW} zH?u<3kZN5aud>pHe~{&^liu%XTNK>-Z0Em?2V$R_X5rVSri1?kpZb4RyGkI2k)qN4 z#PPkUQtl1B*LqkM2iaXiOH+1fE=eWYXZ_(T0ipea9y!d9S^VvrFyx@UXlyK}l?VmN zn_;tRY(%*KIFX4C$5^5--t==Ih=y%jTGog_c&{hG;*?7o;0BT@9h8YME4;6G6CtISuRWJ+=MjS><_Qs zdRFVA2T`Fy4Id9CSL1tLFDKrj5j?TljG4e&rgwhXzbd)pFccLNvze&FUhRg>n5L5BFI@=a0#J1FGl0MWNhqsQCx zvaqlTgVNHRhfF){{4eyl^LY#187UM4@-*Gp2TdR9$n;l)rJ`$U2$b-V`m_8L>SZIr zcCKBvJ&n%?oK4hL4Ww-oLq*@uVqPP$5e&6p@vFB89b|GZHu?!dhHYR!gut^bEHTA5 zySmv8miJpRKHlF^5k-JP+Z?8d1B}pQBM;1F>YN;0+|%9P(hPZtJ*K!4Kk2xLkYOd_ zOd3*lMt@1|A8_Z%`d@A3Nw#-1t8}AZxL~2-d*3&d@Pf}v*^q^1?o>Qxl!R=}8ZNzm zorcb~R=@FN^y~h?b(UYvUP)2&Ck{X!7I?So=9jhNEQz61)PNK~y}q91u<(ZD&CRNo z6VO$AdNe|8S{y8@G^p^XVl9rYhc#-x!{?NIjJy%nuh&lU)h)H!P#brfeTz+m5d|gN4N}DzkdbfTD7xt zmT6CK4P)JsZwwK92V~^q;DbI_IXUki{KG;|_3Xq7bEx~3_5-r~z31RbgMW~hmDYT{ z^!8hvxHMX<_-9&xE+Y3+PMYq4Egc4=g2n2SD`;;5{NBlM!*HwNJJh+oaLI3W6Z7&9 zF6($$yG43>g2PC4j|!~zlthmT-`#*L(zBz66nD~u@^O--54h5isNn-N~1osGX(tatKX63Ao zwZHPm^MZ5DntWoXtqdA%4-?A@0Bn-A>jzxh@y&j)_N<%*F6j-6*!d_q|bMr zwoVDU$j5b|AfYm4ON<}_KZ{-NVB9nSSb#ndo(egR?pRwwkBtBEI!elonQa9oKq`w*GLL{Py3igSTCglOBH= zBV8!=aAJBN%J9*jE{M?t1_isVGve2I+Azq+;xyo3egN!nOvH>>k7sSO?|i-BIlwT{XBzY zInIbYGV(%y$~4+ujnZO&0{srKm4*=?40M#hwVD7RB#BUOjc_=VLt-;So zH)w9J8G@-Vc>1=vRcr4(ArK}Fp&L&TDD&a@Sh%im$v{2_T@1gK=93yt`v-g|YuD);o%IA8zMu4#G`Mlrw zWWnw1EayV%n`M@7&0P=HE9rsPX|@QB_n=)&9lT+5s`+(6*r)46@#nMW+^P;VH=^mX zT5f*+W*35@8B~hQhhSyx$CSmy(#*@{a2?My6K$aE~ zCK)_rRfXashiX1bR+lMlTs>IS$wJYyGg^hEYqXD@WXSNNLmF7P48!+5yO-d-Kg8|2 z%JeU?Wo<`s?hd*kk$DITmEGRR^TXa^N0AIhyVqU#+zkOvwd;tLx2%d}{Lua_nW!e_ zzh9znQj<6uJX$B(Y+nZ1f935Ml0Vs)Hy@)nq9EzN)>CIzydL~YN%?XCEx>APTvQl^ zSoUswKj!?^)RbG%X?>{Jt|GYlVqs<`xv?C{8e~l-s+7WIz2h&VPszd7;+FnK5Aryr z<|P&X4xezMWYlVPA|_#Lv2oS;^6f`)V)7g(TgQv@wya%ik;jd|pkRQ@!~|-ZY~U)S zl2pR&1fVQ@# z;qTfb)owT}B`i13hwc&-0<>VgzdS*7^Y&$J4R+nO#&_qrs$ozuy@c@+7ikL%g|B9+ z$b&oycsifw+svn9>^ol`77G+Ry|$+Xb22}!2<1HPDVlpabE}EUi+welNX9=Mz-#BVznPwmf(;1r{kH1gdLSZt z%Vqi}ofCv+)OB<`a;(p9vwfSciCez>GN-^eGHQBgPy#iS45tl*i_eX|jii0DO(cj{ zgC9|~6YJf)q-5Po9Vx}S#9^n%T*~WI;DT}$RxNPV zn;jxxgPuM^D(m_5z)TKRFLKnwpeado_C_) zsjU4+_OBX}G|r{FS%=!Ycqpa(FJpsv7$}gikY>iGGid&*l||k6S6Z9CY=TDP@(pA+ z6*S?w2uo!Uz5i<&e?k?`1FsuI1p5I|8ad}pE{_~u=b~yNy?opfX3=b~OZodM0ofGg z^wfCE`Q!6z@;L!slQ~UKnE6^4KdtL;O?485)>#tY4;#~)U zXvOeISs;aG_;;7>ClzZKsgam6e<58>R6xhbeO@cCYS$~LCU?Pxml}AFoqEL1$JcLX zUv_wKh>3T^%q|KG*fKNYo_4-%R#b5{LsHzOpl}-7P9~#xozOCfrl^JKT3aet48=O0 zFY=JK$Y%?$Vfr5TN%zykO_8EhyccB0g(QO@;Eg;s9E@mWh8BIIcY|(B;&)ii_yD^5 z?`(r+s!2g{Ysr7JvCpP;2n?`J2t2a_d$7;mD8Rq~6L$5S?eBznFM2exz6hzrTZw}W z!e2vDZ3bl36GCYGu@=G%UdOA$9S&3Tuq(bg>fistoK`jeR~tF_Z6Qz5BcOMH*YOkh z<>d&RaC!s0`@6A+Qy3?9aXW;e-|M0@Fu)c0Bi0eTiRh>0uiCAAfNhDO1;U4s5AQU@ z?kP6IJh&TLHGF0G0t)O)@BL*+Lr`2W@3O+{9Vht>Tii&zy*&_VC2{Bh< z6biMpSY3|LR6;s&k2)q3)f=9l8cvaM$K@`Kd)&#pS5Z*Yr-S^2QMHxe5GI&k56cvC<{2s#+;qC(Mz}0`QTnu%%mGZuMvZ9U`P$BLOt|u}uw?zt zQKhAAIo793aNcOBH~5vtwan@74}yN1mtAb{?-yk{Rv`Rf`h$o;%d1B(z3YFaRos&; zCMrT90D3&wwaeypmvejwXwug%U!^$sGMf;Fh8Ef2uxg~Gw|$9*-UAOi%H&elZE^we z1v+TM&?H5T{myL_&1ho7BQz{A+bp0-8Ggj?Rdt&?xu;BL-hQmrm+CM7oWV?kjcg{{ zTRLwk?tolBnso~SmmjLA0lJ~nCAyt^)_S$=4W%W`m5bRqcrJc1j5I7N*p^X0p@-V` z@1hDe*wf2dS4!?2|cX6>MvqLdJehql=j_<(K2PAqFQtV>}gW@Zpm)h@f1 z{J8bee7?q#5Q5rC9N7yZ{fjH3|D2c3)_ZzZ);i->jg}wSifJ~Z&Xz;*b$<9e49BK< z43rruplBYyL!YUSS-lm!^07F%DtbuQ)(eBHc%{JhzBXm;`dueY&r3xY@EEsjrVeai*OK0)WWP=f$ucCB zCW0njIHCNdquOope&WdwIcGsAO}6YuB$$?#q+x$mh+GA2-Znu2EIbC8s>%%;9np%k zxl>T?;QqH6OG+wKw&0$ zVE}-uEJ6T4K6H5a& zv*0IPyi!lzyErk!uk0zXh#@#8N`{!w?qx0D=LB^hQ}gXp+r_}jNI_V`U|WTbV@Plce?tEQJk$DOSm_^)`}MpG&j3w8N+?L}($ zxH=5Qxmcp)0@5$;=)|W72S|pe#0djGd-xQNByJrI$yZF`7k94^=_R>?`RtUzl}Ux7 zynXUy0qCWbEd6iyvYL&QlrT9Z1zaiNkdu?+;4lxV8)ldVHSJdVM+8wJc&-^($g#T# zV79s1gv9bUCkTD{MXv5j)>Cp-gZu%JuOGp;L)07e%{Qcjq$t>kxm}L&KlrC*1r}3EG#YS zWj^U7=wzhHQS6xYoZhaw%^HF$l-4`f>($idR*x7`fs;Tain&$~r>694(3dvE;MM5E zWjBb~qO_1t=s~`L9f#o^8vp0IpaOdq4xinIkiR)G zgM#ZV(*A0)e)SM0^Xk9qZ1@SxFmsy}KB#IYKsXie44J)qOF$4YHQ&%6L^%?w*J5rj zX=i7aOomECt;9?vP0!2(|HKj!tJG%Hso!H-feJ&#f&?p8B;}Rd`RYDBrik>qfs1BK zQK23Gljzkx^9vPrag{xMz~B74FBr%1zZX*_kpx!0XtYie{mkc+sjYSv)^^0mI2fZI z;gsiriNNknp`p^eL4fs+P6!dAQr6UxDPY45wPd0v8~!@A2Yi`GG39{$ z58VHTmI(hii+<>FO#h70a<=*l?fWdDZinA)#E({Btt$`?G-Y1bpXp3oRbPG(2#P<^ z#l2#{`#&zg@3Uy*1ANT77w<1iF2c->{RxjNX=gXyiu^{iZklZhwoacOPak8V?;kkK z?@K)hJB)e#2Fo*ZK0<5nenj3JD6r~=W@Wb|#&7ln&Y1pr;(kH59noV z2Atsv#UF;J)I`nfJJPxKtKIm+J9i@uu`qdF^g2s(9Zq##TdsSnNDZ7U)<^xi|Le!( zbgpACRPe}L02R8bc0SXO@`S5Bro7eN>$_zZ5{DU?X)?7^yX|dD9-_C%)oeW`2h@X@q>lM2RW;m`WRjejphS{D0CJ8|6&4V;F>K<%^toYo%U!L6 zXzTLa=;cx?O^UE^XDgH!d7{+YShU6^hQhn!ZbGXrMqN$23) z3&!N4k?cAJ$uX?6fLS<|A}oNEU=}rP=Z4pV7-VL1#wliav9V-5n(OEPsrun{#go%w z3n_qA*6?O5ezL{=^lRaZO4wyBR;P`NCt?6ImQu%Uck5`?VsoR}WFcSV>aS-R6rgLZ z)hLT^|A8Wte%InY!*#MEHb%_byl=k7=43SwUL537)MeyLz);t7^EB9x&a^AMf9GN$ zUEeOxkif)OA(3Rca)^(xeZ1-G_mhb_M9Vn=J9uS`n|);4y|uJ#h=s-8ZV*qLbw} z35JmU!9h?dds}G9pNwB!zF-n@fe05HE2|X}YOY6vD9YiHOCOm0UBXr=y?6U_D1nEj z8H*KP01>K0;?fNOc!_$4VJbu@EK+=+!oIthEO4kl2TTUx;$NMi4xLsmyI)Ml+XT@h z|KR~t0#%RgSS#cztY5}e)zvQsSrrnmId{bPZ{ohF?imLeR8e%e_XmzoTK!TgoG#NT zDlFXD-8J5m2M=kDo(g}g|E`)y$D0MttN&_Gb%pNpio}~B15rM-fW&XR=f=p2RZfae z{H8l~u>u@Rsaci9mh@jBb0{3E6~UPthI)HjdS5fYfgM!m9I1eBT`^hw(!%RySiTB> zSr7}^xZ}810JU8R@yo*xWB=rfrwjr7!oJt7Q zL+o!#e%K?9>#|607u8D=BDTxG&`F0 zWT;)~KlPt6S_HZd62Y9k`^y@w$j2+FaDC_GdYi-37C$+~#_~N{qtpET{=~OkdBaP3 zW3dPK-Z9}*AMak#Q{YT1?e+2AT_6%VexZTwke0;Zxm{Z|BzhwtQ0-JLk7?#YuGd-<2BTgsWHQXvESfW z=JKxVexs>~M_;Q2L7f@y2OGG&9_csI)N`fjtsYUk2!^?)UUf>o;Njw zAsH$&Hu)cNk%<(>oh>my6EaNl&#N_WU9pkjGku>5zaxA?+?8mK_miC?c5bRK8A?49p5bb#) zc==h!fMK6SS!>#L>(b8}DqIfTTWF6ozyu40g z!UQ+%+sPGG9UU!)+67peigOl<&iQAg`t~=dc(m{-e_aoV(h@ctgbEqn`%^JdCt;rn zirRM?`7~vl{QZIZ*84`aOt4)o%%b!y=s_I+Et?K400=-0LJo+MwS3LF(LJ^;xL*I0 z8}I5yIvbPjx$p#%{O)??TZE*$JN~?SZ$1|pzq=K)%75u+%=#4jdcf##O>$i#>hiWi zUMRq&`YECH0x8LFYp}KB)+5au>)=K4<}9LSaIE`#_vi38cxQ70PFhKNkT8p+K%({} zQ!TrSP;)K>APVNKC7;4UYrt$6S7Cx#2vO+xp=+Ro1TrG1v*R|~p!tD-gjcQL%b)4t z)m}L_$2(XPgnfpDkS29+82)3*J3|>J&>C4@VIOqPU)_LLnWt@f%<`zq5{Fd)kaMul zh_6Uic3*8TTjK0Kx;ea;m37ah|Gq2AmaFm}^o(GMEKt;v=?>N?luOCuf)(6D`hjV# zJwE~q1T+L(HuTo|jTC}U#31ZfOncMW<*V>yEe1TS12O7HGY}1Jxfu{VPEt+jBwY0? zYa(Cv-2RCs9M4t6!~0Rq67)pyeAq97LX{O26hODxHlx!lzO}5B{C1FNC6vJ-D0n~X z>WJ3p80pnP6#~w`e?rph7QdVL&K{n&9kd@)wl{v6#(yPQl_!1kk3xVm5T_go%$FY_ zdCQ{c|MlW60IbE7FZ}yK76X%}(V}OP=k{7CH4`YsKBrN=Uz8yic~B0_6K+|CqPVD? zBjxjzyzXx!81E>hE>u4g=uBmM%Q1(wRcwTp99|XAtRC_kpLA=iHt~v@PyGna1Ed!N z6Dfy%t~1!V*4F(y`OH^Z^Y{tiS?oq9q z+VyPVx#DZM@Z8xO*k#p<~of3hG=;$TT@YRK-dxoY2K1?Y}HeNI6rs^`-SJSup}A zeqm60WJ&YdxOJr}@pV2m09E>;)q&3vCTj?rc&K!~CXPnYE(-h0Uuh36`)L|RJPNTS zqGVMg!v*eU(%dawXKt$bgr5ZhGB824m;O#|9)*R|tp1M0P9pK^kl6;&qU9yqR(`e$ zPS^$YGLUp{`E@H?)b1z?E~kbpy4sdx%OvHa~0`bJZL3#-L`25p8h)8LB+6k#6MN({2wWH0-+;1x08N6Z#u z71r`zP}?3Ccq8!ZtYb)A6nh;ObnX6C9`v`?e{q(})2|8{0L+&8ALj*VP9J|4@AWD> zefk>X_EL!qY|Pi0EX6r0COP>H8eeQ5(i;W5gGT$ud|GzdZk>iA4T&4nttzT~(HHc) z^<|yuRWbC_d(f*0b&${3kJf8^`He&xzag~Mo9`Y~Z8QBB^NbiEa&wlT=Hc&)izA3n zsEp)^*XI5*bSEYoEZBvbE5U?sp+*}ovA9;FBq7~J6{`M+M?dyflci&Wtsm5K0pUD& zipZdvOEq4l$6>WSM$6U*USL)*PCfp3WROlP^urW`G`3j)70{*IHK&O3AvbVZ^5^qEFxPPhO{|CNV5xZAhrJv``tQcQKFnG@-=Hv%a1 zgfWwV047v`MG`FPZWC2JcrdWy{g@4J3IaD%QK6N#PBN^7LoK0n5HVkoIvPx8d-bSe zef7i4L?NLk<=}nxVJ~c|Cxmv26JY2PmYv_j=LwDuEs_{A zvNQAz&uBNZU>%qX!3+5 z^e9ceKsMR~qTe`YGX_q|Y5d2M&<&kxACOT!lh@H2GerGhEr68NV zJ-I(TarD<)(m*=MdXU1!#AQGSo?vX1?^+J>tX(K@j;H%S3rcNV69srS_`Gx-ruN^s z&IIeK`Jy|f&n<_;kbOIBxT9ChGmL7qSbq0bQB4{~tYJm|joDkrcHzxB^Z19X6d_5yuGt|&98ggZp}I#Y$pfdaNJfftT;h%(^WFl6H#Goz%eU=)k-S1w8H?bO zx=!Xew_(+Uy53b2IfVc#>ibA~O!K1)$Q1+*y6C?h{E%oTh>d(aovLEIlc0LEOqf}6;V%(0YJix?v* z!GipaWuH@(3(kk%{F?N=a{sWydrAk*TQ|)#>R7b&A%&whcKFj*M+Xm|xW0UOz4Yn- zZ=p#0AHTh@OwPz_yI3tx<)Z37a?GZlL$78F#4CWaF)6GH{rXbFgNVtq$MovF+}xLe zZPPy?;4N&)c{IW)wq<#hi~_QJ1%j%Tf7}c{C$B6u=8=d9cciy-diS}C0us_E?2Q=2 zms=@Egqw~KyOq4nFbkrnb~$nWr`Ud3bU$vtqbQHU6(N!11n9@=^RRE6Jqh%;*LS@! zGoyloNHIgZWIQbFVRYCy%?{U?Wm)hEDIX=PL*D(GPQ_K{rsriUez#o>@@c& z_>P!6vky9dQN=Wk&_ko59RxYbbVR>@NzTxmN#=;Ce->9}j>V zw{fkBlDj!ahGek$}itB4l__EZ{0cUcgI) zE|j72eZVd;VlyG%1}V~x6MEVwTTB;0oG9Dt+)(2SQ+`B0J(n&mD$Du*7>42+3YES;>vzWP9+f-dbt;l`; zR!D7KC|EyEf71nBB#}TH%9ma77iUvO7xzk1;hTc_T2N_H=Y4wu(o>iHU!Se_k8U3$ zBO_T^So|j06%-Pl*g82CmdvVM_b0ZGQvcgjf>$Lmzs9=0hy1C}zscEa&wu)*k}V(8 z8jALkc8ZXBn15xv(f&Nw(8=wz)TQdfHe+ohP_=hqo5o+E_cY2LL(0!aoK*>JInUV6 zqg<{v-JXRS3UFRn>`UHFoEjS}Kj#V3Zul!gSmaHxB?GXny3?gcoTEco=(87gZw=HZ zriIA4SduJVtF5w&y&F%yg`vW9EgvN?=X5E+5(L#Seb}*)aR8sN0#q$RGevR6GZQ z<2%FsUlGsOnz3YNe-nG4bcs$a4u5RTwJFoi*D5woM2>B4#XN60+bI3Xd_O@{$jC|@ zx8xs6n*WY=U-Y2pw5e-8p#nc6I2o7PH(GARny-)6k%41=kBds!Y%+9D!r(g?9>Nql+L&oOVh8@I8U3Ap8nf}+HCW>p@1LQZO?q%c2cCbL&B(mWD{tvgqEW6)nXdT=8fkSr@|Mresr~A{MF~kLIfR_UQiB|B8ML`_wbpv!q&&|y}X}w(m zs}|-z;Tn{tBAMOb2C1j*bzmC|M>;=lTz!!{pCZR)Fo37~^X|3-zu5;$ zAx{&PnI9hL3O&&dXkrUiiPyxwp;-HIzK7F2DpC3LGvud!(M7|}{F}2{ns#r!e6P>3 z-f1F7!gRrx*(x;fT%^{EU-+#YIztRTOJ7C8{4;gAI+_=q(l?le<&lyZriwzLziQ&F z!%HYplBQzIrjwF%mQ13j=9tRg@fzkyG3D-q=SJo#(suMHZ)sVvvj1G;F?fWBPoASu zUCakEf>WpK>Q~h4r)eed02fh2tRc#`m{)2)evCLK&u(7c2L~|?<|@$>!y9w0o%5vp z)z8(#eDut`g@8LHI{6@~i?+s6b@^n=Xu)S$Hw(=z5VlDK`)K~9z%u{(J|TtZF})Ng zxrqJAum%*GDtzgqiQmWEbJ8SId7dUoyh~ zxr(YA(|9)R9bC}woh1PwZoB~CYEVOIzRYi>%e%d8ar4^KEA#gLE_?MOPW+@E&Mv-N zJ#sOdJYw+ss@X>$u|)lfahoX?a1{xdM62H@Hym+RR@jY^(1?r@tejq6QbZ6T-)*7~ zDzGw@S_DS&%fP8-sF;r(vW;_}#|{$AKe!z(bizk4^>90*kV9@LekNuV@TE+%Xyb+V zY*q?suD$Wam+2kBOp|!;5Vg%4i&BlEH!o-_D9Y7C1J8S9XQ0;#3T{l!eh20RS1z}* z8PwlR$=%)!hsnl}d)@Ruo@q=vg8y@QMwiPL@!N~Z-Nq&eSa_p?d9ptqwpILLCMNN% z3sfB{CTk$q2jZf>3r=Pe`SMwy6C*;p#D6XIQ^FHW*q3zA>2!oF>-NjBW1V(5;N0AL znAqdqO`!0QTY|~pytMz}$e?;{^JC=@3D3Km;-1LLLx+bQ7lX~xH1$R$!kaMr4X?QWU`FcsuiTT{~ga? zPeK8}&fW(lX^!d!hlPu5ooSJo)oVGG2Eg{I`Cx(g8_`_&R5HwNSfzKqe*QS3Ru>%5 zu&Dq%XOi9u4@2Lclr18cy5`ar1|fCApTTMjh9N4)7YJC#%U~ytR91imQ1O@w0BUpL z?WUJJS%Wx@SvI3O-(&p)*8>`KwWPVbzxg&Unu=1vWhKlL*zCy4v9jiMf`jW57h+jG z%x6AS3uL|~r|<9R#g9Y&-hE8w`1Cj#8kajU_N%)G@rHD2m+gB8a9U9Qnyu>Rn{D?mwIGdtwRE&jcZj1@zD{rSQ01q!SNzQT`vg(F?jaS z;q&LCCBep~CNPF@(b*{qK`Fw$q(Ld%Kq(cu%NVG=D525Dw6JHMI28hBY z$*t8z`Up86o-&y=03ZN1R{}={*S<5Ps z8x~E>TQP5eh%mz@0UMRwG$O#3&~Ew0uvV^WvsV*@xfi`WHxDRh8rbt;?w|dX{hWTt zzV&!f%VIH*N|&1OIy=_qg}wlywhv)RQmm)(f2`OD**B0l`267sbZ8< zdt#N=URa*fl5eaM<|{J|oQ;&$&)yyuXqD)NuImywr*n%j2s1uTz(jyTtuiIWx7B1} z++au;XygED{K1_N%}|X9R3rCE*1yecagOz&2A%Q4-^1j#N&@^yTIEzs?iO?9j#R|Q zk7@4c{|<9!59#DLH0?5PJrtC~1=2_imPiGhY_Cz|a_S(`Nj@DrNd#d-+r25pdNcY4 zxP%!5%fiuWCG`x!(DerLviKm(3Kd#v{BC{m(}2E_ZAPS zsCS}#KaW3Y!S^Es4?rJv=cc#a-0ML!rlTRcEmrp$-sISmy1R_Oid>@plqCChQakqD z?HV6Wv5Mbg?b-Q1hmy<9d^SBvah7N3D->(gJRy4-zuGN4vSBX1#pbGSZPe_1x%^qF z()4^EWV@^k{Cu!ktNBB`ndOgBXGr8aP`z?x#KBzMCYt{g1hQu6x zq$)3CFgd}uM#ek4GEu*NPLlxYlX+6g#_SI#Hs`S>T*y4QXZOKk9a8EbcOV!hySyAs zOAAz3!?5%l;?P(W zYWOJGUiPPq-os}xPE|ySo2Cr0M#Zyxh`VYBILEHa7-4b32eY#)ySFOG$s}2NTd@!i zWkD{S6_{bDfRI7Is=Pv*n6jKdyLFLl;8Z3$(?aX*LB>9}Y*E3nQu{nGnw5S{kJwK9 zn}5yGwz9&uq>3;YR?VlISryFvdF8t`&N=UY?juhJ#|yr{9j_WXxBeL6L4~2ER`*94 zxHTJW{%bw)bCE{APmw`ObU-Jih%W%!w=FNAc0diQGvbvCg*&R_ZGxd_vrA8*4jX=r<#jZ9w<)6W-SB46=z z8qq#Vnmr{TC0wbs@=uvY5h0p~1i%he3#=3nwTc-(8T)r3NDzkrpkW}RN)y9Wo!5|> zIOm<`_`X{++|o)7NyhsUP1i5}USotN2>uar?&|PS3X2chMTHIA*vK?dfu>y8fw&k8 z$Qjy5fTi0_sg?C?I&XAroG=5?^=EA}SN%3(;2@g(@w_%ht(cBq=mm~w_&Z8+6#49H zsdKwrROiWPzf;Q1(MU|kSq!6b%gLxMvms<4U>cOICD2G>$6Bc21;eZ)XU@a2vIf6M`IhN z{5IEVut6m3CsD=rHlSi^M_fw0#BVlQMwzjAX*Ig)O5)sffMWI1^Dl~R1NpxbFRvmd zCj0WsMidjj)d|XoDoQWrO_=bZsL2Mm+XWIVsj2Xx+P5l~#3xn4R>8RaNC6>gq8EOK z(Q#aaqPrunPp5xC6~7b`=SU_wGCaqT#h*IR8NjsL!0B6b7zU}kDI>@ktXhwvtSizI z^Ve`}F_d?`+V}S>F8M535rl6#&deCYK^1$(6p*k4BBo$*g)AMj3vc?Ue*-5ljdQfJHVBLjX7KuatLt z2lktahtmIRC6MH$%AKaRLP!( zW6md;cNGK8x``jbLEWIRGK|8~;LalM;o-rAs*5Yi z$cb03{-oQ#)6MvO_1Z4Vdr01QOHJvx8k)q_hsMsQ>imU1u5zQI$-}EaiLn{oEP{%r zzURv5S{4nx1cdkIT()(PC?^t-Q zy7JAZn`^)Bplakk=K|hgna$Ho^uOMp;gqJP%q0pyxG2QL%1X-0nzz{9)FO+mbex3# z=wMPE9yzwM;jUWyBj35}rg#zd?4&u%vt8Kf?4iSS(M?QbBAdN zmBnF>M^T;p-=lXq ztToFoTEOH(2953qfcc)FE--lJDO9q>q$eQp1c_E9G!3vKy?HwoP6yY zA|K_0-6$P!rK-lU#>0qQ2-_l4=9*TtpBy(~Hlf zi;~;a(S%VHx&?aq^!5d^6I?%7iVnueu(!!Rf77Z?SyqF98bMS|Es-Vy*tO;}j4{~6 zQ80r#F6JjmmsvX#Zi3zogsGLQqJ+G%ZRniB`-K`lT^E)vQcFoiI3tbKS~pVhE_~KG z`+nIwS_aU^a&DIoB1`_3mkX@e+8LokQ`ZY?p-WvaQkV?2;P&^GZib!n8;qS|Jx7h4 zt2ZHGxzi%Js7IW*$|8;<=UD zl{cauN(FEj7+WP6_6oYm-70>_b(UU%QM+ca*FH$(du4jF3 z&m@+!n1M>oc^3!%3f}fBLW>!#hdYT$4}Q2U#}J`@UvV@Rs<9_>|(2&yEvS| zuT6!E`>ZFE6~}Az8gS_Bw+}eoewGf_MD9fmUdAiZii#V6lMmn%17kF=BwPt7Kad0q zSg8LrJ8T{RVpY*8GWk@)0lc?0C{N!*=1{A}Wv9D7E0hhyqpDqZo}|DyA@J^3fFw%o z9nl=EZ|)`dZQx*|gpPmbZt2-RW6@kLCbsS3QBG80)2p_*PzX1UBCgcHGK?$zEWK>W zUkh7{J0QK6Cl?;i7-j61AQRu3j<&Lfo10{uWE^Bc^IHbU4itfd3#U5U$4}78Zuq(x z<{LN$`%CBWkwJ}n=2LIG#7X9pZKfu6!P;PGHQl8q(d}Fke1(RyX`#B_aVdip&8nVC zS9OJzrLmn3*1ORsfnK9HbO8DTBPK4cF~ABMo;Z5_lW56M3Hi#CI6Q5jUw%qKweD)8 zS6E^Rc-|%r01!_AW$x0_E5$ifCR;VCAkRq;^Ptqla?qXQW#V9u{gE|wVsNRy;q>LS z!_Fl1J>}2GKk+Xne+I7%M8ha#5J;1E5lDZkibxl5)mB_8cw9{vVZlvwM5$XV+OVOE zwozm_t0tuuT5jkxb^jVAzV`I)kewJvC$B!;T(T##nTyySgY|1A*T?t$m#Q>SH zr3YqH6(n0&P)RvL5gIe+Nfkf8R1NY?&)ceh{h7>uu3vXKys7(X)7oOi%9@4Q%AxxseU}ZXOR-+H zbsda<(wFW8gk6+Q|7~P7ZZU2&TYU%;Xppg6-riVz-UN;PLH2cJv9rzT$J7LXOAX^r z0iIm7c3`w`)0w4mu^CB@Ho!Xg?ZhwM)Wm7{uAS5I@g9L?tD$r*<|)E*aKG#R^mor` zHGNL^zORf{^Y4CSs7)p)pliSAYp3zZXqY3Y`&}dVSe(_dDt9~| zTA93U`dDNP^45=#fLng9zFn_WN`iLs)^<=>I9WJVjLa5T()=1IKD1%d(# zR_ad|UX%N4dDB)FotG7j)0#EaHTTj3Lk>0E3WtQmlp}R1#*MqD9UWB*DjI4!8fvVr zD}Wzw+Rv;s+`sF5m5?7kIneZ?lXfJStwi^Q0lfCTzqcUGJ*$iBlhx|l-nN}pEYLZm z^#-aaqnk|jNrn#Bx2 zfC?&U@T5Q4F!)wvPa97sSTKus*`D@}tklX8wkN}|l31|isiuq9Cw6gqCIA61Gx2)8 z${D0xYn|2ElG>emrj|s4XrX@`qovB?3L2#)U zILUi`DI7wK> zYdh* z{^4TX{gjVn`BO)(eun9!;seQPexTi_m0!!6f4)bkhb(z9!mn*#s^C?7*ZJBwb$Dux z&Z?QPhLLWbWvo89R&TISCBvO3ojyY*0Xp2q+>>FByf00o@tvY^|J=`(rlX(WjD^Y^3%#$2=0?u_ zZ?dC*gyn&48Q|gw^yC8qxD_P96k$MQ(lO)qykr^+?n!(8q^hRoc>~3|g+;r%+p+a^ zcBMUdrm;*M?4SK1sOreVK8NI^L{bAzCzsK4tecR4T=GFeJ>GW{f)Kyd^3aJRJ;8EwLV z?(-^`BxJ+nMOianBc6?&d)`h$lBCeMKz@x~q%`LrF9`?CE8k-CpkTt!WK^y;DTSk$&rK`D_KL&5^^rvY5(Yc{^FQWD> z_ek+e&mVN~fucPJ#U?DnCG%X>#R^)?Kz)Tdsgoh-tM{M-wWRBu1Vbgvf2BgVlX#bx z;qbgSpLS@YJ*1!!HPTeLI8=GwBvHiB(e)pr~ z@(pdvArRL+-GoZUsz#ORFn}y+uBpMSopIU9oqpv+^iFegCb-G%uuiY7wzTO|tR4J$ zdPaeaQeNKqOCZ872B(iNL6MWpXsQ!LnD}W3=;oV*#LlNKKCIp5>~~(A+gM68D}Ird z8Qv1HnZs`8pyS!Jm~Uxneyr15*t9ci+wAFf0B~i7tUtBlUoTc zL-JeCYT}`R`iy#xW~!(CO`PTZ%~d?4?c{;>QJmOiT`q}YpXUu6f>VO~=^*)cR$fG| zOSiQHjbhm~mrNYg2b#MldYg@;kIYsRvzo^j0Z*UU(rj)1JLsXJZIQydPft&gH5ftg z>67`5wr>#&AS0s^Utz$}2xn?2HXGEF=dJRc&pW@WBSy3{bxCjWu>90)y#sQ_1_}a4 z_?)wNzP|IkB&i)|C@M-HCBdMj1l&yR*P^Mfypo8~YG}oveBPvJH(n$D1M@dP}TrJ@I^EC262c&!n zbsZ^Rl$DKec22<1(1%d6CSHt2A)^+z#ZS;Z?)FHQttfl)KW#S{t0v5Fn(R2Vt^u)r z^VvoOrxT_0FZUbHn;Vp0R8Aj1hT|vP?@CSBe2d~_UAtP1F-R>hq%h;dyQHK1nw$`K z$i4f_&tV@RgMa1ocfdDn-v-p;u{StR&&>ysQ8Z8i9@HP^7Bpcv(9|5fo_yXC=bEIF z)fYBfGB#e33$ZeXSCz=N!}F3?v>7>9;(9j~zj)H$NNksvb?hK+ZvERXqjO$j#6hWGm zD$IzPK#3o&bM&2z5#3%ZM*EaNs5e4a@crjD5&hjfe;g&JdnXf8bH>Cq?`7ruIynQ4 zu>0a=Txgdl`0?r(k%nJj!F7c=w5ZQNL;2SE%=|_WiSe_ao`z~Bs@MSvC2R+y&ksAT z$0e5xs*7m%&Zk{8)pl=->Qup~_ZqRMLbLeBj2}QGrFLclgoyKmb|<0IAkdx0Yv(an zIZR^?D06H?R^cbRE;5Ld*+G8mY2cM6`w-c|Rq+&J>{pnO{Kqq~Oz@1p3=NSgS~ zS(jG1ob|fox_0_qz{FIuxTE_F0?q@k~wgb{L_lm;5_^6obUG}J}XG;@@=Q{0&d+QpSHt zYOP;dZRRF2fb%;}k?-#8hVhF?5QxtIC9E3A!01YSgXfg(LaGH*WgUfE#p?Xt{9l|* z^1eIfDZ0|SjPxrTsg8L~$1yssM%fz9K9o5pX`#ah7Buj1Y+^J9KnTa^M@B~u|J*DO zNX%;_fzBUY_l_1ppui7uGGo#u%D<#3YRg99K}J|9&k#CX5LIf_mpma+@1^tRHFpZ9 zi{@w)6BCo9MgpCOqI*5sXNOt0+Xh0IRS!L3$KO(7G5Ob97>d3z3!vOz6%A&~%}s`i zv9EXg11_KXJF};#$i=EEU;a3Mok*^9g!TJymmus>Nj<&;JZ_(^miMH7zrp$iZ;odf zn^xwFWOJx?9uo<5vDTh#6n`}a<3nS^D% zt+ziBuLau-tq>c&wzWOG-pn39-zx2Oj!$_m_xgZnd^NzYWqiW*6)$RyoSeXenC@)8 zockQA5URB9dwu>|5K8PpYFrVXcekHYyrk{slUg#2s(NoV9G^gP*X_wim_IX@%<$wk zQs{({WQJ^!?Xgl98z5H16v4N2_Ja zw-}x$_kJm~`YgedAq^t3ob+r8sw%Pqx)provjU@&J;SqkkDQ9U&3QSW$~zs;hvo`r z`d#e5XA6(W%p1f_^XYfU%@h}sRP*z9wJFB!7cUgNFV*>PTY~ue*q@IF-#RL`-?%XJ z5(Om3ft9W^7CKSSN*f9uuK%Lj$4DY;P;s>TF zDdO-23f6>(C~F-BOF@;V5+_XgfGGl>A8#txNvXo`0wusPAO=ULZ1zR_jt3qUM*L?L z3RWk|%RjNiA9BTsRNtDKF2F<0xR1n)b`z6Bq_G)BGZ0BmmN{;gxtyE%A*nD|pT9>D zi{EIKW>s@ANOS(JUF_1u0$Cg`E{?iLxvH00ogvqE7v)QGr)MPs9*W+fDnoz)GA8dd{N=47gE7c+8V8Uld&xph8A=STgf|qExCd_vhTk@ z==S-{7~>IjDuLkMElT%A>fGAIWetYgaKqgF21fpH(8OS~paP(jY}FN-mrw?R}U6NKX$6rN7Y-!_wqBXn?-5*dCd~*GmBL)Cd zBnzU6g^5zuMZM)xu+5dp(6+fc%fFRC53R)|bamUP+bUx?xsIRx6jyNDXP%duONL9l zXFg9*u+Av+-e4%g6df59D%_qsJVp-JjCHG*!MAv8WoPug~PWZU7;6M#4k4N z;J(>zd&lSGY(nSDN6a!=;BHXB;4@_jNfgkdQ=6oqtc2tVJ3boaK%x2`=csiAyyBE2 z1&!?Ayh~z(U6ec`a^6v&btHnTC?R=C$&e);JctW2*yVw{J_vF{?rZq6=ZITdSz1av zE_Y`E*pMVAdv7IQfEplI`#8)7@8RK1+9bV2wF)?m+9Y#=Q^g|~9@3fzDWwy{_CmU= z=z9$X<|Ke|6v2Q}{9X#-MiB~yqN1XTAN_;+ZE;9e)dj@q#?qJZP?I}gtODQx011XO z>FU{fd3iZso)OxR;geVYoS}n>e3;Mpaekq@H8H)Fj|3MK5Euyb=sWq_&ry4cHo$|h z@Hp#!=|dR5JpIeBic5%7iB~z{(mzP<)SrUnaGbpV;wh@FI#fg$QX^I_EbIY&=x zM@oU2HCbpz+WoJ3>;$Og^~v=AYXQ&!p3nC7*Wz79w7FI9OC068Uj!P)6qCq7eAHhP z)ZJ7G77 z`rW>NZ?KB|T0$>WIiD`lKt?Bxw%-9+f79%gT&JLAqGT>y6XP}n?nPixvhunsD$27K zshkYv6UV?zNgzq2+#61CPAz$&q^GAhG4)sKsWj@AF})GjL)&S6;&Jg;y1JJB9Ui7t zrSIhChW$IZEqufb<4e|?_P_5Q;^N{E2t-F`nL0}tAl35l;QojH5-Jct3Sxa5q$Tphyi8!q1UwpE5*3Rvci&Zk zyM1>4judWsFS;y3g|H(51AkN#&&TUSVnG85i9NNzDqtq8_0KYe3vjoAb#XlldNWks zynL8W54B>0zCs>L`01%Hh1E09I~U3X^XV)=kH6qL(M7!{0Zs=XiVw4BON{51;AFva zwF26Ny1Kdqy_0~}X8ogAP+4iGtQ?n;f&C1 zsx}^s0qxfM$b3M+RT^nf9LTugl*x(L{53S`F7};Ks3Y(JSb)&l?7xEoD*AqWbaZrj zT4hedt82PIikl13R7V`ZX#xj)AZc8EC*;Zkpf~^oayC}hPs1NSf^#UMxIZrAr?j?f zwj$JoRM9X%3Sai_K#o`Y0Hr{Q)R2%g=#l=vbL5`Mz4R#;Y&?F4Hy^uJ_1_hJgh32` z2Wo6=1VAF9Et$Fdr-*21zCePy)#U)2L7OBF)2FPVfy2bKW1D8%(0l1mZ;|0Y3gNMs zs3-X# zcz`n`O#(guniMwa6VVf+fLqdwrRHK|JC}>xa=J3vPvwr5-5}|^9Qh$c6lxS>_PvR4AWKyiF?XU1IeiT?P4a6R3Gf^`Ol0nB~D}OtFL0mBK zl#St0{R(+R7Cb@o&Z0(fLE_VO>vdr*+}V$(S1EMR%Z?*3@2#Yr+-x-z2wam9hIs4@mge1WDiQ|1U&3&hQ+3~lad48i-wVG+EH{FV( zVYd44zc3cNB&u-4-2+!9q@R)4n5Fc11?qyQQJFJyOp#0;Bt-b?iQ(6~ZwP#Qk1Pfh zNhH9>_$`&E?BS8%_;uT%jtZfBTektW@?*cF4TvN}FS0D^wW8`7;sz(r(WIR6-b!ZW zXD$2np3tqR=1#%yJLczer(kBeHb+sW;$NRFrZ#uVCJ9dF_!pu_8#Z~T7mrCJXa`&f=iBrDMSKKYnU1(d zGsG!A&a-jvOT2xAEn?3~+08{gw*D3p@jLN3p3e*!gdo}>L57&h zo&&hNpcGQg^e|t1V;lL-3qPj4?}1!{Y?IGTcvBKnJ`obPbFsw0jjHK`Zy6d`@}xnI zybv_Evo>z)*-OY8zc8ax_Va;K6&vpdGy|ilhY^R&{;MZePp^8aw}-@x5N)TyzWReH z-(2J%_#h^UqyGMWKR>_W;oRzvJOD5Q%mS8{ zmL@mgAm!Ku0BQzIXTR|m*MVoocK;oO#Kdn~y+DiI?JYcrDv`FqYW)kmF6Moxo@oO% z2kKeabQyvPpC+J$?jyUv)>!|_H#nV;~1(nWHSTnKsy36O@}(0VlyVx_OIpn*$$Md{d+$^ zG{eHmUS=n{O=ZJu2EFeLCoLN>O3=e$XS-BPdJVcJYmUU`biKM0x?MF^Per5y%mf|x z3o=MKBGk%|<3hs>AE3X$)N+)H&|+Qh!1Ek=ys^5Y^EkB6av3d1VUw}+v z9v3|{YCQ&bA#_y4%IJQW*#`o701DI$jEt;W?x${FS7m6aW+=LFqiixrDcFFP9)MY2 zTURHmpn&Dd_?tZB5djPatGv}AYyd?8&j(y*BqStYO9dSNZ79$#lvCv*??U_+-V@lb zC-MLZFpJ(_!c1A^N^R=CzU6YRYXPq9;3JA;h*mLRB8K?vJHel^_a0dst2?f_wB60OU4 zagKZ^-tAUiK@(>hA)ybk#hV!!QfAZ=ivI|UlLYXmCeC&HBmebg7;Lnt3VA~^I%El` zVChVgmt(ht_2Y#aS z3T#S3kt`XGw{pQjh7h?TB}8sfTVk?(>@m7Qi=Ha5uM3$sN7X%Ve__%59LZThm(BU~ z?=jpOjx}~nh_4b~^kw-&!GD}K->NR4f3QNQzQ)j;dQ{)n_&B4lyO?yZKaO{k?&zb# zJ+^3=VzHjKb_-B|Dk=hmXc#c7Gcy>ooT9&-4kiiN+3Pn1c61wn%cpA`(Kmib8tM>m zyFzHFGnUW-`PQof0f{5!N8Oefn<;140bqoH;rah=O|>xI@$YjXoOdnur^Jd`f`j78 z{S#vyKhTgZhFxYH2qGL<*fo3^Y=UlV6`{iIQHSzJ6Q;>_CMRKk}gMqn+uc^^LD8a;t zTzKO|h9$m(%ziX)DJ}Qcv)Fyj>$cx1lq^pB1mI=O6efbBom|EbnZQfRJfo&}Jou#- z5bIF|Tqr`?8&cBC@&G7QuK?y>;9Zi>!yXQqRz_NyIS@eb@tIVe5)3TqPxiD%!L!EHsCiEwr zL*6Bw4lDd~zrU=5J=1=&;4+R^ zq(m)$jfi0$uZ7myTJ6Hl?_7ssIw8d>qu}Q+m!j*FEtJt4{I*}>)CDIkoW>$%QIZ-P zLCRX6@1`ol)^tlCkX>2SPjmiy3BWZbVL!Hr_it4^64BGTuWOh`t!ru@Qm6Zs)5<)* zmXEqq#zBWQn#i_90J8sfSuKT#5kg>w>lW;CEiga>rv+VXU zFpmd#b1|y>Shqyj0zxK!c!oM5M zNBlC=`QjHRADm!c6MLHSMNdd;5fORTtYwX+)}eMIY`Rw=MY5id@(`=(UAX zRyi#=DIQyMfsH7GqFRQy_Qz3^BQrfFqM`d-I(3GnwO+LTnGHImZQsMeeRScC=u$LD6te9-3qJ7K+0=o&SxvTX zuOMkgl+gvjv$3TepPeavN4mSbbarK&ApGnh{WsU3iLCHIxGy@6Ev{()+dpn}&5D_!yi^fcL5%+0`G!1CrM29PUlGfz z)*93{`bd<65qGh>rFGXXm3vog%3X)FTK_Kk2!9)1tNZ|cxQh#&xLMIXoBvu!5&i9r zLg8sETXRY_rtYm(V-EpEfY4VQD6yw4X#)y%^vFZBIm>aIO~y=bh>gxDVfH0O>5;>hG-`_( zT_5BuqGiDjE)K4Yb+tv2fAv;Y-|=f7E1sX9H-vX|hugaHZ)VH2ep^g^Ctw9!Pe!Y< zk>4~dCw0q7Pr4ey%N5eP^g1WjCUf9J^~i(ba=$OsTKyf47cDOkj#trG)V;NNdP%MN z1OoPJX~ii#Ssr$+51PQ2{#q9y^Rh#|5``8%K$FwIQ{Ne?oBex2hUCA*vxY z@%q`+yB`VQKO#{Gal_Z0h+Tiwhl7N*pNuc$M9!tVgdblzxv2wzz)M8?cs$0GO4`Wrj- zp-!mMo^yo53$5KgD|K0%p4-&=xaF1DZNVSAnfl58`8hKQLZWX?M?HPlYF*_AKft7l z;eC~#L`yRGo~X);i>Vv<9`Pb-A8O#bn>S+*@nH_OiN|8gDh+p~sk=B)@?$bAIKf)r zsy$-zlq3SkMgQZ#k?B~FGHS}Fro-6JxOPV-g=&NEa-mjbC>Z_Us>SIhb;aO}%nz`+ zzi(r4kSoaEC2|7q#PhLz8Mmv|=*-0diNfLri?VWapUYv_L4wXVHh}za?HSMAqtQiSKTK{d zs!4k7!7pY>FFpw|kYDwKOubQbzDk2p92uL7mNAnTl_VtRNyB!~$Fx4i z!J!)Va{-EbdC1n9sj%@y^MFz&_d|<8oB7Ta;vOY^!;S z=vSwadFWx=`I?drwcD=8BCM*IlRXzw1d)J=+jDb3Us&gl7nrd*%T7UX$s6Aou|ceb zf9cpb>$*W?f*zLp?e;aK2vLkP+6p95+G?KFNa7MvmtFJz(_t%tSU6pu75WXxew;rWBOn<13-mP)|$- z^CvYi-5IlCf`jzx8|uHXMCT0ZJY2uN%1|^110P#oEAjP=s_R-&(WE3jAR&tW`2PN- z-qM=}+#T`pM~Al);}>CO`1G&V>3r%&0}!QL=4E$Q0Re&J>4e%=joJ*W%eL9ImtB%& z-<3Enc26fAG{x#1S}H0kNxizWXMcraNMf`TZ&XX8ZGO(4sFDXy$8W4M%w+Al)`-jv$sI(q!K*_l=g%uaT@VVD=-~h}q%r)y$CI?2$zU z3nwi+8{9o)SGY)3K#OFCL zTZEhc!Z|2LkUy?dG|J*RExoR9W=ncf0WJIa`N{!4y?$ewmO*sgtj2Vf*FFD(VG8r? zC9T2VZnLy5h8~i_US}g{jb+!!aPOsBm&=u1HbQ09f4ATqtMaE0pq4xR(ZLv=3lM?5 z0#zbJPFlY^P)_=q$<+a0lxdT!saPq8X~hH`iWs#Zcq(?-CdAC-I+41=&mCllfkBuM za5LJ?wek)`-@m#%GV(3GN??M$qXx$Bmr;FrNh!zR(C3~x>Y@gK$9njv%J&gehP%Dm zg;dtC?hQbRI0h-#J$u1fGW}+zD8bw1{}%EzTmmA}>4@OHK8iF7W?xiUr9{6AACncR z>F;HBe_qf7Cf;PH_E_*l)maULy^}p1QqRJqs#Ke&LroqFpUQyUJ_`nDxb(;xrY74A ztI<0DppJ+@d;dFhy<&YaBOO{7!`~cnojHDfQyxHP=e9{_T^8lky4|@2@0t+9A%aEj zHKx_(wn18r?D|1VnCCj{@ICjNm7nk3+e--|4VNb|m&V||GEB)AQ@Tw~PAsrtStzk1 zLO3=s0kW}S0283-;s;%K|BCRjqQ_sbLnlL%H<#7jZMLWA;rsKkiA?i=t6&D7JSs?} zcV(>mHMUxQSZx21D9nIWv+{vm<-Zq&tX60e0vc0%y=N9+*&7tP|M>g!t*uN8a zg3msGy_E1YT(@yT#QiwNEf(jkn^&imB2(krJH@i!9mQGZ?p}9tA^w^1kY69>K5xlm z@k(q|8=0(7PzMq(4BU+fwptvPTV+8aQ({{fKrRBtkAKt*8B5o`&BajC_Q6Tv0?N=l z?Yx$vgvsvrIRBxDrhH1V1sg?#b4LA$CCZsyX}r!U)}m2>>n6OlX|X8K?VJ6!@T$OK zrctHS?&^r_1mnJ_ocmeGZ-nkvn=*F`g`xyqc-_@gTOajFT2D+Wy z$>4v7q>Ri+Eb+RptE+^~r`l$soM(WL2JD3x7#PICG0HmH+CMx!KfHer>Uoy~nq~f8 zArEBmO8l$xBgBHNe*+#B4L@5T<<*BtV=I3w1jo^SgvtR3?o zY#vzOpZ_$iR(rMZO1s{_uh|DpnP;XJ9ud7zwEOiHVy2uPpPXBBEt0|j3)-qD zV6D=ead5&b)}oN-dG0E>B!OCCisj;SF(1k|5H%x){&(UTebgz5kelP~!;W)g5x_Em z_5DI(za;~=Al*~9T3(}2%B{CpiajqD(poY9z3m;_rbff+&&Rx?HPrM`J<{nE0|(ue zV0RqIeb&TT-v)P7;v7eG3A#c>cOl98;|ZwweDWp~%WKU36DOul8y)-3@9uG}%EuD! zDax+Pt*T%TX9~Z^oi$r*<^yiwBdV~pL6vGXWo6*3-3)jRmB`4*#KFH0fSet_$Mrv{ zcCXpP>-+m3goIfztB#Dr{tJR)hUE~h>I`lN4-b!sCfsfE#F3#GLKhboBm{)8sHo3C z9w`kdl>%^wf8X0mkiM3bv*;ovMU}N3zAhctY#I!_Z}7dfGsiaF%oISeiFQ~p?jgc$ zX`D|&QJ1Pt)1|Qb7;02nE)l*SkrOwyUS+yIAkA9!Ex<})2^|&;FGEEF^T`vm*s`z|oiwm!r&6jK@zP%>C zDKSM~O)an-x4gGL=xsUU#!F%LH*T2MipYIXeWx_s(uzX0A>^oohMFl)U?(zSq5=*X zvj}yFP+hK>)Xjr1ISn0(dsMf%c*zBvOul_OvgPEiT!og5)g&r?AoMx#KpxV)!}-6= z6mW~>GreyFD*tA-wwZBpe<1wS!J>GqCjBC(n3G?uI)yn*e66ax(nOJ)iBhv17=eMz zzyK6)0UWgiSg!dW@T>{a?D|L`KUc#1+ltW^2vXJyzeL^)?!7ct_U#`%cJK^#wFsab z<;{X?B7eO7-urAOVHZA+*Yz-j_nj9wy-)mH=B@pY{1`g%YCms+;2e9wckkw1-wr2E zzhkg~K-$x%^f_{OE4o0${*lLRIz9DVziR z!F_Ou;eBDzfHKHrMvIUG+^ran7<+oEXG0T(aIW&*Yp$rTl|W%J4w1p+g;93QrNYV` zR&Y_R_N#?^W+W(~do$1Fa0&%c`8^1PvvxzzJ^22qC!Sfm=dziTUI@3=GDdXl&(Z$I z+1l?VLEq1whCF+ce=V76nYhKr1X8u-_EbEDV!VQlx|kh@f(<>Mpkv-2-D=Kc%nzU#!-tk$bimo`=nxeGy$>4m(Nao{Sr?Q8=&g zM&XN=i(lx^h)>S_9y?k(Fr6UOA~)rPbduPk9z1>4!Xm1;XSZ5LbBB~|mkc1>6Z{(Q z`BD)zYdJLQU{-dp{EHaMJJc#&ffR&Z{gow|1!dmXxtJXApc<*#qVnp>UW&Ky)In#& zSexh4?9+QBMY9KjP3UVh3K&>L%{!i`az13gDyCCcc*ndwPgJzV*r~Z@jEv;3yAG`T zx3N;W9DE2d^X+-MB2c|{7s|$}`^kjTQyHAAqsKL9cG}R?P#=uk_~Lu~ zFcL5tF8J`i!kg)()s+bx^S+0WzwbRhvf|0p^1x3-WJ0kTSJI99r;SSn$3-0XV%?|X zH7Jo+`s5Lhk}59udT<;6clb>-v>PNqh#>wZPflKdITWi_s;;6H1Ofr9b-=Qz25?}s zS*-rgxGFf9aVsSxWmN`@=CdQl;g}WQs)d+O z#X@VD9n1~kzoFO+)XuV)-a(wdXF!f@_bd6=FJ?bybfRV4UY?PTq*ZmzH?-~L0nzHR zS77!R?Gf%X?5L$QtrY36b^l)r(6BS^aJ2TkzFXB#EbM7@XvIr1GEI25;cV+#$0h9{ zSB>NC1&ESdy@*U0Yr%Hz^yet6nl;OM>iu__35RXI=?jel*8DvgYH$Mo3UmezDC5{O#z zNcP{#T=34IM?sp@P;Ue3vy8J_ZJVk<9I zdDq_HM)=GRXhi5=5~&n0>?lhF85tQ(Q0uFrf{K-C0zaTJ(bFG2v^~}X`5tCwW)K1b zf)NWYAOj!``s2A|y-r`E^ptLGJS5wp`F*6j4`%6^?u>ZPl{`gxE@{3#frwAv8UY1l z78?CCE_qv6JCh-h-r`P9U-F*n%&P!*17wjv;2n@e%K5&H`-_glymc-ozVwg!8-Q;d z*@Q7i3akPq{jOTtdU|(ejIg5G+PDrtksJ3*6%gd#0VPMhFuBBVKky?>-oMLu#kR03 zPrWn^h;Vrwk`$fdy@9Brid#_jaZNm%ItuN8KhJ}uuc+!QM?Tfl*^=;A??7@eOm4)d zPZ;=iEvyzrio)^>{USE$u~db`UA#f?7}${-dV1xRm5fl)M+$>PPErz*p1wZVgHnv2 zZEbCDK+$)9A|szkJPrW10E?WZs;VlWocL}0R^o72YDGvyM1+CC=Kc8H)YR15`^CRC zqR$dNg{nMlN|`-JB|c5p`M^5V=DRmhe%_PDn3&0VQ+`!#T^FmOD3b^vY6ae&NUcP~ zZV^>g#gw*YDy#>0RO!UpHgv~#NcRrZW3}kVs8`Rm)RD7XPQ1NOm1)@qZ~G|OE!@Qq ze0E@OH#et_f)c&KzSD;GU)9s7uQ8W`imJ+>v+a%X5(k~6#vy@hdut2e`h9hEJ@yNK z3?Z^@1j^ulUH`f`I`*W#$N;w-@YrB{Y$YS#(~#`|v2*1KF;@vO$9Us`L?(M6B{u@* zuxhrW*kM~+MjK`|f_U{Zzz8h#v`a<}N^Adb&bJ^JxIsVl-z>IuW+Wlp?^~(^bsC?WkS8OWA%|N<+hE1z6;Q~a1c)LTfDe>_S_fCREv6HV! z@$p~fzfIbNQV;{gTRw{tQwP*8;MNNVws53b4TZqbZ_Y2yV;^Yk|61%A_ZlmG{xiCi`#Tj##DGYEQZAsj(9?@7FZIR%>S-oxw9LAuT8{oLKQ72>Yptfqy0iEJ z<7f#zw09uD8mdq=IyyR~tsNjcNJ>fq5%x}t&D>M?4}A8h<(#kj=?v8}DT{yC^8uS0 z2fDE@bSqU>s%qDlc7z9D#?ULW7nI29o6_k^#TFd!457fD-i;1YZORX&p44b%G%K2} z9h(D{CLvO*ykzxu%Q4JnY+hnn*wp&&c^KlAO<+WfI_RyV4Fs00UX*v$&%?WVIoaPX z4@}zZEQ3I~)-OlF>`1{Zgn z{Qh^MK#bKtc(hn8q4!Y~Od6%P3@E5JeZF~{ja^0r)!$7WBi-ISP8xe0Fo#D)U0+_- zH8cp`iUE&(3bvy4D8f~h`VeWgzc6BHVZi`_<6IfdKC>! z)smg1v^j5yo0^&c4UGpdnt?X6o1?{-=jZY9aXw!eN5^wOTND)?ZEj`;7}8>6U?ilc z>uPCHX9j%2U_=ew{reYyI$;qJy&iVoYAEGrXH;D&!?^+N3C(?CEf04E3F=Zd$fT}z z!W|>g4@Vu)mRhk7FvzMIhek|F4Tw1U!bEy&ulQ@1&&9SiP?G#RnuLnFn2E)G_o=KH z<{f%ji@HXiv+?=&z&v=29wT1W)RepgFi!qkZp7|s{U5ROZZXK^Q1Io5!&%!?;#4$< zFOnihP8|zgzLgoC#3|3c{D}ujI=D~!zfKTTxLF6!-Jz z^&w1#ULv-W#QL{a3U6;COO|EM%4`P6b8`;n#L~B+wDz>Js~`SXVo#gpv?lwtK;)}G z^T(R|+b|78(@mk>7Th;EV&?vN8m5JkR!R#GKzLDivrD~osezNXETsykOB&9m*fv2R zb?Q@&rqwji84GXKqfno8?p|iY5Eq9L;=;tuwuS z-lUo91Rli?rQEG|c$ezS$-h%STE1jjanMaz%uiEr;k4LIwH}`Lo|SOmCs{NDCRx_0 z&9N11x<8Uf;Ou2aH+_&wk*3)AyTBI@pRuqgDJjQBM?t`96u1;l?BqQFq};{9?||u# zkL-(IFl+lAa1|H-A5C8!5Y_j*y?`i)NP~iuAR!$~NQ0DgOM`SvcS%Zzba!`2cXu~P zcXz*s&-eY^|8`k+?>#YR=9%Z2(I_3Wai?}&aiDW5W>;)7o42=}I|$roI3W2_vwms56>Lv_C{E+#)Vkj6 zF{bT%irGJ?^bV&%UbRtCAmKC8NQ>nC;93oTMT!*Zqp{tVoqpbCTZJ-?Vtz=Rff`Pe z+ZDWoL_GCV7M>VV6geKA3XQVX(BoZ;W~d3sa|h(_3;H5s-E9Yo9^GLF9<1(8@vG{zl1-$i&@Kk@`#@2Zg zjNWj<)8Gb_B+WS66)aNN<`yQ2cXQ}Ewgp^GNU*2Ho8+uhW2C;&JBmE zwl|cUY)RDm2rCQ9^#A*VAu-+NcvKZ*Awq)f`@or}j$yB3w1mWhdJ?O7lcZA*`%0n) zuL$zItz_|`q0Md|aE8g8B9zGS&w^NDc?7w)mP-R|3pjHCsF8JlxlvC^+*?>-mi?)K z(uqbmGAaz8Yu+{OG${jTH4rcL%UZM2A7WC7LwnGdaxbHc!4GxRB0Zn0YZ;%|YoUb; zZxnU{rpLM>Un+4IO(?CDYMl?1F{2yW$X7s6Ns22fDnN^6f3DivTCSh?qz(x| z4oTMlHvTJoTCxkp_ zWL=t%@3IQsN0kMna&;*lLiP0rdD4aNogMPID0QF|4; zGZ&kBBsta_Zok(B*|w|%V}m5Dxsk?~o;OKsMmBeQr?Sky^{aF}+3zN@*F+Vv^t^Jm z`)lO0zF550#B;X3wy+=S8f(BYEgHqnjxUGqCyJkLtNz0D5#39Y|0XyrHRlckuSe6JaBf-*i4{}kUlR>|z~@UgiP&BOiJ8p^~@8JTC8 zn#RAY`{ew!V}nbc;i*d{`0L01UU#jXA|NoRo&0U^s6VBU1D;tCIwDOxN&gfmrqevU z^p%Pg!dwny*r?siM~M|YD-+Q^6*N`;*+0yYsaWA`y0$5*o2hFmOZnv#upu6#98T4D z{9Ko552|ir<^+b%264SH_<0~`;466yB+w$wxQ4r=qdRB@#SHeu`TYuGkZ-4~~VHI2hl zy{r`tI@Dcsz+4$)RkFP+Q99*Wi>I`qxCZed!F+yBP7XU zf!M(5?0#|s(d{j^$rFqS$2zCk3aFLATu~EYu2n87q$9F(VtG7_E9RHe91FwJ{JTUHXwWI%%6&Gd%v1x1NrIxpHx-?o>a%<68|Amr#$Fz( z$xoHCnqpPzDWqj=$xHYyN$pnTJQ6lEkNi35d@`_1CrBH-^)(a>-?o{}=KM|*j-&sG>XA6kp9BRBv0c<+96 zzkCRAuKM>yjs||Gd>LxbV|5u?vlmm49EcBQ;rKSLq%Q=|=4OG(2NR^LZvdSbDj0 zP$J+Ld?49HuNn;n=4(MsP4{+Rce!yTpP<7uBDy-_P`s_*RJ3b;YK1dFsFA&v*4gdWI3-`SL4RkXE9&EUZF~%0Nu{rVg=16 z$I;VcTIw~qIu*qauF!|Yt7!qA*nH#JhssSnf7e8HuUXtQ^Y^Ik0}T2|(ZK7!t#h9{ zDrs=XG=T0Y(Cd>-sEYO!6igQ)QikC8zwFV zh{tZw-G_R!<{=n`opQWU!J&G(!VlvObA)BV!V%OC>mH@9&Tz-Q1mL+=N9mo{z3n4 zbaXU1IT;pEMl*phL2OQ)FaiI=^CfPIl3O;5a6uojAA3ZHW!rGw1`*CQN%v%?r0fM3 zh-t>htyepKx5YKWq5)+J6C)#d{NFIgFDxvM4(4mtQXnhg3X@ZE@OBz+3trCA`amEZ z3y(&rHJJ*Zr^Yk$Br5N-Ia}#an0+lP^-Cu5c>tFeCmAoG1b+#WV>7h`9RgvF-fT?X zV#lZ-CYq3dqg6R;k6pw+jXxi4RufTY${uvvOuH$sbUuy$br1Ws@XT{ZMHoUJHp8u>5Z zuS%Hdl92Py;&I^rZITIgdR>(0$CDZ+uW0J`-oM|?jPFPkuioWUI9zNlk;d0InbghU z{BZ{Bc+q~Ar$})2YU=8{w^N%($g|(!{t{$2!MXwQQsr#QN^=JY0xa5&K^%P(dAG`F zwL9MQxNo9ecSsDj0RLj^K=01)@%(}ULJqru!9f}#B9AM3Y=}b&=P%&6e>DTNFwio> z(8Hx>M^AoWQHiEu+j5HMI8>X5E-w8O5@aEwn-%@2S-$eBR0c0yJN30?=vW*yvMVC6 z82?K&gTAp64VFxXreNRLZ|@@nR%VNSeQ7RRD`Toz@L3A1-a z97HX#iyQDNcb@_9Gat1T=eK!wC*X0TK2{6*vc#?MA595 zX`(mUc!ak8##RB)v4z#qasXV#1BP!TM9Uwv6!*DIYocM|pm;y5x$AObU?1r`q+0WE ze}+nrU(HvIq?CJ6Ut7^YLQw9)8u=324 z!@hUNhDAk9_J+en0^tB{;bcOXZWY;}GPlvt>05))$m#gT6WP%&ySuDPYyv%+b$q?D z4O({{6j8;kgqA-RKe@knzd*{kAir*iT|8W0ymgW`Peh6#>~Y+pvLCb(oQ5|= zYS6`ZSBay}Dsw@kRh)Z$XXia8=2uf^M@M{4htjjP@R6S0ULeDnNC8Q5APV|SLD4fX zAU;u*s_*9J22j7)0uTle!&a+gn2&0dqC@5ew5O5_>#9?WMO=d0t4+LL;$mT8Nl6Vq zo*b!*iwD&NTR##5naTqk_?N>mjndN$vuQ?VWEc&!d4P34%bFTN4p}C~V}!iLF!%+p z^`R^~+?M3qFd3|4Mglhjr*;SD)`M0y<2;2l2CC@Kf`ufpfVOsy` z&HhWOi?ry5Q_yEH{y%-(_0(6;)02T%vMX<54 z!8>WiZEsf|4h97VodPJly?~3loSrYJ3QHkbor-C9a)5W3`G2y)t^3cH@Afs+gM47r^6OBV zhk{)v4f0`OX&fd)H-HWrTOdOh78?u1g_OIWrB#8I1{WeCA{!|>SOmPjqjf16qlR#$ zpVH`Z_NFP_4a657$0v`TBnN5MV6(hL`UfT0Qh1@aG6zhnwB$^y*EfdqH+<@W8_&Fl zjHsX~d-<3AZ3@!TEhz;xCEqg(*u}jsh#*=~iuV^5YJhD4DXpv&7Zq*&?E>}(@^^qX zKTWWXLt`Rc4#)g0o>;$Ze!PDqE%1kPI+~BJEnFX2o~rd}i}%5mGNj$>uH>9-ar$Z6 zt~QJsT{JA?yeM?{KFepzd2Kg@_YExb#%hh!^?YM+}82JacmY>tFjff_`X=xFaog}r} z@X+C439^q)c_Em;9S2eyp7)Lz5Eob1|8fz}8=MpuJVEp1)fhD9E(I0PhE%ZVU>)b+s*Fym-)Ay9lJyE){mAa zxN<&o10zw^5g%6RSPwQBCI2Sd#&Ju+;1z}YV_{+*cPZ%6xJBaw zqa7a~fIOXh)JBQ!i~@3tKxg#4$@u|8(Un@o`s~Mk(%a?rWx=w`<*T9`PsY5>-c84* z=IAPCWIgklFW>8lf5HDxr`E&M@kjM?#$GJU-3s30a*jmxzRT6+o^n^!Y(^`g|lGGK_Tmw?IK$c3R2oji|1*>me>s9p)rrxm| zYOU*g^@k2J#6Hs*3iZjET&gLfFpzkm6Is15KLf4hl(T7>eq&phD=eg0NbTHTh_h2M z(^t0Y?NQU9v+{h=T1L)U_VPuBHP>&u5%xVZdiNu&r_tpmVrHi42ioTDk}oyX5GeCG zUA!P3y}Catgwwx>5Z`dK@cx2Wwma^=~x+Wf}yp~pX6^j5Ix}=89)2IDKXHNN>r&q@{74w3$ zQJ7x3i)O#ozR9HVm!At(Rnp=e*tMGD$Qu_#73Efqnj!uE(Z7Kcg_^Ec|K_7gOqe%c zJd2P{Rg#-~B15g6Li@QJcT?!m+CUuBR)fJ>?6iyd>8{Y*C`*$~IQj_lt(tH;DPeB$ zAjw*{CkB62a)ZNGwZWK$g#{q-&oi^PzXFNl)9U2MFk2iKx!|2)%?1ac@D23a(U_B# zm+AiqyubWE%WLm<0~p(0l$w(ZZ6^HO#M0T!&SVrHc!Md-->;dbT9J4ECJ~{C#Xg)R zIJG*9d7}f-u%t&)GAl0D>Wc%DwVdqxpvALC9C?cJcg1iC|M7Z8M zTS`U+lQ_=>)Orabw+^$fZ6}jnk-vgSOT>7$7bmBt;^X2@@Y`!dZb?E~v+jK@{ zmiYg;04xd9RYd1PzTXR=kg(AB*9qN2nnn4%5QLC$Jk~eyNOUnSP=hueBKGiYPOVeY zvcm)1lzu}hUV@CK{4lwpK7sOb#8By#u5ny zwBKVnYRSv9+IVb>XidgoF&MtJiufvaQZC^^PzL);zzYeVk z1yvc{AJ~O#=(%iWU!s8#1|m#f>2Gc+?XnWw~tRUQAkc!<&-Kao)K3|~Gnr(NO$MLRd^yI_jQ9z%r(G0&%jQmSTk#%$- zpZ3ojlZYt$AN>O3vB^)O1-TbBJ-)lJFD2CVAhyU5Bk0hqpFu~g82%PKh6d>0Rs`~8XV7)3dTR5%cpurRKi$CyGG{! zyNq38{< zfkOc%swWj<0W}@ZczQ^0{!pO!kD;=Z;xGx@LtcMk}B|oOi$cCoECs9j+>g<}~ zA*nPxE5*@!@|OpyMybaAy(`<$p3`fZWE}SJ5-IQ9fIstK8$ezjO^5%@;w0f7a2ELP zcvqj2QQtbp)X>PeZ-SO|k@t8el;9f`2^mRTF-u)VhYQ={vB)?2PDwGOqR*J0QJ5TC zEP<&Z>GKW;3+^aLpWnC-R)A7;6XVD2Im6R}4!zwl9@ipAMx06#lB~Ksx7^dEO@V~N zNB}KHFw^~3Fg9w~>5w5B^!jAtZM!^9l;hIMC!NPFT?&aHd1iVhPjbC0VR6Y+$xpGqR9<$=|6vb z`&ty7S+{;uYt}HVntWP0e09G6Z_$otjc_#hV+t52b{c!}*NrO2H{8zlPy52j*ZGHj(J0Wd ze$ZwjDX__+qD@gO_+?Of`K?=gk!NAO(-+3jOARWP@WN9(Wx)}4 zF`07){ql3UPOdf$DbXGz9DHxbB3vo#>jJA|OM}wCKr&M6(u4`M6&;0QYr;KR3>aCo z-{9Kk8$*cRvr0K7yaueEcaDT8oPVUJ`I5>w-2%gnzOy5elF+%OME>!p6T5efphjkj zLX)u$(V~}CpWwy-Wm9Y-3Fz;Wi#c%fseT4A^Ou~Ro151Sf1VDnVVk`U!y6b6IwhwQw@#YOW~*wX@~u53cb zUqswh{8Wo-SqX%H%6hFul}hlo9T!p>+n`hO%XEl$W+EA~MlljRgtz>g)ECN<;prKd zjm|UjodhE~z*47Yw4+^+XVjU`!WkCh_Y-!Dq0SY;d#kJ775(WmOFh$yqlx2YEjn7{ zQQ`-~&KPs~a)e1NN(H?%F-T?O7-17Q-~?aWKkmY`5AVbm4spv>t=>7pE&U5ceIkmB z^FukJjIrbSFeVvl6ru`=Ihtq`8^DCxTv5%jU}#+U<%RZF2b$;vo%MozVQ+P!d1L~i_Aeky6I);ueu?+s-6E!&QdK?K7wDG%Cb zdVUwnMxijLd~-WzK|`KXx{E$Q>o{$rlKJ^fp=!)WId<2E-hhQ485iB+m~V1Q4=bQW z!BO@0AG9W;6arSfn&T^R5>XO#JhCh>nBP? zqgMO7EbZwy03*|@NXacnxfMHu28j0%? z7g;=#Oecm zxlo^C>7k_bpF{0j&boPPsAlpo&Z>6LqDo1ZJO>uSNg0ZQ@cBmklmJv5sy~sF?pn*=XPQkd zrkHf18M%TLS9%q{`Za<8GkUyDIkJ{X433HQQ8+EXemfP zz9rxkkwAY70l2f5R{l=_01#7f8No7lCz!F}yKUUoQrXvqmXj$%;XOH~f+wnWSa9Q9 z65WZ%ym7Z&4u421ZfzT~oS6$=)iU*V-zcH0Vb29kTs%<5%yuIOrOrCl9Yiia~HU{SgAsKq8G+r zBe^!vPVrhd#H~d9XPNMsUB=)4G>aOXCb3m7yTX1RsbqSU3?Aapelw?vb&iO||c>RF~N~LdFM1d+} zL)MGuE7PfnT4GAilmRO>`FzO1Yc?{TsloQYKSf_Q%Cjls>VKZRszP05ZO7E7IUXZG za@s>=SswrK`us+T4{F@vgqDw3X=D+&AfaRJX3RB_Gg{ALXL7IqQQep zG+XZ~c`ORYUhqNqmD}zQEplP1JmDIX!G&j*v zKDRJ5tl!g`=*gfa8Skb_Sl>gTOg}qg0<7Y9L(=lLuD%$K`&#Rl`2yF94<;`v=c&y9 z+f%2C@}D|-VyK?yO8VeVFFA{f$@oh*_`J060vO&22MaUNTtwt|d6cySv{K;M=cv3= z`2-zTebCSuolX&3aImmb-bSZqVA{6M`^j%B*mqzze)26_4#QY5hdU z8dQd*arRUMYp0P2$co4z039Cy(X#)KG%R3sZ2m~Il8Soo2?t>9r4#hg3U(P7zY zy*FFf`8#WSm5_S&N%!C&tfQ~;D@vAN$lVrhPFDftMIvr+RKn$!Laf%XBb#`yAsGZB zBA!6qaHEf`QyU@M-0abal>O_00yUv-%nCg&!Twdpe%4n6qOZ2t58_ik-Eyv-c}(0E zg$DF>L492t6CbGTGuczi$dRwfuA5Pr>p*gS-?a}KDutmeLgd$H)I*QyXi9X?)~ zM_9ZHDvz;am35ns3;Eq!UcyPkza4cv{ptj^ps5g{uGsa#@=dLpgyO>A?3%XO7Xf{{ z9Q9Ew1_XW3*)j+I_6&2Hfuy|vB!pZJ_)U6^;-Ib`jvX^nBE5A-&H|Bq>z~dsYr(K* zxwaGEFZ2X;KtO(y?QxAe8-e#EHw?|i+Bc9yH>-DN&nurknP3dY_|q|ao@>3uYy8&V zNQAThZmn;U1%2*NQ!+o3GW3&JvG*32HWa!5w zN*^AMdz}Yd5gKk#j~~{7h zhpWuzSZ}*(`U>;IJ7bA(f$JNbCaKlwy}5g$yzqwU?;pN*YMU%t7q*RS?)#jU`izn! z+H^oXbk&x2Kjb*H@HC3+rG2x~elRxMSfQyk3fHpQ6`*zS9U(eyC~MlheNerzx>Z?C zf$JUMbWDDnJVl_9(Sn1tROA#rMG(!nv9i+TC=5**+AKbhdGslW?M7RuR$!ywr=9Np z=237j{SHX!Pzj&G7Lh*D_#fu1_VR1+FX588j2~iNcx=|Y-%o;!c>K2-51K+I_sc~8 z8UMTgiSJT;LP)heU*1*J47_$84@< zb^@(%vpmr(8GiHy3<^!O4%`$tQ@gvGOPG7{JO$yD#|Z_!Vcw&qwix9`JA zbWEI1$Yhbqva(|{Gd+9!OG+*-oF!uKAmcewfatwqiv@XCghp2hJR2PLI>Ok;6WImS zfk{ZqfP^!%C5A-zgtt(UozMa|fv{@5$?Et3IGe5hxv-9ExY?x`#EI4&#I!-y-2+%`&(Y) z!HLvQ66Z786;{5aabl60>zD3J)_E=BupXD?>*`YzxGE7y(FB7iY8zFb^}^TOkkrNc zmoOsG+KiD%K6VMGuvd(oEPR8YvMw~rxEIe$?RDJUSubd|PVk!K-{5+nhRY>ax_FMm zh$(ABhI))o9EC&KIQoNOCC3!2Tx*A>446MfvCoyOehCytK<0kJ)Ypgp@-Cv5vFT_X zsBxI8F7;O(ud9UGGjD#lKM4vtI$S=LFrZ~+i*uZw)%x|WrFoLISofEATEo|p!+*Ma z2L)xpL}hj%z6hhPIo@eFQ7fYbrL+67vG2&q$u)2o=}k>d9n{-efW+$j`1qe=OBYY> z_8Q)-tH|XmR%UA`WL9l3^F4w)EpcQxinSehOdG zUtGu#=H^ZJ{tr2<*Y!r!T9gbLV70tf4u=@lCZWf5Yq^~9%N<=>d z0b9^~rzR^guPiY?+gOrg6P-O4W4A2ozf0Y0pLZ5MblPlN$4Sm{)HX`n{kreV5rWOq z$sFwY)WL}AReZaKDm`(2#XW(mh+Pb*up0XmYDT{<8LgP2ewpIL8_FpaW2g34!;jRYf9vETv6Qy^EiLE zE-5qE-Q2J2^qeYb*RKGDEdj&CK^jVmh11lLPDlPHMOe=&H22iZmXe;64QnGM)|e?IH5NK%!) zk`2r#%PyHn#=uJq@ zK5lUee3-AX*xKB^cpI2gKy^}0@B$L#7dj0Ova3mwPn!uA5s{Id9S$rFXyCyN1Xz{e zZ0G|lCfnYC^G!`sF2Hi!^x-qUHl%#DgemV$e9Q-fVhe3bEL&WFV|c#M78+YBl7bsj zrNRL4a4};(cLr*QH{QEbYdvqzigdPIxtJa?aaYFn*6;CA3D~`1%b&dD;7K|(d+K_4 z9?QyqskY$GBGWVwXv>^cokc)^z!3k z6AMYf&Tr*l`kK9#9ruEvt|_`oCe(D5woi2gON!xkESD?;w!1n*OcR2jjw&xNq{{VE zhefvUXNx^PwP@e*7^mB<2X(%j@r?9zhigy5=2jsW6N;_f%hOT|UDLN}HVhi?6u~C!yw%FG;xOr(tb`69Yne{aMYg=I$(gvt6t@X zlso;XW07%S$NHfr|E7T{Hr>}m%1lwpmLYk|HUDczM{n%BDs`8j;1Y8oCi!b2fA`Y3 ztnd@(k~cM~d80>d&R_`qcQY4#Ms!VLe9*((s7Y(ngZKAom*&PU33-Lr4vgP?$l+sY|#+XY9`_a4?V`6^gfqrsqop_f`6(yAfu1>ZVQQ*ZYSWke<0zI%}UgRH>!&e@b$40%qTF zCAWZHf%TJeUdg5;Pl_!lKs!A(b!yDb6g1_Jf_$O|a619$yx{=wYznOjx*;{Nh1gSo$}H z(e)qZ_eq)3cx`7qry(Ea&hLaB4-XH6SfIC#XQpg1!e89zMEK_u%tIh*;69d~KkdLeM>442Dx?(J*p@9>;1|N2S=kPuw z2l-e~O;JOMfgTWJ?tS-O{U{dl*X-@-ad2{yO*(P;Me?wK-yyW-edz^zw;XZ-lji8I z1n~gU_v~!45~^qX!~qy6_p#~cYs(f3E~BC0ULqa3Y3zNSP*6aHRU}24nIs~DbZ(Os zOxVAd&TR{8_yLw8$jaw1a4pw;e<<1Z)1Po6b=q%Nn*OK>MZO;IEKAk5Aa)+cWoxg7tU%D zb%H5=wnZac4~HI@oz10_xX5OJ&T*SbK}7{R#5g-REHt|}mz9N2Z!~a#SurM|r46Zx zYCr)si|Ly0Bb566KAt}U(o|7lAv!ubpg90d;O0(iC{ldze19HUQ7lnaG- zrS&nx-hyH9pKeNZ zQ>F<^&2rkPNa${I^Ge?vZejrGWRM@)v6`srit&&8G#|~|9h=6#nMIo~1|~~d({ zdC`iBmzS4LO!^3jh$nT$QYr#EI^yEue0(q$$ACc#Qd$i(I|4rdcnB+fFioqq13-|i6gon8P@bRbfm0lg5`z&FE9+P8uG*RoUCWo5& zl7Yp22)FvXqGhx$qop>rL8^B-0Af4A;NxorTt6O{>oDW`5WqGI2w1h1+0FuDNMT`N z4W$#}@aKdG>U0?1)%zBAWc#tJ584*TPukWoLOSI)hdbLmib9wkm#gj8U7?!v?r0qS zW1By5A&|oI+UnO`L(4+8MC8iQnoz%_1(&jy9Yv8k2XB0G3%U7>ei4=Kjq9wbWpqe( z%)Nho(YK~+VmoPAizO$~Fp%DW&eECR#X*4x$2`kt!Pl!=6E>v6CxJ0yyAA9H}NkBG52NVwhyiQPja6t;kkiN9M{Pg^M{fQEN zcAy?BG>v^}fSv$i82I7JnI=Lm26dD3K~uoIVbAB}t`YG68Qa`vns(#W0bqjB(SXD% z-ws~qw^76;fPxU`Jn8n28HA^D|LX8*;$#HJe!YMc9J*m_#|>fL-{si0K01cN=qvI# zy=_c~Dy!n&EnoAtYRfwL63nr(2?wOX9qDVpv{SwK4l2};sj{`QN4=Ypx*@B#M-I|q;)o>@({yAvoX^Erst7u_f3mUH9d za#B*E7F)Q$_;%y~3$eJcpr)oq!0FJkr?M0S_+9q(z>xVxHH6A!2%QWj;+A~<52xF# zW`6=p=;-g26J;)H=f;E3&vUiQ@RCt(t z^N80RJX(ix^gc&HULFhukD*Tr^}rF#jkj1kboEAL8rRr?bn9+|C^L{<}pq z`oq1&DPqPu^LtqujgJNkNwq3penm}}xAT!GOT3$j>P8_|gP+Vuvi##RAwR=u#|XW7 z4}tub!5yv?HqCw;>1EkfErh$CWQ`Si7dO1`sPhm>$F2vwIWo`aT1|5O#4?34LsTdW z-$Ru8|8W7(U;fLqk!Fs<-amB`6g6r^`z9px8oJ~8Gv%$Og(3zlL?%g)2OKQq%2vbi z!uS8ysqds5*u@T&J0J9@yn?#q$r#_QGS^z(v$)}9Kwow3w9)~NZGCX9Q>s>?5TOe2 z8Z9j?U0q!zB#=u?-Ul@_G&rckK8=izM}~z-%gL?ny$?m@zxR%BH~lr{J@kR^0tJUv zC6(+ADan=9s4vk&=-j1zTehJc4>97-VG*?S4pwW{;`fLTAx*Z$za}8bduNsKmkc? zF%wfp^3-YZHC;2K{eDH8UH<&}tv*ES;a>fvBEv!!&4WU+7iJ3fRJa8O?8(U-$wYfr zv@yR{F)5nfUC%Dqx-c)?Mutqi%kiTam^?yUW4=WIBs6d&0`;APdXLuhg_r=dbRoEa z&Yqf@UWdf{B{Ux&AU^LQ&)|o-dqrVIbMFUY*tuNo89(TUEqpF!LfG-to5!UyM(6!= z!4*LgNK2K%a_G!~UQ?G^ycQD-B$>+$ARVoj^|g-dMOM{98D}L}j2%$yTtqm|0_wj} zFGGoUgbq=#`<;WTp0~_>&iN?=aUqL+;980j$>TsQCta9vKLIPBz=yFzd8JZvlVFEG z4ySENg8h*=BN~#X#nVLv?9T^W91|0}f%8g?>+D1CoGJ2@N$1=XJ=*Q#3*RyD*`fuu7uMT^>QNn0HkZk z^8O1MzcU(it9m38e)F|VsJ>reEUVOIv-pN+KMosy=^w0@5F(>*pRFtPYWB-nFrPj^ zctr*m?cclFUKg^z@8^mxQfC`izPC1{s!Nm?WwKB0(h(CGUB{_?NV!srD(0&5H0=$Z z`i>8Z1}BVtAIi0WpeCJFRawi5(9{gqhW)CDfi6 z>0DL;d(=Alj3nxQ6EYD+b|#z7Rf)-T-pTxD`;@PhU2KD_^h(WGLF3ybR1_35Wje30 zjQ-|2=l%NCc73we+1Xi!75))OpUeO!=vPhDu^*@`0imF!rKO=caOws%f1a3A#RiI_Yp|%!oO!+kcm(Tgh2fLgE=_ z_x7#jVYNwY#738$*pOH8{BsWQY)gxxDHyEW8XZlNSkthh^r$!1c|@$Vf{g z);BswlfI(eeF8laWTmB1v9LN1Opgu^3DD3icPA(tI@6Q@PZ$vaVf$p`pY*hC^)Yex z@bK`|6t=tBC~BM1DDE0FVY}MHYv4^-l+Bu=poE5nq3)&?7ata@*8@k!K^5!vj}hq6 zV{B|bS7`)}6#16y@g30aeRg*CzFCOY$(hUihhf+;kkX%={0HFPEC>t<4El{M$PGhw(ua ze!obyW_Nr0-|h898pvBFx(FYW1B1^s9I0;|I)sp(c6VSQ;9fyb6LWJ~H`~CM&3yP^7&4?V zx{R1fL7x+?Vj%&a)Ksl$n+NdBpUBB|NZ$P#B1DAHCuMccf0IV}MdHihruP-9mXnor z!l~j6LLh5a%;@ED>BnC+ovBOZrvZ7$z*uWc(r>3j!Ok{nV+;0;21^)VUB(==v~NLC z;`>}1_3z(Ba>jRTZUN8l{^7w<9dni8a87;?A6tj#m`_B|9F^u99TkAj8Y+0bFxC zBw<5(3_O~FJ#>xonwp%c*n8;d!dFpCg968E+X13*yT+Eb#h5wJTxMPS6KCN6ed#Kk zy1~P2r4a^*1GkGnT=VMHE4h?Vpdzq5TwvEw0-Ru6P>5eLnN##)Mg<4eYWK&c+5zSG zWR!#YGlzQdK<@UW3vw_F!UgHX-JPAEK7RZq znL|!WiiwG-)ehy~Mh5k)t1By|mj^chCUU_m-wO--r0j5-W}BrA{11b9|UZmdBg^=O!wHrCb|@yS4ZK(9jLy_A@cFkfjj z4hstlqAmw@q*V~aEP^-Me2^Ro@W6Km1_r)b$K9$hZxYSzVhg^0bCI=aqMi7NK)~%H zV)k>3i0_D+gv4v^r;SaAGZZ%+)XvI9CL~O;-J4pGV&vZ*+}V4+K+q2u{w4c>W-igNUS6C;VXk#Z8+%(av>po)Ip8M(>;3WQC}E#4 za~w}`mnidd$3&A2mo=@qM;w3Z`>>WrY~!CEDijhSG*aua^*G1gy~hzIJ0+Zh(?1Ka z8*VzFoyfrjeo<9rC9noyErY6eetyDWYB<)l|F@u=q(vnpE>2F;s%sxCM>q=d^V3-X zxXyi=HEViVXSRW-{WEb?HkkgaQ%dpa8-XLrjJom>Na_DSRT;5JNi7>qq3OZ;fc6W84 zTA`18LC?YxU31ZZ1$I9`E3UEshb;7AkRm`%86HBflZJyHuOHlf`4@(lo}hnpZwjsO zLD^Gu7Y2&Y&B-oytd`3??0bspg`RtUlLj+E6N@-QJG_tQhrwsGNM=I2b1rCV6R=6j z>N2t_sNetNsaRfKoi~&{_QuF!1UFUNVGqx{5Rs%v*fJSxEWcQbDNEFyT=*UO=8^xI z+EyU?Q_=0rJ?XCGuJD$oVN+A_0xQPn8Bhuz%;Mv4akWzYQji{SUy`=&^rT00F1l^+ z`}2P&SN@*5$bZ+*pQWCjFf4q)|9u8NKt066*$Th<{kVdHg1|r|i-J{@i;Ihimo-cW z!jaKkap;DF@lX=skoz7b;gZd2y8}>&0H&q8+n_(1rh#@#;yo^|tUV%GBztAHwtyGG ze<(+b{|o~$EJQcP`TMIm(VgC)U6kyJ3@c36m*=% z?CrAUg3QdUV}dCym%}lCMQ6ZTOw((#dJG%AAbRPdReJZTU(#-c7rRVKFuWys;V(uB ziu85mhV-fzaLXwQY~W zxYgieAyV8fzC4$VAMRSq`F~`+Wmr_*`vy9I5=sb2cL>tbB`w`uQqnD*LkNOMN{DoK zNyAVA0@B?uAl*neoW=V)|MTVSZ*$GeHM90!@x&b|mgE-iml{@=t#$0K)^_?1u`(RA zW{i(&nAzb2ylDuWmYw5OzsD_Fw}UJu+T208{RxQ(9n0r)m+{mnK4BFo6mi(K{kR_tK;yYiGG< z70FE@Ni^&q+mkfWsIe)wVt~BCJ&AR2H z4ANvy^28q-#yF-K&|Kz!4O_Qy{V`25PR+uO2nnY1(Sbl_)tm*3b$|cR_1xH=`1iwU zi6tyVc$3qV^U$ySfSD8B_{aA%ZIW#Fy$cabp7r^zoFAGwU1de9^Kzf+(;9ahXtbmw zATb&-8R#ETAb;|AC0EH>GrR!4cLadbz^S)SUCy&u)!}awd-gq zJiynIrOCuCE3&}e@DFdi_~x0fqz|5$@=h8}b|h75ojxv<^c6^e7sU&^1H3)!s&BUv za70y*VY&A%=0b?k#u5fOQxW(hVYtk7atGUj0@SXi2d45*KzeHVuk)LV=&6^0y+c)#jr>^R2laW;S|p^w)baNU zJOumC8#`X`-mKbkt5WJG6CeCC4LL-5Os5tm@M1?rA`#%N%=vYJ$0qT!K-c<3IFJG4b z##1;F11Ew1MqY8c=dgm{wgi-i9`kcim(IFL|G}{l9FTUuo6MgJ-{hGQw697leg_|o zNwOr_+GCa;{K#Xm7%^w3>lCSlE2F(ibHj zbl1{r0joIAks1CdcYVJL&iHkOda0@J-G&5HAH8RLQcClZ7rKI*9Fb7vlw*@uD>>j| z+oo)kN0mV8OO@on!2z@a62M98aCVfH&FyG8{@dyicn(nh0gn+=<>GNdC@Sc?J=^om zJh<>TD6n`r=KN1?-}?=fMT3R@O|nDNV=p3y?a=qN%~tE+&%=`6S7&hL2%~0}pDnxO zV%uE@^K&=JWj|ulJOYI)mV_45$@0yvOzZouhJadCVPq1gsP1K0X= z%>|SXKv*#PNNM%vu3~h$_|}$LnOj_0)}wB(IV)S>Vb7;8dET~JR;#s6D|xdYQGR`~ z-ddnc7`fW@?q>xN7&ey-Xq9~K0Iej?m3zl-Lxr}M?f6$ziR@(!9pUBFjRp{__%wGo3BBA)+@GlxQ6h*xks zt-J%QOhfI!OOlfcTo218mKQk1YAOiDE{YcM=JG+kf}V{#ba(N*Giy1C+O66Btdrl8VzxI_WpxDLbb;Wy343_m*-CT$R z?|qDKi{o0Sa#a58lh1H-L2N3WC*P|OtOw5}AZZN%4 zs;jD!g`qIOc>@xOtH<9Rl76F__90QzS((srpZmoKK*fr5E3xe+)&b|FEJ$-@Y_rm zgdYJ{(?>?1ikYVyiDX`gDyU3pCO{wo{?8m^gI;9>^r5X;1r}XcP4Hl0px1x6VMsTH2Kj&g(9rM-)3M)j<4+bmfMk!X{dcB>)E8W z`+s3Y09?5d23t`$Ma_zS%pDgJ-n>Tm?*iEsP&NgfqGm?)qt8(xrK!X9mh60-q2J2* zeJt7deF{QkcU(FcuYPVQh)V8m&5fEuyjH8pF6${`T6Sg@rAu@m(=qU81-+bvY)O7 zkj;u_p!V(_!&ihaKsv>fU_Wk!tI#dW)X7uDs(1J4(PQ%1=SiX{jY3i?`5?`O)}=kl3F*|+!L`d zajpNA8UlIHT55Z)~gc@vFa~k zwvYdrKv|Kdn5~%ni%krNuhYGd_TmbX6=gAnR zt=^;V0?JeXjf8F1k~sLCcCL!Q0)`z+Tix-V9H|Q0fZy%5%o}gPvvqp_L*}c z!OLKxp!~{7UTjv#0mSX68d}}{@zXRZRfncy$(y=*;pHq z+1x3Em>3vt9v+(qXqcFF_4PgPB44cV0|q^aI6=&*S)v7`Bb*L3fHx+f6M(8P-z1T@ zt(m>E=ygqPn`>`p@C!vH49e<;4XPrx<71k>&u2U|^;eDMKQ(EWm&koGll=;?1Fmu- zpI}=tMhusZzqcxXJ;sd{hXqC~zM8e_uW({eAc9y7$z*rwWXu4NkN5b!xM}>&M|**s z#0VloM2fL-uP8Ymlr-lRFxs!zBzolSczg8va3P|?4c}x!8x1+vq##$B{UsmEor14l z(b3@(5?UD>Lm&Vb(Qtse;U-x~qOCm*f~;fMSE@i_RxVzPI*ocC;Vi3q?Qh!LP#gGG zMH#S(x_A^_*#)r|AkMzgra3*=>*MTbj?DcJ_MaQk+{w|W@2UB$x4~;{UPBaIzb0 zlX6GJ8hWW^JH)!b%{n@|i*dS_%_IYXRB>C@<8xDZP9Z~ystx!R0!(n?+$;tc2;-_8 zBd;{j5WNqn)>wQm=EZ7ux$L7mWfQ2M+YjPNHViIKAR7Gkob8R<^K0?8D{0Jr0intg z@Y#49{iAS6E5%lU5K}aAiM+|o-!(Y8vH1l8wfg?uf~CQ{Nj|NFzVmUIn~c(knj{rs z1Ix@ru|Y7{_KVq!KEK$>Xl(@&8c7KWcoomZgs!fxfcpJk?KQ_wrE>hPTkiGSBG)5s zgmsJM)%C8SL)Wv`*XuQivswu?{T9jjMoLVOnvK0=x-Oz>dzaFmMQ?kDqj>D) z)sUhV4(50D%Rq?xhis6Llyq-n<2+Da{EKqs*UqW?_ZDX-bxdbI2r^Pbfr=sp6MDoa zx#76F#0SkYuM!X-kfQq`NnVO{!8WJsrieGVfLl21j^?7_6lU@+fBl+}IQ+b!pgu8U zaWknGl9?22@m??MT1;f4|IZ&%5x@7qlm1%#&42V-B**t#EMK-9TQu8h=UK(y`MZeS zZIGipQ@mqkEuS}OIsfu{ZlCSv{L-Bp&Dwqx`yTB5L9$o-HU43TFd&!OfvG(0C~U;r}GEKP2VzhlZKGc`8l1XTe+gvo8KA!$%eMUri_K zm8cyhWLmQ&3)1L^kXQ&3dRb3DtM+5#jGVP)o;&nHM%Hzrk-jmz%X`YTlF|?8U40cu9{{GAST2v|2rYxqIuW`{c3z9S zVnE{b3|69aw=Si1k21i~cQtJ%JN0;2qJB%#${!hxskupM3TI^_gL^TiQdS#PT*vpD zaVRP(7TJS?y(RJcZFQUBUaKwOqGo8nc2T#I@14!Cdw;;-T`Dli8BxCFDOAJ-@_; zf}$cwQAOtR{>wsysi_#eW``PiZcdd=EVL@jY0U7f;w)y9#`SFFj4wO2UfJq@F5q6& zN~9ziyM-RJpidU|N&^VId7TpeF^%wE6H%2RUfIRotb~073ThN$CD5&uQb94WW z%MT}cFR2fGgWk#0jQZ=*FNC`y{bwht1FxaZY$dJY3O++-UIER2;^z%Q^gRH-CDrD5 zKORN52*Rbv-y1EsHVzzl1P=s&cX^H>$$tSL4H1vw`E#^@?v>=Ql{MK0tH-tH>YriGWw7-@_C`)c)= z80;%o0UMQQO)T--u$cJqTm-Z7Y1MmoBRkMWw&3SAtvBG)z8oe2B36_? zxk{VUPMCZDM`EC)%si-Jvs?OJ=zKsHr7NBQAzHZF|uB9=-@rDz@_$8q4YspL*{8LP;w6FAp z>f!~|1#M5TVjM3JE#^klz28vC?q+Oj9+vInDowRxz;OA-0T7)<7qPP^k*%rE%PLM> z+gqSdpFD|AOl+GMeD|1e68{D>5VLKjNPOyO$#1ySCPx^{jSD{1@R9j#UKSE+Wtci` z=F#E&g${%aqu0boKGOfJHijP7vzKVA_A_GB8RbZGM4lpcL=T(H+kU9tBDV8}ZB>2x zPN-buD{!?%dWX3i=++vJj|28JSC3BB6KICe&gN- z6+)L~KJ0Qq9FEl6Ky+D_zZ9f9&nm?BoV)iHdYD)2SFlFcZ+elgiN@DFsB>$uq%SSO ztwZt$ssx3`B&QmTV*c8G=1?^#p6n+0iUZ4vE?*#(h)qUHDKjp?=QvY<3PSTw?Z3RTp)`eb?$jfFy(~ZX|Z|tc@@n z2bxyUBJBj?&jp9S3-iSPrT9ivW&{41Vf=MZqGorQdPj0|xrD^G(VYymE!BJfLQL~# zy-L$`_iXd!;+&s(RlOD#{H3J&#IPFngev<=#8Ll?ds_KAwt*+HQ=v$r zl)>}I5ul!CC>>VbU>VAq8{;L*xsPF)x;p_mjL$hO20rzk1Z>fL6}#fzqRf zwsy5n=`@AV%ildcC$L}-V8O%&ke2p7j*2qyUmt1%yLMVbeqoCrZ|JWM)SvsUOqR%_ z7R^01pZS{!;~n>?85-X7^#qEa(9N?FAfW>>-kB%@o%2gwe4z4SVs2_{4zaZrEJ=Xz zm@Hh?^lt2Y^3LRUnRi2&V^X_EH6;ut{(}uMGXP+7c4M4ABFshS$H_P`tzwd3SIBBS zqToKLM*IH?nb0+%)7o-$h(isk@Pnl;#;FynP>I+`3I)A zxj*yi9%&VjIiTZfCJ2Q?bd>>q?sn^bZAV2>@ ztD#&)O~dS3(Vw@SMU)aDDQWWAO}=}ITBSZNmBZsGu>%4zlXuXg4azU4euWMh89lrC z>^C&tBZ!TQ>*D5?hmSolJPgpehfs9lTXYPJA}a$8jokmK9`I^vYW6ZPKms1PtiF7x zo_$2`nLAyEL`Gs%$a{K#h{1NMwEd~@$cwQHlpO#bP$hXz!0NlHPC!5)$j`3`G$1Al zRhhKQBffkAN>j$wvxmOo!6A%QTy~Sa$E?sQw7JCV9j=1`6SFQOOEsd!PfWULP`VTs z$nK%bXpv?_;`YnnJdFu!m3X6pEl3qB=iAoKUE_ah=GiA2F+15G@brW%4qC69*#9aPv3%dF#B8o8=2^%NTevKBGls7FI?Dm5TtenWx!!Au^vkS$*t z%Di4{{(=<&z`t$Pv|9h6+cq&BQm&FWB=>4lI>S~$eOcojYX9qEq@35Aj>u@VtHb~P z6c{>*uxXNC^_`rW^v1rLK)%=r2}uRLS#=T-`=ue{H$d}Zr?{o%ZUlg1>vD2v9((Cz z6H`-jv$ID=<>Wh{a|e#_$53-u{DrqVv|exev7e}(uRs*j4^**L6r~``J39hC=XQ33 zQW-S`ECZNDcn2ph(Y6%AxNz}`R3lR8{RK}Jx`v7F1>*W3xRgZvfnrb9Eh1Qv7*HBb z@Q!?NdKw(Kv&;iv4lG?|YKVY3HO-`?J>ekxa_%tW`ZrNU^R%7p^TDXo*V>nZUl9P48=v>q+jaPZ z6XbeZ>GiyB-*O#Y*#*#8~MAW(OV+!@6X-94)EPkFGW5lb~ru*}7;S4zH0 z0QyDVjVYzut;jzLk3U>Moj!!BukV*)eGC902$UJwRwT zw`4w89>P~Y^gW0Kv~6mP-%;BnGvuo2Zhu<%s`gIj3UBtkxbBTxTj@eYxMcZmU6U58 z?wn+qy8(`5{EKH(=4BZ}h@7blheK<9CT^RP2HX_`Mn{$~1P zjRPlm6xw1X6%QG<_d=Bu;+#Z~AFYCz%$MskO)m=PJ%UL%e}1@(ihNENz1^9M1o>GS zBAD{^x@@V@>H`H`l7$K4P-I+5tpj_`=0D2baptFQh?|p$?2tc zkdJ>mo)#@NbkA~)T8`4jBDdmLac6{M2+zxa&TUD{jga$iR7m0en{!9IZ~IF>O*%m%h&kM7YVh(wmm0wYPghm;L-su(j5@3M-D?J4VG|(BsTya;eLP-q zKF$H6>)LAyv?quFAOiVWTifEhtyWSKrl)#5UI3J>L$&VY9qZUep1bu&2Y9BRH_A4*QhtW&G0FgRFhZ_bga;xQCcC%m+>asAx= z#BSUB?wY%n-D-q#Tk8|CnS~AGJ`TJn?b8luSLpLNc9Tr+%elI8C6oZxUM`<|rlMf5 z$JR(Y{J+bw8$Dtcl^Z3=-C{CIV_szGEP3jUi|Y_9U83)Vx_H z@TDRGPC}ch9x2qUIyFhTOL2C1kwddwWv+%wUh%s^CMKCCeA>=AOUhevy7zIS0RwC0J}mKEe7$K2M)i_udE zRNPK3mXbm5YDi6t?fQLRG+2B1=isGp=s^~0PTJ{de99t-O+tE!sU|{-Wn_yopE+vr z24%*UzLc-g#4~q*@Q71@zzx_Gxw*PN2>|@t3}LAE+81oP{q5H4>1~z4ooL7lpnkci zE0X&*lIW=^k9g!%ulfG=>+noPLm@5%NWgZJ>C&9MZ3+EFEREift>Zi0C-xUrBcQ9t zv_xh*m0$jm{b3^c15h)Cz0;)Ce*N7Yk0S#1CUBd$7ogNaM@Cd^q99g%Txx|FpiXAD z(4Y$At_im?An3LW7MAE-=yGfdUvjseKFY&3>JOt1o-1Kg>nQp4eo;*`H<+n(Ov~P* zjUU?v8ECHp$iO4;?Rikvx$(w85*UsnR*6Q)J@~-D=<+a zqgK6Q-oD}B(5tnX1ghn^1~TCzJEyePj^PD=%H7GzcK>sJLe1h)XT>r7=^(jI&2I25 zU&grY-v(oUM2p}0z?`BbKLqlNu+6<+q2k*Vdb44mN)r|5D2(-gKY8s8b`=#_xu$1c z>RMFY4u1!!S7_7WI6rn9j&EARsxJ^AR~w^^B6Y8h^P^(jtx>ZC%jQSWnyQ z>Bi2KHp6%DfA{?>cip9ahY7y1gzG=_lSmPblw4q6w_Lj)GFueN*K!1_2ysySe}5T2 zRz_6k^XPSWVIgh8%a3`x`OB|v*&QvrFRO!W0Y?||f({-?H&%>BHSwmAEJLizu8oK@ zicAoon8*D)I{^Te!}<3Y<+hza-)MmLKJW_xo8}F0NDB%I0*+cw5XI&vB0yWn)aqC= zO1bQN&AqmD=N%Q#_jDBS!?$bK2UdA$ zApu`bE*C7=q6eKMASOwIBjwn*@q0z^JHiPpdBdzzR-)MCy?#}OfMIgo4xgm8wZh7O2@+# zdHF?4)>NsV4nro30x#dVJlGRfr?f{d!kjzq2?N+ylXf7ll$}we)=IfvBD zrQZS>9V?_56=uA@-!28aEDG#0AZ%eCyCZXKj=>_W(@|P%b87O~<;eg8kmYz%au@a} zOAZQy>CuwAYdz7QFdwcr!*T{4W}&AuUg6(QN!}FBviw1KKU#`5L6rX;eYv^MI59BH z842ZI`V!v-A#*`X;#ZbPU1TKEmhcbt&7W99eB)GI6odWgU3P*kRsDWvX^u{z@N7YV z{bYMd?9qwT592DSLi2x1D%_+gpryOw~NNnFfbC_S;`b(4?e#97Y7UEJH|_~ z?=c{WZ1;lR4K5>xp?W2Zx*B1fWhBuf`*vNB<+o7HiRBDp-G4g_D1=%Dr>GDJhv45O z!jI&(jtY<%$%;7sOdh+Hk2Kj4ao;*jblCq~awc*xK`yt$u=s0bQ#>9HX$z8&D&?Pw zvz%!z)!q|=b0j%@O{${Ob+NC!e%@>LC@nny++-z>87y~6qHQ)QERS+%gV|1l1sBc< z#`#>zmDECjBf1NG9eQeHH{P z%LS>ayTD$s{#lwjPm>k*S$Ozb!D2*2WHV~3|D~?qL6@Qc(Dzj_z4H)`pZoEb-OeMR zt(fC(yoXmtg-U>&yw~^<3qu9g9LotYnpZpxI4H{LxX^^SCc@r0 zP8WJqM6b!xX{`ArF=S(S=q!u?5Uw81+Q&LQArdkr@fVed+)YGK|Zz2WJ9z=6v z<*2gXUW*8(VKD2jDm{V3?D@3W8Q{*2dYt6?={|BQ<`Qh+cZ1K+@f%YN$^7i(rs z8k27JLW!9|jKqm6GI$4hi8~gj&UfHT4g>9%8LSR>rCmwW!RP&Ws+*!!6v3*8Z(L2e zQs#7B2qUFPV2sR>ObH_I!2-y})MM$JjFdS3Fr{Zbb|!od5O_q28BZeyqUSh@OD18v z&G`6dKM3dUtl=OF&K03{z@XYbr|u1qoSVPLpz*s@n7+>gk(G z;{>S+Vlj2&vn2$1Y6+}sUF(1^(8KA*A;Umk#x6Dq0uZAibDAOyHDhY^2|HauB588P5jzk zI%sRp^Z)qI2)?$&d3~BePN=I+Btp-PyC5ZytQy(i`BeDHYuWH9-|}q^+qrs$^R%X- z_YiDB@0^>jYmWCG%=FxU_PhPH(_IkWHJZpHQy^xTD$~Y}ZK^XJY<|hT3I*qvv;65s zp6B@>@6)%c2+=;9nFNqG_(r{JVb46Nl~(@#oiaE}Iht4OVDdz;Ami4%td4$@;un%g zP}jgK3YZo>*Ksa5_;JGpaQ>~QBlGjMi5C6v|8AW-s&t{|(TQ5d&vyIVoW`Q7VQpHG zX|JyXQElUHbKPj|1%ipfpZKj;C@~NlM*9owWDM}}4nD|0-jsP?AbP5uAo?j?<8E=f z1=nhwrPuSGmoFsmkl`Kl(L#PK@tuAkhmMW3o9S7-spOy<`vu?6EvrO8Fq> zNJR3YB+l!;qQJ;JLy13Eo-WCcy%tSW)uNJX#_`Wy7N=eG)}6VNP)j@C#}Db zbg|%egy>%Gu#=$shR$*G^@U|5rav7SocM`x@Q)@~<>r$|8Mck$1gV9B{wgwlCFOt+ z0XnA|q~Y5$iMIV5F-Xk9&9qWgQVX+=n_tj`aPIZ@6dl6Jx4lr`^);j|O(8M8+MUot z3I!dd8?R!o+1=coZh4#sjPO!SIO9gT`i;Z^?rV)bq2EeDkk>H(SFwYxB%|6acYQ|E zvzB)(q+0I@k`x~Yuu%Q@Miqni0_9B*MCMIGnBmMm) zWa&!P!{Jj05BX*&z48&laabec@sneI3uRxJ`v*PGkM7}S7QUV@(a%U{%I@NCWzz&S zD;At?&g~y_r1!>PuIg*WA#$N6eKXrS}O(8q&apDWmRpzq9311bBOq+3dLb zgw!CQKdOsF?i3fu)+g8A+_LFE$W^p1N^j8L);hSRT@4q~uoO*eiOfUySsWJcWhiXR zUIUH$9^*?D!L+K=l!yBj%RJd>lC%CCjw)F5LWJe7nw`1Ip+rjXOLr5nQSW5e7S$BT z_4z`)rn2q>r@Y18q(j)ZBr+kVl<>BA2hxefaGyd`r(eHT~OMATci`DVbb;!S< z&O&QcojqDYN5_ikS3j=$wP758wPY^us3C>8&2GERiwc{+QvLcLR%yw}y?3MUzUl7U zzx!5E!B$)8aCZ&-I=1nHb_BflX6o$c3++KU8@yIAUFNbtt;OtEIQ;s%qPNFN$J?Qe zdKSL?R{;9`-sP@ENN*!u#3+%AFKi|h1<7QKDEC7<8gn$1XyaqR8DGX#k^2NYZ&$8}~!!a3rCK6>lbZ}kIX?*5@{yDx40wkt_ zmGvYky=i#}86QEk)hj7=m=xx2%!~NvRNlYE`Qa>ZyF<7-UDsohUdC{sv7`l`^!=^f zZMwsfyPx|GmoYEYW=LQ|cSd*s>x%N_fs zW4M*+DHATet9Hc)5uZ!+AMnG)5eI^W#YIqy1r|AgQ$q8&X1yX2QNkM8>w{eOBSI4%#YSXWw{u`V@4&73= z(LGCO);Pj;nkozbO^VHu4YFR-6p>XKx7G1lef^YqyD`e0q0+&|q^)#J(kT6>=b_}J zGm+rTDNqA zrTG_gzpz+rbtV%~I!|H$B#TnmJND*FGFG#9NOc!?f6zDGQRz-$|HAT^!7V`m*H}ziGJwW-tRmV>ra&Hv=*foJML4G_Y};nm%>=- z7`ZE;yq!d6%r-#`Mi?>L|%&j;N<0HrgxQ^OjH{2Xd;RW(Fo5L9{ ze5BTieE@a>f zc+>OqoP-(6=cm(8eAOn>X&=r&Z1j?>I?Ci`|9(VAK_nU8JLC|bhq}Kg6h7TeL8Z;S zjz%2glNxCORl#4{9Hyh9kTJenE90VM)RJr}{AABWD)g@@#pip38A_sL1tSN z^te4F@aBX+zbeKIkDlauI^xlH-i1|gcvOc;F8r|k9;+TZ$B7b0T&5g=re(f^8ndL7 z6fz2mzuna!i+&y8nDx#|dH};6y%!+ZZA)GF=farZO7*n$%zbBu8J$f>=j4ywH4*&u zVy>iKdeQG7FVSVmrzu{*@nU(Cm7Gj)f>ff18HZf6) ziWc*PfsRSu`3b?Wl-h7e=;H;9IsM_yUSov>kDwLQtcvI0%X8d;qJV45<$r%WCq%9u z!Wh=S#j_r)UH2pWq|u{(RO48xlpzj|lluu6m<&zNNkN zEoG$$#b8?M_fVY{tqbeZ%j-~7I#RSn-mj9avV2uuSJy5KkSUkpbi>umn{9mTYn=P= z#>AWa5tTO6YW)`)U)(d^D%+C~6W6L46ygDM=`?`OKKN{RrBczKVpmyTZ&dl=sgiS) z87jCn#JksLB)S#O^T%WvbRx4YE`qg-!-Gcuq5bAj5clAWX0Dj|Qw@efCe%N2 zR7FMbO0G7JXoHzr=~reaE!WC-Cw-%9UmAuSx~W;h3YE8`L)#X)cSRt4XPaTIN;E!m zrw%E7B&VnF$k`IyCCd2|d3SzMCp za*wJR*l6-3XHQO~onR@?3?qb`lTeuP;p6kjLi+ zFstWhWg`5-;`=Y8Y$xELg~LvWzFkfyxxN#4fXiJoe~`G+n<+67GWbgvT~O>Zp^k!L zU`+f!SfQ1s)puN2-m{~6^09=86QfNFGjowfkN!E>;kd7fuGx11^`{D&1J^`@OK)r+ zLOLNI?*}q4JEdxMkIrb9S`4-HarLGyd>_dLP(?{3=w92U6 z5&OEhlq8Zp)~lr)qL^a)WV584U0ox>!{er!sG^~s4`-LOs64ay9%S}16`0aQ%3 znrJbOB)$My&7l@;YQyoC32{kyduF@DSH6nRcq=jf6aU&7E_``XAGCaMIN5fcL3$;y@!O~# zWyp|y>YYVvC0xQSiRXQ#yTw^8;HP`B zghS1(wjMfD|NNK-)ufYq{TkP&@CpZQAR-3HcnH6yrl#O+y@5edK zwa|}M%aiJuonA>T>FjMO^rIFtMZPN%&6fev%sj zZaWZAst>cB6T-8N5GdxYs=g$N@aK?$cLT(B047Or%{Msi6>g9@K$(hNZLdmhQ;`~KWbt$aMrtd;Ib zp13#FFl$gWoIhKmGd2A_kh(FC;vYi#Ci4DuDTX8Yn#14_Z)JwekX_vfxx1w-$EEU z4lT}(GJZvgq_ndl?RcQG{nd$B96OZvzrE@tjhd_FQJPjZZ06#sS67|J(UhIrAY+jH z{Pg1RHv**p=Sj&I@xg)`YRPqiJiNK&e|nVJB{u(p*Xhz~i)#>6iwKZ}3E zW&(9CL`Vn1#{ZCdW4RoHN{^2)!AGVf{x+2TVo$@Cg@Is3p;+^X0Mioo;QR4MdkQ54 z;^}I4GB{6HP2+11zX?^1k2}VJU7zn!(-di7vVEcqo}&Puo&UQ!uJW z`^PHM_bBL+4*A?bm4snt>!F@EMFYZc*dbt-8h7QeX(@REV~>QoE__I zub7bD?OmVlqA*zNs#vP2iIHKkHb1NPUf{-(K7^dUSAa08C7x9m6CWvFlRHkU>mo z!@2*@_J-z9_H-}lPcDo090^0{K*%j}H5SEnyVBhqm{k8ZgxWM`WR*)oPdsgvj!O_@ znSFfkV@DglAt!hxRRoOg-;qX{1-V*s+akS^oV**KNJv0aqb^?F#K*H`(9--8CPy_Y zX;kKaf$0j%Cj5e#hW_%bNnZ5oYTQi80sfDD_2%q>PN<2pqLAZ8F#$6OJ0s@X8CQzY znHAIy zT*|b#Wd6bb6TX0v@U)_eZ=@?;H9bi_`O|vvY$vS#!l#X!K+I;c4V8MuY|TpXJ!~G$ zVaXTZuS_Pc-EcF1{xCW-(Fb9A3~5XfiJX~xM;rTcb3a^X+)l}AB{iCK#B|Sxthx8o z!iQD(PUaBDHX=&J(qPSWYc~IpG|eXhkJSa!?#o)f5xtO9KBpA`U;)_xf^|fbgM8QL zrxPK;DzDd$PHPhWFjnHENoi9{K6B+3&nzicSHh8lP~)IV3NWk?v!8Lt;~j|jnssRK zaGD&4Kg~rnb7GWIKK=V8-L*J=bf{))H_L%jFkYx>2vc&G|HzBNZvqJ-DtUf(LPfgU zh|)b@h%dzSqHc9J_q*_Yj`w7L2F5_T!4I5=C(G-xR0IL+eyd$=7IWM6Cu{{$@1JCG zmZlLkHw-eQmREaY{jA3MZQsWKF}SVBMt}M%NIYu8-0v3+O1v}OhG9r$BL3i)78BjU zMwDr{;9to3mcQQD=p0oWcf}MpJ@IST6QtX&I%*@ish0Ki{7ah z7UK{Vj!tU)QQAoWOG-sGlJ#5sbH;RwO@g&H_MHXug(n#eO&(j%J_mol4hjupbFOwi zdDch!F6qa}R(9M?D%&Ebc%dd<{%?`2jXBDt+*i3Djeja$oV%xKjCw2ik_)nVUli`F z*S;*XT18n8Jbg>*;o)mQ1`u{{W_A{L-N=s8P*x{*`nDz3KZIU9INY!cK zEGuE$J$kA%?Pnm!@LfotQ4l)iO&SKe#4--OeZ22`b4b`pw?6dU>fi(JM-}C|pKz`0 zmfLpki;kZ}7F{%^r$gi8gDsUrgAD zC4#x6hK>B@zQ4G1)1ol5>ZFfX>gK9+2Cg3(nfX$GU9C|=L1?JFn0Og%Hbp)hK5uu? zeh(AWWu~*$lCkI_5Ecwxi(`H#Z@6F~Sb~N^f%INyh3^+VZOi?pY!FJ!HdS2P85WxE z3ZDpVH6oV}->@ERXj+Qw>xX(zzFSX(<)e1Aa*Udgxq5Ifm`XG}ehduutEy;FzqG;= zr;4PNmDPNuD=TLOfCo5gzc){wJ%_Jtwhck=V(C3gvX)H#@!8ss9}i72ZbS+2bSjpT z`QU$x_FCaB3e(jHQphQ%@YfGElPrMKgmUoR&<|krB13+Xo7&TI=A;m-XvW04BEc{# zIeEAD*E(g;bV<5TPO84Gs~1aJxpRHK%m`X>v$%&%_u%+^rJAq`&hPBYO-*S^BYez8 z>8ekTXV|hmqC=}mrtYzgPK&gYFN4=!Wu(~}+;zOj@OZvrEl?_(n~oTwNg$N%ZmG3U z&JID}G~jhP!CZYGgvuSw?CWL=J~E&`K7oYFnGy3Y!y`I@%u*QyI|etO-8x6zb&{9A_vHo;gKX6f>n)&6<(Z{VR;inq=lNdo=TnK7M7BZ6SwVJkTD1?_iRLxiD zERudk57TTK=iB+MAO9$W1!b?4y$Z5$CY@cWeAqu5C{KbOwd=p0M~MMy7`}sH9`(ee z`*(lDD23?*UcI8FV;?}e$5f2{TvEdnM>RDEzQf# z^wrAE(-n&1WDervRYFBR@sgXUI0uO*%@=*!bx75mf4D(%2Zh`XEKx?8( zvTdq%*^d}1+S$7JCYGuM9f`J>jEwk0oS>?Ol~tnM_lzs-Rg|OL?AwDfPU0w6j?H7z z^53wAI+=THGAVG^Z{ErXN%S0AQC1g$;Tywa7w(C!$XjH>qSiqz-Xqj;F4Vw_0<743 z&i|=CK#Z4@r=FuJwk-|rh8bv!TWNhsK05kChF@D<`r1CIoR(((%>fU&>P)+KuC`N2 z9D60Le-Iw|`RWsm61V9D41)_jWxm0m6-LC6Sj0f#OhGD&Pw}4;u4<=gzkedaV&PV+ zy3`BlXEt%x<){2EuOUz3rA{MB|M8C$Z6;QNSx=^Nzgy}%5L$U&smE>70T&E)Vz7IP zJWsBGrJgPUk9|^9%Fgb4K=y<4ZDe9-Z~m9t#i25Vm3YTc%-@hl(^jo?x$clfK6l5^ z#?zJH%I5SJc3HR3}o65fTr0j0)rfOA!0inmYd`NIO`r#o_j{;P_#MxU88?DV*wtXFO(ielr zdbh$>__BPhA6w!p!xXT@3N{cS${fsO_td3}iu?>0lkHg4S06o8k=_WpaoRDuGJiMOP9V7?rd zDWaGRaAyBZK)`TJqba7TxnLF*Gt1Qlw9m8JDePW%Oh9K$`^4Va0V!t%iN)$<#fl0F zq@Us&tCgkO^PbQPPr@BKsfHBo6$i}U4!1IYK|lx%Qn}N@L}0I zjv!cBX9QU_<4mrk-IGm->@t!Wl&^*Ec_GCy>aT?+tYao|VCtK^TCWW|P9B#dpn7aA zsXN7%5L)EqP04?P=J&{?_H6>0G)ObVU!80qmzMSecS`Z6;SIycOc*|vm50;lL(n;N z==pQmc`CbD=rwZaCGGvM|5EC6z}eguvq*mr`1qce-}ey;c#(BlAG)!q(2!j=n5bkU zrT4{5aP1h0rIgHOJpmC7gPGr#RB*X6LBjRbpI{9+Lpb8NV1tt(q=@IiSXmkqfa zKQw19lPXx|Kt~Io8Pr>yA&(8ARgxwHysm>l>XOCGth2VG7ey?CcM&P&l zh=T;$tYZHilHP!v5WL$hNX_mao>%8j>eK(Y6t&OR8)54HK=4Z@#1Ai5vL#E16BS3( zuSs+SIuHQ8V6r^-xW=Xlh#q`pr+l`S^{`5uqbr7cYI$4>@I24)eU>;L7#!{&9QK0* zMY-DJv5njRCg@t*o@%yM+eX+cK6q*4|KF3R7SORnGPdOmP*6#lsC{C(GzP{@FTT*~!kjEA)P?lW&OxaNHU{B9RY1TKh zx3~A_&mZ`{NdQofg2xW1)<>3>T8_iMLC(oAB$V`+ZI&9&&g?ur9|7bBuy+0&7nsV&$ zgSm6W8c1onWF8d4Dr^27u%t}~0{s;HW4&0n5De`8|KiO82Z}ZV?6R1p*lH)h+vEgT zoB^+n??y6k&j+_2fOp>iEi%x$Tqx!aY|Hj2LHhTn>iq-0XwzjwSE|7Ms}WW?g6O_H zH#-q~ru9tS^CoLVVxQ(;n}M19iWqo#nFBIS0IET&#m#o73A~ zP=Jx}qm>pCDhLFT2ofLYZ~V3X=u1E9;YG|F&OJFaWCf_s!ag%5PQdx2vto(}^;Yi% ztGyHKWyF#{SO#u$2kl|1)w?XCFZsI05~Rn+?ZWjd3^h+L2 zD!U{Fi!@F~mIKZkgUPTh{qbGc$HRjgRI1zhaC3YYvg++X99btUN#?@NJ~P8Pi)-mm zz~_+<2EXz9_3LeIum^yRr5LHwBx3sSVrGuTU9I$e1@f*qEIv=ad$>NFU0aR!0+M@Z z?y&zW(5lht>8ACi=KuL)xBPG|At4d+{xl1byTyDzbcW$0TMEz-1?Y)Di%3yXQCd1Y zE-tQyR`~ROH_I1-nhyCi02e+3!5WCrHQH}Vcm@imCJNMxIyL$7iX|5AO+F7t?sx64 z3uriH`6921Dn57MD1w=t1EfnVIi1IkIw~q}FmOTR=0q{$=FkkQ>Sg6XiNX2yPBtZ$)R*z3l(%e_iif8n+3=?_#~$jpDbjMyXsA zn%97hAT6dc=JVg%_X0zS^$W31GXGcaFMlPd&%cX$p2inz{rt4oZo&gC|0)HGi)ua{ zRpdusR9sB6;{a&+wzoS6-&Fy~mqQMwb6s8_|D&O$ed-Ux^6*0GoyXif;{|hO$=;q+ z2?!{ige+*yjEqiW0uV!>9SqDfu!5ow$V$~GWlDsWFhI=w0^Tg(F3DoGhy@UdHDM16 z+m4(7cwa_RGM6Be(>`~FO*L?VGJJ+!kwbR&2e)4QZIp2|P`9mJx42eGJlYsAz%2Lf-1La!7eC0GV z6A1MGldblsrlpn_B$TXf?H;QW3R-)Su%@R zq}Q$CTUd`hqWSPMMa+&pfdZ=Lrr`0^|@0D**{ zfi_o%Qy{lHX07GuY>we!Wz`c~aWhfEXJLA^jt&)37Jg?C63-^O3)Z^JDzj1fOQdJS#2bM>^27o`M9Q?QHWRrEI;#<-vT#le093~L=7sn)XHVBv`bx)YO2$>u?_{ z$>G!9v?JPTWa!>PPXqm!zm{ThEV@!g$v?628w2)dl+)($H5}NTApgU49)l?vB%qle zFr=uCn_SY42`uIAmpk{5kL>{MgNG+=Y*(r|cjxlrqIek~?IakIkM!y@0dUwYi#GVH zgggI(FHa*yf5=%%*?ibb5H)MmpL=W~uH zSdURMczYT6UAMYbH`0~)IE^r{S?-eU(W?ipmBEd=2~Z zPOx6&+J;dqBcx{)DsO9eDle+^w*H3AFs!qXbiRVwwJBu4Fw`Hy^vEd9TEzGB%Y~Cb z5^7YV1G3@ab{Whvxw-JHA#mODDn4F6MBkZ1QUoLwL3V)L2`vC_BhfvfCNsyzNSQ8(|TCtPo?Y+BP zECH~9$`B&sX~k0HP*csJ;24_sOSZbT-JN^OD-`F`Iw}#aBEF26?jiEAAb%A?gyWU9 zNn{pB#2q(h^`}rEB1^@!bTXNagwKdO>46cX{HR50G{w#n6Rb8iiTD&PgqTCI!QV)5 zf!;A>BtM(Fu7R~WQXOrh0PMT?QGKs{uI-!9m<4!3P=`&Xi5p==D2=qR1Z`<4o}bme zdTJ<-k;o={(lJSKHB~?L@@;2W9kYT!f%|GN-0y4@yq$mgzSbA;*!UUAUw0hkKB|$- z+c@%L%gZ>+a8xoq3@>eskfI}E(WwhAR|Va0d=vd8FK9i7=Hh=aUF)YDC~MlwOIs3p zgh~H4M?^zgnRwLwZf!a&ye*HFukFx3^dSVNpMx>(5(Z_sX|G;K;NMY${t7Ux0G4vpgnN>d#1 z;Pw@q#Bs<=xg4!luCY%}tW8g6XvyXyX`S;53{PL~j9B&DXqD%KkzMOE`F_H~MSunK zCd^so<&X3`Cco-m48DD0E)S$oB{NTviDYvXjLiPU=J8y)%0^)dx1 z_ej<0vku>g29O0loqC6&hYAsK_ffKCxP@G6g~|2S4iA8dqNFXidB)B4!7=niFT@zu z>Y%xxPTWURxVXF(p2K6BUA=N}O_Qua&VJ#;)7^SlJ)b0FAD+xr>5CHA;c02pW{3a1 z5`DQNbC6qZ7CAU{aFnyf#(0R4mT zkC3wZU5@E_{>fGMmq=LTCve>)oGB49&{PgZMuwoWu{o#0`N6YhOPIrfl&L4Ax@$nj_ODL#uf$XeybG3M2KplgRYXgFoC~7aqU57J# z_D|$ziU(W&77&6)_IV%mfubxgtjqxfyA041G>j)nde!k$wyUkh^UoUkpMqx(*H#v4 z6PA;clleNrj*y}vV8)REbdagiqj_w-`@JMekxk8aVpT9}%U?`EAn(NQ-_fHf`T2Wb zWz$!9(>2QLy6@dOCUu&uo?Ay$K;nrbf@Zs!R&o_iMJMD8|77mQsq`F>{a&yMpP^+| z+lIyaaVWq`VYM)zh>2`w`1psk3A zmyu1bXrbfVr+1ltW4kQa9vIP7)YLSS)Ad(Uko%vk*EY5OX2hRI$)h-~Lh%Ib|96jp=KbjRWP0uvUeh0E9Gh24 z{+lV)^(WrxEmXy6MeCn&a;JT|dwP!NDoJ^HwHJQXP#`1Y5ahO-3?%|IMc?o3y)j2| zT|n$Y6BnXXJFUA9q4$D`r>^cbShT%h_A+tCy+AapXLeP~SE*;|G<>VIj=S+kyWXXT z-MmlF_Og-_cE!NIAi_fad3stRABo z>6<{D`U-7?9eI}8mbH#_7__T1p%yoaAki@tB+!?B@SDmav;Np19(h2<;0vG3-9 zQ+DL<-~R$EToe$!gTlZ6yYnJr1~ny*_fC;vO>m$}gIBlFG%46a)cmWpcPYH=}Ad&enEd{yz421AMEPlgx zXH@J)U!51z-(4OZI+~p>dgy2H-XH-5fy&@wj8t|yRrMmEKL&Q^+GhSuvo^&#`cNv5#X0HU&j|N=2EePK)g!=LA9&; z?sEYV0f$b=g)`0Nvm9wmmrEWrn|%W?T_|rl^aq1J4Swo zw0*qp%jHwE5|x?qd@N7)^<94%FfII_RQb=>O(1qVp10|4%vm3;tj)UoFY_UMfXwCwLUWYc*S5 zrrWyE)nzpI#{l@;@_Af!WRmH9nje`4=^Ovxp74EH6?%kOTO0gT;v+x@15`IyzeZ$F zI5}Nw_eHqc*?&|~wQ!xxo9GX%x}dDI4`zSPuULkmxeJw)Wc7!T^oTBd@XE9x{A(V0 zT;ovnuoPn?CbWB7AJL590z1%W5;}@3U#|P9;A9zruC$lGHuO`KX``3yY*+q(XAD#$ zoza`Z9g+4l)w~NG=`Cw!wb6N=-TR-U`cb|I%8}nuZ(m3hU^xrcWOn{?18W;h7jBQl zKqj@IP-CN;eln5Qcey(zU@r(1#E!Y&VOO^kFlUrdzjACN&}iy$x|ez~9-ceyN{|r| z|0MefM?^lD-$WpTQt$>zM*1Ui!Y9mHo_%Mi(ZTu9Dmup7oIh_cp%7pct@vCP%;B|2 zs;kDQT={PDHo5lwT>%zVbW2PfYc9PD)Bzl7vBRC7eE=hKwZFVLIT;HWj;S!gX2!g} zH%Ft(%|N;1r5pLHsW15!(SK8wRae^@DzX=h{#aFyS7x{K?P!1WE zit71L_x3gk49BH=u?QVCgF9_0hP{*|`QYGyL=*(PQ-M@-u&k#^3266#B-VTt*niq^ zpwt|Xnd@BAz2Bs)tPE7kY7XWd0p!R(qDS&`LR^%#3=Xf?@Dtn5M-E0NGDHGT~{GlPgk+y^O@8|_0W)%%6@TzVU={Q6UPDkbU)-=X}J|F zeuzo)>xo$w3y8su{=(_BH@?7Tr8y^VDhgklQ(Qc=5o#P8>#vPA|Bw0+I&=R;7RdXe z?cI<1o2(O+l$2B>WzP1$enS?&Xr=>O&$5*yR~BB4D+}n!(cx5p@>Vv3x7l;n1~+vf zJ6~Ia93$SbMC6TvN&)1lOkAC~c#>f!mhq#lA|oN2&q2%dV!_-eAR>v~=gI9KmKrBF zU$6wTox?iwajz0^;3I>Q7<6#iES0OZWW~jQH#ocn9eLcE;NaH%k;Ng`qn#oR9Otpd zEK{q{UG67M`%owVZ*()eBeivqU*cQaP@c#rRa+O>MOj z2`MQahpm2~{tgKqQ&h?<0DB$j1CeS##svTlsVhb!4%`aPeS=!OvdH3b)IdG;^ulY8 z2?EY?Go3eu(wCQa?dSAAL_{adUsqSW``Dbz--CP|k`3?##j#09@)(pi-n^aID0TZ_ zUHOTA(qwu@H64FJvTS??Yy75Iy{}@S*iNV{^dlVl!tE?a42vCO%l=e++4;Cw0U6A0db6J zX@F+8xw+X$5jY6zfD2@yzYFNwzPkpU5EvSLY-yb~R$;WbF3hrBx=3KpSQ**8QqY9W z%VRU(+x)@I!|&C1M0$2#p$FHD9bIr+=l!A0-T68D^KNB6lfb5f=EB>vxkp|co4*}K zBey+qQBn7M3%_UagKKZW7+AEOxA!~bUKa}8gzcF;eD}-0z|gq%-m|xdRv8s^h`*MP z3$_t+l4eVaYB8qvZ%_hRn^*A=-D}fhA4j z&}&;79JB$5vu%FS<2Ew2K|JlbE9VJN_CUP7 z4u=Q1!fNKjf`SUwgp|r{%xr`Z415+&tPgQ#>&dnqY4waWWPFFren}kc=#{U0P-w?N zI$eSv+qK}L9`?odX7!@-jA7>)r{!IjEy}le9w{Sy@uF(iO?I{3PA5d$=7=sxF<5L{ zPF0)++%Wn1RGX%c21KkxEuGMXrKP87{1y&pmXXM)sM7={gmjn$HA-^w#^&YC#W9ta~&p)Of3B6f+-NT;>0B#ZREP*DiZWLgQeL!l0|7oOgON1{|`}4{t zwsRvl7}P?#2w(l#LhfmYqv+D6T_TPyje>>p?YXD#%7fe9&MF#I^xlHte)T&ai=(j7 zo&R;MLW$`crsw)klw#B56?z6`LW!n9){CCXBhIh)kfuA^8B~)!7!_JH7Cl0+=WhI1 zt}VxF#@GoyTT#*ZH6Qq*>~zfFRbr{ zT&>S!_Z3UgAyH9*^xT;WZ)06W~IR(J=h)hN7`xp{BRff5K0pMXu=@_t`4&mOwzAy9!8*3(YN-$}dGGFe?i4MgsA zhkU%~X+%yOzP!P+w8|?ll2FRqhgiv^ZfNN0v39wlF;>vtg2oVU~ix zZeQIqHU^95Rx`3QTw@RgKlFwTXI<~*dd_!|`ZGL?KL4-_T*o&MQJ5__jp0%Mg7C96 zP4)WoUR-+&PV;MpK^J>g$s(NS`_UuQVVQ^N@oOoefoTE$3If;j^h4s;Gf#Y!k|25C zLYntW-XFm8gSn+Gno}K4D~D3L`yrZ-JO_6n9~X-*j9n`VX!+~)HKy2*oY{(`!J?8q z!uAD0buIznf0)0RZ783xuXI_hJO#8rbDSK#g4UkyFWE`wPT_rKh79V7EYMe*uys^) zg<}=BMg7r*GYVzLQT7j#bzI}Pg;>T3^xvmP@Rjt>ry~3shjK%->^O4 ztk$)_QF3|#$_!prZeKD$*p(*%L4Adcmvka2skYBm9saJVebo@jwUOm|`&h2s!B)jy z?4$Mwc&4Y#QyhBSi1t5?oCnP0JJY0;vc3hBmdCmWBgcmV`Gc1$+Aj*@9EpiotT<2( zajX1%Y%=DeAZZyX>CmfbCy)qH+%(7UYg7%#Ro%v|pk7_J>;qE0*}t$u`9!#q~DfCTVlwIuRwZ{m$gY zo82bl%b0J;a_ow$xeD6ls^+$e?OQmv<2%}G3ebR$nD^Wi-;|-Y2Y%d~^mn4Z5=%!J z_mOS3OCp8-tCFx<8-QKBFeLhM`cC22#RELLMkM_>oPV4A(dX zLRSxgq|)JXO_)3^UKTCaSG+R4TF5f?$+>5$rJ<@wf`GEK%mJKrbmRjFccE!_&%7A5 zyQQg;puqx#1ysfQ91J__Mp5nBzHL07Y?`F^a^pA~mwGDA(UPP&%5!lWQQ1Pw1rcrZX0>9)sUfqCF=TiOlOIIPa~cyg595FB8O_!p%k4BH zA5k3-E<;Hl4Gi{OnzSrcKwx9RibL|>LTN+yqmy_(pY5z~*WR$AC!EvfRH}^N(8x%l zH|J(k$=tFiUP*TNwv)vnE}&Pq11uBrD1UCaAdDa1{eq>efcI;dq9j~yhXqy0V2`ER zkU?AT{=|;6Bo|3dHsKs~1u-kQN~f*ho|J-V>R#erxj3fz=>QG~7HJ9hcABw?u{5 z4{hNRWWAU`>j6_ucj(gvyK6^1#Rv1jJ@whhHwV5^a|mx<_yuFm)Q$ z`<`eSq)B(e%Us?rUaacBKkQwf`ET@l7$pkWZZ4~Dzi@x6aL4hp`5MS(ee5|g6x43+ z@ip*)Ub;*ntJ3Si(%#1&Hy39WMEB0LqZ{Jr@;cGq*z5JP>IOEVUvcCo`S1&YlpI ziWr|Hg1008Z{vE1<6~9-QjiMekY^rEOmK~7M=E`$#UvU+M9nH?2B-hSyoRKV8xe9`yM&o z5m3Z-;OSwVNpJ7He1=qUh>LsM*{Ipz()KllkjuisATl0O#?H>cTB;!M^yniIA)>BS z&EghZOa4tNLp1IKnO|)qaaduN$JOMoRXHs#^N4g75v|2=14B(^QKPYQNq*lCpu6qP zBH51~mo!y2I+HU>oz1|Ot^zRvegQYI4qj-`7^6s9*gr6S0N(3+9RR9AXGLXO0F6R` zj7rxiD)rGqen)WLHHba4#-kKEl+xQg5g(I#A8wRR>^6B;{ZYjMr zq%6rFH|K4(EC#cq(q+9_i}&^2bT|})V#sFcu#0O06O~V6nTXgKB5?EkbR&jWQ&a1C zOao=D@i^8-tTojiM`(I<{$dQsXgHz(ZQAMxsj|wtpNKW^$&hhQ1y^+mA`<6wGu4WE zOEB(E4=So?WIAns0iocW5P_#>AqhAKf9|SMncDioagaL*&o}hQ`G6KDo?fj0T;(xxZ2`@3X%-4<7KLPrrv1Xg>;JqPS>F~*ytje}LZ(D%e90c4*CYhJj1+zOk*%OiJu{`Kq)l2H zS69#J`Nid}7*O2c2L2WE?gb5$z+@MSUC56i|2hYC*dBB> zVWt)*2V!_IsHqdgO_&vh*~KM7;wkXBdIwB4x(6@bP5p)E#(ENRnGV|F4zNa0K=69t z{N|qYD;lH!pt$I@rm!qt)vic5jP02IjbE!qb>uibmR^I9dzpxwR>XsvPQ;9EPHPW= zFUJ9yaa*=7T|%w;)L+A>K0mIr74Z=CGfoXw@K?6|J1Kl6kJ_44*}`dw!GR!n zM^Dr!nUa*906ek;RKqe4i-fwwNM*@l3B(j>;X%P~0Yr$&Jhr+aoNBo$#0kF(wiP|B zzX@5CJ5(SN7Z4XfUuSfp$lx_V)Md>C64fd>jkkku|tD<}dAqy-l= zID~7K^CdseD#xajUbUonT@8rwxaw_`OVOk({)l9u!e;U-J-4aJqez(~<~JTLnMr~F zfn|=m_UZmUzw=GMqJhQC)El&0iGXA#gZ0a=xY_31qEKIu;2tyNzo+S#iKw4Q*e74#MD0*Pt$KHSP{7{5}s*UW!$o_jK_ySuc*q1C1K)xJ4w*?x>78a zRNgO+69G=VW%b>4B{nJ8c6Po#IItuw)3M^z3`~}%L`=m+q=kRjtA*4X)s&mWzA$aHr{Ca$nH3Ea4 zat!?XaV>08#lG8z9G_vJ3^-y6+mRBunA}s8J;ZUU@D_+X0d|@4X8M*9%J>0)jq2j!EkY1zr1T)!E6trMBn; zC9TrZu=X2w=g@e@z^J3VN-}Xy150wVymJ`K}5DyUA4sRoUOu4 z7GOfsB(6;QV5%i9>k_$6G(}x*V`5^WsL1SfX9w%uJJ4<@7U{&y{?!nhWteQ!d;`9y zNz`3KA_8)*@zWxNi``=z(Gky#LR)%l9Z|v)_JD$(q2-Gj4o1zm@JEmz-H6oYn+7)U zpofdiiLBk$)|9N8jUd10Yk&EFQ> zYXT+5#4bps#=fhPEx9btGpOkZ_KDZ$L#7b~#PA(t|^+b9tww{ZiO-WHZqPIg6!^bgCadV!}thbg^Pp!>L0DNE{=?#f)ok2;cSrATtpx# zfNUHfpwRNX3*`PjjT;!p_P%@tZ};^l&KcoWQq6OQZkFqzX3M-htx`n!7lGTsN~L`Q z_?MyIN^nSJBxm>F>K4S&wd=+I&m%^8iAA9(bOp(w^bM4kKhUq5p)Z#x0dR}0IC zQI+xn5vd6XP9pZSl~549iuhX)bT*`f_AoR|%7H-OLwwS?WN6X1k!u}OeNJ)rSvXW@ zVSr9f4)5oIWMH^C{6<$+_X8WEs?JRamHMH8ZpFE*B-VDpx!?W|BUzANvMUesU09s- zE88h>%Ib-eY!{o?IB-6FS!{)0UnV5h{)8J?KZ~Z__WzDHm7Rlw)P#iXh7ny+hDOy&$MH0(ePjg{*X6hr$9TuyN?gnP{#!z_@9Wje@n28<(LO9L z)3%{L)>aWLO2WCy^FyU=C4ODLA(xufw1CmJ*1Kvr{^T+)_vh+$xv`^F0xwJLZUHUI zZ$?HAet1)K{DrCRR?w{#PH*0gYQ7YuHeR=_yvin!Al52R?Q5Ex++PG_dgc)?HJ;jH zUa`Gt2m1`!PqG+xRkGmSI8VL^f zRPU3AAXm+Y4>AE~u_Pz2FQ@Gm^&esR?}dGQ<-M*CYlo^PPe-qH>O2k^=&gHp{wBZ%HUp79 z($QDe;vYyU)6|xa@p;QK+eS1tbzA1+7}K;VXE=H-@=&uF%C0=Wafobx9>0~LS^jhx zN22)vB)Ut-h1NPSXa!%;&zwY|9)4ge>EyqGkSw1nduPxMiP^fR_WG~JCuZh+l{)w zor;AO5Im2T*5N|^@vNlS@?=E<&~*`!qXzzzU9Z*#*=2lnh29MeHl<6-)|GSL>+1V~ zDI!Bz(alpAfn4~n^cC@N*Ugkwt!+=k8}31LTgKaO8d~>v^bKCG+2&`h@rAVGle%ER zNwA082n<%Ds#?U1acpla7cB23?^D@7H_K_oQB@=`T|tMe$Q4tbNEABa8MGLHL=BeYv~`OmVj;#pZ+ zuf|rQgD<@)Qa%dqkY~v<;q^k_f*`Uh?GF%e#V(bOKiz;Y;t4^7g&isf$LcVq6aAbK zyww{J=fWQ+B)_28r$Vo4@JMmoZABaB#)!p(+zY>pWR1(U6P~bkyq)o^00Y002Hf;T zi@jK&$D;OX?^mf;i3UYVV-x~bPTI?pGOeOtEgQq>2xIRW{V$5ck{G?39Uo^G>z*-g ztN_7ndvf2Vz57yCe)pkakvh1*hjAO;Ihq{CN>kbR8h5w^fk1d{(gyQvvihA{kyvMb zrY_dI*I5pi+`2PIoRFhIi-7g3qQf{g34%W8%@e^lT0MML^goYgSVD^dkeO+UCS@A zMwn}1?%RuD{lvuNUE{b(0zb$;%Hrk8 zu^#*Qj%_$U5+1k(o-7|(%0}PMMrQ6R^%c20S&~Yq=+UnDxIZE2 z^*B?8u%9BAX$)xeuVgh-BaWF!gDNYF_Di`O9WS$%J(Cml`lI;pv{aOe0k3#;rsQ$k z#T27xWEv#C1hTQCugeEg2nf8iAzZ+Kw-Ru&P`zAAk%g+b~U_RX@7k& z9)9*L%Q#XK{l|BZ%FxIP_EGL?13@}g^8Opv8U6ZX^^%}hvN+ovD{D#RT`+~&4sPTh zQ+-pu=<`cvTB(LX?&i3>uH1Sx(dQ3Hp!#yXYW$PCtAR}|j@YZKJqobs7^ikqUHLV- zRFU$<>4+NK2DMxTY< zPUp4ZEzEkRd$4K3J0 zH#%lV{`XB9g27P>=&KEX5zXhC(Gr-B4!d!V53jQ;oB^*Cv#cd?ak$>T8rQ~#-w#M0(C}j9E?Tf&iC=gM_F|#bvi6xI zWtya4*OCKkX6C^RSRDioN|MVRT3)U&D|tt(`aGh790<+cbjahSm6rBx`#WH;UZL)Z z2B09OdKd@^wL5IS%cE1)BLB&8T zVQ2Vq@zg78N|3(UctP?WL}VOf(7(}NN(yr`@<>f()3wOeb+!>i``N`k)rl+Ghb0^b zgpG}Mut4<68U8O<9Eu#Dr5Ojz>MoDG1fVYjO5a@6hWgG|s^?z*MoIYINo#imDc%3- z_Mf-AISw>CN1E#0&U`XEgv^2?Scf1*WV((lyv@PB0zT*cN`zTYW+hSsWme~Q>?_Ad z46BUo$V=y!dw<_aJU;Vd4K@<69902q9XnUoEME5uz@1mr17Pf6I)Du+uw7zfWgSg_ zUUX~9MhquSNhiZl_(N#{M;vFe#gYgQP-;Y zscK4N=(CEk?%@r%`echl7&cg(jE0Iz-UXA7nukYg9)K)HMxvp;k9_#DT{sh1wrXFV zwSntBJ7TdkGQxhb*k`qrKi_M_&z%ZiHj8|+eZl9`@)De-b7^UpI~`MK9oe*aBpsNs zdX6Vk7@LJjOn!NL3pjxk&CU0I!XiM}k{I<8+l$q~Zl0b9Z{Gr8)Hk}n5sS!DT{gEu zoM%a=>BR=}QUCoXIr4uy;dsI=lf|DypAq&U$cOpRc8sE75xKmP|2DvU*Az@n(Cvh2 zuAbWO2^(1d=GP)@bVYlT7%FCFhmlk^V08-S;^d^_2`ujl#)0l$_4#y9nG--=3~!0oKbeUzYxw0=q+ddwW1uey0+MewAl1>;Z>n6eOR<`6V(dOYe24Q@77F^~I3izrUYB{sa8MC;$lsTtMtbY(_>#QWB-@o(aGgH!?CZ zG5Iyp{=pL5j$$-}|8ET9qlbtr*4su!L=<2F8V_!Z4G|q3uW)5B3IFTXKpa3*1pNV; zy|A#bzP`RD2*1X|uG^ta{HHUuDDA(Rt>am2J_=XFi<31u?#ljuf`KqKAlE}j``?9) z@&zA6?DxVJu939PWQyqguYYtgs#<7dH2a^XuK`pfG94(r^!JU||@ytGtjv%J6!l+OVt=x8e2(A?Y|fSn^z z7&>4JgwM8NM=Wgl!DDxy8_J4kTY=x;CDRv%cx?+aIjbn);p%?B$EX!r{n^dq`s!zO1Qe{aT6M|ft`2SD zsE225lc??3q!;7$j%Qo-R#^2;+xdQK6s5F2hZPPvETGCho5+is__{9eiU`tXxLx}j zp!a&V*z$dla9?+lsLlCnQ*NLXyR-3k$Y+3MOU)DL0(9$7+h6&CxZ(MqM6B5Zg$t?7 zCZP_bLe^Dy?9%hZL{}Kfjq?Dtq-HARADwRWuf#OHE^S+zroOoAK964bvHU-6ePvXXUDWmn5+W%IN`olfDF_HicQ;5& z=g=@92ugQ{fHVV0NP~*hkiyX2-QDludEd3ZKi~YDHTRmi=iK}3v*X&=_2Vv$r$_F! zO)l#9aCAOjhU)_HD`f@pIOV~)M`ROHeuo;=wOzi4%*yJ@BtrA4S8z%~8C#mq2 zhp)RZllMY^vTTLE zJotAb2w^u_?lSVj0>YiN4BoMTRA4N9qy}pBkOB1*I#pZ@H;DrJJA~Q82-;UC>FLb5 zzlnl70!2I=R{BWqGS9b^Am^S_Yma^BwpZi|;fzVqoS>BR1G(HH) z#rd)sey%o8y70YxSx%DiJ#!hU@B-98d}&q$3e{No7uZvD2!mZkZ0fKSm-W{jw&oPz zNy(oM@mU6tM4&(HGNWZcBf*i*DgHFj>@Fv+tHnx<18BWBj*ZTCvsJfxoYrouOmKP- z`F!OxTcfXps;EKd?z^<1xdITw!NJC^uC3M5e0z^e+$1Q@(ABcMRMM z@oL#@QMC6>&u7v;(%Zg13@-O#C|}Ce+1Xs;d1ks*fgHq&e$PXJ=%$KqdExpCeDz^; zeRlz2FVVv-l;{yYKO~3z#Ce%P{YDld&NZt=+#V8)lvLnY{BCGShLs9JHc6qhn+J4N7o{YahuYvV-M za(zlm34!$bv|{7xb9-06;H`ZxZC-fX8N!e)*-)z~IZQ2Aomb zx0r6#l8j5ICvxi4j_ra2V^1C`sI>-YAu720o1h!Ftx1F?w`?c6w))2ly=htwowt-T zw)hP6ECaBQWy*p8g=AM8?Xj{q+Xx9#*3(tjQLE_W5i1loxt$TWL5*^O6W{nG^(;Ai zrdK~s>$|PjgXa$69b)3}7hiuZ9cVnU-G0x6Qf>QM@y!2NjkN91qOamrA&G^j)D{S8 zTYM<*gWfC>^rqmnG&~D&xcrE<^B%4FQ{L0u*Jx;X=kvFwL)~yc7rLoU)ji1lU-@2K zc@45<^@-oVMO!k_gDApY4L|)Fq)PSG5)Dn)`pHN9acmBTXYtQda2S|M1}iy=)U{Ax~I(8-|IVS}`NUW)u2udanA*xX5 z%P>h}yQWENbVt=>PY=%}^imxRuBp`8SXD1*?&Ao_+<*GKhAeh>K8t_WN0>(wOEl~# zide7apG>;kUn#?RG|!y)2k{D{YOBF9TY;EXM3*!vH#c|q#r9w1SKeymV3Cm zoL-Jgfs7m;@jZ?uYvKT%ss0J8Al6LALY6>-Rvs>uJo3BOJt>O!W2rHE6~QPr3ZziL z&KlDx*2Q?h_#larrDDWdKARkVa#7?ib0hjHr$fEzKFyl*2wrR~P@JZxc~H!?sC1CA zkNa6&T*Gr|d6N=;5w2Ju0t-MtRhwosx1M|*h+b8yaEx85BhR85fg2PE&j@H?2sDp5 z*^l|nE~V@+di!U_mr=vxl0UbW73D-JGBQ#<`}_Cz^)bgPiHV6U4H{Gby=pUyZWFln zr`w;IOR%Ct40R1vj`ZW0!!Q22v=dN3T#gcC74`j1uc98m&TM>AMMI1_E}+eOQ%Kj_ zLedOdLbOc`;K_}uKRqIo$C(n2{^|SckeYjKakLoC=}@*CIoenj zp5wdUA%YJe_6`jb$|_x?kCl(usJ@5ZS1fK9zV0;@t@c?kQrjj4olg|)ySx=cjVbn{ z?FP=U^~d4)!NBw=*#I2FmoDmT47MU7N&N8Z@7c1fEVQ~-%LDAzr8l%6Cwe^`8XfvA z?Zs2V@U$+sg%1B|8$hT^U$cG)O>86>10xfL?=F}1NFJN<%DzEedT=jM`N;B`i~z!grMp{h7p(3RUv2lQdeHC{QdR{ z_KOonW0B?@I?Ss%^Mq$%G7t#I-E9$hSd%;LBS^8!z>fak)aI8mkawDNJFXT#l7S!{ zkK4cUqU&iFL81ug?X^CEFzQRBP_aPx3*$BS>$15cbC%ku!rL6|S~#(XUYf((7 zk+U{kGO?af2g>pTe80~60lfUZOP!}@RTfj5jebr^R}xBSQOalg`~!ub-o#C8qrFhj z)}@CNI(eXJv9$*VeWND#;>Q{AJIhaGL{Bi!8PoG%g#1hFYc zue1U#U+L-T6-TF}6irJj<59^SQB3xw(Op_UeyF7c>x zduUP*z-2>qSTY_OZYHV%bvqx6$#%`S#S|3<*qdk558qPL9X+{|edy?INI8pRws0a6 zna`Q@S=q^%pUm3#!-o%Nc#(rlqD0dI8Q6pW9E;5kP7EmRy6g$If%f+H<|q7{cgqH!OifJUNgr)K5VP-JM+*>k@DSYu zjjD(lc=xBUt1#j74!LHk5DaEZQbH8SyIwh=2B!uaQ-{ss5>g$%`NmENgbX6{UQ``R zDDm@jsq42`N6xEx=G5$AMBrQg`k{bXLcM*jaMIn@9 zNGag@v36E6$S|YBJ{`c5&?_0^@lOj!IT}8u$KK2X3SADZeG^)OE z3fBwWGXG}r;q7$1)h}2oBjF=gZsg-0i4B;Dy!4tjb`3HP85-u|8ljzQkPiC#H5trA z>ZgXsg+^7?)n-)_kZ2GZmLH4zKGjjDsxW<#JjKtoG6Qe_%;C-E1=sCbetnyODPOL` zitS&#y>2IGa4o*bLdJY@FSOnx<#|ZagETRfl^@6WE5k+8ocwFrCr_0z&S^<=z0<)n z@WkirG$)&}P)*z-S(viE*HID^FS(yLLg~(IVT5JRw!!jlVPUycy7pXrj5CN965S-H zw`$wU=d{I>LeDhBzI^juoX%Q|=Kkobzp-@huxXq)wAEa5`(qnt)^e?xG}K;LdkoWV?Q&8Ui(>y7FCnqPc;m|E zdj1BKMD~!$M+;1~R70|Ne-Ge`6*F;_O!&OD=1kpLS5e~R`mB*4zxriy2w^q5H&+&W zy#~jVz~L4Oe$zI|%CtjdAccYXEmN+DoYI{8=M1@-?c~&$|0Ov{-isYiHZ?%^yrF=V z?jSfj;*l}C*J$Mdu-TF)XZq;?(?k7G!$2PDT_&qkYIrWbyx3ugF#EkC(!ZZMMypoS z7iya91UiZeJ8j*?=Y>72X}P@NTHDLh=$qjYkKTNi4PH6vUHp@rvG=`ycGDsK89BkV zMDSf?9w9a+RaU#t!36c%tCmP~swUXw$HgOEd(NxBbZKlXSKBuSk`cyy?Nm>`{Ez9`;G5 z#_%bo_rNjmSErlXk}M}j7+`tz7>c_mzn>Cpx&zdM^osT*w##!KA_!}SNUk=AG)mv# z_#dn3patGjP8}hGPX3co95GGUQ3uW=<8MW#-{HA??AC9G?W@D{^F4lL%=t(>D-Gcx zO|47fK|UHmA`P3!RqgEU?bWGiXvAorFz`J`;)p2{@oR5BVAr?s}$Gs*TNHYq&Mv;@Fl+|QHDx}>xhnVy;PftGFrX<%T z&ooD#6nX3?s7u+;XpD_d#eW2G^Y9+{nJxg$Wax|6`ny%S#`P^VaTR5e(L5j@w=Rc0 zAuRf%KWB~}9O!i+w#@iu;xaX8DqaSG1vz+S`cYP#l$tb!o8vfO5*kkjk9c<`FNC5suJu^jYNzzv*tN$;{!EUcZIY{qzfA; z-MJGsY!&m#gr6HtR+mqu*@vgg8SCL%#ty1S?bs@I?ADRHE zn;(g*oK_n zxcbf!yX?3=fP{Cnyr_7N3n%0A-igWin&mgoTX#eB3Nu@=4M{?+VYbE{%J>jgt48%P zeKqy_zfg(-r>fyDfouVTT7kQbz$=C)IlC&8tx9p_HUQTLZF4p>R^ZTLq<-L!C~Li8 z4Vv`*Q$3d8!~lULq}v)m51W5M6cnxxckwjgx)RS??ryzH>vq%r^UfhWiY^l43ZBDZ zkAC)yvXY5GAlC1I3}S_idv0jI;+y$Bhp7=eizLq{UMpJSxg^%EWNZB4t@b0TH{bwb4k!tkAu*% zovbgc_ZH`qE0wwGR&+Wp&`65D{T}S&&fGR)lPbzW$3#rUW7dJhQ`Mg(H-dfbZ1iBx zzRY-5BjeQF-3|F=Q3cAV-j~f1g_Z;jKjYS5PZK*pk4&H9%z9kvE``U-ahOhG&TV&< zelxES?pO9xp)JE|;&Nxn>Fh!Uio7f{nm+$Gpm=w|v6C1ylNY&a9G06zjTGOn3W{bV zd1Jo_j2jZ2nb;ucgU1jTAdjUANP<}BSDiwOA4&`$dLGxj(x_+9lFQ6bN_zT~_!&}* zNVu;_O$AW|FN%Kv>ADs0+#I^#g3Pm9ZwD1|B{;JrYn_PzJ*30Xt(iu5CPA{}%FRl= z7IS`6y1hkNjGFtr%b)$ni&mP&vd=l&!r;WY==OI-`|H4DnUKgi3u z9u4k`#5-ApMEMRoex;Zy`@Uy-HvO=Qa%zDmh#UTx3->siLz2d-{!FcaV&J04PZ)R109kR}RbH{w- z#Qo8>um2dY`FRc9FY~&H8TLEz7nW|UlmoORqz&G7UmCnt1>JL~Fs|{;(E(C&24ni5`ilJl+wch|?)lad=ve;3+};*bg2DR{+%nu%IZ^^#M$j@O^n zoR|+};yRHfoU*&Ohh(XCI%T~X>pe2Xoe-ryMgiCe3|z8r2qV}{jdP$V=wA(du%0n>c5&h2;!@k21|{tBN>fpxw*Q)L z=KYvE_!ZTmciYDs!w`0t{!it?I=q%-M=!MWu(zbS1#>JhC3`|3u zyzt`XPv|VG1BGXNJZ?YSoSi3c+tFEA;J8XB){A`Z2{ilB{^mc-4+06AtvXz1a=fr^_xrcmdC<4=btIB@le)O z{DvY!(Wu$$1RTTS;^IXCxe%)5kSxLR&wUmSlhs!J>2H#?ah4{eMYdgG+c=VY#IZVCYEQScpEA}~Y4o1s)N^Vr%d@_M9@Pe)hxH7#wI zH!COr&jaQ=&=!+RVr@TNkY;#@@_VAn@+A#Tp^*>$*RNmsY5A?~01GB^kIl&U<|OkA zCdgmGkCS8wy6;}?wH6=%Gb_16cr@cLWWx*jf*8i!;SB&90k1~49a$L}0<6#atb8IO zb6s6s|Nf~cDdF8=##9b&9N%1>4Ww|b-G4`H%HaJ=nKx}`Z%@Q+TN`NDmv%~)$G@YZ zXJPT(pBLSlqYV7?X|&8BtX2w{52o3>n92-hrg?!N&~(4zgy@NXYAOjhQNWZ8lujwL z37=Rh0kD`J9EDvzl!tRo2T)2(j(9_uoSWcI~gE@3cE)pI`M98)S z`X=*`!7R`J`j*D#HKA(u5*!(dTH7Nu<*y|eY1r8vR}cwc+$~#?I-cC$d3kj;6D-0M zxnMGLVoBN*;@2L4JOIR@Sm_-{wzuru2T=9vmM2X(FLUs!_c4__mGj1quAE z-RuR`(a}*?j|UG0w5d?F93A(;u>z7`;1;SJfMTBLW1I>L9eax%v^M*E%}oo~dOG8PPS};Re8O zT=YOusNuXLKgd_X6DcYz1cZ1RZu*5HN2jMhYilXrI4^@iUUD+BeVyx-A}(LQxb8n5 zTJEeF04JvC_hSpLbLGqY)Y=%?Sw0Zss|h`M+-cx;u!g$bs3jVgl{MG*#T!5HH|C>9 z6V+B~K#?wMK+BDPhUO_2=mq0K2w%&O*E!_^I2aAEoogyQ>}KCJ&%8@d>xd2R&7}7< zDrRQ1BdeoRQ{I5DmS0*LZSVvBIn(#M|=`Ev?5>)*>=kTv7Jdn@`HyZC(uw!4ogN*&p2E z>fU)5SG4!UZenS3+e`B?S-uAp9Rl5o!cN_MZLitMyZZYRMW`VyTGCeP$NfiQhgsJR zqvGGQzKu^NdWwhIyI;n7X?rSU6NO(rnziQxwix?X- z8L$3}5)u!!O}=~|Cd6303cs|7z$1h#;_WtT4_-gg>gq^$n02G6(1meGJaJiU)Q_qA;IdvdV1q34N=*y?A zx~Qm1n>5ipAP|Rv&5FNk&y>XMiN>`zp=S>`1q4zV!zqLsj=Z~JU*bh>X(ZsJH4gys zTwzlLlUTFVtG%~u#{(jF%Ab=iNA}+6=q>NqP5RWt=~mk=JJlOpu`zIP9R3ObeQ@Y) z&eP$IU40ZWMtTm8A18B7_nUr66-qvL9Iq&>q+w!WLhJ_H&=XO8A#__Cq&!tT{PITX zQ|Hex21{OV_LSQY&f+{fJ1uAPwqSbuKmdR5MyQFc0miYW!FcCQb1Ti|CQzkrB<=w& zq^K3fn|7lrh{yud_T{Ld5j#aZU_T*n9u<8vynL;iCphpJ5Jp3fgscQllH9M=M$7fJ z1tXNb0kQ#L5%fy`{zwuW?Gt723IL~tMs(CxAikK8jDy*)%%zyof+qc0ezWGI%}n!1 zyV1MoJWsruaMz-w@QzEghwP%p>eQ-T-KZFvd{=@Lf^OlA_x!_oZlK*(Pz*CMHGM;- zc$V}0UB$s$RT7Pcqb6lpSu71qT-@DBg2`PQmhuO%{rLbulQ4#RQ(3XcEtXLYySoM6 ze!9t%5CT}FuWxP+_F7HxDBga<0y15Qi0?v&l-%HK1$SR{#_-%)_TY*Zx32YuPXtZwc z-rnA-yKe!u-=w}?0x{X|(}GHtU2)2Q^R|pUdk5(S2x`<90MG)QZw7*{mLCKnl*a(A zK-18Gg^0`TwP!IqIKEaShbYhk!LC*8ORh9-3kL8LaAe|d8U^9u`fb;8&YhVc`fax6 zV3u(TX(Li*Dzj|_ysHCRHdAamlwRB%^qC|YAFDv0V~1>K_aV(d2{_6cmD4s?RwSN{ zmg-cwZm5<{0}<0)UEIovZKtIU=mEVQFJTg%ATGpUw`|(b$YvWn zUcQmhQAr625M!uPMwFH^n@fEk!otE@=8FS+-aKTutg@2xf}dDLMFoM2(JaPxO(aVgBH6Q z>H9Ofu%LR^#`kR3z#Jp5p#jFX#S7a`Xv z@=<-n^?ejhC15vPNUhWgh#9KC8~#v#t1?}No^vj4>F)O~z94ci;<|}44we9BTLjA2 z?(j8M`-fkI20nCOD;wIqDolRvzVoPD!g}UKvLYDo?OeL4R*i{amjWZmDl2`F)JNyeoVZ^<1`!rpRr)w+wd(9B)hx$%N0JkLQ`b zr7N#X#QPMn3`%4MgKe%*zJvKd}%mt$HE8)y?}rKgO3su-AIyQs~|&J??9qO zy22XpI6U(wwJ2JjrhRtbnGI8)WAaHAEzCQN&-z*8)sQD$qUJY)pJK=o*KL+l+SQH) zOQy(X7-5Mb6u|SJSY6tC%rF}M4K*>u5gH5J z03=q2rl(&eWt!2|Y=Q)oemF^;?J1H+825hwWxKcL!V05pc^d0Zd7^ z92B3=t*k*Y?-Pk5Dde7`Jw!65rnWYT?@9!Op8&+Na^(CnNO|thf+JAE#v|i@$MX91 zYe1@d))jMebF=c|N^q|!a2Nm;3^i3%hjJvs3J>p12<^Pf(=wmN0FOca_k0iv$gTPNR0jv+WH}Cmx z8AX$cmSf1TKV}%j5E^=Vq+r_%DLq6-U)-M$1zYSX%uD%3<+osTLc*)P1XRfE?CkjZ z517Yi6$XA?D0yVatgm%R(D7igU8l;T#^X;K@Ub@K&B1m}1Y!vA7yviC z>&4*;2+7+|K+@-1M(*JaR;v{hojh}8As?zHfS&{oW=Pva zi*KXPamatUSp`LtR?~ZH1qnhTqS3{RgvFhmh%Cb(Wd}rme_+u2+|$fn8KE!pAiO1fD9b7 z2qi{ZU0M0tHa)%=DzKCUtW#@yq4#K?H-GV44bkSk9}$eC5F#NYM1?FYER4%x;&7`Q zCGav)G&MKZ0#2rz?h-+Nupa-RV?ux;J-El92qprO`A>m?zP`TR*#sE*qxYYJ5zQx< z4;dI3K$zb_!HtPS2CNH2`vB5P$!D)$UQ-IJ@%}nY988peZ`IY@Y-~#OPZ5wZx1D^LiIEJIC(p zUL4Dwm=+-0p@G6$Oc`RFX-+TS8e((SOYZ`?@*1+fCGr!UNqrPGMC*ExTWN>p1U;k$< z!%&7YJBAxyP1ln9gqy(C>Lj|hYnv=fbdp`ibtG+fQr(oBl5`Sv(r#MPPAAjNx;aUw zI{9wFEl4`uDRxV4Nz$24xjW%bNIKi8bSK@(Zq=>Iz1f-SPP;Qm=iFJ9cjuN2RrreO z?m>F5D!Thr$=#3i0aaELOO|^d(lu3a52{J`kgB@(zhI~-HGRua)853*gnL-#a>P9% z*9Y7O74AIa!(<>;3e92`}BxDjBC362af%}qfTuv8>P4PsgAtks0Y-8=YlR<)960!J}vXzzAg`Sp6Nd8J}de2o#(pGyU$B{YfZSX&b#NO z#Tj=-9aRtCGTayNj*hA0w+waM!;0Z8y{ODrjrr4exkE3^n~~M)f!dvZL-87{sxGd{ z{z}kdBD=WOYX+@;uhHS%bB&#p3m`@0yx(wDx-&p&u-K5_Rf?(beF`6^3(aOZz~ z`<2V*p1OOHSr3t%CwUWO>G-qz{@Z_i{OD76)3|HmyW^7>-TBWU;Tf)}4A=TOW6@Nm zvTmhu*b>Ud;@B#w99)wsrP8=MDx@)l4LvE=F91DR-T?bmvs z@hrDhJRWO(isN}-jb;*6#-Az@sYYB2+Y8`iGdB0K_Uc4dCqvteErg=t${O0 zVaWC{xq&mS%t2aN$juD0sE7K`8Ltmc)m>)W*pV~0P!Z>9aQeq74Fu!4{ z#GElG2I*lYEQW;*YfuUk!z^15OT(O86T`e*)5s}o7*803bWm)U!lJ@fzs>%gK{?EX zrs$t=#i6^tGXG1$~FPLgi2Qy*jrm38&5oY%qJTSMd z!975LX;U2C^Xkq-5D*xg2 z!^sV2PzjU6ycks^+Q*-)z1;Wx+8bvU9$7@$ zCGMDdquIQ{tka8wdoTXUoqIgv+n7BeMvV|l)l_(E?;YC1%?uME0Ns_ zyl#{X`Wme#x{d3R<*lNd>nn|(syF;7(Og-J68>5@N_4y)0AeL_8oIyMQ#!{HIE#JV zZ3JLR&9(SgM-%mCzq_&)c%0OlZZvO1CEp7$KEJ73fRcVMO5w#UwR%e!KyR@>UmYD- znNFSX1D==*xGO^NOd|I%JzGy1!Jv+|ZF(kCO2k;hp~!8*F9~m! zcQ1W3i7n#~cPsh0@SeYl1kMrh#%NnSAGjuzxoqi6f!$6@nX>_zGDrn!mAGvW63EZA zvtc601$kxPP7G|E#sbjE3KOBd0lBkxX8W^25l@%EH?}`p4kk9At!#f*&ZKie&ePKG zeT$#(0o9{)w@(29o5||vD1XgsEnVThXf|%lCp57=%63~Fymkl)cqL1XZnuGw21@j2 zNG^lSPs--kPcY*nBwT;ZHHi|hzVKX>U1{h>*TW&ftCkf+Z{YX9*X>t!HKb z#9L=OY)zw`P1yRNK>@d&y!Fp*DsI|9W;5O zevw3u=$~ckWfJn%`RNZo2|vk7uaaCM`ST>Nk^B_NCrCa?;*tm#&o4HXJ;;^RG5jb^ zo(@rTQGbdxK27pE$zLF0@4F70hO%ya9$%kb8{peJ`8av{^n12#gM;tl=Qx$SIk=PA z&f)D8WbHk29zU2RWGVmmz_RM~C|j>Xa0j{}ov+td*BYI;L?n292!qo{NXUZp3JHfE z6-GKi)9ZAgMr^T?y{tmnTTg*dSQddEkd#a((g)MW({oZX4{_)v@XW^Lj)$~;u@PKh z-WbG@E{3UD!y7p#Ls-VG)9O5!@)L@~Qx#xEPW*jqggq1O@ERFJJ`Fq}ELR zeZXcB3IlkZDng{6=s`IgO#Bo$qK&_s2>@~B6x0YmeG*)cf1f1xGa&Y7!ig}gN`PcO zUB=TDjAs(U_h-W_TaQy=1|^e0bvT9ZG-}L1t(XLVtp*eA*^qQjP29Gi1|&DkK{c$j z_b3zjdsPLqpM-Ewfml!pOE)cWt6X~@+Wj?E4a-+CF8ymE^Y&u|)j6YTfU8dh2Uwft zpxX(o$@E|b>v>-|HLPtI8yJ=U{otUQ86FzvP?HPpA0AesE<@2cBJX%=_yDgM--GSM z@FDDi`3)mX$@{a1M@MB3506RheEWFZo|?T)5s3G8dqK_38EOx{dq=q^)V?_bWz!rD z-qlH|vH!O0iuMV0KuTxcU%L9s*fpo5*MsW5Fn<$IsG2%>i+!CKp58E?Gj7tHa_EAv z>3deKb_cpTl<-y;Vz!Q~m-Tw?jjN5$ns;X5@x=j@*4(vLPxY_WeMq)ZVzC=UHr0%+ zYL6}s)(e-YY@CVpkx$(-Puzo6iKg?tey@JH({ComVJ4U63^r=6G%e~~&&E~mNXvOTs z>BrVfPhSP+S@IN`X$1b6CZTNnq2x=Ah zj_GEXI{nLFLXE3ScM_4ca56HN-K0o_+ZbbPk2tGZ&43Eall#%<|3U-C`u#G6I)u== zz~;hPEHbpjCl|JA&p#rI8QC7zD$1!A>@Tg$(0WvqTnkzq7({?Xbr^9vDt^5wztLTx z){>z%6e}zFS3Eq{d>Pu1r`=3Lsk#i)Exg+8_XFrTuW`FXiB=C=1B%uaNS*zqQz!K* z1{!6Mu6tOz8+`q!Q(x5HYK-bc>nPQW6+br{PpOMtAs{V-P_4zqQ;G^*tZDWxkF7V6 z6PxF_A%e#rr4ovLQD1CzI#F^F3_Wm5D{DZSX1|AtV|xbf)UFJad5epBo#RNi{6$L8 zo=6f@w(x{V22d_<_*lj3tzetb0j_UErDmtKB6_QUL|rc;SLBh%Zmy|Dl=E-&npbqc z$J67)*c#mA|hW_%5dVQK?t^ZBFT?CD_|c;vJ*VP=H94p>2UkPEX^X=9Wh5UscFawEKb z3+y5byoFq#QozN6;4So#FdN!o0fwxElo2zF=p(1U2lOUV;~Lt?En)n5pl?EDZ(Bh@ zxD7?x+)LJFxIaC0JnToC3|K@|lPD}%`}-7c%roB=&Wqymv| zmqHHz^E^|qSf(u z)YR>S*jA_246qHghSQtaHhV#T7B5rZ1X(Z9a&@NmsgdF9bwP+r`Y*A$vu?dejtACjd|0>CgBwr!<8p+=w`MV@k=iGD*{k>+Cp&6y#X)SyD zH(31ZB;?2XACvq;k|#;NLGn*X{yxb+1%XX&^FYU(?@boT3;kzIk)i2tlKc+IKPUNJ zl7B(+FG)5TnEqF|b#q{QkWRK(nEn=P{T|8ARg2Q&b*f?MeCnY_A`6~X1jk9^DqPEs zd6o@>TH2y%Z5q4`+zI@u3hs0cbscl|y>vqV0iJKB*~;is#x;wtaHWr+4{#Vr`eCXK z`vW*o8isjWSu>F6V4S}^$OJHuiHQrGfyQ^p{uyut7`Jbyz-zK9$rGfUFtcc##knPS z!M(M4pO$;@tWi1ZqjqTXesv%EPlB^Je1@DQ?j2Hkj=Yb0%^(kJLpsg}Cvot6PSIA9 zQy9~R4V}bR&aw@e{1~3eptUUYnYcA)WMe^ZbCA)M5!#eRTo-x$1}5uR{{#&?Lwa`)Kg9goZSX8myHe4j>v03zu)GxN;I z3_i}7&&N_xlBRN4fnjEcEx}**#d;$9CvsHlY72>00_JwHiyY0AqK(Mo*vPG#XT|C)qOhBW6kBH!-U{?d=c^0B+=sOi+1ky* z0zaBwY_QpNlqSYUsm}F-Bz5goo~dK#$4Hcc>_B`N5qT}+g2?I3tI;J&I~hcoRHC( zqcaR?ahoMx|5f05rp?`a8rP&;bJX=mDQnh1K3q`1Yu03{5Eg*BI2AMFGVEQHrIu}f zJuJm39rpfSAiEj3Tt+I7-AsgrEfTu<&tqR_fcxKpT`%7*;IB9=i7Ya@4}r(yvapP^ zj{IR6Se%kn%`Nvz2 z(JX%y zZ;6i8OnX|m^0`3;jz}YRS%f*PJmfR-qFbdR~gHAvEI2+~2w@>oQy{kxO)!96Okzuo(!b5podU*udfU^Yf4#Z0(tqHs=vG;H1)Iw& zeJGYNtfVjoxLjZfhGq#jVt;wPwpHOoj3V`|oP`xwJMFkLQSmNqHk=k6w*g74g}e_J z68E80EECn(G69e^-!{H%g;s2x0JJ%=UMAAcorL}+_VKqsBJ=7l2}lo-{Y^0HSZ;hE zevnStF+2Yt3Gz=hxTJtiJdXduk?jN$8PRW#M|#3Qae5 z8@m&>@?}0jAwJ3y*gdpBKehNG-J;*P3KM&eVloU5B8*3Ne+Az6H2sMP#ezKuFdn4= z|BG^)yhbAh+V|mGPb!2|`SWw}X8uET6NB@v!^h1Wt7BW@yqqSUK$q6Ut9-W_a)+e7 zo^msE^2eDeCXoj}I}$B*eN+u8*&A*y-m;vaufM~|mbQ*to$H~0z@Eg0CAA`Rao3)U zD(~O-tCU`D;YlM2y98vDETsM{T%bir{T6H!S$rMHFSF(}G=r*n8nVq+9a=iJm(i{P ztpSdEXcIqll1^+5f!tFu4?r4%i`9&st+TY6ppWuY3j5@aG`BR64na6$_K zHz~;L>D$i0Mvcse*2pq;O*!~0s4QrnIHJn8CkC)(3}HGYhX*>1QV98+_zDBm5|9ZJ z`rD{u6K`J64jfeqoqBThmr&c0yaQlzseu!wh6pNP{hMe_e|=Q%>x|J79U%!_Dx)fD z5@?l-+mjF}=qK{jyj)f~nyY zY*M9Q8WIGwtgr~#A_v*fgjd+OnWfc)HNp}+9OYmx*wfw{>SxrvxQY?i0}FL(t=D4w&Rq)X`hS9) zhxSs$aCM2KOfo@I0dWf>-i$z?C2+HX*S+j1g(=j@PsiMxj8y*-8~ooOQHI=I4B-0v zyf0sF1kEegTD~`8^-*cmEE*y z97C043|0rmEy8zXuI%PnnX^k?4-r)Q$&aDC7{6x-3)>KTTw{F#ja~M1t-n}{rKU4W zXceE8bNWByvOf7LdGMxFrH1!aTTj(ucRn~x%J(3jD1UwG62qLfDiRpZ)aLicCY?Or z;uB~P5nI~xpyn>(M$^0!mFZvx@@_@V5SDb{Q@;ZUuhDBkt=G-2^?W!&;AhfNR$gT6 z+Q@FXIdp^V3Vi;MDTJYqA_utk`r9PpyRbPx;h(WYNECuIAkg|frZOys?+gREgZueP zY;bxLY1y1^3P*`2DF(Lz^AEAFFERCpAZ{K&yg4U#Vq`ho8g^{U7aV+*4-p^*ku}aJ zQ8pV#xH9b)x6)L*cfE9dKc4+Ay7%8i!bs*#JCg=3Nl?z1-g7d$a>gY)um9bd9HAI2 zU1`zG(^h)YPEN-*c;uF$rGo;sY+xYl^9&>v7 z+1w17xwzciF)oOeY%d2Ydc^^zIMGjH$o`{97}(R4ws!m>hY}mSPV_`280a>>tJ-CA zy%cZB%_#QoPW;#7Kl8<3T>W^{W|O0jgKGxgXYuvVfY1rWsQgJ703=id5s)xQS@^>6 zi&Ab%vz>%_7Xc$MlP0n8l2yLIFC!QwWZcouUET@NKEXnS z@8EQVxf=#`8v9_Pfh&>iLQwp5v`jM)1W(k>ikl7*`nWHk^fysj1QWaMT4D#>jbVTTr0TJ=`bnzY-b7qKeno}d^o2#gA{U75Z0$L zlRe=y<~JJ0UIWIc8PuMFNi@K?_d;N|uz#-5IEC5=kar)xHJ&lB$eG@0sWl_DzBHU;7XYZ3aV`M17dSND#cVeFd-voKqDmLMMr>W?@uIz_E&7AuPpbo=%oI%ma}l z^LsADST>J+9|u^TV+!mI4mGkah@aQvc*{v-uAjWn4{G@B&Prb+2BrQ;fSp??sZfR-LP%U}x;gi3_u;O8=krH7fy!Ed3^P|P39xGKSciny35s9}#; zsYtl_>#=+*ha6%Hf{Z#1m;r9MdHhxg> zmgG4S3XF)#QPF4_r)&amg3}AwTg(KTedFUxQ1WQp;1{S4m&cN zZgFQVH&&eDmm0tG1m1c4dIeE8VHGRb*Zh&kO>+Iq7^wkJWvB)XAys@Ih(XnItTYTL zMTn4DI8OnrcEx-YW(@{;R4p61MTCDKOq8%IRKQq-&pi-6XHmW{QG}soy?9af>B6Y% z`D`4Le2Vkhk`m8fxNu(fA%6a*KZ-J3FTi6MV2pZ=y3ygu)42g*faRz=@ z#^3kFKlRIrTJA4g#*Za65qDfGfZn_eRa$g&%GPc%e!O!U!9fTxbJOEg92!6wN>h8( z%Oo9=D@w9P>5<5O~I{3VYZaMAAK@PyxIM_(iP z1Cn=1XkSANvR;BF=Of4qk!q^n)%dZn5mn<==1-bni1Zf_6V`)~bbKErB?E>?4lQ10 zO+6U89CJ2pbE2b)B^ke$6x~TS(6$}7b+poF(Ga4upF}%`{|g{;HgFbp{y{Z^Ns{6! zD8$xLO76-(ZJdgqBE@q!g$!!QJJ`TKq%ku6qx?C?GTz&FTIhYokE&+=z5L~^rj#(myD%|ED4GJf7%p5KLmSFIm}JwiJ<8EfsIYSG$^}$yBEL zde^WFp=*Awo3nC4*HOo`OraZozFV-0LeKf7ZrLge-SjKn8Ed9nwW=bW_iNp{RTp}} zpY6_BbKQAs9_gaBz)IF)OJe2U$krj~ORQoYW;50@^dqdwYAwZj1o}}{w~nz{YlY2O zkG?FidA4vzVhdhkT4;-+q<>%CrqyO(XZ+E`*RI-=M2(2H%a`CO8$56-+y^+ zV`}SdG>3F1{yel2yVI&!**z z+>5z`#bD>x*Ii}bL`bKp&=Vbph^1|5Bq#D%mZexpFr95Bk!&@QAvGqULuLJPLr?Xn z-}88C+IBl=$F|M$XpLmx#WzB}nQBI^S15!P88YpJ?Z}1S~rN- z{C*U3Z;jm;TnqbaJ-6+zbwlR)(c09kS9`bkA$0O6p2&cZErS};&0?WTG zUy!c8azz;$!(5WPq7KbreppEIBMoixNkO~|iIU{Ev1v=vu$X8gJt>Ziu|%7h=tvcI zQ(MR11Yl_RZ>zEtRG*fjADonA@qY_4mOHsbV+vF6Dnm`A^hrvO%~2jJSm+eT^00)N zmXi|Z_QJ3dmzb7RwiOyJscb5v@>m*Gn2vfgiI&WWSrtb!OhHaHsiK!^r`D-=W|PVp zY3mP(0`oG~Q?uzgai4n;r>%qCsI1)`lwS9GzT>uo*6Ql&&d2yQ=2FeO?)a%5hupoE z=7P{}afhXO5B}?MC-zd!?K3AeoLo3uEnWF9W-jfRIH-tli&M1}c5RcDYzh&u+R;V4&zJ2uU+FP1clxT*YI`COoVnui~lb z6EZ5dXr4~;|`!EVO#NQOo0 zTN=^|?8r!zu{z8N=uy|F9Gm=&T8+WK2)}}OPSNcf$0O`>B{O! zeirH{*5eGVs2g^BXRo68KX|0UlhdBu?_pVK#a7Swq3ih3Stz@$8ft2IfeY+P4d%rT zHnF_dz;>>;8m6#}Y;Q8d;9tWiuq!W$5Zd^2NQrId=P2I@yc-zo;7s#-y?zI^B3fYy zn^MQ74Hh3V;i)|#q^4yPdqVMt5GPTwFV7EkAczjn#qy1SBk)rL&~LzVfjT2ac&-U# zQ4>?3iU}XqI(o-Qba=2C=Q=3e#f=2Z-_X`MfunF`{yAukJpVFfoF?%Mi4!EgLgK3= z$nbds0zn4rxQPY=eMLU4j950lPTC73=tKDPBrcGcMrr;{JSo{ac0CanR!JX zEI(w;XwRmCCD}2B9$`QQM#*YxsI5))JR8rU-aJK;P3;4jYw-drY-6tR;^+`cFnVWx zOmibIT|%mcr;cZKjCgXO^(o|Qox@aHcyV!khj4vR6D$lsxDo~{w}&Tg4*}%FQ{bjX z6!y7GPz#5;4BsjUT(qM$@HZD->o=Remlgr@Hqi}_XR!jr(Tn1j2IbfKe%uBLTzUE8 z8&P8!%){+xk(wXJ*l8J@PXun!BYzZm0?_%ls5Iw?trjMhYVDvIrbaX5UBrwcHJU^T zev&F04tIi<$4^nBBtW<81P*d|j=Fq<#G9fpi!qacP^4+vlf=^gA{%4b0eeH0PAoz z>?Aog?g1A{YwZUNI~$~jPNT;mBt(ZfJPXPZ*}%UpAC~LT>q=FC(wtnC2gjye>=V6B z!4pB#UdW)$g82|9l{%TbNB|#lEQv{EY-Cb=1=SkbBqYZO)i(34w_T6Fm=y^8aG-NS z0`8JH0258*y@w~F@g=3A4Cbb1oqf1VQ``Mf%po)th%kUr$6!vevJJKd{-!26v9=H7 ztCuIdO`yRT{EEp#lPSmrhudR!BczL9cE+$ooN9!K*vXCMq`a#sk|>czT2dOzOa&@e z1RJX)<*@>erH%^JdonXBinlr{iMNRyaIt5kp&3^?Gf9QvTydAiH^9U4Np)1kD7B3Y!X0 z<0LBzv{*u}wn)bb6TM+=mZhXN)`#`rB+vu606bAJ|5;Xgc>QE{Yz*sQvCB!FP;+Z8 zJ~DbF(*>%O*Z;Y@E%%fx+7QZs&4sOY9Pw`>A>z*tgSJEd17>M5&uw=R%6DjAMYH>- zFN4qnke*t3u@}16psn2=t~_^k<<&5XR&GDL`o$*7E)!DOj_cl{tW(Y5{jYzvcIb-0 zU|&y-qnwN|)djeu1#!aiXfFIonk6w${yK>ciA{)9ekm=x922y04(4I+R+=Z$n(<+C z17Y-r!x@favDX8v#a=fHv1-@3nJV5EDCW(c6ENF}_$qa%M*S|QrP@Qfupcl^=O<1K ziN8g)YBo`(K3Eav-{+2dE1dy{hWkY>Z{zR|T0=C26VGVOWr(!rh-qG&?}6z&%}4Gv zk8lfzO5k71-f*~JYN;+hoi;K8Q%ZAg$hqgn#QJe&_{uB{}$auuR!3u32Z9}7Q6tAtAHDWA>&^Hoj8KD zDIW%%0DlGwDyXUa*D$BjvT|QDOUhvB0IrDb6{Ago%K`@71DB5;4lc29W1;|{8V?1R zKYkW)Ny_NeJR}Wg;{2!p*Z|tTO9aQHcT-sc)WC6Zl!NQ56VRzmKu0CW0T`J{ZB&_n z&rGK}1wM6vPaVKh9@WW}0X_hp(cB*R%uj)j=r3OAEXIdE1$<_wz~}7=G0C814p2lu zv%C+A<|j}zo6G@P=7SZ$&qM0~uI2$gN0Rvo_&hQ?n&}hp;XbyDcS(@xJ%Ae?Qj+2& z-y#u_AY|fw5-~*MnDBRg9XbvXNl?hl4pcuP?Z+g3LgJ?+$T#`V zNZchsD`Sq@d1esNn9auz&#i2$pHvQPTUdDwI9LE zUr{w8P?X@A)1FFRyYR85Y4Fhl9!qE2{XPpHHS+s?Sj*v->G1;|`>7LXl<77saJrst zrv=;2SQzw@ZEy7*KPwTpSo|^iF5)JFZ;B%dZ&Q+32>%6%UqPhh$%UEg`9AL09<0O> zszR4goM2)Fk2u}{jSN#YpEh%5#auLxnI}Yf<1w+BsV=TH1)yg(LQ&9~6}Km=k;mJP z-yVq7WkF$Jg+I_@nD`o@IkY;KPGx3j51o#q z;#OE1mO3RBw^7IKxGL_1#QMN?X0s}=U7i;_21~7-DN^ZVCfEDbFX{LD$J^%K1*G?0CGk5d`R`l*`tx(w zEgXo>TeEOe%d-Ma3@3}|?2n#=Ro=RRsgliq0X&vKrt*bMg8LzoG zT9a9?X&sB%%)!@YCFbJmurjOQTVhpK!`EeX*1)&SmRJ+t3R`9^e5-?T60| zqx4)jPEx_ou^W@?@%Y>*=!fTqG2>x!ZeiZtU~_a+97i`a%fx}u^cwz?;|uk!T;AND zQJ_<$`H$kX*vkk>4Iv}?ZlY_aX(_R z;&)Rnywp#&Nd*!bMAIiHkIw2&wSbI!|F?_??VOqyY*)*ST$^cAqo++R1UuDpW2XI( z;W%&zyZV#bOV7M$Or2>dE4^q=-D!DR$;!EfHs!3MzLm_#%DXta6WX+zS-G86b7!W} zk!ChhmHpJtgI@q}SorUnx)wDb){<|W*L3y&5@f0mN}0tBX5KcYmP*<4l%2YH8GBe6 zRA>6MhMCs08s_%Iw2{`Bl{Iz^8ZB#V8+m=E&PXrwvw8v!q;iUFCJ_`MyAN4jjH~0RBUwc(r{1rcx zb{dP|x-3Pp*AqUIWez*$!cV!hf-&=@V$uvoV+cP5?u?`}j`}-eF6+ulJO*${ zfFn-mUxOM-dnEc%O1(xCE@D|qP?=BA=aBo6IDxrIqZ>%)2JiQ-r_!Ph+RI8=WL5P^ zum)SPm0~MfkPSY(#u>vhys)37vZ6*S;(A}&*ZdSq(LM2T>fctSSkQhWSt;zOXgd;d z8W(M=MSeeG{8d?@slYxdj%A?IUD_+uZgQ_1d=LXAAA-;<*RpiWsOU`uS9gt?-q1G@ z8g@gskW(>Qg{&~wsp8$szg0cl2>{Ls`=h^J-NliIEhy|$-LrsdF_nV##= zX?w>)+JGiGnK3h`C1tPX)`D@9&5=>PfRL<0^kCXA)uzBPGNvCHJJt(@;lhY%ov?Oe z^Sro(=v_N~h>F_83ib;p@ z)W<>A4;py>E4{W`XootVLOBF&40MYp3EZ4`94U45;!~9GMEnK@J380$L9gUT653-8 z2h+mAO-|f%$_wXGMibgPbuQEk;ZA1aY%Wc0;D`SRe`;rNd z^Pf-Jr@%vkapAMY0~lM@!2y*ES3Qai~aRf>R$on=z0lP-SAa6NYKf1PJg92K>{*OEq;jTiC z36wtt(@Kq9usiTQGqZ{1y`jH!b3{912xs}ysIx~Ua4aMFXS$$@JlbLyide4^fs`{FFO?_SDfV(}WP2IFHSjrj< z_mSH)z60J?&YF1>qbz658GJ+Df|poPwZTJS^YvXI;k2@I0Ukp2m#z)g(_@O6I;&b% zIi=C92F!9fE3xvr#wyze(B(XRbQVKbE(Q!Y>gTf&In3ewv7@NGMOWN7}GQ zb_OQuG9L7YaQl07uBzGFk6y(U0KoO3jZcr_;5uUGZcR5nd2!>}I7v2cePZ*&U6frV zMDzS0xJg-$bf<4W{qEYKD*}x}J+;pWG8Jhn5Rw&jFXJ>9@d278F;($-5;2Jpgw(Id zdLS@C5BFygk8a8`k=%kmyVu}wZ}@`Y5|{E3;4kIF!Wo-G|5a)59Z=F&M}EXSKM`lC zLo*oF+n#j@iGZzX3HKCtT7lA=06sGa8oXUbHg5!K=g6x2L&SbB|M1s z>Jf94W%Vuq>=&|}1lKv?8?K_j$)0!37mC+OTP>XqG6AcSr63l92PxeWaLW@H(X({= z5oUqH;u#{51Ca3N~7InaCyajk!$FK+R z@~eBpOR5Z+$O1^oJ>lh#?*?A7I(l`FYtyB)oL2xGz~FBZ6>{m@HBJC*V7j>c!T8NN z>@3e=$0X1J9J$$Y-k8JB(xABjKP|vd3(!-~TV%|DA3#sOdH_Fb3-F`*OVB z0>>p15~O^007r}|`3i{}ByN%*Y!bIfOd#6#D~lHy;<$2js+*(A!T z_%UwDDg$(oo)OiHq97!bRrPX`Y+l35jHpn~ylNW9;dVb2Rq9$fnJJZ)6f}lHE~&_{ zAHrASjW?0?c?n!i;NK)Fi^t32<)+9IzoZWAP(2$}4z;YRta&A3^0Kzjj(EM=vO{~V ocAfTGx)+K=y+{_EoQIgJlzPOZhs+wNF@jA>p7C<&)2nqozKy^>v)d?Bdxs&-ZKR#5BB*>kepCD@c{a?}V_vc&M&RLXq z-XQWjYWd#>|NhJK>le=Nyg*f-A@VelH$mDDKY8q5e}CkG^E(zw9-f(>HZ8@kg7KNH zaAvD-vkiqST)k;wvo)^YWVX%?ZsKk50=Musd6Ad!F7Pt1;BE0Lui;(fbG(jsi8pu? z@ABJBXLbb&pU?N>+3nFL2}7 zaXw(TPo3(8$*EuvC&E9)uMMw6gHwI46P)Tr+z;YYGg7biuZzQ&ojh{$-CqL?nTC`K9?ep9%qDE4w%JI>Vl-_CEij;x-1#fl93e0CFrl* ztzeZ3;F-xP{ z?9O)=(%M%8)QZj3GYf) z2qUL0T%Hwu_^dD7#LskZz+JoGMd4MLV`jSjz8~^T^ZEl2Hv#nYGjkAjwgx@`fLz1j zz?Yah2kttoFEje06DBl$IP^u76=HPuhnT$QyP-G&Av1Nu%gk$jr+p>Kbef>mkal91 z<(!0Ua1le=vvdKw=J(cp&hb4?(1{bfBv+GsrgSo6-A(XKiX$GQ`3>2MFWViIm2&sV z$M!{(METfqUf&71|3+3KQQ(%u<1^6dBZe!aT7753djJc?4}mb<(se~wOG+J&rC3@? zsVS>?YDP`bQBzWzIV~SF@jKJam^hiYfk&D+VP{H4d~zgB>n2O}i)>qgNgE?GWn-Al zL>VhEXJbo8S%n#ysXEa{1&KS_+Kg9|vynet!V@omwBggQRYm|W@}u|EE&WpNvG88{ zK)AWOdRm+V`w@MdC5?JfuYc~_X#Ni#Hhgf_jR$?Gl{B_`Hi$eoh|hu99fg==`k@El z%1rJjE;g~cH^6bPwp*68jC^l$!w_G_DzGa*juGX=<0#2(7w4(o4E<|Z?4YE@y;&)6 z#H3+{O=)7&hRb&mcxF$E7(i;1dqQ&%+@u=z<-*8Fp>sGc)Heq5!%a=#z6r+#2D?iUk+eyGGYc4LGos6eMh@TqD>>q5hV>CI}g&Pm521YZb*8sNxYKj}m#9 z$QOxxi3l0KXn`QYK#!XkaBQ!hG$Yj})`)w82z`jSK;$BkS%?;2L!E~wZh|l}BbMqFw!)fe} zMa!5Z;o*q`89|TDU z`QXYkFT51Djw0>w26>nk_hap>iiA%_Z8@VjfjSB3;_K8}2%>fyf@OLq+=w!BBN9Et zj4}1c4u9rXwfHOz3P7_$?!Mo7bs(E}&d> z`T_QleA#JnX59lWlr=hclI?tvEle7p1Yw8{3wV~)V~UA?Q#qnE!8g^q1f>O~t_+XQ zy4Xi%n}H{SroE6s8w2y9gp_r2cVPe@axBIf85^0DxDVZ0`ZOdb2-ObvuXa3NJe4;{ z{BXgGbs_`@2fk5kpzeowVp^Z6HFdZ!D|Pp2m z1zucYytJtTHBR%gM2o|y)tBikVPQ7t<|s=W6Jyj2PXj%G3&0Z<^3U_i-TSBW6LZu| zjq%a6NvOHCkQ^J|lk*Z)s%!t;-Ifp3JlYV-Ae)QYog^0DKtaZz9S&}X{0Awf#eJ{S zLnz;&eU-y*KXMt_JOJsTm8bfVcLm(38>5xS&#gQc#qr9G$5ubLfws#sXLLR9I#r$7 z7~OvMla0ev1P1$NYTYZy2s1;1OIDJ{E1zT$50aFWd5RZ_bct+&WXjW7)e{iV!I4-* z{p(qglGZ#iw$>3wuepNbESC6vz*^$>@(`=_+&41S-$IJ{M&Av&kX1B=ti0UP9?V4s7Bq$zgUvNP))u zEFX)IYH_m=%RJuaaaPRn4suFiGE1T%6=xHMtel@`W%u7=n)s_AIBWvjN-_&x0>;&F z^hAb?e+B8pF_bOk2+|4U&p<&HJ=Oo3)=XJZZ|hb?9UeY_D{_0~YBS)nghltj<^8*Z zOCsHvQUIhHcLkR}eiCp=tC-a~#71-S%vZye6MKsi9E;u?>R~_)92X}!xV|<8oyHV& zG=dy}k(D;+94NtOu3Mi0pC-Vk3E-)Yo8-y>9{|sIVGn#3XTV3!mn?OclS3Z^KJzo+ z^ZGO~$)RNdP((m;bRQHgPN8T%T>!K!hAV)dyY>NGEdqRwrHfPWxo3QD&QHNd1lTUo zBSNNk2W~_}#Xb=Vnj$7b$Rq|t5|Gw$>F?qycpM_g(c`;%y3Ah$-9U-!M219e5J`!Q zh*0t@zC+}@MBXGqPA$GiWY4L^_o*m7dYiaCw-z_4c*k#-4*0Eji`sucZ5wPT|LN14uCQS9FaD6d62o)>FxAmp;^W9X@)%W9e+W-)G^YW^um{>jhjfojl;N zA1gqsN>^c_+w&bKD>+V{g@La)&ep&U@)mi8B~H?JQEnphO?gBS9V${5B7RQfmmpbn zdRylCL4XUkJDRwMx~vdc1|dcsZ-7RorCDdJf>pDYt>f0ivb}XuZf0i4J534bxs6a1 zwCCkD%4+P3jvI7_Qkz0UOKz1E7y*`a)v_9ff=)ov{IYspH%K}`yh>@L%z5oTRFGrRrCx juXQ%x*>mK?F&zV@R)q+$`(FT8r@Iu`8>qI&$g`kf(9+_6cGIwL?8o3AjbiSi&=m~3PUi1CZpd-5!pP83g5+AQuPz$Lpy2TC@_c zdaFuyqZ9F(w-%rDPNH1*PF>PigEenztjSjoP46_*uV~%YZD_1*b#-YZ334k+14hq$ zpT&6E=`in;;O>l)Fz=^H5K-x3kO!C1m5Z(=TdyXe;4x3~All-&T))zfhY^1%&H4$y zr|p5)Mp52>Q3Of2`=uQk<>0saIi|u=^%sMTU*>^Gu$Cd4c_4Z`e=<$@Zko$R&XX(^ zPbQ;ycyQanUP9W?@t-*_WS1du&C{9Y86RppI@6hP)4{=*%(|(07PDCePn%VlgJ*@+ zn2TqX)mZ~ihc#IX&)SEYrFkx1-jb}&6$_1HE0Ro$Qp$dhEY}XTsb1(a9VdY&Gfr#o z>tZ7}n0cswpfhV;rwK1NA=`&Wq4^f8Ah%Jjo`%JYbn!+hCkAuSQb{)Q>Yd(@YNxe3 zBd7=J-1*p#N*QD8XmL<#%v*j_%4lh#rB>+3U8a$BY{_cCf?>{uY=mj@e!_DxF|LX0Z;T6bgKk4UwBziYie|PC`f_Ern>)=Rf!gA? z41t_W>GK(c!kB6^ZDRDaiG^h6`qY?dzhJsX27IVr)ZY8Ay1ez!{G>7kOPB6cQ9sK)S1qGtsrRL=d{pYbI8XE2suep~4D#I4Q_8VJk>+VR zwqDlv6UMK5E~x^4g)D8==?i8l_%I4W?(c+O$3oeo$TY*zE!{F)y$!c<@NeiXeG{o= zw9w)h9n`sc=a~84ndUA;wlP};Nty3?oLB*48UTx*hL`!|TtMY1Fn0j+J4udaLMD^c z1|}I~UYS;zF$L7o{)1i|E%IWb_|n%_`Z+RR&28Z0q8Z?noR z3oXlwv8-Co#~l1=q>0|N`Vy-h_3kWs-?dWN-$H-Pc-QPJ`6_BokaO)>AM3)+*O+@q z`#3p0g%;|6Vy0qDp-oStw2ah2TA9Jc?iu|I>aD>#w6;`KsBhgSqaQCT(17V|q{+sO z$v19HVDn_OK&MKUjzmbt37EXvH5CNIW_L$BI}w+4xPnj6%th%zARIi)Ul}(QJph_- zTzdV@ta}Dv9F9u75)Wfh*+lSH@TCHWAP`fw@6|>_m>F4+tVU_C2fd`#Pj*sS*-1qV z@2b<#xj%`gXtGKG9@69Dn^b5hD~*FBK#i!<#42ulPc@d_IQO7BDeKP*r;j)JSj$%? zj@Tp3h$mjlkBRbNPkwJW%E$xyNiGO^W#b||<8^2eMLd!fUrmLum$FgByLJiTS4ovp zMYZmH3*LIU@Vt1}%J=%$a{mrPD;K$lNi%wT8XRgBY0Wr`7*)|%5ua8euNrN*R|jov zeSCTmAHFcpU15S)@{yl?OZnM=n2$Cn{RN%_i!@-759JE5Z<6n6Q+uXe)D96g0N)ir zx%DfMv0od?V!?u#`GtNq`q`>qwd@BUu=h;yOEq7W=6mn#A^;5>QJ{aNs7`A$?a4frkggEj&MKMY| zGcwFVd#^Pi+(ICJ3v=z`rg23RUqKUSvrWM!`oxgN=HpTy-fIxG^F5yM>qBv28~y)- zq>Ue39CB$Po9Cl645IA9=272I^~c7Y4aGY)hOpY0gG^+b|65OrO3-ta^c-lyfVTP< zwFRqR&`1?3g^yw4CzL&Sls!aJh>RgA-eT$F6C2702!oF4BWvo`0%9qone|xj&wrxd zKcDNG(uq?dilbN>kx)z!BoKVAYls1@K-^H)^HIW?&xdKa3ox^<`f+ewRym>=7-@HH z9&K|(VNllh`ym&v(Ks{1yzU)M#5+{mPVl0bf2?|kR#KZJ$|)#3N%P93izP=@D%jxD z@!PTy2|o*WIpKkj%jJ#fg(TrYgiT+5qolF*k_?yVaUX~}*y<<**o{CCx7A(ihzq}r z#ky{ZEN{~4Z&CI(GOtlGN~LJ#t(7G;X|N6SD(j_1+X0Hv`HLRL$=Z>I_ny~}(EKh^ z_6Rb~Xc-OT6z~MW{BviHFBOzq`u#}6ixKM0aqDhF{8IB$p`2(!4g(t?&>b4e87R=I zkPTC#FlW!`I%6ugGPbUED}q2uksonD5}Bw-DH|e#84zs-&6-~w?lC|bat$dXm@Dc1 z_~C`R|M$g*bjpk0Dg?Tts9n4TH_yRhp{+q|SOK989&Hnk{#bwS(K(PQN(;6VI1@LRpP5G9gv}zZ%qCIV6mF1#6!xeYh0i#|~y}|TBC7Oqz zSrd2vMR0sIUw&e+mY-DIzq+aw?imf#rv~m05;CVJ2K4DLS5cGoqRMO{W#!u`9L$MO zaYieai_WYvSt;!4nPPa^F-Gw}-5kb&MN zXt_;53jlR+7wyNuzwvjj;)$B^lz`NI}`0 ze`*MM6#4$KJv>1DRLE;XT1Lv~prCNaF`fIIQ_dO37Sz^VSLdMuKE6ND*(yU&1k+nl zm(b0Oi#{U8I8f(jQXjI!GV(ZOpCl0kYT^)?Y?pPTydPzo#4r^h_L^k{SRlm{L7xQF zx0VuA>Enu5SGUV-a~s!IL7y(0n}KoAP>#Y%k0510XR@g=7ml1qL;a>=o7NlpTC$<;BCe6M7y{`z%|wOIR}#@c-4*z_+l{ifDkc!1Sw+g)86 zX^Py6vyjodIAjUlb~Y$_BzU{SG%9*o8pfnu4U6ywnsU*#Wc#f&5bVK&TsHgq?pT)ts)eAym&54 z`F>W&X2H`u6VIi?WN?Jm9V{d)c#uc?oDE|x9=OwxXVKdEb=?I>uIMZg7rMDpj z#nFJ@dhj!BLDqU{uLxqXcX#DC*Z(rQ_lop(!Xnxa^4^Hc<{jSK+b;sfqwr|&>o5QD z=XZ8qeMLG+FQq;&fBwv0e{uBYm98i4d^q4jx{!$2fHxSVwm#H!I6NgYs*MEO{(cl^>g$~MpO56&sf*j^-*Vb!khC2M>;j0JtWg{ zzk!rDk(FAXjs-S`PpWTYz3rgL;49TSc@F&`&F-Xgy{n7I&`)ffWKTenUbtIu5fov5 z@FetH*_S?Any==xIwEkIKZ_8^3DkPNhEN(4ZK{opJ#B0u*@ZqarrIx=t`%mfAM01O zkKg{t7(3%yS^LNwyW{%UE9(;red?vB-d<^x^<#L;nl^5f*2FFw6KATCF_$(pZ>}9a z`5{c#!tdDBwY2lBmjC=2O;^94A}{oQt+bfI%=^aJQkp%}?82SERQq1PG1bRSth7}& zv9_1S?b0sW2gam1)g~=wp?0CP$_2HS#-z;*lyu4t`Y!Z4{l)%L*}kkD{-HGR{q1e( z?gAo)uxmj_1&c9RFM04UJl$n!@!Mf6?IIJ=zO1ELuqQ$$>l^^X1&~Er(U67G2?qn7 zGHFJGA+oRleh#EFOnZkz4v-QdtegQPbNG%EhdVGf8ZI4ik;z()RvbQJB|J>UMJz)a zyODG%mys6r>@F%R@Ed9{!e#h~t?X8ry5Hu>4ri>F?geo#FMLl;qhzV~q^*2W>VbHf z#O=jk0J7r&<7);E#bMjPd>v4k>DUpiyHr^7sioHj;tkd*me#)*|Mramz9MR-|2 z&IR6t#6V+7S5!`jESpSQW?wxW~%q=g*O4>p}#5x zvHgIIezd4S1E#Z)r5kt0Ke#)F&6Cjrot!)yiinI8724;S-Ap9#t_F3T`IC5-26IAih&?WzC#9*ZGzrrXC89bHZ0=nD1}>(#_F^`V&`^{>U=DZ8pBc@=}^-$ACqp;nMqjZ3&@Gqw5-A7EYvjmP2~#~F&5OeSRFO+ z;z=8xXrRuXf<;4Lm?ChbBz2UxA;6I3EBp4g6{yR);;o<|LYny=+xycW>G}IBT~oSA zMpSi}NFx@COn^s#5_Apm3ASC8G>8wdfDf{0A1-0v3XwP$@E1UHh1mtL90ccf zFXH0I)R{PlAjp#D&FfdCH-LR49AP`J2n=x-X}^Z)K9tQ^obpt;Sh3;P(v*iW9N@-# z6~}2*q^Pp4$H2lsWrh))VF&`Tqj>ZQQ6h}&I+f+!q?O#F?0w2UMkebgwD=Jt6F)`u zhe-JfG7T8*!K@xfBEka_VJ2u91bXHglq$V4& zr^GQ$bz;W$Ei9StTGhu(NTRPRIAy!Ya5BV)5I<*xaD!@8_5?}PWe+JQ`>i+QsMCUjcf8)fz-|jzNq-mCVv6g7k&{@h3Biq+$(OBs0b?Gf@h< zbV8&i&>J98&bUeRWkSAE!|q#i$dXR(X)EH>E?ZLr#KxR>G8f(yP^;>c^|Ax+XiXN&&J=fy$r7m2vT6+qhZxa1M*JLm zhp&UYsqu<){ng^UB6YPDt@JKxh-r0@vcBi@ini6)DP4?- zFLY5W7pL}kDSZLniKv5UWndS}to6TomCI9Syae)iv0S3~e7Ig*nmks?%1+wbVh4vq z=d2zW1({&w?hc|xugJw!D)}z52aGbZOI!+FTtRwC80Qb@xK)qQ3%5X{5&53pc$0h_ z($?Ma#t*M-+{p5LdHqnUT7W3u5WguELlC zj;CpP%J$zR>BIq~eoeL_ft3Y7E|Cq6WSxj)MNZwFFgmys3Wj@7!3T((1y2OIp)?7# z41NgO_vs)^SrF!8OI75HnQ%@qrgL$BT^d3MOfVnc0wVpK$W(_Rq$f$i06JS%7oxiL9`p! zB5|48IK31zgEUtc{#*>G6#q};xsNc+S?<@XAd0oA1^BKCv7~OG?HPcGp;F41n$#+r31vMfI66{1C*Xfu8r|2rj{rvYn$YKYINjh*=#8&pPLL%Nr%k9~!JAt#TT To$KrxPnY=%NFBXtTE@QtT_Y{* literal 0 HcmV?d00001 diff --git a/models/__pycache__/adam_ssf.cpython-39.pyc b/models/__pycache__/adam_ssf.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dd0c0d223703dc471759b2a58673bae1207219d3 GIT binary patch literal 5847 zcmaJ_OOG7ab*^_;S3job!5Pkwlv_@0qZMmND*3qS;_A};F_;9%{P9o^>wB*0Wp=3C5W z4(>LqF&B4-)tQHTjWt*kcbBzT8~6IpHB0k79NwC&EftH53p&0Eg>MI~cy13fNk&1KI&R5E&6=&6@FN{?w|9S6Rd=Y83Xvh+d9 zi=a3i^Xav_Q5f^C;3t!z;CZ*I9){8JA>N8~(1x-1p8gb%E^EVdSOl>cJlgu_cm8ww z_zmeDhDCG~ka87!<6Q{{#Q5t>ldeQ-Rd=@ zolnMGNH+-7f`QyT$cpchnUURb_LdNt5Ovzp4Y2*gOdN|#sEREr8dPjkv4cWdFrpmR zWaT{Wi>qk*M3?m-h~hBMgWwbG(^sx1X>mQC1DwtN z_*Cp-KJAk4q0scE(bQeNiNB*?h3x5`v8V6p)6UXb_E!%P=`>;aBbiL+JGk;Dic%ZV z>x8}Dg?a~2w;dE2e5_KF=cpg2*~4^I@9E+i=85ik@iHXoh5dqypa}EhFQc9-Th*r< zD{Y>2M+9W6-#P?xD5cMDA(X~kTWB+5pv^2?cA?LWh4$A>*T`tk^xN8p-}|vKb7r-& z_G5GA&gwI-tj{gXsh6I*d!FWPOl!ZR3l@@~oJuzmMs@cn$UAXf)&dwV(7W%9SO``g&%lXB? zU@#^dCQpBdUvE=d{6QE?yU0XzBx`9F3`EFeodcM-2n#N)Xu?A2gyS(!nKYyE1VvZ? zcgND1q{EX52e^q4md^mLIdIg8!$a5_EtgKY$Yd?YC=S1|5+0`F3e=EBKax)6M$)30 zy>(>;enTxrcn-g@mDwsw_uD)<u6a zi8oflm`1?gufxj%axNh895_3I z`R$OSnUKjOwUJ2%Svd0=Gv+{AjHEW02h3C-)eCP93|+M%d?0{4F=qyhTML>BKU+nc zYNIt~7ait2vCy-=n#*d{dd$V2W||mHyRWhO`RI++=;wB-##tkJb#WwTK=s9-gm(fGxFD+DDDYf}7YU{W*aBVK&VlT}82HLIB9<;VpBxoEwC8M9N zE6{-HbhEVkX!h!(8El@67U<;U*+fKSoPfxCJyXFiZ1!l<@5fv=;0gi3G8dHxfn@N! zczfDXlmJNHedoRJ=e;Wc;%HLgmH0z!Dq9Hn3cgg(5PyI+W&3`8GKQIv1<6{R4F=Fl zTEn!TNvEHQ1m4x4sY`znuhC*o2<|c9;!mj3R92dVX^0k4qlHz>_@3&lym9G44N^8( zR(3CT1=uT4CXTox&6uZtJ4lG;;F*HKc#@L`4AVjo^2+9Ic*c9sB93`1oj@&xke9JZ z%zJhP;de=uN=3Er@)7*KYU5?~xt(A1ul3u=Y5Pwr3B++Uokg9cyCi7V1F`)%Q25EMNKsR$R(qYZN zvsmlz4dr7^*8ZGvHH@pNackAM(kW}lCeGr0wPN+G-AX&s13foL3k1XVLOU~b?PLAq z2F6;f1GcuV&he)O!qZA4Mjos@nQcC?W^Ve*`RFY*`X4cRb2a*W>?NR%i=M5~rIHZN zV%x+WN84-%JqVxk9pIROzWM_B3hI{^2)ijsy9%oio(VS;I$SI5m)T}^xuUC0+~in#G7hBWqXNPYaGSJZC~c*;K-i(4lc|G&7j z>B}oiuiRw+W}HFg{MP>Y*iZGZPB*%Wd|l9Z|NqyMC?)jVB0Wc%FrcmeS#6QdtXQ<- z3*sl3CITwD=f%q;9imVNK&N5p(@R|i1kaz~zO1Ws4a{0dGas2Rd2VC?+Xq0Uu}45fI(J7bM|Fvc?h0LCJf2OUhAzih#2} z7)D%BR;74_^oGdy@&cJ1_iM|Hp@KZ;T?KBklj2>o;MqGDoM!)kwl;I9=3VGgXF*Y1 zns?v1UD0@@r2TEE{ef)80;UCf0c;AnUL}*?PE#JnIF>u_S4_WA#pXSxi zHGJ`n`sn@?`2{*0Ad1Rhmr@~H7`R38~UA#c49vOX-F@jl@bfI zS=09OM*O*^skjjFtOCY40#i6Mlp_%?E6xKmC7xg}C{%TR-`583CkSy29gzL)}%CHj*NR$ z!_&7`lA);{4d>EaAX7)O&{hdXyKF5CBoOA@qt(l`c|+YzN-t1%(SnHabqzS%9<7ya z`n>SOP+E1$dfAzGu*-VcS>U_Cd;=-TrW%b@30$mo2A(?ePToY4q}D4gjkb#&l|0q9 zw9>nxfjicb`~drx{LX+r8_ZMOZVz$CCRSUWPU&JzM6N4Zxn6E8oY^KK%$0JJ812bk zadm#Jl9yah9OIY=t8>TeB|KIIa*}K?EOPM{wR{D|Q$_{B62R9*B0o%kozp+NhjbGR z@vGgp2uvVde>CfU^H%pxmgn6^ukZhHAAR>!z7s~#DK)*OnuumyXz9HqEG#bVinbB3 zidU&nHuyG4=Y}Ztw`D65*dYMaiEMl->y$WE$(T!@C?1A_A?+{t7)-I?iTER0+$5MY z_zBXSkH%rjf-o01XlcS+@c|Vp@|aChCfaE6*Mlfa#uM;Qs5=p1bSmDXk%WkH4GA0| zGLOU%U(m8t98lMvqv&nDh%kf{Vur$ZK_dsjVJO5Ks7hM_F`apy^PE6c2p$!bX5uq| z_$G~UhADJFl30Bh%f(+%E5769xewaOS?<>rHj$RTtJowYrOJz#bc(4@aOJO~(10@= zKpSLT`ZiDo2t^o#e-l~QRn%R5&!nu(0!~p@X6VMJmfJL@n=9wO0KWK!gO4`^qd_3O zAgK5*x zt1)RR2-L)56tYvbO^RWh?-NB=8HL}f8bG5d(g^ydqCQkrph{n+{05~=m~@D|U(mN7 zxtvsFS$qsq)uKuCFoIGO^$Zstyi0BL;L^QWo$D>~SoJ+>`N|aP67n_?V48ZzvdsSj D-8J;u literal 0 HcmV?d00001 diff --git a/models/__pycache__/adam_vpt.cpython-39.pyc b/models/__pycache__/adam_vpt.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..954232ffce14709cbceb3cd8b415a2269ddd3a7d GIT binary patch literal 5658 zcmbVQON=8&8SbauZa>E3nVrqdZh{ktz>v&_fRZR|HV-xlL7O4UE(x@hc3_`MRL_&xoCyt!p!m)0^X(fb&xCJEm{%U)6cEdx$R#(^Kud2Vg z{{Q>Gs-W3)HT-`0WP{y(SkwMQg_A!Eg_rPTKS#nerbn7rzID$~Wz#e9HX<{&JWHoG zGqU4~S5bK@s>W5Xs`55u9M4htN>qzoudedds1Y~4rph}}D_-$d;*p$qf_yQw-KNAPNQ7+9(rA44c5G^u_j+XGQCZvU)Fl9yUFLr)5~Nm? z28`bMA&c?0(@{R4jJr2Z!hDb>K}4mCK^|O!C>K3Tw%$lW!DF7}LA1kjxpjFEk0Son zG#e!RfwoV*IgawdDxsIK{ zn;DzhHC=RbgPBMA_jP71>NMfyCi3=?QE0x!DoAaVtDCTxu`b>&^NGP6v{aIAUcD!c zTy0ak*MlTb=PvsGu*{=x9W4$@jYZ4P$~;<{XsH!CQkQ9D9Xp3 z#3!5Yymh53_~CdUc-E!DZV(>qVn<{JHITb<_j_2jtPYYv?nk13YyB6m|6y|bMd|Ja zdARRqg9(?7>wM7P&wa+j;Aa0HAN}?BZ|=VIqO8P&g!(-H!Bci#v?AI z`5R38B>y!O-d@f)?d@fEIk200_@m;n@H^nCU zckpCeNHo1+G;{|~L$By(@pg6B*wVN3Nqb=_+sj3SI(1n5C{JedRXkY(NudpCD`Bgb zq23PEZTWc$|0>nw8OjGqdOcZ|d%6HJX`*|aJc^ujgT0&!KM%5l$5GCdo$8a$Qk#3t zA%WQPw~7oolhWrO$P~s*n`={}uT3pHcCOEix%MNbYhIBjwV8!JwZc_zw=jy@5xj0wo7M|!W*7BYWv-D^6gEokvUK>^yD)bP|07e^lJ*z1 z?E6n?y83?`X|4~eg~i}JcZ{i}O7@~;=gzE#Z|4r{bA8%?rp=-O-JYAa3cF|>7_-J) zn>Cq*)|J94R+KLFS&JE{X%}tuT^Y8AtHVywI4Cz0c;|G@om+Ll`K1XZs{du z-;qf3wCr0i>jw$rH$0bAfv+MjZ6)*xvlM(31tItM!p~u%?DI%8!_h6>GF-h4w{h@q z=q-I4Ps?bb#W6albM?*%^Sw9BeF$u0vcd*@&VZlGT7(V+k2~hnfN^VnLt$q#Z&7Ku z!tA`wDt9cjtS;-rI|~tiu(2f@LmB4+6p9S^nCj zsptTZy!-lF-^_Yv0L0!EL zHr%U&a$BElF5|-|#NrU z@_)mlO&(ota%myk=c6q%@1daje6Lroaa zR{x~7VDU@(s6wUq4!Q}SlJ0TxDCKsM^zkU(V(F7pUF8FWK_~Q)HFZh>sT9)8`mF!E z-_q~TFZN97#3_-)aV(8UC>96;2rkz%#DHd?FDvW$IN{9aqcq$HnAunSIJhCJ98nBx zw70Q{wmG6O=<2mW$ORoHFd4>m?`a~wMYZh&ABy=Ws_C1Ti=+=EC_G8?^6M8%eyUWk zUxrfe%0?voEZpaW2STovC#08?ga;88ed+Cz!q!V7T%yOrAm?DJ;}Bps20`3aXRBi# zd=ZoN+!9%?(Cpu&iON*8R6r=qYd*~-?#~MDkUmlvT z;>jqgYevg#m=6I@5X}GS7;TgArJ}bH&L05+Z5dm@9M_t(?l;7LYhEgp6HUlrU_%7D zBSSd@1zHudVQLiS>^U7@OyyR_&Xrz85J)NVBW_TRh?7VtNs+(|h&F>}EshOcYEg+Y zKzP)Bm^7Gm^#cvEm-w2-=3jd2iLAl8^m&FI4lmUSbcMuGc?tr;kz{EIAntBwiG{Z z70tN;{%g)$8ogMV)z#Z1c8s!vn&w=4Rs+Pf0Ddhx8Q(FKR_&rzv}bM1vRbs~IMU8K zVAN|$8cZKlqIm?GHFXbP0moP4<)?-l`Dw-dtLs|fp3y*kYT)jxMU}Z5xSUWS;yi08 zva*RqC`wy_IPF@N z@yx4+xEbJKPNy@Rp2Z6!QyCRh;mU&=noMm23!650T6!09y#HiSX z>D`UYZ9paG)aamvy6aFDU&3Ez*QU}k?pc>oDZUGo0zU5=3cX`02`&aCg^HH z-S?E>P8U30U7d2X?Ohz71zn-YMW;MXqk};%n$$^IA#Igf!-<7=sg4#do%7{)y+9bC Vu5}h$sL-DxKtq2BPusMNe*rJxz2E=< literal 0 HcmV?d00001 diff --git a/models/__pycache__/base.cpython-39.pyc b/models/__pycache__/base.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4b56622d4b7215772a8d4364b57e9f34a3e309c6 GIT binary patch literal 10276 zcmcIqTa)9)bw=Yx5WEbBGdq`^y^PknQC`iiB$w^DWJQUk^`+#I%34<{j1&|Gjp2|3 zg4_nUml5caHB#cZyt14)cR!Goq~diFSCVpW%BAv>UywIE!nl;W|RT}nvJXP6luv860Nc#mhc37jdrhP5uzhOZ*c52+qs=Vg3ltXZWN1F`QTU zW*^`K}0k;i-6=$sd(z1wVr$dJ}@PL~*c6ka$r@l`D`~K}bC| zMrKPHv&mVd)&dTTYmmi+aebZI?1YU+E1&_e%QMeW_S1z>Txf2gK6DzLO;IM6Z_4w(+l{d2BYeyhAV`7CoMZ1rj4=JBTg&M?2yxflMNZ8>JEaq-4A@H)gMGKU_J7K z-Bib;qI-%skJgS-a!3!*=XA zRooS=K`6|e{?0N}wOLjE4wh%EV{3AoM+-3dqcVAeG?PbBh_T2WcF5#eJ!Jc8bnXi< zo5 znOeic(26VwMaA-{b#`!m_JvO9pLnV1x=>Hqy2NTnJ5viP}U}8dKvKuZps%R z-Eh1f-boC+*qK_wAZ+ipeV*1F;q!s#Pp_m)r>HkHX^3e#^L96MTfS%(r^|JGL05JL zZ2*Sw_kgYuASDBUXE3U4RpD|~wagAK%}&SZH(i^X8d{Vh4r#F8g9jq=D1kE6Q6!&=bt~OdtEa;c;qz{r(VwBMLI|wZ#cj*ab8a2kuhLxm5 z&n?YgFC}_sjjqW17~M0iS+Z8-3(#2~o#nML%IlkBJVk=VS`HSM z5p_1XaBpqKQRr_9KLWr+o5KbP&aD6`(ZSVdv?rj>eyYuwbhL$ofWl^eo zUALdIC+a{_xZZxlz%4F~Lwp*o;#m^U(apBcTfWx|(0^*nAmxN^*FUgmO1AoZ zxA9P_52(k*t{eA*UL3S{5W%RFn)%u2i*^`i;QAyzwS?FzBXWds)~LkFX%;nV?kn1vlQ+1Zh6qfqNc@dJi`qTJH-p6s`Z1oSpt2T zk{7VtXgU|o;!R+;B*1AQf9=$~gW(SRW^vbZZqZ`LY0>ZQ_?)-HRw9{QEcx@I8bD@-dV^D^b&cQS3YFP>uDGk*Hi9nFtQJ zI+md$58QAGSa7r-fjtsQa|kZ~nKLRh`;$Ln;nS4}QE4Lz zfoZS~;|w@VYoQ)$Lp{+G#G_|{m0UwRV$ZXPP#v8#V~th_nkF}Pu{!(esDRZomz5Y% zjvg4I?80@SQ%0gK(sZ z0NU)6$}yiPIzIev0JEZdBjU9RTwz%{ht3{_8`O=1GqaODy<=`^X{5BGa%r!lfJjaD z&c+n0*$^{0AwyULP*iYkReTPuI%(lR+&`hIhuC^!eY8a1RV6RSRz*@+@D<<~_+a)1 zju(b$Ndig+BH}j2kQVmb2pA_OT4C20cW@z}BykhvY=$g!;9#ST4G5HE>BL;oR7pb= zxGf^WtN=R7Aoa@x9P0c$O||FdiOl~;b%Dip=m`nJ^nBsA!>`Z)a!;6}-6u4E?1!7# z6Tdq1#QK~$$N`;r3=K}&!X*?YyDn^1R8c3V&(|CrD&MB4nXiaVsGmmF(~?6GDr2c{ zLn))|1l-3c`;6WkFD>DwDKGs2D%jjBlKGH@0tI3a_e1uA^7@;o8?S+V4h!Ie1=uY% zRbY4pLhG^e&2vC90MdM6X@N64UQzam0GZOp21vg4Yr! zg*DrRA&CVnvBt_LfJ%uDrfVHCl=T~HvE3=f^Uvr@>J`(w}lKN z1G*?7g(!-MNRW3oN*TBqG9J;>DPxruBsKG55kMzp?UXsG*$(>yVhwr1AuaR+9$*B9NnKZ6OMm5sRN-b?I>P8_pB=&IM&!`03*RJt|nz zJSex6mZB;{m_Ur(nlylSVE{S|09%uxnHbo)VVeR2FkVN45R`(c`%8)}4Kj+XC+9~Wvm?6tMTGK4p$k0RKUF&FYLu@grM2XP5 zYFt=VHsID+b*ut{w+13$BnDbc=wjQ+b<)c>T-3LnZTa$0g!v~NM*`@RIDz}#@CRPd z<_JrfHbL6gA|GiZ4;=Rfx3k;w!Iy{-u|<1tlQT0wv6_YH~Ti>tt$)rW%_c z${A)sPy9UHF5|jEd!7hO4F~j(`;!<#@4Gm%+!JQG$}V6_Z=O24hWbl_zZ;j>#<3-j@%|of@F_D3Jw9M;aYZ$Hqi4|;1_b6 z4?ojT0;OUKV6i-^NI#%+?gv>Or8+a$(r_QefMsS#JVqtLF7Y@-TAklo%{aqvkdB1A z-$bcl9nF3=;nK3n(3=HhC<)1$)+C#X(qyZ4%;IU~{}jiJ#aDr3RqQa9ADFtz>JN0? z28lR0Hy2gshZ)ToJ~5B!(gd)=7*NI;;uJuFrV5;uSQX$YFJS@!aTMcd85Ec|%o%K! zlIA6}m1S}~Kg;Aeax#t?SD;rVNPLgIj+h8dQK#O*rx&i{9$euD z@bS8m;0l-Vca#HH7Ajh*x85m@BR9=MRuS>^Y8RES=XOWAM zt;pU~NOQE7RL0=+RL3}x@tNd$o>gw_$bFf;eelSaC?khGDAFNWk~rGh+Iswr;U@BJ zlN4(si_nJTN8$?B?4BmB;v}AeNQ)WRV$*s}JWZ|7koYZ##s)yV*J`2vRBwmavk^B( z_M%7WagOW=m5y@I8A==2X!oKl)6k%4$bcZ5i%QPeC9KIv@;w^PF-Q6++WrAYL~u*I8_e&4r7g3wY7K&)l*yBU#L~03*%@&36t8CAz{Zn9)hn_{X2& zdV;y?R7TaLEeRVJ67raAcfE9k65SbQX(~IqLrj&B7_5hmU9Hk&N`(L@iTvD z&h|+-ba5`&pZxkW2Sw``YF@Kk(#7^**Ngc4djM1&)Sd$tH z$ueNI0$8<)af}I8&m^_cIyBCLSDcx`>SEqzAGM)}4fK3QK1GxBw?X&h^~!tfMlAcm z*Pm0W8DTKG4J zsU6hlinN1O%<<}s9nc5Pg=Fpj#SZ=zcA+>IC*b-Mr-SPz$kAr#-gCT! zOaWK?IaU9H#6u(`&fY<(v3?w4sa4{v_#>1iXiAWqRx)JG5wV#AEeHRiZz!~{Je`64 zC3X2N5`RVFeG-37f`C{24T--cLC7oq4x)MVe-E*RyPvD-bh;7k#P@ufC z1#GQ@tu3oa_k3U)r*XJCINu4h)+NI`j@I}6MAvMkLi4 literal 0 HcmV?d00001 diff --git a/models/__pycache__/bic.cpython-39.pyc b/models/__pycache__/bic.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b34dd0531af661423eb4a5d509c90f58966c6fb8 GIT binary patch literal 5505 zcmaJ_TaO$^74GWW^z_WmUc48t?Ic5##Gc4mCqWQ!d_lQ5L~@pZEs%ypO?#?mws(7a z#?>`;JnrGPj*>u(qVNJJLNh#J-gtw*fIvb*;-MblA+q3&CkU{7r+W5c8|YD;u3J}~ zs&md)UrjhO;~RMX@Z}%2K6%J6{!E2~kB!0?kg|^;xWQRsG>bQDn!0Q?ExgUd>e@}4 zQJaOl({yy+PTX#(>FK=tjO-tEDWFn)3AB&$=)(i2A zEz)84p}d1clV(pqNMqX=vz$#BGo+d8Nx5~~%8dXPgtWPVq3rbwm$5UmSFrG))e|Dp zo7ZrZlMVWjP+kzkX)J?4lrSoaX(X@r#FnZCtt8B{pc{p0)-aWwMM+!Pq3C3Hm?{U; z=HQO;kH=4SQ+X;GWKu+@`1RpNZ*Z#Lijz~_9*>gj)Rnlk*1sVZFu`HcUppzQGLyMG zo;9puv1_|(v>1BqK9r|bzlfBrLFC3JRy{Gs=7iRW)o$8&JJKDOI2(H?mnV8p^yQMcsZ9ih33gqto@9qlPaIVJKy{di@*9ZHp)xMp$ky;d_g(1+oXPDb^Tesupae zz3b^-xs>!eojC0%J5Jj@4AK+bP^y_|I$1!IHJDh!L}GPM9HDvruq`7I$S~V_1m(4cnyjG4M$OEgZilg+)rmpt9v{K?D@cbY=Kel>z{Don zj-vk1e%tt1PI^B?J+U5R^I_}&+p^L7LzL{zN3b5Oy%<~jUTys@x$kgZ0}_0t-b>+q zPu(0TM~W~`8@6(@-axb>WmBN2a@g-jDF66E~Ca3cZog|l$AuOa+0viLsdC2uc~Sj>~^crQg;h;$aDe?n0=_z&D30w z=`9WPM-=ueMa{ECw|!hy%U1?5a708KIEFGPGF`*`1I7fVbO(uyVd9|EA7sjf6AY55 z;b_!QRobGU8>TR(P=28=*|h1=0x74;T5rnU4>I%d{@GWk{TTKpYeN{U2CT7xyftRC z+Kz4e^adIOkCAFS<|lsHWFC6hXsPa)JGN6}i{@Uhf4Z*C=5}1W=%uCY0mG_n&a9bp z%xA+T!mHLpy} z(G2D&b7R0zPry*#?LYP>#;7Jso0YsKt4E<7cW#@bS=7wrRgfd&ZO|rsU3=SXo2$lw zK67$*jI=qQ&*pQB(=R8iVvOeVd3><|D4pR1Dhr2=MPtPnEu?ln19<{Ex?0B*q$9?t zU@kqg)^;Iv<)LlkyDYcHiyF!n#!LEbjhFSilrM}|CdPBdsC4zo+@97NV}Rc#;1eqb zv`tGUN{hTSUd`#CIFh@P~R;9Qm|pLoyIdgYB!2K_J&wp5ump~zNwoXI#LP(^=R%W{E7SOH8vHBW)@ z_pu#25Q8+pl-W9j!4`oHHapCYu$t)uFRE+-Kbsxdae+3#6zhoTqR(N|H9g>zhki)- z*}x`BCR!o?ZkLCiW{#MkMZ9?TAtFnO>ayQMNJBDkWbmL(d+baAF5B!qL%RE9o!W|) zkp(tWk{-7{B6d0bg4_6^relI-lsA2_iV}Cw3yf#vpr*13Xd#xwOJE|doW-|u;0WH9 zEk2a<;{_Uv>L$=b^Hyt2{9k#)o{x%A zr`G}9oen*?Eq!s~z>r@*u6h1}*5hyfzoBIBL@{;|n)h}UksOG0Pmu?%19TB0f{|6V zVRKN?ca3XwWn}uc7;_*G>*I=2i#usgL>yjBr+9HfY6k9|0V4nbI`<=t@i!!6_=NVP z`;PgwGfA%%CfPY;iVyBs%AJw|GZCtKJ&HRUU^7uGyrEgs$G`h6J%2ra#}YR%u=2V+ zLI*&`5Jl8dEGWMlgW1VmiZFFLVYeGAA5($D((T8Z1?gy&M74W<1)~njoi*`*xdmz3+ zLZd}SIkJp62XPxIBPM5*ml5a-L{I<~2>E~6|F%6F#Kg5+TpeAz2I!~(4r=Cb@xK}H z9#|meL9n3N2K@md>Es45Y+BoW6c1jp?^q+xQ$aE{}1R8r;B@e{q7DP zaUF_^ntok-#bBb-WILaweu?(}f6AT7n+>Ex88=-}6eX}J66 zcM-6&bn(zZTR?|+4XWxhzhM`<`4&~sKTyiiUG8J^;w+kegOt%900tc12iKmuJe|`A z25(g8FP#*+2Z8c~ptw{ZUk!q*gD@#t^nWpet_0#05@cE(e1Z(7=TkG&zj#^^j{e|) zN-NYwTXz;YDrGeYk5x;a<2~=Kd5?Lkg4!CZdM}g%P6o#H464LeNofDn*yI(bQJM;} zrg$Af%@uV6kdADPu0evXc&eh0nQV=aMNl-WnW77B8T8MM{s&83q+zt1?|ZYp9Darx sE|bvyPY9!xJg~Aozg6CuV&l%?R$^qdQ3V+x!ZQ7VuE2qj_I}oX0P&!d*Z=?k literal 0 HcmV?d00001 diff --git a/models/__pycache__/coil.cpython-39.pyc b/models/__pycache__/coil.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d36671862d5c7bb33aa9b5159db2f4a36388909 GIT binary patch literal 8233 zcmcgxON<-$d7n3j+~ExM zpP^Q}8WsW8K`v1uAcdQxFR~Ay0($7Nz4p*kS_Cc7!yMZ~A)qMQOADl~x4-W{B$vBV zt{q~&`M;mvgq$t{@6ZqZzE4Rh5kx#hNGt|7hdR?H2zYMycz z%+qfPZq2QKBDi(WIFQXVu6R>uHXdU~!d6pEmH3X^!&eRaafcFx)*x`=P8ire<*wPW zeIvA8k0Dq5njLvJJevi`%4zxSz+qm`3u4>f^5V4dMi_Mh??ma^%}%fHdrzxtN@*#! zS=);*hk>^p#;Fl|K@_sf!JyaQ!;Df8v|s)1i~sz+y*FQfJi?mWKmHLte|_cgd&oV$ zN90=|ouB5Al8|!n9(&|TF8UA&PcTJSFs1JZEzuQS=@Z?QUD;JW5paE~tKqANx71?T`g& z)v^7~ZD!M@S+U1@&3vjxgTBX7-Lg7CC$=n>$IQqEUc4K!oz!q~uU5|v>^7RM2Hw3G z?K`2%+jlVFJ|8bNrqfu=YqJK8qTt3NT%8hmev7T6=#iM_Ez9xkD6*_a!q3iM>ILy7 ze-OpYyX5ZfZ-;|ReW&AJ>V>Z7N0*$i<6rFWu_g39gEuOJ2%>?XhF<}{{Suwz#p(JT zky=b1O4G5vfj2r2k_aPVOvi;I>PjPdEQ}PMSCQ9Hp5w~4$t=IMQPdE3BuiAns171gT|qaWMq>@EZ8mF zg%#V;&KHr7G!OwBtBU&-zO|$JW1FL+pv&T$`wMI}2DglbhvN5tTf`Q-;FYdS+uM=Y zS7H_113vlyDoDZafGeY$)J`60t~3^hIFeD0@~+H#i_5|otuU&Ne4hBND}5W>hPKkQ zl((e0(0WoQ$3nM|NC}Nnbd^=XRUgPh1wD+olt>5Q=j^*KxLQLPs*@V|i5e{~k1C0p zXo-Tcs$*rCm+`Sk2nW@?oBr z>e)IP!nFOcfbrLp0(MyBsXS6fVkE_NVOUB^V33Vu)(fLDl_&7W#edkJ?CJ$z9R z`R_MDPxgUi=C}6G#>|G$1?|o7uE#EH2H|EeWZq^NZ%$b6F)=VBlIDYu^|;VNhQ$8A zq72V}Twx3NJd)U#@JVGCn)!69#vyaI(_9c*ZDzadIn=QgYEXj)oo7;L&s6QRP7tRh zUIiw!q7FE_VL{TF?Zgvl?Rj<(rCO$S(>jF0>a!61zSsAxtWtH{4g{IxJXN5kqg3|2 zAeA6Lby~q1Rj4gLv}1_Tz_nAU*Gt7)sdzox9w`$q6Emshq}s06X>X$~Z9P?Jl1#6Z zY4*U|wV=xaXS-SEa%I*%+u62YoV>}JX?L|2IzzlNos%tenY5TSQDgHJrS~l-jAE#} zn04+!3-$&tS$92$r!G^9y-u50z{vy=AKA6G?b!Oj%s~;*I?WswmL+I1WXv2TP@Sr6 zyVtW%$Ui%a5nsR?ZGn((tBN{aLsZ1_p(0NyeWgQ1RYc=ZQ&$cp1+QEaYiPHEc1ygy zCX-IoMNO(6J}XZY$dMvDra(w}=y$0A{L?azD+m`v;_s`as!^Jhhw6-)*z#q+hR=|3wxf(1tKJ4jFdB@i)yZOVPaoK z8s^LyWXe0;e45BuyPbF&0s_h2V`PFdX}m-!KuLkUOym_JTyS!4zIXHO8}DwhU!+Nz)Be~*pYOw>(u5gB3~!+ zHj!T?!bklzN}aG)3rEZZMMCD!M62j4Ac9y14;phkTRl{itVH@>=}^+vq$8eH#Pgpk z%Kj2JaZe0-3>gu;CacI9^}movV8Ob8*{{I5Tf~Y$Nh4x2*rsLVEn-|{L`I!b+E{`P zCRCJ*^-CCxoFBH^SY@hof>sF31oa5juljZmy1z*FhQMV6hd>JYsb)i|wmq|GQO!hR zRGQ}l5CDGKp#|#!7b7jw@}?Qh;&o;f(9<0_9^}x2Ct;&mHVogbS%-2DV4wytYSR`? zCBtO2V6&7>FAE4&h&SgzM_Pm_L-(h zqJF4DvsVu_nWvR1^!P*;o*3AXY0$|-2Qrfd7a4HPOlfmYhJP>+*?HoosWJzOQ@SJ# z!blqnagO{MR|Y4`DA!3z$AuVJjc^*c(}1iHepAK@xsXta>Q?*jenQW`U1@*z=s*AA zo!eKhfY~G;oy8&7s5H zj^nXsQNd`hseJRTYiXemhUj^~#u0c1UJKy1&#cJV#wGj21Mdv{xN|f1`ZvD8cBo&G zH_H}~E?E<%WL}?n5jmga_%zk*#&6SX4v{W(&CkX?anpp$nQm&mcjOgE#O-Eo?0oVlf5;J`e$WcTLOD^%$UaV_ew;A-es92yu2BVr`!qaUKA^r4K=%EL-h z&aCxVOe)iqB=8ym%CQV9UrnlRZdDjA#QCIx(F>Td78k(+3rPj=8Oo|#B8GY>u}{Go zCF(^zMrDBSMqKPxl15y``gAw{KpHNhWFc9|Y%NC#wan(uwONYmqefisE+&h~62Ld_ zIfj)UE@Sul4&XEUdyKP$wG{|8phXd^xiUXC)>WI@^SQBCvG(PJT9P_lnfAp_)=*YS z*5=x-&DOD+!1B+nV?9|$?^S^NI#=kcD?&q97F;9Ill2Bq4>y9pecBq6l3QBB&Fx4y zv1c*E+F1Dh8VTlTeN5(M;|XU$iq__xg=0I|nA^cAzW-BmJ2*Al!D4di*bYuVeFxap zDYuL;K|VP>+lP|WhZ~7JI!%0)Y>dwEuRPl1-vUa`j)hkQI7f9L!G-LcILDqzipiPL zIdIoA$(b=6yV0|7%NBX-Vq(P4C6xmh>7jAwn~9Q*7oYEbA^u`gJP_Rq&etGbcdOu> zO`K2@>r)d6w{_NC7(I{Nmxv>lDg05uXy9^~=*{HpSR0;8)X|0H9GUMsjrfJp#VpO$ zRc&hyw~_nQ#!JuDX+nT$ql(PfX|PnCIO5h7Uiw8;q@jHvgr9UMy-v^eqy4Ynf)53&d|~s=e&}o?b?N?a^Q*6K-Uy>;^Zu(BzuZFI zEpE(g$JwKz%e;uJ&0I6 zIukHVOAf=_7BnD3@Pg~vRCB!;?oyhI!d6VQdDwu=7ftoswzIQqGZ#@IXl`JG*z2W|cLy80 z*S7=Lvf(#6{XvF(KgO7Z48lR+vLa2e04v_>4*)iBRs(pbd+Z~Wb8M1U04AXRBZqYm zr3?f1TU15iyyi1cQ~PW{L+bEk+tzL11<^`Xe$HljV$9%aQee!iALX6UiU;Jko8X%$ z@**?mgh18~uni9YrD7-5Isq0LrMWC@3D-Fs#3*5RX~tX@#f(hd@mqi~QDheA&@w(W z^W;KVd z%@3C3&EF-tZ4ZMN?qlA?JwEaxc4QYQpGxJYH z6^=qXnLGx*GspVhkwGkpdnjZ8x^h=(!Rvr2P~isWU>^Pw_)I6nbs$|6KKdPCFtLjW z6}%KB0VF^iPJ)s|>=}_X1;$ZVX zUj!b@val|6%RKs3OEM1y03t!Gt6LdWlj0camGWE-)1MUFVSY`xbK#iLs=g#d|1@v1 zXjh{;hb4p-i4abWsJy{V&CgT%?A4z*;FO9 z5wVx3p4^PIIMK=AzFy`ybM?fv&wXlT5wVF749gsc>dY-i{05P+C{uN$#B9IK2%E8Y ziIA4#Yi=4xP61+A;clvIBNCP7vsh8by_YKF7O)?nM_NRF>LsKF%1nHe+;qP0QM1BP zpy{MIkn;?rEkqPz5Wup8Ehi|G`wY@&2KN~>xh$Rrb5_O6;{IyJj!y~G9b?7{hMZy0 zpJJAu$Bei{M8LZ_AVdY|6pk_vCVvaeiRcXix5S)(gE)*ZqTr}Of!Q6Es4FcCJ77{E zLqvIoIj}7dB5FV}nhxA5c1vxbBH*WdOPa)ARIqXZzhYbl3s#7cfgu$HTm>+r!CAQk zRxWiHz@W8molSVGMrFc&W!mJ?$78e%{|sbFQ~IV)>(to&NcgVTIB zpAPZ2Nf%>`mJnp-`{2$##fOSq)Ufvd-}OBd?-Y&+TNY0Kt0%;5c_wZM3y}0=0jeXJ zgKNv3_ghp)9z6S9ka?N=Ar*cK0@wayO8=P1Ox)P-QTAmJ)8Gs~amvjK{;Z(?;-aaS zeq0Jub$8oi2r7^j!e34nta>JyGofU^Pebz{(;rakgb;EGrkyk^N0P|?kXrr`2t`yW zo_24V*kdCqRv~=AYh!9hqRB|`*dG)56C!^~gzU~q0o+0vFuD+tDiXNMPNHUTJFohC42^9G;T5^EO`0c~F1!m@dJX{O{M&^)5 zCDUqFHUM&oE^@6&TwoTnfA}umM*N3SK*KC$O%VQ|Kh#)`Mj$Ji{UbT4jlBJer zB|Ml#N6F%{b40i%Pbk$VLSX?G5&3|KM})H-ZFKG^c$9M!WvTvJcIK~xAB5<65DpFq XAWVJ@l_K;8po#)k0rUB@e{s;jH3AJg-G>}YrNFwjPtRmesrguD{ARz}2XENhp*(a01vl{Gy* zQ;(jkZ0$~OwK2?scz7L-Ac6=ub#q(z3%GOe0WNSM@%a^*(ea+y@eC3hBSEA;L)qS3`yGCdA6r(eCI!xK}h$ncOX$o1% z2C`2{Z*Q1}vY(|vOsTg58QjSN9touCUkUQ)P85g~NmDs*-F|;3lB)Gi8j2{1QW?ZZ zcP!NovtcUzK_G%8l99+&QwE|J$yc&8+RvnF$SBP-@k%;O2FJbEzWeeIzju85*69z> zx89%s8U6nHjgEEtI?|^PNc<(`{N%~M{r$VUZ@wYgNFn<@Jj5Jj|0#*@K=fYv>gK=x z@YT=1aq5!Fga{EX{j6Ckzk^6*bQw3g=10aJaXg;_2`j z_wcOpI&a|V@+NQLS^LPajjo5;ol_R(B&1f)eiWw$Ddd|F(l{|DtYA~d3~3f<^u#O- z-{J-;Fn8YW*s78b2a!;&@AuQb^nF2NK{Acx{Y)HU>g_y=_mqt}&7Uw;^ZhUm^4#~I z7(f2p%_NmK<6$mEbd%p7?PtTAgRmdpOfnwD`Asb4_TX5op~`2ePqgn1)?lLsy1sqZ zWXMQ8`%%1t@KuETIz(X{Aoxt06N|HnjdW$IM>HG7X#VztC>g|o*vk8(=(VkflBstRm=v?PHh)R`LM0|0TmTd>SSeYJ%lCFs5vk zhHN2jPb!d>UO;D3<>tiIB}r$JmdbPM&=!TPQT&9xWqfOEO!e3YUSX~qlls(v&KqN= z6_(#PN{_md>Z(j!*`#tOn9lPSoHH%2o}dNY20LiUxk(#k83J17E=NBm^HYP@9@}GE zE)=$0w5mu%qje4?KP+y**)Rp%+^!CAZdQWeW=FTbYs?og03D$foOR-5_ zdN>xOe|yyWP#fv1i>7|CA0zcN2R&^T6S;C5=6jqiDW7!|X zk>48(b3r6h4Fdans6CFOSUJ9~L#^x%`!V-Lv=7MVu+*|eHJ=ACW^@%;Dy>yHnCD>} zbsTMtszJT>lOPRxAQ$T^HURT!ZO+F7=A&n`(Oos)`To;0r0fi>MgFT025T{w*&n-B zi`lI8v2AN=iz=th!Lv( zE&=SWj0Sh!S97}bg+VrF^^}tVRO)mIv`Q?1=CYH&Mvd0BVpep;pP^es1Av$uSn_WE%0^FDU$k@s}Jgk+3oza>O!2-+|oU~@`C!aX}-Ck&^%Ra6K)-)G-?{{wUE zjH@s;o4~`_*ySd4wXug%C=IVI>(q)i^i03Yuy;P zq&uk@u>DaU1*w9<4eZam92xhqQ&83Kc@Dbal5D$ZTK_I z$$ZhC!pE5`OpVE+E?p};m~rz2e%aVNdaba_`sDHf`6)FRx3gpv-Zg_f^~dITzNqrr z3T{w`CVZD|UZ1Ryu|sRGlGE*?HjMR}QOr*(Vt9 zSN+cD)q8Mf;r+b0b$gJ7`-t6qINtj5t*tv*o^L&TZTky*$h)VH*&qy$N%hL!c=Rj( zxA#yL9nlwR>RcCeAeKjprwuEjo~T*`|0&YOrBA3|r&8Y}p?xCuwrYj~o%CT;A{!j5 z8rYO+S4X+KL3nsS5FFPS*fhL-871)KGJ%S%BzU0A=m;%57z8QzgIrnRV5qEon3R_J zi)e;CBiu2$Xi%HvZE4o1c|XjO!4U2kMllpYc&z5&Ou^vuQ1o%fg?XQQX`Q7{vtY4D z^}7R+_59rc&Wx`r<)E>Ic@C^r!%T=Ml+t%$naSbT(G)n5@#u-Wg z92cdV8YV4(p>>83fPlOuUrjd%)7-EMAlKgN0%qp9QKjXss6YVs%2jS}ImGw3Y#_6W(+kFY@O9?!9SsyaBjfFE(ayyD`}+;~H+Q9nr%U5&Is`}>iIv?C!Tl^Bxvki>lw$0SB19+D_Xj7jLb03lEi zAxi%JHpR{%D856 z#&^D_gWseg^kGY^Kx`r8EeJ5*c5V0P-MV|jeM#qaUeHUeD*9Wh&^rV0HuN^rmwjO) ziNl7@*X=YF+P|P(soJG(DEo1~O*eD>Z9&z`PlemN_zoeeRIphV!PS|rp89*Fs9ngA zSr#AmrT&T`=n&ESQdMuiclVt;U*8b}QvC)Aa)8Bek@$eb_ekuLAopB^Bwi$;?-kpK xDRxhf^TOolX`;!&jjFr>-(pDRuRxURj(Ix&x7m_u!>gtp1Bq_~vlyk2{vTo%|C;~+ literal 0 HcmV?d00001 diff --git a/models/__pycache__/ewc.cpython-39.pyc b/models/__pycache__/ewc.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bddea5431eafc290759286de9d0a800aaefeac4a GIT binary patch literal 6594 zcmbtYO>i8?b)KHTo&CjP@ee{2WK*u%SOQseVkIhuWFnHXT@*}cU+%^_9v*_YgM%)vR>zPS46xGF}RORlmk=6gK=xFEnB zoZXr?um9h4zxTcO+N)Me8h$_f(szCToTmK;6~=!i3Rm&OzXahLXQ9^0->jvhtcOO| zXc;W8Gg~I=jL_^_Elbsz;O&;Jcq=S)omNrtc3A3`TV=%;!b-Q=sw&_x#wFOiibh~l2O7!A7po%L70dEpn|+PQvt z|KA|h`ak{y{r=@j)7Za+^8PzSen>4JJ^G)YzH#sBmHi)~gytJ~i5fM3kI1_q>o32y z^y^=I_V=#r)2Z(3AnjinNA-(WeAFEKp;7(fJ9q_Ji*c=`zpt$^&ba=-X&Kz$<^v6< zZgCrLix;?qx6O;Zgm;0Lc?EBWS9uNZBA?)Oyi4zErq(KBw{VT8eB+KECcv_gA z_!A(Bwx?w*Wg|wakZP`;7%4>9)4@aGFpa)?qiIPi9`pkt9oKC~?ZkDZz1EI50wIc+ z1=UfIZ1=<_w%m+^a7~)n_xKT$Mc4I1FOFUJk@n$>=etpIJ{-h}2+s5E-HqPheBW<} z=es=~gz@=c+h6JLhjpHaiXk9hj7*YLy@kW}l?mtaF7 zJzIrHH8<%&C2~oE80F2Vw;dgpA2D$TBiU|^Bze+w zY=0f}qesc|JxG?~Fgh^pS0JKGl3}ki>}^LIu{n~`0B>d%s6l6t*ja(=nWK7AoldnW z>lBkxR(?_o<$!uTG=_RoN%f@4t)#~7hbCs6IPA>}YF3>)_)}j4eGT+29`&s!ljzgf z!-;R2;$M@gY#L?E(rJv82BcaxgVF?^`bdX-$KKPa z9ea0b+a*Y_bq;MOL@B?X*a^7hgvUJZ&{0%p5?*_6EZw%Jf!a$Y*B`#@xAYwst0@{K&xp7QI z$=obFxADZJ*=>k*cL|@(hI*=ZG;A76^^A?QOdo0A)wj&sxjJImstC3QZ4t!oy70KP z{19L!Z5||^zwyYB`pN~#BFXwE^!E~P`-$J{_Ah^#7A9-5b(`5A?>0$?+fTsKRUf@_ zDeU=P7++p_GP1b@6ZQwW83=+5Sp}dft591ZUO=5#q=w?4&%GpY*L+zBd+Y1iS82AR zwI0BtC%RrD%QO^yfaD3Vzb|c1c+q+wNH#%MCO%CwRg^q{b;5pGrX4*j7-0~>fV+b* zY4^jxUGEQKK^rV9d+X$8q}5J>ZY=AO z*A2Ma$M9|dvyXA^ffqF$1r4%7Lc3ir@*pg{3hBtvhrp{qzQfP5ys*FFwaQVj?XI;W z1(Fx(%RWQoDU6vsaR@J=S9}XZV>RY5^RZ*pn8|97O%s&qO#iKI>C8dB$?A`7L-A&f zHCP?(CB4GxcyhY@Sbw}^;>dO9}>tte@wsshQF3hw?|HY(3N^9WMw;OuW#T$0^i#~Yf(iPuv#HY z-FAqxf&s#TcVcOvKmCMf!$5UOeAUhOi;euc=FXlU_Bpr>?E(4oJDCngmn?JGg= z9MU+Ze`MRxB4}GfKUue1QqEb)>XQl`{-(iDe4x~67wDL12}|@nw#SAR0u5`AT?1g> zy>s9Ilq(!*2u0Y?;W}!I!x9A~)bDPFa``O-uA(xmrj^Ww0v6z8s)wAesTM6W)5?f( zV?i6%5+^IBwX{0YhZCuhm1y>KA}g!6kyX^&K}~g}y`l}Bqy}eKvxnX?=u1DQf_wr&RA^T1uwV>K@$tu(Wk4HS_tBna*r7mlokx?P)D7ozWq;4m%coD6L*9_k7?sM>BCa-W{ zM(r-V!akj>8uQ_2?;s|EzkF%=dcWsyfIEMGxctYLmv15tU%vn9%4gQlc1MAv=leTU zb#ZO@;pcw0ahQq#c5J5RSwYd5w3PRdC1pDS$s&sZJaVAyg`AT36cLKC#h=j#c0*Qu z0Wn?JiRkrrWD&s7%YCbJ&+|98J;4#g!c5`3;2RJT^aLh0y52id54NzvJAE(Wt`|$g z?+>IA54z$itw4DLxwPIO;-XA7uM#2jkdrQX;sM+vzGEOfe@9Nh3&MtDU$l|`^dj+x z)ccP>n$xG*uDC}NIepPvckg*ZGFMvpN8=FV7?@V@dqM<$LNX)Zm8>o8c7#>YlxIS* z2xt-5KOki%UoOGb$eUR^l%(bbu%R0tf}>jRTCBWsAUz3ebZ76}`ch zSWS1>qE5ccL;$a&r~Vt;IaX%+ho)1}cPA8ns6#vnBC}ZP2t@w**&y<+GFOUu5l=r8 zM1K6cg9y@Z?LCU!OY~kt9GX_CG_NUe(Fjxk7ZZrG>lC5F+zo;i0M2AqI{=)C1HhTg z>IZ-`*=ZaDoCW}=kxc<`rpbH(I5Pm8*(1Q2I|evvzGS|$kSv}8I8(;}Cj^|FIf9dE zz|Rs&&yB&)+yVSdrPF|unP?f1^UOYgs2Kpx*>vUraGuYWbN&EugpZ>W3QS1p6vT)% z;x>p7z=$@HO(GOLi!PB6r1^rfa1nu*w$k!g^r`g}s}@_-Ac&|4wR~u5NGRbvy)O4e zOsxqK(qQp6k!>QB=ZhU8yF~62Nr_w~LMAPSM3g=MC2>baE&hs%Ppo$KnN};_rJi3W z^4CQ6h&&+j9+7Vld7sD!M3jj-WTHq>*IGG;ZxMe(t$$18?}&Vp2;s%khU%eigeRtG z;e?^Wg)coe4Kmad1`9KO+C=T*xCxtu3fo;&277AQaV+tnkclqAQ~)5sz_GK0qDr_q zt|RAs8@%2zGE-%O6bR@;1JR<^u_(5DTN7tfmK0P*Y^7v#2J%6gp-81XA3QA0YgvgS zkDf+$&5>`yEwin&(bs5=|6_%B|DP*hJZ>7yJTA?(oOHWc&Q<77z~@?fjkwo|sBFaC z40hsN36;Wq4lUB?_qLm6u1M;*FH-+gatHA(H2eflOg%K9Y>iFh(ZM=?wF+32VRYKB=y8)wPsuIvQv5^np zB$L8H@b+z-tcl>wQklM;Ip|qL@P?DkIRtOG1NcD8wA9AwnF!vTq|_;QD##z4v;d4P z0#9=sEoKV$)M`kY5ge{y{?f3VmOFLaLoo3J=upYhpf9M&Cb^XdZm-~jE@@jo*t(2# z8$ldKUOXIm6ntUiB@Op2Gc?BnaR}#t#7NbtW$Wjuj`OxxkEuFs6s1m6v|$uMs~rCv zCLhc36T19?x)YaGj8h6BEs9Fuq2mn?_ijh=i?~2NK1oFR=4Il3ALPIA zoXGG`WV@$k_$<$W6hte%ozJyZDXI#e$czQuA&9RKafv7_ej=BGQrZJy5m8ut1i8|r zs|)cRYFQz2gUD47L{;rb+WEDtEarEL2(+4&F(@TtQOQe=lV7Leaj?vR@hMADyR=N5 zuY>I4QFkvIa%AUY+bY3Y^0=k0q85r}mXSl(5xeBM@`>$hR_FqhZVGS#T5{d|Y5{!3 zb+-myn763=OL2}CeuKzsL@0C;9uYEIC6TNi+>QD{7`pBlhCfGLmWfdCL*EhCK>*m6 zV>lO_1?MHudDYtdlyX4QQg_<9zoOVU#}#!VbbJWvD2iQ)qWpl!W6FA7H%QuHyh8UE z>K;ax^Xu4^d$^_%1?pHmsKK2(E_ww4gRJJmaDhRWzxh4zL0!`8g`4d}5cDg*DGS$c z-TwT|FRqFnRsA`Uw}{*&@>L=ai73r_len)DX%e|b( zD)(RIB<+-0*v@Kq1-(SqA zQ>vFlI_s1>GxeEHrC#aG)@MbT;Z!@v>c={B^|{V`eO{F1oQ2L}eNm+I&hgHP`iaiT z`bm*5I4^Zh)lYR!*H4Rl(W!OL)XyMYs=xf6#L8^uvBYNV(ym&61^Eh_MSj*UApa^; z?nz5k>oZxB41?Q#&k6jR=hklAUU}#48|TlRyH+!5hkw)g5LI{WuEztLQObAi;QW`y z?NN(L?y+72)p6~`3pL{-?gd`MbNpH@ZW8xjxq<9tw0j?aHK`pBFs>ixlNN@tR{L`8 zay%(5=*p$#tH~0Oy>eyw;?#n!TwcC82s{^lN{~!%7x`~Z%&VPe);NTHZ0aK_Q9Ug z>XIC3!8YsQt+*~|%IgL#${Nj{+Xz~oYdK3wWZbj@>yBqJn@1T+`*wgvxp$GcV_V#{ z`H~t{Z+L#owQsu(Zg*@qu$)yph_Wjm-dnqMmy*T9Rp?S~cs(~TyB4=Pc3^WqDg+ka zu!9SpYj1i%ln-pz_xJ_3*XizH%IqD?VDWn&-@vkT7B9t*jDTCN-}HFLM|nXo-T8AL zc}}abgK0a~hK&Wpd5n_2(F?p@uyO6T&i(6mcW%AapwpQCsY%BF1PNP$_)2xR6r%WVaw8EH6MpXV?h3L#++JiZmB&5n)3i?XJFzhhdB1|6a1+fI`& zpyg8>x@k5X%lA$5sr1vIKihGGvrf+sxP6vA=x=(xv)x9^Iot7=?f7S#cEDTCa(9Q% zqxWlgd;ytqUQUarUy3)pJYH4{O=z#ZqG%KRky@zWDhLn<5LWs`PJtdHb>WE4-26ot!gVZi^!!*h=EKQ8Dr);EY8>SxO+S^%{zKT7XS(F)} zJj~&p9m(^w%{JL!d%HFw7DsCM3UV*uIWO7M6KJeQcEEK^Q-$cT-!eB zV+eevYcbYxH#UZUK)=6#8)e2iZ*6P_uI>B##C7{+YO6#U*WC0R@KxHe-Hl+gKO-Ox zy-A-v8ocpnz^VO8zxa;h)C6|<*HF=~u5qj7TI-HIRk{|X08T9zXqWcA9&gx@MxqjB zt!~$L8GsT?v4X~?>9_iJlmpb6B$z2&r>f|}FH-?EUwSRd6Hb~P%LNQ_K7+!@xNEu0>)ha8H>$?z_dVX~dj#7n zPOBT`Zg`HzKWGJz8eXA3`kJ>U(3n>#n={3}K>a2lqS1?mi@!pZ1z@<4HL)rFGRpWO zRb+cztj0E*jc9fZK_JfjSZZ2q_y0p7o`c)tsu%YJoi2j3!9(P72MwL zwP0+R8{A@?&?3sw#ED*jQtL)(w0Adft|ryQ$HvpN9zr)L7`hzOiIrQ9j&qz*asL39 z4osO=7t7n|_Iyr=9OZA~KtIC3j$=CjP%%c7j<>Otit*=7YAJW2(ZmenH9@0ciX?Xw zq-I(N$}=97?$o~vBFSlmAk>gGqCeI3qN2&hv$Rr_b(9sKX{sTYWnGy=9_4CP)={g- zbI)|NEQ@sYnXZ=QidvMN(UU5cCl8q=`7h zuPwDnpaBN$G*h+|{zIfPOkE^!(}Qf=2yz6IkKGa*fYFB~fUtscZ3H74`SM7TCHINwor`B0m*PD`wRl(P9n7LtRkX}w9s-yQ zD^iM;_EH0xR*14x``8d`OIB7+Rz{%r{K_!n=dO%kmu8zM$P&O*S>^X|m;ynfoY2cK zdmEenb6SuvhMM}stQZ3seg?@UnZF;u*6=#YLHc#a>Z~*Ct#wrX7apm9daQ2+BQ9Te zyoTlYZy^!4-1xh{rQd(N9c9FN93FuWVK6+f2_Q{oH>@4Nm)`Nntm}0m1x@mYYLzG} zOh+IXk5X#@n}<<4-k89nHGxI^60P`Gh?pQt3o#}rffr>k&HWa@F|%p;RuJ&`nBJwf zZxa#HYM5e76Bx+}+9UPedp7|REr(F*Ju1mN-1HlpHX#Z&65FKjxUd=>z{;JwQ7OT# zv4sl}Z+M+<51upd>-^OG73g$wFFy1u$_^Bl#0$kRD`$OgkT@ znZa_9W*9`m+|kMW*k1S=WJJS%SA{&6&rALf&Ov7B|I2uU{Nz_T{BDVpQOvJ`tU=>V zv|pt21pWort?-|tW?v#g;sje(vmUnmr4p2~#~Yha_MT84QA%hDp}b;kMexmU6ZwG1 zFAzCU8)egehzoB7=6$fQKIET0 z9IJxJpiH8NGUHKYGcw}tV)(;0Eq5^613W(IX9*lp0Y_-h@=yUq{QJkow||Ji4{-q@ z{RG_~BSTz(TTvKcj`~eyQ5wm>#iW$Vn3R+%0iiwWh&PBUn6ig{RL`U)dY+I%YD2Zk zi)!1MvIv6g34Rj>ur6*{O>X%<7KCG@8;DXC=hlum;n>E2X5SDv1HrZ&SPU*~g-Bk5 z@J$4FNfmcc&w3DEA>Q?@Yd?sSN5!mzX?_EbKwJq&IVblQj~?M<=tIXy!ZbO?pMckd zGZs_%h&XlsKg9_bA`Kx>i9aACEpNy{Ax!VeRa_m!>Uf|Cdt4R5{IQt57|M&15I^dT zF+vhWbwLuUt`tHEFAxJj)h*~mkI2ERKu!*3 zsD}CrceNtUW>(N@vbGp#_~)FAaJS)G{jY!q^UY43O!fJbc_MFLN>(~6Ryv#Hsv=hv z>lOxsSog8d0U#dW@?VHI`_J%OnyVKQagLQjJ?~iSaFyd6?^EwrhzKkBXW|ZjMC}Gd zJ|yy+LehEQW=VGAM=Wut+NiOT&_QtFc)Nxh#22{SA|2RPnR;eorXFg!`8h0+WGb6W{@jE55esLza2gNl0r`y+om z6)M=lEGz!39^vdLJ*b4K;i<4fXcC+r*5cImoRF*1s!-7z^R?ZKTdV z8q~i0R_%`G`?W{cmM=C@w%J3ZVQ(iEjJ|F)wjNlV zA$S+qT{zHz-Qf?ZpGs;{v9~el!!8_u)AA#=(d|X54?|iYiBPf7sEEfhUZfdsQO}u) zEsV|dVDRijC0MX9*8K)=Az19W{06nY39?ib%9bCtYq?JY=?LF%nCsBt1R+sM*zt6Y zzK>7~++Ok33}x8Gjg8OC?Tgfv7qs;$S2WS2Sh2)l&OatZcRQ82b@>r+9R4D~&EkO?!xjBuPOf~F6%EHwnqwR0%X$K@%I3`Fn zDcBbiV_me@r`siCWru{Mg^`FQ3Yj*h%((q8V#g%MFtibQGW`Io-><+ys);kZR>S2{ z7>)l98uITE`94VhBsC$C=m}p;v@@NSA5ddh@_OQvfi=We0x*?Oud&_!Z5ro0L`oo$ z+H3^}3@dn?e?}E4A1r9|6V#IQYoZS837>)I>oJQ_s+inztY!U45J}8iF9=qgtSX5q z#dSrBu?vH~)oj5(RbRld+nD}u@c88Nl2sr7YjTY^7oHud$w}b)z|mYPP^jO{h{}E>4e*%&?76%1WXH%m_^C16-CvhegaBKbf z!*<#K;0uU>5wIUJo#k=-M~T-amO{sOWM+g?OhhhC^PgyTI%rAYlL%ZS79nO9O%8qQ z(9m*noTre%v^Y@MF65rU?MDhoMWMH*da#&-Bhdu>M=E)`nS}qNl#s@wYs{GxjwKCc zn*V~Sl2ZbwEB)ioozkS~>Qb8j8JhDyC-N6W=$!dq68UQ)2k0+SH7X;<`3W8|3Bn^O zT@_^rjpMAC*8pb|5Ai=HLYgpE)^z(K;2X!9bP+OGc3n<31^-=;s2rE|f|lbiGjLHH zBOq5W%Ee#Ame=u>lvAW7D#Yy&Z>7*TCuP9thT~6iNp0YH&K6LL!r8n)gw%08=WjXm zsdX6$5@T6T%zeM*lM*gdLlN4ai6^n}iPP$BP{dAz@A)i^l8r09t{(^SGq*mvcl*wV wt3pb}p}gn>BmH_D+kT5oGXD!8TA3nEq%HH}Pnxree;x)s(u$^*zb;+=Z>}*<)&Kwi literal 0 HcmV?d00001 diff --git a/models/__pycache__/finetune.cpython-39.pyc b/models/__pycache__/finetune.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7c06bafd8354a672ad123592f1a57ce45c81c72 GIT binary patch literal 4748 zcmbVQ&5s;M74NU<>G|B5*)Oj@Hvs_?#k(PaB#M&QB#x6Hdr4x0pq0>RclXTBY=4Za zYhv%Z2O(`xE?A1-L{2^2%zwZwM-FfSPIZI>tjL9{2@1bgJ^SU@LQ0SN*RQHxSG}%! z@As<4Z8WM1uD`tfYwv}Vit=|V9K8$_F5*uA3h)(Q4Hdh1sg?ePzhDvRE zSc**Bl(G?)qq1F=atS(?ZAsY-D^b<1Nx2->qlVp(vK2O?IeSjZm9Q1H?Y5MwVJDim z=c5IC0p*&#=-2I~zT!82tJ=$uPxwuH#h3C zrAsb%uRt|mC^g@VJr+bk%-wJ!;Has+;-j(VlBn~zcF6ump(}X=8 zkE7A{-XCB<|C8U)_4^B5eeXq-_ihvXj#@sy_wPSny>aov-dj}lErQDgKL_;Bzk2## ze|z?c3wsty9`0k8J}ae{AOwo7`iiZ6r1Vr@^|cQzES>HfA1b!tmwXe?l3(^MJWapi zSMetbIE=mlMRb2FVOy9sBMriyFfe!NJyld3#|zywb)0+3 z{U_F=n6HQ9l(S&nzqPZOjMqorAY6|UKM2$H-ptCi(KcH~zpJ=Ya%V-Yt2^!ek67Cu z+*7G-c9Be9^eXPO4#<=tZU^%R>(vdI)^vD+)h;XwQk1vi+{7&n`V2yb1sNh9pM#wanTM&)F~LJ+sxFhM zI%Fd+0d=_qW?uHS+>)A9r%`RD4lBHx*A8pp5O9gR`b6XPOydo|#GAf(*FcYR``Uh4 z_Nw_7ex$2ISBGxpK)1!)(CO@A*|!Y#DWA_5P)0Aq&Xn8$Qoe}N9PZXsgNq#1|0n4V zmq<@ecy(isR%E9w18NZWSCe@4&gA@^i70XA4q~i_nI>cA1;U{9zuwhF^@hv6O(z}f z1fra7j(fc@5H&=cL(Ubjlu@6E^C0DyyUXw{Z(OVlTcw5u;#Dq`?1G&`9Xmk{mGQ`T zdEoRsQ4W)SANB}i5cd*c_7WDkT-4}Q=zsz^Xk{c!m$`92U}fs$8w3=_paC-8A;v)* z)X0+kf*uAjd@34;d@u?Fr#~8}jJ!qEiFw{ahhZ3m!gORG!o+8f!=P(Ow-a?5-HF`T z#i(E(InKUCU>^)LD?R9;?ZF|H6@7mbqo&US6t$^Zs_})TH&sJzeqk6uRa3S9nk7xO zP;aQMFHBv^MpNymEwt~PJaFzWb?p&d>|Z4a2R{BQ6yOC!A7 zq@vV!qsZ;*>;k@%y-aWs(48+_Oyh|tqmK^;UcjE9R+=24Uwi$MsE)AqQ2-iAMOhLj zhQ+FZL643t>LGJdZ!;jbfOB0XPrAl~(Uo_^oYX#IPGifP z@A|vi#GDYXfitHH_898K@-@^}CROM`|B~`vj&ki?9h&t?Bdg~o^vhE^Z^o>C~W z-_<7Vtn62o6u-KqeXO#l{91kj->`yOW0}&lh2HS3rPMPm*t)&!>C|j6C^s|DSI5P$nv^Bi1vKQ zdqj7runwN8>}v#<2wtN% z)XSpbF_`HfSXeUJ78S5V;`Ev83+smKZQXL2kF5`46?ugR5t3ZO(6Ju5w}lqm#0a-X zZtOd5Ds*o&7J534*mr4!Z_^0KHpw{lS&eGe35Y^O+wqcUG)CsacZ`|qZHqZ1Cp3uNzBfZ_8OqO@UQ@8?@&k9h$VgJhRcNNh*I&_fKpCuR6HX?*nKj*O?K;l6q2Y zYIRTs{pwmrJ&m%Zp42E;8L9~)sjAw4OzT)#)$SWsUE5hWBoVoi(w<{XvV`#tnB;cm+|~%36qXea0~D8N=wr zFJK-kYv;`w)65N9$C#!Ars;rX8hMA}7EA+{$rlcoX7L!)$o}}!aG9TY2-D0TW19ED zBP$0yvH;#8RylQqcNS;7GoLMhM;7B%aLpriz)g!_n$y|hjA_o~XA618H0&Cdi(Mxm zZ9HNsxI>T;ObCb=9*oGhQTI#SDN%!h2s|EcO8nct77la|sp@*b%id5+2#ufo|29U==FpOd8);Vj%IxkzgkIVHD zC3&KkG+Zn@+%308bt+m({xlWPR+IA*t>QEXAVX@wXemKB-|QsR;6C0bG=W}8w6Jft`fXS@H2uR z5WGS_e>W6|m5U_NiA8?vk?D~WMbjg!7mJC1nPMq@3NTx5aP-k%3ugq{!~qW_S^6LI CsG<@8 literal 0 HcmV?d00001 diff --git a/models/__pycache__/finetune_vit.cpython-39.pyc b/models/__pycache__/finetune_vit.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..15b83a88442e9ed304465556d7922adc6405c0d1 GIT binary patch literal 6455 zcmb_g$&(vb8Sh=KbtKJ}89VkQghfEcjsp&hv9p*2sAMX{W@{3fo_;M^nqD-oTXsBd zL5gt*n@QMrXmJbw0aY9*E^vSg$1YBApr+tLaSK$1-}hRxI)SPbQoXO=zJA~D`@QGX z>lF#VZ++xd_w-vO=`WNx_|uSh5F!2(1e2KTOIGofEk&eNOT}05)j+c}naWgO4-Cr? zam_CUC95RjI_j8~DdL7-4k}hv#7llHs9SXrH~mI1W6cCjs~NPcmdGpn?O@iL74eEc z7tCAp!Gg7bbk$m9HS3O^#OgnnttG^lS;IQUW~}3g-^rS+)l;k!h@WI_YlY2Pcd^gY3HMI1Yg<>M(W#-^_fG{8^ZfW!7pItvLeIY1PgaN5`4alL3n4ZjB)O*4d;}q`L8Q_E!NxXY9r7Bc*RYca>QhY1i;><4qif-Ax+C-BXva_P zi50~1iqlOzZYNH>0Xvprv8D1>dv`Yb#V*4BIJ%vVa4Oqxhzr6-Lw1U->6 zWg@+}j=jf@Z_5{@XFl|_GB(DgwDhz(Hpk_0B`s$f+LY6Z_*T$|o|Y%_xSDF2kyf+P zM55D6b)+i0sm;4T0kqKY-&SQQY~Clu&z+WJ@&6QLA`i?|BelDwj5U$crzt%tXBF&R zbx^}7HO#Y~)-bb&$Bm@U)U>g!&}eC6L&+KwDVt#$>dmBDI{oPSoavidThTRCssv_CgQo=&-HaD!J1^7znA*gMJZOc+bFLT z7E`nx@+gUlw$&oPA2RRRyh2lfB~l#AK&9JMmWyrm?5_KE3>3c;Lefl4lQpFxHxW$P zR4Z~rUPWl=4Ov4@MQInZy06^(9UW3`QfwVTC^tc5O6q&hSSZy^Db+4Y+cK0{9~-HZ zL189xCPR_+O$})U3S^|pL>-reajI*FESU_9Ea?(LJO|N(9Y0eW1F6V@zM*VtmkKL| z#l8dOFl}Y^H2(mix9!9^+DkVIhUY(n;(sDYTW{Vq-O&(RNxQUq&W~KjkIy5rTOUrx z@In{(lpD-T92{JIuYv7c?R89{4aIpB%D^8)|Ij5bj^V~Q?Tk2TexCA;(7T4gj?S^X z*Xz@$71Q2IIEXe5YO8hIR1cj)8C7Yk#5quF0+&?9Sv)w_Q|vR$6kC=7+F_IifZc$Z z0$?(QDaZi~WNK;(2r*%>+CU!|sSa~Blac@lg{cG*DYm?$t?{>D3}M0i9>h9j{w~UR zH;J<(NYnTOB;G@U)SSN!0xkqQxPb=8_ws2^#D?)FDE1JEhe=!@K}$NaQ~V>ya}nY; zghWbPsH{v%-H^BDZ`&<8r$bJVw3d8|*s-$&%hR$@v<&^g*S&kzom{9oP%W7gj202jZ{fyn4UD5 zaSOI!w{}~yQqdbT@uzkw>Z+()-mBZ*t$S#cqPB+G+M%9jlR4Cz-^ST*XfMfJOBPsV z8*@z-vpZ0N(FgMrnj6{V5>hRMHp1)#ZsSPnGsxEl%T!y~Y;|pifPAW;0KAnbT)8p6 z_r@5=O^6GlG~#H)U4m2?(Ph|EMc|(u_kpve_{yl;^}W0bT(^lvcsv%W1!CyM$)j8K zCq{nK2gzG`?8%SConv4W?x=9Y{BDe$*TLe1Ulu*`yO1YVK>T+ zZo~ulhAK6hdINrnDjE)V!k)+PqeM-BYv60ptK zv220N$ZLlK@u6)o`L-a2`CP+OleRU3QPtHsc=65*oZP?f?Ez^C8jP>?Gn?LB&Ofe zz)iG)IVgi&P%fZLd`l=bVIXJ?t|G&{G&mSgtnjJ-OP|$)eF|kS7p$fChu_oh?-%lt z;K4r6YXv>9ndds!q2Xo_(H&(J)Xsnu5sP}XxtBS9h)x_6CPJJRhAdStWF_=jkbVs|Th*>q$a@x$A z7^RgqC-Adb8yz*xbAhz zI@2A$@}W1@4qXu%ALyxbk`pz`^#a}LR?3MD7cSzX6e9rS93qz=&Fe15%xstikA~Ot zGF=ObtB1J`>T=ED3>VzQ8^WO^UQo~)m26n?HbLT_9XcViotWQC9jfst;KWt-d`vV7 z8Lv|g5jA1kc}3tz0SzsiXzB=j6pI*f$Gx7<;Kl+hh+W>t6A1{VMJ+!K(V07HwAZO< z3nDk+d3yG`!v!PG^(pfBSRF3CxHfU_xd~mca2e+BrzS=}#KI|^zl|1!)&X<2ziGi%D$>>iqkO%u~O1Sty`ZVytj{{<2*U+>v4M>J(~PQ>al1XaGTG+M*k_fSj2@ z^AO~;0Xc2JO+9Oq0RwUXH`&}Cqp!q2P&8Bkzl=*N4p!2$Q08;aSoIBI`Dde2UP8RWlki&1_5GXATL3(m;DNcCPIL4J#WnZ#E}e3b-URruFP+#>OH z5@gQ&MG|}F%wM9Uu;p)3Y~P>_VQew~7G>{S>%!|=EB`jt6j#^pQ0%)TzDMHwB*;2B zSrh*u3ECd3)^#>K@Lqbvj4hLl6i--P{v)dKV-i0h@lz6?CUL|-={57W2=P-8l6urY z$ygU&)ie#JdBjlZvEZPo!ZeNYLDSVrc&9tDXQ;2uH|q3|7=p;yc3!dVg6JY%v+d21 z;}<331(3f96XX|35TzIQKYoQ`L_7r2a9T}XpT0G^p6}y%a(}^hQoR)tix3nMcSpdF zVXEdCvt%~RMYCnDit^4q;>>bgyafy7E>wd2oHZ+6q*h~(_rb5W1bN~rN(v1m*9TOj zSF6=9#8A_Hui-ch+p^$EVjI13g02F3ai){H>Lpzp&K;&nT=>QkOt@KXy{06gCwq^$gB)K2%syISQy z%c{sK+7&k-BD3NeB)00n9*f;xL1}KDD-QEK*=T$kVyZ%NZSOyaSd(6&pj?pZUQDF^ E1GvLbrvLx| literal 0 HcmV?d00001 diff --git a/models/__pycache__/finetune_vit_adapter.cpython-39.pyc b/models/__pycache__/finetune_vit_adapter.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..423198711158a3b259fc8aa77bc7b632c503b8e4 GIT binary patch literal 6410 zcmb_gS(6)A74BWFbtKJ}89UCzF%X4h#&N)5F}CBFB&3|F5StJ>4o#14k1V+t&Fz+# zaSL3GA#5fITS(X>iWl<_sN#t#s(64G9=mvg2WkplDBcWJ@SWSzY>uIdAk{s6_w}6b zeD^x#azVoH&4$TZ?~|lIQDWy$L*gVt^altgG1-%>>?>P}NUN5Lui~k`W@$2&sh;i| zmLcMrm-BO0PQ-Q8F)dTX4KME(tfGkLypmtG$|7!h6@S8-@T*qUuUR#bm-p)aq%|qx z1#ilqwx<0VYX<3}HOorYJuQiqe=1vZh|jZ%wU13$`w_pFRavd2SO*ZlkJYUOHfi0@ zrmTZcOKh6W+>qFeJGr4+4>0+x)R^7G3Z>1@s`~+k?wDQP)&c*~8Cgn@e@jMGx=22se51Up>Dvk+=@gdgRIdfBEClhfZvoNHr05{nSw^I**7eSu&F> z&bN)me_2_!=zF3i#$&k(KZ@S(#Pv&9ezs#kas}td4K- zRY{Yq5;lA?&5c&uRCY{(;xq{g(Lsn<+K>ivA`fL*ij@Qt*-#S6RudUgV-mVjmd`cx zRF8UHm#3y}w}W-hx}ryVM8PAtQNW6D&L2~TXLGW?WX5Mk!|0SZXaFp zgLuj7MKO1m*yZ&LVQ;D1YhA6J3NoOOe=GdA&ZK&318Tr=?e(Ij{7MelE$KSNmo^-!CNjfrd8uq#(Wp^r0vDq1-Pf z+Q3MPgWOP}Q%rQE3ggt;!A}D{P!JmzkPWHWV5ysa#YBm7z44U>fR8Bw8{tCt+5_L6t$T z`qiY0UTU3sXRGrG}}RIadC0;U-&hqQ_a2P zc&Q$Tym=wb1)<&I4omYc4AtdM?50|?#~dry41-J1p44!MKr+HOV&rF!isC zQmky;C@o}`leO*gFb=b}#Vo%aF!xGYpsBzjDUM~J(j6+x#kRWkYV#os6g>tRbhM)H7M!R_^VNb}2V0wvHf_o1iiw^*z)VN_9g@wA0dt z3}x2)Mj{QMFhhADLy`404QT}mWF*Q^?dODXs>{19nGB39=`2Dt1<`^Xzf$T0vB-kn zP}a1wnU%s~-;Z*bwz7Dbe+naA%x$MjSOiPWeXQ zUdCX1=a}E>^*m}tw6_utqK<=FuiY`#UFVQTRoW_X4%C{!B~ft}C;NJWeTJE0%LagW z7^MM#H(;g!n1R9+y5VZr?px9Hn0O4_BgzUO-_T<4DPT4 zV4WogqSC$qcBPIg9HueIF=arz1$T)&>?0si?&Rabptzk2uMWR{Q|&7_2gxo8=UR>{ zi4spRJ+3n2CTzj3ja#x@)*CbNr*}8IaAH>*c8Ej7YWzi$QA9(_}`7)K}ys*{6 z#8RyttcIzv8gd`Lp-7EJ-heMrMZ@7v&~o{slqd;s^_{>$4$o1S=SaLL3Nv3a0{$XR z(;g+}b{5$f%NEE?{z>)Rz^d5(Xf<}L+l%;P)F~ldS~?9le;#x2Jl9JNTXd8f*t4GJ zHuTJ3Y^t73P>ecSfHj{DJ4)^vcaQO;we~iX&IVaSrxDq)1TQd!Fr)034gBl!yj(}T zu2cmOP03YxeQs=Acl5IhlE}EW4D44(Kyip|N}bG5U@}lhg?ll;MwA&t+;$MlF?Eh= z=a@DpG5w|nZlZO}P9E%nasgf9n?tDy13_zW6&dEG!NGuH1uy(x`Yi73Qz&~rV=b-U z{g!@zJ(1=FANF`!%IJa3+@^CC8gBX_-Bo&is(74tu>7=0{Fd2nH-rZGGG*o)coF>e;7Hr-d%G#CO3)BSAOPlE>|+dBG)3 z;b|?q6g?3HuH!-R&%BTpGvc19)QmlHG;r$Njp#~6m#&79so3XeFo(nnjhGj=GgvB5 z^IK!=(JAht=2sA+Dukp|z$p&N2cS2Q%6et2F`2qGX(i(y`E~MM0*9{yN@CDb9IwLU zIdF#!83b8PbfU$t%deap5o3YGLr_?z^i8HB7bJO$J`a#Cf;t=h646#rVyMkfPRiq$ zB8d`dpe3NUq-LPC<{+O`lJZaija3H)>OGkl6vbB^l*HFWPI)LjF7?g0(wRsqbXU8n zh*?#Wd{P}$F-k3|4&i49b$HxKQ5(b^I9lF-cj*__PQlxX{^IG*Ogx+9fz7!oDJkrO zLbE*lY%R&L!VIqA7Zo7(QC1R2J%?Opu zeE0gv z$q?Hl00Wmdq5f?j{ArVpSTwtR^c*fs@CXkte4I2Nv88MMg^wLyI1@(E!nKbqKDLUo zbAp^Z&E{3gIKVGz?UWN6&RoPNDMkRuIYcTym6n?v zGqYh9JnUXg^K>o9t{&zJsLN%CGhB3Iw+n|7yM9J%RI*{kT?2`~(scr6I}v|`I#i>c z&xxyS`IxX5FkYq{B5K06(}KW}3>s=S(bOLJC>GJEU3d&D)jMFNdRJJf0aj`wSgE~BSgG%Vm1hAO83D{dqa?`R{0Lwt<9yo|0c6ah=2$Nf74n z8zeR$MwZOKL>d3lj`^3V%Bv*EE%C3C_!^0?lc1Xj{|1ShB)&<4td+k;V#`|j>y#9x z`YnoW+p8gLDdOLz>}_M5zN<0v?@&!~GyN{bzDI(rgMXg{86SU(#M>ljXDpMf4sTMc z{0EfxLlQqC@naG%lh|W%^i26{gy;-}r0%geqoioCvOP9O_x_z$2P-r3JFQU5;mPg5 zmd(90btu!@U;s{H+iAhJGtP>5$+p*ej+d2)$2|T3rp8Z`Aa*XUa{L0tToMEtoYs++ zN6&{%*Yoh=xV_YSsmcP0SqO@VYao!sFje!2nKLWqtT|yWit@%GaXzUoo_7TZXU9)2 z%bFCAON)`q+n`G8f(LOGCBkK7l*`QO1OX>HD>N&uW_i81?L~`3y2TB_DrXrW#`HAG zO{z~wB;HG{qIe*R7FX~9#f!+sJA&vS4nyx^JLV;-D%9-_inSrE2?26`+k;obgLPz; zN0c+8nikkRUQBngPyQ2Xr+eawtgxeHQDkN9vdawdQgNdZTeV}4#cnU4G&PT9hk2Y# bF**z}QX#pt^`AnlN{>oVE=YANCQ|#u7_V*c3IL)vfNSqbI9c z$EX@ijKEw1!!ElEdsXvh{sSU-;E~v1U-q$r2Lc|533wrRGZ66mvU*xG5{3wZ?#M6i zqcXqm_x&>6dcDH%`@=6y{;Rhz_6y1!{prX&fe`-`f-|o8%r0WZR%KqZHN>i~1-h*( zRHykyVA`fk>wYOH*(I4a(8jVYnKu1$P_e5rUGi%|-LA{DB>?W!70ez!$Quj*_ck2&wy#%PCHFd4Sl6M#+E@ zmEJh)CW9z+eac;O6ZbhZ^+ZR@8y^n4!VA1Gas4eX$=5Frf|2h%9mRvtyQi*J)cw;S zK}(EOeael!=R8-07)s6Si7WbEayAORt5K5I5-*G+aW)(Wqnigm#!~y={SN(paIvEu zoJaoP28r)e&F^3P&HsO7`-zJO&r{J`NjyX1Wr+TxPoMnN&(FQ>;(>)+7vb1Xo8{sc zk?@$UaAvEou%5yduHLqA*c#VwGh62dHxV1W#4W@oFY^lG60hMKlVb`2Z8n3rZ>+f|P&fzq@H3h_H261LAWB~_-1!V)#bLiW{^Ia;bf>d32W z3@L*%461Eiu`7vi2ceS;e7|GnMm!#QBDWl85DpT@5v1_QhF)?#5<9t$ZI1J@>?6CV zxgL9dPnKdAP`~Le;Y$rD<25 zMu=+=DI3xk<7n{3$`+KR;Up1Mu2`uTBfk?y*TchnM-dOApV*iST0>rOd$hj9jdwc8 z$MP7Iz4qa1?pAjR2M&KNBxw0^JurEesu`QIiP~oq9l=PHOr5fCY~#FeO8d$s_R@!5 zR43-7l$Ku9Cf1}psifshN1t+9k#PlM7-@N`Osc7#nQ1jEO&OhdY9LoR%)M)nN^@)UeKaTEohonlzF+ z*V4wmO0%Vn9W`rAS+>A+v|C8^bYYEQRn@G?p@WlV+QcZWVSBhZTuK|~+3pWh73W}V z2|6ffjK&ASUhM^M>bu2Phd)?f;?hX1^lU%0(IxVRoc2$l^&2(PkBtS6T(se~wD@qf=QY@{aG?Yz*hS5-TlvLDqA**+l z`|d!;l$#XWK#qcc<-$r~vGjc~ZFTc8aRJHucH%tkWj6{& z7e0gP-y*QRHy@hrc!aH_UD`bFM_t#CFCcN)9*)ZN!Y&{yH@TO%IJo+}gX3Q7cPyz5 z#d#FUAfCkd&?PU90cGL?$jPG?7pdF~z3Z6l-ZhrzqkbB#W|(IwZ5-5I>%OHPJBKnl z&{oNFAYVk_l4>}MCniRUeTJE0%QCnF7^MkkV8ToRFqz6#lmG@YEwv09<20OTe z4iaKzIqiwuFtJ6cr%2F;h$l!~BC#vdQEMo(QplpxBdpP7YNX_6nhXB@9av&P*3t*QGHR)WNk?2y8mXEra3g7Q^A2pmX&v_DrD8O0;ZOZEwAIkI zJa5}RY z6^VaNJOIv?;;Z9c&-d~waNQx2;fYwP7RaO*C(rEFKQi`{0f^zov(JA#?wkP0=#C3V zEZ&5<^E#NH^viNY@d(N!aEp&pZOM=NeJm{32VpPD&0Zt|_=YNVnt2297&SCq;f8%r zoTW@nf@|Q0E=oj+hWt2*Psqx`m&|~_O3QR+nYE);4(4(sGLwJOd@r;cPB8l#r#~9U z;@vbTAzfa(1UP>lYw&%~&rL@Tl$+SIvF~+^!eBl_&5H%fS!W+$uNTwKvd8AV$9VGA z;4bSfCW$dVqm3oIe5s3#Q@|F*KOw2^MBO?ZSh>YB>l+M#jXH_kCgBIBAH*w2uG z;t(@s!@^KtGNej7O#B3~2IW-6jf7x_+JVn-GCqrI#8(mPYm6Irbnp~oSRPiuG^i9X zCgT!nEf@>T6UL{&)O0u-(5>B{{$0i-vz?C_g41O6l-AEUBO{TfP%b!4|NnkMzh7U> zOOiwTBCi$X!Qo!ly$Sudf{1QC;~-akA$G7id6kGTcf3&q|6&GUyH`C@CXu&_ zJJyq7=(#>@_VZj$snoVDSd7iDXZd|+;UAD(DZ*?kJ0S&o-14_A`avJO4wd<+>$ z6An012LyKD{T7U25P)kP&=JdSub;bu%N4xGJ2pN@c7W8GTa%3sT-bOnisOx2@85h^ z4|P{0k$1b@n^bhRH+lU-e_KBcMaX?*q|QTvXjE<#o`5d3;_W2p9w1&IAxIEn=E^g9 zy(_RX2lgSN(apR}SBT=8Vr_%wTz3V>{W$SPa6X9_6ck7!i&ni|5dIq@H{^~Ri%l9( zi^qYWJ(~NYXdH4;r;?XQkfrApi7Ew9v>c+eV{lh|#8|l9oB0B6H$aBCD+YLifz5u3 zdJ-q>EZu9kdsKCc2DRX&`p&j1Bzw+{84mgQI$V%(#S+5nCUhCY#aW!EF6JP_%&07$ zUSjcCD#b%kEN@jFKYS;kwaEqKmh@Sa-9XzyJWMHijQLN5_#Yq`$Xpw^KzET9c%TNZ z{kW&#Uqj7FQZ9fOn=OgD#=?5{ye zB8`^_8i?Qv97_h}fsHiMeh3~Dq2 z8bFYhw&*r4A!lLOJO(*!Ku#NQQ_tFDzaIbX(%540FI0TjT36oCTE(}irM%|; zE2aL8#J`jH4hgbO@m&)CL4vl&uJzm<4;+~uNn_h0BgJ!9Pms}x|040soQizGJY zLK^5zWd9c-CJbcSy#`9gy7EU|*I}CX7%Dw195q##rdd8}x_YUm?46n$>YpVZojN^9 zhF~;~lUE$4AjL@69A|gz`bCX=FBGS-KuJ_2{g-z|F`zOc9};NW!Mau+H=e$HN$uUDI~ zCk7x|dj*TqP*Z3ixjvvGy>M-Yp&-62wJmQJWn;Xt#+$_IrX3`Qk7yD9FWh5u2%y UQk(ymkZRJa6_g87o2O*%Pv-Y-LjV8( literal 0 HcmV?d00001 diff --git a/models/__pycache__/finetune_vit_lwf.cpython-39.pyc b/models/__pycache__/finetune_vit_lwf.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3753e58d01d2ab63abc87ef55539ae636c6b66d GIT binary patch literal 6921 zcmb_h+mjsES?~MwTz6*nrd_S&m92zKDqh)2Awy6sTUKNzakh-5Kpy9K)58_W>Q?r{0CI=ln0UrsG@l6M|glOc%gVRP~rETp4p3pd7zlB z{`%bdT)*%4{l2qay7oZ_uIn#}8tj<*)* zq2U-R)#-s5T8<^tMoGi*66 zSym3(;i9uB)0JQ;Ty~bj6=wzcsfckHHKf;B!#T?qoDHO(U`^KQYtA{O zpJZ)klPx;u*^=`VTXvp)MPVy!^?|}x{lx>_xxmh_wFipx38vmuI_rm6t+L%wb1jM> zjUe_Iy~Q32@iya;7*L|p8%JF+h$Anc+;vZQuc4{WJ9^&uV$|h+=tsf}wtbPW-yDRa zz<)VT29f_rU9G75nSnq{^i;j(CH`x^$0Ky5<#pllzArAtk$*cDc}@6H67x&ZI2_$O z{2`{=|L%9__m@{Y`r#Gi58om27gY0~AO73le`V*ztB0>s(Wgkd$5EyY9*v{Z5RRE0E{g08gfTOBhulkv#sx$U}xXdqmdlddBh`QmQOcXI=~nqYg4 z#1DFW1C<}DdD(TlftMt%`=RpDrHf%CE(YU7aQ`B^yMH?#UmSG@!No9Uevn-3;V8v8 z^4&Y=Ke*f58r|dT=!d46SP+U@(`xE|bGERp`Bb_pRnC5trd7U-C!uMi$`Fr>^$fL% zK2;{hl=7xro4Vbx@`fv7=v6UQKS6#sitk4AJU@$Ga-$oM?&VgG`~JS4*W5T@^R;(W z{sfxv%_DK1zNmOT;d57b$u4P1Lgufiwz}V*FW|UAS5{I3{91TOhvX6f14^n*l&O+w zeI+ySm_nUsQ|0{~92pMkK)tTK_3E2iW@V+c^roKKSvjkuWm;%fPAl?VK_6yXo~l_j zH6~VCos^~u9eQdaSDEMbp88p!j)A`eT~(syb4v2=vx+MJehyNoLpwD{TOVqfA#>&| zXNvNqf=#XtYZ#@5dDhb!W_B%Wh&t2L#(_qorHx%}(!en+FazxtQX^ehQ!uOQq{*O$ zSu<^-m)5X7TpTW?jVsFDU!@w<(A?&9w$Lh_4Z{7}8~!Npx`U{{wY7ElBm6qcx#8dO zg4`4_@7~T!QSA1)$MUidC?hPWB0f-W69C$m>)7%{KK_saAXy4~?UP{oIl)?yoFDkS}>vePMu0QDC7P&zk zbXrnJoT}`TAP~l6O3jzX;xznl$7c-daf3l3oQfPxrgL?Wn>(JsnpBs69-EMxvKA|w zf0S1WLo2==@mR#gx7DJ25HbIqyh2lf0aF^wK%>W$mdkDR-Cp-I7$_m37`9=khE`FV zcx=_yD{4dC!qYGts)3S<)-GiASh$K5)G|{f;fQp*W918TfVd4_)$;1TK)jh@3oXevQhl$iIuh z9-U+PsMnuItAzHJ5T=cT+HXBJ)l=tCMpN16GszttB@kckD93GB$7s8uvULj*Hxo)C}Hp z2w+_$AEVR00Ctt3F8!-DDKTwAyG7nepJE>YiR!Q{DwFDQDM$clz(YOLa1M%FlfJkv z8mT50m?@gfdI(!^Tk|hjspyT__@nQ1wAInJeAKo*Z+mK#qP2n6#;Kkc#S+>rAK>hF zjSp0Ah!s{jz+A=Z!=6(j`Cd7qNT1hLdcE8{>6KoB=ydHt(N=PZawcU-t){xrtU>tKM=Gs_-1Q49&({0*ut1#!QR ziRH#1>cw#SF%RJzs`Sy!8}QFkL(Ah{)c5(1Q>G@tHS{76CA>skevO19D+^yT1O6&a z)1773PFA@X%azDX{z(t~$Z5FYY&C9wG*0;Q)F~ldUb_xBe;soO0zb$tS9X+J*t2oq zcg(_Iy41Xwpd59!0B5}zc9uOg?jy#Nw+6@TyBH)vr^y8f1zunY&x*F8TKH?L>uMY6 zw$_wDw4^rG{k6GqJ=V`DNFw7pGO+I>1H~b73gRXW1ttT9)Oa8OHnPr|B3eSK#Pl_4 zU1G+X!pw&TxQQ{ehh?w}Dg|`OcL}vN3#&5tg4RtPQTG2-qOG9BP@Acm*5@fr zku``OQqWscGtgRlQcfFbeX4=R>XQofo-RzP@~uy5@@=D}K2@GqGFvo;3u%KwyN8;b zRWmK8%}EoZw9@7jesxISYkm zW%$`vT4I$Igz>u?5c@K#Nu*vwsky3%_0&dh*qgIT+M1eKJ9-u-a2BgHz^1fpk<~xG zeY!ZcvUX}tHqtg3;@-M=VsfrXOC@h?^F18MKh00*L#=T11oNQHaep8ZehV4t`VUE6 z4hg`(LH<<|V-j~Da`i@D?{dt{ zg;{V463Y}86rqQ`1L|_u;|y`J@JDbc!VkHmH9Fa_=I?>Tzccb8=6VT#jylwnamf4B z|J=vKdfCWjH4{*H#LP-RXyAYkF zM~yb4s$ZZ^ZFru(yW?@mh;ws>JONgRxENs*=YCgEfQ5LOU!_m1L4=V}SzN0m{1>Sd zmpqA#lzf)t<-*foZgK*-E&UK>H_^5b=P7j`;yZYfFF`0EXl>9liZ52+RT_x>5qsdT zhMEoJZFL>g4Uq}_j)s=ne^~aZysCX<*fni`@d!=irpf7?f|N5Dj_xxgq|o&NBM_Ox ziZ+843vgl0VCC;%z$K92V_4C%GBZ$G$*OZ$0sO3^qy}IqW%UeDF^7}1F;4+lsD^;0 z4q!0>EC31gYoAfFri`fDS)pSpvQle z#Cs5(3ld8BuTjDO3M7)4a&n3Mn-I8K`|@>A#3bRLq%0!-ka_&qss15}-yrcm34#;; z42cg&e2c_yk{~$YzeVB*PWZPeE3xEvDD^f82`H8{_Jsc~l^mnRnU6;c^%nm+kMUA}9fYv zupuK;9#D!Xk;Fi`w!A*OJni~Hfcw@>s+Hm#rQ{wEZc#!;wE#xTHtb7w$!^%I_JX~| zslKa2%zu<7M=OEu8WKXrqNxU>D=zh@1oyxvN>ru`1+hLj)>Na*L>Q?uG>HS9rTbxA z%pLjm<0v28P`bG)+`pT|Js~OUJfS0`k(!C<{D(iewYHM!F8+_?A3yT1ZE6$?43*1V4J^-`bm$a4BE)-@-djWq$_k~+g z#3}laCYU#ivauKh$ri;(GN5znMFHX%y8PvI;lhcF%G+?KD(~}>tsUGK@+x&bYcJwB z*c}L7qo&gMew$JugtH)hQaA`eLhwI{#Hr6Bgd#F<5Pac2{A94e|BSvf19_uaIq_vx z78TzWu_uKQG63a&N-5F@xoIahcazF!hjaT%p|&qzmzCr)#LQe^KJz~^r6%15!jK^K Oqm(ouLvMav+4*nbak>Bi literal 0 HcmV?d00001 diff --git a/models/__pycache__/foster.cpython-39.pyc b/models/__pycache__/foster.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..de5228226106355671c40335e4b0fa36f391f76d GIT binary patch literal 11175 zcmcgyTZ|jmd7c}|;hoFnUbT`a$rsU%C0ULyQ5E?n%W18ylSs&gjXN0e40p*Tht!#& zrCkq`)+;AL8YN27rfHj|!!`}#6zFRU^ra{Q^p>Izd7GyqeFzNnp+F0?aAK?9{}0LK zN>1W5MM=y*|M}0EGw1sKmos)gpHcAp{)fKO_~&O8zEc2;OuA8stnKwIy zZn0iuKHVvG=jwCaa=qN0ug^o4saKv?L{{W(C?eTy23o1C z*VM@HuZk}2n(KlldVRk|$xL(LHvCr4wL6qLZ~OMeo-G`SRQ4I$b1pizbUB&6@ZzP* z&wY5^@oRciYV-!KZ}n|ycOBo6UX=H3x#9Syd#*l!IX;Bm<$TLn>%!%?1Wmm1|#Js3LmKO_R5%ma_etAuKxj#O0%)IhN`Uk}j2T@5@Y7&L*jwnAJIoX%W|b4;WSPN%^on9G1mGM5FH zVlD^HTv5XOh-xVa1?@|(Ux*S{9N&I8zMYQpEzj~DyRqp=>zd6;&mRZ}bAe>0V|T9z z+v;tJ$h>A-FWa4g6B)f;vr#KX32)GMWMo=a%We6V6&aW@Pnzf&dMsVR>L> z9#UDLtRfz=8XeEN=Cn38eNPrCt3+guNSO*HI=yR-l=I+gT4Z=mrzsbacwLRs7M`{} z&$3=u-aLM?>-r}<1J9SvNpWpwvo|=|Z?rlmyFKA_ypzqIht+hVzbzN=G*#yMO%qGAdYp&$l~qHyjV?Ew^{gou*Tr-o{4D-H42q+w4WjW>0o) zKgv7fx-6>ebyXff6C|}GCWpaA8M}!NTfXgW9YNY7d{I$K~^wVKR zXknJ0pge7yi{*_Pze!KEbF|cH$(~nU$_zDO1e#U2`Wu{Ayi5Kyt}_v4{SuYfMGL4- zmL?}sk=n(Z;;q)6^ULAe%BSoZd8bjS*2&hMkBD10h>cOJO5+`yu(L-w~ zP~#c|l|l~y5Mc>Y_E>+LFP)_I+k*59C|xiiP$iiK0{xmalry)u*}zpzty zy3?sLwtBP#cPf{q-E!?K9cM;*IZF5>1{x>5-at06(`gq)X}jNdT*8DXbH(->o4o#_ zETGt;Ee^D<8S){#DN1>pgJ!dX9Y?85zvWt8r`wa;&=-#9*EIP6GUS6GHJu-kCrLoX zYe%A-fWEN0wrg(ySkhKw5U*rOdsZ$Hp+z0#p%{3Hb4z5EWa&X4Pdiq#5zSA)$3puD z9mneXQXZwJj}f_tikpp|`!ayZLn~*Lj&!aLTGH{X4QUIgwT@^uWSSnx(mSbrp5sI7 z^`j&nb2pqQYf(Yw^aq|K?GWWi zVSW%5cREfdN?Ke;w0LFE>Ih3Zn^2w}fMYzSbDj-sL1SY?1tvzxxW~0b3=XHMQne3d z%8i*cAIcQ18K>?3Wx%zEC41={EgeDUZA^LxNR62wF17R zn%W{HB}gDwHT;u5PA2JY#@CO`6y+!NsqDvQs-%|jYysEK@))M>YQ$YgOlpIpE| z5m2ckg?3IG*d$co$Rw=`Z7U()JI*15F{mW(9b^5C+xW@jsDyV5hVC)qYeXI=@(4&| zI8c7_K1%OLTrT(t$URbg3Pz_yiX=Dg2M=I_CU^29dytk1$-Hl%7oX*}gEAueI;F`N zkoOSRHa-h;09+Ei(B!B6R2x_tn=zChW=6`cCa6S~i~zpF1P~@8TB@CE=L4!UE7Fi= zZfZl+7#4so8qm%d8LDzib1NuO+)zPF6@?8ZvI0$ijwHCsyV`jeKTFCmA@crwNaYuT z#7J9KhDnj9)UpEkVgS@XpbS$%3aOCNar z`g&;G!y*}KkdAu@E-OmtLEO_haOJytIKS7!_F+-k(tx(#@vC??(LTZ>r9!hj_6coh z4%2~2BZ?Aze=R&J7Dj52ZXZM0l2BfQ?afc!^P1Mw&MLRmt9^GBlEY$gOUEP%wc?{{kO!Xcg!J>CFJQ~Y14v_61{gt0N`n7$d zU;Doree~o3>d6Rthj+;8x_ll>l{d#-n~?O&C0fX|qG^L&f{C^c+vI&n}KX55KhbLwiZx^1N7Vv|LJ@5`?n{f6x*5|83CqXbjCKM zC7g!64aH@4djwpAZls||ZZGl(-C?bA4am2wcb{n#bi4 zo^POcFBOI~(J+lh>X)>u#-+Gxut}##>^j$CGY)UiPMnI=)(mFE2HM$9%kvw(ZvUJ| zm6L&%)U?|>M}FcMOv-PdD1QI`MUxf`_9ox`VQtSIVKoj#0lOxw5moF0eAdx$; z>_d>fO7agA`6!W(fk0O?|B;CxDqiWqvWBT)#RD2?_RbHH%fCo#`fnE9o?mT|exez1 z#Lk-NRY?{%xoUS>8*X*%dmP!<_Md0)nWzYDsEltlB%$^58O&ZGnaOY|m@C z&eN{zpil0`#SgVqIBFBiZ`I7K^vauOuFTlR=FSBbbS16YjY?O9jn>iBZ5T>FLC1c+?GDzb`sons( zztxTAtfpi81L;_k$z)e-x@~xd*d2=7WqgN7}F$=%npn4Hd&I-}NxZcM}r$SC1=0(=W7(L4g7wH#KfsDM$yUuoUFOxu8V) zIV_KqaGp!2gN$DZ^1JW^hncHS1x8$-ztCRvmx45Wn{-79G6!H`h%9W3xd2|%q9XFF zVV{A{Uq)Y6AWe~%7>`k4qE7VufD+7&62r3l7)ECmxhc{=!+BBsSx*J?qvWt0Bw$gL zr}hMMS69crk^DHONID>|YpleWTn1|A+@5P)fv(viXFCb6yM=JXaVn`sI=09g0{j~U zM-Z4$GOaGu(i^k}xRRa6F2m0T^uDM19O)HsCtn#>-+!)pvFCZ!SI(Sxq=~%CY<$~| z#x{vgH-|eP_}@K;s>pDksi}5IQuE|hBBZC}H6m=hl6sXbB1<5VdLha;B-&*Gk7cjF z9i^fCdaeYw?{vA`iaPl>>TChi;5C=PFR;df#@1WqXm{=nte%RWmp- ze8C?=3{S``i3y})!+Fjkv>3oML$e3cZfr+IxKO}fuOV9q8}{5iaI00`2~Fgus2yfs z_BO05wv1FOO0e3YIrBV3P*RPalzI}_j3bGj+$E_&j{s|>BfKch z5KS#(Y>iBIj3{}BV(j73*Kv8I7!@E3VT*x_+@0iYLo2+Uggcajg#J0?uK{07brsa0 zGT<|+_M@b^FRf~C8YbL@@&sk}6vsZOq2}apG1UA3Q~pz+MtkY6c{}fi8UT&Zh6uq7 zvkW@`m{AL42dg83-zwnu2-ltq^8V4F z#PFNHdJ*u;@Ar?j@A2;qa)4jBZ9yLJizqeV7Y>$~2mDf>HF*Q`MIpNwR@X!&ydSCK zC{x%6#z)6soW{FiSk6qj?36nYtc}vcgYJ2;@Qc+mGKU8N-L>E#L-zyWgE2RT?!|T4 zz>;U^mcIe+viwa-^EOU59fVT|u%iqaR%2ehED5t?oNp3GMsw`l-yrVOL_S00vqU}z zva|iJ(3l_{_FHob>nvzG`_o%un!XG6?}qQ4jsK@t*WUB4^+ZzOfS|D{e~ZTF^F-LN zq}V*d`Y)WvPks)gD)f6qexJx6fZ%|G z^#@IBJw4`|&XnB}^6OOg4~dXYk>4QlM<8rQlX~IhD8EVaKP9sZZWg12#s0%|ZAu;mL_qqBjh{8WfIJ)JmE z)E#ke%}-NW7>JU6jJS3N(yTi!WfA0RVGcA;wc(xx%;T^uEP^kM`MHrY$qh+GmD=>Wq6GG5LCi@_^yZd^4)~wzOk=SZQlDQrPV?{@R6Jss z3uD!{zzVYl&*OmraX6^H2ci1jGxaF?+mc(lKFjA*cMrY;6GXzfrN3QINECx&PDVE*)#DSkx5m2>*%ptx~!5Ud>r%%T(oYUSfeS*DwSgcM=6PkfiWQ=GE7%!(N1#`AxOXKrs@6z`g6-TU(FJp0x5 znSl1m8O%@-|ARLZbS?*@VkP{g-~i9f+40_4g(N>Kxusvd;6D_;KUl$fTVW2y%n-&@ zcrI9hVWSP#Xr#m;-fP4i#yGEuDpnL`9y^C*M_6`rB4c0f*m{&3AKO^|cHg-W>vfLY zs6du|K_n8;b%`J+34#tG@h$>j58*KmfbBdqvC1cbl9|BA-9gGYMn><$IoC&VlOF-8 zmuDUtN6TcLa>*sV)2?A15^IxhQT9)?Szf$^4=2u4-nd(mY+4c3R-|IhIg}3-JADQFu z5b0^oqj%&yT6>Z|I*9$>Wg50eiByR^P2?FOIuTMpvQ30mO})e)C(Irb?m5kmqd-)o z^E=D)>10u~8a~@NQD#p-u4jMS!YSbysuikDky}JMMA#1M66X?GBJ!(5;K7ZZ_G`qk zzs}xv${l;;`ZF(_pE0oZOeV6H@a@H>jTrDd8BJ%Y!gpye#h+aa>4_sm%2YRv_7;ve z`P4T`AWVl?K^7aYfl+WNsF=2BvIbyl2Y)Vu>nEC1^(-xeu@x`>jM}nGgiI>l4zW#! zGfTN8LlTaN?6;tz7kYFI5mSaA?*%9QV2|$4&0EvObAwfRKv!H({Xg16O2N}u2$YNJtwE& z)A;(n4_l~@wDq~4N5N1(W5W~2sb!&aZ`DwvQ=M_&jP)orJ{_-{jaJi^51%^K`cJd~ zCn1ZdL!JhyCvotPqkNz}IUP}=g^whz-`VzGN7m=4?WBV!hHYq5qj}Qu>8VkJ@MH2L zK7or{-UQ=gZg4cG!^_9lVK8)5NMP2?NGOGwLd1kPEi?prHHofJTm?t7n$Ud(pX^YN zQG((T_^<_GX3E9+U1EL4U%rU<@&TsY{+zXXSG zLU>pV-jW8Dl&sSzmKBc@2l@X3$y+o)0Ca81ka@s zXw$hQ31(zQ0_O+wgrk8#na&fAlPB{e{}$EMk~}N&@4(ByC-M(O*n8$l(B(gp;Cn>g zY4pF1jIZr2PeL_Gvf5JYRsR(|$SQ?4Qv5RNOng9Q@-fs2*Ok;c4jDwHST^um9q$C~ zBK{~Q%En*how$MzXk-edqkLQhUwY8@I`MadW0~LUb+%eQe|aNmU9j&TrJnoXrC+}I s;yRBnH*;o=cv;aL;ea{*3g#U8rFd14I3_!Bdt>LfVgT4brO-e9KM=~1P5=M^ literal 0 HcmV?d00001 diff --git a/models/__pycache__/gem.cpython-39.pyc b/models/__pycache__/gem.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8597032e7bf5f97c39c9b421917c5fb12535b22 GIT binary patch literal 6824 zcmbVQOOPDdRn328eY?84)IYVPmPTr+G^3AaV8$rMGlTWB9U6IT39yq9Np`*L>gvwQ zs($%OQn#|q;FcIL7APQs2nZHctQarYvm$~O3)q<5TY*_X2W;35Hsaiu)unFD3RP&wD|+T+s0Q$8Y^{^9QeK+P_fd;LkzkJv@n_YZ}v-9%)|o);$AxBQiUd zXX#XCMs~;cY?am{$8%6-MNTK@xTU@s8-)P8Fk+qS?-zH>YdAro|;)qyKlc*to2*vNz8v z-U6HQ7F!yt{)O%xMS6)%d&gMKJC5`Tl+U1i8R?U(?ww*syjR$)clxHr=GgpWjm?Ks zyQX)B>36io!XE9;?S?KL7T#|+L!>OR#X5K!aSUm9d!kL*!dgFWigq^+BFbG0L~yGc zu#h8H{A!Sdx59wODqHz_+~i>=j71RL4h0nISG!3&4qqxaWLX5f6^hrpak$GId(ey6vMW^U4r?uY(XuXW{zmw)mH+t=UO`#a3H^=E%dzrTLBVeY++{N6(nKcbqS ze)69`et-47clQ`_s6LJtpQ7pyNPGm*x^Q*r-+%Jv`FHm$zXi947O{enr|`<(o)i98agv>-)K0}PWru&OV{_?aa;I4 z&!Z1y<4`>4@=aOpaeU5hKk*s9ldMd#QLq|D4MSQ<7_CVQyPtfb%e?P5qaaCq{}b)g zGnYECxD@ph!NW`J!OnWOf2r4ON0&NX7DmaXR@k}N+vW=xk!GH_5Sm`X-_dtUSn0+6 zSvBEglOLsRJXi4~7a>w@14|xhLt{jXG?BOPwuLjyF@5MFpC73W883Wi`ufAL(~APW zoV0hsE6a}tjRIdpSJG;Bd)v}k<6*cH%97uW*kms%zUaabvlWJktoWO8_dz_#=b~<_ z)s9=zYR79`bkgOWK*(}99?Yk~8ah9Qf%wUZI7QN^$M zi5Jy}K0kdeQdN5)urn!aWdm^ZmS$)^?F>v|2m{#AM*7@jcZN1fOl2apM~2!XU6mPB zn(7;QQ5Y6gd(t#utFYFm=D-l8)DUH6iwbj|(mqe^YqMO|19R~wT@$({bn_G4Dl{~f z+q9nPe3se)WYBY}4W~yMD?GIZR*Pm?Q*tJ?u4(sH26k$HXbzk~Zb09%Hms*PHA`({ z25FblTR#>@hO<;V!aDpKa&ySdsN9U0M|xow&Y(vAvRE7*MG0oVu`nX{Kx(*z+!UVb z2$+uiv6Ob?IOSey-$S{zae~?@iz?pUBct4zR>%blSnkHKsSA$=(iS{u$M6#;>GpXu zlolEP-G(U(tAS{)E1QtHWWB$(7KO41!1BrQLY{Ey6CfHU;zq;dmyqL^sVE=xdST3f zij8G>T(h6q4L?hjWuTxkBh@osp;pSW^Zg!NIP}+=oV<-UNSvo8MQTFsorD5l-IGqh zgSZv)9I1SX#2chnQfu)e1=g}i6Psjw>Z22Ox_p}-r4~!n!pUZeUFh=WI>1H*%}wcK z;4IB3jNwF`ek9txDD+#se!>Y#Wr>2q2be_^g^_f8HL#pr?YH5MJX}YBNZ^Rscyc-k z;LKs-1IaRHln!>eABBy)ayD6_)%l$u4zMc33i>9K6(d$)A>;LX)d|Aw?}*-(x-V_j zg>)3|`7gWgbo)i1%4SPDmQbkB^a{M#dhVJP-O?-1EeldNbmM1^ZRjq_Exr2OF;&_s zL8tWGu^eEas~3QU75W=hy#_R>;MqAsc<`C=niK){e*}*cSfFqQgQcjTZDj5Zmu6aD z=PM{N50zVR5rap<-f&c#9Nf|nuE;YJ&S|2&uu+6lm*Cdq;^a6LIEs;)sWk%MiLV@N zE2hx8%IrgJr$uejcIIGPO2ST%SJ6+-c{f;yuE=v!>c55A9ENlkb zsLd;40chp2&}qXC1V}(4?N-p~1aO=?SU>+ZiMtSudgcy>=*k@WAg;pCQ0GP z=jJt8=)wLvAy7GyITai+Hoh~W?1!L7&f9$?Ign=f+h=@Wp>@1oPdEs9n@hj9cKbg== zhRScCI>3_$frW_-46r=Wtens1YKYX4V9f<@t*k$B zWMfv$rFjGvr>>=iqX=!xeQFG*(;UmsBA{&=2y~ZOai{_iO079!WiEQdT90b!^vE96 z;>%deBJE0QVE!|#^u_w=%*YwkQX2sbp$>tsE|w6gR9g8@`S!o=f0j=SnSc zx4KEP{OHQXH`Y*hSHWq}Y;IH0>uZBgzxMOBLstaF2YPB8=Y(t0Rsl&Cl>dY@7nu(c zD?{Zo6yG?}CjK1~-=!A%4Own-Ow)(2a3XbiAVk-1@qkHp6>;c6z!~T(Tor^(gdP4K zwKF@xLurIt=>K6ah?yTG(rotn(oFgtP6p1ekRD=Ww;wZJq7n+D3P0qu-|TjJeZ*O; zqR)fowwyv(g%>AH-o^>28@~v@z#_#34|^|PqyF6<@3#Eafb-jwwX;RzBPI#JE!XUF z9ySF?mLz6vqG zlY9e$NO=uNL5MJ`F9I=2NYTG!)bu6fU47A@@Ma-w8PGKT%W)6ob>maZEg3sg3PIFo z{49*%p#1_yz6Ucr2qP_RKtNy)t&s+hr~^hE@Yvi280-k_7Ot>JARr2H^O|wyX?LEwvSpY54I+D&TXp!n;rqw z%%yX~1+{u>KoGURyD@e*M-&0@^)Q`hmIAUrVb)+FuF?);{P`ys834bqpQja$%NA2N zT}T(77-?-h_9DL2QOt9R0m{`(=joI(n17~kJ>1{9t-l07kG~ecc$@;T zC7+=_(lb>pU7phhD_CD6U4iGF$4L9$N2SNomH%~}XVbIHJ+w}Q=x0_^`G)p=_{{r4q2+TwFFCgO;L#=Z0Uj`2Swp- zT!cO9EU&0UesC@=rRRp1V8NHubJU0WF8Fq^_flFGUrMWrN|d)g0+mqxi8C9oi#O5| zs6+u&q6{j5+aFMgMp|HHPzmxtn_mMO`~vx@A`0hOW%!HqWuXJ#1FAq;rn6bW~VN6eZ`sPAczGu|r)wboJyO#Jt~R^M? zQ)+T(Z@l(}tby-B(<{yM|%*XS=1~+Bb&p)zM*3j{@ZwN z*(LOQfKFkwmkE@UKS;2wTuI+qRCMa|kA?-h)s1n4^?h0J{p>~r>5}hn^@Av@QMa$0 zh!_7B2|6Dt^2cc-WqEv`+YI422X8%2klra0a}eZn2^lSx;JCJX(Oq&csG7z~X1IS& zs){(f>MrP_f0Y|lM)Y2}xVrht&fjB|IOY!3lE2Do)~qkuQF4*sLETQtVs@8&aTS+W z+$F_wrUDzLi(7S{%?k%AM7JAlwgoRz*EDC(#da{F+sb2Q?)nFJzJBYQw|PPuACmY@ z5&?;OB>s>@lLQ^qIU}L2$>@SX-Ib}eAK=u=NNKyI`*!vr-qDdu-iA;-n#>N*!G9G@ Ux(Yt+m=>;xh4Sy|c61mvMDX z>|OVez&hjwqezGc;AMt4=O6F~5J(^)c&o>NgjVpvs}p3tQ$4%hb!;I)kNV6xr|wnf zeCJeqwOU!j^OtY@%>UJ*ru~(K{f`CVYk1SYfiR8fq1MU&^^O7A2+hdsn0l_WIu>+h zXhnriLFp{;cE?tHAuL8tr=)m0EJu}2Me)V38r3>A#XDg=n(9m`z68F}X(+xNHltRj z6-{@hAy+yxtlBxmYMt4x#_GS;J9FUY*;HqNH9CjEA7M?_>KdIz@JHEn=OH%Jd6*sQ zEV0?nvGW?6WAisOHXqcs&CVlifgQe~bslB_0u^RXCuuJZ?$H~v zCOqB^#M4O}tR+HLMG&V6e>xsS{mt&FpPc;or<-SA*!dl1*8Sx#==sfyZFA>2$UE1F zd_*Om-ul;{U%ULZ7x@&a-T>is;@$)49zVVKkB^^u>ct(0R0fET_rPOLr1T7!Klhzst2)z}al`4D%w!fe%3=j(m(ldRm>zXh!47Y7$nrku*T^K9p+LkP&gMPrJ#zxSH^dwjr%F2v?<-H?)$xP zIZ9X%rprCw@~frZ`IZGPwWos>r^m#h%ewhjW<03l4yD@{|6iv{!i$ zYz4CFCL!BhtgO2tfkovj3{uD&adI`@lZ#=}?e^lXw0iMsf=&`1c|z8L$zU!G*4Ftv z2I6!8IXOz>mc3OGa94Qg#uJcJ8${C^x}$G3^1ZlQ;lQcVmQiK38yfZ|gAuF)F`m^7 z&85A;36F5D>)KeK+uNT}0UA@CJnfR9c1c${gS45xUJ~U|MYX4z28<9pJF1Q~X5F-g zhNxwRsI!8YV)jki@5XK$R?Pch4t}MZ=Fmi46Lrgbb(^AvI@8;*xebf|R?LhJK}Ij@ z(_^whaHCmB4ZO{Cj zrh-(o4gM4a{&|pd?ZdEte~_OdCxezXfQvd*)g3=Zb(Ni!27TsD#B(?>>Yid;UXk4r?M*)Yfx9 zfbBDFpz|l7F%Rf1*sj4tQCPQCn<6~J5RND@6aHaBUtX`k?yI5(k1>a2({UL%vdD1!clH|3P7 zy>CvnNk94AVf;$YO|%el_>P(*e}!gIQHn{`S$E=FJm~Wvr7VEbhISzb-mlQyze?m~ zYFgTx&^<=UpGWZpyy;00GQ+Z7H_9+VN12}i^RsmuhSxGatJ|thBAA-7Pgb`3%z!E3 zRfxR>ED@T}g|V$~>%)RDv%j+&TBE`k(9ZOs!whIk!!qij{u%A< z5#-8SCTdoPwX8ZKTP%+CthUP;nkvyoR#qMB%v{ihb>WOkSv{+bjp0;gj>^<~HZ`iK zf0HmCvI9+RtUap@9Z_GO%IXXW$W0pG9yYR4))+M~N;7LvY#p@_f2URLQdSl-S#28; zXjs1TTxR9{i9_qNVlFEI^X(ZeD<9D)jk#$In^}>SW;IscFh11zNmd!nV+{+?T62__ zIOq*)J)&jJabehsPr{E5)2?I&=0DA<53Zk0kL_VAD~yh0EsAPaX2jy?XwEASs;&He z_gHQj%9(O-w-L&FBITb$DbgB~^`adHo=@ir2OianyfEFNlT~eQKXVDe8-eBvOK1Cu zzXopk`f%y_7nUw0X}Wa%)QK;xqU;hyAlLK#O;SC*I=ubz|JDv&5u)zvsr?YA98VVV zcu`g!6VO~_DM0)Rl-E%B;cX(X5+Mth`Z-ziIi~5tTR3GvC16D2cDcu-bJ_DZu6mpy z*MqYnyAeTzP?~UbY)0NSX#`g=!nM8^GuKO{>GucHOa~Ds0Owz(5fH?ZLCkoCG>;P@ z43SOOPon++fe`B$aL?bAQ;2);w` zUo;LeO#yF3KjA#^1@1Cr<@^M8cn8PFzej{jPXP^on>d#U!3_UCk;_CV;p7T`d~oeYl%Mke zytI|Ar@Tv5?&I5>Oq2Jhs^h+T23cX6s@zmXa;ab9>r`QbNJu0i5)(;?^od*{!il6r z_S~8a5|vL6h}&~){tk(c5~0ktJ^Mh<7klRvSFxVtV+{PzA?NSQ9vF|sAsm1Wn>GidNt*S#|E!n{P?i20M4B0ocfQW1qS zoE9K!lUueQgdy(ai&Uz}Vd9Pwp(NuN-n0q=@UfhNbIe(Ej;oRb8L){O6va+c^_M^d z-5IsaK!nBz2t;Vo;QPQn4e|(~jgW(6+SnLbV+|2qTQ4wcK|4ZU4+<-HEqdn;9%aI9 zTNxJ8HDGm`tO^Pq#a+h46-C(>zB;UV{Q&u@MNo{egR1<;nAyIis#U%Z-S27Il)-BN z-j>cu*M3ado-C*jIVu9>E~D(47My3VjtXKY;+WG0C|8k)*4dTc)RbI=#O`N*rohOd2dig%k@v4>X+Yw~}v@OzkE;p+e2scs+m&%YS2p#w} W3=63%g(^_oq9zwW=Yh`ezbw<^f`>TVqt+Ru**4kjbwT`-=weh+USK{h>MqEwm zhjwc-Hm@7a+5>#8vE4MKli!R7xLaw8)E$oUE{nnLIBn+WkW zv^kBFKYL!tehHbxXqmCmvc7EWnz0#M@A)k|wqxf#17dJv4|g{%#Xjy{T#f_WOM0;{s;=y6(@Y|on7Gh^z^)HHJsH7=(vo?)(6 zLN7n{nP;vrrsde01}Ilzdy~(p#LoQgZW=S2i>Q{#+m;@_eB4hC{Cks7{gZ*3OPE5# zq^}mP-OpsXyU^EWRo26N2))${lVm9Gr^DOnqAXVN`O;|*M|Y*SE0W|Ok=1b6kB`@U zFUq1k7vc%_TszSe>hB3hxsXd)bSnwldr5nLH0-819ySz%C|9CsnqnP;#kr&8Jf9Ur zyReH8wzt2Da#lrRn7-M70IHU69xN#l9QQh>Z-7wuD|SuTPhbp8-3CI!qfgBlhn*6FXln_$@1OdDDpb zWh-&OTwJ-&uQ*ef`Ux@L=gchFGc5YPJ#jFS^U)*u20gNIAGP6Xeik)rhgg$+=PRbT zlCMuUP=jao)@HOxip$#%XaNss`3K7iExT!FYnX1` zn|$Kli9et%2hUUmgl2dg_G>ZZ|+esZFAN4hzyfs|f09E)}$9cr^IN25`a#z0#c z+=%k_UYK>Eloh~aNDu&$HXZQ{-YrYn-gtMnpNMB!3P#;D93+FGxGVU5a0E$~H!blA z6aW8)(C@!;xgPP}MH3s^JgD8zU0HAW%9&449HDI+;PW1ueBOJ5EM43!^Ji@>v z+-*yn`hI3%Fej~|)E{;_SR3hdQ`p0gb94=ftnk@QRE0jHt!~llYtvby3V=!mOtUq*|HROVa zp1o@N4?W8=*B<(oV>VQN^`UP!%oVd?HOzxGW%Q3dyv->seq0#BwX*&1BNLkw+74{@ zmLWceoH>QpnL4mq>bW-uRWAmps|ay7SmwO*a=(B&UZ9u zd&y|n-pi!b7qWUg>2~%Y)ubKWg>h_i^qU-A&hliW^}y{!gF%GWx7hl5B+XUv9P>)| zm#Bx~@oQkpL($%o4(qj+E3LKiAzXHx-0HRy@o^5P4-r3uL!VFwJ*X9E`?4{Nbfp6@7fpSl2*2GLFZ&GyfN;=VQFg6^t5Wf%r{X#q$iSW3lO)J?A~|YwkOtv@LpE%9>h>u zlDoJJcR`h@Kv~SUt%)}&<)vP=@Ztc@!kUm@)TZ@9D{%=<$LlrV0IAN)s4r8wZ*F`E z%)-InAzWSBc-F|i^$b*wf4_iBL49>w`~>D8o@VkXBs)#}7*C}y<}jvU_38r(Qv4JX z0x%ev6W!@%%@uJSMOjLRVMj!OdlVzQ(ecqS{TTL7C3SeQd=Y4Hj( zlz|?u-qqtLo#rp%9n^7R8Q*C@KUT~&(|K66)}Sxcn5ucOqV?pYhPKIE7C%0cyhkCK z509D~xuTJW=Ak)p!JNS?UNydAzVn}2&=8?Bbrm+46F;_~B;`pk0ewV!by6eO=7@KG zh;kjR-ehUgD3*F;6eXotR?#?FvJBP3fD$ehjhTgU906)NFFTXv$;#x+WVKi+&VW6w zt{aoHg*^@U6~)#3U_+G*igRaiRqI%cXLgd-qQAeER+N`mm`0i zd}t{HQ^^j@teOtf@J4^w-Ur!G7K=(d?hYQr9QTt*P~ijfyMw5oJ)j1v!3X<0aQ+}^ zPj9_OAA#JZdy}otys&j`m}Oh{u3Y@&F4|bYj$!w7#qNH5x_i{F+9&pJ($I*&NJwlhL>%SFqt6 zjK=V*+&2O(GVHV9O6G{>--z1#w<8f_Lkc$qzLzHhje7PVx+AURCf;^u6s2((WdN(u zSlaMP;#IzlphpIXfI%#lSfiLC9j;tHdNHOl1`6KAUW=%kWNpz!Y-5;;=Q-pvNSfy! zaeg9X(=Q^CKBUtLZ$v@}7G33jrN^0ITNvm&g?yBUd}VcBuS6?L?V%9BU^;wkDy7+# zUN^$>RCF+C#g4RCrzS&^>areiLs2ziBeK(4$@bwlf_5y_HDd^ z^T}FB3?S@^^2q_1C2-gD9+GK#;E=08USItYgZZ~&K7sx|c&YW!I&pbmsgC*6;HA7@ zdFWVyd9ZrIw;$1rQ~Z-0p8iRp_7(U#-~#ra;F9L1ft?)Kr`pTev9)OcTZk6ip@EO? zTjDC(oE*E)*qND&`>u1Hh@_|hTs2v1nX~|TwM!*)Mc<%s;pV-Wy=uVeS9(?M3DI}y zkY2+kmsJG8p7?nMC<=!^GYcPIRWAHVfG4cb{S!}nt7tuo-bBi>Y6)iMsaggmO4But z;LH%{Gu}rC5Vh;?Tf3Os1bbp&UPO&}x7Zns0+s-6)nk;l zHa9g}HLuEATTmgfrxXZus4bPIvx=8^IuNIMY5g?U5KpOZi@OLTX)^_PiH8H}wfBak zwF*tR^N}u0E4r&$Nw6!119Dr0Roq>0i(DMzinZGoTO3gfvjiK@yxR_eG0mDb-rq%& zGAFez;`iCp-%X-?3?eJ5iyVVA;_Ga5yVx$@k##leaM3Pny1bw~RbA8;D!##&JYrw( zqV5n^wuOZDwFYb0uxLLU)&%RE@uR`M7uwn+jPKT?}AE@dcxhYVDM z8!4?s_WM*JZ5U85s!ib_UWQs5;hTv|W(z0pQR#~kRLfUf8MFuCnz=H@#x5df$9>QX zs$P^g45)i)TFwI%xT(%yN(G)w;sFi}lYwbfz`GGZJ#T4?zlZTbi#(^VOEW9qQWF(l){yj7Tpk?CtPY%iU&d^-wxUvQk!&)^j z06CumqJGr6Xf0UoNo}5!mruPUZeYME&Pxw>6uEl}=-Gj&ae28>IEuS|8`wq+HGv~g z3>HD&?S0j5~$UIXW3nS7J?AcsOhIqbqljIYAD-lU#8K-PL} z?Hfq#edlE(c8a=cF~NH4z(*C51?n9s8Pp7cS@O+E9jvWZ)Qc)&g>F$t)UZtTAd4$J zv{1cj1+Y$Svl_VxhKnF@>;Y$8Rr2N~PnFEV?O5{~Q`K07cT|H_D1fz4ywbMPw9Yra z&foW5?0i!Fe)M98GBJ!Fyr?&8caU#3G%_iG^NCs16!AA`(#rM6$Vul$0uA^zX5HIR zzC>%^k)~$1Iy5ZP$1FC;Kg-duVFuYiF4(O%T@2i-d%4qcK~6}j;UA$?K=S<|IC zASoIm{sPZ#Rrqa-UR70Cpq4WLKW)_zKb)BG~pDR=VR zy*D&3KyX?8L66UwPub_d7_eirR=^YHLFZ%Ycgpe8=@Z`+o_!G+3TNM*8Z%hN&lnhQs5Mcj2@7?jH`T!`A9@4E>vI32MO!c!g1{Orw^;c1L)t7TA5lh~Hyo@EaKLU0m5J5}vM*EHjw(JfsV?x8?!Y$F>=#4qGVRBN6`-3J{8l zYncG=4t+HQjrf)hJxn2HMz0X10(o_(>WBqmc0n?s;~zW&F_SPOdi4mcEX@U7dQ8yy zDM7nEs1LpYOaS7hz!htNQbEjx&(H+J5(<)S)OaUrZa$G$0n>nIb%z2fcy%=qRT#R1 z!(Ve>+{H{~N!d|`BQ2cP=1JVF>)qma(D=Vg9h`PJ+814Rr0k`u7@;PTmG&?NiY6(p zW?4fiS5|s=u}ex~-g7aA3UK(5$OVNd{+Y>X>&4Lp^)PIw=?~C4mv;b3Yk9u_sDs3< zgSxinMdwq|OBW-LqL=jdN={&GGn%qGLR;V)9%Efx6hyeT>JDp{sIes66`d1m~^YtiJOpJVbm69y&)cgx~kCgg5{?obe%1rb3n%fBOsO^9B=b^`AZo|wo6 z`<-V(&c{my#eK8tIp%vz6pd9um+LIrOz1wb z$LE2;#WWSvIPr(9Yv{Uh-tA`>8LrhKU@}EnQJ|PzynzEVK?RjH-4EwPJa`iXClN5I zFyX#q(Ip=a`}^Ho9lVJ;TM4(a^x9k3-?;YXw)*N5(^gi+X-ofkycK5;FEE&$aXE;d Q3@zxO0?(;8U{Q|!Khh82h5!Hn literal 0 HcmV?d00001 diff --git a/models/__pycache__/lwf.cpython-39.pyc b/models/__pycache__/lwf.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8f32fb432cbe06ad0c6ebc644f337bb61ca07dd GIT binary patch literal 5224 zcmbVQO^h5z74E<3>G|E6*+1`(IQ4KdXD&z_I3ffRf?J6bAi;`UxNtCu%=fBy*1JwDr1Yp?y?XV! z>Q&XN@4XtQRx2xb{`|^M+z*x&Wm}r#e%TF9nUT*=b6? z9JIpe&U84_nL)kMnf0ojxedju{ao$LgFoigI}6@aXA%4o`aSt<9*4wJkq6pLoV#lMB zU-m+jg?PYw#FV?k$mP8_assNIb+~gLsy;)l{$k`ZKlCH+1nWMBr1FfD_~(6xMNrU0 zjXP|^=cnVy-;BAaaz9F9b~+k{gDZQ#g#jBs`zbxYdaAAOok4x?Z6X=9e0<~Ie|q`S z(@*W)M6GSG8VXhi+507tzYuvBWaH$s%m4c8qhESz&mxgVLD>z;uHGQ5IkqAYkt&q8T>C9jOK=vBNbO6z^aP&y?nVom7hx6ipW zur-QLRA=UJh&pvF7y2zUsUZl@FsH*b(_cVlH71VL47NVx1&kDt+FL_!A=~I z73#6f=y9hFRID^|2ipHleSK z)%k-7Wd%rdxlLwP^t8;9nk3UmnyUR0FK3n8Quqrz=DI%8cs12{%`5P_XI`hJnVM*O zMcJ$7S@@Bz4qY9(r32jtZ$hWF3)k2(*l+lBHiJ5P>9@w@D&VqN)TZz@#v0t{u>L)NshNd6~!Rl*nmC4=tbQaZW6Q5;i5vHLI=3PfkgvhI?RbSd{(4R9wu^JjEgpmV_D#6t^(P0qyfiP{^hcGeiVc-K($mc|rMz=#J zaxf|&ix%m?B0v@f+PCjc_iXlV<6qu)8yzN(f+%WTwN&GlrPozMt=}>XP*qd4Pt1a* zT96xRo_39FuOr8Rm1NZ&~DsTWo7^R=PDIC^QhAMjq68(tW zfP-l);Dx>^zbTFp(YVD+o_^hcV;1`5eg)21g?o~#lH=7=HPuo*HO7c~(c_1|JA`E1lC^myWdYhu_ol5l~-PVN3yJ7z!<5qPp$(Ha4-~zUy2;Ye|M7 zxFr+iuoqx25K0nJ*l@zoY3q!joY6sN&w;dOa<9{PEQ;vkt)A<%`>Az}2(6sZFFb!% zln3yM&b!(vZ5jd`>J#H{wZPpfcXi9Qwq54LXL@FBip}D5a6gM2Pt`s)O;a`DbS>@EvYoPw2hv`bR%-IhfGOdXaJmB=;0jl{ zwyW-{qXO5{!mfH&`A~iH;v3q?91+|ZSz)Xoe^E!4r$Jg8m7xdyv&x$p>XkQiXjVtH zw3?A87RPE@n{b*UTa?U5t7FyE7nD(*TUjZsr?s&*no9MoOueU5Sw)t5R+Xg%No}k= zri?6J2UylU1oG=NzBy{7rL>VXFiJCRj1h~o7NXg-)GnoEK9kmV5gtb6%V$y}?~l*+ z=lFbD0xX*|N?Kl0C>^-2jhboEE6pihc}x3HWv9GKb_{b^fYg|$2ydY`%ymggo8!W$ z6`jH7wnTL_^1QuaM|;cXacG&-Wd=A|yT3Q~Mr9p;{Cq-ixw4KR($aN};0tB~ZYLc|?fc*DNrFrnI_kWU($m?QwV zq8l^jyPRxB$|TgDFnbY3MN^(nC#*xAAZJg2g}@9i5v4qC!7e$tuw>w(>d~BvUBIX0 z8wo8XWr8i>1mVG)x(K|ef}>wmYpKhqTk4`lAZ*3ru&Ub5l*A9Y ziJu0Mc??BYFbN|6xjTsbQ6AU-10n#M#m@sHX${BFI;M=KcquCbAg~VaQ66Yf+0_;R zDmbDIUYWFc6kM(SHYKZ0Ac@y@^-+`8(e-Zpp;p$|ho$B|EVT%1BrGAb z8!I2YgbZ%FKYIkArU6jX*&G0B9w0yfbqoNtZ~#z?6M%XRHXHcu>VStcUjgtY>!$#R z^m`qz1TZ-=U+ORO~$jK#1gXDcZqwG2mu!R9+B@8p|q1-BI1CwACQQ_ zT=2q_rzT-OwSERl$jp>=sda<&E&Cg1kufHT%m{>fQ4|6v^c=QHDm@~7B3ncP5K)YB zNMRv~A|f%70g=l@4giBOs!Fs-h&zA`#;JN66Xx%V32aE3ZxPuhLN{S{g~$$(t3*;F zBOS8e<9&AoZ{aH5tAaHXNY0`h-0 zD*0a%tCw)+W0YD>w0Il`TRqMwQsjq8CNVEl^%Wwo5?LqmBO(`xP(IDRMMPc}pCgWL zjQN_pM;z_9ocf_9l76v*24S7a{p?8sn}nkEzHh_V5C0m-nki0#;_|l7QB6u3?LG4cE^qhSul=alU-XOG*XR3fzgBZJ{Ql)@e-OO&Q=0Z4R5|%GQTb&&=?{=_jk8GW7CGzc zsOyn2FuDdS+RUzrHX||zrEW>JnJ8ObOO;Ded0=-ds%%BhpxUht+^(zY<)}8Ock8Nb zM@xf7w=rmTn}g-d{mj3S_&gM{OyxApFsS-v6mEpR zh(pmaWa}$Q+Ka>Y<3NOiFwXpFE6il&`UkhR-}^ch>!-Ugr4b~fIP-?S@CRWQid5Dz zU+jk26wvZPe(e+=B|dc#Hnnl za+zyb+1R=+7$$dYSxQGkT)FLey||Zoo-}besi>kCjd7UWPsF|~dqa_ANs5B-d2f`; z%B&EkAfB0q(T-TgoM#L@UJ&_d>Uqz!|9<1*AkHpEqcjuYMSlNqFBx4N2EFLwAmL$@ zUL5+D)6L<5Xksossf$EoF0<5QvBMhO!RCB>0d0H!qq2?XD|pg7NOG-D2gWt@^@%an zCgxOKKxUz(M3*8iQK?_XC_A>POr>0#RJcBIP_J_146Re;<}5Fr(WVAnuev&GOIq0b zC<+hlhhtCuOGll?i@{;iR`(c0xT4;iO*^ivdszZ@Ev_a^QQwb~`|-RkRfZ~!U@r{zheQn;?neLR_J# z^}&a4Yx;~JIhz=`Sa9kZ5yhm-pAcb8uD3MBzXdtUnGJFniX7nFzB$#YSGAfGC#%xT zBdoJ%EzMiGb%e2ME!KCl+N6%Lvo#=(Ap2x#s&V_NIo7jAPCZRtIiqpfN0U2L;?<|b z6wCQ+1@58`dw|?Ra-SMw12gF)#muF}%*Iok$y>=%H13IvA+wwQ}KdP!Fu8~^vfC)v*JXx3$Z~Hm1tB(Lre_4oj|V4W#S~` zm!o8N7uO=qUJRE198Dx$BCE6_y%PonCTJ)vSb}&LC-kUib^wdUlLCV|plkA?!W32M zO!JBng)szoFp9F?Fbcih;V2cPiKR;x>n0YCqA-${r`AxCin~)X_eC03E7qRx29^hl z&uTAAWpklDpf%LI&tQr48WIiKv96oUd12|x%w{I5zc9@5w0GHd7!cGyDIj;Pd!3ihhGk^Zw<1M-Nvn;HFD zpCp-)^s1if>@6-Nwg#Cr)tE-VnUh|zczxEtg#HHlN#-EYka#mMO-IKO=>Tg zTQRGiB{0c6yw?jt@mZQq)46_)ONkJEI;9Hl(3`$Y3tJ*gM=*UA1?`|cAlHj!U*cWo#q5j3!7w%sa+kf%Heh-VtA*WHoPWyd6I(8<{=cJ=0RT<4Z~qYNL<*hQ)I8GP z)oy<>CL~bmTUmKxgZNO%YZ?@}PEd`Po|+#TV|`4_Q&DW=a4;>wHeP{&AeO0uS=fjA z*cz9!a^KA@?!Y|gV?qvOTnZ*GPnL44- zLk6Weg_#0#ul@#bg^9l-fcLoZc`g0XC9oy^{pOrK-7WFcIH!1%5(0hO9phNbN?Xh{ zoKm*e#m~^NpQD7#1`wI)Kj@{Mw)h&VvK%MgEzf;HkBNVf5$ z>qs;(*D`A}^M$Lo!ED5CF1x@EmkTyK$>2eWCd_^`nd}HD$Mgo^ccx_uUmmd|HZB2d z1B$$>eWKsqEcgVpH76ydTG-g;I(V!ycE-S!=y%68g6C=myZt&^>gczgXiwNvbF`Mcin&YK$W854s=Z(AxfVRA*jaPDG;?ha-l}SzI#-y%t8!b@9Z)=c* zO5T{2E4h<5XJsd^=F78kHM9GTdxzbe^=6uWT}dv~HF*ayrxr4hlE3WAgANJ$FF z)w{hsW~3P(1CVogFyGPdnsEJCM}JP^pA$pF{wSSLf^+u^m!9cT-@GK*euqh`7Y|2S zdMr-Txf=D-EJy~!Yx|h`UwE{`&z(@x3wyD76`S;<^co8DDTPuZE%y8ce&T>i8%SO` z2$?E%5RyD~kj5B?;2MYy^*ujEv7i$9hZqRgCPsF=U-7%B4yft(kl^|Z-WxpU)bOBa zuwkHk1MKrT$-BDd!~JbILBP;&ZoEeZ9i@v8#~WX|ws9j#(~XB$Hh*>peN=(twsX2+ zXP=*LQ08*cw5?#J9|Q-~ba`idIR1Zp*_#{Ngf&mRzVm58d>{f!ejSOdgn$a60JKpc zq%b$PsGHR2j8lL~f_-V;>xK8_vPbMN0u&mOzaQaRWChlLW?<|)7<%6q9J(LK0?0fI z2hte$52PO6#p@mn{g``x3bQdBNdp){5He8ahhRDaH(b=IMZtN3N3ywiC(bg0fjNNB zOZG2J1JOfhDT&1|(v+)6I_ob%6>*!oev1~eL7rXjjxU6uYDs}5Y0)9zo;yWOR9=)p zVCl}<6>ViXNQ8hjBzOkrUa}r7u-nU0Y12S8y(rOSwP=B&2+85LN<$HHT$(~TL>;wl zX)DKos>I-s(%+$ND+X7lt&6IUC#92UFlNgNJ2~(G)`6j{7vvFG`UT*q*I5gO&Q^b9 z^5}1woyQy-UWEQaKk-<49kYKPJ_O}gUYNSW4qHD-#83GOHSA##0r*v54k%E_DsVrb z54>e~3d&zLxK2<=sZ~Ph@E!JbaUFeThDZR1r!dcVpsFxJ!8X7u@-Qj{;ZP4*RP?~Go@%9_ zQW*vXGt_-h^upjZ?2PWEs1^6(Oc4heyL2CUId|y}I`u8_r?_77tp~%x9%NK^B7bm) z`ySXz=`#iWwpA#Oy!WyQhY`TnvDy{)Xqr2lrr5W0zHm0!by*7pu`7ZA0udA;*$N%$OA7KLVLE1v1DOK7A ztsQn1sOGQgZ&gh|O4g$Jv=gWD63nfz1up-_1x z6*`52gvS9@32*lj$V_S${wb-I6NJ`hgf72A=>Er4GUxC$ zfXJO`5tud5Vx4TcdmgIOg(_z%2LtX9xvfH79aDN6@vkw?Djp$`<%qt#q=-u6gG&~2 zNb(AG{xv248{MJ%lHtDCp)o{#SyHN@&^FZ|i31>(7*D5C=!GBBAnO2zHsq9NVgx3j z>Bk!5Z>UOg_L6KPAW2N<1#$eB=$%pA0j&#)UxQqO=K5j1km~v1Q!uuKnTrU;AETlu zh49ssVhL;vk%ibopy6K_@COlGLjVjxwF#L;GF>HjWG|02<1IB-$Jpnpe-|kc7 z!6HgNRpL%UPvVZq!%tcXp9jY#85%{k(iXlo9DCELQp`oj;kU7S`0M|Otcdj#_cJ1p z_*pd0?<)py{3jYjA~G`s5rWVW0AS>V zVaALW_$mQy3#kK5E>jqEPK(9hPMtny(Z?8k&T_nYl-R=|y}KhnDtgo>C()t(yidt> zO1?n}p({Z;UXb7yXa84}5Cn*D}- z0clOph|ZY;qy8PuyGDsl2^m`zPcGthwu<>}#<3uV5`Te2Hj1`U){D|j3YruV8iHT4 zs;^2aoopP31!vfeM}jR9N;P$=XR4>W zs(tFT$8J}%5#!ae5*C(LAhBAB9tnZ`z=}tB;6Dg2@OH!lXpvTyc;NDY*zB(PeW$u- z#*QVRT6OANzjNxV?|i@CcRHxoT@BYieB*oJUtQ9)A5r1xRYu_*+}U3vVH(rBnpZq^ zPgiBbGu6}bEIf^_)hl@=o%)zvyI1xcmAAT;p6j_PU+PwSHLs@fcDLSZcny^=cV~LD z-YoKtH^(a8JafGTR`nKH&0Au1Z<#f`V{FD-VYA+GHs`H=ps{(j@IYe=QEkuk*4PQQ z#FiiE-U)V+t+F-bPBQ(L);jeBtJl_Bx-6xA5%-XB+k+$&ahe2ODqRmma5D{9#8Ik# zHOQiyQNR;bTzoIh;w1W767r}QB_imqM*`hyB1*E9zm*Jn{d-Twm|^Ste?r&aTzj%e zrTdFN{NSI5;@Xo>QF>CMM5lyQzV*|`|Ni$MZoYHviG@;#`lSZGrps> zb*3}pfdieJ%zB_f=Ot$2Sz=}8;Ayi8bMY*HN3%4~!Mj(b{q^6tjunT-kr^mY3!-_l zSCI&9PaEsGzOU<=FmlWzOlCYZbIr$hMOtDSq?zkCT84D{JQ6&BQkZm-O~236yZ2hQ zEMU} zf2{rF+{IoZE_Mf*;L%04JKRnO7yDt{z1T}x)XgsTQwFIQ`uF$(=J*osOqsJ@(+%Bz z>X^fs;@vMyR}|6*m|m18CA^0_yMQFuI#9yCHa7O@mGSZ&i(XyW<1*974$75%r6Q#a zV|_UH{@tk8?*_b?#lz@w^Zuyia{Bnv3e)~QX}5V44I{jC+GPhSX!s(985M6DWwPP# zBF%1iXN;~jAAx^ z<%j0T6ph>vGt3dQtn!dP&Kzn?)VK#V^N_TV(|mKd=l4y_sErp@%@VCjeZ1;Aui+hs zb84eQ+08V;=X>M+NS2gsKQ^VEr2`&D(jt44m7w2`5(W>z$#SG6qD-_5ehvlRLb8D& z?q(puZS{d*DbyA)UuFbZ)11=)V3{s z_Ni?ex`Q4nwdVEV%EZXOAT}foOs^_34sPZD9>5#IQef#j2A^lJ9A8gXg)yEu0CEDI z(u%eQyQeng4h-cE;0%;S!!e9q^r}uuHKq-8&QMc(roNbCi24dy`^ZW@5)CmUW~rt# z*O{lb8uLC6+tKa!V{S8~w`I zPc~{+I>&QctTN}JF)Cv}tmQ^-=5}5dtNXxf$Wm*2r1M$YAL{QDM)P5N~PL8mW z@hR*ihxSo&7Imj_msT~^(ww%S*O`O7-g#+DV=jY!#xG-sSFxgsIGb0*E3777Wp$KY zHD6=wh}ZJU-bAbai)LnCpqX?3C(WFDfo57qG~-S*)0$}J#3O@$iuZh-kg#*IO6e~w zYTq`FBj>-4()ky*oENW&H^vvJ9nXtoS)CGmAigiz+9MWc(*?ZFqW$m}>W4xYA9-pqpJqrs&y%TP);AqswJ& zH;T8m;eVnqxVQCppV9TN*IFk3RS1(#FD1q_=<%=&=P322>eo?k}~$eHTUTb&x0{{>j|mJiw{ zP*{MpxAa_nfx?ANDeF>(bs01HxU)Y)qUo;g7zN_Z8x52ihO5_%6=2~!aM3l0^BO<1 zer`Dy5YaYmAg684qu#>LcxsuBZevsp?6?L=HHDporT8f{twGL#Jbm%!gtkW|0gDyl zs?;)6$#`m%jL(gycG>vvVf`835i06QYsb~ZP260O%S{-h0Nmsy1|qpeDdRhYTHr=zd_$s^R%J~l`7QDZcdH^vH}0nRhioS~@} z3TYbq`e-(vW!72yi&v`s;JQswW&*d|F#%K{UE#Y3q8p;ITfV|g*Gp_7wf4Hg$ zWLZf%_M4!O7Jhr6mC5WSE&J?Ete1Y@#GM~g@>lRA`NxGjuIBUk%DA?#L0Ua;bQ-LD z97JNe(tU#QnMF;^E^4D?F_$mo$M=Y2Q{6o3mh#1X8FYK;m^NC?&G7>1Ctn>es;4<# zQqOX}I9}e@-qrx_L>G>+@{Wmj=;upRpBVWP+MJxI__Xa0^5sd}GTJJ_>8#{)pb?-5 z&~JUTmb>}d_&Bt>ny>8xl*em`jZUap+`KAILURZXM%CMwa|DS>XaAI2#frqK&S}xi zT|ixBQOm1qL;kr8c~2rdolVCX0Jd7`i_p@7dC&1Bi=zz{26#4N!AH^7-cb z{mU1=)E>2(U%A>;066^S&0Ft*S7#YDz17B$Tm3-9Ak)uO-slEIzmIg=pzzuc!th|= z22u3Giq@;Lq-efE3x$8O5+Trtl;8h4z34ZPNHYn80zXQi65+NiC#k>1119wwaw$}2 z8-CQ=j2Ht1s8vXtMIt~~D@{l-*>(V~{~<}M7c{_c`{WemEJAQUJMu~g9m1kR1yDbk z&e;sYo!x*lz@Lcv2yR8xE6@=*c2^qFZM?zVevmLf$fOze2hz;I9q21?<-ZZxq`-ep z$5L_*=+WyB5EDTw10IC;ig>7|U=pvG4)8h8(BPMlw3>ypKk)OMTsiL`@yw{dBTEWR zJhz*LN()iuHIBX-%GLJtLqRR&Fy%Z71>!}D=A|A>-IsQpU`=S9;Eyb)13@*sOVjlz zNoY=II*(U7I*V61n8T~mp}}OhSEIs#u~t-x3>H)Xh9gTjlVKUc#;M^*N5x#wjaQ;$ zgiFQat>*$T{s|WIx45%ANHl#B&_aL#*RQYWs{}jh=K?YS8YrKmGD---^yQx!CF^s; zHhuyiaOa;IR;}o-;Ak13bHbR{Pbjc*Aguuzi>S3Ahkk&b&#m&XIt31pj#h;ZkdmMm z`Mz|0AF(&c3i37IzdZ=LMT5TftAs# z3<-xVkap1Gs4>w4-8ap>A78KP^4J&n+ zh4U5@{itjolx?iOjp5H^cMI-Sf@uPgFbKfzpd$sCtXMPgBup&Qbk@J`hhS_Y{{p#(`_^Q!Ra` z)-2z}L)yjpEGJh;9!(MH0Ts+)+~*^e>!-W?yOgu;#L+ICP9XR_DpwFP!P_H(0B6Th zrEIi$x)~+$AoHn7d8Jm3{}DQ?4bA_Ya(_(8BNCtoQs4B zRM(HL-3Ti*d~%Uh6BSc7Y)x!92T+b+V~E0W+?iK8Cew+QnUdQU{dA&+$ng5Hj;5zKI1bc=}f_T-j zu14O#X-2J6hkQ^Db&hDk5}hK}R@TE+{6;1RoHu)M!vBKuhO~?5R#sHxLVp*u8ijGW zgln1Kjd7+&p@cFg{%y(;U02p2O>hip0@4Z-DjeO2437&N#z`LPEwnpNIgM=T8?EyCI{!=L z`45q7+*og&;_FmAgG83pKP`$0sE<&@wT@yn7m|c4zo0hnY*9B5aW}g_LBEP6WwrR< z-~v$>PAC7eUUb1>Asx=E|0||-B2BwHu~1xte@u(`ElS#yM3kt*)K4k*yOhx197VW& zk#e-_)b>>Gc4VI`osqhvbEUvG!WB*Z(^Eh#^mF)we!lQKJ^L-~L$?V>arpbM{{fJd BQeFT6 literal 0 HcmV?d00001 diff --git a/models/__pycache__/replay.cpython-39.pyc b/models/__pycache__/replay.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a26000acdbba842a9bd995e8918c557413ca68e GIT binary patch literal 4841 zcmbtYOOG5^6|PrTS3jq_XFTI)GD#q$L+pthDM2PAQQ`+d&XB|dgd(A+saw<2GxccS zx*dDQ)gdsEvcV_?2;m!U6U>egS%`LvVYd|+5Gd5jQFVZa6LYf&_ zamg*IGz+rr+Da}(<=Am6O17hFTytwmE=Tpa;Wm`)M9p}{ol$ZnYQ=509e3Og^r}0{ zYVI7XyT^JOYy3uc=OHh!rhA;txF;Z=WG&Y28SW{_r&-56!)D!w*qr+?JLWE4(bzm& z_&{R|VPnU1A7T16t$TbIv(lEky0pY~7UOFq39yt7M4!~^+As-3KTZ6Iw2Qv*uckf= zIkei#eimL0eV!<_`DzmIFb)&pN6Vq;8nPjL-V4R~Gzr&JA?qScvXq}shVfvlcj0GG zfAaIKOE2vH5hLh*{A+rC|6!jn(mWKGZDDtzgj_(ta}>ME+Jl9qv6OQ(yFErvXCS(Z+hNPjp=qMvne= zj9l|frlAdE`Eu8irEE9|xpX|QpY(<2aR>FFCZV{I@=YwEm4(row6KoZ9bHyDFNpjs z^SnFS-6xjfL@Y(aOz?1t-Pm4Fhf9N?A1%cx3!`j_t1+G%Z1DxO`X~*HR!P(A`u0pQ zfpdEu2Rh|UACgOWU%{Kvg6G->UQZXs*ktI<*A7x)Rj4_xPc&xTwnm0%>d19&Nmk>(GvqcBJ;FNVPUjP%X84wdGyf@a?wpWNgP`nqdFV-Wti0;C?^pn$phF zArC@nk!i__KNy4wgQKq?ue$1sU|sE%ENAP(wY4aeHTbwkCLHpNQ}f`LFasy>C!p{y z;BxmQ>^T?~o5&ZbPy=kC_Ea^;AEvU$DWftRFkgh;TEHKnM6zc780FNc3i3cj0^~K2 zw$J^fhaq^B6Nhoiw>WJ%|0=;*s#I4K^CHD$vPMJOQ_Lt#V5#vi68%9GdcDCgsrf%uY&#V$qH}JDR zwQWOpkYnks¨4rKaA|TSgOQwvSIe=W~q*H2uAYh+*G>egOsML^P*u6pjFwfRP$} zMOYi;G}_pP3mBp-9A?51Or%#fsxa`HsKYhP5gBg-#%kndZcQv*OD-I)3r8^E2&E%+ zTcW*Rw{y6zGHH8R*f1>AK2rANvQ;JN(SQ-(R^&4Tow~qA<1Ahy-6&3ElHe6QY zK6XtYWM#6{^W)g>nw&xhC(7WL0NvTbC=HRyGTL~zAB22~a*17}dF{$YSslPo;t(8_ z$+9AMbc=lh%OCDr)+6p^!FouB!Da0=GP`Rc99(@<&ZyFdhSQj~Qw$phuJpnTOA&{E zsNwM|G>V!^m=>M)rv8JNz{89}G{y98so>UUY3$DtT&Aj({Q=!$c>L?gUdEd}1t3$a z!pIzDUIxs|)@>M8$M}5Oc4IH#sNn`=MSIU2gbH3--)tLXAvA$dv!n0mqmnT5(vE&n z`$&IhV1 zrvB|wE3f3OaSOe)^A?51ac829XI1G+UKMkBV+SE-RK5OuZWZl`V;l2gA+Lbc?Kv&4 zp3>;xaoZTR^D?W<|D4qC%pPic93QW|xVr{O;*X;pFq z;)Od%`JX)OO^Z(Hsd%83g0wh^lPBIA!E7v~U@ z_2XRznbYPW!GS0BV?WAvX=hcPyU(s5I3togwRmZe2J4WPZjBbd@xtQOG|LumT{!p5 z8uC^s#CU!XY*Etrwb9*g{(tq*6jA1(mbwpdI+V#$5fG}%T|yd*tb_i3DFZ?S6 zuM&KVYUr0`Bj6aO2Up?gU`tlO4XM}TK9kO>A8g+6Im6)$o{H0p2xG*`l%ruY_HRlf zypA4j4*Z09ekRReFqCFCjQQ87hi9knVHB{~4&1fmlB#a}$99hBx^dTX>c)0wk3tS;?+A|^LvQ;$^4^1a z%-7`Sor zKFrclOfi=%f=wP+2JADZxMc1SmneD4CHyUH3I9F;*^^=q{sWTUAt3JHKP31O0UZjZs*4T4RAh#)3N2vULp!F7Utr{tVe<&hal zj}fT2E=YQWfDWGBV-NH@KBPkL65JrTNw7t*O>m1KCm0bZCv(Wja6(+;?@`*13En68 z34uE3-s523MaHl2W>*3Cb1(}|rCiM1cQM53d!5Rz9CkXZjBDYUJr}$GaiK~#)da^H z&y!WpD~_{}>z;Rg=to74x^40=V`Thg0y;A&PT(|d*_d8ogD{G4;XRnzDJn;IJ5G*F zi=EX0APUQ|oG&>ioU5IpSyF!^C^l6zU+l6;4X9(7qX_w2l5iE0$UC$?hROmZ<01egVK;U5qnha7V7!RO>w2Ro2~1VpYm$%>iZtDf0e ze(2(47WKM5UR77Us`|a(>-CnFsv3U(_J@D#|Dva9zof$HpM}EfNZDr)ffle>bBZT( zm@4azp`NB=;;F}GVmTHobnL{^HKrvcr>ts>xRO+zs>+*jEvY+omAB%hq~SD@WoJ2Q zI!#q)$16$8X{mfEUQOCgTjk5~TGDYkDqo4$lMQD><*RWw={jANXYppTu0uLLcWTj5T)ab!B@P_r7eQPW0E9X00zc17#0 zJ;3_4y&jW#nnG6J^n~}$&z`Ua!F~>sQ3T&0PkuL>hl`jRApbFh`Pz&mKR)Qs%qajQ0fBV}sNngjf^B?6+ zq&JYV=OJ>IYcAxJ1+1?f>nQ1}q|Zt?I~dN`b83mpJrTMw4B*uKMWJU(%k_9alSV%j zvhJQzMCp{~eO;Pa81GAy)bDG29W(gsxr<3EF2>_b@bF@A^YHp`d~xJQ@x^2qgmHF} zC&?~tjO#{eBwV*kbGIQhX0rx6Y|a+CJ6G@-HO_vNCljj`H<1xqt_`s2fxZc|UeWHe zJsb-oN`!{hN_JuO8FM$ za6%S#zOO60%3754XGY8(GqQTydGyRSAvC?gY~9qGtje~L$`5s2|F!48xy8&MDT(LmZZj1JIv1QOjv~^R$7!0-dZ4FoZK3tzK`LSL;o%#8X z@+Zrtt;@A3-F*F6XOw62Jl%CpM(*jd>GCib`=J}&3US0eE>~TDnBta>eQ{FRGm7o2 z)46dIFZKE%mz9&-@Yq($&IjWt4lwj}U}uKwRczI3cxOga)L5Mzo}Dl1>B^6+jjkJc zNOgC9j+H*KrG~mYhG4lN^judPtL_bLEN$kI{ z$GCTWZDJwWqKx-ekg7-eo7#8&V^Yf5peE|mrM#4v0&5-mL7AXBsNFTM8I$s)GO3D2 zUKPthJy;4Fcg;ynG^Z=a+K)d`VNb|B5+oBb~|X z$LyF*>VE=eVB+t{U|QOGMZ3qcKfVNxK!1OVlw0TvUG%+!erIzFy;`}2dG1bD^VPg{ zVBTZ=C%FwhWiQfA8?!Q54&{67#-XxmE4Tkn`+H@}M{Rgd|4!pKvm#FPGkNWp!D{VE zBX19OM162>aGr8-o2z$q@$9;`r44o_ZM1CX?V@(FhMnvnt&1{FiH)7Zo^EJ_wuF>e z;U|*~^!QS~KI_rV+j)0g%Qv>P$tJAv9MbbjV=dp@07qFxUwWgm?N;8IZb9SAc{$%A zJCG&v^;uneQHK#;z{p$bEry!q3)E*;F5lB{TvOw8VEYboz0zC8hQ9iPqjs>*aiVrnL>Nm+36`H9EiOg_d9qr}c@` z`y1Q+OnwF{J~us|pE+i*B+8iS1>75)#qa6+`q$~a{7f;C&KJY(;l(%R_!Oo6ojGo0 zySuwDd^Fh^@*PMZ)6U$bD~Bd6!M!L2a|<(Z6;!~^hGXuB(j@*N%RCe=f!pN=girSk zS^dBh{`G<+l(OsN{rxzUHLyqb=8zu{qqO5;KN6WV(_tEd6jl_HifEV~no0KR4)H|q zR+K#;#}5SkEj;*dL&%CZ8ii>9#@X8~+{c%xP5F>j%8?I|C5+$45~UfX`$K8(4|(DV zX>l*@6LC36hd0wXaVflS4b)66gM7Q5j-(9&3~f%nfq#ugwZxG7*JUXkx_#~ivJzrV zA#pybi=iyJLHJ<=yOiBD6tq;yF3TmK53|h0g!vF8%??80VGzR~jipsEYP(ny>=}Bg ztigx)7&E1D?Ia8$R5syHunX`jABH}rkx6zHR*X_F#v)TJPnO<#@5S z``{CxlQ%dP?s4v^m&c&=(%c`%u{7c^1=T40lHwb38LWiE5ZTP@5IZGyhoYBey|V?w z+a+VLguh5nJC#Mo6T#3~SRT0RvX z9m=2qm@Dll9gRiCe~&igJqV{Uvnr-XR&r|MI=u1B(xthjs*5Ga7T7fm7zQW)Ig`!# zJy_TlJ9h=!F?L6iMW+;I^XGp=Qy^}>AiDelir+-a{ux381tAW$&9;ASnkH*KB#9?d z0|W(spVw_2G{rWmsOzvz*3dT}>ULef1P>3^hSbzsU~%oI^l7lO(5Df%kxYzIXXlZt zKQs*`@irT^B&%kXb&UGMwyyt=S(-o1ViwH)+@8%tjfr<2Zq3=nQ|QcIuS$VNirCO~ zWz}_o0ppnRb=SQy_Tr+2Qw$=@Bb@n=i`$$?h+?VyCCXhQK?XZzL}gA28Tc{;Win#R zWmdLyRyB~8k!odAwfks#40F+cD? zVu38oncddyI9;Snb*`gWPLFq2u1 zP3osSUVTpm6`!rB&kf|+hABMRg^wou6&O-EvHRx2iSXA^>)1sUeibcWBdy;hK_QQ< zj-qr1fSiI0L|`Wl=raE*YIznG-s2 zqT&29iEojhZyK`Z(hvpFFCw86r&G+Rt~~GzBxQ{p@vOkG8_dR&AI1XcJyK0Q$>d|$CZ&6pbXP!d!rQ$_|*5n8AE$(n#F0z=You_k&qnS!C>mhL|B zroHE&f+IgA9KoF;`~aHZE-EykTUOl5=aHk^sIUb1LTCa8uJDDv zofwGkw1KTSIZDthi^{YLoFPP^+icHphDJ1S)Bk~b4e9|t5lokkHN*$ii4MHb-NucN zRon6t^*>Q#T2sV&q;Z59C`6+-+4I_0r^`J(%pso-O2vm$s7)fhG6iRWZfq1u*)7y%kMUHwd zE41qhVAEi$Ks&tq`(j%~7BtG1!aL*;7it4SGKFv`Kcnz&N8#P0{r?48#^4<7jKaHS zfp`34Iw{yx?ebyc8*#j&KG}jje>6F4UnR=!eGrGIYOfY>_-E8jK^~zU|8o)qbNmho z1#jm$alC>{TW$tAs}N1$9seFR(HATJ7ZCX9F=M-PDliiKF}%aKvSIom)G}b};=A$^ z4{wYkj&FmCPFuq`094YMeGDM^Us4^hfr5>9DEEC5KOjN3n14coaF72Li64?sG(%C3 zyC~v&llnke(AN?F5vlx`!~!ssza|j*Poyk>D88W*ADaUjo(j^cU}j|iRax(#ruhiS zZO%Zh43I-Q1#+s)%Ac1@%>~qLpSs9TVjcf$Oykrf7pV3(X!sPVKs5$bqnh5c()1B* z6uWSRIxAoEHszFSSXGhCZkq1yk5ixGE-#kVMV`M4RoN`G#v+QdU2+1-`f{oG?7X`_ z%<%64PIx7k3MDZd#s`ro_K?0nb3&haD^^Blh0|Al*~$E9bbB`&c_Q-SUHY$qm-->E zP-AWOiVxt0IDNd5^;u2idps_vRvq=A{1lVz4o4zNq@^DG8XAg-t$s6%`q$yXLf^Zs zTxr4A2>X?(o@P~9{(9jVU!%K~(IKMEr@uP3td767l9a;M}h2akhD@d9yzV3C&>cARX41=ewphJ!|H$dP)~ zj}e=*8_PX_0WILuBKZlUqk;T{{EVD<%}Ia3m*gw<$SbXzV0Do!Ru$P_Rdv$uhYZhu zKIn<}`i%XT2Dcv%gWsT6{{j(A@Qg*xn@5fvyOE2xlexJUd7ReV%+Fg<%ko~<&f8Jj z@;+pOD6o7h>*Qh7wR}74<^8B{`5+tQ8_`BSjE4DUv}xx$*(l$Nwk#iJ+xbqklkY~m z7zo7O9S7MNq=9korZb&Wwix zS2(#>$!w~&vG@%)oj9IkNma%1H|*OVALfNV%odea@=(0EI4u{4iAeIpyc9C44l8@x zlQcV+f1yTDu!UZ=Kp5|#T@2SJI=Jy+!fA2+5KqVZ5WVVw)NFi&0R51a{;sg^3%Np(gmmAy}XF}m^Q_40%Q=;ql3LtonZ z^DUU#S+Og&bWYgPL-#dbIall-$GCW0_a%SK{`ULNon>p;uG^ox%V61AhIMDM zS#k_ik>27$s?xM8NXZLm&1F(30+(^76VrMj)00zeJlZ+lv=E58Rv9G`VBWU?wU~|u zGC!6=q{T^`rIn6CYsT`1r^Y`{G!DaK^*-tM?JACRvndLjcxd*{RjJEnZ?~CG3nBj= zh2#q247~BJ(DyK@i9-xJMzz@&|&H(d)GqQ7|TiAWA z=1!XMp^XC1V)p}lx*AxTp&lwN`E6tO6@Z zbB;wRpLKM&>b{;uK0`)(k=5nj>>?yc@)1=eRfSL7SG)gDHh z=p*#4E5z0tyB}iSo9&adO`YANYr_4-rp*-q&4aRd@a6I+UoHWAf-90zt11_2La;;{ z{1jy1x36j*Inr;F({lctNHh$+!ZRoPfVX_J-y=%W9pMW$1fsRtiF{c|;bE9%6P zMU$=6+pul=s6#e+S&fR|V!XNhZda!qUDTb}sVw^(U&14Y`;(UFk z_+-AQC8MJv=QIuBk&_|ZOAY7ZmPrg^xP>DR4^#m zY@ks+27ss@vsn``Y~|JzT1e*vHv*N~huGL#Coqk~77O`2ok;b|X2I65KY&+%B=UsF z-IAuBVvgo3Td|Rr+$bDi-Bm% zR*Zv6sx-;ci-dmXluOHW97XqOf<~q&6y-6)KQhB+W}(xpIv@yG#76yQ0-uR~WfWCx z)geM|MO~ZvtAk_I6xAh7Yk6Ik*;%R;{b(rsOE;)*@J}bZb!2;Z`zH}RYVPq@loTq8 SPRi0a`>nPOe&`~PJO2l}!Uwnj literal 0 HcmV?d00001 diff --git a/models/__pycache__/ssre.cpython-39.pyc b/models/__pycache__/ssre.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dea6512be1c84aefba8184ecf7e9f3455b6e5d87 GIT binary patch literal 8445 zcmcgxU5q1FR<1vn%jMs8dwRNOCX-IG1UX82W)qaG!X%sQWO_GYwv(M{?Shq1s_`wm z?JifjZ(>3RTj!qpo$s9T%Vk%???>PG)%K0&HSHg%aQbJV@CL5rFOV>e>7mwAcfDnx zY=q{(Y?->MvsxDF%+MO-S~*!~A#b;Ana_p!fzv9;ydAoOVyiePwMw#_56go}t0MDG zSRK?_wZT$rX;5#~WnCd$9yD5w!G+d^!AfgI*16%u!D?%Du-00WuGXwQ!dq!uO&9vk3pL=nl&i#+9%AVsI*SCF|K_&f@D(-e)Hang%|x1%->20@hi z;Z~4}!sfeow{P8{Y~^edhSb{eC`!Gd&;3D=20Rhv)aTtGeKC%Loj4VxG>DRzzZi`M z!vn0OeDm$iH}8D$`g?KMYad|7f!_@z_RUcmk5Y7Xm|AEe+?4xK(uw&X>Aw2$ z^?&&8!L8TZB!JUDb6!Y7WCE?FGp%KOq;+(rGvh<2WipdlA8Od>9J6uHu{?8dw^@O? zxaV1sm2h`hnN@HvuqvzJ?y@CT$GymwSp)YHyTDd(FMp(2TC0N1xG1cf~- z`5Z1ukmi_%S*kWNeruW`htfH+`+x$m)>U$Gcat}$S_37pC>V0Es&a|mDlP{avsL3(?(BRi8 z*UzJu6FHQpT&7J6%$T|;7n!+6^AwpizvtGpnMop*BD1%IjdhR0VC+1cc=BJk@=d%5 z3Mw2au6CG+LT{nv&622iX$);wVhIwIccb`zv?%j6EMHjdcz7V}4iAEHAWB{wvJ+A7 z_=%sUoUc>Ql~YZgT8}qOxu_=oe&Drtg7)q(?nNmkgG&49d}H`!4CYsllP75|*YDt1 zc{s7VS5Qt$NHpEiYuJL4;ppS4+=7!{XS9$suz*W$!oOonq~?(bp?SLoeGL=wF>-Ego{yr%<|#>&nc)`<@-~_td-pH1(8{Ser}i8IqVh z7MHd-oyJG&l#pEnlH`XzW4)-`o&F~Me)wh7x%YUlyOTyil01?FYShi#J;bdHSGYr?IVMty@)wr?E{a)nX z3xiX&+ad>S??o^rb`pfgH^bgg6mQ01 z%-`vyIKjM1W1Q`HTiQdugkr1c$(Ki(boh@^hcZm3JnZsK`4>>9(5EmOVl{!+X^Z6r z?3~7yd>D7TkiW2cQ70Dl<37P+lQ>wC#tA<}k)ATQt{@IWVZ(k$T}alW79}`kQ*7#A zQp*e#fhdwdy?MceN&qhhqcH6a!@%nfM+qmRCrWQYdhcQ4FbqOrt5;P3**IQZ`$Xc+ zoC3f%X>eIRalS^i#h?;+?xN}(PA(neIdEJKCfOw&BpPg$Ymmiqb&Jx^TvwIs|8(pM z>fFb+QPCa4#bxQ0$CkAMh()d0&>gg)H`)xn_L$mhs4?`$W5=xNb)#n0FuQAvS0oUB zZW(P7#^OiPB-$$R_@9ws`UG~r4-T-e@vkALPr(bO7Bio zV!-&3>2u^9`V@waQz;U!`;~0UKw>G6=8&?-* z8OC{IxEu1PAO!w2CC?xcH9GvEm!zcYtk+JrmB!e?FmCT8pJNyVQ+IJCBv~w+h?7pV zxXgpSQ4fdN>vEr=LLZ(ttqHp;4VN~4Lh%`?KN~l~xb26@>&U2n!U(x2 z-46&PJr=Zm5L)hI94CGw3P~Ca6+-2@{$SvvwN0%IN%I2#3gso0sM6s%^^yR=n74O? zMfK{O5}^}D8iU2LV87P}L8syCl)O!Ct|vv}0Sh@{y0G44{s5Gs7XogCdy0~mLhdD? z9Yl<{u<4Icv1lar_27@eVz}hBQYYQ3dH^0EmH!$J3q@ zF8?M*kZ2M*ry5R;tG|qsRsn6HQ&`ehBkO&0Vob;w7o-Ba3>U`83}zWj1qJ@yM7E4<&LU57yr5w_s8&{2iLQ7S+C}CBJqZW|sbbW?^ni zTl^)gmA_2Mmyv8Y%@gY*96kqMiE5jioCE#_C8W%s63KtiOPWZ6m0&#^)g*oi!7qjY~aMf@*{aIAzdw5zxR~?Clov-UFy7jnZtUwbg z`naxk{&NflK1$v&djGfSJ1%UqudBxj-0Mf>SYPJceRCAO|Oh&II8%7y(0mYHg)^lT3GLw%BiYnMCJ-_kzO zKlodvQ-DK|Yb;NnILv@H7AEcl{tQ}6lQOx2)c=FOLb-xgds3a$vMPB%WCWnWMpipr zGBnvk1L#+?+RS+0nk-G~ljTVxt7prg)s2hV%xm-~(MLRJFRbsAb$zO0esjT3e;gX_JbvC^&y0{k&6Pgn5PRBMxa z&^Iy%^FcnBwXATQbB}Yy?ib+gp*;y1Ml*3zfhj|3&0a62L#{uS~Xau1irIdG}yccEQsmM4-}Sy}=_g5RXXD zVW%7zmfiq%=Mm|f96a9LhWiH&{rvhZGK0undpKEt?e+EBagwY*e0Afc4%(;ym$h@Y zqO;4+R!De1uG%K{?fLEY0ad-&nT#j@j}Lo(eVaJlsmC{;;3NosKuI5oCfcfU*a3yN<9L%yrV{J%r)n zh`&SQehNwR(m89&2^M)uvpLw2u6NJp{E&({Y1iRv;dS71x#~_E+gjwYr>)YwUT&wz zw`0x$^u#3Keh9rM>|O-fB>d-SQq0!vrHOE86wNiyVJ7tiW%Y@HK!`bbK@75#xJSDq z-$6K%fK%}t%n$L1Z{tdS8i@vAuY*35$#)3ub^9@qYTIbQ>^t%gu?zWE*RP5m!HpeoA^wBDU*R0VkJkGEaxc@qssky4OpG#`pcPvI)q{6|w44R-BZx(xgO^$q@9JUY==v zDHQ}`mS7W9OL?ZBsU->Kfxs9c_mRRd76oPyrZO{CQu|q+8CX@7b`-0u(k?ca6)E7q z11+Jb0DOH3*dL=T;kdi>uP^-k4}N{`H9+~#QspmGPcs-M{Fmey0C)h1F5q;GWN;l2 zf7}tLQ@RNr&XlBf+cNM<0gF5jhM^A*{}HCa$yNR2$pzjKq$vE*H$}P4>4YNgfw;nf z^1(awa9ah^75T56C3pM*P4OIcb2~g9kW+xc62fx875_l;K7In7g@_`LAuUed=HI8` z4{5kX_525vwLv%$-w<|tCnotgX}r@3{B#7L zp;cPsC}t{b`QC}hD@TfmDXbg+JsOkiA>{L*Rgqo1#jsXcmFKLqRZ>L-Y5e!;+2;W1 z6V%Z*CqIFNcxMGbv}%+9BhML^P<|Gk`_rhsWYhsmtHxEM2B5L@OZp1D`gMJLQGw0p z49ppNN*4MUZ+-&>oHfD)oN=%uokEFI1otJnnBaK8BDmym^QROLlpYJ1+xebWv?co* z==&?D`og#AJAHEPZ56P^7c1=-@GYwYOuGzk=f9Z=U$qpjbqt(<96l+ipf+MW2;C{I zkj~0JR^N0HibKr}>tegN5aCxDSm_pCi4oK&BiX{cX;AmJ=BGc)#ezQJG^ z&}U=b2eB9#b)is$tsrGvKAejKdW;}dJ%jk@lNwn}DI$$`pyT)=lUfK`nO%)p zzonDKm|5WOXt|-KHvANPr=hcOL))vfyqxR1YFuXikNxbV{*Lx-eXlOpC9SQryjs67 zb(!?ZB@Zub@gHLK^pPgo7q)^H$WQ9_*#ue`M+h(hVTqm#>s}A+&sKogEGi^+Pa;qN zCD@9D{{cGlKco>RV&{rSy2LgUFYF-{9Z>sUQt~4t^CQrje|@4*7XNE%{2L^#{4N+| zwExoi{rnah{|Q$@{-p+;r~M?-L~#=O=u<;oNguB%bXx4~=_qQIz6z2?dGO4=MTpqL z!T0t?eyCdH=RCfFXYdv!-=u_`bw#m>&vC*~{^yjCmT_7dLPQEa+CdoNtK$jk(zhv2 zAFMc?*kxSF3KA^d!S6Mv?zm3QdB*v1r|P(}PxC3sjz#XoRw?FB)=ox{RancBp|K4B z66vbifSRfsrM)oOpooPEeB{wV6lDZ4*|>*K+nm5zlvO`OW-08SWt~I_`wp&q9~WO-=psboFX3TLqY!5 wdw1Wy{qB~0?Nc(7(vu(goEum)f^)BvLdu94(PY|pMU&}cUju)1uNjv4Uw~GKR{#J2 literal 0 HcmV?d00001 diff --git a/models/__pycache__/tunedshot.cpython-39.pyc b/models/__pycache__/tunedshot.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4252653a315fe14c175867ec3e2683c260f57502 GIT binary patch literal 6821 zcmbVRTZ|;vS+09mS6`>6XJ%(_9ver{B$+V=*}*L9^=9pb$Q~K*l0eC(Qd6geN6F4cicDEvSSQyCTv$+L*Q}(ne4XExRVtm7pFr?1o62K{IUGt*~vk z!zFtu?ARSqUJaJRuH6-BD_99v?bUG2UK9CRa3Nf`*O9K<7g@u;bX#Ig)_PZBEx&uD z+LxJpN9whY@deU$PcD=wg4BZ8W%N!DSctbCkJ14ptle?srGq$f1IpcUQ}}grmU!Y@7@t|6JV@uZ@FraFe@{xA(CP>SgE84^j+;scJXf z#DCRyd4#!?qLI41@2Ahkk-rzGMLqSSB<9aX<8br#V{|ybV@m7Ty(BV|Bbu z)?iJ%tE|P^cw20Vb?~mSW!A;J{+^^sb_2V;QdDMN;wh(ALTOqIJ;|R!q|%X;%b7ft zaeR0)WnH=_@mEp>Y4u3SBnL}_)R5QLpVv_`73E}itC7AP}k zeGkht`c_d>LC%`he5*{OriL0blOd~2!pGFN3xoSfgafa8adZy}Cw(yT?;gK{Pbeyb zXplMq??3GR`R#u?eD}q|+Hq5F&q)S{eo?>g5Bhtl!+g(u(EqOw|KlHC+qv;#VT6N- z`n>S|GynY62QOdmS%sdAM?No1#}NZMyh@{+4yL;w^L@UCEMFy2BXNPmIz*vCQ4(m3 zmiWOgzk;F<<)Z31Uf?E)<9sN6^vSI-O1FY>l5&5G-9Oxm$6F(B5Nw4p^Mhn79Y;P( zpk|w+2mBHSpt%y7OOoqKU9QW9d>L;`wv>zVMfI>fJKfFsVm+A}&wiAqqy8+Oqz;iu zLpl=Z9kx^6##)9d@knmN>TXbo#qc~Zwp8(AG|yE$p5 zb(lr-NTJ!X=Dw0Qrc&NyI@+}|Eo+HY)$$foP}0uY7^OX28g_=uS@W87@Yk7wlQ6am za~CKwhIVs0H0X`W2H|1-4Sy84-XQ94Zf+j`5Wilx(ENLDQ0QsQy}hCm#ZI5Qtf=}x zB%cF{3e6icw=mq%$d6c|dZRIfn*wl0g)xo>2V)xZ^X1 z&v1f4lG>J-O{5EXQ0T%;3)$h?56MRdY{5X2??=09tbe8f0Y3RGn0WeX84M zjG1hVjOh-Zqzlo9DZf>p0IkW0zOEc-cS=Kr(Qcw1)~#$l!*3w@*iKxdz4YR6bo~uf z|0f>l@Dp?E9gnb;v`d@Ug4lC|GoHz?fVSc4`7g)XWqZV!~pzp*}P+9rkLb zl_6^9D3M{yd)hW95EO>XLEfZZl|N4fpCR$HB(_MrK;q{}kZSNxLcrHx2lvrIpsXmT zJrNtm>AU$Y5-*duN#Yd}Pc8(ZF@Fu>^LUaDghWbPsw~_@TWQLNE066KozodYKw4|y zDK!@mlwkvhpaRdsfjHzhNXi;@-c8~fS^GPlxIW*yOYtmPL_L+w|lv_i7Xb>x=tbnq-s z;XTgv{tU{s;YDgKEVs6Oj4hOAY6{rfh@*{%lTSaK0JaHtVVXt~kGV&{3RAkOjrnU!vMd5cm67QK1c@-MBDzV;;ga)TqPbG%W1|76VOG zcutx`4$|80!kGfK_dp_O2k8sb^7FL1xth?jVLfLkz7$&>5cCd+Cq+)HK({91DMVHHtAmtYM&48}f|MfWf=fkoB`ZHL%rkje`A9SC%Hi?? z6U9A@Sk4(pp?i@9q9%`#x63b3&R*nANn zLd3QNkuIwVNV$YI+M0xWKQmpKfzEQ)1$?YT8-OQ*osVq;U|IpdT+UVmz;x3q`PDKl z0H(3ce;voge}lw3Bo=7Gf0MFbB5_3G%OnVR_7F*|VU* z{I{v^?~wRi67Q2BH^#q8f`TvpH4-0?_&NzPUH%Oc3)AJK*QMQli&6{Y<-bSS6YE_4 zk=Dt7pIY7^@dqTnP2xKwzDt6vk^doyKO*rCgk9ft_kB=!x^_-%lZ+Dgal8C`)Z+Uj z{+L8gg0SJ7K^CjdEOf>-1#Xn+mYl%kZ_zY)144pLp0!c3*VP|tnx>qyRkC3GmDZ|| zwZb}$>Z$w>vKLC7V0Ap5t_Ni(3{kN*c$({vhQI z>O>yJZsPwX;pPeXY*EdW1chhf>rQRVlT=0S?~Mw+%FnQEA7RKi6N6pB*y_en9X{eG^D)ZH^W z-Lq?Hd%&O#=Hg8p2oS=pr-~O>QTzpYpo%A|;DyIi#Y5nMQWY;0Zw^)A_njV%F6%(S zsQT-3Kc~<6zTfxzj=XxkqTu%f<)8h(|0zZJSIV6I8OYqklYAe7DNGF&r+BN5CiA+Z zvUzh5-vtd&Qi4OEX#Z~ycDfCD@fOzRaSQ{zoxJTYd%s~Gw2@a z&K0KKReG&se1WprQ-v1CkVZK47`@Yd7U69UM`@oDmF;otr~P5#1@988vKrnt ztFs2)W!7XZyeq8DI(S#v0_);kdtWgWr;goT6s6gh_}ZD3P?{D)Px4m~sdA*`YNk$A z93S3HTT$*S{MA%LT0hb<#l_Mf4dl%gsMlEKH;QzuF$*xPsgPx+;A3i=!s0;^SsIJOB03fBe(ewr;*8tf(JT zpBF!P_Fukv@XC!|MVQHW6mVg?t{l+iWg6XfG2Me9-{Z^3@+A^g5|>Dmi>=B~uSOC#yNF_6fpHer+VkLgadYR!XMj%2b(XJIchsW2S1ZO_gtLsYXCX| zdHbtxX%lNw%1UqP6MIsgRI+kzpieog$ae)}m|1zMPO6!aTUj+PO%>AT%tWqolG}g! zO=z!yzaw2$;?@_G`_)4r^}&qtNsF@y`1C`tgtOgEPdI^_PYD8)l}hkHzv1He(h zfkVRZ$INp|Gy4#hHyQ;o6S_YdLwG4*cO zl!lKF13na`1dRh2kr4%6%&%aPLfiI*RoIj;XnqumzzMq~*rFx`f#%<+Dg)*BHz-A7GnX0)8MK<>Vu(<{WvNCO|PfF4_ z_04mZOa?}lbQe$3h1h`|zg?REv&n+Kt?e6k3oC`iejfEOZEgKo{t}W;?8J52OMe)R zZqRW5iAOnn=EQWzBWxw@()#sq=zC#u1BsLNJzZEq>;tug#e&p>GS=rET<`u)&z9Oy zoJXMy{1(QCE(J*fVBOQn?ie4=~xqHJ0b2{t{Xxw6_WlqNBFd!}b$PJ$DWS z+q6~k9Nt2Lz$Mdh7B?qmhJA*aV#{)PUl=7^S7yOX0Wi77G?V}aay_#pgxD}xV_*)f z%!IkxX-R^F#&iOS3|rnaHu+~ThqPe+lSuW-{CO&Pfy9?ctdsaz5-*Y=vEV zIa)wkYvCz0mtYwiH~=Ad0p7zUr$K54GB5zJE|UY%XJG0~0oNnchf#$z`AsHcwB zG<%bk=dC*@t1M@dz5QXMCvgO1i`Kb!!+33p+dRx%uOpMVvn zbg!pNaseCJ9dB=kfv5u3T_PO;Po!pnCI?CS>S0}yT2S7#*IxhSq<0xK!ygxZn133R ziaIE(bjy2jDEa56xQ~TM{o^Q>NIl({0r33^1z#S0{%tH z)Fie>UhJWSmuN`Ig-Gkbi=``>;eM5t>CQ4MXRBPy@?hnC_28z-5HG& z{xdW(0iCGb2AaQ)HH2Xh3d@xPfwK*XY4^;+Uf!YR#RBE5vk!1qi)m-sbMs!X9ntQe z#uLRP0R~NIClxq>E}kXr3OJXA^s3rLx~sJ$4s{8MR!(f|iE+*$64}<=zWy2+XbjPA zWl-1&EC!mW@$Zr*5d~*Da<@~$9cl?eLnIv7qYTQ-d~8gN_%rji7TShrOB6-6owqE( zUJTTLxJVl?h=!xK&dR8P_2vy&poThg3N{v_E^kWfQkX)TF7>%m&=XOX5gIr(7kaY8 zcK-1n==bk0!6(?!kb<&tB(#uAW(-oyyr`$~2%lS&R3*!GgVE66g$bDV+{k-Zl;A~N zknvt?#xGLv4zTt6eLvt|r_MyEIYE@r@4j|hR7TK?C;6 z!-DDJK<8;T>%X`Qo;50%p`{Yq5AeaGmbBv$`ovz zX%n02CSq(%%J>|Pm^urKwKl%~iU1v>_&&J4KqMr!BftdZ5H8W_2r zS7`LCnOEgo&uj8+qoh7nUQi}>+88vm1_k1eHMy!*R?b>^3$wJd))bT>?|?ik$kvEK z;0)?VaFmnE{w)v(IbOOrSW1_(G8{~)t7MhS3Y=FNl%buKSY=6J)jbVv=Q^uNce8?0 zliX^tJDH6!u|t=YtUWa+o%nf}CEN?#l7_EXVD3#%ScudDFyM6rp zJ;Z@BO8KU_# z7{O7dK~%6co#09f_7R)BJMv=YdI{g4S@mQbaoVG~?;MU}#v4@fHVJ|;(RPV(k3p@m z@G!FJ zC1ui-R{{;(wkqEssJ7|74bPo5VWJ51Ve0w}WZE;3(FtMzK6cioYXS*2%|Q!bgL}Uj z*mM9k9RN)|?~u6yYydQQcMdj-bFjG%u#w}XOM~U~k^~!T1~#}o1lTM9R>}aIiUgZ? zW=ysKkuIxBNV$wQ#VYk(($olk57U|IygT*($Cz;x5A`L!Y~ z0j9pmCpa$tYa~e7PSJ$FN7>&X@tY(bks#pVzeVC35WT0QIrI0CM!ZRwlJFmA&yx4@ z-=@C5L*jQy{2qx9NIWL-SrX*c__s)WNP*Rk(EoIdGN0jo#-+nhKy86|Gtw)vk?i$5pv z7bO0Y1YyGkgA_~dELO(71g=Txa-6{A@6a?M3|C;2=WUeib?HZjVQ3d@l`I&4g|%vA zt*}n3d?x=xqXa^5ZEmZdICZZlVBn>6+{AgP{~P@_v&)LtjK3Ohza& zn&o}U61(EWMmQl3pF)WG>{8Va!q9ckY{XO4ZNdg$^*t-3U4a~8Z?GD}> zvPJKzJWXNB>wJmZg_4k$a~9+^+jlA;;V9BW$1&9%dlX1Gskax=kg`>-4rz-EulJzaz$MG7CKkJkZhv8m7<#p;r zn&~v~Z<1*Jl$189CMQAhnEbjk8}lhDqm2>v_2L+AkgX-e0cVFVIUe!6ntBOOORZUS GC-&b3UY}+F literal 0 HcmV?d00001 diff --git a/models/__pycache__/tunedshot_ablation_1plus1.cpython-39.pyc b/models/__pycache__/tunedshot_ablation_1plus1.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0980033a056058c73443dfabdc0f27938f959c3c GIT binary patch literal 6685 zcmbVQTXW>bb;f-#m@9`va>>1jR^)Ofu}Nv;k{o%xTCY};}r=onPF*CfZl2kHN zI5>T8bf5E`uiI_4ED66~eEg5zfBw28{R<_|{xl?R;Yoi6!6YV!l3l%JTM=p1R`FIs zHPUQNrZP3uBf~a?u7$OzX4izSqmF5tLN~&CWZ4a&*TQDhvRgto!*a=cqv-7SD`oUHP*5(zap_V>pYTJ$L}Ah_7x_- zDGjNJruD86LNOtO`Cr5%~cOnGEt2UVs$l5CCX%)nb`HD=;%usXBwuCWGd;%%}PYvWyK z9oEI$VhgN?cY`gmKHklDB~7whSnZ{7K-2&J^hCk&;UerUt1Yt*^qo#xlQI>9N8Ll<4t#W}McBR$G-$TTl(;nbWq1m5#P` zl+=*2rX@eBbd)quV&*brok{qZ=2mHNKaH{BO)rV>L25rGaJV~OJQb?yB)e@ z{LV8YRDWh;^nhPRhctFdV@q;VY06F6kgwov$(FJvuc?RK>27b#Chy5qdHN%rw*Eyt zX%ixshO|YnLoBbng}Jw#EP+#1bM{lD_v7S#JWCH`ehuyTr{>~GXr<-uWIlH?H{Bl~ zoeEo%4|}uOoECe8WV7D_6mk?X9{`nJuxk7aH2sb4$FfT*>Q4@TgU3(#W+!&l`ntERkX7I#Sjwb@1fx z!HhNhk5pNTyU$4Jx1N?{@&5zJOdi&9jVVlhUzunkrB73OW)?8pp*3tA%abN%+RB@l z+wDm^Yr;+1M+%LWxA&Ezbu1NarlVdb*Yb{-Rio%I1v%Zki(b0Jg<)^Fn74082Y;U{ z*a>5+Gc_x8$KoH)DO zWo6w5LirrfRBGOsxuxNbMt;mn)f&4h9RtNbH8_lkgqxqze-lUSYTW zXxnEDpW%c-n%R~ZP3WZ@l)7-+Qg-+a>R%V7_@G&%(yBbFYCGadmQ-yURelgN|DDpJ zslbIv$7i6@V^+(>YWmKO_gM^-l8_Wr(_~Gt_;dYs(sPETuP9r`we*r` zbn^`q|2rP(@Y6Hz9gnb-v`QN{!o+jK^d^*9{efB!yGi(3;Nc%j2jv1A2M173+BmK*R>U`(M1KQ)0PYHkV)G2yY=P#+q( z4u3VX+7P8Pl*qB<18s{F2nxsLpMf^0^XDkz3nYG<#3qU7Nqmt6nFjw&2*e(&;654% zlok23CSt)jeK)^D;w2KdNxV$rQ*%LR%wL1}I-ax#A(7EmCW|o9Roe35(qpSd`*e;V zkkuM^D$4}~sy6g8$4v|0$@$OGO1 z6Xjt&vkCykuA3m=_0tmC8W-lMM1ic0MU4P~E-%SUXK{&i|6^i*&7_@AG zyo$&yco6?J&&hBWO@@J`M z0=lw!2Wb8}<`9N{SQ?J#2<$B(&OOj8e{rdLH9;}z^aJd*YS?M=+_)Efr(6h5PZrf6 z44tO5k`jVIAJ4LK1+>e6z9#pf_m!@|p*|te>ddzu>*pLIk#Ef-1C0QSAp$N9D?fq9 zz!DYyIwFM>YCtPs4xtRvWb-=HbKL<36}4k<6+{ZO2U(ViI;CUuS6IR;BqouC3Oh8m z1Sr}4dmzf&2=vZ~2=X(}I@hm%{w|Er8gXB&mCq`R69Lsv13D=T8q;{6fd_y`-wQmS z+n75{F^D3&IXwx9Kws99F)S#E_-~+wU7Pat@+uUt-?4|nF?i;-IG4@2^(&}6Fsj|$ zhR!__v_L!|@g))$;BFUr5AjrRr?eHTVPY!`Y>KH^#$kK5CDndD7DVxB70%I2Gl6YP zK2k&=B@_~P2DBp#K!k=@t70{kt-F{io!bSg;RmE$ArX;?AqGkn`)aeZ!-c6+lfT82 zl0AUZ<9`}<|HqZ`$a-c!ivASiUc{5W2dK%wUdfh-1s{PogGsBoPW<@;`R#8^8MMG_ zvKFRHOr|0iYFG>>7Gh?s4`i`#kYSOMEr z3rpB$r)Y?`S~SJm%sa%2pOYqL)*g0puw!7nB4*Xi>v^~6Vw8ovdyHsY^bnU9MQt#A z;B4y%F=%2P+(t|m{bftTQWhq}c)+fFAS$GbR8;GyqD;8Pne`}R59vhP$Jn?egqZoIvmR%{XR=Tj0 z_7<2h9qb%`lz4mpI@`4mm5L%@zW}em;UG&nu{l8+PRQs_2>U?(O`KYT2+Zq*wp}#) z=-InCZ6S($cKs#l7TV^+$@-UWuD_b3>H5RxH-2jeWp@RwcRlX`WnJHyeDsz7UpsU~ zKz^pD!8NX7Hlmz*E-6WfZOtcv+uc@RTNk)-Ldq z3oc#iVq0kkQ!v^RP{+CNWppCLNt)k8&!rK>m<0-}%cPW#DHnHNsW@Zd&|lW8Acc8~ z$&{uDVU#=%8fBxpmk`ze4&9_*g^)mmdmwmpLRm&&YUA7j{KLNq@^=MkQ(Xi3!`Vh4 zqa`c9GR$*nS@}pao66zh952QFhZyG^+|d2Q9BzI*g_}%y3}`ap(nQxl3rUz01lA_KcW`@^ci_G_U;_r!DF7`@0Z1kI zLC`ewg`zzLqRz03UU2a?1)?5+s0R>g6+Log01-f_=+A*@X%0j;07Rm{Y= z0v*RZ!kP5H^>INr`5#f+KPK@_5`RMCTO^K1&>@FEBJph!e@cSfnZHY7?#}$rC@FmT zJEYAWntzv)ryjd}!DIQKQ~7%&-Y0R3#P>*ipTq|wen5h}lK%yXACjPzv70;Yz7MKQ z*S)E2l5^tTYlr`cYWyXMzasJ1BnU4qI_9!7J)+_|05>;uJ52!db5u>=fRNym=Y5p? zb@^jW)07LoN*;`V<*f?*T7h>O^)u;TX*H0AYja=y#Gwamy2gy%$akF5a-51TLvK3H z0q9p%B5oG>r|FA`e~E*EI7IRxC5c~En%-)9ei%B=nT2?Qsys=e2SG?7 z?j`_3JySKGGJ!efig^w1O;KZTRqSS|i~Dqe<&}|8*nr;v~ z=5$4h+YHf5mL%bRknt8ZBFnVf#CkX8VS#t3gxDOXW03f|GYca?oED}uZ&cfGlYA|` X1~J`y!T*Q@Hsworx^mONhAaOCLTG(O literal 0 HcmV?d00001 diff --git a/models/__pycache__/tunedshot_adapter.cpython-39.pyc b/models/__pycache__/tunedshot_adapter.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..608cac73d8d37a63dd80275b63cd59fe8fe94410 GIT binary patch literal 7584 zcmbVRTaX;rS?>Gv^z_Wm&d%;?R~N611Ri6rgp;y^Q6yWItvJYwv4m6Uva0Fr(>pt} zJw3ChdnE0sr=Yw_?a5ht_>7b6mNzq^Zlo1 z_9mGsn63W%-23$D|NGAOpY_UROQPSiKk{!pCrSSzGKY+Y^vk&7e}Z8WlS9eL@3Nza zyy~d9E1}xa98IP=HPkzXV~DgC7CHr|AksS8n2ssZMp*1vPD!K-q1`DvWsx?+N@vEI z=~SI+XV#hR)SQ|qFNSlSx>FZvE1d5vI18OcXA$|5bAs8UVlOdr#wCKD<}|!L@%L+`rsV_v*;+-6ivVnC1uHq~Bj%{_wva z{>MMQx&HFyy|<|Dbuzz94W4`T)W81lg=a4BnaKILj{MX~E@nssk|Q(8QNAo~$V_I+ zeG^`xGVQ+PXiR4Y?m8v`gaarJdJDhwXGdisZ zZJRHB<-!lYy7StlMk&+deh;%ST~`d`W^-%Z?4Um}r4;k5FgHl58uwNErV{mmjlh!gVxi%O|7lE%_dX-Y#4 zm!8NYWh{M-$r5?*u6$Lx_4#*|p)o9^g?H7VIV=vXv^dhxr?-uMw6dj)%42C%VLIB)q*^*7W>p%^Fa;&m zw2D!x?b&v%J(pH4O51;vDnN&^mYExYm-ON2oIV2iNFD@0wl$vu`815jhYy(Gvqzt6nP@Or%Js=I>=jKWc6F z1JEn?01XCukHO?d=&i#UGrh-KQ9`3egMfFlLX5@%u%&eZFXCr0w@lgaGb4vurqKY6 zS>Ys3NsLJw3-F~2_ZEn7DnVyGU<|L}hOIbpEb%mv&g5363k+wn%U`4Mby15Kns&;p z98G!O9`7dIyl*KlZ$&J)n^`m!V4Bi+4K#YhYq?m>z}@gaiwDKe!bpm#X|kqRaut^; zP*#yw<%(RaZXRP0`!$VJT^qRxgHK-wWeQNKD%|)XWGX zAKaB0EJ!@~V0piT>)mcPOyLdr_sE@rU&r|Hr67(WEcoY;6Q7!Yp303VxPylspJQ=< zR1d9UT3ZPpq9#}6!R#YbJ@OrhylJV#cX$^G;+Is#w|IG|gD-U86iYS&YXFtl+foBK z1;>mOfm0JaqNb+c5bTd_t*y6>R0m$oq|ip~lqFIuc|%*{&*K>axcmZAjUs=U3SJ>| zh0JHj&}{fsGUVp`(=gbKu!39YAXrwE)0&6{<6orIn`G9={4ANDBXg9E`8z0U;)-i9 z5;<+|ve+i7N<|*bKeAf%osK9C@>&B|?zzOwSim+k#*5fMT-rCt&7dXPVAe(2LsVK9 z@T=5TX`K>dRA9=8Rtvj6%D{KvM7do|tPym=v=q__;^~1pRPY@n*A^O~oKz^+o?&`Y zWyS-b!JVD-`y&YwcysS`)}Et?dzuPuvu=x6+MP z?hT*2Hw0}H?*f`e-0gFph!s$}*-(YV07f?Z8yjJemB8jMNsoZ%TLP4N5GSt>%0gE| zt*l&o>lfn2Nr(-#}|rH1r%VZEBuR zP&{?=0?u;&*h%)tbB|#sn{6G0H2FjN7&JZ$BViM$<62NoL3SB(U0asxNY|CB;GsG( z(b5!KkBoDKkt__ikFOU5zS^YZrFM=JAO@bOz!RTJfFq{fC(KDBb1c!Gu`5H77~mc4 zTL?P7tq4yoFyk@pbhOh&yF%U$zOQeo`~xvwk;c1qYO1N)7EjUM0P~f`5~-N&6zyzg zLeUqu6#jQfxm}sggjq<<;Ht9H18rzVPw%(2(e~fbmQW&Qz2CA7#G0rf#Y601vkJu) z?K+!54OG==9vq;ct~f@Jjr_tGYAmAEqQJStB*L6PR&`C#>?0eaN*|4`1sH6LmFc$l z4#wI>IFGIICsBnsQzxAiK2-S6BcI8=;~r7G*bJiJZjWCYqXDPwcJSo%E9Wk9XkVPR z{YZXxP+JjNdZh=q?F10W@#-f}^c2R$MC&xsHk=;x3Mp`Wx`>0PagmUDJfS&}iyoUc zzk{;x;u2b11L>~_5D@tsnom{~aXz9Yo~dz@HUHu7>Gy94qBFD8r3k3s$rMCa6yhMd zVP2=9@F6`=l$1oc;s(90zX=x8Z@L}tZdQO0a1m-XPD~_X0_li2>~_l!xKATlEu0Zx zvx`tb02ffl6!0|bV18LO!pbrCpoH&GWAWDKC`B7prrx-AHIK%`oODEtoiAi|$lciA z42U6kHk%(hyb?u$7h)Bzzs*;uZz&J^ay~hYP!w@l-v{~i5e2P_Y0!AH?gTdgFzE`TpL^7t|H(;Tdvr=;d`{< zL5|YKE`X-2-E0_~XiG?V{(w@%iu^q?yD;zzQY{D;CPDHsSwdat&>|+{Lkx?*1C9B= zF1$})dHN%%OM*_k#ap1x1fd#ct5C=%s8L6-s8g``zI^NKgxZ8SAtGVQ&}1q~5jyS% zl_TU!2q=xAO`+Tfa=cx@+n1*)MG`d<*2Ny=$vSca2BGXybHlKw6?E6 zfQs>w`SwDxC<5Fs2^t&Vg>#_OlCqGHQjZOi;ykT z^Rcl0F#@No;Nw&(@DmFBabUx*Q5T~XVLqtL4@+bI>r{%f-B^V3I9JMwf|O{+nJGjP zQerB{`R6O>`q#K(I?<63XVnltPyoCDt^pen6XDNBY;g*CQ(i7nfUH3oDQKz4%14@M zD}%XxMiS?1;_*iSOM1Y3WKjI7lWf@Q=C>@6K-y;8ifwD?a zPH{fjHcv>ynWQ+f08*%nFA*J@bl1Mg z7t|oqf*Q(e{8#b0_^**6=Q`j4{&mWJgUoNh>=9?@yzxz>1~2^o_+Wy5{sG3oK?6}q z%s&bKBH-cQq6hybnQxQ%Ei%7NW<=)GWQ3CY4yC?J<}pa;ze5!SbN;(z_JPfRpRxx4 zUid@+^Y2m1H^@9B^L;XZK;{q05McQqk@*1`T1>~@@U{X3YxJLv*f9yWICzYekpSDHFeXSZe5`320DKJ2bmnyo z&_?kP!2e4tU2E%b&@@fzkrs5cb zlZfXeg>uC~q{ROMCM!?Q)cqg~UH8yJ`~)>2b;Fln&fyZLV_>SfshVd^!>pLgX2aCY z=R}LfIq@|`9DtK{qWRoOXiIbE#IfpX9Pk!o<3KDYSE&tcO1bZQk^G+@4#HaOMo0c1^5@h~55)ON<=HdG7c0^t@+hU*t+!DO9X6?4>LrmaNdEOwpn($ZJ7lY@{Ql+1u4SJF`7K zv(+_xEPE2iD?=y3avpZ9JUn9rF_=RR$tgf!3zdBRC=aIX5VrxSvI0Vzu*>R*@TYm+Olj##lGW~WVsNP`xUn$%XU=l z*W6ma?$-Max6yC9P1#CCg5<*k5v&`pfPz>LquDmEF~j!YV&h-E*w!o@X_8 zjn&;JP=1n?Sfiu4Poccdn(otV!F`6c++Vn^utm1?p2C*G)`9L`VCoH}y?ltTR5sgc zro}O&5esy^4q`u|+BIMJZ$dTXZ6m9GISzQ(4`bm+o1w_g z-RSm*QTW;*>BixiXHR`)6p8Lt?#IE-V-?Ez%U|mXbcLZxSN$Y>GxT|kv9zoreBKGg z3xhb^8HlVb!Z;c57vfQWc<=Bze9KSnxBmS4zwEzvrL7;fP(Qp&;?E#DAAX;He|4qv zvxoor4`11O^~&K}^z3yK-ynrgzqa=8|N7ZaUOBW;3viwK>9blQPzV)QWs0l4uWYMK zW!if-P@yy9J;gPc$t>JWR$w;n7ArCb_X2jNjJwS$tcrV))mR;Ohc#Fe_YzxRE!@lb zKDiaP%vMmZzONXHTf=UxW`)^G1lozIQJE%3S2BbU%7HRbQ+28WF1RyoRk^M3*M)|% zexRj_hp9pusGF;R@JQvC^Kz^)3s1~=U0BE3C>2(f<2OhnRUr#Z!Nly%%;I4ZWAn?w zAij-)C+-cyn|uk=$qL=LE4+yFI@O#lUj#c|(%lc+c4j7{VaPMv^Q4l;i=<yFa!;9?B{?1@@ zaTs)?i~Ru$qvWC(#UV=o9&an&7~bPgq7MR2(u7ddvQ}2hs->>s?%;2~K3npQ`3wV< z6lXsw)B3-LD=9;yN{`kF;J~`5o0w|V69ZUAKE*IWeK#K5iRbmU%Adx2{KAoV7NyMb zw?)W3;U~MlgnA-3T-|TZXLI~GAeS}&HBcb*%K5+}DXBJ5rpj3BC}RVcDb$HJRldzs zMd+z|pk7nned8T%Y>f+P;T?T!kBeg`Elv!)Q%oIscTz1a9suU6%D9vo6EiJMtf@li zN=?+9d2R2RuLDa4{tk3iiR+(Hl5ak*sPgYuA%)s2qz2QN{y-ZWvS!X|rm!bPtew*< zP1SK3Gp(d$%g#^msKKks3e$1c3+%j!D$gDeIx3eQMgF0w8BuLzn z?30`{Y{ry`mPX)K!~Ryt7(T;`x`}WdIhrhIYBw{b&1I^`uTcM{e2Nd6zmz$-0p;%w z`9KWvcT0JDH)i48%%Q2kPN|H~fYKwV<#IJcZ##Gy10|n=P&C^xR6}#rIxbtaVOdpm z16S3kss>sdtvQp&_&)CF6ut@DCN7EZODLp-;0t3zI$t|%zmCWMg-h9gc5c0+A(oO>Y2#8f2>d9yjKW;Mt!Gvk2XI%J z#X{i&gO#HfJpXp5ZA&!d`;mizQxpT1!X!bE=2uaZTg|UgyA_9bFxc5S7LR&;2U-cO zt%8kcs&#e0@yJw9?E`sfS}M5@^fvjIRL5StIyT`iOxP)wYyz(Vt3)J8E!Zi1%tVuR zYQvA{sV#ko4U08;X3t7Z*sCoHJv^Ozi4;rTHa0mSNg6KyBuecfe}x);nFPUrzd+*i zBrcO6nDY$?1Q@K~E?$scR<_fc$OYrH)BHM#FOm2ni7%75a3shZb0P{`xMo2Dpv@tR z08!Vf>i*&*t3~^CDoPV-EnGQr$unaCdx+03A%J)kHwb2k%{_S5Wr`s>tqc6C($i_3 zgf%HJZ9=Ps$cr}kJNQJkR}{{qbleIWg6RH0A8XhL#VgA=T@h8P^=ix%b!I)F=--&X z%L;ja%*G$RsY9Daud(8hc5|+MYNTA>fIfOW)%${Ip=FWc#;);!${n%9oCErf<;e=3 zkpAM7W=NK~idq9#6W78N(c{e9FQeV)og-}-SxcLT(}4p-)OFz^iPqaF>tw;Jf<&ns? zj_m6;DnJaeV5OJa2`mOk)cBTwuaVEdVHnv&LD*m<1{jIaEA||TsoDYp}aDTG;xOlwoRsm|a39sT|Tc}QmW2b3L+`k5B-tegjSkA;DM zFDpqt>V?BWu!D5Ryy^A*yY1SH2MF*3DBkV{A%7R&#L0_f`i<+?GG~ab>xUqTiR8z$ z&S2T6Ixk1uOM;z{c9dt0JmLC69EW}cfWG;*O2Yn))83imQ2-- z==d8Lzis8%2x&swBsw&!qRbV+>UmZ?8uSbpcn_NI<4RT_6s>AHk^yLt+Rv=Q5^)4$ zzdGOY950UqX&vuMKT5Gc&`!h49B497(~e4i32#b11`GgRLzjD_ZQjJ>9lRvJoZqJs zF)p5vpv11N<*~IePe9H@0(#{KxDvt|(gpk-hxPw;?M{6K8Tb)e{}C0;6leugQM+L4 zlGS`T24YhknOhTnUrY6gPO+1``?rwMA+8$0q=x5L_nNWY1=mlFOR}e)0o)?OlD_0e zR%%VmslsokCh|J#fVjN5_l_tKBf`p|oSKNc=z(yLUd&!`R;~sJ6(7;$)e#@eIZ~}vq3x}(OGF;qy87j&rALrCWNP8H)PTc zJ}k(sb{)y1<)H3SKXM9c=-RvCJuil#Wkn)$ICqGJtJ9dLfsrlnFc@&=&4;Fm{0TlX zc?LoO^ChNht1Gw+-PUZlJzSCtSINztpJV~XUKRQw%rYwCGLI$125a-L!W0nv< zVDaEO8gg-J*I_xx$&sal_aRIF3u1k#DYNujvn(C)YGTt{%oy9uL@P3OI;)XdFQZ;U z9&e4yltJe?H9TNtUeXl#M41%QGJFZK^6GC?_ipXIBKtGOHQej0AQ@#j#SzxBA}Xm3 zhKCW(D`_pQPpxqyH7B*SK@9Yfod2h%a{l_Hk?YO3a1{%}&ym=J_z5ogd#L3*N85*F zc>o(vKKjy4oPU7VeQN!S6d+N$cz?Y9`OE8X4w7X3{wo_VZsXZanP~e#aF3c^kWEC9 zPP9n4w41rx{|<)W290Cp42%*5ev!oUBxFSRI+c#X0{5v(AvkknoRtBa?@&Vx={$5MZD2Fgh2dj$#QS?LJMhJ-SVXIxDy(6U#QaZMB_PFlt3yz(ug z1UB8bVR&h6s?jkKAgoVHWJqaaQkHi;Cw8fgmg-b_Ng3Os)~llzoasKG`B~#;T1=aO zbY;>^n^PpWlLe%?EvbzIGqIRf4v^A~oxMv4z>+A6rQWhwkw<72JfCwOL6q5`mJ2Yi z;*!FgT@4A+Ew}?pgAmd&!xk$|knZ%(L&ty#YB0gYnF%hWEtue9yw1uWuQN5ri!jo) zbdij7uO*(CJeilJAyzi|Z)3ao?~ouInSrhAl!4Ms8JRXlSW9)a>0dHQ;^Mdsr`CKz>QfT8NSujnOWy222Zv`k=cIqgkUzpdjg;hV=qrd{ zWbsGVE?LRa&ke&sl%u~gjy2X0=WOW<=ydL+7qE&a>i=yNKz*M)@(GU}x?QJ}eWFjE zmpPu7CzmLfJ#P=`MgByd!E=hh{0$P6M$01?-ltNJgrw>pP)T}k@@4!lA+pNs@IMHn z$n#Dt3GpueED6dr2uU*Oz(-bW-M(O3cGX_CS8Nm4Gxq1CR{L4Gy_qThp(NoUu_QBQ zcR?NqZzLh_B7xeMi_3Kyfr3e%o7uTe9CM=g{6`R3J#QO{Zj@|L$}IDHw~{v?t)=ve z+oVtKGfzYrr^G`|BjO1yyb3(Tyn`WA8q)FMjJcf`fbYg)e`e=bO`%=EKmw_}T fp%^-Ylgy7OM=FE(meslh5C||RFhNO{sAm2TYGPrZ literal 0 HcmV?d00001 diff --git a/models/__pycache__/tunedshot_cam.cpython-39.pyc b/models/__pycache__/tunedshot_cam.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0c3faa6193ceb6a599dd2902137a1b2cf50b9cb4 GIT binary patch literal 10716 zcmd5?TZ|;vS+09mcUSlH^xS7J9R>T84oDUb0*M%bdh)hNgv%QOnQ$kNJdg(->H!6KNJbze#0xS(=KD`o z&rHwmMuI1*RezoToWIWXzgN4ZQbEG?(Zl~%|JjR@^v{GGyfh%^aEH&ylEfq?`;whL zWm^%rYO8oEzS`DoO{O;0*V~3|h_dGA+BrKX$~tIF+Z1KP&$kP9QIvDO)h^j3Q8xW@ zd(0kdSL{lA+#YXN?W(BH`xEU+ds37O{#1L~o^H?BGXgLAv+X&1PLyST-k!%CEdNOR zsC~43%s$pWZXXwICI5l;g1vxp*?y3X*(Vwjt9(PYA7bP7NmjKVW)t=!C_l=|Y_g%) zkD**+Q}*L*+Wr)qv7fjku~|0vj>P6Xt@Y%#YA>Q@o*hBWk*sD3HAmSo)Ex6BLGcum zuS&J!yR@e(H91j&08sNgE~96($=Z18oo>{mLSd~J)T3r6aDBoqx{-Ssq#mznN%`fV z&b_u5M6SQ$MalftX1nWq7dl}x@D8;-@Ji2*n&-J2)YtDxV5Kj=-i*){hAN(SL+`TZ z@&IEgNhxx9!;6+Xfw$g?5-ak8u)~*wUb}l6)VWLCWfxv}rKTq1;d4|ya<;$`srM6LU4*i$X^{5Ad{sQ3whJ93hDoI3577~cD%0{V;0_l6 zVrhUy?O>zkq&^lmkOig)jCfpKNz66Rje6V*6T|N`no(F&_E&-OL+B@wZ}CU)Ou?g6 zlUwWXn=WV10NI;jS>CGbO)(?5W6-VzZrj7I6&wfa?fF!;9A~rV`e}<;F8>tuT*|hs z?)kps92l35pVQ#tES{o)Nz<9KrHh(6kdce_g0BA!8$`Cfd^kh?TflxY&QO`UTXT!5xyc zq?`fXsP0Yt-prwcmR4~AcX$q9ggLhStHgv5=~CF>Ff?E!-BgOdEANIaN|qVI}Ia2B!~0aZyr%9+L8K7L+Wd|^++be^h->ZA~lw`<%`lAU%0OHjeai9U03^NKi@CJ`GJOa@^L{t z3lQ7=^3U+<{*#VE3}F`a^J#COR;z+-#S<_NvDi7UB5KFtQ6`G*M3EW}*wBNo0q=PW|hm6$=`^ zloV2LB7GOmTGUD3EvEI&fO)r)0!;-@3YD=85FJpAq!W_@{5tHV=d9Ij;2|kbNij7| z)|7%=!EMSWM5-(=lG4@6vWA+1QqAB#Tkm%?!q&vnWNidnlc9=EnlSLz3)m_&<|`BmH@`I!yK@EaDq8<}M1 zYs#huX(suEtiDVSB~cci;JKP zcou~rJsdv63+nLQ5`%e>3lmq`d%P-Q0Hge9!N65ch{k#itzOIsz*;W`SWR$>)rajvgJP3RC$Y}o+H z0>TJW5E~FrsFne&K59&;4>dN0>M$XwT1#&ku?{gcqg)HEL*)=-%WK*Se-UHs3vDgW zf08P$5TN7Yq`Ubw0$(IRY@Uj*9%2VK@B(ew{+@^p<1MQD83GOgS_prWz^D@D4b;7Z zJF^a8+LUEs87d0=hN-)Di_U4p%o5icxKqw0g^Ues!N5KX%i)m0AU1<>Z9$>Vknw=8 z*CHh=wN%=t$Qb09GN9dpyNx>RBgGZ1d{h_|_iGWcz&yXN_7xFrSi;bjqB3EvF{Vcq zX1q`4dwlpV%ccD>6F+)W1vQO6#`1gA)gkrBNGZJrdh|Hb`$RN}nkh0D8`=jlFGSO< zuubcj8O)-E=%;qbnIOrW12&GkihE)QM&r=i&!Jvx%@eh78Hy|XX>4Z3rciqeonYau z{*!O@;T4kJg=iXKr^oB0SRqQ+YO3%FAS3I&wKd;MiqPf``3fEn1!p1p_QL3;t&)h9 z5lJsxdhLrL+@`!!@1+)))9EE8M8d)#3onF|BPsMY-=NkULY)R?lxWRht&14>{u=d6iY~D(LYu#aIrzTkCx#4&>uYGqw+e!jixi76 z9nlGB2P8uZ>h&EYw1C6$fLrqf-c^}_KEH~(RAMA15%&Tgi-pt%MpW4b^t)S^lMJry(Q!T{=qpc zIZ+}w6rjG*tb?mj!%6KV>eWjZlR_6~-S*&2hQcwXgF(7sWCY9SPFP>}D2(RGc$z7C zAqYIz2S;DNBAkt+m?kPxhB*e09f^lt9o&tO+gTMkraeF0rSVPjpD6hNzv@;qLf9&p z>n)mKOrTFxd6D#jU(b{L-Xsoj6Mq|&cW{Rb0FqMH3&Mp^0F@sZxoL7T)B^a&QVeCQ zJmj5}@kSK#Jl-cc6AJl1ArizCEjdzPm{Rqm?2+q38`3hMp6QAHV%@RQH(I4u8F@+RU1UvT<6UJ9QC6(C ztX9b>qu+FtJVi|ykNPJieXyt8-95zy^(T7DWj&2$J%OF?>Jwf*Mud;tkm2L!vQi;7 z;WLi|PM}4BPfdB5&q3FSi6=CK}(d1w%F0njzDO*Rn8eqkD)io9LWzbB=WoGRt z{c>y#X5um#tqe&cD3P-XcUsb7tbAWo3`6VcJQ+&V@^CmxF@Vij*GYz(44iRj=UJ{6CT z;M4IWgP|GjCg`U|@Tquu1fPzM$7Q^uunJ~8Be1Cqn-$n}hRsEjtp~6p2;QMwXnz*s z8Pq+99l{gXY=+@EIKd{OhvMos90E28r=Yxp=;u)V6zXU0Q9q6P*?ZK_pni_(aSDGV z)X?k-=MV%kkpZ5(1CW?JvsXezH7S^+I*QUyJ;y%>k|u?`LjCMAdW~GPnzH+SDt{ed z_Ys?0?m+iC_#_d01xImxA&=Fl# zL59_2J4dhc#wz(`lJxz~IZ}tN-(7dNDvL`nz~K_K4c+0=met`+7-ZL9Y;_x3)=(D- ztTe=yN72)S)=qL^|0vrmOg6twB~lpsM&CBGltm~Kg{cqv0!VCFd?e3PKR->j?YpL@ zsE-E$C@#c~JbfUorPgmO+XfO|9)eCvjF3ts{}xN+9CjeL2IC){Ugl%eOO*hbb!3y* zU8IvMDBEK+E00H45nXsMtcIB8jp#zh?{K?x;nmBpUU9BmI=_7W(iJ<`Y~SQf1hp#E zcx^rnEpnMQov1)5o!7QC0OalfPotLI;=e&%(rMieX+>mA?D7bT5k~OSm=V50h))o9 zVXwG+pnUqUbs^8>+qlDZ07-sCvy`%`!9&s1s(eD7hEH-rSx}J?QYwmt7E?Yc!r6jc z)-{##h-F1XRDdv7F6gjH^!sl^tH3VdmMbZZky6ge^xMxZ4Vm$-s8q#lgq?aEg@`mc z>=JCkFk}#}2EqrjMhJN+co34UtXzY!E#Rdz>Gd@#og?rI1bz`d~1Bk>!tj!dMnO{r&n z^Lmt{6aZQ4&FipzCiOs$9(vJR`3y4zM%gZE%ONPOMzDe@CG2QgLO?sDm=k#}vE#^P z=6Ej%C;*Tbg?3D1SBg>-^@ht6HCT=ASgFHSKy->k0BhEf>1lPE0fe~gHzPZjX%%EY zLk9H)cV4885Jk9A1kW%V#G$NFQXFE`{{#g#BH`nOpGwz=Wm7N1m-JY_HN6)K2qmBt)pjAVj)^$uswaA=yycnmoCJG0M>G0@)g zl<1@s=sls!NZiZn*8K3S?(VEk5ytQ$TSTcLp{o$}(Dwt#dG-Zi8bJ^w9v+N>oHNzK zN8Cp48QEwgI*^V2eY7DBB(l+OX4z;2Xakepg0E*X9ks|z)7J$-^&Idbd~>63QKmS} z9YX__h9w0)C0e9GPWYIVhAMxhy!qzlXGDKme+HrLuu)){=(tos#e`c-<%L{_2!rNl+DG|@$e^i?=S$})kn zkPGL7_9w}wNe3)6F-0~4B{A|Lp64Hcnh}#PqaY#6Y{GmaRfn6D(lpWx_~s4sjUUXl zhSY2z!F5f!D!xSs#1P^DWJs#Es4~xhu22Z!lSqbq`eo#O}A)*zk8%8bh7-9;12LbP@ zgCa>#JU+0*Q%zGCh@{2`oqd!4~ zp^Xnq__GTC4!!eT0;F0F33xK?WS=hRe?(qLe}N)h^i`7iE1;L)W{|}1NxN`Ira#d% z4d#u0783YJVdzYu3+Ox4K`mh958(f$<>2!k-_r^A92yz8CuyRK1eORa5V%OdB|y1h z@g0G0Q|WC26mD?glYc}dp{YrC@qZxj6oGwfev;5f349jdW!xd9u&|VpshTHE!z`OK z=7M?M)XiD*gn3fX)lP|HOmy*Atu%@i27tKUo)G__SPVVhM22rmY$8|DlCmxd0#q&i z%VIGIxIqoyqq<63*Nd8dxJcof$XMH@v;v7*%Kveb=t)hc5l9gwyC{6maB&qMMLB&b z$G0ufOVsK38_kH9h+Z&grF&ZpBiNq;zBJz5p>B>IZd=TTrazpC2$+vkM^z#|c^~4` z4bB*mPZ0f|6F5sRjS0ioZu+pTaDqd-l>H4MO~&%~iSqA=QuoC-;lcshqNvI!MIQg> ziF^|vFQcAN0r>L6F^T0uiBd3E-%rlrGQn6>j*HTD+M8>=#3?tvnf`4rxs*;5Qfq(0eIJnupiZQyO^RnhBVeI@r1@04*$6s`z%*{-}U*cC_VJi03zr_oZiYiOyZEsxn# z#)s@_<8k!N*md;OooS4G!WM4`joE$LueF94OTG`PxLwPpdw9$Ca96v%uuX;BM&ECS z?XGXRRJ&}2)>RC3SVM_RFZxa9c#a=h?wS+E3pd(c&vh1ZyTd4Scnxm zX3*Sp?7oXf#5i+>S@xypt~KPi8ftdf=2IE{iM7@{p0(woADXp6S=x(;EJZRK^rP~H?2 z!BBAkYOG&+@#?jgEh6KX(;Upq7b2M%8>-B*uo9rzPJWf zYS8aFjLl$ptea-rZ-=HC>+4o?YaJUF%T0F!yPyP)yTRr$iYT++J&4J5tT!986h$T@I zcdDb;S{^N=DN%3wqcXkUC0xN0NF)sLR(sf~d0~LJ8;ZOp@fwM$ycX*lINUySf>?9A zt#%kRq=VN$`55@b;$24jm5`IvlG*68I~KD~p)y`#N!+Q7mzWN?uhF*omgivGa;AwF zh76%xFwI+i%T0Rt>#|3Qb1~hwrsKM%d1&56S^-VY4})p}fsrm2RNm4=W#UJBLBAhG za1YMdR(&Qk})RW3CRg>N_e zFpd2&A|y+Uzc}B2{`SP=30#2=l1K(*qc&dn!-Ye|SbB>~xPl8H6T$(ZPlba-MGmD7 zDT3URY{{12k_HO;l}LdoDP!SL*Vq#H?86R_yvugmV~?QRP!C=pR>Ll9ZbD0}FnL8a zc}0?&SlW0B*Q%jF27^N<&0q6Tj9A7MaC-y=D2X*(2THKIoQfDbjbVo#KM)k!^v8(@ zy*UI$9adY8P8_XlR+() z_Ix8LqbH9Zo#SVg1iw*Oi#0%s55rYxcKzEZnBjKMxw-!(yh5DeDC)A-z3FdV`RAS2 zpN(_tR@mG$gZ7RS7w$Oi)@EqhPSe_M{m=dX{HJT{7oLqZukD+7=9#aY`IoP5KYzZF zOB9RLfn|xUZepH0UAD!Zp|+0`$rFJeBd~KIu>u=I2uG;K&J%fva?#TP>8>Caq=Hxg z^e*C_6LZpnxFGLT(t|osG*W@lkIIC^mv99IkVxpz?!s=3AVG>tX4r*gPb>tmw`6Yl z-8=rM-Vl=&9s%8)wGleZ46R`6DsiW21gQRKJqO(r;8n#al6Jl#cws^$4TU{nAhm>n zf=dlWvVdQ+MIn?UaaX)7y!Po=rGYlcM44CRfj-C%a#41u;F)Zc%w_^$KKA!QZYd3V!7&A^4dyu(tH~Ib5OG$wbNsW_V3aYig(u zA#iKUS8gRAHT+QtT?wVm#v^b z&iGxk1cU5>$5X7R`PqX*7w`D`VKrSWhWAZIm2u-LY-Fgz2_}(8%%GcsUl^vC35~7EN}8_^Z4ZFEn~6&LwV3@+|zgu$w%a zPuknQ?c9xXv=n$lRK{yy&>;Zk9zqT-ANJBQH<~YDqTopoLDCgPRHU3(!KI5j)T$&d z6SylSQ9(;is-^TmtPdDXh&72cSv4-!q}>q-?&*QT9YY~fE(^ON#8@2w^o9_WJ#i>P zgwZAj2U`UTYrKVp1(K8nj2jG)N0N{8|BClt{Ysb?KG@Z6^JMuD(Dw zDVCPcuvbugU?V!oUphhb+PUMr^@8iyA*__S4j^`OY0oc65!|G1ma2 zQrAL<#jU{NKCsk@b0F-ct>WkKDhdQEk&LsrFi-&#D%2BOHUwBe8DV%L4eAMK8N$0j ziw^jZBb`Hs4n-y>Z=CW!7V(%;h(qDp70H0Eh>GU2)zUQ9FfVK2#+5 zCo>Q17M;_CQ75S-w?2_v0%UAp2j=`~SPqj628kI=c?W*nruJYojy(M_v@4wj8qGB~zs3k#QZWVWEp`2+dxMYM8^HS{+=Xgt zLATGE1gub{n+=(J2+)zu{>Fyu#Cf2(NuGnl0xnqy)}0`HVW-GLYXsg)S6=>1&^U&u zuGvp4FuQ?f&G;MnR}K4NLcGQ^TQG(PW4Oi7CSd0M8KR^|_PnV8Guz)W74>^iCDE?fqhzl=4wuH(j<$r-`p;;|b1d9E*aXn3+fK5O~{#zHb}TAi5p zi0;JI_CZ!7ndD6{yyZ zMFkQ=PP5QS^aL6MNtD=-B#9hoTSo12NVr2IGkXHL(7+y{leN{il!4+uF&?X7tc$U^ z(b)0G4D>}oPrg&&I-rmfjozZ2MGv%hSb_#h=u`KQ8AH8Hi3k5ljFIge_@%R$R-z4Lwg{rj`9Bf8h6bW-1oC6{rR z7~ZgLdGLl`$D>J0o+lzqr`K(6LT%KWrf1!aGq5}++;6UDjj7Zz3E{B7Nx$80I;=+= zxIazU7RxuTT#j=+2$bi*V+|5t{1eorK8HD9i3=_}NGQeCByIJa?>m+YeYtu)ao6)n zz9kXIpbB#_W8d++;#HKD2D>C?QT0n@XbZ{E-jBho9z(!G7Zg+ z^I1ovlnv1RE4YG15J4)zRVj%laKZbB6jB^eL1dswJEf64Cc-=c@-#8&E(iH#IT6B+ z7)nD39}ZmEqu?6hHDFeOmn{wCff%Z`8X<~9Y^TAIAu!fRh!EjjwzCi}YSAH-2-A5f zw+jb{N`QS5*#clxHH%4Krvw;=AiaUaY^1y)Z9vo`wNvbr%<`BMqPa0A692=Tv+Q%11St z8H!5xsqjQJwJYKcr=qHzn-%cP$*8z14yHpjDt4C0)Y-YHWIhs=qUmS~+-180Sec2+ z6ZJYGmBanB(F~kf{^qpf7+;^L*Q42q`fPYAD&iU0uHwz+c&(n+=6P*4tu0{J9*wYf zozt{$xIczHtl3jY_Rx+#8qKABxT7|o)^Hyp^&LJQRd?Z<*fVfXu=6m|lkIi1&po7l z7VYy7X`e&;0=47ZzRMwPel79wy3irA*ju4Qc;fV)IKN`|pE*?@ioBDR9%ZwgFeM-VYoYdfP%KzR>oOoHmoB zIQvD6HgZfi8*Ul}Br6<0tu;g=Lyxl7I=PdA@NV}4Qr)WM_BO4Z%JK^QfnWvj zL|3q~Q|Pib`~%Bf?(|wag%No1T5(icovfavYU2sU%`wFj+=F1ZsdRw|H|rdWDQV5Y z+R)Q1Ou+qO0}cpM|4G=9_$&)F^uS%F#2HTmQ>=4G@)KC$?eJ39?J}cy>7}bL zT{o{^xwv}q%5|emOLtg!1F@0w+@2gS4On>2YTk)!nZSC7=s8W|iYK4r9!*Mp;kxxH9MmC~A?p@1NV33?W- z_FeWFnq5m+vR@|tbe7uzy#U!Jqcl;)bQATnSRTHn@J}g5Zd_bFR6cvuWc>nW`CVKA zxt)S^TFyxd43;9-;X0j4sI(+=(WDH=>2{$T3x zC-X5Xrx-gY(%(TMY$U1=gwQa1i2F#t@~D8AvI4PqTKU5L*q(&5T4t=)v} zrp#~RReYUFln-G)N95;0M&Bkfqom&n>fwavkLi&z6hHksL3zA`p!4`9Mn4i+ zig39Z{|fjeng5PNCYX!@(r0p{Qql~@;frb74CZ~Pl5b~A0~i{#B84_it_@Mo!{lp& z0;RNuWOg%n`{Jl13A~4L?u9hxUgF8H5`isJT=FnS+@ta^&+qcE!0$R*kP3TR80aC4 ze}t46B0Pz&suE?R3cSlAJj%)*V)0=OQTY@fjkF%3ESM}jnY(on5jp1v(65KHJkgh# z7NXoS0a0NVCZZZ;>|7ltdrLw*cg`;GIBp)TDu5h*Hg`~T@Wc)s6Qb&#I;i;{h3|^E z4e_3Yx0tev|JPH|)Sd=&s}7GvH8SeA7Q)5h@ubYffdDWhd4D(Fv_E_Un9Z+{CO&IvF}i6Ec>ze zLR@SztjmN;&$_+sI7=DCBss}HPSVFYe3cBH9)eE*R+0gc$(@&+Tgdm_?ODEU;=9YI zz#3TE_ZY|M7paPPwA=S>R-$P`BIM!6Rg^`%a+i@8QO+*1fleO_#8~G+i zqbZ5hm_F)!f0??zL5y@nrY&9>0Yb!bWBMgD-Y?BTj#*puz{-aV33F(; zoMKBg#TJ>61(>5p)nO^cm~y8AE(&7g1Y>Hzm>TeAu)m`be^2Beh|LcDoVgX`T)P|`;0PTT+ZT3($){#Zm>)qEl)BU#iRkbHA<>5oyHSjC4N{QB_q#W z>0o&s-?3N$E%+S4xrE)WyVVX^k;an*!iR2q-ts|`rA&>L-Zr0`8!GaJd5pk*mHHNs z_ARIDq5mmjRbpHtLh*xf`U3{@^^Js)#~ZPBji)jB|A-|2hVc%u@rlqLA37j@RCGYG peh!@Qk1FR$vx5(Vq@o5la`ZQc5)wfWC|#VE3vw-AD?`+z{{_;=i3Gv^jvm!cXn61x_E4)z!-a_n3NriEZMSb#X(+-B%Df@T}^MFp4pl0 z>6ty@M4i z$g8%ByAr4!&DLb9QvCD=*ojH3>l;?x_PSvi8v=uCL7VX8(lD&j{!9K={_Ht8VrEkgh3M<>k*^GUH zRqT^UpJD|z+f?jFkgl;g`%yM;KgO!|k6n@20$aQ*u|>bSquQsLd|j$9?cyz^wYr=t zVF;@QU6;{4-eMix^=>b2QNn8ULoaT1LpPw@WjA)OqN&g8T3UW3^tj*g!`Kbh{5V~? z-skG$Fs;?^bZhTg_Q8x*t7ueV|hg+~=GxsiX>cX^0sDQPKodDD-d z>W2PCH%^POA4XmNRM_wIws(I5@ABcD>i4ev)8OvKy1H9Me)l$+_hFjv{U-hW)x{70 z=l*~E!>jAhU)+6->Ru-EC2H{Gizoi&hfhC#ao0r7!*%FSo#Y~hgfH1LlWgT}sUb6& zDR)hHh03(MlC3eF8Mx~#$4uM}mS-03IeeKS?j|d-GVXac!z#F2Y?jU8USRXAihJ>G zNt5gnzRp6Lo4mQF9GD8FX)<(0e+d&yJJLu_?>~CS?-0 zV-;zAM@b|{V+O2_d~O+T+?V;aEFCJ$M2!*F;@o~~r1Hzseh+GaGHcTJoh*&M1=N_x z6(=>{&C;kTp(dZmuojc>g5|YTk32W<)1uc6Z-##C#M?c8aPm48*0{gbZ*f1WW!ZJt z+g!(oPYslz?G3(!mrrx8uoXK2=M~CX>u&6AI8keWL2vo3=0@x=-*dM)c|&S+S|Qpt zpZ&_YAHK8w(uH~fGwAnxo|=v$26EE*wQl?hc`&)O&tBrZ%Xyx9n+_g(tIId}DP;Lc zG6gb^kg36>8vHYYqiT^KH24`5-Ivq6<9LA^MUHb{`smYVJ7Ih_=tnX4&$3&Cjc)&J z&uax|J6+}n(b>2k`YeKXJ7Q^8d)xdm4D%^m(F%+t7uBL{Dn+>{8}bRle?u7>!(5VkLmisK{Lo7BBMp7>i6!n$Sq>>gww5$M~yP zQ_a8W2B{u*xwnz#!miWgE=%)1DADI`?5CR7XKre^y`CSkRP}m&7&ivv_fn%Dwzm2{ zXqCGF1_P}}U~nUF*Wrq(-s7z>rcuLz&%0?ZLSrAu(mK8y^3#}Gsx-XR$bgn=G(de; zIEh^lW738Kbm_vm1s?3O-&yw=!)rJ}D~fGPJWZrixs~by!Kv)_cb8s2kwQ1k?hq?no}YlVa-$^&Ec%$p=>AJgueI?es2qsQxcp(%{S#z5O1Rl2&Q;e9#3iM;DNowy&$H;fEfW zD>axOyYRu%UI)j$*{qww8?x__IRn3n@!?B;6hTn%&mkv1HUB)78=-#-4?8-?{NAW8 zT1B+B5<%V)P7+sG!u}yjNB})Ib8E&>>fvYe-`Chdj+%B1ab>&yTsxXHwEq0x?%0k z@X0$v&^GZdplL+iKKF=N0i_#tRR|1VWTW3`Kpqvq<_-xDpJ!VF6uKY9FAqvWPeZHJ zuDtdOQT;fihS$%wV*Vr^nwFrJ#m+1w5hs!pyv=Q@%>~_N6EjM+R@msKMx)C+*cuAd zX|fscpP`0^3(4N}IWc8g6x`ZzLl-4HM?(rdM973B7hB1M_X{*lXOdYySmoeZjv!`e zs#Fd9&@MY2iVpBLPP5mK_(d9-h%PN&2AjWzIRt?pq=q8~f|%>Fe&E;j3@>eJo=s3Z zb@BrCO7_@E_Rw>WU?-hz?T0eiL;4srIt?RX6R6@^R8By48Az|lRivv*MetCSm}q&5 ztp~>~z6!sErs6&BhQ2&>iKDir{doIx(YwL1cU?%x?`1+3=?c8LvY?KcWE6}JP0Dl-I z(c2cwZz}v(VyG+{Z40SvTxcMR748yCNTh#gm)ilaqNTx#-_Tg;n~K0zk(D3PE{AqG z(QYPdhg~qYsbVdzi4n7Rt!1;Ld7$PT%>w$aJeEj%y(@2>LtBki5P{5!b^KC1(Nzlo;EbY$M;%SMeFy`dVbn^uk8ZZ#zalEy^x4~fGtuy)LvvusDUCJErGuj z)aA!G7(o8m7)mmv)Ux0xiAls4f)$Rfi7mobZ?&<8(> zDg@a&iN)Z)!hat5RPG&xMgDx#5B=Lce&O3_z)5oq9-X>+hLZ#0VA}R0`H8_?O#~DN zro8&e6Fq@(G0`ebv<>8AkAWUKI$azBOanuqb$Lv4BAgzXHot|k@8A*viH4o8CYa#x z60W9samayyk*8|ZWX-?-KK=bGP7qSF)1@G=-$@k&m=xt9z+-NwuJ9o}P?Quz#OCkGvIfi~lD0;Ky`P`$P_T``C@9;fhw^Uw0)=Q^(&XN2vtl{e*x3}LRt(sdB5)82MvJL&AS-f}t%aMW{ujPN9^7Wt3@y@?j@_sA~!l^f(?D!#UfK2s?^E z5)rcyJ8)AH2$1h2gao*iwHtMV6U~dS%-^9D?IQdwGCMGEAyRn=t|mtcM`Rpz`D$) zBb~^LDMOQ~C`G8f7leNV;nz0PM1Z{Tuw@3 z1);DyvS{>VW>gS&byO606D8O(o{>Pb<@QX1&=~?y5wof!`J^(c;3>07Well4nuBPc z7p)QgLxh!fAW(aQMo6L=k;XF2%6UjV@*sYcLX}|6j^R&}5#IMErS@Dfm0wOv9>)_LkR0#! zw$nVt`PmVIN#||1T+VQo75hC%#Mtj-af(U=r}$fl%5V4FkU4I|f0~AVn@mV%sv5if zknu7~(piTh%sxU%Ot;TnZ<{x%C2=8WkPh(#g{Ct&f&fuQ9&h3F84;pS%^EQE>QSB{ zzTjV_5lm>wrnByH5#pwLHWt=DLZF#~S~>TuL(z;=Q;15W#8i$l(>-+kJ6sW+fJtDRIdBdg7%YNoz*EFT_%9+BJAu3@ zuN3G&K?7SUXsO7`N19nw2J?H2BvwQ`{t%#PH{S#FdlNv%P*l)Z2caAb&>V!a3@8(n z4np~Fk^jFy89HF#q}K%0Nt=}!4$db~p99qA0Q1sl4)7zW1LmXZ9@H21pne`u7vsf??WOpbfVwe( zI*#7~^?86gzbG+FD7#-2S`74tlNDuHWrY#qwDxhd(ExmCw|HR!{Q0B`;4g$VR{Z2P zV|}vJL8S_kQzMw|Ko!R z`uR671`cM3N}^AMzX%ZdZ_$IlN#?i7{0^BBnK7Bqkogvw-zD>VWFCTa{%xutnDgH! zvj=ScUCQnQc<~bf%bFl=xr3q@~F(5S||dj&ookew><+y5Y+(XK;zrL@-s=RL#?- zVV2D$^PH)hPl^`xGvaHCkO99;5Y1;!LR*?WFOH#CBcHb*Ce~Wd6CAS zbc(y(U=yKLi8=|-{0^l~Ql}Zgt({hY%@aQ;MRrLPKpTsnifkMMU^hDS>(QT6KRpm9 z>ehjt1yMBliAkJ-klf}j8CuZ;OI@cdEp=+1&pzx0(6Yl9X zm9|w)5FQbNz-!CG5Lm|0V+aRJ5QK36fdfJq!hivp!(67!X?`d+;0FW|{E$tE?`3uO z9J_J^{9`8SRc6-9tjw(UzISAM#iAwQ?~R}QdgGT)OVYnkW&fXs%6+(^e*$3=lLN`g z?y{qZy6UL7D}mb498IQYYM^%v#}H*L$aQi~PLy@cI~AwWsXEn8&8dm@d@$3Qb!J7`3g$ZV&U|OVSwOwuEHc|!+K^cBGqSVHO3o2h zc2-!$Ig0WzR$$c)#kmgURaSGZXEV+XY}UE)5sA&Q`4=TN@6Yb2&P_}{E7cdSV3pE( zT~3uS1l5AB$LJn!vkvZhw->jlU^V+;BW`s=FQD2PFZLb>tIz9NT6%xj;C{ysV=q|u z<8=9KtJ4eo2fI-#^bb9|?umX7w;teL*x0-#gPnDLx)oz6ysGek7x|C-9uM&@B`wAt z-|*v;-O%6c#;G0qVbtX(!+xiC@ya`~mN#CQ{pCmgdGO-Bb@j?D>Q|m8@{1rFuYQvL zzHsjw-@5!CUwz{IefM5@lAb+G)tCSY7JZm{?u_TdJzR*a%3hs z%1csHW-?P=G$9o#(_WMujp@w5U1vFF;%=}!vvAMh%h#$QIZl>Ls?smT@n$BWwlt%1e?aIaPeUqiJruGYw_mRH;mpr7QYXkXYK0 zMsgzW$`BvinX)8Zkod!~g0i}!B$A7%gKDVjOAzV4%ui?KP+~W)%TaxyA z5C_`Kao<5!M&CT1;(PoR$EJkA%n1sa@*HeT0Q3!#u8{O~% z3T}L{=bz&XSW=p6g{{~PIImF6I`74eO*d){{M3HVZ*6SGF7q4SMNTr18l6@M-i>=c za^f2wz4*}SG}q(ZHiKl_sUG!vK2J^86(hPlPd!W*?|ZJxxA;+1`3jK&kz+)z14%VV zV#HUesP-Bjgj0+BpviAS%VjyuyKW=!qR4eGOK;wJyc5R9gMJiq|2TVYu-WY&?=@P% z@lKcdL3BLshdzrS(Qef60^h}WYrTv71`PEMT+uR!B-@HD+p;0A;BLv5vMev7W+{Wp zSPa%Ci)hHyVf;sBl8Fa#MK(wxwP~Lrz-KYjb+NV%7fZuHhYg71gn$3Dhi}wVm?4kDN#n!t~69Oq@jjO zkL8iFD}9W~QmiKOj(kRX_OWM_p)t%Qxo6a&Im{2OBtO#7C!bj2ZY4^R-+?MFNy9>- zjr622GIk}B_(Vs|n$)&$_&&(DhW{N^mcq*IQuN^yk}Uo|02<5fT%s|Bsh5QHkl`D<@h~7PBgh%1l8^ zC8=PPO1s*wwP%vjDQWw)M1jN`>#5oFy|@pt=k(DaMJj}JU_a&e0db1(K&t+^Ledw0f@x74Wh`FUovymE^s-+qY zP_GIhNeg03G9*wdT?n|)iL~T*&ijmE8E(*u;?xpv%gS;q)rAhHvdbT$@pbVO3z~FF ztxTV?zEI$}oAoVZ?X8gc&!-km1*%MCECWneNi7$<>ATIw`|zUZRuD-sWexw9T)}1H z-sl4MwurHpSP6VJ=vqza z8fG?H&M+W*@2~(=Ig&tg1zr^5g{074*&%!=jE)MuoJnTz##;MW`?{oh3ai?FOz35K zogc?16)MS3pmgQSAZc!*HD;sL^!Uk>cU)oAUhH`g@MF{`AH2UV zJ2o)2+v&l53F8ROcIE^<4o}JBgSn}quSOj&2v(a7XgoiOVe6~>-MH~Ph}=o!E+Tgm zc@L5ABtkIkm_i?b148Zi&D1s*Z3W~+t-&hN`=b^)bKId$t01Y`Y{jWA{3xeL_BL7z z#H1qr1N1y6Ce&>3_fnTrL`Zu$d}xW^OXNNx_k*N)AV3&3ySxLxk#{?;7#5ax(}Ssf zfcVZ3Aw93pW~PO_F#ZS;vY-4>BJU^i7?JNH@;DLd!=E6sPDCt#Rw!o4zneP-x&{&`6C^Nz2PQ5T1b(j-gle&Ju^piy<(@T305iV^&xt_I$a01EXeq7ns&Sskf6Hl@PUqD$qk!|&% zJ+n>`@{;tNei@wqiAx&XJXvGE2X!D!T{{(Y;lo6yQJC=8)zpAOz}HC)=Eoi^R&mE`ccd-qP>;y$!dW0LG`Ms z9{3JlF$^pMlZv@IK4% zCZ}(~JXEw8`&+hG;>xHBJ(I;<-Ibe}$0jTT{HE?kD~_&^({>IKvAo4^M={L>6s^M5 zF%YE2KC9P+@01}q;diRU@U9T5Txd1?eb|08o^h9mfc0mo^dTYw?>~qVLiBELD}#Q5 zPMDYQ!iimd99^YqE;*oXBMd)-*GD&iNJ4z(fs7VJh>-ABRg>+vG*$U;O|Qwb%AOKVdmq3n z9fM4j@SUg#k&xO@If#!uM5qeLlkp)ST1+TtqS2?|-vQ>j%7?hY1I3=Z1PG+lMGqo&)+pi!pFz$WDGB z*49P@em8W(C2b)X#**5Q?FH0bp%aK!q&6fBv4}CsF$Fq6oYV&SgLKO6d~87vrmc`@ z$n+%@+e5LuEu!jTT%uaL%=Eazj7t>XS0{bRtE2IliGS*;f}7r6X89?1ZNh!vO&Pxi zevEiv^qE+AY!roUY48cHc%E70VZ{rhMeq{;+%C?8`e3Bec~qYYnpPN^*-m5vvBXyG{;lBUaFq9N@p zFdLh~AEc(G{Y@_30i8y8LkSW!@I$BMb||-oRk$0yev~<79ZJB?7aT{5A%SZUT`21_TK6$_i2z z2FlBTgB4sAVV`HoLN85h&Q;?a00~+XZI{q1vXaPYL7!k>r9d9av69)Ak-kg10v&=R zAxR7EL3|!Ven}>UOzG2d zjoPv(*n%yF+d~MMYJy8^cUz&@{JN6i&8INM!&_Cz#ue}U8$2Hai-FBJrVeH%8?le; zfU{CWm+2PVf(@Z1BE=`QGl+n4LkX3a6>cyB0gX%@Qhpa=0vD-mkV~?o@Yk~%nL0Q+ zw~&b*>Ieh13o<{$^zq#o-*buXTH9yh0yAj~!>7OPXeY*_Wutr!X+#?JbATR z^9UthiIJ|M(Y8^_#*J+e3?hNDD=`Z*ECZ4Xm$ab~SDDRs;4=kZb<|J+HiM)OQalJfVPoJ}=Xa6eM}3Z*8o@QY zMB(md*&IY}9xl2REDWu9QNWorT87A3?IWWV@XoXOvB(wJg4nMQg6}9>gd-6_;$hN0ur%Pu5{Z6VNElYXQtF%@nNvCTb1c69t5{NvF zWo$!#Cx#p+55Q_axUBF7SW~%oSb_4VHvG_kzJ~x5mX-V98eBhB8-ZE-B9RI2oARxL z+N#KP?px5>+fQ^Qo9HY}v@LP?9^~L2o-WQMxWCgDTwh;;!T5NSFs&n8srhLWjVZoiDJ~+=y-Uafjl)3Z7@srXH|^3_tEP&WTuZk zB#(*wC=o*W)N0|(2ap&xd?z=~+H+xpj3Hhth{Zx%0YIeXOwva_wGM-kmY|8y1vMI=3Y&`x{TFBD+naMkfDPd{- z2JCVl&g*$`4mC~nze+O^XnhBjewxUqiTn(apCv*PoN8PC#V9S)=D2(KpLzA?!B8(| z!2faTLAIN}BD&Gik(8vRr+IO*hVu&U*jesfv`5Z-RudD(*CpEn{wU2)mzhoR5gJm( zOb(0KPvF5n;)*DUm6Vcd!x^u@;x5XjYN=GJ(6IsjO@&gM3Vd1RTZXxacBDSvGL?!l z4?a_|6=9H&&amj{0W||WBEezW@}P83->#`e57JENv>+yQf@`<~KYr}?3MUt3atx8y z0HVUDM{umHpQ{_3!pkff^y^f*k;qi$#hUqnAw8omlzEHD0msP+{`Ybf-&P75 zB!!c|Srn!r?vG%+7zYHHtwJHHusu4CD0B*NUzVS3!s5{(1*{10s>)UXww10h^$Wzt1nJU2%PDOgnh9Q+-e9?OXBFEloS#7lo6gH9ELq8j|$?h zQaFsdi546@+>OvUF15=E4j|z8D`HlaB%f496}+XIRCWXD+YcbMA>I%p9gD;%9kk?TVd;)Y2p{sWnN`?dYaVCQ@Hu4uD6@sb zzl2tOL5i0X6XU~DACZ#ku0E`VC$PFBv`-?En_oKvcsGhxU$|%O-Az0@Cyq)eTBN3v&EcDm{@?mxD2h+_jZ{C%De6u2 zEIc46&CUA0nPip#9#H2Oh|ssC-zh}mmtyv{c8?q7twOh^dlWE+D4du3!|(q-79O1H{w- zFU3)fR22{dcp1%3A!cp@G25pAC}O;LzP%7H3P3T&fPx=Q08nNC9eDtXB>?3^fD6hO zWNc`jsb42_ng0RG$a)i?L~pCiBD=u9Onv{5$R82; zVzXc&z&2mm)w zKTM^cA~GUEei;|84B-$b7cDK0e63Hs3m^C0*Lk=`IiJu@YbKDB_#)1ePqP6q*bpf~0f@2>s zO5E)RTL?iZkCDmjSE)oxbjkukJFNgl64|ZDDT)R-`q?jfeuX&yfjIR*{LsPL$6F9h zV@{Dyqd-*T)WrVm+hSoHXp2+xRQ8cilO{(eK*pja{Ko98ZxQ8+MZzS0DS*88xI_Y{ F{tu_*3#I@7 literal 0 HcmV?d00001 diff --git a/models/__pycache__/tunedshot_scale_randomdrop.cpython-39.pyc b/models/__pycache__/tunedshot_scale_randomdrop.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d6742daa47066d3a8c30f5287cb315ff68d4400 GIT binary patch literal 10262 zcmdT~TaX;rS?>Gv^z_Wm%+Bs=S1W0Ki!;VvDT=WZTag`EmK-B5#uA1!Hl5a-p52+9 z>6tyc?bo8fFS`KF5wnBQ~}|^Y(Y^)QN@E2 z%6$Lnnc0ivRKXi7_21{-=kovGe>vmj^Ol4^>$Pvy?tEC1{+Tkn|1@MC#uNS{2$PuH zlAQD{JBrAwj*7R^QrnuN$yBGd^tRy`BCWMD?TnKVX&r4$#}sL!m2F#2PNXv}yPbFP zB5k$`?I~xfU37}=l2dAzow6v;wx-(^ry|l;YoGAs`r9g)aV3Wi*l#E$UBFoKCfy?;X^@<`)xmn zyw-{zB@5>p?QY9|q!Ts*e_!2^Cwi@@af*9EZSAHF?6mXKjR-^GQ@K-K=%4dF9^hL_ zl8-#T>PN>rfxp&?5|D?_Peo`x1O(j@$5hKuRc^&w=2kRKS$(OK~`V+ zEd71yp|`&K>VJIWiHi?EwEZO2ogwl9HMr-I#eevgBax|th18<#Wn2EQ+vdqFegOjoGHd&q(@XoR+R>a$4C052e z$EH~YZ#z9FC(q{C0pts8o-N=##SXGXyo(=`G|4I9^bRGN(azMAT~nnrO_rYUb&yEf zl!kIFZ^>XEyqPjDU6S~jNI_cNRAR}+)Il}m^?9&#Pv$4nbf7Q;HF~fV89S|!%FIhU zJ*WlB%u(NdnnvF&YBI=Kqnek~G-`6FF=H7ti!xh`W-%Hw$0RH+zmgc-4+AiiUF!sw zkZ_|bUH<}~!;+FrBWOf!i}NDotczY$TXVxk-%sqzeq(hla+zQAu5jXk#Ar7Hv|WAR zlgGaOsVk43Ofp^GX)+{5PT`{7|v(&?M@x99(zRnLJ%NL2{h#V$z1SHYG zi6LL2q}r`{U`{RcTXlXrN?w(dtn1cVUKqOWtJ3uoN83SkwABkE?jL2B`)i%v(Qd8L zI@<0qzZD*hdV$YEaI_oNyq51mK3J#Cc&EGEy~1zBhy^~=@&=busmK%O{NZ`KS~p8Jc1{*L1L*%+YLTGj|s0}0R=beK#0?c`yuk{LFaNX z&R1prF7)Ho3AqcY#PaHq&)vui*B?MW6uc<+%j4PXRM!Yy$A2XxNYKQ50G?8;45ck; zpsY#*4UZnlLuE_)NhV8?8q1sVY3bR=pHT+JAQNYvQ3vKAJFw#HP(z<=Y>BrOD{*!c z5;-poaq6yGC-KXFWw#lH`OMshO~YfNG4 zHD#cQoIc9wkvRlgH?3xFOCH#mX+E|ww^M^cWJ8P!n+km`F03oV{FXEys zwZupzF41U!YDq9jk`rT+K7n-Ug3pC$Bn7{H(Ps?Ha9fQqN-XiUG%Yt0U5IibyZlia zUl+Al(701#rDB!#g&aqnv~MmgZv@PLF0p7TkZ4L{8EA9^*K)C&zFV(-03QnP29Xp~ z*6`1gi+D`@v*m)kjHjR$P;R2MOjIfF;_sb-_VD*K#+&f>8~=~^Tc`pPyqOisz*txK z8!SU`9aJWzCQ834;GJ3mxO>t|GJibE4Qo>^N^Hv>D$H4DS3*cK9K6TCNzFGBqc z?ARWbwn);-u^mmv_NFZ4#>RInNLsE{Y0nH`BgJ{@gU8%dNXAh=i?7U~ym*#S>WD#z%y77FPIC9IOybehjJYuYn|)vFw<&M&0AbkKen^s65|k z2aSt9tev*U*Bf_(4Yo<9VcFQJ{c{dRlE?cq6X9M8+g_`+RIfqG`EhEy#NUe-zn94O z6SL#3=ja)pXCQvCvfUhCECEkD&_8Z4=Z zKSa&dmLE8IQoG@DOH452KSmW9F}Zq;pP->haQI0g4-@$y2=wWi2Zefy3QiLtF|JOh znuBaDewGMnN&Xm-4-t8s$PW-XM}+$DCy19BGR34J=pRO-kUnq|L086tZa~~kJceQ$MF9XJhg5|?KplaAS|;TS+CMN>bo)QI z2M!1dbUZ@c(eA}ESoZdTCd@}E)=o>CGSr(sfW0+@0@;#>GE|$s0n2QtK%p41vZW3( z!a7n{_ULv}xujcZ7eX?5Rv{Iy>Hj76SUK{ zPP=>Zr_l00@ksqU#%t_#!RZ8=%O_eL*ec;kB*yKlYGQ!bVbdfA^CJ%`DL?7pdY4wK zW~v+nJg2%roF@3n4@0;YDGyAJKQRLTGCsEdWV4e|U&ic0+FJ>SRt9eOOE*k)&pBk# zl(tHo!!t;bxe=>4i-!j~%m*Eyfh`-thyZB7=8X+N1Wb;hB7ngJys5D%0F9teYwArS z)}g-5DAPpk*l38ck_w3sVVcd3xCl{)Bw+{F(LoqplRZh z*q_AYW-`E=sCWfWSO$?uhe3A{3q!|&<@+-?>=vEV9)H3@N7^;9WB-Cg1TR8>rPLHA znv6YMg>5-d2I|CA`yzN*gPEoYj@F0Z;wBkFdSnbUFzZeNY{@G@8Tli$!xLCG1uSEC zVQ68enM(CH7;K-|W3U-bD~6JSDk29AF4#4lY3P@QS($^GXyZ&_j)`+qQDW>a*|+vHC6d6f|&8#;Jg>n2^#~qjTZkhsITve-tm+nb>Q0wDxqKJ=zsL6`MZV z@aL(aiP^het#&0bgSIbHH!M@Ek{p?@pvMSSNC#-UU}e%~kC4z}3_K~kL&e8fTN8qE zKLo=?u1Q!-HV0U64tZBanmiP#33h`W1hHVskRSz4G0MP<5D~eVjV!S1q!dvGIP2F` zY!5l^ws4{IQGs&JDW*q7us%8SrEy=f-Do^!;ve-?(VD(K#j=yu<#FphUrO6+Xiwt1 zd-UmuY{41wwbnIQepWQgECPjSZg>DS)P81*W{A{q9=Q@8vZ`;L z)LU*vh}DRL;3_P2f~DsN_dGv;U^&pd)l!jl!T1WrytD!2jFW+3vbm`$QJ zf_f)0>K)#OE0v>8BLh&#CnvoktfRyhj!4@Jyj7p`43&#{ew@a$Q@?AZ2w$M0`Q1e> zJ^*2c_kw5y_fYQ?+%`qfumpFt+Y3`@A@!fwr{VHE2|KUV@>_`k;m1B9^wD9x)^dn!zpyBNx63nIwo>^rP5eWn71>nFUo}$ps z71Gc1V~umeID626)l=7)B34 zW1@*xTC>v##O7C(6lh4v?#Gi{I=V-@9qot#~hF~F$`Wi3?qr^JYaTcE9XCg#nXtWKa z(s3iZNuC|zBU=))keUJ(<*sQ1BPubQ<*%lzMJ*Y|oAMxoF*2<1IgL$yUJ*!BV#S-Z zGttfz?MkC|wurF6ti}$N5r~{dOa~F1VTF_n%&5l?u$L7R-ok6=p>;O!iT%2R`AAnEE7C`Al4kMjm z2d`0tX#ZJC!z02U&N1PM20JrRLhR3QNnkcun($LF_MsKQ)Q)raTpimV(p(U zZW*}0(Q6?7CZcKl3+TpwfyiDcL0$L*`7xIEWENb6xTPA z3>&L$58pH3Hv_r)71A?HV$7XGuildCw}_V4Bk~C%gyxCWzzq(dFR1xWW)y?xLf05i z2(6M718@-oaE`f&|1$Na7y>7YC{fR!J)Ky)>gfm)s1OA2cgN`yH3X~KEe^H?FG6GS zq?86oJ{TZCf*{1|xu=p`8l+EE+aXK|OY7C3e0#7?FTzB}CBQgv|5=)ZKxGj)`5a|` zmdNLc{2USD;AD!n#oa;p)JA8hs@f^Ef1299K;$zZiAIY?RFM{)WW@~`?j5*er@?bk z9y+sWPD~Z&O4PHx(P9eec2~ zB7!3rdz-M#M~190TahMH4_+Lg5>`6ACS~P9)!^hcrcs{Xpww+dCQLR`ng}hAZdvwi z0Cl;S1dB{++<4&MP6*0(4s!mr6FUil= zp~~oTVPsVRWR-!*RFvYPVdB^RI`TRAwZ_1v!2gicN(QUXk5ll2sYb%SAnYl8(1N%! zD8zYqGE=cSw1lFW8s@}XB~KIz!HlQKL%m-bm{FmLkf!ir6)~$~oQ;daBEC|Ji(BxJ zhGqD()1oylKVVMhH{ojFLS(c$hByX41swjbjZ=tNh%ZDl&Dm&9T##fcP+jKZp2N|9SF0<4^bV4GkE7a=BY^%{tehTC+` zqR;jFF2G@fcX0R8qlBs$8mW4SlOLPtBF36BJ^A=4)4xcm3FjyBnIvE1n3)UpieSQ(Bukeo>Aj745fPuu z9%l&DMt&E5M&!3sb5w=SNECkqS0T@JaXsjIVH)`TH5%U|@*oIBdON*<@dA~6iU6z+Wxj0kJIfgAr$P_68TzAw>Gzez(OO3+z#FM2!; zE$Hc>2tR}f;w4ZNocl18aVLel#5A%SQ|AE@I$QdgK`5@5aOs(31w_(R_%$)Sa7-w> z4-{TU?fVH7L3NU9ucz4-#{9{Yyy-py>Ag3 zY_Bkjm;wI->idU8zDne4ME;1#9}~Hk$e$2-lgQVJhzpy&a?AgeDumGfIi)6&%m0G1 zpCm%!QJsB9!Q|hdMkE~kFNypWk-sMLH$)yMvR4otyY8*~hbD4euMCh?jQcBk{5~Omkl#;~GzO88*u3C4bwSXLC z82@(3DrD2h?#h2h%fJjcJh57C`sr$!e#$`>n(M-^CleaRB+|C)Zoo=RYs60~_%f~N z0ui$IctE5{N&mh0}?oI9z>T}0+VD1IL< zf&iSRP1Rg9A2fkO<~-;D(=?wjb@RArS-m4Y>C4oW;5=m?Qhv^~xZ__AecpgM+!q@y zEZCG`NgpO=+9n7%8BT&TlVV!diyEzPnF2f_3gzU}0t8{mV&*2bCx}Z8H~OuMlM}u| zxO@@63g9+MaGN7Wi8`IuI(*JNwHG}5XOyBPI#U8d+l>~KB*IpqlNSZB?9(6Qyhbg* zMJ@G~_<@48t8GpcjarI08F`+Zf@NZVc5Sgx4z$IIc_KYz+$dn5kAaL>OW2KMJK8Xu UZMkR>Gl?61M5;$AVmS4G03u`ZeEDDa#bhh-+FIOEn(M?Ez*m z(=)i;*ag6zib_kBZIxH1R8A_E?s)@7FF}qgP(phJAV%>N(uF_T~(M?nnDjh=`yK1X+yp!yv zs%e!@bTZv+HLKEgC)dqa^W8$V&@EPr-BPur%9EX$Zn;`k=~QR7Td7vMbJaQI)75#A zsV;14BKt|bx+rqhGa_GI5{2qnq|b@8C~g|n^GGj?QuQ7&Q@vM|s~4Wt#H^^itci+K zzG+tP6Z#9<%G?p2Qd?iqL&J3;txm5lc=mTh7w1^7@3)vpHFw>H-|D&b4s%!Qe*HN# zb>xZ_=DyKwNT=(#e!a8q_~GIUt!}^LJl*qJu5+sH{Cjpge(Nb&cN<%GZIFpBezE1F zE8HsmRNZr)bL!H?y^JvH*X5?;uk~DKtLKLq-*LU3TyuB3{r#h_!?S$xrShxK{{F$s zPp+6pW#o_cD1HKB^Aq34->*FR#joD}^6$K7Qj zoNbX6Ih>OsFA6xPL{XG*PKy~)#yJzcr)pNriFxF6VnHn8oEK-r63zv2R-D7R_#K+1 zRZDn_=flKkh8o7ykeTM7d3Zkz;cGXwp&samI+hX6!dTF*Yw{W2K-#=%1X>M`0BIo~ zTfoxZ)#VdW+BHNRH8FSDj~};2DzTs)U%?hAvq#q+L}^@`L`?#@)Trirqcm#LsIdbb zGAT4XOlCcdOUHAukTQ*)dmV|Izu$MR$vHeim}t2zzt)kmz+7sh?l-n-UhBXKGdG;p z=9XU*PNTjr`R>Dbx80a^J=x*+H03Dc3uBsXV>$kb5Yzl z*y`^EASi`|}ZI^IQp*L8%4Wn1$a^^Q~X@sO+ieR&W1dH{!4fzb4fk_N;bkBKt^3o4VJP|i23@^IMlvJ=$dZ^`c3_+_JMc5-x$OPi6HTQb6^jWgH(_lTDT?|q|`YT z7(wzTlx;y9qyuXh3(~{*p~kfw#E?skb36BbD^{+BzniA6xrGNc??aa~UH$zvNMCO! z0!tXeyk!h5m5YsXG2b3yrM6S;^r1e;V5Hd~gRwm|$oUy)M((D;y9K#zW0*bEhB*;K zyL?~;c{Qr^FfR<06oLYJDYT32QhO%IUD0+v9T-^K@%7MdI(2^+3taN0VF{U{)ji0( z*Xeiajh4H)y1IJwHT+h}q2*k!cfy$8lZ~w~;r42qvM$1;1F&;s-FHH(u`B9fyx#9S zt_aOWe;1^cBUS%(q`0JaB}ig)T8Sc_q-FI%q9Uflyn_QHgR#tyW| z>N<5--ixt?MzaydBc%#0cCb=Zi=>)XebQ7wlVWOhD}|`$obHAr1RkT-X?cD%rS7KE zq23B(N?Sv{Cb=-en5xACjV}sQkupWs_GQoSMc1aI@|G){y)eb0Kzo_SW1!I;t5&Mn zbZX7Uo&A2r?2A3SvlQ8Ny;dV86eFICG0xhtvYBzPP=hy(4ImC)N)Q39Oa%>0aIyA6?;(>8!4iXCS%=Ht>OdX># zy@10bjBP@dKad#!u&G2JGj^;OB1MJLK7x8^x3PLjK8xfXGjWA;+30oqPt;KTCpff& z`^Vb5+s9OLmR7HHdVnPF2_(ktSIjW(xD7y87#EIT#~RE|E~wS7Z?4#CHAL?tS`6~t z=pSpz@jOrkNz1BUwR{)LskZln5~8E7rQagnKB(m8(dtB;^wP0BeZ zYQp5VGWAUq&ry6M#d|1DqOrV&vIY*X1fg-!MoSiEqG06ogV{S~i|^?KC*Z1$T9?MH zPuwYLZ=pSUJkk4%Uq;C+?Z&qCiY`-rMWk->Ip&7*sA2orLynM|xqw^|M+wKwA*{!# zs~{1aTf(OX9G9@G2Pji^HQD$Mfs)l>j6fl#-nH|Ti=4xH?emqTWv%l-fZ(z>^x-fGU zX#PCL(CIjxFkVv~fw=X=ZpT@PMf$SE=FtGvT}Ka4U5su!%AUCIDcuQ+tz#!8y2%bY z^{9>-j6fMj#aIIEiX*+Kmys?T1%*RpBGJNFx9;fY4)Er*DgAm33Gi&_6>voB=mXt1 zfHz8?4yiNHCFI~hLW{sGe5_-xO|`aS>j%c_MFlnoNo8YK`~KEJ!mD3*#Om@&vYM(f zUF+BVth)Llk^AX{8Xzd6p02)w61IqZSAQ)J}RBp(1pCF^P33V4d5aF=#A+S#gIanvrO$Rh{@W~c8rLChX1w@;N!zYA@+$hLUCiki0yT9{%^ zf#q-bX9S)BJ@GzE3XHKh35=>$^cMdZb!RyyE@79T%Il3+3`z2HZd{`l_58Y8&{gw# z%a>=cP!3{SPF<{4lb+X%+@fkqCAV5lzxK?**`@?7C9?K`5nTt~h_~E?>*j4a;+|`p zzx`SMz65VSw7WgVAa=W<0r!xO+Z~!|ay9dyrv*}#8A22bUq12kXdwS8#KAoi)lj%J#j3__U(?@mP+C@=*wh29zWzYz ziDIB-4zvx(gmDIuJUv{*MWnN{+k#u6oCJBmF}*}_ib4!f_KP@_Cue~&EUTfOoZ!_k zt$fv*)9*F5I2qS!-TEG060k+V!a;g9@@f?(m1zZ*RmJhUkyYGg<^0H%@Zn;>Gr!(y zz;)rcEkpV8bh^2!&g0 z-lo`A*YKg?+wh^oqzb_yz#*%d$c0zsUbPbC)KKdiu&hyen4CP|>6h=K^N->1UWCw$ zoSA{^Q-Hmj*KIRpGF5>6L(l=tk}i<}j%j=~ZqK6}F4Na+qhM6fj?PocEWu{7Ez+@q zNp~uvALLHzz+G3iGLGacK5+!p?53dP$R~m#w+>w1fD)S zP8pi2(J(KWLtR*KV{$53kqfei20{?#Fs1B3ewbEgbC^+QJIFIs@u)Vi{aiaAAY=jJ zZKzQdf@Dw_7I2qhP@sJnmf$SSsMZL(fc$1}!ZAQ7X7qFd4wZZgQ2A#&Pr-FkckpN1 z6@N~JW)fvBNS)C@-;=OaMcBPm1@UTS>)$0Zif}KWG&ZOCi-C>)VO`E>LGds)D7lwl zEzaOh7A?tOMr2>VeK2zvAC!U^Y*LBt$IhZ!Be`|?LA;nR;!r=JH|e<&hh>%wE6lidx;1 zkFx8JP;@B9enM~86*9*Xhk`f`2q)4iLlG8hj5O?mbK)U(WxcT<<`HxP?t2Z{!cIcZ zT`8YNyuZP&{wBM!;WKOkAf*argt4eoyf+U4g%I{ZN(WRH0WkzJi}e0)HR}!~+82 zF9V-@2{Nb0En8p28DS-$rh(SRSK{`Gyl%W^*;ABRO_mza2}r5nh7(Bn#0XM+&_xi) zX)y7X+R|r$M^5z?I>3a=JPjtlhWwugOhCT}fD!a+&ufFcpB$zD6CipYCLr59H?dO? z`Y(lZk5y%urAY&3@Knm@kNMfwOCf_cg7i@-%z@`MSDFJA*!xEJiU<05TmM35{ zI|iGbD}WZ&pI>Q%yeepkkDvuxDu9+5fJhS1l2Xv}?EnsX?h$a5MS2Jiy?q94EC7uG zpqU*3%}h`RaLl^P0HfD!12CFZ&;rT~Xes+=L1I-}flPK?{w=&N`F#{zZpVlszn|H^ zP4NR1KS-ev=#Z%)1PBpTT7D7&@ipR#_qsZ*{CxRCZ2!X)KSJ@N6hB7s;}nc8$=eh^ zLGhCmcTr#YcUYlx_f@7Q3M_w`*#-sGW2N%C`YAufM$c0G48_k<{2Ya1z|=?i3lzUd z!I`OMn)Pk$bEqvluWC~*vEk5^zr+^5Oz|reze>TopHj)NQX9nv5r+l0`zq@#IwgD^>!8;M|12-1XKCleZoHW^rb? z@@X92XO!AT{u11Mv!i#70X!w&ko+B-K{O9u(AqwMTdu!N~a7#v?3gA0iCL z?GG&JBm+1SnyVKNO^}C6XLub{_CE!)%ZaQk&d{NF$dE zGC@-1P-iv$5y5{|DxO|HIJ@4vu?&{J+i4xBHMdOSzO42kj`wb?YcH(7Q@$0iKhk=J zhe0|57F?tt0^ByL8SJycGGf!(g|SLu0}_!i_yzk%7tmm3LHRRb!Q1X|qk9$cw^p;) z5%SaQLuvR!Oex!cZ1L%$V*8L}0A`&Yf>Qj*T3=TI=PPr5qsV# zkdX9f-c3ZfiUf8J=$jZ7MDu8~GthikXISY~cr+Hub0(tYBM-)c&E`%6eV4dVQkd~( z^LVp$?695C`bFGjVyr)HFvN%8p4Qc=?c4zB7+rIK*bLHU$LhugF=#E?a19Ebrj%Qi z!kRx_nqcW15rDWbzQVfe6;bK&6~GJ;gZAO7AT_<>febqIlSZ6dr$CfJ24V6HyZ{}( zMi!y`v@-e7W@-Q;O9>eA?8iH$SdSLt|?$^ zW0bpsHai~~^@ph6iT7h;psvH$A-C|o5lGqGX0#n{ z4(BJS1#w1g(3tH-vBWP&Zt)$P!!wFL@`H=>QA`d_Q2CqI`3c2FPkK-Ds|KZ)D9=5I zO+iOI-Rlr2`Vrw#(1!1GTm&*k7ajd6)KDSe5zXXE?6t&&X0!2_5J=P?6LryGVoj_y zo2_@2Y$$zE>+wb`5aiX!u_G%b?$P=8?*nX753AyC9d!vU% z$7r51#|;=&Zl3)s)zP-=;!~YHIE=<#AI1a69{k(=eVq3(C~Qo@vZT0P@NEHJLYP$P z4cgm(&c{(k?01-Y(?on7Q=f(SAPvp)NJR0YjIpM#8J2u2A(Mxeg zm_SewWFt+r@kP%#mC={X)tIecf-&MZKG?i1kBR5BojqZAPSDNIVIV$Ci~^#+q?@VW z{~g^-pP-xRY2yon_=a@TUl3|jNYT#B`oYp2v=dR{6RcDIeJ;h5lvDl#mML%Z*O>Zs zh$Dtdp=MN0`41`HkW9+oU=vbF`Hv_jOj3S^+25pi2gOqqoFag_Zob?g}={#;}ZU-#TW=8QK(2M_XN%^F4BPbk%S z+-Ab&z7wGS{}Y=7qXT;*2MDmprWY1C99Z35%Cjhk|2G%;IAg}g4DaJQ{2jfGyCa8ehO*6i{VrUD&!ie#`Yp{qXUK}yLf?|BHs*(|BQ*3OJ zyj!ufam$iofGVe0+N*dPGRm3P=|AeN30Pw@qagT#f$>W@A6XakAIHy#nrBTPIZ9@$t!qEQ!X z4VFB^-zD_(2fR?R;Xh>RKTv!z>gVCh4>cdE^YTX@s@ESw;^9k=KX{ovKl&&b{^LkA zFE<|+Yw|zxvLlNBL@`hCpD9SUqfqIWkvdXYa`YqOaO=w;ISwf_^Ok2yJ>6YWpaddVS0b9{l_-Akc{eh*WM@BcS8 z`BR99(XWxw%m2>8KZ96VlK+G0|4H$`C`i-gYZN5r@+%a#DZWaZ;YjLJ`wZHfO*(c~;eg@R#OIZv@bu}E=-Vu|7` z#e`s2a}(Abed3#lyeBVOcXy-T=g~pY05u5k7$@jA;yUxzyk4>u1-DL6aD$)Im&}NG z&rc}$-BL?;v|3uU5fiHD?>0|R_4~$z;*B`Y$_)R>fIIoMT9~TU@OcD2IzT#8tL@-x zf~ZD){4d|a7qd?BK8kA;I}|r4T#AoU+@kn<6d$9YLn6tTBcjL^&yg(#4^@;j`tSiP3ab32(UAByM)K~SXskUiVG-`R4Eje+rM#1@+nq);}Imw}Y mC3@FS;H7BZC5X{pgmMQXr*IzWg4%sxeA0sR$el(?=Ke3kVRY~S literal 0 HcmV?d00001 diff --git a/models/__pycache__/tunedshot_vit.cpython-39.pyc b/models/__pycache__/tunedshot_vit.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b985622cf042a310736156bae802fd1711735ed7 GIT binary patch literal 6644 zcmbVQ+m9R9d7nFn*Cn~!UF}M?*N&BxNxZ8#h@6XRE3#rafwM3yB|#ax{%^Kr#dsTdrr$yX)DV0a;=<7+t6`au1Y&mzE@}!RXP`ydgWGGrQN8~ zTWBrxs;z3T)~faDt-31DM~l5itD(|`XsNf{TJEj1R*)~Y&ahHzwXL!8_x08qtF+Ft zh1NM%Z9RtcB<3M!!u^&kY(ZX zb|{|h$Kh^Y$dU--q|cv?hrPk=BkZTvzSH>IYyWa^?}es$)Ik2|EfRlCHSd0xet-8u z`{xh-^Cw^2zWl<`t5o+AiMyom%!}v#{a-JB_=O`Exd6|JpE=DX42e){=}c=GZ)-a` z)0uJ4g;ki$x~H`)W-|wGo8_2`x5M(RfOif%Q^MP2WmdsE&lXq}?*gl_I^IRL$QpQ; z-qtLwRmSct$=vkI1LN3KC{2^0CwU4&XouQZPxXneYr;q|p+h6pJTuiHEv8-99vE9U znzpo);UMJF^}J5p5uV43XpL+fid%iYCoL>)f>pMXFxuhgQ29WYdCv7cKgH4LC}dddwmv0$)*^_Axp3|-p!8K7~JM-=;VWV5(h%lOJ+$w zs7}9XV>XXKr^@M%()5Wh;z>#nsn*5gVZE^?`WEI_@kAfSmdz_nkl%~@x8hm8sq@Fs zj<3(flSs*ezav8K2|w9;2Khv-seVwO&E{csKqxT#)sP^AQ1gMqYpF5TCfdkoYaQ}WlUVhyeIip;fd)*wlqx`6l=EoM=&jkYhfkt@t{`%irZ_+sIIXzE&A{g{^Ay`bsp|7(y!@4Bf)_IKYHSt@5wbGFEj z3s|{gw}eqjm}faHVP;oG6;WnpS~)amw6wBkj4Kmuyud8zEu>buu&QBJ#c`FvphnfS zie75ndUvtgNGq4L{l8BQ?1R0RrQ;6Z80nF#4LPE7HBGPMM~o{p!A5za<#0Xw-a2zK*>iTG{dzl-7*S#6_2aC zWXJ>{x3Y*!TFi%4hL9DTBVIkQ9tmbIbz)rg~x@@=&FvQv2{a5(F-(iM_ZyvQw-x+!RYT2DZZ~9U!{{ zHwD0q4Q8MOFfcY#S3!sihqbzP*GX--t1EH}Bn)N}NTgWurnSXCgfWx{^YchG^ZX(e zJV)Y_BsNHVio~Z$keTx)1Y!tQa1RX>#Hwa^O6p5Hka%UcXpys7J=&?@raNo%CtZu!$iDv zP*$W5lyLppt6xf*XF(}~VHSw_2QaoQgVL$Eta{{6p-cfczd^ORsNZg5V$$lwJALWw z^mz}lp-7FU(SU!96dfPDvK{h|Q>LWAwdcn^N_dXCY>^=60&i5YWD5L6nx;3+tRAoO zFqWr~nc|Zfg)z8Bk0>m@#%m9T3I7CjN=PS5R{`g*Vh&LhM$++AN7BKX4WqDWXATpR zdNx5d>huHTS~l!7dt%&E&LeA`hkQF5q=QbAry(>%fd-ysEPR3<^KIkcP;1M+M;z`ax zXhy{;C=y`k`2XC=FB46$4l1)v&n)JVIBlY>g3=EoA!x7FF+#;^0#OB-094vdkO07k zN&CZTP|ThV)#}svZ&K0jB3KnyygQ}aUO$7ZN{f!$_imqY;86Y!8FkF;^uFr}582KVd zyfZ2hfd)C(x;cD)d6qIXRij~HK(xusL9pF%KCPtXi2*`wjtkU#x-c%Pw>d7Uw~G=O z{&U*M6_xHnTA^d(eM8Nvn$xQbC~X?22#KdvM8FDh*uA%N)RL&UjJVgCx^zUogb zb(h6Tnn#q#HMF#FRzvj3!{KUajun=1ZLnt`c3fm76*E>*YOiQwEp^cw*5<61)+Y9- z9$$bPp2b&M;C04mk(J+HKV6(SqdJZ(=h8aG&iys<*!b}*t(>>A#R)imHrt^GMn*X( z;DEsQ+Z~bcFC(K!$Pq*8h@cdw3K#y|>4C`|(H5&_?_IosqZ?w=N7p|`{*Bb;ozeQQ zKEM7-KS|c_e0t;KJE*&%pxqCG+f?-I&gi|D|9|b!6~%>PJvE=;7G@*uEY6&xnSF$W zvUlR84~<^xFUfMiF*6Tf#ruQXGEWDT>=5K`gGAl(Im207gad>&5%w~IWdgCJ5$@wS z@z%hPndc{blSVa@VUH8k&!b&`7&Bg>l5G+MX|kX&O97y)c|@3pV8-}}A@_sZasg*2 zKwuK^4z6}UZQrDpK18!|8iVdq)o(#a7csEyZTnntb7^PYV+Rv3WIDic9tMJrdN`Kz z%hbf_#26Ws*^N!Yzec6FU`o`P4Yvd`pT#`PO-(?$Dk4&L9lDu#l2X3`L032lVWozs zRtHg}lg={YTg50LMgI~=PBeGMcMXCy>G1DeKvJMM|RZoFr?CF5Xm4vK2g)O1cj z$})zVL(01_TY?l}JOU;{xh)Yt5KILopyZ64`8b&TBl7C(lUB=&uF6sb)v7} zzES%cA+k&F9b*;H0!PI;9jMp<4ym;Ff9%Hg$ka9Y9ka z*U5DOHUOG&V-7Y;bFjGtuu-vEEO%GL83i`Z6l`$U1+ZBJtmFYU1qC*50!oOEW>C^# z#W84S_bhZQK#u|FS(-x6V%h+dEXC`9r}xzXJS{1(SyEurP++6d3T(<-{1Ds4?~x#5 zdx$Ljw<-HJiQggdyCkri8vlI~zXvgOZGInV92W^+lK1s(#aj6vP}{GQc!$I{Nc!a8Q1(p{K16f*eZ7-^kCd*E_)`*p zM&i#&j7eN3F(L7N63PurJN{k>#!YvriFC;+ae21G|AI7rK;nla{*nZZe##-`vNt{I z;#z@+30?gXnEVu~$!icAeDb7^lD{th!m=!Q=P6&Mn~US#3h#9C$Gz9efe<`C_tjrI zbhAR&nlUJx=gETSWvmPZqS5a;ozI}X55&NbchE-=Sk!P|D9Q5wzjYB!~=E~^!WXGUV6 zX=_p4l5Hd*?|?}jsO97)^-d8dBdTtu6UUr*xw1c5&B}(N6D1pTdQe9lTyUrYFnr>f zoUS4`0a-@W)U|I>U3VoL+qh2SCDhFtiGDxY>j+*ZePx;7rPKkbEGVGtbs|JY+`%WZ ztO`Ji760aUNb|2q(~i^?U*VW;Q58)!)nP0l_18(z8XjBRb;{D>O7~K>mCx(QB^My3 Y1_YKk`=3Fos!YZuLo&&DR7&Ok2i@}u^Q>Uh< zd#bB@PF35UaWz^pMk@ghSuTRjve3QU)(b3m30@FSJRo@Cu^tdCNYMy{gn08Jh3`Mr zJvYZl(4+I$x%}tUsdN7SJKulCEtM<@zl&e|NAGWcNRs}A3Wt9h3Kx;0e}Z5VlLN`l zp0cfovTCb%DuLS3Y)z&%HPAbTZHT-UQwAXr)pO_HM=J2^TBkdZr4TL3T8U9_H1X)oz|$75j1Imsrhyf=$~`vbz1$6^YHT*}D>(_3OK;eVWPFrN-PIzEWCk z$f**BkXq1n89n1I*1=Qn_Tm<0to45A#jS4W22{H2#_m<9`n;i~<<~=x`yD@w-C)&^ z(}nA;PA~9Z?nbT9Ki2m6oBbeeUE*%&ZG5UiG3)$xE5=ZGRpF8w`B#0Hhj^EgmSUGT z{rLHA=x=o6v>5wg)aB2I{Z4Ot?-_i{$9L-AyYkOFcP}*5y*kQ!w@G{#qWRG`==YZw zKK}3f|M?GZu3fya_ZGFiO5&@e@Z8Ib|MvZ}pS`eWqU0eR`KjYl#E|hNTV|52d`()H znaq^CCagkb+Fi-kn9dA5b(Ui$o(9V^3(p*OriiD>O00}$o=vd|o))XJ8lD9<&FXj- zzb0vtUBd3nq`C2zd&;4yP@X13Df$bDSlX3_aw3mpmL};tVa9!#U(52L!c4Rn;ZmGC&_*slFCFwC4b)lVz8_?H^ev#p zM5#D#`A(KcO9?IcM257OgbysQ+Um_7w$bZ`H$y*m;_aTlGynEmHTB5?5 z>us)K%Toh2(7VCs@Y!ju6}Dn0;JiX5Yt49IM?W9_VVVI&WVm}w~~Vs4iP<+unREAJ}wwxqnN3`-+vSY|r(rV=fg60<4{rafnm3FmVYfmTTbJEsdB?`R3SWV4!-;Mh)a!#8D`%zh|vs1k0_X5{zh0W#V<-Jev zYt&QCzv%|49(TF7k>SdVL5t2GaLZqaU`m z`aa;4yKoBzIFEqbM&Pc&5L3O!TVYJ2hC4p*rnv~pKKx4S_-@Fb!rW42-Aj$k%~Fj9 zXjFxf*aa~r9VGBAU6{7;2D|Kc)_lhB8BWlOV%rjL6ZuqbrMmFnRCf4f8ebQ!_@GIr z)XF?5>)YeqxSRDYWc96(`L|PxrUHMaJU#jU8L z5Ee6%hcawg-_lT4U_nNrjMPC+V5GWw#F@#*$eFGqMFeb3xbnNj0brk;=u@5{9j1Y&eId`?f0;hv`Wk8g0ANV(RpMh`VBQT z{Llk#r3Uk37dBYh@8GyMn+;P~L$)88G4LP7_^>5Eia;dz6_mtQ^Vg}~2>n}l+3`8% z_eWiWRzzznVIyjCMc%1CG}R;fK)_8)CHCPRWC&jp6?<`UpaU;-_$iib2-JXA;w(!H z_$e@Es0cqbfg@^S3Jk${*w)&5+emcys~P9oXq})$f+cTgtNe3#hj3iJf?OldFHprr z5-*W>fy5;eG^Om@KaU&^BCOyhItY}#OpUZAV!`;2Q|>B>H%YufVwJ=*`-0Gze+gm( zDXKw8WVD&d;+Uu?WqD`jq1B>&I>Ic-Y7L~!atWESfNgM!7jS?$bZ(HDfjhK;taEgR zsI)G?SE;SiI>p8?$CM$h7EXE80q=l`ayuVeL-2x0EvOR6(mi#cU>_u>C^$hWE>o#J z#q_wsjC=3~r#k7&a@lyy#2@ukp-pd}V)=dT+C=-vn=*Y3`WW%Z=+ki>H8XT>Y-%6M z+=^$JwM*YIH#~tB(w`X#t_07VN2!WbLz*7p^f=c03#ixH3z)SgJh!mAM<_lvQ^4L* zH(a_ic<#;suuZrN*EFJTpL+zXaHSg!RnQCg$VPvCJ@C^4(A*)~;q&ZB05kWa_|=_~ z;L%_zOIP0dQq(vJqT%(kqnHz*r6sUraWV@+#1ZO90=M~Bs5KXKn@!9p)mq_tH#OF~ zyn~~mK%K^i0e_Jc4Hs0s>2rdyv?#E(nf!+%M2HopE9QaFc^~ zIRco$s8TiXL%ZyBC@R3$IL%%^;snc_fG#aw2AaQxIRt?pq=q8~0+s8se&9Fs%wJlh zo=s4^b^HPLLiXBm@yL6R`A%AG9fUC1OKxJ&=rn|cL!gc{t1N!LZEHj9nRw!~pJS z-$an{Ek#%=SUgi7($k@*3%y*X2iw;-RsN9}FGu6OyEx%gZIdU^H-LQkkwh$JD}kQH zOfdNTro#UwF0_l2nJ^1EF!^?w74B&RGkj)Ww+P+;fbP^pcVDvvk2TQ(){_uASgcI3 zM7zeO&;nL9oCXFcXv>cfRHIxUfsKWfn-P94F^POmcvfXq!0bZ@qe`1bX%+&%TVCgAa~dxzht9=` z?gRPRo!XM%(n~#ej+P%i(M61liPmYNZ5Tb^6;$B(bP@MX;vqrvIDI6KNgO)_ z$ZUTFNpNuupuZ%XfWYU_eA2v#>k%pORE?Uf`S*WEzkf&2oSL03ML+#csvxqW@CK0$ zb2|-%C-g#5QxKtwFX%0ldL73H;9aF>@)Dbb*&6ekLI==BTjB6dt zM7Vld39(SjIaEzSd-jp!f}GB!RK0%Xau#@rDd;a@QeRGs0e2#A!zV!CX*Ii3cqt5h zH^4_0fb%5q9ifV z$eztgVo-Mt;FZ;<8hsAPC+$DqM!z(_Kf~ku)kX6?q=@!WQp##kJpn)g1O3EQDk>2j zLmc0y06`1N0zk*ocFIQ$@l(x9oZ)2bF$}DY6K+=#-asd;IICf5biRRo(m5{tN?E

WB?>iVZ)M-@P$LGQmX%H<&UonTlFOiu*C-5aj~mM`KW= z$ZZJr-Ol0bOOu=;i53a#QXJvd1Z=P@V!m=x8Yzh6)S*SACsV_Mc&fvqc$%m|#P@;( zj4K2E5cz?>C}LKXB%f4<6}+XIR7PN$!x}i}w9rQM3Jy`)1-}|tTbIB$#dz^ddp4dE z@ocUxCDus^oF|WXw3_6YHH#}iaYmeFMZr?$QLE2M@j_x^G@KGArKCF22et4F(DNj| zQbRnU45nG>(fY~s$QaZTeYlv^h_h`i#E%V6WqDy6;yRFs%+Eh&|e3(c{6{8@M$98+vx>6#^0DR_+XzzIcA=YB!3O z?!37C`E|725aFrodD~R={QBU*YyZD~7>WqVp^+Moa}Bde^^D*MocUpzp0M|KD7SC) zsr+hM@;GMZ02+C>x1HuGFwSCbldh+3xt!tZC-!^bRI%UTg4nABi;BO6F!pxO4VmLc z{0lVnE{P6_3C-&EL&nQgvra<5r6umA1j?pWhXS=eA~JkLpS#|6I)#`W@EUo%g=<=b zRJ7>ahiKG~;}_u&{{SL2LB5*Kn#)B@nCjW^*ue;qS(@{3<>h-Z-7Mfnh`&l*j8=#@ zqcOXEjrcE6EiQ{AaaVv#t28fQkme>PkeY&uQSlf+{0zGOIa2fngamx6AwZ!Bdltm1 zjJO-vhrc3%mqkS1CQy$ea1A)9KvVgFVIC>V$^*?TDm&Brm?#!aOy>xs5GUA&l#j-c z5`#p6RviYDShy{P8whL!n8*MVa^}Ne^4BQ;U%*5qFiD6unJ1*dRGc4LfCA8+;+hhukiVGsRx;h4#>KJ5Hf*630nN)}6G1yGCD;Nc2a11szfK3fRQySLDbpbX2 z8lqAF7y_G_eXuzPuo2_Mv+cR~ga8|33^us82G~pkR`LKFOMuOLfD#ITGAOCD!VrN@ z`y_NUKo5u;#?GK;I;jInX2K=F(<5~NPcs5+W(3&O1=xtZ0GrY({}9{7zfOXT?EqQ$ zFH-TBNC;*|Y$rpaZ&2=6A;zxFKSCbY6ofAkf0S>FaEX7D`u-Y;Z;|+Q62C!WNP>Ks zk4Su*#BY-L6gTF-MGeA-e}{7Wj?90TiU-~~`$+HP-yzM{NZcp!T@t@X;`d3AH}XFq z@jVi>GInv@-SiRa(f>sv+a#yNz3n>xL(-t5nEx?}KOsTbaLggoS!aC3g6k~YJkq~$ z2u%JJs?jwF2|jt$N6BAjKhZQzIpV7*!GrO4$Xnr^M*dLw2U-q<;MBgae(I|mWxA6m z7UejplpL z9QnKj7P=#rldGhok)T5m_(&JG%VEe3sw4YLD_LDXZUxaYg{>mSwo6$BLQCSP+$4Ph zvW%z|kh2Tonl)Np!!<51(pck8akm?6wqjl)ePNluO}RztG$nwt(+Y4h;-4##T@n=_ z)Z*VJHiBaOd*#S~QvQ_s>4CUkwhr|yh^q0wTg06JadiG330l!ZOD&EATI$q1mu>8M g;t|mqh_N{d)=12;D4#&82m><7tW+{3krSo=0UzgnjsO4v literal 0 HcmV?d00001 diff --git a/models/__pycache__/wa.cpython-39.pyc b/models/__pycache__/wa.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b53c4a24737c32a6581b67259db2b079a0d8fda5 GIT binary patch literal 5394 zcmbtYOOqT&5$^Z&Ja=aHrPa$T*_aq(X^kQ5Ko~zI*#>#Bk+5N~XuCv+}bY^Hp zcH34u3%t{I6mN%x$ZZ!D?}VkO+%7A=5LTjUyQ+9MtVL7pDa9AVdemq)qGr1ZxzwI! z<@OA#v}d~-tNvPV&w-z3we}%4)m{L982UQ&N5CItjrK9tY#(RS?fckFd+~XV&9b>0 z8k-BMTW0$Nn`eh^Xzi0szo@kqwy`>Gxur`>TwxLZjW`CilfLMYSXvpx9nni-KP2g_ zFZ>IM&jJpq{H&h_7XqKhO02yYcX$v5vGBv?K(q{56+Z6<;?X1yRudsBB8bz3KN=6B z{zmuAdyjndvyF34ZvO!@=>GB-^!)Crmbv`|bax;_~@~Rp4xUvWq@?>9(c@&l%53>XlWF`C;SeaGu@4l;9TDyq7t4iawXF4?4&Ov_GnCp?eS5+KHbxSH^F?4p$h;fl1dmFX>A z7Co;M`f2KUx3t?2EJd+c3J0m+!4kW=xta`?`kh|56eTPO)1|Ba>HY?vLysqEN@UZT zUePzpuRU{mXTXk5+Q~z_jrTdc=>?EXTZ8SdTl_dUeT3PsnRu;38m&p`?6Ed-#^Aj| zUg}xG9Tk}|DnTxf)#}xX&1FslY&Ona3!;AL^To8c8Jtv3{5-j%IR(%+EI3J-$KKvujYWIJ0Z3t`gj_TsLz zdhtqvN(qmAA*;b;79P!_rSn4=meaB2%(`g}4N=VuQDe54V$Mwq{noQd8>S9uY%mwU;!SgCqOOU$rQNy> z(L~SFTQF4|`tQZeXcjVhS(_e{>4F>0L8{|zj14k@(R@bpJ4Dif)}KIYtu0V1b^PV! z?fm!^X5c^;lNe`z>iSUHg8RL=WlASa2D}qUi!4|c{eC})86f2n%1W1g(OFfeB@5~5 zU}Yr?WEp7Uk*5SaRn`S03sRsDe;Aw3Pl23o9p->|jh_IKRUncwGu07aqylAd#X+C> zBJfr^{BxwF^X8uik!5OvGQgbxK=!5Mb3g84aF0r&AWHZKr^Dt;M7~IkDr!kysL)cD zX;M3a83tt0(I6DPei(S&{vhQ98nQwF`!aeD!yuH7r}~g{mj^xAGY?h)wkf$lr_r=&Dx9n*zW>ls`-?5RsuZ6oF>wz4ZqRbL-TwZw3X=@ z5KXVa_N_avS<@}OcE_?obpt==6UQ-h7bTWnzvGySw`+P6-crMFa|ZtMnT|WO^qq&$ zXV-211_4Gy7^=6B@9t#_JOu=kKMM@``d~cj5!Opao{!|(D-X}a16Kw z-h~`3W2!^$azLwGqmUzG_m3aY^G}39=|%~~(m^DRknf@)3p)M=N{cE$1Gl&=MLpPn zK#)zP-Swl$N0fXXJIBuxd4b47An*cZ=Y~jR!DGRjy-vU%pi<==gjdqM`21N}>caq| z0JxXRf{IF*0PF$bX@bdW7P1m@FYT-bWN=)TUm|n9D1!clZ^YO+6OCI!jkWzX<`3V0i&75)t{tS`(h%6EzFK^j7ntz29 zWV8H55Lw)v?x)P1KZ)X(@$MRUNv|0tn76A8+kjy^x&xDL8lTmXal3lyj$>^WDYB|% zb;y!;o*6Jjyeb%M0tJO8bYX1iTl&xzW@c~cXSMhBH!i(y44omtv^7FpMfRr;U1mUA z9F|ZI_0MW=j3Ad^H&L@PtY(!F*?VEEXVo3f&{T;wva-rpXXb)7tWiwPYFTw`45u=4 zRHELqsZm+|n-o4EyU9I3xX7=bv)}(-WWkwtw9m{#; zYt`ky?;Oi5LxqeyZa7F2dLmW6f<((?y@*etCZ7g@!iDMfB0o&G>10)#+mBsBzJeI` z@Z!0C(pd$!bbYw^wI>%ZBx$;M{mkhvt)T1@1vk&{bT&x!=*sZ+3;$a?bVZQ8ucy{= zPFbh4^MF)RZWYj6WHCT!43uk;-}5gJp`4DtL@o65vfAO8rU#eelsy%JCW+VOK9lZc zzq5YT=M32=JQ|sk2qHx3grj3K@~=rFxPlR`_5GN6ek#pQe<00t5aoXP7>$4^nG9ma zDQ8lFQ~`}_c%3Bb4-jdwjsf>O8*&PP74DvPcn|juIQMPfG%{F)0Q%Bcs$QHq?W5N{H<=el$EbX|UxYSJB_ZxGoe za-B#m0as#^0vG9})R6ku4%BY5$b7eisG5!kfMdayMr+ zaGk5@#=kA60#{XzYwkL(N?iB3E?m_q?vp>Z3b;`(kmKGpQE8Rvavvk1@;q7cygWe% zU-7&v13%15)Fqv_Fh72d2<0O=%Fyg(b#m+P1YwA81bYiSO!bZtp%mi;-n0S&=&{^_ zd%t_qJ*7$x8Y1h|i?K_lkWQWD!CeAp^;@u`#m78vIII zvzfJ^9iguo_VS)3Z{NYA?6>78t3pNxq)w9+L7}6t!GTMmzs%ABuTj56Py5A`Gzc< zd->vv7rwd7S4c&IeVd^yfObT> lPvjQ$B;iF$3s(kA(P!VUZdkaJQG^2B<4A&Be@pxFe*mS$IJf`+ literal 0 HcmV?d00001 diff --git a/models/adam_adapter.py b/models/adam_adapter.py new file mode 100644 index 0000000..eb74416 --- /dev/null +++ b/models/adam_adapter.py @@ -0,0 +1,162 @@ +import logging +import numpy as np +import torch +from torch import nn +from torch.serialization import load +from tqdm import tqdm +from torch import optim +from torch.nn import functional as F +from torch.utils.data import DataLoader +from utils.inc_net import IncrementalNet,SimpleCosineIncrementalNet,MultiBranchCosineIncrementalNet,SimpleVitNet +from models.base import BaseLearner +from utils.toolkit import target2onehot, tensor2numpy + +# tune the model at first session with adapter, and then conduct simplecil. +num_workers = 8 + +class Learner(BaseLearner): + def __init__(self, args): + super().__init__(args) + if 'adapter' not in args["convnet_type"]: + raise NotImplementedError('Adapter requires Adapter backbone') + + if 'resnet' in args['convnet_type']: + self._network = SimpleCosineIncrementalNet(args, True) + self. batch_size=128 + self.init_lr=args["init_lr"] if args["init_lr"] is not None else 0.01 + else: + self._network = SimpleVitNet(args, True) + self. batch_size= args["batch_size"] + self. init_lr=args["init_lr"] + + self.weight_decay=args["weight_decay"] if args["weight_decay"] is not None else 0.0005 + self.min_lr=args['min_lr'] if args['min_lr'] is not None else 1e-8 + self.args=args + + def after_task(self): + self._known_classes = self._total_classes + + def replace_fc(self,trainloader, model, args): + # replace fc.weight with the embedding average of train data + model = model.eval() + embedding_list = [] + label_list = [] + # data_list=[] + with torch.no_grad(): + for i, batch in enumerate(trainloader): + (_,data,label)=batch + data=data.cuda() + label=label.cuda() + embedding = model(data)['features'] + embedding_list.append(embedding.cpu()) + label_list.append(label.cpu()) + embedding_list = torch.cat(embedding_list, dim=0) + label_list = torch.cat(label_list, dim=0) + + class_list=np.unique(self.train_dataset.labels) + proto_list = [] + for class_index in class_list: + # print('Replacing...',class_index) + data_index=(label_list==class_index).nonzero().squeeze(-1) + embedding=embedding_list[data_index] + proto=embedding.mean(0) + self._network.fc.weight.data[class_index]=proto + return model + + + + def incremental_train(self, data_manager): + self._cur_task += 1 + self._total_classes = self._known_classes + data_manager.get_task_size(self._cur_task) + self._network.update_fc(self._total_classes) + logging.info("Learning on {}-{}".format(self._known_classes, self._total_classes)) + + train_dataset = data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="train", ) + self.train_dataset=train_dataset + self.data_manager=data_manager + self.train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + + test_dataset = data_manager.get_dataset(np.arange(0, self._total_classes), source="test", mode="test" ) + self.test_loader = DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=num_workers) + + train_dataset_for_protonet=data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="test", ) + self.train_loader_for_protonet = DataLoader(train_dataset_for_protonet, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + + if len(self._multiple_gpus) > 1: + print('Multiple GPUs') + self._network = nn.DataParallel(self._network, self._multiple_gpus) + self._train(self.train_loader, self.test_loader, self.train_loader_for_protonet) + if len(self._multiple_gpus) > 1: + self._network = self._network.module + + def _train(self, train_loader, test_loader, train_loader_for_protonet): + + self._network.to(self._device) + + if self._cur_task == 0: + # show total parameters and trainable parameters + total_params = sum(p.numel() for p in self._network.parameters()) + print(f'{total_params:,} total parameters.') + total_trainable_params = sum( + p.numel() for p in self._network.parameters() if p.requires_grad) + print(f'{total_trainable_params:,} training parameters.') + if total_params != total_trainable_params: + for name, param in self._network.named_parameters(): + if param.requires_grad: + print(name, param.numel()) + if self.args['optimizer']=='sgd': + optimizer = optim.SGD(self._network.parameters(), momentum=0.9, lr=self.init_lr,weight_decay=self.weight_decay) + elif self.args['optimizer']=='adam': + optimizer=optim.AdamW(self._network.parameters(), lr=self.init_lr, weight_decay=self.weight_decay) + scheduler=optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=self.args['tuned_epoch'], eta_min=self.min_lr) + self._init_train(train_loader, test_loader, optimizer, scheduler) + self.construct_dual_branch_network() + else: + pass + self.replace_fc(train_loader_for_protonet, self._network, None) + + + def construct_dual_branch_network(self): + network = MultiBranchCosineIncrementalNet(self.args, True) + network.construct_dual_branch_network(self._network) + self._network=network.to(self._device) + + def _init_train(self, train_loader, test_loader, optimizer, scheduler): + prog_bar = tqdm(range(self.args['tuned_epoch'])) + for _, epoch in enumerate(prog_bar): + self._network.train() + losses = 0.0 + correct, total = 0, 0 + for i, (_, inputs, targets) in enumerate(train_loader): + inputs, targets = inputs.to(self._device), targets.to(self._device) + logits = self._network(inputs)["logits"] + + loss = F.cross_entropy(logits, targets) + optimizer.zero_grad() + loss.backward() + optimizer.step() + losses += loss.item() + + _, preds = torch.max(logits, dim=1) + correct += preds.eq(targets.expand_as(preds)).cpu().sum() + total += len(targets) + + scheduler.step() + train_acc = np.around(tensor2numpy(correct) * 100 / total, decimals=2) + + test_acc = self._compute_accuracy(self._network, test_loader) + info = "Task {}, Epoch {}/{} => Loss {:.3f}, Train_accy {:.2f}, Test_accy {:.2f}".format( + self._cur_task, + epoch + 1, + self.args['tuned_epoch'], + losses / len(train_loader), + train_acc, + test_acc, + ) + prog_bar.set_description(info) + + logging.info(info) + + + + diff --git a/models/adam_finetune.py b/models/adam_finetune.py new file mode 100644 index 0000000..dcd8a63 --- /dev/null +++ b/models/adam_finetune.py @@ -0,0 +1,157 @@ +import logging +import numpy as np +import torch +from torch import nn +from torch.serialization import load +from tqdm import tqdm +from torch import optim +from torch.nn import functional as F +from torch.utils.data import DataLoader +from utils.inc_net import IncrementalNet,SimpleCosineIncrementalNet,MultiBranchCosineIncrementalNet,SimpleVitNet +from models.base import BaseLearner +from utils.toolkit import target2onehot, tensor2numpy +from timm.scheduler import create_scheduler + +# fully finetune the model at first session, and then conduct simplecil. +num_workers = 8 + +class Learner(BaseLearner): + def __init__(self, args): + super().__init__(args) + if 'resnet' in args['convnet_type']: + self._network = SimpleCosineIncrementalNet(args, True) + self. batch_size=128 + self.init_lr=args["init_lr"] if args["init_lr"] is not None else 0.01 + else: + self._network = SimpleVitNet(args, True) + self. batch_size= args["batch_size"] + self. init_lr=args["init_lr"] + + self.weight_decay=args["weight_decay"] if args["weight_decay"] is not None else 0.0005 + self.min_lr=args['min_lr'] if args['min_lr'] is not None else 1e-8 + self.args=args + + def after_task(self): + self._known_classes = self._total_classes + + def replace_fc(self,trainloader, model, args): + + model = model.eval() + embedding_list = [] + label_list = [] + with torch.no_grad(): + for i, batch in enumerate(trainloader): + (_,data,label)=batch + data=data.cuda() + label=label.cuda() + embedding = model(data)['features'] + embedding_list.append(embedding.cpu()) + label_list.append(label.cpu()) + embedding_list = torch.cat(embedding_list, dim=0) + label_list = torch.cat(label_list, dim=0) + + class_list=np.unique(self.train_dataset.labels) + proto_list = [] + for class_index in class_list: + # print('Replacing...',class_index) + data_index=(label_list==class_index).nonzero().squeeze(-1) + embedding=embedding_list[data_index] + proto=embedding.mean(0) + self._network.fc.weight.data[class_index]=proto + return model + + + def incremental_train(self, data_manager): + self._cur_task += 1 + self._total_classes = self._known_classes + data_manager.get_task_size(self._cur_task) + self._network.update_fc(self._total_classes) + logging.info("Learning on {}-{}".format(self._known_classes, self._total_classes)) + + train_dataset = data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="train", ) + self.train_dataset=train_dataset + self.data_manager=data_manager + self.train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + test_dataset = data_manager.get_dataset(np.arange(0, self._total_classes), source="test", mode="test" ) + self.test_loader = DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=num_workers) + + train_dataset_for_protonet=data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="test", ) + self.train_loader_for_protonet = DataLoader(train_dataset_for_protonet, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + + if len(self._multiple_gpus) > 1: + print('Multiple GPUs') + self._network = nn.DataParallel(self._network, self._multiple_gpus) + self._train(self.train_loader, self.test_loader, self.train_loader_for_protonet) + if len(self._multiple_gpus) > 1: + self._network = self._network.module + + def _train(self, train_loader, test_loader, train_loader_for_protonet): + + self._network.to(self._device) + if self._cur_task == 0: + if self.args['optimizer']=='sgd': + optimizer = optim.SGD(self._network.parameters(), momentum=0.9, lr=self.init_lr,weight_decay=self.weight_decay) + elif self.args['optimizer']=='adam': + optimizer=optim.AdamW(self._network.parameters(), lr=self.init_lr, weight_decay=self.weight_decay) + scheduler=optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=self.args['tuned_epoch'], eta_min=self.min_lr) + self._init_train(train_loader, test_loader, optimizer, scheduler) + self.construct_dual_branch_network() + else: + pass + self.replace_fc(train_loader_for_protonet, self._network, None) + + + def construct_dual_branch_network(self): + network = MultiBranchCosineIncrementalNet(self.args, True) + network.construct_dual_branch_network(self._network) + self._network=network.to(self._device) + + def _init_train(self, train_loader, test_loader, optimizer, scheduler): + prog_bar = tqdm(range(self.args['tuned_epoch'])) + for _, epoch in enumerate(prog_bar): + self._network.train() + losses = 0.0 + correct, total = 0, 0 + for i, (_, inputs, targets) in enumerate(train_loader): + inputs, targets = inputs.to(self._device), targets.to(self._device) + logits = self._network(inputs)["logits"] + + loss = F.cross_entropy(logits, targets) + optimizer.zero_grad() + loss.backward() + optimizer.step() + losses += loss.item() + + _, preds = torch.max(logits, dim=1) + correct += preds.eq(targets.expand_as(preds)).cpu().sum() + total += len(targets) + + scheduler.step() + train_acc = np.around(tensor2numpy(correct) * 100 / total, decimals=2) + + if epoch % 5 == 0: + info = "Task {}, Epoch {}/{} => Loss {:.3f}, Train_accy {:.2f}".format( + self._cur_task, + epoch + 1, + self.args['tuned_epoch'], + losses / len(train_loader), + train_acc, + ) + else: + test_acc = self._compute_accuracy(self._network, test_loader) + info = "Task {}, Epoch {}/{} => Loss {:.3f}, Train_accy {:.2f}, Test_accy {:.2f}".format( + self._cur_task, + epoch + 1, + self.args['tuned_epoch'], + losses / len(train_loader), + train_acc, + test_acc, + ) + prog_bar.set_description(info) + + logging.info(info) + + + + + + diff --git a/models/adam_ssf.py b/models/adam_ssf.py new file mode 100644 index 0000000..43a1a37 --- /dev/null +++ b/models/adam_ssf.py @@ -0,0 +1,180 @@ +import logging +import numpy as np +import torch +from torch import nn +from torch.serialization import load +from tqdm import tqdm +from torch import optim +from torch.nn import functional as F +from torch.utils.data import DataLoader +from utils.inc_net import IncrementalNet,SimpleCosineIncrementalNet,MultiBranchCosineIncrementalNet,SimpleVitNet +from models.base import BaseLearner +from utils.toolkit import target2onehot, tensor2numpy + +# tune the model at first session with ssf, and then conduct simplecil. + +num_workers = 8 + + +class Learner(BaseLearner): + def __init__(self, args): + super().__init__(args) + if 'ssf' not in args["convnet_type"]: + raise NotImplementedError('Scale requires Scale backbone') + if 'resnet' in args['convnet_type']: + self._network = SimpleCosineIncrementalNet(args, True) + self. batch_size=128 + self.init_lr=args["init_lr"] if args["init_lr"] is not None else 0.01 + else: + self._network = SimpleVitNet(args, True) + self. batch_size= args["batch_size"] + self. init_lr=args["init_lr"] + + self.weight_decay=args["weight_decay"] if args["weight_decay"] is not None else 0.0005 + self.min_lr=args['min_lr'] if args['min_lr'] is not None else 1e-8 + self.args=args + + def after_task(self): + self._known_classes = self._total_classes + + def replace_fc(self,trainloader, model, args): + # replace fc.weight with the embedding average of train data + model = model.eval() + embedding_list = [] + label_list = [] + with torch.no_grad(): + for i, batch in enumerate(trainloader): + (_,data,label)=batch + data=data.cuda() + label=label.cuda() + embedding = model(data)['features'] + embedding_list.append(embedding.cpu()) + label_list.append(label.cpu()) + embedding_list = torch.cat(embedding_list, dim=0) + label_list = torch.cat(label_list, dim=0) + + class_list=np.unique(self.train_dataset.labels) + proto_list = [] + for class_index in class_list: + # print('Replacing...',class_index) + data_index=(label_list==class_index).nonzero().squeeze(-1) + embedding=embedding_list[data_index] + proto=embedding.mean(0) + self._network.fc.weight.data[class_index]=proto + return model + + + def incremental_train(self, data_manager): + self._cur_task += 1 + self._total_classes = self._known_classes + data_manager.get_task_size(self._cur_task) + self._network.update_fc(self._total_classes) + logging.info("Learning on {}-{}".format(self._known_classes, self._total_classes)) + + train_dataset = data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="train", ) + self.train_dataset=train_dataset + self.data_manager=data_manager + self.train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + test_dataset = data_manager.get_dataset(np.arange(0, self._total_classes), source="test", mode="test" ) + self.test_loader = DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=num_workers) + + train_dataset_for_protonet=data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="test", ) + self.train_loader_for_protonet = DataLoader(train_dataset_for_protonet, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + + if len(self._multiple_gpus) > 1: + print('Multiple GPUs') + self._network = nn.DataParallel(self._network, self._multiple_gpus) + self._train(self.train_loader, self.test_loader, self.train_loader_for_protonet) + if len(self._multiple_gpus) > 1: + self._network = self._network.module + + def _train(self, train_loader, test_loader, train_loader_for_protonet): + + self._network.to(self._device) + + if self._cur_task == 0: + # Freeze the parameters for ViT. + if 'vit' in self.args['convnet_type']: + if isinstance(self._network.convnet, nn.Module): + for name, param in self._network.convnet.named_parameters(): + if "head." not in name and "ssf_scale" not in name and "ssf_shift_" not in name: + param.requires_grad = False + print('freezing parameters finished!') + else: + if isinstance(self._network.convnet, nn.Module): + for name, param in self._network.convnet.named_parameters(): + if "ssf_scale" not in name and "ssf_shift_" not in name: + param.requires_grad = False + if param.requires_grad == True: + print(name) + print('freezing parameters finished!') + + # show total parameters and trainable parameters + total_params = sum(p.numel() for p in self._network.parameters()) + print(f'{total_params:,} total parameters.') + total_trainable_params = sum( + p.numel() for p in self._network.parameters() if p.requires_grad) + print(f'{total_trainable_params:,} training parameters.') + if total_params != total_trainable_params: + for name, param in self._network.named_parameters(): + if param.requires_grad: + print(name, param.numel()) + + if self.args['optimizer']=='sgd': + optimizer = optim.SGD(self._network.parameters(), momentum=0.9, lr=self.init_lr,weight_decay=self.weight_decay) + elif self.args['optimizer']=='adam': + optimizer=optim.AdamW(self._network.parameters(), lr=self.init_lr, weight_decay=self.weight_decay) + scheduler=optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=self.args['tuned_epoch'], eta_min=self.min_lr) + self._init_train(train_loader, test_loader, optimizer, scheduler) + self.construct_dual_branch_network() + else: + pass + + self.replace_fc(train_loader_for_protonet, self._network, None) + + + def construct_dual_branch_network(self): + network = MultiBranchCosineIncrementalNet(self.args, True) + network.construct_dual_branch_network(self._network) + self._network=network.to(self._device) + + def _init_train(self, train_loader, test_loader, optimizer, scheduler): + prog_bar = tqdm(range(self.args['tuned_epoch'])) + for _, epoch in enumerate(prog_bar): + self._network.train() + losses = 0.0 + correct, total = 0, 0 + for i, (_, inputs, targets) in enumerate(train_loader): + inputs, targets = inputs.to(self._device), targets.to(self._device) + logits = self._network(inputs)["logits"] + + loss = F.cross_entropy(logits, targets) + optimizer.zero_grad() + loss.backward() + optimizer.step() + losses += loss.item() + + _, preds = torch.max(logits, dim=1) + correct += preds.eq(targets.expand_as(preds)).cpu().sum() + total += len(targets) + + scheduler.step() + train_acc = np.around(tensor2numpy(correct) * 100 / total, decimals=2) + + + test_acc = self._compute_accuracy(self._network, test_loader) + info = "Task {}, Epoch {}/{} => Loss {:.3f}, Train_accy {:.2f}, Test_accy {:.2f}".format( + self._cur_task, + epoch + 1, + self.args['tuned_epoch'], + losses / len(train_loader), + train_acc, + test_acc, + ) + prog_bar.set_description(info) + + logging.info(info) + + + + + diff --git a/models/adam_vpt.py b/models/adam_vpt.py new file mode 100644 index 0000000..93a5c89 --- /dev/null +++ b/models/adam_vpt.py @@ -0,0 +1,178 @@ +import logging +import numpy as np +import torch +from torch import nn +from torch.serialization import load +from tqdm import tqdm +from torch import optim +from torch.nn import functional as F +from torch.utils.data import DataLoader +from utils.inc_net import IncrementalNet,SimpleCosineIncrementalNet,MultiBranchCosineIncrementalNet,SimpleVitNet +from models.base import BaseLearner +from utils.toolkit import target2onehot, tensor2numpy + +# tune the model at first session with vpt, and then conduct simple shot. +num_workers = 8 + +class Learner(BaseLearner): + def __init__(self, args): + super().__init__(args) + + if 'vpt' not in args["convnet_type"]: + raise NotImplementedError('VPT requires VPT backbone') + + if 'resnet' in args['convnet_type']: + self._network = SimpleCosineIncrementalNet(args, True) + self. batch_size=128 + self.init_lr=args["init_lr"] if args["init_lr"] is not None else 0.01 + else: + self._network = SimpleVitNet(args, True) + self. batch_size= args["batch_size"] + self. init_lr=args["init_lr"] + + self.weight_decay=args["weight_decay"] if args["weight_decay"] is not None else 0.0005 + self.min_lr=args['min_lr'] if args['min_lr'] is not None else 1e-8 + self.args=args + + def after_task(self): + self._known_classes = self._total_classes + + def replace_fc(self,trainloader, model, args): + + model = model.eval() + embedding_list = [] + label_list = [] + # data_list=[] + with torch.no_grad(): + for i, batch in enumerate(trainloader): + (_,data,label)=batch + data=data.cuda() + label=label.cuda() + embedding = model(data)['features'] + embedding_list.append(embedding.cpu()) + label_list.append(label.cpu()) + embedding_list = torch.cat(embedding_list, dim=0) + label_list = torch.cat(label_list, dim=0) + + class_list=np.unique(self.train_dataset.labels) + proto_list = [] + for class_index in class_list: + # print('Replacing...',class_index) + data_index=(label_list==class_index).nonzero().squeeze(-1) + embedding=embedding_list[data_index] + proto=embedding.mean(0) + self._network.fc.weight.data[class_index]=proto + return model + + + + + def incremental_train(self, data_manager): + self._cur_task += 1 + self._total_classes = self._known_classes + data_manager.get_task_size(self._cur_task) + self._network.update_fc(self._total_classes) + logging.info("Learning on {}-{}".format(self._known_classes, self._total_classes)) + + train_dataset = data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="train", ) + self.train_dataset=train_dataset + self.data_manager=data_manager + self.train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + test_dataset = data_manager.get_dataset(np.arange(0, self._total_classes), source="test", mode="test" ) + self.test_loader = DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=num_workers) + + train_dataset_for_protonet=data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="test", ) + self.train_loader_for_protonet = DataLoader(train_dataset_for_protonet, batch_size=self.batch_size, shuffle=True, num_workers=num_workers) + + if len(self._multiple_gpus) > 1: + print('Multiple GPUs') + self._network = nn.DataParallel(self._network, self._multiple_gpus) + self._train(self.train_loader, self.test_loader, self.train_loader_for_protonet) + if len(self._multiple_gpus) > 1: + self._network = self._network.module + + def _train(self, train_loader, test_loader, train_loader_for_protonet): + + self._network.to(self._device) + + + if self._cur_task == 0: + + # Freeze the parameters for ViT. + total_params = sum(p.numel() for p in self._network.parameters()) + print(f'{total_params:,} total parameters.') + total_trainable_params = sum( + p.numel() for p in self._network.parameters() if p.requires_grad) + print(f'{total_trainable_params:,} training parameters.') + + # if some parameters are trainable, print the key name and corresponding parameter number + if total_params != total_trainable_params: + for name, param in self._network.named_parameters(): + if param.requires_grad: + print(name, param.numel()) + + if self.args['optimizer']=='sgd': + optimizer = optim.SGD(self._network.parameters(), momentum=0.9, lr=self.init_lr,weight_decay=self.weight_decay) + elif self.args['optimizer']=='adam': + optimizer=optim.AdamW(self._network.parameters(), lr=self.init_lr, weight_decay=self.weight_decay) + # optimizer=optim.AdamW(self._network.parameters(), lr=self.init_lr, weight_decay=self.weight_decay) + scheduler=optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=self.args['tuned_epoch'], eta_min=self.min_lr) + + self._init_train(train_loader, test_loader, optimizer, scheduler) + self.construct_dual_branch_network() + else: + pass + + self.replace_fc(train_loader_for_protonet, self._network, None) + + + def construct_dual_branch_network(self): + network = MultiBranchCosineIncrementalNet(self.args, True) + network.construct_dual_branch_network(self._network) + self._network=network.to(self._device) + + def _init_train(self, train_loader, test_loader, optimizer, scheduler): + prog_bar = tqdm(range(self.args['tuned_epoch'])) + for _, epoch in enumerate(prog_bar): + self._network.train() + losses = 0.0 + correct, total = 0, 0 + for i, (_, inputs, targets) in enumerate(train_loader): + inputs, targets = inputs.to(self._device), targets.to(self._device) + logits = self._network(inputs)["logits"] + + loss = F.cross_entropy(logits, targets) + optimizer.zero_grad() + loss.backward() + optimizer.step() + losses += loss.item() + + _, preds = torch.max(logits, dim=1) + correct += preds.eq(targets.expand_as(preds)).cpu().sum() + total += len(targets) + + scheduler.step() + train_acc = np.around(tensor2numpy(correct) * 100 / total, decimals=2) + + if epoch % 5 == 0: + info = "Task {}, Epoch {}/{} => Loss {:.3f}, Train_accy {:.2f}".format( + self._cur_task, + epoch + 1, + self.args['tuned_epoch'], + losses / len(train_loader), + train_acc, + ) + else: + test_acc = self._compute_accuracy(self._network, test_loader) + info = "Task {}, Epoch {}/{} => Loss {:.3f}, Train_accy {:.2f}, Test_accy {:.2f}".format( + self._cur_task, + epoch + 1, + self.args['tuned_epoch'], + losses / len(train_loader), + train_acc, + test_acc, + ) + prog_bar.set_description(info) + + logging.info(info) + + \ No newline at end of file diff --git a/models/base.py b/models/base.py new file mode 100644 index 0000000..3de6409 --- /dev/null +++ b/models/base.py @@ -0,0 +1,401 @@ +import copy +import logging +import numpy as np +import torch +from torch import nn +from torch.utils.data import DataLoader +from utils.toolkit import tensor2numpy, accuracy +from scipy.spatial.distance import cdist + +EPSILON = 1e-8 +batch_size = 64 + +class BaseLearner(object): + def __init__(self, args): + self._cur_task = -1 + self._known_classes = 0 + self._total_classes = 0 + self._network = None + self._old_network = None + self._data_memory, self._targets_memory = np.array([]), np.array([]) + self.topk = 5 + + self._memory_size = args["memory_size"] + self._memory_per_class = args.get("memory_per_class", None) + self._fixed_memory = args.get("fixed_memory", False) + self._device = args["device"][0] + self._multiple_gpus = args["device"] + + @property + def exemplar_size(self): + assert len(self._data_memory) == len( + self._targets_memory + ), "Exemplar size error." + return len(self._targets_memory) + + @property + def samples_per_class(self): + if self._fixed_memory: + return self._memory_per_class + else: + assert self._total_classes != 0, "Total classes is 0" + return self._memory_size // self._total_classes + + @property + def feature_dim(self): + if isinstance(self._network, nn.DataParallel): + return self._network.module.feature_dim + else: + return self._network.feature_dim + + def build_rehearsal_memory(self, data_manager, per_class): + if self._fixed_memory: + self._construct_exemplar_unified(data_manager, per_class) + else: + self._reduce_exemplar(data_manager, per_class) + self._construct_exemplar(data_manager, per_class) + + def tsne(self,showcenters=False,Normalize=False): + import umap + import matplotlib.pyplot as plt + print('now draw tsne results of extracted features.') + tot_classes=self._total_classes + test_dataset = self.data_manager.get_dataset(np.arange(0, tot_classes), source='test', mode='test') + valloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=4) + vectors, y_true = self._extract_vectors(valloader) + if showcenters: + fc_weight=self._network.fc.proj.cpu().detach().numpy()[:tot_classes] + print(fc_weight.shape) + vectors=np.vstack([vectors,fc_weight]) + + if Normalize: + vectors = vectors / np.linalg.norm(vectors, axis=1, keepdims=True) + + embedding = umap.UMAP(n_neighbors=5, + min_dist=0.3, + metric='correlation').fit_transform(vectors) + + if showcenters: + clssscenters=embedding[-tot_classes:,:] + centerlabels=np.arange(tot_classes) + embedding=embedding[:-tot_classes,:] + scatter=plt.scatter(embedding[:,0],embedding[:,1],c=y_true,s=20,cmap=plt.cm.get_cmap("tab20")) + plt.legend(*scatter.legend_elements()) + if showcenters: + plt.scatter(clssscenters[:,0],clssscenters[:,1],marker='*',s=50,c=centerlabels,cmap=plt.cm.get_cmap("tab20"),edgecolors='black') + + plt.savefig(str(self.args['model_name'])+str(tot_classes)+'tsne.pdf') + plt.close() + + + def save_checkpoint(self, filename): + self._network.cpu() + save_dict = { + "tasks": self._cur_task, + "model_state_dict": self._network.state_dict(), + } + torch.save(save_dict, "{}_{}.pkl".format(filename, self._cur_task)) + + def after_task(self): + pass + + def _evaluate(self, y_pred, y_true): + ret = {} + grouped = accuracy(y_pred.T[0], y_true, self._known_classes) + ret["grouped"] = grouped + ret["top1"] = grouped["total"] + ret["top{}".format(self.topk)] = np.around( + (y_pred.T == np.tile(y_true, (self.topk, 1))).sum() * 100 / len(y_true), + decimals=2, + ) + + return ret + + def eval_task(self): + y_pred, y_true = self._eval_cnn(self.test_loader) + cnn_accy = self._evaluate(y_pred, y_true) + + if hasattr(self, "_class_means"): + y_pred, y_true = self._eval_nme(self.test_loader, self._class_means) + nme_accy = self._evaluate(y_pred, y_true) + else: + nme_accy = None + + return cnn_accy, nme_accy + + def incremental_train(self): + pass + + def _train(self): + pass + + def _get_memory(self): + if len(self._data_memory) == 0: + return None + else: + return (self._data_memory, self._targets_memory) + + def _compute_accuracy(self, model, loader): + model.eval() + correct, total = 0, 0 + for i, (_, inputs, targets) in enumerate(loader): + inputs = inputs.to(self._device) + with torch.no_grad(): + outputs = model(inputs)["logits"] + predicts = torch.max(outputs, dim=1)[1] + correct += (predicts.cpu() == targets).sum() + total += len(targets) + + return np.around(tensor2numpy(correct) * 100 / total, decimals=2) + + def _eval_cnn(self, loader): + self._network.eval() + y_pred, y_true = [], [] + for _, (_, inputs, targets) in enumerate(loader): + inputs = inputs.to(self._device) + with torch.no_grad(): + outputs = self._network(inputs)["logits"] + predicts = torch.topk( + outputs, k=self.topk, dim=1, largest=True, sorted=True + )[ + 1 + ] # [bs, topk] + y_pred.append(predicts.cpu().numpy()) + y_true.append(targets.cpu().numpy()) + + return np.concatenate(y_pred), np.concatenate(y_true) # [N, topk] + + def _eval_nme(self, loader, class_means): + self._network.eval() + vectors, y_true = self._extract_vectors(loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + + dists = cdist(class_means, vectors, "sqeuclidean") # [nb_classes, N] + scores = dists.T # [N, nb_classes], choose the one with the smallest distance + + return np.argsort(scores, axis=1)[:, : self.topk], y_true # [N, topk] + + def _extract_vectors(self, loader): + self._network.eval() + vectors, targets = [], [] + for _, _inputs, _targets in loader: + _targets = _targets.numpy() + if isinstance(self._network, nn.DataParallel): + _vectors = tensor2numpy( + self._network.module.extract_vector(_inputs.to(self._device)) + ) + else: + _vectors = tensor2numpy( + self._network.extract_vector(_inputs.to(self._device)) + ) + + vectors.append(_vectors) + targets.append(_targets) + + return np.concatenate(vectors), np.concatenate(targets) + + def _reduce_exemplar(self, data_manager, m): + logging.info("Reducing exemplars...({} per classes)".format(m)) + dummy_data, dummy_targets = copy.deepcopy(self._data_memory), copy.deepcopy( + self._targets_memory + ) + self._class_means = np.zeros((self._total_classes, self.feature_dim)) + self._data_memory, self._targets_memory = np.array([]), np.array([]) + + for class_idx in range(self._known_classes): + mask = np.where(dummy_targets == class_idx)[0] + dd, dt = dummy_data[mask][:m], dummy_targets[mask][:m] + self._data_memory = ( + np.concatenate((self._data_memory, dd)) + if len(self._data_memory) != 0 + else dd + ) + self._targets_memory = ( + np.concatenate((self._targets_memory, dt)) + if len(self._targets_memory) != 0 + else dt + ) + + # Exemplar mean + idx_dataset = data_manager.get_dataset( + [], source="train", mode="test", appendent=(dd, dt) + ) + idx_loader = DataLoader( + idx_dataset, batch_size=batch_size, shuffle=False, num_workers=4 + ) + vectors, _ = self._extract_vectors(idx_loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + mean = np.mean(vectors, axis=0) + mean = mean / np.linalg.norm(mean) + + self._class_means[class_idx, :] = mean + + def _construct_exemplar(self, data_manager, m): + logging.info("Constructing exemplars...({} per classes)".format(m)) + for class_idx in range(self._known_classes, self._total_classes): + data, targets, idx_dataset = data_manager.get_dataset( + np.arange(class_idx, class_idx + 1), + source="train", + mode="test", + ret_data=True, + ) + idx_loader = DataLoader( + idx_dataset, batch_size=batch_size, shuffle=False, num_workers=4 + ) + vectors, _ = self._extract_vectors(idx_loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + class_mean = np.mean(vectors, axis=0) + + # Select + selected_exemplars = [] + exemplar_vectors = [] # [n, feature_dim] + for k in range(1, m + 1): + S = np.sum( + exemplar_vectors, axis=0 + ) # [feature_dim] sum of selected exemplars vectors + mu_p = (vectors + S) / k # [n, feature_dim] sum to all vectors + i = np.argmin(np.sqrt(np.sum((class_mean - mu_p) ** 2, axis=1))) + selected_exemplars.append( + np.array(data[i]) + ) # New object to avoid passing by inference + exemplar_vectors.append( + np.array(vectors[i]) + ) # New object to avoid passing by inference + + vectors = np.delete( + vectors, i, axis=0 + ) # Remove it to avoid duplicative selection + data = np.delete( + data, i, axis=0 + ) # Remove it to avoid duplicative selection + + # uniques = np.unique(selected_exemplars, axis=0) + # print('Unique elements: {}'.format(len(uniques))) + selected_exemplars = np.array(selected_exemplars) + exemplar_targets = np.full(m, class_idx) + self._data_memory = ( + np.concatenate((self._data_memory, selected_exemplars)) + if len(self._data_memory) != 0 + else selected_exemplars + ) + self._targets_memory = ( + np.concatenate((self._targets_memory, exemplar_targets)) + if len(self._targets_memory) != 0 + else exemplar_targets + ) + + # Exemplar mean + idx_dataset = data_manager.get_dataset( + [], + source="train", + mode="test", + appendent=(selected_exemplars, exemplar_targets), + ) + idx_loader = DataLoader( + idx_dataset, batch_size=batch_size, shuffle=False, num_workers=4 + ) + vectors, _ = self._extract_vectors(idx_loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + mean = np.mean(vectors, axis=0) + mean = mean / np.linalg.norm(mean) + + self._class_means[class_idx, :] = mean + + def _construct_exemplar_unified(self, data_manager, m): + logging.info( + "Constructing exemplars for new classes...({} per classes)".format(m) + ) + _class_means = np.zeros((self._total_classes, self.feature_dim)) + + # Calculate the means of old classes with newly trained network + for class_idx in range(self._known_classes): + mask = np.where(self._targets_memory == class_idx)[0] + class_data, class_targets = ( + self._data_memory[mask], + self._targets_memory[mask], + ) + + class_dset = data_manager.get_dataset( + [], source="train", mode="test", appendent=(class_data, class_targets) + ) + class_loader = DataLoader( + class_dset, batch_size=batch_size, shuffle=False, num_workers=4 + ) + vectors, _ = self._extract_vectors(class_loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + mean = np.mean(vectors, axis=0) + mean = mean / np.linalg.norm(mean) + + _class_means[class_idx, :] = mean + + # Construct exemplars for new classes and calculate the means + for class_idx in range(self._known_classes, self._total_classes): + data, targets, class_dset = data_manager.get_dataset( + np.arange(class_idx, class_idx + 1), + source="train", + mode="test", + ret_data=True, + ) + class_loader = DataLoader( + class_dset, batch_size=batch_size, shuffle=False, num_workers=4 + ) + + vectors, _ = self._extract_vectors(class_loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + class_mean = np.mean(vectors, axis=0) + + # Select + selected_exemplars = [] + exemplar_vectors = [] + for k in range(1, m + 1): + S = np.sum( + exemplar_vectors, axis=0 + ) # [feature_dim] sum of selected exemplars vectors + mu_p = (vectors + S) / k # [n, feature_dim] sum to all vectors + i = np.argmin(np.sqrt(np.sum((class_mean - mu_p) ** 2, axis=1))) + + selected_exemplars.append( + np.array(data[i]) + ) # New object to avoid passing by inference + exemplar_vectors.append( + np.array(vectors[i]) + ) # New object to avoid passing by inference + + vectors = np.delete( + vectors, i, axis=0 + ) # Remove it to avoid duplicative selection + data = np.delete( + data, i, axis=0 + ) # Remove it to avoid duplicative selection + + selected_exemplars = np.array(selected_exemplars) + exemplar_targets = np.full(m, class_idx) + self._data_memory = ( + np.concatenate((self._data_memory, selected_exemplars)) + if len(self._data_memory) != 0 + else selected_exemplars + ) + self._targets_memory = ( + np.concatenate((self._targets_memory, exemplar_targets)) + if len(self._targets_memory) != 0 + else exemplar_targets + ) + + # Exemplar mean + exemplar_dset = data_manager.get_dataset( + [], + source="train", + mode="test", + appendent=(selected_exemplars, exemplar_targets), + ) + exemplar_loader = DataLoader( + exemplar_dset, batch_size=batch_size, shuffle=False, num_workers=4 + ) + vectors, _ = self._extract_vectors(exemplar_loader) + vectors = (vectors.T / (np.linalg.norm(vectors.T, axis=0) + EPSILON)).T + mean = np.mean(vectors, axis=0) + mean = mean / np.linalg.norm(mean) + + _class_means[class_idx, :] = mean + + self._class_means = _class_means diff --git a/models/simplecil.py b/models/simplecil.py new file mode 100644 index 0000000..f5d616c --- /dev/null +++ b/models/simplecil.py @@ -0,0 +1,84 @@ +import logging +import numpy as np +import torch +from torch import nn +from torch.serialization import load +from tqdm import tqdm +from torch import optim +from torch.nn import functional as F +from torch.utils.data import DataLoader +from utils.inc_net import IncrementalNet,SimpleCosineIncrementalNet,SimpleVitNet +from models.base import BaseLearner +from utils.toolkit import target2onehot, tensor2numpy + + +num_workers = 8 +batch_size=128 + +class Learner(BaseLearner): + def __init__(self, args): + super().__init__(args) + self._network = SimpleVitNet(args, True) + self.args=args + + def after_task(self): + self._known_classes = self._total_classes + + def replace_fc(self,trainloader, model, args): + model = model.eval() + embedding_list = [] + label_list = [] + with torch.no_grad(): + for i, batch in enumerate(trainloader): + (_,data,label)=batch + data=data.cuda() + label=label.cuda() + embedding=model.convnet(data) + embedding_list.append(embedding.cpu()) + label_list.append(label.cpu()) + embedding_list = torch.cat(embedding_list, dim=0) + label_list = torch.cat(label_list, dim=0) + + class_list=np.unique(self.train_dataset.labels) + proto_list = [] + for class_index in class_list: + # print('Replacing...',class_index) + data_index=(label_list==class_index).nonzero().squeeze(-1) + embedding=embedding_list[data_index] + proto=embedding.mean(0) + self._network.fc.weight.data[class_index]=proto + return model + + + def incremental_train(self, data_manager): + self._cur_task += 1 + self._total_classes = self._known_classes + data_manager.get_task_size(self._cur_task) + self._network.update_fc(self._total_classes) + logging.info("Learning on {}-{}".format(self._known_classes, self._total_classes)) + + train_dataset = data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="train", ) + self.train_dataset=train_dataset + self.data_manager=data_manager + self.train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) + test_dataset = data_manager.get_dataset(np.arange(0, self._total_classes), source="test", mode="test" ) + self.test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers) + + train_dataset_for_protonet=data_manager.get_dataset(np.arange(self._known_classes, self._total_classes),source="train", mode="test", ) + self.train_loader_for_protonet = DataLoader(train_dataset_for_protonet, batch_size=batch_size, shuffle=True, num_workers=num_workers) + + if len(self._multiple_gpus) > 1: + print('Multiple GPUs') + self._network = nn.DataParallel(self._network, self._multiple_gpus) + self._train(self.train_loader, self.test_loader, self.train_loader_for_protonet) + if len(self._multiple_gpus) > 1: + self._network = self._network.module + + def _train(self, train_loader, test_loader, train_loader_for_protonet): + + self._network.to(self._device) + self.replace_fc(train_loader_for_protonet, self._network, None) + + + + + \ No newline at end of file diff --git a/trainer.py b/trainer.py new file mode 100644 index 0000000..e6554f2 --- /dev/null +++ b/trainer.py @@ -0,0 +1,128 @@ +import sys +import logging +import copy +import torch +from utils import factory +from utils.data_manager import DataManager +from utils.toolkit import count_parameters +import os + + +def train(args): + seed_list = copy.deepcopy(args["seed"]) + device = copy.deepcopy(args["device"]) + + for seed in seed_list: + args["seed"] = seed + args["device"] = device + _train(args) + + +def _train(args): + + init_cls = 0 if args ["init_cls"] == args["increment"] else args["init_cls"] + logs_name = "logs/{}/{}/{}/{}".format(args["model_name"],args["dataset"], init_cls, args['increment']) + + if not os.path.exists(logs_name): + os.makedirs(logs_name) + + logfilename = "logs/{}/{}/{}/{}/{}_{}_{}".format( + args["model_name"], + args["dataset"], + init_cls, + args["increment"], + args["prefix"], + args["seed"], + args["convnet_type"], + ) + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(filename)s] => %(message)s", + handlers=[ + logging.FileHandler(filename=logfilename + ".log"), + logging.StreamHandler(sys.stdout), + ], + ) + + _set_random() + _set_device(args) + print_args(args) + data_manager = DataManager( + args["dataset"], + args["shuffle"], + args["seed"], + args["init_cls"], + args["increment"], + ) + model = factory.get_model(args["model_name"], args) + + cnn_curve, nme_curve = {"top1": [], "top5": []}, {"top1": [], "top5": []} + for task in range(data_manager.nb_tasks): + logging.info("All params: {}".format(count_parameters(model._network))) + logging.info( + "Trainable params: {}".format(count_parameters(model._network, True)) + ) + model.incremental_train(data_manager) + cnn_accy, nme_accy = model.eval_task() + model.after_task() + + if nme_accy is not None: + logging.info("CNN: {}".format(cnn_accy["grouped"])) + logging.info("NME: {}".format(nme_accy["grouped"])) + + cnn_curve["top1"].append(cnn_accy["top1"]) + cnn_curve["top5"].append(cnn_accy["top5"]) + + nme_curve["top1"].append(nme_accy["top1"]) + nme_curve["top5"].append(nme_accy["top5"]) + + logging.info("CNN top1 curve: {}".format(cnn_curve["top1"])) + logging.info("CNN top5 curve: {}".format(cnn_curve["top5"])) + logging.info("NME top1 curve: {}".format(nme_curve["top1"])) + logging.info("NME top5 curve: {}\n".format(nme_curve["top5"])) + + print('Average Accuracy (CNN):', sum(cnn_curve["top1"])/len(cnn_curve["top1"])) + print('Average Accuracy (NME):', sum(nme_curve["top1"])/len(nme_curve["top1"])) + + logging.info("Average Accuracy (CNN): {}".format(sum(cnn_curve["top1"])/len(cnn_curve["top1"]))) + logging.info("Average Accuracy (NME): {}".format(sum(nme_curve["top1"])/len(nme_curve["top1"]))) + else: + logging.info("No NME accuracy.") + logging.info("CNN: {}".format(cnn_accy["grouped"])) + + cnn_curve["top1"].append(cnn_accy["top1"]) + cnn_curve["top5"].append(cnn_accy["top5"]) + + logging.info("CNN top1 curve: {}".format(cnn_curve["top1"])) + logging.info("CNN top5 curve: {}\n".format(cnn_curve["top5"])) + + print('Average Accuracy (CNN):', sum(cnn_curve["top1"])/len(cnn_curve["top1"])) + logging.info("Average Accuracy (CNN): {}".format(sum(cnn_curve["top1"])/len(cnn_curve["top1"]))) + + +def _set_device(args): + device_type = args["device"] + gpus = [] + + for device in device_type: + if device_type == -1: + device = torch.device("cpu") + else: + device = torch.device("cuda:{}".format(device)) + + gpus.append(device) + + args["device"] = gpus + + +def _set_random(): + torch.manual_seed(1) + torch.cuda.manual_seed(1) + torch.cuda.manual_seed_all(1) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def print_args(args): + for key, value in args.items(): + logging.info("{}: {}".format(key, value)) diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/__pycache__/__init__.cpython-39.pyc b/utils/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a5014954c987be1fa993ad7cc376e9ed1d226fb0 GIT binary patch literal 132 zcmYe~<>g`kf(|Y26cGIwL?8o3AjbiSi&=m~3PUi1CZpdfbFM?|esim}7(43V-mEu$KgO|R8# z9Mt#GN#$0(BfNI&pwig$Jm0N&9i*K~u;*6px_(euf3tGCclWg}HEFs&>MMzwRokw_ z-+TA!?QY9`M=>s6zFgrp2bD{zLDcU`bwnzV^^U*e$+jyX8nszzlBLx#{gt|3saI|u zYSJ_Njj&`dQLY>O{tJl1C7Xw`RV5E1le@N(->o@PaE9-Avtezi}OU9iHV_afL z)26rSc6?8+RDTeS_sHn~6e}jpgfvpfB_HK0NY{}3Pp~inb{Fj|+EcRO z=CBI`S7Mv>EiCraH~4!}3r*<-^}y}tzzr?mYt>mOL&bc(7kCY?gAL{fv95kCwA>GS z^%kijuh7pDq5Tz0{Grw-kF|@0?;o{3?Xob_@LHaf7@53!?2GT6|RD62MOTZfhsZ?KE30R(_<(FGK4;&#$#R!CI^5 z2hv>=Kfb@`_13zLW^1h%G+X`}4XWPTZM&UdwR;d+j??Tk1IL+0TmMb6D>bdhzmw*k z++P?yr>lw9k@H6LMKjCijcg6HN4n61)SA%?tLZjt&WQ8gX`6~WGquJiwwj-zJG{rrhmWTjfu*{CrsJB{<^9D*j zEtaOw>&FZsl4{VRB_HJ}@ersf14wG){_<7K>(yvpmG7-Y%f@{l>1^FG#z?m-9p~#% z=F=Fp<4lQUHmbU$2|UWuAUgH7>o}q9IBie#T9nT_4sFi3ChIuDYdDVlD%!|ZBCivn zsUIRVDU!xQs(m^imY&YrXRS1I8(@rlw2W7fo*xVg*KU0K>a7c}N7GVwF_Zrt3*Q>U z!dT0Rp>U+vmxhFwhJfjR$H8aD@O&`Hn`Y!F(nhNw@jm`s=p*3E(S=C?T|pE9Y9-LA z7)GWEb^r%+N@f~vdloVaBB!7x0*yrl5oNIi=vfx06vu|nJoKDWdX}N#w3r5+jP)!N zc%oHNoQ^qJ=rJHOp`g!(ya51$4Y5oawoL(iiVD(8=!M{MNy$u$GZ^VB0M%q%Y3Qk- zuX#~XqgcSW$f~n3rvwdo<|IH@fc#l#KBF}2=xYk&T2gw-s6CDOnOA+4X!b_tkZN;) z)iX+_fPQBg_7e+v&RGx7&k*#znn2(0YH|syEN6(!5m_cO3-WY_zrWSw1hE!~6~W@C zwb16h+X6D)k$I{v5jjPKcFxm3enQ{hYm!hShPuB3O&PPcB%|p-$CjL=mQzH?y-31L zxk!Yr{-J$WHh1>|f<0LzZh=Ud2>Dt$Pvi`dvqT2&i#-=Px9dnR4qo#khwBT(F8MXG zcZJA#BF|V(9=VC+lk;T^8nfhiAZTeAg5n_!BamD}FVEUQfY3%tpl8g2QRI;qMs@Ir zco{CD;+v#s1VD)U1ptk z1fTP;a7o#-Os_liHJC#JgEBaCBI%uMCOsLYIfqsRHgif3$=Fzn!5orI8ITog9s-Ti z8!>1TS?oRo(K3#(AWD`NCSK~6d<8<$C6zQMA8jt$ zJ$y;M0(T$17fHusBl-XAUYJEO$d>Yvc@HrX3oX0B0)CCuGSLOr2%JC5gs*0nf{gJ6n^u-M;H*Um6cCrSxf@5LdW(7X|a zXj0t^q3v~DSr3|C2YJLw5jN=w_d=NJt`F5AujI>-u4Or`Ksbdv~Lo?{AU(>mXVxuMd8wRNm6@`^=o?f8^8Szt7AmQ@8b^ zPW1k?3O60^9La-_593ow8v-XrdyE~pum2ElghTC7iuUQgfmjZmrl>BBy0=l6;X0G* z4k`AoZ&b6g3N2F2k-S4iH8Ju76sxIdB448_zH|<5o^*}=6OvCGTEj4+-{?Rh_nHJy zsu{TsSxFHfNkOln5XnIMpbPR(zd>T>ZRz$o_E=Rb%yfF~?t!Ga2o0}`u$^~z-)#gO hIguAhGr7me!5yPHwBC(^H6`uFbzU;N*#A#l*kOr1EUZlnhK(gQ;LPCg#kdPo!E65Lon3gsYNUf=aDg;85w&6Q>cGup- zZp<2T=n#-Q8HxdQxemdeokV#oj9wOOz;LvfKS*a_#}9fCBY{JpJFMNJ|~sZ zkY`vH@~n_&!1u6T@V$c1g3qx&@O^^s0pHIyf!`$fUho5KGx*Jd&w<~<2Eh*sz7PCX zb_@7h1m6#S8`}|{T zc{$n?)IB#WyH(4vgR*B$*P#*f;(7HbgXt=kIKT3GOAgOJ+=_T=0tf3{#Tt6G>1T`wrmj)&`KxQ;Ivw0aZ7 zlr`}Y(rY$KLl4qW2GZUrHmBO!jLsCMGHpRY+h95~7G$&%EWykL*#S~;5J_8WrWPg0 z(=18ISWq0Ll(DnT9+p{@o4t!tDaW#`2Z-roz0goDGTpHwiPmr3z`g zT~svaP{vS$&x3^0j69<-8Ha)6SWxHXjznuol!iQ`hFVj{e`8Trq=r;ZpqdS(C09D~ zB;+Y3osgFFi8*6f=xh>j)denqTK~zK=UZ$%pXLf4AL$i$+Tu~7b!L%9T8@-jJ<=`C ztus*~w7Ap8IiMx_vn`mlx?-e>o=gn7xMN9;w7~YJORe!V0ya~M2er5r0P-GMQMSEC z@u*#NP&p7u?#BmkyOL3f&TVPydAREf6Zw8l=;R3^1kuQx4eWB=3Quu5{wQ_asxiMh z#{GIU5VsHe+@14lq2-NxZaqqm`JT^@x?yN@o}_t5k&!v+PujJ>=TUM3=5BfJoXykJ zy_>oX3y#3CBO~-Z7dDikp1njmCKk77o4gl<`hv?H@+6eIjYV1<(T zZPY?wl=z22ZB2tZ!Z3;>eGRmgX=3q6@EZVWYe287A; zN(27v32k0&Nc;d(=QXCy>*U!QN<(YtOkYsJDU=E(Q(m7fkZ z1`jtNot5X!MxsgIHW~mbOD!mivMj~2W}*RZ6_1%>4ve^ zp}bjyEN;GDbWV!$cgP|=s92sI>D&rkpATRN-wfhBxBCeG{#slnB_P&k-5MN3-S=o* zEpNu|r7w0V$L#>_fsvPqRYh8gRD1=FqF%RaEHdITQEJSliD+m(Su))KK^sv#(%?mr zU%0jUY*?n*N=YteMS8*SrDfemb0#r5PCz0#gu^AT)Q!QYcWT!47)=`w{3N~?5Q-(Z znpOx;^pLiN1A?~Y4Am=|nv;7}egq>cG<ToALg=`7`lsi&75-3ko*(;)_u9XH zUqt8561(@+hh8i?XO{QuGr#vj(RuL&QCczNbs)0~JhX2@>lkWq41~!+<|r;LXXP&g zpYt-jlL9X!H{eO;;hE6FPUp2yX{a;mS@|&qd6kB$L!xyg1|*4Aw#6yUo7?+{KRXm4 zarW3yP`8m>of!&G*+bL5hirSupB_5n&+?&G>ND5>=%LZc(Y?n;Cl5bf5Jq2^%dM83 zw!FfU8K-J#ad6{eCQeyxq`6fmSSt`M>HH21D3NQRoXzDWQzT~8q6-TNDHCgDZfIWR zGDPC?Jmv@}h)Kd7XbRw^^viSGyL8wM^@;qznpmgv=S3nbfr}yse}D*SMkHBrF#9h< z*rzeeIn+)FGi8LZcQnJ4&3853kOx3v95k+i+EqHnF$3}X#sPpQ0DqII!X%O{lRzfb zObe69z!)@P5;{!65GIkedz!s00h7qVB>LhTFPR1fsVKAhI9nF+87zWN5P66Q(T^&L%5-L39e{)fJ zbz*L7d?74|w-Yrx+{@ceXK;-kygewqx9%^}kx=N7C_!v{hpj8y>()De5F=M%NT7+_ z3;!I*&rsWOjHZ$`6{bo(H%%R*PzR@5`TL{}Yh87q=lbe+rG0|ysAF`a>Uabb{kAPKSJFc&VzqJppOACG2)kcrM zgc;sw&(d}L)N_N6Kj}}+*cH453vad(;^Xf;enkn_bX;EvN%+%iJ=0)k@{OwEQ5-m> zq}`5*EP;A%uqv{CwdPLQwaTfg#h-vW+&i$8o8z0dw*o@f651-+wszCl>jMwaPTzrIE&mTV+=%ToJZftH3lhu*q#h#LkZ zcwOF-n7DH!>CQm`y{#o$&0g|g)VNwpS9Ow>@+onp;3u%YNUw8xE0KwR76|>a%f2M; z7km=kYh5qaO$PUqKE%zqlMuA74E?lJoolBRZBhj&5>T4t>Tx6-L(94=<~{{ z({6xwF!8Qb5@*Ga7Q>H`=1W(Q{=IMkIO0$&Isd{YEkzL#0u(>+={ zp{KQO&7Eb*CORoJQ1in$d!@5Vf3EwCF5GOU_cX z>?}ts&I;3B)qKk@KhpeiuzGAbYbe`(1?5U$p}g)_{n{hV+3@Rr1J4V7(_g^zqTlit z@%)ItwWp&ZXllD-p(7drD?ybS6|4h)0W#a`ImF!##dg+E6Ltr&=qkY@{@EN6fK(#TVvO3cgeezSR5`e-(Y}{+hpzXTz_6PZ#qVvE_?i3?}ieg?T5x zGBPEk8%XIRkc?%Tt7X~=Lr%|(Q-f)lDfCP`G>+L3OF!{>h{`d0i)H3M^)*EaZRRn{ zO6%IRpT^&y+7l})oie7~y*g=`^xn|MR%ZG7k;(tE=xI?;U&Dfp-O@ zhF+Qm=_Y4bU#FbceRale>_tIdb8+0mflHOS5e9KycjX&y!u^1wy&rdZ5CySFb0Zx_ zx#bN8LF{8BGY!HXU&OF^IXxKmdSO6yfe&$s`vT*n5SPj8xs~*KnCxARU&br%J+~Fb zVk;b`f(Kjv{n0@(-12-c+8T;}m~PR6+(_|!bMP>?U91Fyx}=ur5{Sku)?{nUW*c~} z;5k~EIjGIa3tigyouEFd6jt2n>*{!J+8dxIDJwE2!5|zm0mSNJV{9K-*7BnSQDi%8+k= zO;{)8eL7V89VpmghI&rtLHt@@~-n`gk)pdtraDua5YCJ_9?#2Jf{doukB;y?#bo_^zH;!sNaS zGqWG$)r0>2fy#FbzJ+$a4U$_CDn+2cYXxv5SE1mlSrmlFA(v@hb!ly?t)cSMra$(= zVenbb6P{aS)I7n-g6HKMNi;}O0zx&i;AxLAFo~ihMjcAUxRiHRZhNtxM4wA|f0V>f zgdOZRF9StGSTPBwlyP|!@pak-;YqH?gS^&F;x4ulBjxp*!zg-4>HJ z(`8FT7a^pi)ZDc%Xt*25em>ySUX{2x(lKiHfsTcTY6I`M7Ej9bY?!$_3Hb& zVg9fF#44j6tyQ#9YyC;xG+FaW)u^(%PH$9MRbSIBwx*BPZ({tylR>ikzb|8PZD{7$ft<>4w2s_Lh!;rP2~58yh!BtiM&MQ8c1#d`v4}n)jdd{ z?g}f+u%8OJTEGR%li(H{YQSuM+tji1hgTaRHwN7%F6SY#M+x-lW1D^aSD4--k z=--fmTM5M(o0)mSPBjVw;JcWIh7QlfHIYgvSx9AB15tD)j5`>P|0$8FvO1Nim*CW8b|TJaT1*Nx5n{5bQiY2bWtkM& zfkmG>2}-|Trhp@AC*_lwXBdKEnASW4l^jph^#$QhiH;UnT?ju$rX8ObG}3c-KX350Sd6baJkd^=QTv^B?K4-4}a;_EP+V> z(DWH1bqf|6F?v?o&{Sy!is~D%?_=rwkO+w$3>O#(yP-t$4XQHJgo{o^iO^GcD&0zcgsx6g`L=-%6$6`{ z3JKygb8cPXG?TA97b_E{kU&W=CA9$zX=7cm z6G{fe24UhUKy4B`o5~84P37bc%KamZkfC%*YKsz?DIO{Cv49)6^7tpbQV5B~MoY6o zqB`J4We0Zyg^V`oE`lykTo=&e!ZRO+l&`z)-Jut%7T(67xjon>AWF5>FUCXa~I8#{AqPAX4h-9SnSE|r-B)p5I} zryJATExOyH8!*iU0Du}989fBB7+K#PwTQ`W64`b@LD@$;CFT8CeY%Q7;1ha#L6}O< zMEN%*C96Zs%I^^&r$si+@Jq6)jfNj1rSb+Bu?XQAPIh%R==znfhvsfb;XxS9X|_ctWVD#FI9X#B&&DfT+CkpqnBx<6}KuleH9M z0BNY6G+|7=EGmsr2fiB=%Bv-p6_b>YH|U2hnMZJ4?tzx*RC+wYl`HoiO3rpn#cFcz zU{B~k@{&A{^SiCNWP1A|h+Oxd(YxR{MR03~;#2UQ$Q#Wy-_TAK+5(Rpe(T`Z^V*tnVZDFGz)~aVpPkhj>BaB~|u^ z$UuBF_=0fK8Ol<3Cm<3cY=21-|jQIzQB?7A?EQVFG9Q7TFnN^c6QD3wt< z6y>61qx3-7MX7?)+oDpGswn-fs1~IfO7Dmol|h}4P951KFUCgK)FUUY>X`0@%$28cLSa@}&1 z491MC%VYJ>^6yCLHi&f80ATiqO9rR6=Y)XdnN}@@;1y)()*Ig;^zVFp%wSV-{1QX- z6I`{Pve$8)=^G~G=Gb6h|MLL*Z3Hp&{}Uu1*wQiIJE*qtyB+jil$iq|2I-4iTZ4xp z;oXDHgzst_CQfy8 zJfab&JR;jhKoLSwM!2!T_Z*1h>)?fnuQU%g6)yxV+cC4ilRMfrKKT)tjQ;cgiz{BY zD@D8APupJHMiFbM!v?DDzRa5r@~|FY>=Z-3QKy&PmGgCy-HR6tOV^UIg5bOu)!xvYGTnN1YF@ zun~*G!FUTVs4#=&i!ep5WS_g|b&v2tORkc3RGX1c{w;ck)*%IEvLRV}ZbMB!=y}6X z6#Hqyj-(_{TB%NV4?6!g@||aRfJa`!my^Lm1^32lx4xj5*HVfjKY(o#2ys$e{+I}1 zxKo{JI>RFgMG}-g1Rh!Hd#m&lrR9_+Lqdk!qS`F2b>_t6RtxK<{M~D0CMUiqofiHt T7_Gn1KGj-$t9l&az)pYQHxf1j18*XshtOW$J=1^CoWGjIk&T++kDfRSeifzdJe)>N}? z*jVWP0+(E4s$dQ8u%Z=Z21xYED-~n4a+~j4T?SoRqi411ZROMnhD3*0+ r⪚y61*r+FadG#^Zw6Nbnw*3CuTQ}jpQP$-80lI6Mn8$zj47Y3Urx# literal 0 HcmV?d00001 diff --git a/utils/__pycache__/inc_net.cpython-39.pyc b/utils/__pycache__/inc_net.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c41adbcf0b0aa7d480f03a587857910890941552 GIT binary patch literal 22898 zcmdUXdyre#dEb2jxWHnu*yVEh6b;jQK_Qn9i`2uEMlxk~oMASK$jB-p3j%TP?qZ3@ z^4z<8z<}eLa_qKNqS%S^@^qFRk2{-q9RHEDPMc2UrfE{QY2Bnr+uO8h>vX1?rfFu{ z{*lZi>hJg62Ob1S){15lVD7;==bn4ccfRwT?|Ck2v$Hu3pWl2q?|gez)4t8Z@W(>o z8C+gL*EC0S^rlt`fAxwkWus!?*Jv6ovtsJJXEv=?s*-A@E9q9Il94*SX;*CLJx_>UOlowGx z;2cExppqOOAd)E0^9PrdG;&V6@w1Gss_Pv^JaW`9|BVsbK!XX0zd+ z?RY3gRr%r4xw^r}qt7n<&fq$W%llzd(|W-5K5bjOrS*-T;TV?<^^#*cR#R^oeY0iu zt(MhKwNm|bE8Pce{ge6|db{nsCmQ)5Mgp*w3SUw4iOl&0#T@1y8x zE`oL$W0*<-+GD`eQoo2kOy_uWPit>~9+1xi2Pd49&ikE5dvn;!Dcn8vdbYm+h)>J2 zkN1ij`%s&59_uYQkH2oktJyw@Hcv>Km(gbb#5M)M_~iD1o^eYjM#nMS9=q>g*OWi7 z66EWh_SLrQSN$8CZe`b|a(z|9=60&e^-zD}aRnZ!l~aKoSH!vZKN{yw9S^e6-Qy>c z?5UII%0`eIbQxGRwdQrp%f|+F)mF!Go8Gaw^P{Wpf9&{UCs&>2O@AFsDDC;GQTHc6 zcIx<@dOCId=!p|G=l!d-)9k6TI5cDeYWI;lfO7oA(Yo`*@yAx{C#Usr>f{}JICb*q z>f0rqKtzd2@02LUZXC(vZ7)= z=enB}8_mK^l=0SU&1UCXC37Ll6hlC&e&@2=u3|mQW+j_+5EP#mII9=q+kyS!^B1MF zz^-C}HhotGnblQ!Hpt78Mb(%r7W}ql8{D-&RP$~;hb7W%+-6|4J8i(-?5JAT zJidIQoULTpyF69NG27YnK?;?FNi}GpS-at?Hs~V0yJz@rqYaXd9$Of`;ni!+no%C*tOE}KTOT51>$#|4=L4Am8>*a2F}zH8>Kc?qwQB_JC}Sg?TFqh zGXh6QbLu{{tJp4xfFP+uEFPS=@gh7_4o|#0f~z1+_E7W7yVNW?3>lTh-7pMWz6TWDqBPC;6+P0q+6&u8YMHT&@);4 zBx+Mqn?|k0d%m%4QnT(`J?g9fqGxSej=i1gnYVQ6Lp^Of4Gk&P(E90~v7Nc4J*R#2 z!IzDG2HI0*+h#kZdUh|}%lO%zc@uWjr;L5t8SRR4vi)o?9X<2a_?cb`kbK$C^=v>6 zE%v6Nz6l+Fwr_WrrM7^Y|BN0}z04U6x*mREJIwWLfd>}WV_HA^5$(#CEzK{!q2G8x z*Qg_Qp8@RiJ!tlT8aZbMv&ckGFWjQdfbW~tzwVYOL0T@Pk;q{OO!7$S+Iq)xOJH9vL^Z^9sR0g(wzAzzHAPk`5_?+1 z`np@cyx9RGi&`&tk9}~}LFn>R}2e=eabU3PCg?3KgWeyrbp_-qaQq0@e{)M|LJVb|CoygPv6Nj%tn zBx$K!XRGbh0Ns?f<=yH5;7dJ-B*-*7YisB@$b#*hM;?;fYP7*FkFw1vwn;U7x8=z? zD&M=-@Ye&g>0)6a$JqtX{b&)B&k+8A6uQv67RlkV23K23H%`gET>gl*>Z)h0-fXNei_s z(8{7$X1ox?)q2|xOjl7Dv0DyQvH2)-Ak`4DVM)JSnTzpPzae$PzQg1MyWbHk#Ip$Xj5+Iekff!B~wnSZYqsnPjoNW19ou@~3S0?r`Dm4N zK_-%^fw5ZOF+qH8 zb9JB%?C3$Z8k#M5iXvNaxzzLEF(+U?*KfjVdGA(bY$om+_K8gPB!mTU&}+S?d>Z-xDX5(cKjfI&^0{!|p6b1J>m~Gv!UUlnG41WoYLqPQI zQVhswlMz@52OXCJ3wccpOCN-?Euw#)1_2Jo+O|O$A%=_#n))c3mdy}H-;aEd-r58+ zad*rRS2fm|V6WrwWJ(iU5-|RRtLt0L#TC z3UcjBu~j=ha5e<6x?2AxTE;kF>gx^jQPaKfFpES z!Xu;wQORQ`*k3G?WMlzI-EG?xf)FPK&Cm8}MBn(CUWIqD(w649cHQ5?t%w-7Z*8Mnh@lJ{lxN8;XbM!TL9R*@2bu};`D*pbR;?M{5k3Hm6lR;N z{2S_Jj(CX){PmhzXR^VBI8h#wU^ZeWb+_4!S&D3N^6QYfkXQX2E^h$|7)uuNz_d-f zfD2#YWp&&d`H)2U@m7$;eBUVw=ZEtATDAhN@1(%(EeHG_zi>V~S>gUUX9m9+C-2PS z*LDid9DcLH?Q_EI3-DnmvEgb>F#U7&csLHpJTKCY_aqWuyD3{4UOalYO>lO@M+)w5 zsMBD2NF8jbHA(;r3!I{K5Z(_~l;J|wYprrxQE&>5RTsG+vtILRzORCUEH>Iy*E^ld zUQmRxFj58cS~zH8)=39SaGWqy>G^jFbi~Cekii_fdq!tFVCE#)_>)G4vld3yMux9W$Ta5DO(KBO(BXm+HZ0;XOz~^Hatd4Z6OufqO|= zdlgKPxJ+14IUACf(4BC_JdN;}mxpB60&GG0n%h`g_tmp3SeF_#59k#X(Iuhe*(`V& z9^V8H{XtfKf{9>jfHIQww{Zz=Qm3IYSPZ|0rFZv^p>DwChH*#Pm%=5u`yw*H3oy3< zy!qw^@X*s|Ko6f{^a|-JEHGMxjtMWf8MXjc&}L!_%Qt~(SVnzl7sy%2rD4&f4ru+1 zpLVQX<|gSf8gqQCl(tXnWu5>zpAnkeNv*s^;ML^M76;O&Z)Qc z&7K*{1YMc9vlH#N2J)d$B1Me>D75;=sQ1`2;4-@V$6y&2O~VAkF_O*p-6+UrgLW`o zYY>J5VAVM+v?VDvlw)FJX&W&99Sb>V7Uk56dIdcShp5a)%tyFM13oLSvMa8F($lyq zdZ$;;3^Wl{=hGjHy9gE%PQ(YLk6bjE{_NoRRwECt^{l?jbSU+ zTc|L^ALfazv$(u-NJ!=quGGn7rG)LYEhQ@5yh}n4?=ZIpn_*2iP*WwlG0ARr%oVYz zWfOqA_Rw>X2JLpPy6VSKP+wp&K_3Vfw%X(D*#?ve(2ULxGb%cKHlFEKTp}<&jRNPl9yTBTkYDsGmkvS(g=s%KbAa!Ppn5(~|+t zN0cb=E)C$tvYxtUc!&EKqP5HrdC@YdWD!IPU7^8}a@QDFQPq*jhSa$`xjEc3CM06- zZ=ox7i-~MNSphNa`ly@Z6sUb^`htWkP!~ zR0luDoalt2BK{PM98z6pGGGP+b?^pm{0CgZ5#S)m*m*l=r*LHy@6~aQe8@PT!R0L? zNmN0|PbhV!1I}Hxg3O1{pL~|)yAT%I@xzcnB9^IfB`S^4GxumvYoM$}YE2WORZ1JF zjdUbx!*WK-5VsW3be{Gswh2q%xlvB)(Ke~4dV?N@)f;mCZt1!22IeJ{SyYvv<-Ehh@i3b^3d zDvQ29SVGB3-(N!C?`7Xy8XG4(@N?U;#Vg@0<8HES_MJ3%(a_Ua&}^_6o?jUyPT|%g zOUyxdk&k(MkxbIn2l}NsP{9Km>LDzh>o;y!uKx}?nxtGWbIvj^Xo}tYCV@Cmt_f27 zArEm64nj6Hm(hkMDLq_-X^Rg9X*WC?}hxGp{*(ZF1-pMWQ-vDT`ok7G8hI9W~& z6!41e*SyP&APSXM7U0ypO6id3@+T(I>q+8w2^|h;;_xf($G1Rj{Ln;hB_2KM?bcpCXWT>^>WF1?W~bmjiF*36Xd4P5)-k?v}hC%iYmNqSzS2V zdB|&YTPzQ$nUHg$*Js8z^MGXwx1P$Lzl_n3a&#gO?AfyC6zl4^Mm{lpdq+?m@9%+* zLe@;bEbQ~pE1ODocP7Yw5Ek2c2ChexAPg&O#;(1mk&OFWw=JKnH?*Cg+JL=g$hc8! zkJM&`l&uv(29``+L2u<;xc$F^d?>=mztl5K9zYUU2*7(ms@B|GuLY^Dt2!QxOV_W3 zilE4_0T(1jzcU877O&TRJ{qn)mPD-`Wlj$@@mKjSp>@wes8b2L2ey3bC|EB5T~h* zJ440IbCFY`MXSM&bm;8`vpYYS)Lm6{%4qH-D&*Wb{9UEPvw|B?RtO|QF9@{=s1r5p z9Am@c6Z3w6z;Kl`y}LL7W6%JSa!hIop|9cz^@~iV5<011C0;(brBcv8Sa%zt$LlBN zM@;BW;ZA~Vnk!1SKo)#tE*LX@K7x1?=XVEJ)FS|Hl6d0=Sc6*=n<2FZq=#XL5aF32 zfj4&v{UheC6g=Gsw}sKH-1mqp@I3U+U`ATVff5A5goQJB<=rkJ{~l30VL!Y&AfAGm zMi60Kzl`TO*dDUF=#C%q9G(V{?LnJts$C4yQf){}Y!bRqKx)=a9iiJt5y-}2RXCeW zpYAE@V~~dX5FZ+p9cX`H-Ay=Pt{jK-Lr=7yw=)ADeHrZI6~G{&xQ6eAV$mM5l!=8P z>q5+lgIaU&nt8fjk5m)rSaw*#wc}Q60l(O3&@OU9-$C?m)v?i9Y+qdjoeKgf%Po1ogzQ5sTfVO&2JVcG=LieA5g2BJQ};y+|UOb6-O=BC?rWC3Io?}Q4p zT4NKD(0`4B}Rg#)UZQ26*NQ1iXURoIKnkTOEYAQX`_&VPWkOHaDFmcIB zP_$2QKdH~7Ui}yokuw8fL!td?LL}UX-jl;qMjuiJMuaQJPDIaP6xK9)n>IA;nguv` zbp%Hwu*o&KlIz7`5-{ywB0z64Axn)+PDH912lv!J;*C*ogE?|&;U;0D&l2=ds8~82 zK!zM69CE=YL_GH3$P`G}2@1J1BzR^&b6UFMYD)?&4f&i1tr_U#S~btrLjYG%5|n2{>Gi9~hx!_8MQo|BBNy0Fu-lWPZUaXx z_`%k7gbPD9EJ2eO`Q%GX278K{!cP(Vp=%^`+~KTO^zF`t(;kQ_v`s$DC$xA71#Ts= zFA98PgC(97fbF0!GSo_|Z^sN_q-v{vjfwbaM(I+bZeJo;oUI1?2bRx51j@9KI3bA{ zjCVf@1F8tOI8I6#zK#$yT*ea5p{v+hxovx~lU=xk^KoH5S_GGXUy(=0_Gv$AT!G?= zpgIMkpb`-TWBGU(WY<+Ku9nbWkUP_aUQx$+-=}{LGy31St98rKTt=hn14rNB8Iia^ znxN_dJd2}OH6In?@{+LIFw*PMS&p?XDr*Kz8IcGqcxfS4)L$P3FtyX{^!ie z)CY9^FH!e3%wPR0WV(lA(h1KEOvSS(CVVx3dk2S)sPPcH(6AHytAEYmM4tXSb3EM< zsyUJIKUfo&w|05({iRn+}FSBE@L=@2%!W) za}oso9n|GEW}tcIw`V0^*q8$$SV_4kB#QTm$s4F3;lD|fX2z7DB6bi zQPy6~37drc_2=|uz4fxNnpyr^JvkF2{?C&&_m=0h|%798=trS3`wZ5E~){) zc@y{5NhZP-$tmRYTey-Uwq9nPEb+&gySw(KgcQEP#$%j}V(J@QYK}uSYSCEG_k*Jq z@Qov_rGZLibc+!Qyjv<%KxoM0BwQDB)oCgdv6=Ar913G6NYrK2P2}Ok+V62J(Z%)d zOf;`|er80qIcA5+1f@m}W&BqTY@>xHr?_w$WH#Pea$4jzrk~7_>ja_`>m6!gd~C&l zI>=fxKGz^X>{uaV*d#2XWj(~)1gLP-O9qw$-#o?|iZ&1d&X~oEFw|sZ=W`-t&9d{q zhzvpbiyB;4pCk%C&E#(|8QTz_L){;8{zYVNi)8PLR%tkbU7nk$YF=aKvQsJSrXJ5x z|CV<->U16SS8?|z3DXiX@Po|YcuopB=z_fuscC2IlwzAYu8|L^hMy4!|3GHK0cgt+ z9er>_CvgF0gZ<$ftdsni^|)vD&OA3Pe??Inb%BuaiSdY2QZJxMs3%U-_Wp$PjJD+a zrUQ^@e51iq${)(1vjl}k+ORFnQ(fZ8H$_7H5U3PBc}Rp51V8hc4IGh&XM*|c29D-$ zWUDrn1{|uX__U273)ZMkq;OPQ%1HB2i$fkYjDyX^l-(gL@bCCdbl?-I6T|I-D#zH~wK6nQhh6F`fUr{{O zHnhdXybula-{Hm|bFL&A>?t!>P^_!t8u^f_`4MmTB#RD**m%rsXb+A17G+P_&?aZ& zx58g+_>-))d)Yo5I=Q32=uyG$mK?IPz*_Be%Xmqu{$ zofE41j|5`x5FNuYfY zmdNdi4TLnIs>;5O?iX4yF3b(Fp;gl-t>$pZf zoF_kjKHK*_-?P>98y`flroMhxxyap|>}VosR4)36ZSp+?o-W*u{V#BbNAG*(`gl(P z!o$9D1pb+<_-9~pK#0uX9W}98@gCUA{0D?;zMXa;!b0B+4iI%4XPfxf33-npa=`HI zqPqZpq=dj_-7Q^n<$ao*`M@Bba(%Zuu=h;eV zLP$gigJYc_VJsp8V+Mz4^3-7Y;){V(*GBi@yP)7NJ(q+sx`Dal(D;xaO7SqyJ5evD zvfZ=9_8r)6`EK}rw>a_l5io8(@}ur|-3^Rs%z0!B{}6hkWh4HKbD~D`oIAY_h--=a z4ljd3U$nyG*mO97lS1>`=3l574KQ^?R^V2WXVc%=v*Jg{tf>D2Xl^qT3GXgkck~V3 zgKz33pfB`Iec*1Wi+KBVr`0QW1e9Ai9lU=#>3 z_o%TE!3o2B!Z=2~UM~*lIT!l9z&|WQJYdLmAG$|u+$L@l^Y}D$PP}MBGJ=~69Q?fa zd{GXCt~kHsUoTWcR_@hIf`REBfFm+n|cz-)A=P3rKSC&eC}{ zD?~qo_j>WBst6iYC-I!Hzs7P!bPL-T4e_RmViHek=HKe6g~f@Wr%Pol!E&p zJn^#q7T#Z@;H-v%lP}rh4c*IYB)A9Bj=nd(RG)|0H6dp#;Dpg$z5CSN2=I=h-HQT7 zMMeD&CQ}u#-^1PG1Uyo}7IFpnA?6k9rz>C-^ONkSKhKC*_}^X?ocyq~${Z}NJwf)| z_g=Vom*Cj*0PURy$0SfjUqD2z>Yt)hP@T9u{s8$&@v)F)<#|_#jZWPlwaORi!mxP? zM@x|xSMZB$HbLdN$)$)?9%|o7c#L7P?m{yEM7bFe8{8&TlF#Du4k3}9M@RegD4F_a zXgATTN{82v5S}zLQDkiQ0D>W<+1K@ic&OAz!GWCmV5u2-af3`Y;&N#qOR4{>^Z0o)j} zCOp|Dp%KK*!gFs?)C`AswrP8$V&w*<4E}u%5w*AmyU>33qG&&pEYt%7MBY^=;RzoE z(z4iTLvgeqhus!cRA%_MGf|;zPt+~`CwdtK*wi1g)5sAwrfTrrDD_87#wcPHbw5pt z2=_1fKVF_PiZEFY<3#NO&Vk=Htr5bYx08e++m|Dd9JX~IVsit3CK8ES98ScBenhr5 zg8n#(9K8mKnsoB|V7TbNfI4_W5jo3lQ#5DzCnHc!tfV|E(5m~d4G=a`hN`8QZ6D=CQ>pLCw?a3xE~gkGqz7#iYg z{5J+Un6zuKs_(!A`aDLO>Wunj0z;-MCeRQQ9yWH!{o~JwB%o;j|B9u($>DHexNc!E zI|-*rbug&09<(4f#1ZXqy_1rYh-{QO}$csnOQ{*N5>&jl{05}&kHsb({ z=0*G|k`L#bVlt%1<7LO*=NF6 sE}teB6&05Mm_r97{wU4NHSN<2$BXBR`Qm#Q=8I1()QWqGrQ!?!4~_@qsQ>@~ literal 0 HcmV?d00001 diff --git a/utils/__pycache__/l2putils.cpython-39.pyc b/utils/__pycache__/l2putils.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..03360c4aec643561666d4b6fab04b3700bd3c5b1 GIT binary patch literal 7595 zcmb7JO>o>sc18mj3C+q09D^1TLzeAO{~Y}7_cZN4>ErBU;^RZ4?4OWvt*ddy^_X>;dh1#U9b%()e1fx9(Kf;>u->_={t1o4nZ$+pX*EEm!JVIFdKCyWQ7$krX_R+{DUz8j>2 z{kH~wmWS?^Nc(P{ieP6i$}ov5!mUt*Nf4s{H#i$@v!ly79&Y)=IOkE2%Nh?K48wrh z&OQb{K19j}$U+S#snIFo1TU)7U^v4yUg!2B)~)ddcd!#Htn)eEe54(+u8s0MZ>e$v z=@g!Vh*`-_1f1 zg_)afxqH-*xqiakfk^iv9=d*jjb=CiH>*ZZd!>7!aDB(c^Ds(y`p`{|&tPl33?RilW<(~Hv24*x`V>UBclUewUR%fndecJgOqVTDcBB@j- z^)?EH*3%}eV8@K1Y-&B6%ln#6=R;pc!3up@Q#Ic<+>@V2WBvPt7uLyH5JFfTzMhy_rtUBc!)s2?zc zHM*?$jF|EZUnCIp%PvpB(@C1U*?to2h%~`nDn?ae`pU_GO=$*0?n9buJl`LL(&FJ> z6oiUb!~!Vq%*mIc%p=tS?((D9gQ%#>jrWpTCHg&TPZGALb9kF!3O+B&T8aCdka% z$TZUekG%j+bC7UTKb*a(@0{hS3q0XFmNV8^!8I+R!;MLB45R@Ti8A=( zNLQqS%!x&$cJ$2-6RYUksh`p-eh(ETx#gYx{+9IhFB605(^~~G#N=U5_|FF1a&2qih&+8#R}(FbxzgXRa%@T)b!EX?9!*5 zo=*V@J~S=XKZFpKH>gVkDpQ?<_#xWQtuEd|-F2ktvgdY;vERKO?U=Jm}WvtyJ{={JH&}MZBzg9t6PiZefP3g-nCmH>bb<8m-K3rs@p^Kw6VnRH>d8 z*^$KWhn^=L&+Dgr7*o0Fc@KtuT(w{=o(7&LXdOY;R**1A8wvstC*OZQ>10vP+_K_v z()@y?a1AMY85sDW%qp4XXUR>{CyCDk`Tx0#2o^+TlWkC2@JpGDI3fI3DA1p$5( z1O)Uc!Jk!uzj_7!Y=Aw2JPc5>g$uhNo3}zhxZq~Gy&Z~gStC#)fHml$G9|?UbPzer zjWQZK>|%u;N?tbwbyTziD1J}#g#wR$NJdFSz+XQ&p6b@YlT6&wW_P-)|P&#XYc*7 z_7(ARz0fadFpvfzfCo0=p>jtJStAvjNe7rU$#NfXRjD{>-775PFjj&0yMUo-{>5!b()oF+8LcmyWP_s#vU;G%wsRcG^o^$(`bDp;y=G5YO z_tCkMYX>YqKZ`j}Mr$)?ac00%+(0Ny9mr)AwgGbJC?-tt#LFd5fJf8;6grN;fZf&A zowv}lHn9jIl?P!Rj4!(fV<;?jp){dbq)hG3fFe?iEX`gTCDm1T?KsUcQc}_+JtAvL z2+r+Ukb>wTRS?$1eVeTH#4<+9=XR>p-OMhD1xO93T@r>kv%{t`%R0fm2hJ%E8|8gj zrBkBR*i;nB6r88-RjNa(L=k}J%^9k`8}ATp(}&6@w2~erZ-ZF#1NgyH31f=73N;H- zY~mSI$%fClr%rPZT}opK$Uj5bNtew)0UmUuhi+{SDL~5p6_5cA5w1RKa#9f*sf=;0 zdRsZ1Z*Cs3as8xZ^IERwP$W4}@?rJ62uJ&(Hf|IKuNMt`*+_6T$;sTd#!hY$I~{A{ z%fgw!-7H;;o;_(44xpaHTjRNcK~?>3+$`*gQ?6fFYntlW{2Z59)Ss$13w;&z@9Noq z6lO^eUs%(2^}k^vF6zCxvaT@Kv}J8P51O03`Mg!kA2QxP0`240lroI?C|@X?UVA!I z(IPN67WFMmzVN;_HmB$N;IFEk*4DJ~ z!U=r~C-k-R1%6@DE{vjGER?%>Q9Fj9A`n74YI#$+5mFm94o)0XrclMV?6PIfz1N8z)Mm`?daB#^gjYrKq>W8SN7rx+;Q73whM=JnbZa?g!0(mO- zFR#D7HLhIVx388kg2{Ro25G{xPE8t^y_7POCWLQ-iO2-fXefuD!Lk!!ytoNms)BA0T-!MY0KC(B#bEb;7g?e2Jy~fX~S$X;IxW#)DUk= zgb$>(6Z#y!7mK6=+bD8Rtzy6-?lx$I`UcFvX;j5*;0K|H$UwIqr`ujX+m`yRkE8*M z)1AX9dexzJ>Dow>!t<1liYt^ev`)*^d`^4G2F)5GRI?3F}?Yd2mX&ZvtW*{3W zU1FMRG=Nrs%)tmE45UvGW-M5QfX1;7>C`7CR|c?v+le}oG755;qtBE(|3xZxeSvTe zcY)a5TPc|5df#97;DmE8*a?H(0qkE+aV?5)4RQp%Tt60IjQzK$6QY78no#q}7WgC& zOXLvteI?7A9ZTv_D(fHZ=VA8A?Gj*_86sX3JqmYa`}}%{D>Z=J!`QIxvCf};S1Jdf1!P?ea*H2 z!nyg#9#c$>A-+bD)7t1`we5R5QFanNrHCnsQij+dj*eoe4|l>uMOLE3?NhW=P~4CL z5o&mON&(hAd59Xi;_o9ViU|7>RmVWSzhv2Rp^I0433L#_-u^xN>#6!QHUU-)rYL!zc!22#0_|*?^uY$tn+3c{2pp|Akhc2C-~5dMyj{ z;XobinXvHs2c%8n?8>^D4sjv7nm-dX3yl8&FT#S;bv@Nku9l)r(2{cHnbZFxnojP| z(pxo=p=Rc^@O|pTOd~JH0AMP3rcHpEvNbBJTQfFq+yhP2PuxQ$Ti8EE9;@Kg8O}Nt zP;%^kg;W7~ic3-wRR4Th$R|D<#v@7;{w$4*p@u6;WD7XidzgxtmqFB1_C6(Ii0OV! zM3i?z-m##%|1SZL_0r1X6o~$k0GkL9ezN<_{Xay%e>=A{)jfY{2d%Vsf-^vagzKO< zY0R@0kPC_l6;CS3_;+-yaG>}S<(Y*DCX;xO;ms&|Mhl76GEO=x5~ZVOs5wg8x`qBg z$O&GhRGc2QV#5|`CNE7YnfMK@L5op1LXg>+(H#_{qU*SZl>H7FXB1L{Q+9(RE?<@a zgixY5P8p7^zTuUsZ?pO~s_$CzAqS*VWAm!*G?a@#aBlVL6Z=?Og$D3y$x`0nH3gw8 zd@J7ouoWfMZ;pH)rTT5uHPN>*nMWOklUv6+{z-9;U@Xwr=`EnHN%cr@A$ta&J2K0* zNm~V)7gcR@7j9(*+2#=tzH@c+XVUrN_MMv_dv`zSUYE|z+t;q%^r*huynFMrJGZ<~ z@7%uj$;YD?HW6YT!rh+zZ;90XjeBscf*>Bb#mxUsB%JOW#GB-@0k@5z-QbiDvz?Af zG2&nsm_*L0zYhQbcTz5$Q(2OxVr#mo&XUyAOxD9>FG7U7bm_lBgZOh~(%2ph&jPSA zu5vF=v*eSXTI3j8cF9XaFf~{FY$R+H6_3!JVcK{fM{4>?Z&~Pw2}=4x9Iv z7}r@7@6w{2vO~%~MTV%YpYO=IAdO>01?d03GR|XAkIyK(MBNru$bS6{LE>v_Bt0sv z5(KE&kOC~q$g1s?Du&clDf}eNN{kdu&(sC>2iPjYc2_8vn7xL~tP#AUUz^$3Yg2Du RY2UFOdjTNH!q2p8{|hIzxL*JO literal 0 HcmV?d00001 diff --git a/utils/__pycache__/ops.cpython-39.pyc b/utils/__pycache__/ops.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..770d594def741d0d7e622ab63ec629bcf5763d50 GIT binary patch literal 5755 zcmb_gTW=f372eq!mlSoeEL)1}x=P?)*o&1oz1MAJtG26N6xT|Cm_WN=IZKi1lFQ64 zYuQ)<1>!vPp+JHD0s4`@r*HGvKIt#?A<1{naz#=S3!3&4Gn{XBF6W%@oHIk#78gqf zo`3)0U-iE)8peMKj6XKOHcGrH3_}`HghpFbMcb^JZL4Yt>N7*TYV)^Kb@1NIm%02gHmxWu>&ydcZKWyXuZi*gBgiSZKfvRnaP zVZ01{Nv;B~GF}0`EZ2b77+(UuBCi5pWxNW!E;oQT7+(f{OI`!M#&`|*EAl$8aDoz z0NW_>byR^-71F4hQb_ZKQMEEd!o&`?aJ9SBOQK#gIK#^&^bt&kkpQYu%QZ=qC8XsLy<%L@8(v!lWc{(I8niteEPf@ z^|re8R=CwmT4B5ub>myzXWH}qR;QKtzKc0A`HCT2@q9Ux@K*k|PNL;c84ViFqW%Ys zRP^CjFU7IREfXy(5q%TFTMaW2ht^B;*iP)!ICMT0zcT*#&tnI@ZoiPySa;aA&@Nz} zm%12Pl;$;K13ri`*0DF)zn=FOxxX+m-op6OWIvCm{sn2LcIu_Y&&;%xE_`M-%sa*> z?puf?RwbiGG*FHB2{ndQbR>;ztd}jm|}3$#X!Fn zSIVl0owU_zA866k;;FXVwfHYubhLP;#gR6jy@zsJn@9LQy3PD&)dEcf$Msqm`abzZ ze1oJGB~db0%_UJ5WphLD$GmEyS9sr~NBV-<;^98^@;%RJ%P+*3!-#6y9wjlzPS22w$q(t05fN(dr^lr3Gb;%Rrk z8PwFr1MB2(n4`X%R}&6Pq)}mqCDF3jqNq~j$OEB^jaC@eqcBnflg+Je;l)`MNWphd zVp`2Wm}I+;!i0p+A8HeEyz39qNW~!)k(h^;6seimeFtiZ0|dbuC0q1LtKs531wcXC z|Db%~>?!1R{j9SQBg?qa@ zAEB^P11nQ-lSUMg-lDa{FXDw=`=Cw_p{?kNE|s zLw<1lNLt!Y*w~RI83`u-sHz@pEgrAKB@e?dCD5%@` z8ikp9pCHAU`aV^hET$1gE1VK`OW2=3pHyL1(te6{en4x{G3^ON67{?>(%yJ9ZIi6S zOgYl{|3nm}xv8X4Oi|Wk>cn~K&XJ~mhUKP45$}_ywlk!>N+gkg{gMENK^$=;^*6!mdLGaWq49Oo}6@bOvMkBv#_2~dH`xcC1WeEm%7K^+&rP_t5 zEILA7S}E>5eTo6^WpLZ;QRlCLN^o~!$;KR>^wGt7 z)b2)cD+#pK37&Fz;BFGh087%*r#7kj5skI5#6{WcH+admGsF!XsF?{ynQUOIZ_*my zqN+j_rNFU;D9{xr@Smf#i^4AF2}O7=9-JMz2TagFl#sJ94ey|l7$5-G#6J6EV)bqE z&Wwn4gA6^evU8j5TyYdFp&p>pg&_MmF=ozbspApG=OMJjVvS6r?$e5_|EOY1&@Lb3 zGhqqBb6KMJD2kIn;dXn@8swO44LUb7Yv7`epHO_Gs+fuf=px)BTVQG+euQzkimRvl z&7vle{b4>EY7(AHO>aL6Yp9 i7_@gM`$y*&^v_m&6V;l)KQ#EPTwf^-KkkY%`S}v20Og4Q literal 0 HcmV?d00001 diff --git a/utils/__pycache__/toolkit.cpython-39.pyc b/utils/__pycache__/toolkit.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7733d5457a3a39a9f7514ea24c45fc79735df4d6 GIT binary patch literal 2221 zcma)7&2Jk;6rY*>@Y-?Okd}~s2t@))U5TVBHx!iuRgpMENJShhRn~N8>a4Tgb!NtG zlCC90dg0Lj!9MnH;6Irwr`(XZAS4LCw@yov3aKOQ%$v7w-{1@&u# zvES%&dIh-L!(+e4rkLV6>&{Q!74<#n25LzKDty7ap^8-ef^{R+PzlahHB}2|L%pZk zY7r8Nx}YwC(^UK+>ntBZE8BXI3OY{~(IK91@YvhfDsHYim{#HitJsv!*nM`$51|0u z;B9Uv+)6pVd%Z{@#bN2Wj;&&kE>C#H2h50ylfoYvXS;~!A$!HQI>Ij+y*ti~w!OZQ z%16a`sPk9Ci}l-{k6tm8K+Rt_H-?4V$j8YhnQef(`6==v)8!0ai3DmCn0OXJl5$$GJYI!!Mn8K87w! z8{;8g;pQT!Gu~50$g{evAA+bD@_%_=(1Dl+Gh`pU6KWns=o^{srDG+1kdDSaRMN@1 z9HadX_k8cXUu~xgTbes4*l52EE0%0BZr;a!24;i^^0)Tq3lI*_BKwM80pw=z1GjL- zC!!LANC|iv&)BpPU4lO+1-i0U$BxF|5q9-E41`1eGtby0bbJ6eEARqiRZIGWEEpD3MX zL(D1yvXglY|AXlHL|vnQCT5YV=SS zuG^3~G^D9d>K^H3nBke))9^Q!k9B?ymOg>(*vNGM8r-rp4H#d=APf2GaVuJ(k#;PE z_$LfP{_!#GYsc+qIcOcng6Qz?xVa3ewc}Q3?&0e*ofU#XgSZCs*C6_A!tnsK0O8ss zpb=gLG 32 + if is_train: + scale = (0.05, 1.0) + ratio = (3. / 4., 4. / 3.) + + transform = [ + transforms.RandomResizedCrop(input_size, scale=scale, ratio=ratio), + transforms.RandomHorizontalFlip(p=0.5), + transforms.ToTensor(), + ] + return transform + + t = [] + if resize_im: + size = int((256 / 224) * input_size) + t.append( + transforms.Resize(size, interpolation=3), # to maintain same ratio w.r.t. 224 images + ) + t.append(transforms.CenterCrop(input_size)) + t.append(transforms.ToTensor()) + + # return transforms.Compose(t) + return t + +class iCIFAR224(iData): + use_path = False + + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ + # transforms.ToTensor(), + ] + + class_order = np.arange(100).tolist() + + def download_data(self): + train_dataset = datasets.cifar.CIFAR100("./data", train=True, download=True) + test_dataset = datasets.cifar.CIFAR100("./data", train=False, download=True) + self.train_data, self.train_targets = train_dataset.data, np.array( + train_dataset.targets + ) + self.test_data, self.test_targets = test_dataset.data, np.array( + test_dataset.targets + ) + +class iImageNet1000(iData): + use_path = True + train_trsf = [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ColorJitter(brightness=63 / 255), + ] + test_trsf = [ + transforms.Resize(256), + transforms.CenterCrop(224), + ] + common_trsf = [ + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + + class_order = np.arange(1000).tolist() + + def download_data(self): + assert 0, "You should specify the folder of your dataset" + train_dir = "[DATA-PATH]/train/" + test_dir = "[DATA-PATH]/val/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + +class iImageNet100(iData): + use_path = True + train_trsf = [ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + ] + test_trsf = [ + transforms.Resize(256), + transforms.CenterCrop(224), + ] + common_trsf = [ + transforms.ToTensor(), + transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + ] + + class_order = np.arange(1000).tolist() + + def download_data(self): + assert 0, "You should specify the folder of your dataset" + train_dir = "[DATA-PATH]/train/" + test_dir = "[DATA-PATH]/val/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + +class iImageNetR(iData): + use_path = True + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ ] + + + class_order = np.arange(200).tolist() + + def download_data(self): + # assert 0, "You should specify the folder of your dataset" + train_dir = "./data/imagenet-r/train/" + test_dir = "./data/imagenet-r/test/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + +class iImageNetA(iData): + use_path = True + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ ] + + class_order = np.arange(200).tolist() + + def download_data(self): + # assert 0, "You should specify the folder of your dataset" + train_dir = "./data/imagenet-a/train/" + test_dir = "./data/imagenet-a/test/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + + +class CUB(iData): + use_path = True + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ ] + + class_order = np.arange(200).tolist() + + def download_data(self): + # assert 0, "You should specify the folder of your dataset" + train_dir = "./data/cub/train/" + test_dir = "./data/cub/test/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + +class objectnet(iData): + use_path = True + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ ] + + class_order = np.arange(200).tolist() + + def download_data(self): + # assert 0, "You should specify the folder of your dataset" + train_dir = "./data/objectnet/train/" + test_dir = "./data/objectnet/test/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + +class omnibenchmark(iData): + use_path = True + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ ] + + class_order = np.arange(300).tolist() + + def download_data(self): + # assert 0, "You should specify the folder of your dataset" + train_dir = "./data/omnibenchmark/train/" + test_dir = "./data/omnibenchmark/test/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) + + + +class vtab(iData): + use_path = True + + train_trsf=build_transform(True, None) + test_trsf=build_transform(False, None) + common_trsf = [ ] + + class_order = np.arange(50).tolist() + + def download_data(self): + # assert 0, "You should specify the folder of your dataset" + train_dir = "./data/vtab-cil/vtab/train/" + test_dir = "./data/vtab-cil/vtab/test/" + + train_dset = datasets.ImageFolder(train_dir) + test_dset = datasets.ImageFolder(test_dir) + + print(train_dset.class_to_idx) + print(test_dset.class_to_idx) + + self.train_data, self.train_targets = split_images_labels(train_dset.imgs) + self.test_data, self.test_targets = split_images_labels(test_dset.imgs) \ No newline at end of file diff --git a/utils/data_manager.py b/utils/data_manager.py new file mode 100644 index 0000000..83fa4c8 --- /dev/null +++ b/utils/data_manager.py @@ -0,0 +1,280 @@ +import logging +import numpy as np +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms +from utils.data import iCIFAR10, iCIFAR100, iImageNet100, iImageNet1000, iCIFAR224, iImageNetR,iImageNetA,CUB, objectnet, omnibenchmark, vtab + + +class DataManager(object): + def __init__(self, dataset_name, shuffle, seed, init_cls, increment): + self.dataset_name = dataset_name + self._setup_data(dataset_name, shuffle, seed) + assert init_cls <= len(self._class_order), "No enough classes." + self._increments = [init_cls] + while sum(self._increments) + increment < len(self._class_order): + self._increments.append(increment) + offset = len(self._class_order) - sum(self._increments) + if offset > 0: + self._increments.append(offset) + + @property + def nb_tasks(self): + return len(self._increments) + + def get_task_size(self, task): + return self._increments[task] + + def get_total_classnum(self): + return len(self._class_order) + + def get_dataset( + self, indices, source, mode, appendent=None, ret_data=False, m_rate=None + ): + if source == "train": + x, y = self._train_data, self._train_targets + elif source == "test": + x, y = self._test_data, self._test_targets + else: + raise ValueError("Unknown data source {}.".format(source)) + + if mode == "train": + trsf = transforms.Compose([*self._train_trsf, *self._common_trsf]) + elif mode == "flip": + trsf = transforms.Compose( + [ + *self._test_trsf, + transforms.RandomHorizontalFlip(p=1.0), + *self._common_trsf, + ] + ) + elif mode == "test": + trsf = transforms.Compose([*self._test_trsf, *self._common_trsf]) + else: + raise ValueError("Unknown mode {}.".format(mode)) + + data, targets = [], [] + for idx in indices: + if m_rate is None: + class_data, class_targets = self._select( + x, y, low_range=idx, high_range=idx + 1 + ) + else: + class_data, class_targets = self._select_rmm( + x, y, low_range=idx, high_range=idx + 1, m_rate=m_rate + ) + data.append(class_data) + targets.append(class_targets) + + if appendent is not None and len(appendent) != 0: + appendent_data, appendent_targets = appendent + data.append(appendent_data) + targets.append(appendent_targets) + + data, targets = np.concatenate(data), np.concatenate(targets) + + if ret_data: + return data, targets, DummyDataset(data, targets, trsf, self.use_path) + else: + return DummyDataset(data, targets, trsf, self.use_path) + + def get_dataset_with_split( + self, indices, source, mode, appendent=None, val_samples_per_class=0 + ): + if source == "train": + x, y = self._train_data, self._train_targets + elif source == "test": + x, y = self._test_data, self._test_targets + else: + raise ValueError("Unknown data source {}.".format(source)) + + if mode == "train": + trsf = transforms.Compose([*self._train_trsf, *self._common_trsf]) + elif mode == "test": + trsf = transforms.Compose([*self._test_trsf, *self._common_trsf]) + else: + raise ValueError("Unknown mode {}.".format(mode)) + + train_data, train_targets = [], [] + val_data, val_targets = [], [] + for idx in indices: + class_data, class_targets = self._select( + x, y, low_range=idx, high_range=idx + 1 + ) + val_indx = np.random.choice( + len(class_data), val_samples_per_class, replace=False + ) + train_indx = list(set(np.arange(len(class_data))) - set(val_indx)) + val_data.append(class_data[val_indx]) + val_targets.append(class_targets[val_indx]) + train_data.append(class_data[train_indx]) + train_targets.append(class_targets[train_indx]) + + if appendent is not None: + appendent_data, appendent_targets = appendent + for idx in range(0, int(np.max(appendent_targets)) + 1): + append_data, append_targets = self._select( + appendent_data, appendent_targets, low_range=idx, high_range=idx + 1 + ) + val_indx = np.random.choice( + len(append_data), val_samples_per_class, replace=False + ) + train_indx = list(set(np.arange(len(append_data))) - set(val_indx)) + val_data.append(append_data[val_indx]) + val_targets.append(append_targets[val_indx]) + train_data.append(append_data[train_indx]) + train_targets.append(append_targets[train_indx]) + + train_data, train_targets = np.concatenate(train_data), np.concatenate( + train_targets + ) + val_data, val_targets = np.concatenate(val_data), np.concatenate(val_targets) + + return DummyDataset( + train_data, train_targets, trsf, self.use_path + ), DummyDataset(val_data, val_targets, trsf, self.use_path) + + def _setup_data(self, dataset_name, shuffle, seed): + idata = _get_idata(dataset_name) + idata.download_data() + + # Data + self._train_data, self._train_targets = idata.train_data, idata.train_targets + self._test_data, self._test_targets = idata.test_data, idata.test_targets + self.use_path = idata.use_path + + # Transforms + self._train_trsf = idata.train_trsf + self._test_trsf = idata.test_trsf + self._common_trsf = idata.common_trsf + + # Order + order = [i for i in range(len(np.unique(self._train_targets)))] + if shuffle: + np.random.seed(seed) + order = np.random.permutation(len(order)).tolist() + else: + order = idata.class_order + self._class_order = order + logging.info(self._class_order) + + # Map indices + self._train_targets = _map_new_class_index( + self._train_targets, self._class_order + ) + self._test_targets = _map_new_class_index(self._test_targets, self._class_order) + + def _select(self, x, y, low_range, high_range): + idxes = np.where(np.logical_and(y >= low_range, y < high_range))[0] + return x[idxes], y[idxes] + + def _select_rmm(self, x, y, low_range, high_range, m_rate): + assert m_rate is not None + if m_rate != 0: + idxes = np.where(np.logical_and(y >= low_range, y < high_range))[0] + selected_idxes = np.random.randint( + 0, len(idxes), size=int((1 - m_rate) * len(idxes)) + ) + new_idxes = idxes[selected_idxes] + new_idxes = np.sort(new_idxes) + else: + new_idxes = np.where(np.logical_and(y >= low_range, y < high_range))[0] + return x[new_idxes], y[new_idxes] + + def getlen(self, index): + y = self._train_targets + return np.sum(np.where(y == index)) + + +class DummyDataset(Dataset): + def __init__(self, images, labels, trsf, use_path=False): + assert len(images) == len(labels), "Data size error!" + self.images = images + self.labels = labels + self.trsf = trsf + self.use_path = use_path + + def __len__(self): + return len(self.images) + + def __getitem__(self, idx): + if self.use_path: + image = self.trsf(pil_loader(self.images[idx])) + else: + image = self.trsf(Image.fromarray(self.images[idx])) + label = self.labels[idx] + + return idx, image, label + + +def _map_new_class_index(y, order): + return np.array(list(map(lambda x: order.index(x), y))) + + +def _get_idata(dataset_name): + name = dataset_name.lower() + if name == "cifar10": + return iCIFAR10() + elif name == "cifar100": + return iCIFAR100() + elif name == "imagenet1000": + return iImageNet1000() + elif name == "imagenet100": + return iImageNet100() + elif name== "cifar224": + return iCIFAR224() + elif name== "imagenetr": + return iImageNetR() + elif name=="imageneta": + return iImageNetA() + elif name=="cub": + return CUB() + elif name=="objectnet": + return objectnet() + elif name=="omnibenchmark": + return omnibenchmark() + elif name=="vtab": + return vtab() + + else: + raise NotImplementedError("Unknown dataset {}.".format(dataset_name)) + + +def pil_loader(path): + """ + Ref: + https://pytorch.org/docs/stable/_modules/torchvision/datasets/folder.html#ImageFolder + """ + # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) + with open(path, "rb") as f: + img = Image.open(f) + return img.convert("RGB") + + +def accimage_loader(path): + """ + Ref: + https://pytorch.org/docs/stable/_modules/torchvision/datasets/folder.html#ImageFolder + accimage is an accelerated Image loader and preprocessor leveraging Intel IPP. + accimage is available on conda-forge. + """ + import accimage + + try: + return accimage.Image(path) + except IOError: + # Potentially a decoding problem, fall back to PIL.Image + return pil_loader(path) + + +def default_loader(path): + """ + Ref: + https://pytorch.org/docs/stable/_modules/torchvision/datasets/folder.html#ImageFolder + """ + from torchvision import get_image_backend + + if get_image_backend() == "accimage": + return accimage_loader(path) + else: + return pil_loader(path) diff --git a/utils/factory.py b/utils/factory.py new file mode 100644 index 0000000..ef9c66a --- /dev/null +++ b/utils/factory.py @@ -0,0 +1,19 @@ +def get_model(model_name, args): + name = model_name.lower() + if name=="simplecil": + from models.simplecil import Learner + return Learner(args) + elif name=="adam_finetune": + from models.adam_finetune import Learner + return Learner(args) + elif name=="adam_ssf": + from models.adam_ssf import Learner + return Learner(args) + elif name=="adam_vpt": + from models.adam_vpt import Learner + return Learner(args) + elif name=="adam_adapter": + from models.adam_adapter import Learner + return Learner(args) + else: + assert 0 diff --git a/utils/inc_net.py b/utils/inc_net.py new file mode 100644 index 0000000..b7b44be --- /dev/null +++ b/utils/inc_net.py @@ -0,0 +1,733 @@ +import copy +import logging +import torch +from torch import nn +from convs.linears import SimpleLinear, SplitCosineLinear, CosineLinear +import timm + + +def get_convnet(args, pretrained=False): + + name = args["convnet_type"].lower() + #Resnet + if name=="pretrained_resnet18": + from convs.resnet import resnet18, resnet34, resnet50, resnet101, resnet152 + model=resnet18(pretrained=False,args=args) + model.load_state_dict(torch.load("./pretrained_models/resnet18-f37072fd.pth"),strict=False) + return model.eval() + elif name=="pretrained_resnet50": + from convs.resnet import resnet18, resnet34, resnet50, resnet101, resnet152 + model=resnet50(pretrained=False,args=args) + model.load_state_dict(torch.load("./pretrained_models/resnet50-11ad3fa6.pth"),strict=False) + return model.eval() + elif name=="pretrained_resnet101": + from convs.resnet import resnet18, resnet34, resnet50, resnet101, resnet152 + model=resnet101(pretrained=False,args=args) + model.load_state_dict(torch.load("./pretrained_models/resnet101-cd907fc2.pth"),strict=False) + return model.eval() + elif name=="pretrained_resnet152": + from convs.resnet import resnet18, resnet34, resnet50, resnet101, resnet152 + model=resnet152(pretrained=False,args=args) + model.load_state_dict(torch.load("./pretrained_models/resnet152-f82ba261.pth"),strict=False) + return model.eval() + + #SimpleCIL or SimpleCIL w/ Finetune + elif name=="pretrained_vit_b16_224" or name=="vit_base_patch16_224": + model=timm.create_model("vit_base_patch16_224",pretrained=True, num_classes=0) + model.out_dim=768 + return model.eval() + elif name=="pretrained_vit_b16_224_in21k" or name=="vit_base_patch16_224_in21k": + model=timm.create_model("vit_base_patch16_224_in21k",pretrained=True, num_classes=0) + model.out_dim=768 + return model.eval() + + # SSF + elif '_ssf' in name: + if args["model_name"]=="adam_ssf": + from convs import vision_transformer_ssf + if name=="pretrained_vit_b16_224_ssf": + model = timm.create_model("vit_base_patch16_224_ssf", pretrained=True, num_classes=0) + model.out_dim=768 + elif name=="pretrained_vit_b16_224_in21k_ssf": + model=timm.create_model("vit_base_patch16_224_in21k_ssf",pretrained=True, num_classes=0) + model.out_dim=768 + return model.eval() + else: + raise NotImplementedError("Inconsistent model name and model type") + + # VPT + elif '_vpt' in name: + if args["model_name"]=="adam_vpt": + from convs.vpt import build_promptmodel + if name=="pretrained_vit_b16_224_vpt": + basicmodelname="vit_base_patch16_224" + elif name=="pretrained_vit_b16_224_in21k_vpt": + basicmodelname="vit_base_patch16_224_in21k" + + print("modelname,",name,"basicmodelname",basicmodelname) + VPT_type="Deep" + if args["vpt_type"]=='shallow': + VPT_type="Shallow" + Prompt_Token_num=args["prompt_token_num"] + + model = build_promptmodel(modelname=basicmodelname, Prompt_Token_num=Prompt_Token_num, VPT_type=VPT_type) + prompt_state_dict = model.obtain_prompt() + model.load_prompt(prompt_state_dict) + model.out_dim=768 + return model.eval() + else: + raise NotImplementedError("Inconsistent model name and model type") + + elif '_adapter' in name: + ffn_num=args["ffn_num"] + if args["model_name"]=="adam_adapter" : + from convs import vision_transformer_adapter + from easydict import EasyDict + tuning_config = EasyDict( + # AdaptFormer + ffn_adapt=True, + ffn_option="parallel", + ffn_adapter_layernorm_option="none", + ffn_adapter_init_option="lora", + ffn_adapter_scalar="0.1", + ffn_num=ffn_num, + d_model=768, + # VPT related + vpt_on=False, + vpt_num=0, + ) + if name=="pretrained_vit_b16_224_adapter": + model = vision_transformer_adapter.vit_base_patch16_224_adapter(num_classes=0, + global_pool=False, drop_path_rate=0.0, tuning_config=tuning_config) + model.out_dim=768 + elif name=="pretrained_vit_b16_224_in21k_adapter": + model = vision_transformer_adapter.vit_base_patch16_224_in21k_adapter(num_classes=0, + global_pool=False, drop_path_rate=0.0, tuning_config=tuning_config) + model.out_dim=768 + else: + raise NotImplementedError("Unknown type {}".format(name)) + return model.eval() + else: + raise NotImplementedError("Inconsistent model name and model type") + + else: + raise NotImplementedError("Unknown type {}".format(name)) + + +def load_state_vision_model(model, ckpt_path): + ckpt_state = torch.load(ckpt_path, map_location='cpu') + if 'state_dict' in ckpt_state: + # our upstream converted checkpoint + ckpt_state = ckpt_state['state_dict'] + prefix = '' + elif 'model' in ckpt_state: + # prototype checkpoint + ckpt_state = ckpt_state['model'] + prefix = 'module.' + else: + # official checkpoint + prefix = '' + + logger = logging.getLogger('global') + if ckpt_state: + logger.info('==> Loading model state "{}XXX" from pre-trained model..'.format(prefix)) + + own_state = model.state_dict() + state = {} + for name, param in ckpt_state.items(): + if name.startswith(prefix): + state[name[len(prefix):]] = param + success_cnt = 0 + for name, param in state.items(): + if name in own_state: + if isinstance(param, torch.nn.Parameter): + # backwards compatibility for serialized parameters + param = param.data + try: + if isinstance(param, bool): + own_state[name] = param + else: + # normal version + own_state[name].copy_(param) + success_cnt += 1 + except Exception as err: + logger.warn(err) + logger.warn('while copying the parameter named {}, ' + 'whose dimensions in the model are {} and ' + 'whose dimensions in the checkpoint are {}.' + .format(name, own_state[name].size(), param.size())) + logger.warn("But don't worry about it. Continue pretraining.") + ckpt_keys = set(state.keys()) + own_keys = set(model.state_dict().keys()) + missing_keys = own_keys - ckpt_keys + logger.info('Successfully loaded {} key(s) from {}'.format(success_cnt, ckpt_path)) + for k in missing_keys: + logger.warn('Caution: missing key from checkpoint: {}'.format(k)) + redundancy_keys = ckpt_keys - own_keys + for k in redundancy_keys: + logger.warn('Caution: redundant key from checkpoint: {}'.format(k)) + + +class BaseNet(nn.Module): + def __init__(self, args, pretrained): + super(BaseNet, self).__init__() + + + print('This is for the BaseNet initialization.') + self.convnet = get_convnet(args, pretrained) + print('After BaseNet initialization.') + self.fc = None + + @property + def feature_dim(self): + return self.convnet.out_dim + + def extract_vector(self, x): + return self.convnet(x)["features"] + + def forward(self, x): + x = self.convnet(x) + out = self.fc(x["features"]) + """ + { + 'fmaps': [x_1, x_2, ..., x_n], + 'features': features + 'logits': logits + } + """ + out.update(x) + + return out + + def update_fc(self, nb_classes): + pass + + def generate_fc(self, in_dim, out_dim): + pass + + def copy(self): + return copy.deepcopy(self) + + def freeze(self): + for param in self.parameters(): + param.requires_grad = False + self.eval() + + return self + + +class IncrementalNet(BaseNet): + def __init__(self, args, pretrained, gradcam=False): + super().__init__(args, pretrained) + self.gradcam = gradcam + if hasattr(self, "gradcam") and self.gradcam: + self._gradcam_hooks = [None, None] + self.set_gradcam_hook() + + def update_fc(self, nb_classes): + fc = self.generate_fc(self.feature_dim, nb_classes) + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + bias = copy.deepcopy(self.fc.bias.data) + fc.weight.data[:nb_output] = weight + fc.bias.data[:nb_output] = bias + + del self.fc + self.fc = fc + + def weight_align(self, increment): + weights = self.fc.weight.data + newnorm = torch.norm(weights[-increment:, :], p=2, dim=1) + oldnorm = torch.norm(weights[:-increment, :], p=2, dim=1) + meannew = torch.mean(newnorm) + meanold = torch.mean(oldnorm) + gamma = meanold / meannew + print("alignweights,gamma=", gamma) + self.fc.weight.data[-increment:, :] *= gamma + + def generate_fc(self, in_dim, out_dim): + fc = SimpleLinear(in_dim, out_dim) + + return fc + + def forward(self, x): + x = self.convnet(x) + out = self.fc(x["features"]) + out.update(x) + if hasattr(self, "gradcam") and self.gradcam: + out["gradcam_gradients"] = self._gradcam_gradients + out["gradcam_activations"] = self._gradcam_activations + + return out + + def unset_gradcam_hook(self): + self._gradcam_hooks[0].remove() + self._gradcam_hooks[1].remove() + self._gradcam_hooks[0] = None + self._gradcam_hooks[1] = None + self._gradcam_gradients, self._gradcam_activations = [None], [None] + + def set_gradcam_hook(self): + self._gradcam_gradients, self._gradcam_activations = [None], [None] + + def backward_hook(module, grad_input, grad_output): + self._gradcam_gradients[0] = grad_output[0] + return None + + def forward_hook(module, input, output): + self._gradcam_activations[0] = output + return None + + self._gradcam_hooks[0] = self.convnet.last_conv.register_backward_hook( + backward_hook + ) + self._gradcam_hooks[1] = self.convnet.last_conv.register_forward_hook( + forward_hook + ) + +class IL2ANet(IncrementalNet): + + def update_fc(self, num_old, num_total, num_aux): + fc = self.generate_fc(self.feature_dim, num_total+num_aux) + if self.fc is not None: + weight = copy.deepcopy(self.fc.weight.data) + bias = copy.deepcopy(self.fc.bias.data) + fc.weight.data[:num_old] = weight[:num_old] + fc.bias.data[:num_old] = bias[:num_old] + del self.fc + self.fc = fc + +class CosineIncrementalNet(BaseNet): + def __init__(self, args, pretrained, nb_proxy=1): + super().__init__(args, pretrained) + self.nb_proxy = nb_proxy + + def update_fc(self, nb_classes, task_num): + fc = self.generate_fc(self.feature_dim, nb_classes) + if self.fc is not None: + if task_num == 1: + fc.fc1.weight.data = self.fc.weight.data + fc.sigma.data = self.fc.sigma.data + else: + prev_out_features1 = self.fc.fc1.out_features + fc.fc1.weight.data[:prev_out_features1] = self.fc.fc1.weight.data + fc.fc1.weight.data[prev_out_features1:] = self.fc.fc2.weight.data + fc.sigma.data = self.fc.sigma.data + + del self.fc + self.fc = fc + + def generate_fc(self, in_dim, out_dim): + if self.fc is None: + fc = CosineLinear(in_dim, out_dim, self.nb_proxy, to_reduce=True) + else: + prev_out_features = self.fc.out_features // self.nb_proxy + # prev_out_features = self.fc.out_features + fc = SplitCosineLinear( + in_dim, prev_out_features, out_dim - prev_out_features, self.nb_proxy + ) + + return fc + + +class BiasLayer(nn.Module): + def __init__(self): + super(BiasLayer, self).__init__() + self.alpha = nn.Parameter(torch.ones(1, requires_grad=True)) + self.beta = nn.Parameter(torch.zeros(1, requires_grad=True)) + + def forward(self, x, low_range, high_range): + ret_x = x.clone() + ret_x[:, low_range:high_range] = ( + self.alpha * x[:, low_range:high_range] + self.beta + ) + return ret_x + + def get_params(self): + return (self.alpha.item(), self.beta.item()) + + +class IncrementalNetWithBias(BaseNet): + def __init__(self, args, pretrained, bias_correction=False): + super().__init__(args, pretrained) + + # Bias layer + self.bias_correction = bias_correction + self.bias_layers = nn.ModuleList([]) + self.task_sizes = [] + + def forward(self, x): + x = self.convnet(x) + out = self.fc(x["features"]) + if self.bias_correction: + logits = out["logits"] + for i, layer in enumerate(self.bias_layers): + logits = layer( + logits, sum(self.task_sizes[:i]), sum(self.task_sizes[: i + 1]) + ) + out["logits"] = logits + + out.update(x) + + return out + + def update_fc(self, nb_classes): + fc = self.generate_fc(self.feature_dim, nb_classes) + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + bias = copy.deepcopy(self.fc.bias.data) + fc.weight.data[:nb_output] = weight + fc.bias.data[:nb_output] = bias + + del self.fc + self.fc = fc + + new_task_size = nb_classes - sum(self.task_sizes) + self.task_sizes.append(new_task_size) + self.bias_layers.append(BiasLayer()) + + def generate_fc(self, in_dim, out_dim): + fc = SimpleLinear(in_dim, out_dim) + + return fc + + def get_bias_params(self): + params = [] + for layer in self.bias_layers: + params.append(layer.get_params()) + + return params + + def unfreeze(self): + for param in self.parameters(): + param.requires_grad = True + + +class DERNet(nn.Module): + def __init__(self, args, pretrained): + super(DERNet, self).__init__() + self.convnet_type = args["convnet_type"] + self.convnets = nn.ModuleList() + self.pretrained = pretrained + self.out_dim = None + self.fc = None + self.aux_fc = None + self.task_sizes = [] + self.args = args + + @property + def feature_dim(self): + if self.out_dim is None: + return 0 + return self.out_dim * len(self.convnets) + + def extract_vector(self, x): + features = [convnet(x)["features"] for convnet in self.convnets] + features = torch.cat(features, 1) + return features + + def forward(self, x): + features = [convnet(x)["features"] for convnet in self.convnets] + features = torch.cat(features, 1) + + out = self.fc(features) # {logics: self.fc(features)} + + aux_logits = self.aux_fc(features[:, -self.out_dim :])["logits"] + + out.update({"aux_logits": aux_logits, "features": features}) + return out + """ + { + 'features': features + 'logits': logits + 'aux_logits':aux_logits + } + """ + + def update_fc(self, nb_classes): + if len(self.convnets) == 0: + self.convnets.append(get_convnet(self.args)) + else: + self.convnets.append(get_convnet(self.args)) + self.convnets[-1].load_state_dict(self.convnets[-2].state_dict()) + + if self.out_dim is None: + self.out_dim = self.convnets[-1].out_dim + fc = self.generate_fc(self.feature_dim, nb_classes) + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + bias = copy.deepcopy(self.fc.bias.data) + fc.weight.data[:nb_output, : self.feature_dim - self.out_dim] = weight + fc.bias.data[:nb_output] = bias + + del self.fc + self.fc = fc + + new_task_size = nb_classes - sum(self.task_sizes) + self.task_sizes.append(new_task_size) + + self.aux_fc = self.generate_fc(self.out_dim, new_task_size + 1) + + def generate_fc(self, in_dim, out_dim): + fc = SimpleLinear(in_dim, out_dim) + + return fc + + def copy(self): + return copy.deepcopy(self) + + def freeze(self): + for param in self.parameters(): + param.requires_grad = False + self.eval() + + return self + + def freeze_conv(self): + for param in self.convnets.parameters(): + param.requires_grad = False + self.convnets.eval() + + def weight_align(self, increment): + weights = self.fc.weight.data + newnorm = torch.norm(weights[-increment:, :], p=2, dim=1) + oldnorm = torch.norm(weights[:-increment, :], p=2, dim=1) + meannew = torch.mean(newnorm) + meanold = torch.mean(oldnorm) + gamma = meanold / meannew + print("alignweights,gamma=", gamma) + self.fc.weight.data[-increment:, :] *= gamma + + +class SimpleCosineIncrementalNet(BaseNet): + def __init__(self, args, pretrained): + super().__init__(args, pretrained) + + def update_fc(self, nb_classes, nextperiod_initialization=None): + fc = self.generate_fc(self.feature_dim, nb_classes).cuda() + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + fc.sigma.data = self.fc.sigma.data + if nextperiod_initialization is not None: + weight = torch.cat([weight, nextperiod_initialization]) + else: + weight = torch.cat([weight, torch.zeros(nb_classes - nb_output, self.feature_dim).cuda()]) + fc.weight = nn.Parameter(weight) + del self.fc + self.fc = fc + + def generate_fc(self, in_dim, out_dim): + fc = CosineLinear(in_dim, out_dim) + return fc + + +class SimpleVitNet(BaseNet): + def __init__(self, args, pretrained): + super().__init__(args, pretrained) + + def update_fc(self, nb_classes, nextperiod_initialization=None): + fc = self.generate_fc(self.feature_dim, nb_classes).cuda() + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + fc.sigma.data = self.fc.sigma.data + if nextperiod_initialization is not None: + weight = torch.cat([weight, nextperiod_initialization]) + else: + weight = torch.cat([weight, torch.zeros(nb_classes - nb_output, self.feature_dim).cuda()]) + fc.weight = nn.Parameter(weight) + del self.fc + self.fc = fc + + def generate_fc(self, in_dim, out_dim): + fc = CosineLinear(in_dim, out_dim) + return fc + + def extract_vector(self, x): + return self.convnet(x) + + def forward(self, x): + x = self.convnet(x) + out = self.fc(x) + # out.update(x) + return out + + +class MultiBranchCosineIncrementalNet(BaseNet): + def __init__(self, args, pretrained): + super().__init__(args, pretrained) + + # no need the convnet. + + print('Clear the convnet in MultiBranchCosineIncrementalNet, since we are using self.convnets with dual branches') + self.convnet=torch.nn.Identity() + for param in self.convnet.parameters(): + param.requires_grad = False + + self.convnets = nn.ModuleList() + self.args=args + + if 'resnet' in args['convnet_type']: + self.modeltype='cnn' + else: + self.modeltype='vit' + + def update_fc(self, nb_classes, nextperiod_initialization=None): + fc = self.generate_fc(self._feature_dim, nb_classes).cuda() + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + fc.sigma.data = self.fc.sigma.data + if nextperiod_initialization is not None: + weight = torch.cat([weight, nextperiod_initialization]) + else: + weight = torch.cat([weight, torch.zeros(nb_classes - nb_output, self._feature_dim).cuda()]) + fc.weight = nn.Parameter(weight) + del self.fc + self.fc = fc + + def generate_fc(self, in_dim, out_dim): + fc = CosineLinear(in_dim, out_dim) + return fc + + + def forward(self, x): + if self.modeltype=='cnn': + features = [convnet(x)["features"] for convnet in self.convnets] + features = torch.cat(features, 1) + # import pdb; pdb.set_trace() + out = self.fc(features) + out.update({"features": features}) + return out + else: + features = [convnet(x) for convnet in self.convnets] + features = torch.cat(features, 1) + # import pdb; pdb.set_trace() + out = self.fc(features) + out.update({"features": features}) + return out + + + def construct_dual_branch_network(self, tuned_model): + if 'ssf' in self.args['convnet_type']: + newargs=copy.deepcopy(self.args) + newargs['convnet_type']=newargs['convnet_type'].replace('_ssf','') + print(newargs['convnet_type']) + self.convnets.append(get_convnet(newargs)) #pretrained model without scale + elif 'vpt' in self.args['convnet_type']: + newargs=copy.deepcopy(self.args) + newargs['convnet_type']=newargs['convnet_type'].replace('_vpt','') + print(newargs['convnet_type']) + self.convnets.append(get_convnet(newargs)) #pretrained model without vpt + elif 'adapter' in self.args['convnet_type']: + newargs=copy.deepcopy(self.args) + newargs['convnet_type']=newargs['convnet_type'].replace('_adapter','') + print(newargs['convnet_type']) + self.convnets.append(get_convnet(newargs)) #pretrained model without adapter + else: + self.convnets.append(get_convnet(self.args)) #the pretrained model itself + + self.convnets.append(tuned_model.convnet) #adappted tuned model + + self._feature_dim = self.convnets[0].out_dim * len(self.convnets) + self.fc=self.generate_fc(self._feature_dim,self.args['init_cls']) + + + + +class FOSTERNet(nn.Module): + def __init__(self, args, pretrained): + super(FOSTERNet, self).__init__() + self.convnet_type = args["convnet_type"] + self.convnets = nn.ModuleList() + self.pretrained = pretrained + self.out_dim = None + self.fc = None + self.fe_fc = None + self.task_sizes = [] + self.oldfc = None + self.args = args + + @property + def feature_dim(self): + if self.out_dim is None: + return 0 + return self.out_dim * len(self.convnets) + + def extract_vector(self, x): + features = [convnet(x)["features"] for convnet in self.convnets] + features = torch.cat(features, 1) + return features + + def forward(self, x): + features = [convnet(x)["features"] for convnet in self.convnets] + features = torch.cat(features, 1) + out = self.fc(features) + fe_logits = self.fe_fc(features[:, -self.out_dim :])["logits"] + + out.update({"fe_logits": fe_logits, "features": features}) + + if self.oldfc is not None: + old_logits = self.oldfc(features[:, : -self.out_dim])["logits"] + out.update({"old_logits": old_logits}) + + out.update({"eval_logits": out["logits"]}) + return out + + def update_fc(self, nb_classes): + self.convnets.append(get_convnet(self.args)) + if self.out_dim is None: + self.out_dim = self.convnets[-1].out_dim + fc = self.generate_fc(self.feature_dim, nb_classes) + if self.fc is not None: + nb_output = self.fc.out_features + weight = copy.deepcopy(self.fc.weight.data) + bias = copy.deepcopy(self.fc.bias.data) + fc.weight.data[:nb_output, : self.feature_dim - self.out_dim] = weight + fc.bias.data[:nb_output] = bias + self.convnets[-1].load_state_dict(self.convnets[-2].state_dict()) + + self.oldfc = self.fc + self.fc = fc + new_task_size = nb_classes - sum(self.task_sizes) + self.task_sizes.append(new_task_size) + self.fe_fc = self.generate_fc(self.out_dim, nb_classes) + + def generate_fc(self, in_dim, out_dim): + fc = SimpleLinear(in_dim, out_dim) + return fc + + def copy(self): + return copy.deepcopy(self) + + def copy_fc(self, fc): + weight = copy.deepcopy(fc.weight.data) + bias = copy.deepcopy(fc.bias.data) + n, m = weight.shape[0], weight.shape[1] + self.fc.weight.data[:n, :m] = weight + self.fc.bias.data[:n] = bias + + def freeze(self): + for param in self.parameters(): + param.requires_grad = False + self.eval() + return self + + def freeze_conv(self): + for param in self.convnets.parameters(): + param.requires_grad = False + self.convnets.eval() + + def weight_align(self, old, increment, value): + weights = self.fc.weight.data + newnorm = torch.norm(weights[-increment:, :], p=2, dim=1) + oldnorm = torch.norm(weights[:-increment, :], p=2, dim=1) + meannew = torch.mean(newnorm) + meanold = torch.mean(oldnorm) + gamma = meanold / meannew * (value ** (old / increment)) + logging.info("align weights, gamma = {} ".format(gamma)) + self.fc.weight.data[-increment:, :] *= gamma diff --git a/utils/ops.py b/utils/ops.py new file mode 100644 index 0000000..66dcb77 --- /dev/null +++ b/utils/ops.py @@ -0,0 +1,121 @@ +from PIL import Image, ImageEnhance, ImageOps +import random +import torch +import numpy as np +class Cutout(object): + def __init__(self, n_holes, length): + self.n_holes = n_holes + self.length = length + + def __call__(self, img): + h = img.size(1) + w = img.size(2) + + mask = np.ones((h, w), np.float32) + + for n in range(self.n_holes): + y = np.random.randint(h) + x = np.random.randint(w) + + y1 = np.clip(y - self.length // 2, 0, h) + y2 = np.clip(y + self.length // 2, 0, h) + x1 = np.clip(x - self.length // 2, 0, w) + x2 = np.clip(x + self.length // 2, 0, w) + + mask[y1: y2, x1: x2] = 0. + + mask = torch.from_numpy(mask) + mask = mask.expand_as(img) + img = img * mask + + return img + +class ShearX(object): + def __init__(self, fillcolor=(128, 128, 128)): + self.fillcolor = fillcolor + + def __call__(self, x, magnitude): + return x.transform( + x.size, Image.AFFINE, (1, magnitude * random.choice([-1, 1]), 0, 0, 1, 0), + Image.BICUBIC, fillcolor=self.fillcolor) + + +class ShearY(object): + def __init__(self, fillcolor=(128, 128, 128)): + self.fillcolor = fillcolor + + def __call__(self, x, magnitude): + return x.transform( + x.size, Image.AFFINE, (1, 0, 0, magnitude * random.choice([-1, 1]), 1, 0), + Image.BICUBIC, fillcolor=self.fillcolor) + + +class TranslateX(object): + def __init__(self, fillcolor=(128, 128, 128)): + self.fillcolor = fillcolor + + def __call__(self, x, magnitude): + return x.transform( + x.size, Image.AFFINE, (1, 0, magnitude * x.size[0] * random.choice([-1, 1]), 0, 1, 0), + fillcolor=self.fillcolor) + + +class TranslateY(object): + def __init__(self, fillcolor=(128, 128, 128)): + self.fillcolor = fillcolor + + def __call__(self, x, magnitude): + return x.transform( + x.size, Image.AFFINE, (1, 0, 0, 0, 1, magnitude * x.size[1] * random.choice([-1, 1])), + fillcolor=self.fillcolor) + + +class Rotate(object): + def __call__(self, x, magnitude): + rot = x.convert("RGBA").rotate(magnitude * random.choice([-1, 1])) + return Image.composite(rot, Image.new("RGBA", rot.size, (128,) * 4), rot).convert(x.mode) + + +class Color(object): + def __call__(self, x, magnitude): + return ImageEnhance.Color(x).enhance(1 + magnitude * random.choice([-1, 1])) + + +class Posterize(object): + def __call__(self, x, magnitude): + return ImageOps.posterize(x, magnitude) + + +class Solarize(object): + def __call__(self, x, magnitude): + return ImageOps.solarize(x, magnitude) + + +class Contrast(object): + def __call__(self, x, magnitude): + return ImageEnhance.Contrast(x).enhance(1 + magnitude * random.choice([-1, 1])) + + +class Sharpness(object): + def __call__(self, x, magnitude): + return ImageEnhance.Sharpness(x).enhance(1 + magnitude * random.choice([-1, 1])) + + +class Brightness(object): + def __call__(self, x, magnitude): + return ImageEnhance.Brightness(x).enhance(1 + magnitude * random.choice([-1, 1])) + + +class AutoContrast(object): + def __call__(self, x, magnitude): + return ImageOps.autocontrast(x) + + +class Equalize(object): + def __call__(self, x, magnitude): + return ImageOps.equalize(x) + + +class Invert(object): + def __call__(self, x, magnitude): + return ImageOps.invert(x) diff --git a/utils/rl_utils/ddpg.py b/utils/rl_utils/ddpg.py new file mode 100644 index 0000000..555e466 --- /dev/null +++ b/utils/rl_utils/ddpg.py @@ -0,0 +1,206 @@ +import logging +import torch +from torch import nn +import torch.nn.functional as F +import numpy as np + + +class PolicyNet(torch.nn.Module): + def __init__(self, state_dim, hidden_dim, action_dim, action_bound): + super(PolicyNet, self).__init__() + self.fc1 = torch.nn.Linear(state_dim, hidden_dim) + self.fc2 = torch.nn.Linear(hidden_dim, action_dim) + self.action_bound = action_bound + + def forward(self, x): + x = F.relu(self.fc1(x)) + return torch.tanh(self.fc2(x)) * self.action_bound + + +class RMMPolicyNet(torch.nn.Module): + def __init__(self, state_dim, hidden_dim, action_dim): + super(RMMPolicyNet, self).__init__() + self.fc1 = nn.Sequential( + nn.Linear(state_dim, hidden_dim), + nn.ReLU(inplace=True), + nn.Linear(hidden_dim, action_dim), + ) + self.fc2 = nn.Sequential( + nn.Linear(state_dim+action_dim, hidden_dim), + nn.ReLU(inplace=True), + nn.Linear(hidden_dim, action_dim), + ) + def forward(self, x): + a1 = torch.sigmoid(self.fc1(x)) + x = torch.cat([x,a1],dim=1) + a2 = torch.tanh(self.fc2(x)) + return torch.cat([a1,a2],dim=1) + +class QValueNet(torch.nn.Module): + def __init__(self, state_dim, hidden_dim, action_dim): + super(QValueNet, self).__init__() + self.fc1 = torch.nn.Linear(state_dim + action_dim, hidden_dim) + self.fc2 = torch.nn.Linear(hidden_dim, 1) + + def forward(self, x, a): + cat = torch.cat([x, a], dim=1) + x = F.relu(self.fc1(cat)) + return self.fc2(x) + + +class TwoLayerFC(torch.nn.Module): + def __init__( + self, num_in, num_out, hidden_dim, activation=F.relu, out_fn=lambda x: x + ): + super().__init__() + self.fc1 = nn.Linear(num_in, hidden_dim) + self.fc2 = nn.Linear(hidden_dim, hidden_dim) + self.fc3 = nn.Linear(hidden_dim, num_out) + + self.activation = activation + self.out_fn = out_fn + + def forward(self, x): + x = self.activation(self.fc1(x)) + x = self.activation(self.fc2(x)) + x = self.out_fn(self.fc3(x)) + return x + + +class DDPG: + """DDPG algo""" + + def __init__( + self, + num_in_actor, + num_out_actor, + num_in_critic, + hidden_dim, + discrete, + action_bound, + sigma, + actor_lr, + critic_lr, + tau, + gamma, + device, + use_rmm=True, + ): + + out_fn = (lambda x: x) if discrete else (lambda x: torch.tanh(x) * action_bound) + + if use_rmm: + self.actor = RMMPolicyNet( + num_in_actor, + hidden_dim, + num_out_actor, + ).to(device) + self.target_actor = RMMPolicyNet( + num_in_actor, + hidden_dim, + num_out_actor, + ).to(device) + else: + self.actor = TwoLayerFC( + num_in_actor, + num_out_actor, + hidden_dim, + activation=F.relu, + out_fn=out_fn, + ).to(device) + self.target_actor = TwoLayerFC( + num_in_actor, + num_out_actor, + hidden_dim, + activation=F.relu, + out_fn=out_fn, + ).to(device) + + self.critic = TwoLayerFC(num_in_critic, 1, hidden_dim).to(device) + self.target_critic = TwoLayerFC(num_in_critic, 1, hidden_dim).to(device) + self.target_critic.load_state_dict(self.critic.state_dict()) + self.target_actor.load_state_dict(self.actor.state_dict()) + self.actor_optimizer = torch.optim.Adam(self.actor.parameters(), lr=actor_lr) + self.critic_optimizer = torch.optim.Adam(self.critic.parameters(), lr=critic_lr) + self.gamma = gamma + self.sigma = sigma + self.action_bound = action_bound + self.tau = tau + self.action_dim = num_out_actor + self.device = device + + def take_action(self, state): + state = torch.tensor(np.expand_dims(state,0), dtype=torch.float).to(self.device) + action = self.actor(state)[0].detach().cpu().numpy() + + action = action + self.sigma * np.random.randn(self.action_dim) + action[0]=np.clip(action[0],0,1) + action[1]=np.clip(action[1],-1,1) + return action + def save_state_dict(self,name): + dicts = { + "critic":self.critic.state_dict(), + "target_critic":self.target_critic.state_dict(), + "actor":self.actor.state_dict(), + "target_actor":self.target_actor.state_dict() + } + torch.save(dicts,name) + def load_state_dict(self,name): + dicts = torch.load(name) + self.critic.load_state_dict(dicts["critic"]) + self.target_critic.load_state_dict(dicts["target_critic"]) + self.actor.load_state_dict(dicts["actor"]) + self.target_actor.load_state_dict(dicts["target_actor"]) + def soft_update(self, net, target_net): + for param_target, param in zip(target_net.parameters(), net.parameters()): + param_target.data.copy_( + param_target.data * (1.0 - self.tau) + param.data * self.tau + ) + + def update(self, transition_dict): + states = torch.tensor(transition_dict["states"], dtype=torch.float).to( + self.device + ) + actions = ( + torch.tensor(transition_dict["actions"], dtype=torch.float) + .to(self.device) + ) + rewards = ( + torch.tensor(transition_dict["rewards"], dtype=torch.float) + .view(-1, 1) + .to(self.device) + ) + next_states = torch.tensor( + transition_dict["next_states"], dtype=torch.float + ).to(self.device) + dones = ( + torch.tensor(transition_dict["dones"], dtype=torch.float) + .view(-1, 1) + .to(self.device) + ) + + next_q_values = self.target_critic( + torch.cat([next_states, self.target_actor(next_states)], dim=1) + ) + q_targets = rewards + self.gamma * next_q_values * (1 - dones) + critic_loss = torch.mean( + F.mse_loss( + self.critic(torch.cat([states, actions], dim=1)), + q_targets, + ) + ) + self.critic_optimizer.zero_grad() + critic_loss.backward() + self.critic_optimizer.step() + + actor_loss = -torch.mean( + self.critic( + torch.cat([states, self.actor(states)], dim=1) + ) + ) + self.actor_optimizer.zero_grad() + actor_loss.backward() + self.actor_optimizer.step() + logging.info(f"update DDPG: actor loss {actor_loss.item():.3f}, critic loss {critic_loss.item():.3f}, ") + self.soft_update(self.actor, self.target_actor) # soft-update the target policy net + self.soft_update(self.critic, self.target_critic) # soft-update the target Q value net diff --git a/utils/rl_utils/rl_utils.py b/utils/rl_utils/rl_utils.py new file mode 100644 index 0000000..33799bb --- /dev/null +++ b/utils/rl_utils/rl_utils.py @@ -0,0 +1,20 @@ +from tqdm import tqdm +import numpy as np +import torch +import collections +import random + +class ReplayBuffer: + def __init__(self, capacity): + self.buffer = collections.deque(maxlen=capacity) + + def add(self, state, action, reward, next_state, done): + self.buffer.append((state, action, reward, next_state, done)) + + def sample(self, batch_size): + transitions = random.sample(self.buffer, batch_size) + state, action, reward, next_state, done = zip(*transitions) + return np.array(state), np.array(action), reward, np.array(next_state), done + + def size(self): + return len(self.buffer) \ No newline at end of file diff --git a/utils/toolkit.py b/utils/toolkit.py new file mode 100644 index 0000000..87ab8b9 --- /dev/null +++ b/utils/toolkit.py @@ -0,0 +1,73 @@ +import os +import numpy as np +import torch + + +def count_parameters(model, trainable=False): + if trainable: + return sum(p.numel() for p in model.parameters() if p.requires_grad) + return sum(p.numel() for p in model.parameters()) + + +def tensor2numpy(x): + return x.cpu().data.numpy() if x.is_cuda else x.data.numpy() + + +def target2onehot(targets, n_classes): + onehot = torch.zeros(targets.shape[0], n_classes).to(targets.device) + onehot.scatter_(dim=1, index=targets.long().view(-1, 1), value=1.0) + return onehot + + +def makedirs(path): + if not os.path.exists(path): + os.makedirs(path) + + +def accuracy(y_pred, y_true, nb_old, increment=10): + assert len(y_pred) == len(y_true), "Data length error." + all_acc = {} + all_acc["total"] = np.around( + (y_pred == y_true).sum() * 100 / len(y_true), decimals=2 + ) + + # Grouped accuracy + for class_id in range(0, np.max(y_true), increment): + idxes = np.where( + np.logical_and(y_true >= class_id, y_true < class_id + increment) + )[0] + label = "{}-{}".format( + str(class_id).rjust(2, "0"), str(class_id + increment - 1).rjust(2, "0") + ) + all_acc[label] = np.around( + (y_pred[idxes] == y_true[idxes]).sum() * 100 / len(idxes), decimals=2 + ) + + # Old accuracy + idxes = np.where(y_true < nb_old)[0] + all_acc["old"] = ( + 0 + if len(idxes) == 0 + else np.around( + (y_pred[idxes] == y_true[idxes]).sum() * 100 / len(idxes), decimals=2 + ) + ) + + # New accuracy + idxes = np.where(y_true >= nb_old)[0] + all_acc["new"] = np.around( + (y_pred[idxes] == y_true[idxes]).sum() * 100 / len(idxes), decimals=2 + ) + + return all_acc + + +def split_images_labels(imgs): + # split trainset.imgs in ImageFolder + images = [] + labels = [] + for item in imgs: + images.append(item[0]) + labels.append(item[1]) + + return np.array(images), np.array(labels)