From 936caa84a89e40073fcc4084033f9ab3692faaab Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 12 Jan 2024 16:01:51 -0500 Subject: [PATCH 1/9] Add base ResNet implementation --- README.md | 1 + resnet-burn/Cargo.toml | 22 +++++ resnet-burn/LICENSE-APACHE | 1 + resnet-burn/LICENSE-MIT | 1 + resnet-burn/README.md | 30 ++++++ resnet-burn/examples/inference.rs | 16 ++++ resnet-burn/src/lib.rs | 3 + resnet-burn/src/model/block.rs | 151 ++++++++++++++++++++++++++++++ resnet-burn/src/model/mod.rs | 2 + resnet-burn/src/model/resnet.rs | 108 +++++++++++++++++++++ 10 files changed, 335 insertions(+) create mode 100644 resnet-burn/Cargo.toml create mode 120000 resnet-burn/LICENSE-APACHE create mode 120000 resnet-burn/LICENSE-MIT create mode 100644 resnet-burn/README.md create mode 100644 resnet-burn/examples/inference.rs create mode 100644 resnet-burn/src/lib.rs create mode 100644 resnet-burn/src/model/block.rs create mode 100644 resnet-burn/src/model/mod.rs create mode 100644 resnet-burn/src/model/resnet.rs diff --git a/README.md b/README.md index bc62fc2..985ea0d 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ examples constructed using the [Burn](https://github.com/burn-rs/burn) deep lear | Model | Description | Repository Link | | ---------------------------------------------- | ------------------------------------------------- | -------------------------------------------- | | [SqueezeNet](https://arxiv.org/abs/1602.07360) | A small CNN-based model for image classification. | [squeezenet-burn](squeezenet-burn/README.md) | +| [ResNet](https://arxiv.org/abs/1512.03385) | A CNN based on residual blocks with skip connections. | [resnet-burn](resnet-burn/README.md) | ## Community Contributions diff --git a/resnet-burn/Cargo.toml b/resnet-burn/Cargo.toml new file mode 100644 index 0000000..7da50fd --- /dev/null +++ b/resnet-burn/Cargo.toml @@ -0,0 +1,22 @@ +[package] +authors = ["guillaumelagrange "] +license = "MIT OR Apache-2.0" +name = "resnet-burn" +version = "0.1.0" +edition = "2021" + +[features] +default = ["burn/default"] + + +[dependencies] +# Note: default-features = false is needed to disable std +burn = { version = "0.11.1", default-features = false } +serde = { version = "1.0.192", default-features = false, features = [ + "derive", + "alloc", +] } # alloc is for no_std, derive is needed + +[dev-dependencies] +# Used by the classify example +burn-ndarray = { version = "0.11.1", package = "burn-ndarray" } \ No newline at end of file diff --git a/resnet-burn/LICENSE-APACHE b/resnet-burn/LICENSE-APACHE new file mode 120000 index 0000000..965b606 --- /dev/null +++ b/resnet-burn/LICENSE-APACHE @@ -0,0 +1 @@ +../LICENSE-APACHE \ No newline at end of file diff --git a/resnet-burn/LICENSE-MIT b/resnet-burn/LICENSE-MIT new file mode 120000 index 0000000..76219eb --- /dev/null +++ b/resnet-burn/LICENSE-MIT @@ -0,0 +1 @@ +../LICENSE-MIT \ No newline at end of file diff --git a/resnet-burn/README.md b/resnet-burn/README.md new file mode 100644 index 0000000..5b48334 --- /dev/null +++ b/resnet-burn/README.md @@ -0,0 +1,30 @@ +# ResNet Burn + +To this day, [ResNet](https://arxiv.org/abs/1512.03385)s are still a strong baseline for your image classification tasks. You can find the [Burn](https://github.com/tracel-ai/burn) implementation for the ResNet variants in [src/model/resnet.rs](src/model/resnet.rs). + +The model is [no_std compatible](https://docs.rust-embedded.org/book/intro/no-std.html). + +## Usage + +### `Cargo.toml` + +Add this to your `Cargo.toml`: + + + +### Example Usage + +The [inference example](examples/inference.rs) initializes a ResNet-18 with the `NdArray` backend and performs inference on a single input image. You can also run it yourself with the following command: + +```shell +cargo run --release --example inference +``` + + +TODO: +- [ ] Load pre-trained weights +- [ ] Replace dummy input with actual image +- [ ] Training example \ No newline at end of file diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs new file mode 100644 index 0000000..67d69f4 --- /dev/null +++ b/resnet-burn/examples/inference.rs @@ -0,0 +1,16 @@ +use resnet_burn::model::resnet::ResNet; + +use burn::tensor::Tensor; +use burn_ndarray::NdArray; + +type Backend = NdArray; + +pub fn main() { + let x = Tensor::::ones([1, 3, 224, 224]); + + let model = ResNet::::resnet18(10); + + let out = model.forward(x); + + println!("Output scores: {}", out); +} diff --git a/resnet-burn/src/lib.rs b/resnet-burn/src/lib.rs new file mode 100644 index 0000000..a6708a7 --- /dev/null +++ b/resnet-burn/src/lib.rs @@ -0,0 +1,3 @@ +#![no_std] +pub mod model; +extern crate alloc; diff --git a/resnet-burn/src/model/block.rs b/resnet-burn/src/model/block.rs new file mode 100644 index 0000000..36a98c7 --- /dev/null +++ b/resnet-burn/src/model/block.rs @@ -0,0 +1,151 @@ +use alloc::vec::Vec; + +use burn::{ + module::Module, + nn::{ + conv::{Conv2d, Conv2dConfig}, + BatchNorm, BatchNormConfig, Initializer, PaddingConfig2d, ReLU, + }, + tensor::{backend::Backend, Tensor}, +}; + +/// ResNet basic residual block implementation. +/// Derived from [torchivision.models.resnet.BasicBlock](https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py) +#[derive(Module, Debug)] +pub struct ResidualBlock { + conv1: Conv2d, + bn1: BatchNorm, + relu: ReLU, + conv2: Conv2d, + bn2: BatchNorm, + downsample: Option>, +} + +impl ResidualBlock { + pub fn new(in_channels: usize, out_channels: usize, stride: usize) -> Self { + // conv3x3 + let conv1 = Conv2dConfig::new([in_channels, out_channels], [3, 3]) + .with_stride([stride, stride]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: false, // TODO: switch to true when fixed in burn + }) + .init(); + let bn1 = BatchNormConfig::new(out_channels).init(); + let relu = ReLU::new(); + // conv3x3 + let conv2 = Conv2dConfig::new([out_channels, out_channels], [3, 3]) + .with_stride([1, 1]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: false, // TODO: switch to true when fixed in burn + }) + .init(); + let bn2 = BatchNormConfig::new(out_channels).init(); + + let downsample = { + if stride != 1 || in_channels != out_channels { + Some(Downsample::new(in_channels, out_channels, stride)) + } else { + None + } + }; + + Self { + conv1, + bn1, + relu, + conv2, + bn2, + downsample, + } + } + + pub fn forward(&self, input: Tensor) -> Tensor { + let identity = input.clone(); + + // Conv block + let out = self.conv1.forward(input); + let out = self.bn1.forward(out); + let out = self.relu.forward(out); + let out = self.conv2.forward(out); + let out = self.bn2.forward(out); + + // Skip connection + let out = { + match &self.downsample { + Some(downsample) => out + downsample.forward(identity), + None => out + identity, + } + }; + + // Activation + let out = self.relu.forward(out); + + out + } +} + +/// Downsample layer applies a 1x1 conv to reduce the resolution [H, W] and adjust the number of channels. +#[derive(Module, Debug)] +pub struct Downsample { + conv: Conv2d, + bn: BatchNorm, +} + +impl Downsample { + pub fn new(in_channels: usize, out_channels: usize, stride: usize) -> Self { + // conv1x1 (default padding = valid) + let conv = Conv2dConfig::new([in_channels, out_channels], [1, 1]) + .with_stride([stride, stride]) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: false, // TODO: switch to true when fixed in burn + }) + .init(); + let bn = BatchNormConfig::new(out_channels).init(); + + Self { conv, bn } + } + + pub fn forward(&self, input: Tensor) -> Tensor { + let out = self.conv.forward(input); + let out = self.bn.forward(out); + + out + } +} + +/// Collection of sequential residual blocks. +#[derive(Module, Debug)] +pub struct LayerBlock { + blocks: Vec>, +} + +impl LayerBlock { + pub fn new(num_blocks: usize, in_channels: usize, out_channels: usize) -> Self { + let blocks = (0..num_blocks) + .map(|b| { + if b == 0 { + ResidualBlock::new(in_channels, out_channels, 2) + } else { + ResidualBlock::new(out_channels, out_channels, 2) + } + }) + .collect(); + + Self { blocks } + } + pub fn forward(&self, input: Tensor) -> Tensor { + let mut out = input; + for block in &self.blocks { + out = block.forward(out); + } + out + } +} diff --git a/resnet-burn/src/model/mod.rs b/resnet-burn/src/model/mod.rs new file mode 100644 index 0000000..dcbbb7a --- /dev/null +++ b/resnet-burn/src/model/mod.rs @@ -0,0 +1,2 @@ +mod block; +pub mod resnet; diff --git a/resnet-burn/src/model/resnet.rs b/resnet-burn/src/model/resnet.rs new file mode 100644 index 0000000..f06665b --- /dev/null +++ b/resnet-burn/src/model/resnet.rs @@ -0,0 +1,108 @@ +use burn::{ + module::Module, + nn::{ + conv::{Conv2d, Conv2dConfig}, + pool::{AdaptiveAvgPool2d, AdaptiveAvgPool2dConfig, MaxPool2d, MaxPool2dConfig}, + BatchNorm, BatchNormConfig, Initializer, Linear, LinearConfig, PaddingConfig2d, ReLU, + }, + tensor::{backend::Backend, Tensor}, +}; + +use super::block::LayerBlock; + +/// ResNet implementation. +/// Derived from [torchivision.models.resnet.ResNet](https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py) +#[derive(Module, Debug)] +pub struct ResNet { + conv1: Conv2d, + bn1: BatchNorm, + relu: ReLU, + maxpool: MaxPool2d, + layer1: LayerBlock, + layer2: LayerBlock, + layer3: LayerBlock, + layer4: LayerBlock, + avgpool: AdaptiveAvgPool2d, + fc: Linear, +} + +impl ResNet { + fn new(blocks: [usize; 4], num_classes: usize) -> Self { + // 7x7 conv, 64, /2 + let conv1 = Conv2dConfig::new([3, 64], [7, 7]) + .with_stride([2, 2]) + .with_padding(PaddingConfig2d::Explicit(3, 3)) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: false, // TODO: switch to true when fixed in burn + }) + .init(); + let bn1 = BatchNormConfig::new(64).init(); + let relu = ReLU::new(); + // 3x3 maxpool, /2 + let maxpool = MaxPool2dConfig::new([3, 3]) + .with_strides([2, 2]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .init(); + + // Residual blocks + let layer1 = LayerBlock::new(blocks[0], 64, 64); + let layer2 = LayerBlock::new(blocks[1], 64, 128); + let layer3 = LayerBlock::new(blocks[2], 128, 256); + let layer4 = LayerBlock::new(blocks[2], 256, 512); + + // Average pooling [B, 512, H, W] -> [B, 512, 1, 1] + let avgpool = AdaptiveAvgPool2dConfig::new([1, 1]).init(); + + // Output layer + let fc = LinearConfig::new(512, num_classes).init(); + + Self { + conv1, + bn1, + relu, + maxpool, + layer1, + layer2, + layer3, + layer4, + avgpool, + fc, + } + } + + pub fn resnet18(num_classes: usize) -> Self { + Self::new([2, 2, 2, 2], num_classes) + } + + pub fn resnet34(num_classes: usize) -> Self { + Self::new([3, 4, 6, 3], num_classes) + } + + // TODO: resnet{50, 101, 152} use different blocks + + pub fn forward(&self, input: Tensor) -> Tensor { + // First block + let out = self.conv1.forward(input); + let out = self.bn1.forward(out); + let out = self.relu.forward(out); + let out = self.maxpool.forward(out); + + // Residual blocks + let out = self.layer1.forward(out); + let out = self.layer2.forward(out); + let out = self.layer3.forward(out); + let out = self.layer4.forward(out); + + let out = self.avgpool.forward(out); + // Reshape [B, C, 1, 1] -> [B, C] + // println!("Flatten in: {:?}", out.shape()); + // let out = out.flatten(2, 3); + let out: Tensor = out.squeeze(3); + let out: Tensor = out.squeeze(2); + let out = self.fc.forward(out); + + out + } +} From bc48327ca6b8b6ecadafd1ef73abd5e9facd419f Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Tue, 16 Jan 2024 13:32:11 -0500 Subject: [PATCH 2/9] Fix ResNet blocks and add pre-trained model for inference on input image --- resnet-burn/Cargo.toml | 4 +- resnet-burn/NOTICES.md | 16 + resnet-burn/README.md | 20 +- resnet-burn/examples/inference.rs | 70 +- resnet-burn/samples/dog.jpg | Bin 0 -> 83281 bytes resnet-burn/src/model/block.rs | 11 +- resnet-burn/src/model/imagenet.rs | 1049 +++++++++++++++++++++++++++++ resnet-burn/src/model/mod.rs | 1 + resnet-burn/src/model/resnet.rs | 9 +- 9 files changed, 1151 insertions(+), 29 deletions(-) create mode 100644 resnet-burn/NOTICES.md create mode 100644 resnet-burn/samples/dog.jpg create mode 100644 resnet-burn/src/model/imagenet.rs diff --git a/resnet-burn/Cargo.toml b/resnet-burn/Cargo.toml index 7da50fd..b2bd614 100644 --- a/resnet-burn/Cargo.toml +++ b/resnet-burn/Cargo.toml @@ -18,5 +18,5 @@ serde = { version = "1.0.192", default-features = false, features = [ ] } # alloc is for no_std, derive is needed [dev-dependencies] -# Used by the classify example -burn-ndarray = { version = "0.11.1", package = "burn-ndarray" } \ No newline at end of file +burn-ndarray = { version = "0.11.1", package = "burn-ndarray" } +image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file diff --git a/resnet-burn/NOTICES.md b/resnet-burn/NOTICES.md new file mode 100644 index 0000000..25bf435 --- /dev/null +++ b/resnet-burn/NOTICES.md @@ -0,0 +1,16 @@ +# NOTICES AND INFORMATION + +This file contains notices and information required by libraries that this repository copied or derived from. The use of the following resources complies with the licenses provided. + +## Sample Image + +Image Title: Standing yellow Labrador Retriever dog. +Author: Djmirko +Source: https://commons.wikimedia.org/wiki/File:YellowLabradorLooking_new.jpg +License: https://creativecommons.org/licenses/by-sa/3.0/ + +## Pre-trained Model + +The ImageNet pre-trained model was ported from [`torchvision.models.ResNet18_Weights.IMAGENET1K_V1`](https://pytorch.org/vision/stable/models/generated/torchvision.models.resnet18.html#torchvision.models.ResNet18_Weights). + +As opposed to [other pre-trained models](https://pytorch.org/vision/stable/models/generated/torchvision.models.regnet_y_128gf.html#torchvision.models.RegNet_Y_128GF_Weights) in `torchvision`, no specific license was linked to the weights, which are assumed to be under the library's [BSD-3-Clause license](https://github.com/pytorch/vision/blob/main/LICENSE) ([ref](https://github.com/pytorch/vision/issues/160)). diff --git a/resnet-burn/README.md b/resnet-burn/README.md index 5b48334..ff1e141 100644 --- a/resnet-burn/README.md +++ b/resnet-burn/README.md @@ -10,21 +10,15 @@ The model is [no_std compatible](https://docs.rust-embedded.org/book/intro/no-st Add this to your `Cargo.toml`: - +resnet-burn = { git = "https://github.com/burn-rs/models", package = "resnet-burn", default-features = false } +``` ### Example Usage -The [inference example](examples/inference.rs) initializes a ResNet-18 with the `NdArray` backend and performs inference on a single input image. You can also run it yourself with the following command: - -```shell -cargo run --release --example inference -``` - +The [inference example](examples/inference.rs) initializes a ResNet-18 with the `NdArray` backend, loads the ImageNet [pre-trained weights](model/) and performs inference on the provided input image. You can also run it yourself with the following command: -TODO: -- [ ] Load pre-trained weights -- [ ] Replace dummy input with actual image -- [ ] Training example \ No newline at end of file +```sh +cargo run --release --example inference samples/dog.jpg +``` \ No newline at end of file diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs index 67d69f4..4b209a4 100644 --- a/resnet-burn/examples/inference.rs +++ b/resnet-burn/examples/inference.rs @@ -1,16 +1,76 @@ -use resnet_burn::model::resnet::ResNet; +use std::path::Path; -use burn::tensor::Tensor; +use resnet_burn::model::{imagenet, resnet::ResNet}; + +use burn::{ + module::Module, + record::{FullPrecisionSettings, NamedMpkGzFileRecorder}, + tensor::Tensor, +}; use burn_ndarray::NdArray; +use image::{self, GenericImageView, Pixel}; + +const HEIGHT: usize = 224; +const WIDTH: usize = 224; type Backend = NdArray; pub fn main() { - let x = Tensor::::ones([1, 3, 224, 224]); + // Load image + let img_path = std::env::args().nth(1).expect("No image path provided"); + let img = image::open(&img_path).unwrap_or_else(|_| panic!("Failed to load image: {img_path}")); + + // Resize to 224x224 + let resized_img = img.resize_exact( + WIDTH as u32, + HEIGHT as u32, + image::imageops::FilterType::Triangle, // also known as bilinear in 2D + ); + + // 3d array of 224x224x3 floats + let mut img_array = [[[0.0; WIDTH]; HEIGHT]; 3]; + + // Iterate over the pixels and populate the array + for y in 0..HEIGHT { + for x in 0..WIDTH { + let pixel = resized_img.get_pixel(x as u32, y as u32); + let rgb = pixel.to_rgb(); - let model = ResNet::::resnet18(10); + img_array[0][y][x] = rgb[0] as f32 / 255.0; + img_array[1][y][x] = rgb[1] as f32 / 255.0; + img_array[2][y][x] = rgb[2] as f32 / 255.0; + } + } + // Create a tensor from the array + let image_input = Tensor::::from_data(img_array).reshape([1, 3, HEIGHT, WIDTH]); + let recorder = NamedMpkGzFileRecorder::::new(); + + // Normalize the image + let x = imagenet::Normalizer::new().normalize(image_input); + + // Load pre-trained ResNet-18 + let model_path = Path::new(file!()) + .parent() + .unwrap() + .parent() + .unwrap() + .join("model/resnet18-ImageNet1k"); + let model = ResNet::::resnet18(1000) + .load_file(model_path.clone(), &recorder) + .unwrap_or_else(|_| panic!("Failed to load model file: {}", model_path.display())); + + // Forward pass let out = model.forward(x); - println!("Output scores: {}", out); + // Output class index w/ score (raw) + let (score, idx) = out.max_dim_with_indices(1); + let idx = idx.into_scalar() as usize; + + println!( + "Predicted: {}\nCategory Id: {}\nScore: {:.4}", + imagenet::CLASSES[idx], + idx, + score.into_scalar() + ); } diff --git a/resnet-burn/samples/dog.jpg b/resnet-burn/samples/dog.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c1487ff578ef63d2d074aa42d6831311342687bc GIT binary patch literal 83281 zcmbTdcTkgC_&@lD1VRbDm`H+=L`tyGO@JUpdJk1VFHH~>3|)dKML;lwu9O%+dX-*9 zgLIT4O$5|PKtPI0^Mc-cm+$V*Z)bL9XLk3z^Zs?td**!RywCG|p7!6|e=I=A)X>Bb z0D(Y&+wlYZw*cq^++18xE>3PJ6w1TH%?lIbhw<^j5F(<2VyKg+q);c3NHq4mEL!?3 z28oorD1TN_NmW(#lq`XWSJpbOqN@Bqn}B$Dcwl@mI6psJ`84vh^8e@Y9}^Jf=FsJM z0R~9}9Ks;5FzCNN;1mFWIFC>JKL`Fl2Z#d<;pBpH^YHQ=FK7`0I6z=92L#N?34t81 zjyj$PAi|s?riG3iCz7`=ubanidVSQU?;SRP)96L7@IGcvMr@;FsB z^^16dhW=#(LnC7oQ(HTG2S=wX&YoV^y?t)@`UQuChK1jWpu|0hPe^>2l$?>7mHqfh zPHx__;*!#`@`}o;#-`?$*0%PJ&ey$t{R4wT!y}VZ(=)U0=RVATSz<4*d|h2z|Mqid zcW?jT*WZUn|APwzfd3Cz{|njw4VUmSE)EC;41xX+E)Ylf@e3A)aGqA;64AARx(A9% zE5~q)>7^Gn^zdL*NI%3qg5L2;U{$}I`T0N4{&!^m?|{Yr|3db^fc;;%<^dQObbNVW zVc-%l7i;rM_BrJx$~0#B$iefpep;chZEx2~5Bt*#`|bz(XWJ!%?rXQefl6?irtQN+ z<3VDuxq)n6Yt2xTxb^dH0swH}e^m-mK{P>1%A7oWBhO)SMMRm(fm7AYsl%7M=SFd= z89lZX2>!%qoe=s8gqDHYrT(3+3F7|#KO0?p7XJUR&$-e8!U)yPJID38WDq|_Y3!0I zxo1_(^{PJMx+G;{L^GzJ_nsTY=Ct?NJA(l^Ya^7LJ<5t=kCqdA*Yh>8g10uVhamPI zws?rfOboy>d=g!7Yzs0xa;4YW%~^;5-AGFm)eMUct=_-`Fb?Ukm;$Q}B60*eY<& z2r*bIdGDH~RSPCo2mJh+oYU}jpoM}!uh(g#HyguyWrizr02yb(B?hy*f>YEe@Usfi zb8^8h+ocWIldV)arPY>G2eM$XhGff#qBBuoo4ObN&~Cj3QQFrsoKW!R@@`x^+%?#& z;(n9eTiQ+dAbLZk&C0dx`wbz~IuHO0e-54p?ue93^Sm4FSB2S)I>dH<% zTq<=FH{`OSP_ty~%gZJOTvAPa*UaF{KF_uPOnLUwmsyNRTiPrNx?&6(8sI!%j78&q zXarMj1u+cn_2jS>rtRU8t&_yi`t;ovle)rY1-=JHcmh$0#50p4LByAxD}rK!qM~r} z?!DQ%^JS#^&8uhI*&XHJ^zF#Ja*?Nn>65i_EkTr{@Y41=-U1cMZ(9^^L(lU5w(svT zGPG!%^013*2*vZ~hNLq$yc)1nn$7As*BnX|^g~>UW^T&s%$3W+mr3D5KwqpCl$vUq z-ouy4gq})qIKeA5k||sHO-@Oq zWS1Hjr#F4xK?!~8T63*PW)GXDX%{PlWGu3#xI>qJX3jop2GH{_c4=iffn8A9b%!Yt zqn-+jYM*;#DEwD(P{hOK!zX-;;ipP7J^yZPpZy$h>x;eK1vcEBR_+IA-0XVKImQ1E#!%sBJq4eO@QO z#$;1rUeiIV;V4Sxh6CHf#x1Cvnz?R4xE2>PT*f~KGYtNen8P_hAgCj(_Vf~QowBG%_~3(g9(k7^0sTmPEp z2!E|ZCj9YpP4|FU&z?#~!g=S(utfrkjz8b1_IELGHq z4(t-!ze^2a&!+ul?M`=l*9Oo%>3Q(&sDOWgCk$W8&bhC0LY0CR|Q|^UwNXT+_uQL-o zB6(epoP^Q5&}%OOhf6|S@AlFP_eNEGWog7tJLRTW9ipbxG_lUchjKyRhCl+K1-DH- zFb%_7q>{6ujPjNZ*zrdEB82YU8X}g8v#tsRi}3f{n?3< zxATW%v|lz4`m=--Ewt6o(1QDU7J|aX3T3e@(f-q^VZfUFed>9YxJh~@u&e}5zH!zr z%m3EuPB{b>S@4(XQ-(UhT2au}c>6^x{y_X(7>5?;;48%FW$P1Q6eVum0e+qp_LIh` z_Uv<9!Cg|lvfybGCQCqx)OpL*3q}?!Rf_>w!SCT)R&v*=Frfj)nwr;*p8QI7UiKJ( zU!U60X~lI#vkBw1L_@pP3cMD{q|i9JeqjpQt!2<{Xc%ldB$hb=qW_*M&QxH0T>n_x zBcmqj+k+NS8Z0FwS`6}b8+GG3uB1X@n7=CEDvIEk#TmbtT0^eWhj^)8>U16W0=U5P zbygo#7T&M7oUk{bLF#fzlpDriphPRe*T?wAj)91>wAEAc+B3e^4}UMK zCM(Sy8^lBV!``$*YCQrz!7RcD_uq5 z9}y*|s`2+TJgHR5OUt>=im{b=y5E z#NMqnyEg>S{IhZIP0yrq^);*deK|V&XEpWAxwI?v;nOdh*PdngEFjMh&N&vk+)!7q z;&7=J{$r`^H@|SXAWZd?uJdoGztgl8@ubCGBA?&iQthY4o@JW7vK_4{M}<{h8tIlD z*8PLF_D}nLG9cEoQLDSw$L4W@j^+x#LEL1w?Hh%0?5Vp`*Xvt5NliV-xYm_ZW2IkS z^z`j~Osp&qPK#1#?fwx}!_4=+Cb22__R>K8?Yo*4ALQLd#^Kxaf{~_dp25NzFZx9Y zjZosMh(`yG-6={k9DJ<1o=gINfX4n7WTo6V~U0_4s@4ne z&~mD&#h0c5E_qafz@$8N&=*=O@Li`tp-_n`a;a9SSBym_-f&!xV;;*wnWL+ZxZ}e* zdrNu-+3)hHOXXq%p|apMSCW!vtjb`ZZHRr+f%;}%T5BH`uuyV+*Z~c%f{rnehHi} z@{HNv)|gQCI7G0CID~%vkT%m6)_)>BBl4j1ikafLm9wPzMGWT%%geK}ve@wTD%QIM z%9p*fvyq<0kEsvU&iO`}#zu2-MRm;v{F-o1HQI%T5eM!RzRpktw^969Fwp*L3y=D+ zQd;mrH))q^`0eQv;E98Z58Na}#VGj!>oKUchG(~Yc9>@+5@h;~eVGR#8>Z+|DiDWe z{o`jHyr7ztxXgmF5!Jfn_arHI_+qGc=bvx0qs6( zA%*3A{CKU?+nWgNE|vbyJ^PUod`uUwy>}q%rrDHgnN=;ljo%sxrL>?ZA(Vu`DgE?z)AQOu#&MrMLjROstD9uuGe+dDHb3u@-3f zBaGdqYZ1mLoEA8|LE3ZGkS090dMZ&(#2?4ggGIt@Kl6=S6>gZ6x7S>Ji*|z* zOiwvA`LwIj?1#Xyfq&cf@TD+ApRI2zHB^jaE?G`@{aEyR7cGcbt(CXd=~-2sY@E2t zdAFX_4J{>}ptfBqV+K(c#{Dw3FJj~9VRc6# zZ1K7kXJ!odp(MCYg~Vmqcak~dbDdhE?A9gzM}D`amDzP zDm{Dr@@@|qik^O9uW9!whQ3RC%6SDSc4G)y-VBYrnyVRtG_*JmFiVESgpa)}fjzWX z%$T1F{kkRRc`hrhOq+Up|Aik`_@b^9>DOt!=&v*dtv6?tSIBD-n8`YW3ps`CS~YAR z+oNSR*C)I{eO^|ZT`<;g;l%~-1KNSLc}9spfXPVM14oiMv;&x^Cb4Yl|%J=-nGL@m>=br{II#uaj5Mly+`9 zxycPZb@6_w%)f&7?&GYhV@<}zX0PS0XHIonE=i?kK-V{{!apozPKk@2rdE5AgH15E zu_31{r(y0 zGg-G|On~3;IayNZ(@IlMbyb{})W@nWcTqI2@$xN^h-y>jw&`{W5*cg}#T36(d)YMs za!(CUez$knZNcZjaqMi<1+q`KJHUw=_Iil1Pc4$wdRAqAt|htZ(@*It{)48J27YV3 zi%OpTPNC2ul?(rQ_FL&Y(e#N6FDju5Qam_VI57e0g7ksmObw(%NIW6*+hN z!2QO_#7JA0n2R}1!8UE?tECDVz`}T~qC8=^jCq1EW>U`9^4Ti|0HZ+j!z9E~V!a4Q zapw4J+TD-PFUBK!`(o2UPlF5oa=y;H#=ARCtT3JyV9UlN-y9y(9c;n9Jb@Cloqy7) zkP)qvApjNY0l+vAc9+py)PtncL;B58c0tGPtJur~C1Kq!bWsz>nNa$Om+U)@?&)CwJ6#S+MxBd?s=T$gawm- zZ>ah}h0+gtqaE^O$0}JlL zX8&1z5a9mCc?c=`iz1{TeO1z?E%O2-L6K51EUx@uHiiH4^R$RdA#Nd+Hq)zTLED_t&BnQz~N>-fx@+ z^d}7yL8X!P7W?pY&a{kP?Wcg)_ZJIIOm?|nWPkheDf6GRSt-%01SGq%)T2(-tyO1h zo=)yI8X*Up8=h{H3T9~jBhNqnT|MhC(7tinzrhgIG?D6@#_0;%i6n`ffY7~ye9HxN z$AeT%k2vQtUw=ZIXPu8tO zKogmJRfYV8beI6NTMn|XD!VB1awU{MX28%$Cjb13e{CiMT52UKZ4|3QlYjf|kAKg; z>Q;zlMNL}Aa;)(s=f-a^x0twZzJVPAD*MMgU48R5*2Z65ti(KNOeJQ+hLwP!+r0T!P7tk0~w!W0cQfKwzl$D%<|Jn+K4_Ima%zG>u+d64%DWLgFt(@>gCh;6wP^d0H zC|pE58i*Su{T*0lRsBt5ux_3_QJkKz@yjK3SAs>v5uls;B zc6jm?@H%22%YHL0h$7|JWvc59Nq@9$$6_+ zCO{Mv*Ud9~g605zk4`r3cWpe1^rlVJW4;>~=Z%rVI!ZjAd)_mldeZ$-#$TU8-Zn-X zjRcJtnX+%~OVAeXZbzA%OPQJSeJ{N8>mLO+942|A1=Q9qsI4{0B~pP?aUySM<~;1ry%yG~$vKCZ( zWIp_ePL&FLFP(Z;N+5WePErAMZ;#}7o$aH>EaX|V$0zB|lY{@P4|7OhV@CDAW@fL) zOxql!ME@LjCP(SL{YyrtohKHjBAFShG#+f_5<6n`)2G(pAm%G9ow>Sh`1qt=Gve9{ zfvlRiArxizyOl`ws_#;%5;b&`*O6<;oREq^Gn8zqqsw37MpjG!pLNpU$)|&0HJfmc z4ie{LA45Js2#Ofch;dZ+XMqNtYtKe5FSAl@Og3%!+U7WIgZ|sR8ng^H}IO{K6o~nDiMx#gj9C>zJ*dUcbvAXy`9^wti_WIwG2yWRj`Jbja zl1Uh%d|4fMOcAQ%iXS%A`(b^gm3(mnA$*MSDhH;^kS0HngNLX3lhS@FCey z_y&n9C5go4i{BSDT~Xfbi$@IiRI*HvTii|u@XE)kL$Xu`uvgvC>1^;)&GG)VoFW6U z;;@adiso(Pa6`j*D zQXQjzHJ^|VVq3LzSv2X@Xy??Jz2Y%An_oySu9IZk^j2uOM-5*4=oi}VfTfVK+|Tt_ zY5lf8=8{Ri2N}gk+D(LgPMUt;uN2A#&NTjtzWk=)O@ZIN>bo@sK7F?qe>czn|BVFoAQJDh?+m%%!=#PAC7#g;`-><8+by`a*XEoXH zwl1TD7HLb_RJR;IAEXi+IrU(aiDzznH%=bSqzde%G)r0fZl1Pz)N0D|TqmiT{)Hd@ zX4TEDA2OgBIcdx~jadFSB>Q`X(({;R@zK+l6aiM|=h+k}Rs)-^9+G?u-3dZ%oel7*<38$EEU#i}{dDa-0)^TKCa z=h8FesodB!rM(LNH~y?33T^rz$3#L9A`obL);t$=l`Lqfoi%KD;syfP@uM3&c0m6i z4EjA?^@fNe1#-UTOH>D5_G6-oX$j4HM1`%kYhFXI=au<{@UoMQZ1l2dEXBL?4Oi#{ zhUcBrHn`UAgi7a{tN5p~_>c3_-0nIUFvy`XL*2pG3oUZb+KQv<(>ECbu?oNw;$Snw za66u-r@vn}8QI*!@SWr9eEODLtn?;Bn?8FlE)tp7{NOzAVEW%Pk?$isgIqyv@V!yl z1T$e!J`uotnf~*Y?k3)#BdB zC{_u8mY!Fp>N9>E(DM=>ZJ#gkFm3%<67t+Wj&ngPp5^i4Kj3w;qwTb}R;8P$_Ws8+ z<4?GOA6zbr?+s+VuK%^9%7d1oQ0N|cS0#&%Wyj*WIi~dUaRr z;kn|Q`uZ?6#)<$%)r@JL{p1?QmIFF9E+-xXsrw#qF?;t zS*$(5brrAW(d4M#=-pqdT={@|3QJ6dXZ+GS5Zj&OFE zV7LE$f8Cn7o`~C8Dj0PM<9zgQ`<6S4uCg(Dy&8 zUj7AHey?J=diHT~t3+z}^Si!*XM6x9!6!bKr0w{Wr977|4fRb{;5KXQ?5W%~HC`+{_(ntQoN zZre>Vc2zqIJE@ZP57cxD0%xLwH0-bV_oZgB*r(#nvnZSsGdYzzw~3+?Su`sbmD_C2n1MF)T~_d&Aa9R5^%ywG`u07GVwgwQgag zTXwaX*>F7Y!neE{l$v_N5U8_ zjuU?WT+UAZIT7=|&Nu!|bRp6w&39)A57lB!?}s2k!~cdfZ(FuWzIfKEh~wUINvH1a zSKkt!K0je#`11iNVnAuHYygeN`VjWltq50N_R{T4L0Sjj^CFQ#DjA$I(mY#59VBcD zTxNS}qQoh$*<9jlL_I5=eJ`#aXx4IM} zn&E5lbia!0jxD&Ip!UwRPe4E*C~&qax9lFrv91xsmwXqssu{1O<;4$dss;i0WtRO~ zs0>nt&RFOW_f`HJp$lXLPl;vByI3YFMO!(b{IcfdJ;d7di)Qr6h@my@yWY*`o=->f zHH>TRC@J69+%I9byZH4Q~0_tgdr zC=0ZUK4m;T)jvPAg3cDBVuou8acP@N9cN-hxb}OPweWz8R1t6}JMHG9wrSlSM|9wT zBEPS}-6CBrfqqMVQhI*UTz{CNX8LRmwuh$jzBH)*nu^UT#h{f(bI|2e)tE69Q)S9pwQ`u;XDKOX@Ro|~=#%fmp>hFV!mHSt_YDFall zBLYrXx6BrY%b}ufmi`oYFe~@onxK^V*fm6^gDq$m>MEfJzLueHaIm+KW-w{t~Nc z?|NSO!p9+f7%Y6#x6bUW7^Va;Y%m+z0?4mnx&#wVu3D|D-|45Ey2T`xT6(G=hMmU;Uo>dif6t-#2UxM5*8& z)X{XEe@yAbq?B`=$T?^W&c=|hVACp$cV>IOo$NozL@P@+LFPz&@=PIVTk7S zXcT8~Se^Y~&LAdQlXo*fME;qG^onFEIWl>9`xZm;m+^Gg$97TuNuzR8M_P;;Gs4_Y zxtK0ssc_Ts(dMji`HHvZ!);4HT7efBfyF2cXw1e9LE?O|u_l8pu1;nR9!65Om7Y7# zzxs{FZMC^iWMha)H9yScJdA-=sj`|Yov}lfCdl8BGl-U=Cxw4OX21I<8e}pkdE4x? zEg*XCnoHqmL>VpruFw6iqA9Fpq)*H>YF$+kmmb4V(7FAF&y~nNnj?a!37Vdf4avW0{YO3U#%{CGT6HE0PRZR<9s=UpG~qYS5>J-1dLfe#D8yl%(&#_lO4OjFK)@ z-xv7Iw?MYI{?#_0rrjb)S~(uo;KGBoY=^>^7|8|7;v)I;#m`X2zBUy#O$DqL*f>g4zAb=bXj_%6CD$dF4ZFYQY0vl<)P z1AxdcQ*bWDb!zU-G@^gw?hAq9IgtI`?Z4s3lN-;Qb8bJ=bM%V##I*ma@eR2$?QVXn zW1a3uG#)n7&+cCwj&PM%nD&HMZ7HamY%CtlmgE_Hxt(qo$tBkRuI|ezEhDKypF;;f zkwu4w9FQ+%`Uz4Rh)|PnyYpV#O#T-McZRfiIS#(gkXxC2eToC=)4uMYzZ*A9{8rVl zs|P@5pLS2$c9`fMO-N>F?vRzCrRws$FvfpC>7_g9N>}i@_2?g5F|B0!!?opRm!vn9 z=+y-=?wq*v{iUiw1w#rwR4t5-X#s8DuP)*3 z+$ZJ*3pA0@3)vXG{XC(rWbDvGM=;20uiE5`S=eD|SCEEK6{qiJ$%t9U+ZyI&J2uvH z(M}*d<3u2)5OuTTQSZE~U;(zn)orMc`VVeX$;E}LD`vApL5oFQ``ws<*Njr$UeX%$ zS8!vYT1x%D)EXpQJ7d1bS%0%ZMMlv{DllG^hbu^22jz57>#4`>KnVPzS+=t6VR;9T zkgmHB&z^@oQC5fQNt=cSyZZ<+0#JA$e$g z^OEuz`mc?Gb>~S<4tPAKRz7I1S6nJFnSg1D%Nn@Pd!;i?CSN<6YUlsu%(2R14jQ@6 z9)^#pq`$8P5(XxCe_@xNG$y(=5*&ZD>@>(tv}>(~@spnDbUn!M&j&)0{{gt~h-?6| zON%Y$M2hqR!{%<;s{fjvyN8CoOE9Hw0Fza^QM$i?g1T)__Y;}r;2l!*@{QLg@VZmnNYiQ z^nga=0T9SomAuZBZ0Ky0zg>;zah1Y;VKP}D((!VD+otI2Cqh~zSTRiaT2$5C%O@f4 z+_DDjyNyaIYF;;wLL6HSjfU>T;EbH}_72%)%ZxDikS@_mYAIL78W2PcRR;*i%tsu@ z<|P1uj$^CU^0h%m=RHW<0lHIOl0S$6ew96(bQmUJR~aoz+|?~QtJF!80N9Pw5m?ZI z>dcdM>c6TPMZ7MMQzVL=~2MBVa}dJf=e5wLg6(nlrcu zx}iV75xW?9G$ZS!DZMbp0jmq*znqcajbNe#{T>cn(W7x`T1W8i&Mm070rN4hh zgBUZ)u^`UqgDM`3%YQ)4a?D^Y{#wFu#%I3R|Cczm-Ru0HpL<_xI4U}1;e?|ZFA)7! zzf(}58{^Vb{K(srQYbIixHJtwkyA~Kz+8*yDT}V=0sTA*Gyw~-L^F=_bcRQNx6&^Q8p^)xzUVx-gB4?sVlGTJ4`SHN= zmw~WuuhI_@g6J^h55M9v>effs4G(^q84;K`1T;5G<+gv6p5aIjt$D-$Hu(?^&{}GA zT*B|}PSxS7yl<^MBfnJYawCICVz9@@u}c|%;|{0JUVor^W}lXQP*Au8xEvaDmFkB8 zH;I^hqu8blRk)aF^*$i!;ALlGH+58SGOI!Kx=6hevtEo zuLBNBqlhTpJn3fdT92kFcn#H+G`X!-u*I)#z4)AMjHxrQ-4VNzjk-qzx9)CeEen1=3iU=F7(-{ zs(*v`sQTtukMw@ti*_F(V6^OI=MqIki`N^e`B8~4;m=n_9W#3$-~PAG*YMH<)@=SQ zH>Hf=399Yvl1dROR6&E4xY+S?ZLN8_1?rJR!rf)V$y^d&7NpT_Ce6 zenzRi<_?ke%`okWs|5Ck3sh_|PyErfTMy)@@*fv}CX>=-vZVMlJ&UqvRsUg%2Ji2>HQ@oKKYZ(h$(= z${aIh*-q*(>$t4++wfJRA%6ppdo~GbQ2LV2-M8{cC<^D=0(rFk+m&Ne2fiL-X*3^H zu3dE#|AMB^BL<>pDDo>ComH%}j>`!55ew!_u~>tP%IL0w8j^#}{+&aIjuUFidh6c= z0dJ%tszJ%rE-5^O%6s?Ai1TkJEjiac{9Yv{yLDNyXN_m6sW0i(XU(Izti$jI> zkKzrIXj1T7Qjk;JnsY)>2R4ncT2Uo^><7p4AY=Lo(mc^)b0{v#RQKyyh0kmA_bL-| z)r~K$lo&zYV=Dg+S{ijWApbaUA6&h<-e#Lf(j}&%CmcgA!+Ygf>@=gv#;0nhuShx9 z$M#r)kTdbo(&_Edfmfpj6Z}zc(+Q8X*a&f+A7c~7FJOU8d8&iuEeWNFU z$ze&X8U$kI3sNb1Dp%>PGAuc?aI7-_DO2YgMmaZy^8KA}Op3zK8rx3rRy}kDdJ(IW zo%i9DqObpkctTM(=^{0Wl7ZCUj=2t6O*cfWNSDr87wq<{1Y9*nAW*;UhK0hgKQxYO zX+vSU|DGCD%g{0PbcDMRP)ZZF&@8cd@Mp2)-!Vg5v?yGWCMue>kKZsES)S|c(RvX- zSlF-kldA`V-(M&7<2vIp>0>69*Fo-(3)f1d{Mrxz($kF%NBI`U$c>{qL(3C!_r9u_ z?X(e=k87gNAMtDQsS0;MXa+Vnx!;Iz>3(;>=CF1st0-UXzaiV>Emp*jc_);{#2)kT zNIqnzPcr%%uQz2nX2k+Hf%_*DoZ9GeOsVb)zn-8*19@~@&-c9S)9H4i^k3qB^9Zq^ z4sf~Y6_r;IF9D?uBL!#LavJ(NbA95th^^HCyNV{`WP%=L4G&(ECmCfc8fpgjR9l>b zU*#P>38Ng0`Nn=ybc!3pFQxT%yb%D_XKSj2B{b8ggD>09DP-%%oMwY(J5vzvwww|@ z41@T>>b#g^0tBseMndO*0FRSC|AR@>o;)30<}gP~sl+n(==5XBhx78So%B^q@={&% z<+j|C7LT9%I7{OrN2992P|nTdl$VNw2yxPz@OdR1Jp|KYHZ!*!*Vu`j%6SNAiH>RJ zO+VH<`$IriE7NZI;b2e^mra6hRp-i&riROBH7X_WOMPD}Yp?3yout z{hB-JO?@VKRJ@&mvPgd;m(l2kDXFs&r}S;0uc)BaL~FlRPWrU&)_ZV^Hy4jQuq?pH zE`_k)KwFhY-Nh?wa<30FIqQ|l2Hqbl@97SWaTC8?Df3<{k;BsXJw}7iW_};W2?#ou zVfBN>>Xo2H%baU|H|6tAxOVWpXGG72etpirHwshcudjDpm=<-eH#m72)*THW2kddz zyxTgbKN%%|y_BJY1jldQB2Tmd5riB>_xn1YXT-ID0a|3v} zZV7r$Hgnl!+eQmef?c46z<8JcbV2fU8S=094BNYm0!0b`46FljbsKU z%}hJ+n;-0M)g8o`rN-1SNLM&6eE=`sxU1~>;oY6HgiS&*dJiJS#iKP9{_+z!t_?W} zb^qhW_?oLOi^kCp)(rG^#908(Izz>Z%g{EX9hIW)$P3`%t0>*FgX5HTxaNR0jOw++ zDV*6QFn5kg$H53WnN`S5xY6iyf} z`4jK4Fp5#mMFl~1l#*|Rk%7CS4DNn=l6cdra%)Qqe3#f+TXgVyj`CbuAH&er>=h_O3rlCIT-Bg&Z* z&qtGMu6xGxV5Q95rh^sgybO1KKvBW^w|u3UPi z4=&S;*_WO=rO=lf1O%(76FI8Rk&T8-ec>tkQr1;gWm+52*nFw7#J?*LPP+GvW?}B_ z^fycwKay86+~8jxlMfo>Su?b_koWv`10XdU_mZWiPJTKZ@0RTT%^Za46y&?mW1z zi+^O0Bwf#0;Z`ht_{}n+S}y=JaNI^vX&6YAIq`Y6 zhR00iV@J>Ms@PKBJF?jz6w^wOrt@6KHmpLVvTgLL;(i6cM^%cmE#@|+Mb{ll7ah*S z)OBt%r!PpJCwE)vZdf`BRPiA8pmC(_0Vo{)opGblW3DPgLs2tgeGn`jBrl+$Ze<-& z5n#O~Vl=@pUggTkP_)Sav;s{>mDWbN%OLSRL#TeIn&tH5ltJR|qJ8`FVzDfp%}}5O zue~ivs(6;S)nL+tnQz6ezHszCNYQ!=$Y2;6$*c!IS)lhto55GdA8!9UeEZc~yOcxR zV%#@{SP;AcW@%|i7>#Hz=`M?IJy*lpUECY@)2n(buzBpt<~}4}$~5Lx*O#(#)0}+aCw$L+W2zKC*nU9?a!cp9n=7wzfD{c} zF~T0rgl4sa3;}kzX>)3WojUhG@O)mq3rR)h1eRXlY+aP~=c5t+o9}P?Bbe0mguSxL zFgrO6L`!OkblcI$)8WEHrp!=_;R8|YJ-G=w*htd7%eh5x7+vaKR@Ivq=?jR82er|P zRt>7VAa4i^99LjVn)wtj>;LA`EvE!H)w80EXVPk>e>wO`seT%S<^N=XLw=ahfz_HHK0k2GxOPQ9Yjh<_$;2%+~es{$(b$2p$Wbt&PwR zvqG#96-tfrZLfJ#9a@fmIw+bp^X6MCC&T9Sc_*akC`GJwEpy;=YwC$NJDtEQf8OfmfmZOzG~xX2I32BZ6aCL$3|*p%dQs)&o>vnSus09Qn~HP@z|a1h z&@fLmuYmsRSS29w9r^%%>k48cIATK*%$WRsjRAnL=J&cHF;Td4r$536)a za&Dm=JrxURZp$RaDaXr+11pKnqnF63p8*m4Ich+4N_k2SE(!1F>)Kj<_K=4$o9 z`Z$r9cA=7n3%NRPFy%}`yiQ|G%-r$?jiNJqJFurTIM(MGY4-I?u=h?j_;5~bwHb8kzk zcKEzI&HMw7z8=laPU^^}Cy;~mzP3lc*$79p@;+h$NFjw;?GtO(S6%-D{LeMm`W2aP zs&q6=kogj`!owxSMS-Y5XE|ro^OZWK z9<6DaZP)aY7$J%CHN6vVyhd4_31|d}6-P}TyQq}F-+i1xPkyo`%-`;f*d~bIot>bO zJ_VJBSyHG|@Yq5wZpY&Q&Y7HxAIZ!>dfE+vn!?SqIz2t>2Cw@XSA)9l_5~T=5;pv? z-0m=@$nEEQS4CVnEi&d)k@cJQXBE&2S&r=2eKjXAZ_SabVT$)PVsZwS6W9O?`LzZ@ zKMyXORH2pFhb5k02wP@(!XdWdCr@5S%+;Bu4?(3%sAVSdWABE=g_3$a1dQASbV{9g zpF`ZF3@M~JiYU~vhqp*~&@{mgwvrEFB)*h8lid1RcW8Xr?C}~R8b1y%d%%=#fRo|j z_=-*=@%a zT}&OK^Q0kKQJ~!x52Tw%khKf`t_IzXEj2A)*J0u zmt1x#6xAs)=7$a1ir&f`1DN7YDG!djF7U%!78y6Lv??^V2Jh5y`8jR=#hGpBB};Af zyPW1vkd=DYG#NU^J>+^{0ylgkm+X0%N*GmDkno6NpKjz*jGSBb=`Jm3xvL=1{3??SVKm&3uy)ye%O zxw(xco<;C#Kg%G1@U5&#+KmAFK#Zxrab-{zK|s1qnVY{7P3bpgQH5~LiNon>^a`Xm z{i6J$7(K`-RP4=+uep_s77~400!x z^9b||qWO1ljoP)9v+)zQeu*|m(t;jXIltXJHr15d7P1?J=l_^yc1xVH-SRH;v1c+2 z=(A+)Fxoeb6ZR=O0hbncuQaMFjn!Q%__<44k z!8W6&g~d88tEF7>(O*5Y*H))4YAL@v=ETky$M~5oVXtuXYK@bH!Y0NV5UjT_=;%=Z z3Sk4BW4ti6^&QcVCTeF+iH#W)Sj<8lFGhYgN)`>{76@Qx?=X+?0 zh9${sP6GI|b&JbgMrtAP4s#;g@6qo?6>SB_d&+FKg3H=4?9gZ}bLHe2rziBz7pc9s zCB%%L{ndE?PUzn=RkF9?VUjsI$>b;cY$H>k{!R9e*?TG${6lVwt=-p>yiHl6jAD26 z7Oyw!%1YSMR8I2oUb!|Uo+XziOXM>|!aCYhRlK|{IKY;R(TRQa0jz8eu)Zl%7Y>lIOJd1enE!%*=u^_@M!mz{zgX^Y|+}E)3nR;+jb&;rQOR&3$q4qzh4n zk?2?m@UCB9_p`JYs7v9`uS|bApRD=qus+VXx0C7bIO9+lF1$1s05yF*^G>nruJadj zw!S7^AM`#h_u~bT_d4>qt{KO4#ek`_tV%mP4@jrmd1Q;U{e(rFLOuofe7F6TY9l({bBvKV6=>kS4 zcw|zEvnjUT1{t-ZOB2Y+@qSN&hpQ7eQkkwwaJPlcN_2~p1!{d<1dm?DPB`0N*UB52 zbl`jRLWDd{xZSweY-^SONcofY%`XhGRSmQU5k;x$2u)U4Z~35S{Nh|Js^nR`&&syP zr-js|1jqQ7Z^K{B9c4ETH&5jx9^uqIk{k6{T`rB5e%~!>hv2WuTP_5<0B$6X)IVEH zn96={fyKSS52H{`PS*{h%B17$EEJrl58DGX%HDRRwo6L8>vFqn-z)f!v;_V0fDi~8%)=8?6v@* z336inLdZM2d@6}-PKeeZW3JVi$5XVWjEd|sU8FH~>kZjIGu52yKHGmTCm!_+V2zO> z0oNKP6Mo(;2!IaZ{;XA*(xO%N5jYuvqsx1a| zZfD=C4G+X?3!1jGFCt<^g`9)NT_2Y$2+`yZfyJ$2%h121T%Guaek!x!XKlOu@=h2E zPjK!|ms=na5>m}XKA4+;GSBu!il>dXVtLH@=1Phlf+jj);|aG;kd)^*H=P} zznzRj!b8|0==F-kwTv}IBmn=dVN#XQ%ubz^PM&o=KQ?B+10RA+w|ZW#H28;n**FYx z4Zw0#aTyikA8HgwsfUl40lBr8J#AJ?4oq+1rBT z8LAnX>E6hIXc+pois8|C<)X`d*ILhZFgbKSjbuP&hu`lg@=;l(IC>VhHd&F> zFjN_w23t95lhv52apB%)@9Wjs=b%m|#VtEa?Jm+zLI>sTO=k8A>Eypu91{bE%bhIv z#f-p&9do-)Q}NO_j$d+27??<>ow0-`%3lcXAc<>LGTNj{R44e1%gsQttzpHhjEz4^ z-266Ad4(2z2Qj1&^x{h|3h>_lZ)DS{GYPhTM|Vnl-d!TUATVWxTDkzO>A`(dO1F-YuE)hhJs@ zasu9}(tT<4E0pX6i@Vyei1SXDiU~)A_fl3lvEoB2X9> z82^1%J7XF^5(JaoX?2URuo5$)CeReX>ndoQ_wj84~kKd;yny3LYn=eeO%2;R@ zDwx!$YkKQDKfNzNFcCRGoY6mjT{W;zZj{n1%8|*j^#TevE+1)(Ing0>g4|d9?BdVO zUqjzfO7bLP&0Bevx)en*ID-ZsP zq+o7Mx@lMWS6`+7Dm#LnHT>Y`Ww+tt_e@2Nm=L9l zy{UGO7ew(E8mUgZC(E;2dzY{-BByNU@@k%-CQx7cOM`SVzcEh4mPV_T?NjKC+_ikJ z)pX}b7sFYX{`Wg9_r?DJnGLMvnqRlM2+hNaU(YN?*Bq_-3X=0GRtDETwKxTot%)5} zz3P;b-Yj(Oivy8gC`z9C1^jG%7oS#~e{m#vMSY7xxqK&r=aCxB4;u1sU?3ZV}s;T^$>H_uZ_ zrBJGk#m*t-M~A=8yjQaSz2F$81KwsCxF!~I-V~&l4VdQ_kDxZlwI^W*5934b9clt7 zWQbq6;e*Iax55IfTpFHDBXH}%erGQ1)YNg`sKfI5bE3djWg(u0{O&Ff`v zCz0a!bjCEQ9EU@7V31VCtyv!3*dI;*>WUPyk{QGTuGNdW=h$LfW*L^%W9KmJ+|()P zMRqq;mTY{0+Zw;(p4znrf+id5$gYb7547U8W{cMwI?q&jLV`@~0}(v-Ol1Kf(257u z1Jq4pnoBr5mW4cFNLRceb+S1)`y9p|C_GB;t2YxbQX=THLue_*6|(0417IU!yUq38 zX=5MLQAjp)E_%~Z)|819B0H0l-RDZ~MMQ@*D)0AoLRVSNZpq@oqbF*%8%`g`b*VF-FADzS(j&RcC{}9k8GHyyE;N2?o|q>NF3V4->xe8BjMY1zagX#kJbOHrs?ki~uKcxs! z8mXnvKJ@0f5PW!7B#Ya3>J+s`wn~$l8Vi=!Xbtp(AzXb-l|{Thp+8cZWHC7w{j^KF z79-rlff$Y)k<0#$%3Ug^{&&HOb|afuwisKP_b7szAmZm}Z_O-lOn(530- z(UL6>vl$kA##4_DH{JV>19m(`ZSl*V@qft*!`r1?u&NdMPqTcbSyB!&%HWTWG-{+^ zwIto&b<+0QY;#v0%Y$wa)?OogOyk0!+ZpncJlycc98MOp;s_PDmSMc&R~ z$wT%1ruu>9S7L8GDsGN~c`pD0lSAYDzn+x_9W4k-V z7X-Vl4)>S2Oo}|&&Tz9D2sW13=;3(=3@2;Rg<9!!!64|4dUODya+O5%l}7mdjn)wb zy(}AkXzRyr-b%erW}7^~$5eAitP0xCmL?>JmkvZ=U)C6Wb~WDYL5E(!HU!TWt2wN- zJ+R;!MRp0LBgoF+N2x5U+?*s{h;TFKe}L0t90nH2ccJE@fBGQ8oGe;&TqSEi7z~c| zSjpDT9^YQViQti6@bIS7GD{$aPJ7RZxVz=`jf#W6>Cq>lS3}RGslA2`LCPwx$8{A@ zVnb&4Yvle$14PXMAKj`eHd21SAHlYGrWC1s64yk&PyWG*a(SdVN>>*8XtyiO7kcn# zH0k>9X~4eWkK2^)Y|#roU-VrJ-2&S}{+-Qa5p@LUre+$8002oRYE=8pM#1}u@HaiW zwimi<>q*_7gJ6rsI!Y}3C1Mv>vkE9Z=X&weQJspHw9uG6cCNF&*|6A6$l3kHuw`)} z3K_6Pl(R!21&>p;iwor!R5_AwlLma-sUaC_tv4Jkdov|$VH*oBqD*eUm*mpC`TJ#$ zEDJ;LOAuXW-qq|4;jq7=AGHeLex;-+&yedZf82JKy%%`Q*CK}EtWma_RVrN9;7`H+ zcLj6%auLb&+JNCD%QIxCK)bme`N4jwWapdab+R2+IWIfmf(*s>FTb=uG{dks>>#!= z%+-aG4hYR1SKb4UvH~7Y8MlFM*bwFGUQq~QtEu)(Fz8I&yK1{2aXW$FW2n(meiEW< zn#W{I9IRIuPo~mv>s7~u5UMo&B|{n?E{defM&Hjo15*t{(?i+e(!v5M%qiuh zmi)!4PR0eXpb4Fp-1Y3D>NDZ-U!r42)~*KHouxgr?J3tK6IYE&3r4%zB?{gSyO4D3 zPlYU0<|7!3&1H*C%o6m?5CeS>fFNa*p0Isc1@PVukpOGp{6RlGfj&&Geix*Uu4(3MKnOwzhTv#J(`=WGzM7U>~4)|gq>oN>jdP5N9`(~Ks z2tA(e(u00X9I!F^`1o!21X;?Wm>DFZC%-rQ)BPyg%?`kv`r)zADf@7-hJ87ou|gtl zdo>5zC0P^lOhA-Dmbv|TvZpPYjmYPOQ7(0^== zjkN*}Mzt?T+;%`5!c;ph=fmrk3sUIqNioo588`o$0DVO65U#P0i)x@x9R&HsNhEUlX+yVepZV~x z(S4ogP&_gkl5<|ux{H2J*jrORDa~Qj0isy_4lD7hoDA!(eCiH&*Q7rgggyU@oYUZcp z@Cpf{SZ~zO%_#kfPqF|`+4p4r-nj48<*G0GBgk(}-SlCxJp_FN5E|aHwEXBa61c#0 z6l%kJm%u1q3l7G$9c&T!t9@(6J}g1!q_sbrDTX@C#<^QA9z9PD{z#+2Cnid`o|gf(Bk>BhDaJw z>9$k{TQ2vtl4#Yajk_X1P>6H5Yky~0@!F+8$j#0hE?~3@&}`lkM3RV7lRSWsUb9lu~ObGDDyAL+DGD;r8oT~?ok*F|9 zByRIqYa`uZP1R0ChlP2@9n=mBmrQ&2Uu|m*?2f{lpMd{l-c@CWqFF+)4^pZfe(>cG zk!6fe1bn!rRm5p^T{>>aMVn{iiIR}2b#;}ltY|O&6=%~TmX}o!vmN}rMC+As%bCkS z4O+arG;x(8b2+%&xOw%ZoeS zfqG-q_>DUFwx!EKr+#*A5vGQ}d zy`sz4^sNQ&7F1b{V(ZHnOa?&y@W!YeQgeH`I5hGW%XLMrkmRFCAlA^jb<_?3RxXJ?`8jf+$MjawBMxWNWLn;o@|J7pgxhrld2MdZTe%x_m=vrHxhIPhJmj-OI)3OY)E4%?~Q0G@lOS1?*2Pf=V zM_Rv+vVp_Oorsrb35a#4ncI+8 zPIB0()LiCt=dUwoe#BcpgC%SjRf`#&^&*Vj@$l??LQh@hQ>R()NYBG9zW0FQyp8S$ zjtx-JUiQ+|Khc6CRR({4`C?8sT+@L4apxZ=Jq3kzNmqpk_5bzx8Hq)pUXb zy=NxmMoHMn<#H!T+J+0GqC6=pnuvb&(!{TC2Sc6KSvCvuU=W)zS0Nh4aoFfqV1hom z1o9%fKl@1!``!#2MNxh2jWO?Pa z1}l)ILWZh6*m^<^p@AE~>E_{|_eF%xc7ppOV*b7`ci(%eV>IzU00%f^R3C|`Thu_c z!6!Ri^Mzac0H&d#SzVE9{%Uw&SBR7Nw7_}H_6(Mn?S{jLRBL1_Ns6EYCGE6*FOaO_ z9~REObD1Dbk_j{Q{IRylz@EZUwdt}yoU?fEm>_X8~V*lH4B`vZ!eJSU=9*Q^lUng z!A5wY{PlaH|Ap!Td`5p>HB#Im!9Aptf3dSV(3xOk8vaAdar($p2_)PstjF-))Ihj6 z=9bVDkUm$};vco#Owp&sXWjml=jJ2eWg5Y6EF#&-a^rx5(?@7N0Ngh7UFoN4Q5CD? z@b=q_UR+lJJi_gJ!z-2rBH~>GKkL)EOUTj%NLHh%D*0yo;<2_`_)W59n?{{#p0Ljv z!I`9=#Vp#^W%!7Gi=d{h*JO(Y4VnlG4Mqw2BoatRC59(IiZ2UR7!A|#jJT;?j^?#a2y$uF z`Bp2A%ly(JaK=;!Ce-UKYbkL^ z_;HrQi%YD2svSC&cB5gs?yq;p+!>7%Hm4VhJP^{Nn_548G=<_GICZvX(*s&e^r*DE z_fY_4ZSEhZ#Vroa-1YvIq*#0q9Y1%Rl0<-iFkpJ*v}j)n2hku78-{|$2^VX&YvM_z z{i742lwqXe;x3Y;)aU8rZQ$7E^^uZH|w!>;C)7S5jITSrHpt*tl zyb@QcfYS<@N@dSyNq?n{;(_K#Mg^N5TF4Wz2g%d6mzlY1hgb#I<%00U(pQ%aI}0OE z9|<+qrv^k_36^K1Dr}u;tA3mJ1@^BtUJ3Wm%r0cFNVKmU$vAkK>1hA4K94C)^*p1_ zOXBR82b%UXEC6H?XVZpL1D{_bDB7B692cUL)N*mi?%7DMIug^y6M9#vxkj|g7eEJ( z;^9t08w!5sT61A}t1u2Ei_ALrPSWP6wehE2Dfz{Udk$XG1Y7qZBQjFS>XBsM>#Td- z&i5Gtwwu5&cvW05bNQul+{mO~o9yG=v7QUdo)!E(I?d$g3=tW!#@(7U@>6fZ9L==* zQ7RkpUq}7fIWBCyV#M5X@JhjZ0U-#B+l<-@U$gj5xbfAk zJ!$&a5G_SHU(b>)S9~Ao-IK4>8tm-)-Tk=6$&RmrT8z8Hx)Cgxh=qN2I>Xda`jkwz zs~x`~Q)`rdqrhjuKxsHRg6&av7Cahfg?JbTGRo!{aT;K0MMRqn3?A?uK z7Z;zJ^WEN7O@RAY$@?vjTyrn7k|tqFI^x@)-b@!kdOmtTy74}D zp@tEf@gjibU|ALx$OoJd;lUAiUWDkJkdH zJEJ@kz$goE+%g7H`Xvyd(78uwt3c%YxT(XYRmDo;NQ|rC0ybe%Og+9lRlZ@8@*<+GhzJ>0; z{z+titIut7@z(wRLH-gbP+D}{hk8zH1JYu*^5|r4N_LaQ1j!lPwXNcb*8@Xt-m53J z|MNuGwf&L{(UQpB)c%!{ZZ5hCd}f*9kz-6TjmWwu^5;{MZJZbL{PI)z zm;iZ{`H+#{F^N$tW7D}*`fmJ#3(>eB0l7ad0Kc9);^bn9_T`XOF=;53%|H21ZF6WB%(an#Ae&ujK{ zc?CKQ9}uCOj&^AYV3)svc=E%`T~IDU|G>9OD=LyCdmjNQ3w&km7Mo6VF}}faFP9CZ zX{Y_Wn6Y5+0jpl%8A+MnA#Gb(tWW3T4TMR6A<&f?ivs?W(tf$Dx>&t3%p>8rA;a61 zI*06#a^;Sx;|{!7&v{E5#2u<$g}5RYB9?0~p`nGlFdq zB`0&9()zIpLqWZnD_joDBQZf*K?CCdg2wJ%ZA-P`Y<#ko^U?Ln|K$f#01`r|2liwvgib$1WJQS#zf&eJ z=Oiaj;Xw#t3qI6Fx2l~=lUJ@ftoyhf&vbSib zKYvb$R+U;xnbmw8byeo%cG~m+bpkAfGn^5wD!V-hhC=*itbN{lq9ljL-xMKm-RM@q z#1MSZ7MgZ_?P{CbZ5prCi2!JD=kmLe(!a^c;+qL!P6ClDtNbc_@dG(-&=D%^K3g5S zIv%6~H_N5R`PW(YHy@|nT1C}dJaSq_+e3~~TOGCaI{NGP2qbi{rKEsA>280H#sbLP z?|}^~m_dDx0m{sUlp4hF;ff|XMC6{^3;R0RGiTFVn;+IQ1J9WCM$C#ApV1vN-T}T& zb=mfEA%Zq;e|GB^!y-<_&lk%C5!N{8%MLc0?Hc3&p%k23VvQCDjsxtE+dce+Vk5q8 zc~Bmz0v36i{FMsbWqmB-$HqtZF)KDEG$OS$AFf@N@5Qe?IfM_JC(l04wM&8^ z^;f4G1Bi;!VyIfkGwBITnaRPB+0D*lvnlu6uoK3s<`yjwcWU5#timx+8(MBc=ZPg+ z07Pb_Pj~+r;mM0znL!uwPr{9uQc;h8BH@nxH0X+_f5TL~vb#GO;{haox+xlf;AL*Z(by3g8{c zi=I&RRpg_g-HrNZMDAJ*f2*sJGP~)QBZ>nRcPiI4Mw%C^_I5=m2{lZ|#z_CAoqck{ z#dQ$>neEOVrOUWe^e3CoU21kUR#13KvxR+#r+X&!cDvu^CDcCkKGM+!(nl!MIx6U} zq2_26y7X_#Ec-b)Ysf>6>m0yokcu;|Gi17vl%CY13Yby&ngNex0 zp*XAT`HBmnQd)hOQM)pP8xS2bGUmZce`r@6N%B|3U z?Ub6i^x@Qshp8}l2siGMc#HL(_+o=G*sM1C!6e2_skd;A!_NxNuIoq04LNmgzC7<( z@nTTV3Tg-XI_3WG5{M~(gq3u{k2M_Hz5#(xHv3%>nKOB?VLwvwFx;hJR;{5Nj*ib7 zc>Wk9!lj&uLasc}^kN?J zZ=blQ7o~W&EAdVAWBflRbrX-|7=IZ>eKQgcJ`( z+4hD+lj56u&Y$h;hhie=ZZT03d7zAlhYm*JFJt{2m=r1OkU~RSSgXJzj~hx3;rJ58 zEkOJe_M^l{)}JsF3)*Q|B=mXyBT!VAS~{xSEUOY^gU5{^R?8cifCej)RBT#(q((ie z`#kpvwBJo>0xL)WZh+hj>V|bD%RgJk?ZDl;%D8jelt&s{pz0C3>v6dm&m@6pN za+a5imj@?ysQg%wB?(C%3o!ABk!pL`ucA zRUuPZk=CX~qa}P9CUr3Sjq%)BUl-@V=reQx5R=Q^So?SQJ!7|P$@mYwe@HiXfP3Q zBiU2^n&(SBanmc~x`o!(OB47_9gMpXzgIOz1*Xv%ra@lmm<=2FXdqA7&;Zi2ikBRq zo?)O~1hcrOI?BGCs>fp260PMCz2|s3buQ$nIQH7SHQ=6|AQZn*5VnVOS+}6Q0wv6AyBTF^o$gS4f^!>mT zk5(1rZP$Dra3Xn@d@-&k(pUd~fXes&xD9ZfvvfEho;j;5;_EU|7CYi{PG*{|swM{G zs&|l{s2D(fC}cr!DB^6s52`WNYvV*o^!X*k8M6r0N5i3L&E=$*%AI3#CD|1_(f$|i zWLRjWxpu6kdc7~v14C0bN_9%Rn3ehYPGBKz9Vhr<;1!J=N54XHXm-aHl=;_>?3pg+ z3kC$eui&R4UfCtsvT)OvYTEr<$F@zggCR~yN%pAY?FxNldrETxBbZ*EBQ8R`n;p4F zj)q;%QviN#7~f^zefApQQJxua71ik-B@uH?Vch>1({s&q!Gr|Gh@2HtWDCwldYtQ{ z`d~X`DbFoqA|8%A7>Jk+9|P&)p{eCNI{9C(KR>@LfjN7@Hsrk8V@nEIij?eij7~NF zDl`0JL*CoFYWSi*V&GA(3P4sAy-}Rf6NtGbSaRUv>-^ZN0LXBzK2P(hTEk*F*j0sf zOOt4;^WsF^iWr!a-LHm`p0hq`dhcr4;_#vf?jUZX^OOqN^-mTKz317kIMwv$70$6? z_=rM`byl4_?1Q7_$sh*ViX}8*P#WE5omG}8oH|q5v2Xv$`x-1PoJ`?eh4DP&Nw4&{ zFC!4j+9BcU(R6Rl&nfZ^~F@{j69Gvrw<6 zvQL?5mgP4-R4nf+WetU#S~C-zB(4)&54|BE8Y*Hds5P+=@eKBiui0aMtoyK0M|Jtl zN+O0Qz_INO*IW2_Ao$AB>N%*3Z+RYH3Xs4^wJBI>^|DCmK8<{)UKieK?oYy##>FMM0CUa$MIbB z0JrbTYNv~D0nUdA7j&_vFJBU4h4l5>dkirajr4&d+RmulNd;i}gTpLX;D1K)EYPj@ z?H+prYO*m7{{yJoUHx%b&uo)XYuBGKD;_!Hx{MJ7jo=z)^DL74k zp1KC(&S(pYO6!aJv5L$`NW^Nh|B-wy3_|xrI}W2x)LKGZ-Ewb(XqdzuEv~V?_8I7E zd6m#Txn2OGUMVUAaK1YjLnb7<-{4r+O{GN}#xOLRBNY-4Js z|8|<7rikMm&xGpPvtT5xom+LtP`RaM37aHXQk{ib{P4s@f-r`#9Jt@)3$M%rEQ19X z^DNEHLslqg7MkxO|4seb-RtOwE?D!6c{e_iZn}t?Nhh|n2RryPNm59t&i!7x+Uug> zlgo%y`-gEeTD|wG-Jk}8Hyr&9(rs2jSBGE?EMD3R79?oCNIp#uX#9RwX1=|rI?Ykj zAOw4z#~if5<|D6K^B?Ad-(gC8YF*)APO@KtHe}vD# z%m9h?VMOT|G9d}Dt6kB5lptcPwK3~rT3DDIYuQV|0b#g$k7Gf-Yv!q?b57vz!)TT$ zk4^V%2Sxx|l^zY3i!nrx07{?PO=z)VEz$zxbE#Q&Ri@t#lnA!)+U{#_vsjPB zz0ogHKqQ~<{+1ikt%nQ^$-*VPNksF)y31L*HgJmRzk<=PJ=Ajq2?aykeRrKaG@W3= zLPl~u;#F)Q*L<_vZ@qoFvE?m4kXN9j-P;R^!LY9k#d`z?D~d0Qq{Mq)LKdvJiNchA z76**yj}}y3?};r@0H2Xq@6x@pmMQ#X75GQC&*P3G!%ijGA^X26WwFST9_)XfOTW*`dM{Fa4OOXvJXn8Nh#r z+|xxmgNdlxo~HV%@_XN60m87{#pL7EgUHpM0!vO=VB&wDo6SKnlNgaVwk}S|5_6{K z3L{}UYli^6q@m{U5--z&a9gn1Hc84d>GwtWRbSnpo=C!8?DsH40sZiV{r$$}goxN& zUV~77dNlnD;In(5))jMrB*iiA%SEp}JhO=-x2H1c*$kR16jy3hL`#z;{c?&U@E<1Z zkg^zDgPb3hKg2#T3UQtdD!LhrX&}NID{L+m;k+C+o;j3NQ5`Gup8B4=r&LDGeX;x3 zaovX@;eY6XLIr&EgNmn@=*p%EEpF>m7REdR-t5wPPdNKRx`Q9fBr$w+W9)Ff#TKUx zxKIh-8i+Uc9+fKa`Tqt`b6piWNEFQ$Hg$LFIc03*u8V^WRokbU>2tpAH|^)o0>-JE zCUf`D%R_9T36r^i`^Ub;x{)Az4edxb0G)kc{$bkKMS5|+#_$du2lz=g@Y{T)*|2=d zK17M6t;8h}c(7e(oYAGnObTbCZrs|_hh6$iY*W(KwO~7-U6?>zZ4OLq`kpHgZRnvw z{%tJPRXC&zF)x+1k{^~91{1ME|Cnz6D?H-ln1HfK&8G~KM5{>*&R{|+cZ2Hc4yx&ax}`xeqoZg+hccJWsG6y!ZND)L zVDDv3`s$&SBZ)v9nicI*UMT@%^qJSkUY{*;4|m3Ey=sJs>9-}!+H5-wzD$NDfn7HIV%ZV=tG5)z|4`jZnhGbd>eF* zb1Tet5|>T+O6ZL_d0_zOS`ci>zdK~4MH+3`w6!R1Dnu_g#VooJ1xk6ie>anYLQDLP zj@=2k8gk2iRR8eE)y+63*2%98gLpu^=RAM7Uvm&l7N|F#yLXIj;qZfI8HWtJWBddG z*-qR-6HTSnK2L0T?LyTkMz9tR2hjM&?_nHJ+`a&~Uav+QIiZq?<9P(lRm-Ysfp+za zle$DmNgGHxde*Jn82)WFXQ5>G+$UBEZAoZJHpPR{C%ZtNz#DnfbAB_Ot29Q7T~ zQNlI0PSo|wmMLT6=&#^CnDdLpx+lGEWOL08ys~e339#U8I8^StiYHO1qEH{I+s!Za zH(%@Pw1EL!(1wD&UGs?7yftZwGAV%hPi^^E&xH93srh3esy!puCh9zCGNn)X$@7EM zQ)-cXl0z0jC;HUPtk+|XdSpv}rjY!GlFjkwY)$9+f553d(Yi3MvkaRJV#j4M-i;TY+JKnE?7I}{LZu5}ng#>W6@i4DU__b42pS>}9&f8j-xm(K0*L8xur zws&ygR~`lbDLJbg41l@CvE>Av+7EW1Ra9HEEGAch#$&P>=`~UBVZRpn2e54ydy7Db zM)^>jSBNP>ls_st=nTpAl5xhtfHUk_8*Z@Y_d$}apNW{5 z+z`@i7lOhzYEwND&>`>-+8uTe_e5Wtld*phI5cY>=3M>Jl3_x0zQ~L_wqjJ)~}7yq4|bxHRtgexRPjXV2?zd zhe(*0AW;dfAEmdH)eC0wEw3WqT|E+R!k{=}u>=>)E44s0tLAE(>GJdV8~-M+Kah{t z*M$x$cZPgg8_gLlX^naTVDsg+*n3tP-*bk4y?x)c)*D=VYD|I_VD9V+&ELk?mzKeI<0y;rvO-fvgfcGl5_z-joF$9Xxo>h*w8fwEq%1KO z`_+Lx$!Emn+XM;@Z$_!F8(Iq=bdGKkLnr{`N}&J+beD?sLq&^y?zYpWrM=xB#$o`O z(wYiwc-v9?(s%n)X0HR*o`jUYT)N34f&N-%bAKaTb<) zJ?Jh%BD@@dopcDVVc2+z-@xq~HK{(vuy_2j{wxH9-PPD~Eqa~uzI*7;&+S0l$83xU6dKcIlwbvHZ*fFKQkQ7V;?j;ITZ%nR9U! zv8z;eM$UM*y3az0w?`dVk;<9s;afEU*yAcc>H=LCPMbxw{ttQ%-wr5u?JlapEB*)pGD?X{}yD>il zp=e8s;ovJx^-{7;_nquUls-JoA#{nYu@&6AliftVbAhl8CikIHUx| zNLGi2V!Hrf2vZV?u=n*;`cg^aDCn-M-D>ZdbW3r0n6Pk16wW?MhON`Wu0{42fp~0^ zc85D$Ma|?A@5k&b5ubMdcxrCe#T#PD`j;*Z^o|Ln=SSTP8{BD+zcm%j!i&`Z2T+^W zf87{SUu%6-6EEH}3fP*6u6Q@BIM1tC4hZu|(?F-ZGI}BgU*$>77wCBu>L0*|%Z86C zE%MkRswEx0nsM2_*RqFD@Xs#9BRh)3L38bE%-$eOb(bc;FbHQKDimryDzfuRF+H2;x~j ziRdU_d+AY&5Jo5;a5yb6-dJ8aHq-mYB+ z#4v_v(HfoYU4LRcz7Go*N@;j06z3-Y^_v(+;c+9-)bAje_Q}Nib`%zwA0aJvG;>qQEEMvM2Vlb&Wq| z@U3xZ2#Te(TU{5A0oFQ_2W$p0rdNeoL(aW35hu|XgB(d)X%W5Ow$}sQw#A27kPG1GgV3X{6n-0=Cu`s(OW0*_L_cp8kISjU zVq+Mqo&H&2Jmu%zT*H7Pssw-n>UXt9dHKlAMw*n1iEGb!l9Sj5lRhhXYvdF_AdTAx zYh;BK1W;^U2-sR85`>IlPC8YvQ0tc0o^%gd3DM_#!cUVGQIV^v>1EAv(|X~<{C(u? zeBCw4l1wx-ts-AxSr29|z$QHk7wB=gK}nel2Q+8PPXP_IxKcc-cziAb)DbA%2bl`* zgwF-NrHk;{UJ^SK8zsvJMo(A6LPJZd4O9X}FWBPhS9^5(G?-KMd=DqM1rk_VbyZ&V zC?5&B3jY|eV{GhjWounC!mY0yW#sY6;XPUE0&gx&@@@|AurJQFkSU}XVm_f|XlMyL zD~8vfkv}_KdCCfqcu++e6T6F4F3YM5C!GyPM9>4LHQR&kkW~6`A3sRiYAb;Fyl-_k zr`jc?Q%H%zl^PcT7_mGg&T#N;P{;gr=Z9eG;t?I@yKdWrI2tmV`Y?#|v z6$VxLmhPKg&i;221vAqA_3>0_YSxL`g)<_-tsl|BVK) zfJeU38ND>7S5|6EO8o5e)o(%piV=bfcU*cOfkm;PQrA7JL$sXMHUv{#_xr5X2Y!^X z&w4(~U|{TYlDRhcae)@l^n`BLS&B!Eg!Wk7wWHr}=40+B!*@*Hcon$zvU(t>RJT#) z#5B@Pf$Hr(T&Ma(ZP*ix?^iZ)nHiBN%OBhuHe-9~-*CO}#}+xb@B7a=i6RjnieQ_W zW&MWa)QQn2e<^T|3?;#auQx`tDU7D(cJNqaJYwZ>N=e)`ZD_Ff+(o6KjaJF=D*Uj*;2yVBzs@xI>Q!bd-j zWjp6BvjwW7_u4b*9yg06mR{0;XTk3lRsB5Pu2hVNeN%#pagoHCbJ`7dB)f@E{Nqhl zZ{7+Dfu(>Lwz~#_S`gV|1G;UF+plA7?_l1!qJ>K6U>$SiAqRBPo2|lg=)l84+_qoC z>WLKG$AGk=tD38^BrI~j`QMh8(gLL)RQyJ^y^TFzT?o3)0hY2?s%T^&FbDFmS?}!T z@OS+7yLw1y_;);f16;pVm))Vcq0$abY$z-kfSFyUv-hSO2gqCwGGunWGx#r@-U9OL zxM@9j(90nLO_yOkv#6)jbP|}<#Z>nG0OrS0>T4kKLEg_F?lJ2aRAc@BQFIn=O}B3w zAKgf^NjTW(kcJUT4n}vQbc1X(iqhp6Y%~JW&7?!nXVe(oNQ<-}sG#^5kH2^CKd|H2 zaopeQzV7Qf&(BGfJGH95{m2%nD%+=u0}@yuY>r{_mTsCEX>9jnFn#~7td^FWsaA}hp!-V6q&s2p%m^}+9GOMw-fVAycmswT-d`gID#8;M-(H#qi zr}EpVX3UqoqoczFPVZj!$i8q&H_!@bh}Y2}dk!EJR$L`E>sJQmQ+S=z(e@Kj^5GG) zS#?=ql>{*->KRI6%aEHdLS%SS-hcv)rrR|DbbuV=N9#lNQ?4O47_P(BSlwXyEvwjL;$X71Kn zZ-dlZ>vyk_o2i3?v{k>S@8kI#$#FLw??m<`7gxy1qz& zF!hx-^}$*>J^7WV?w;j#54^A^7tH_d&~}PC9I%&o?l85;|4z)P_mx)v0orDK$oj`p>!8M5v#`P_?pGuRgnS(W(re#tiFkaN;>$$OFhvBCqrQ-0R?RcN$r44yX zgFUdM0Fp0IIo)v~_=Y6q{UNnkUg^ffce&f_XqyjALL>b!{^a3CAMF`Z80waA-6!P% zF6$wc*2DoYnv>AN^YxSEXsXs=vYp(>9+8j`X80yxlrI>Jy7kBz>EQm28{iB6HhI9c zcPe_7IhJa2E3Qy!s}MM0gQ;%+KG<1gsq`!!*sj&$*aA`w_q_ZB1)-13S#?@#`}r{D zTov>u+-b6xg$P=gDc`x}Nl+7(4ANk|r5RpIP64;e?hWImDGDVAWSV+=yJNSRlkPLl zBg#9yd`N$3KN<_TBviPt&#BugV0{CM!Y0(_w9jpi2CU!9Z8iWkugy_o91e{W{40>- zM{jG{v>o=Vs+Z}c+SLoR`aNNd-e$ER?UTHUZ?8fLIJE88;qSHViV{xukaM=*eywa2 zpm?utUgqP}q>MfbQ+4!l;@A&Upj5Oc7f#Rl&h6N1FlFOMD?#RDI!KfRI z-3@c)1tuP(;YBLhr6?K1#3}-(TtF{?6TpL|0vxZp{L)Nl_DgAV9pz8~L5Ce7bjCbT z3+cv8h6veHJTQPwSQ;3H?K3PgY|?2w_N@=Ma!@QVAGfN^AnKh^!MIUti(c9m576rS zRVI7h8m#ZWi2;t=%wI{ii~)j^wADzhcF|H>Rs7qdOLxB2U6BZC6M+K}9fqN;|48BJ z_z%~-Oq7C-@__(4_CtbH{|PakfF{T%Z`e5*(DwCsc*tVANJ&3NjbyRm!$Ey zrm{b^>_cC)Z}56!y(U`AyZVEUxln-n?OoZknWb{G;5n2S%NBrYxP?gwqa%!C%%#xW zZ=s+2giWH=7HD1q902&SKTG>xD9Lf?fW#{DN2ogJsITs;7vwgA|C4*hs7-+ltL66- zn012oE#ub7C8@6_Aew&+RIx@3#epH)U;KOiJ6v-7cMNJT=Sebftc{ z>plRY`)8?iuR2sGkBn@gXi$gFFp0jSQ`2fz3|s&!7|+z^O}M^;JU=s7EO^#9&y=T* z1=3MC)Hy7DcB)pbb55$G+i;IDFC-_kQJG?J~9E7pC|P)Sf`f6{3!qro{W z`7?3!kR#@^8_NqiO9s${%`DQsC^V1Tot}z1YCQ6Gc!CBRFeE2(4)r+z(dLbG96DX=klzLd#b zf*hvy602-}6inK1Vjz3PtiMIyUfHB})AjO59=3)DLVu5Sen_Vim~ZnNR+L~>FuM5Y zP~q-F0$f_^oSIojGx7K0>w}t%kXP8@P@UGQM>lDLE@e$8T9SVo!EU;qPff`*o-yv< zogC@-$M0TRo0OFFx?(ovyAN>_I1S5?8>N&!5*f7JL+_;W$w7z}CXnjZCm{vTb5Fh0 zScU7lO>w5-tS1u_Xlb1L|J%s)0!}5M)Jq4U5`|c9Ie9{(NTgQ@%x2{4!Y|9sZDZ$> zcPe>%q`?fW%(FnpO57$_xD+5xaHA#Z!Us(QWTc0FQ+ijy8BCG$)BYJ(2!K;M{sRJv z7)c6hANbPGd2qPNqkEvvhvd(X-`cEg)V2p20L@N9BZucrmwJXbJ31*X!eEWZ(NX*N zC?+B3l{z-~viK>Tz!`pXF|Wp6PqadaVOQ?($w+fK90*V&X`XCL+~Dx;8)YGJx8g=d^Ds4TXZnzQ93DY+`R%wNlfv5g zO3T7IV9?PlLf}~9(F+O-z-=tPG82)jvK_^L+um#uu%!?tPZly^l;vNy2m$}ZH`j?K zikjobp?srgb~I}xSut$Qlb!Xt%>6*wtv-qKipY2Z5DaWR_4cuPLoqCvV13Mz4NV!^ zpvN!u=r+7ALR04yZ&71HZYMi(D7~5WC`@eow%RQ6wkbdK0nYpzi)@4phCpl}@LL zB`GU8C~Wk*$9()GW$Yr{8nwrK-0ZpqXy~|Y59@a=qf|`Z?7akuYJbJ% zvYN$8XG_~W{gkWDELgA=^U%?ycy05;4-@p^7eLGL>y8)MJ#Ydu7jqfK8h)cHQsXCe z^!JA6-eGd#iDlnso%)`8@}G$1#!C(an?wU`qvi~$Bb7Pz@#foCkF^>|eqZCZ~3c&v;HwNs&im2C67enB~;t192U9t$Q|$rH|cXnaWPukX^*< z0a?WTXu1T0Xw|be4@to~`pJ*3;pi{t-N^~JOxMsU_84Wi7BJyV%kbY-(z_t$CxmaH ziKD;VKtLmFO5y@WyB_ydf&r(VU;(Q`7MXS1$o@MAjHaSqK{*dt6iZA9t+oeZS4_bi zAYh~#VOD5uRU5`?@c>4P@#gVK(`}6Y&_&#n|I^%^Anl%nuOWLu?2cBVIovagD-ag> z{{tj89^fv$#$Qdv6}ozzTW)&A@jQ>sd@amC?JYm9e|bqr#MLrW;R07wXcdzea?I~D zVnYhVHbB&Unm4T?&Qb#m2y<7_qr)y*LcOsLW*{zHjCZnl#kZ9KxPi5cDhwfr|3;R6|{!tgVdjVR3*`z+y zoQTlyuw~4+pZ$~9+XYcj9h|6BF z{cXI%XV&U6TNk4sTX}&GV`1u(LK@}=OXZpzO%yL>VPeZ4O&tYkNnwZbA|#^ZcYLK7 zDo-+#J0@g^-FE5eM+e2EuEO0h>^j=FG@IR~LPRt4UJtbBE_5BM!yHClF=Q`6eV7J{ z<$vZFj~1HPe-r)T{dJR6Gvm%L2i^PDh?o9Qp=liMYc7mHLTY(-{zq+fS=C+BKU)_Vm%KPS?d@qHF?yg3cPm*fFKjB1}TQno4& zI@Rm}zxm6KdxIM!A)rZnp%xbRi0m*qzN~*deIn_oaVW#4HsPI_jSc51b(|_Un0>AE zuXS^pCE4T_zSRDyhc?;9B!J?l(OkULGp7!frRgYaG83Ht$vRO={bKX99BZvnQ1P*n zej%?6r~aSB70{_RcM!&uh$(J7viRnB5_2sjtG$E1+op3o*9;%qWVK5NWF4I>ZWo1) z7YvR_doaL3A5aEXrD0=+()vCLcL%8%2dFIgjmA~w`LV;4RoPBQUcwY}$d%HJ>HdLM z^wz0xU$BAfN8J;D(Eh6cUi3_5t|jKU{%S=$zq-&X5j!U(;bo#_P(x9u3ayWs=!qaY zzpyD-EqT9b%7`ZdzlqV}c|8b##5tXhjy%!YU_tjusyF^y+e!&kKi++2{*KT8#ZaTK z(8~}?b_U~`$$r8U?75}9FMkN6QmH2VFq8mk|7><9#G|(10-|Q#Ls~P7XOKB~C>Xc3 zl`vcMiC|5euto36B!rDSk*={(eB>*$4fMGZVgMm*{z|rOW)q0)($|jSc{L z?X|NE_K^t+!yze#%qQie=y&{kKg-d61LjNLZ@-I}IG$Sq5CifaDg|eC3`)uIrK>f_ zg#^A%1dGSfT&{t61NMa}DEuEvgBhMhs- z`ZRL|u6)?=0|u_5-l!jP<6NO+r1jLK@#Lq;&|kS`YK~}mfhjn5I;>$_xXD__Z0N9} z8w(8^d-7&}1?+(?eJD+;WV1bB9t)GKVTL=FI$)SG zXa=y3jX#@e%u{^~Sf4KgD!NMpwhrEopkYBTjAu;bA;4qu+W9nG<3l{8?zHSauwS0SK%*8?Z8W zqsemc_BuMGz)8Wr$9nj#W5U*-;Ng;gn5SG5`;zFjW}W@*Lh_Ax1NhnP=wq{P41mgN z4;f>nhba5U{Us3GUXE7)Xp+1cW)s{$<9+9~{H%WGYh$mi4s}}ogqIbx^-7Mku01MPxq@(IChg_Fr-G)ilK@{G#LC22 z$NmA{>yf5^;5nPVmq?Vi3{X0TUz! zE_FK=1ff}7$QiOQUiz|JySc3&L2oo3!1Z%szZ@SBh>z|N$UNym{lN3+lc1{9(Za9W zTrc1RVmSJplpKdE9s63PF!J<` zNKjn({n9wqhfIJ=TyBGGHc2<2sd(nS;tRRJ7 zg+0UyYc4S*6LCeKvRhog+jMlmV-zdv|Xlho#lV z41cbrAoXs|uyH1ZS&)s&AZW}g>#Ftp#|vhpM<;U-rTVWX%f$sGs$ea3My-=s~po^vAAL9|A+Ep}FUG-H9B z!1cC3$f!bDL1k48Lhji;e5~DgJ&A_x~hUR5NBM5Zo4UCahkkbLYSy`}VYINjLnk{rA`x$uZ8} z7c{g#Uf!{+gYG}VcpTtJzICw)9$gD zN^f@*8KX?m{44bj65a#?$)tt$w;m2)e!3~xB`3*m+szjh7hjS)T}lm3(WMQbW>Ubc zaKdvA6x~DTHknS@@gP(fg(WNUdvyS1!817?pAX^a+ZFab17=P|-2463PAiD=$_k3+ zagayB2lq}X@0##&bqF8)!=yB!Z+eoO>&A*LNfn!ierfd)@wR#aJ$4NW^$Am`Oh^W` zL=lzAi_Wxpa=q>984|KruvA%fb-&f-7t1Evm(gDjUT*~yl<%`E1 z++t8vKQ%^jjo{j=pv`YzI9QbnT~Ye4mg5PjsIR~E|BZ-(2m<&9PX_#Ao;Ngg1ve84 zPcTx(yOTANqd3->ODT>`w>%0A{V?TgH_?a6QGQPqbOf~V#Zo2HdQ@YwYwE3vG+_0E zu#cq|UL}z)nX_|BAeO_1(8Lvc)r?O(zgBBDO1vJ*(;o;o|Eu!XXtBx_v>!6eK>V<6 zYb?=QG1C&R-3fP;3!4D|cupmk$5~c8CkAjP@eDJ|~HO=1C(zdsbmAbE^kNc@xHJ z9u^=Iyt}qKJ5%C`(9TW+FIGpp&?&fZNuwJR(qurxo0!n-!VuKV)HI7!@@HzsGw>lxOJnm%e;z># zefaa1S`332E)8R47Igmv!zmR1oQ+*|={Kf_1Jq8%%k6JgZi&~^E>IA-0iyfDv=9k` z&Bt>q-&%%lGv3_PnMQzE(M8rPX}N7z9F#_}nDm!jUl%@L)ec-m;v}uXnIp}ATFs9? zn(f+Kc+{u2kbgN;A?g6T@+{jPw>k6n>1O!*Snx6>v;!m@20`PrgUhU-x>46)AxjKe z!5`5rr68aFZU#R|b0Gm+O`-E?ok6PE}r+Wm|W&U2J z(syS5i=|{A!CfN_{F?OiW7C73?95`L=#wPatVpxVe|5Et2u;~bNzc#63Q%no6mJLD zd+|*Nsd3Fz<0;+yWNJbd%q?_A-zP2&lI$QGQMuGtB5uEK1o)=bcmeV2l?SoSRHkZC z)QKner1T4`r2K5H8XyEGFX5fGhSGFjUGwKoG_`I>?~C(%z<7D;=nHOLdOW3dHR5KcZ zvjX^X9yz=I{oZPacfHFpURF3h97{_jSl#yTgGCu-?rhXj#=5*#3-{;Bkg2nJgHvSE z{oAKcq`$g8L_kB5Is`_q+CCb87gpI^WNP)R2v45%*cyYrrJ_zU;r!_0ypmd;#0HoP z-|hOg>JSS($0*RneG@#e&^~^p z%cEp!?6{1qqx_RUn0Pwl=#mb4pnDAXdxKIkQtf=oat(@~Z~$O657vi*a_G38GUj53pc~(5xy(CFaQ9}G8j8<+tZh0&OTB})&6z+xT69aQY5iikyg)O zXXladwwS6%BfwXPGaW#cmF`^;AcKwo#EX}n)*>Z#;c}I%Y5hOj28c%VQfTa;#=wC< zlSoN)D$9ES)vP|mb82P>Fv7Z$L;cME{a}FCFx66B?`V~$(t9D4Lx16| zi5X-?K<@?r6Xnyxd9@H)wMk5Zw(4tpT!i51}bdWE5A zYPieN&dFxJT|h^5-3S?+ZT%;v4BZ6cLPUDSeY*eSz8{5a5=@rWT;l7UA|zj;nzW&F zvuuIq>_X?FU!Zl4>Va7jX6l!>~{qMbqOAo;q(s{7M^w3SXhvW0jT_65NvZ@amSc%<}RB2kq@0AymYhi#^|=Dsjd7x3dFYduQej}E;gdF%stmIH0a*( z!iH;Rje}g_u-a!Eh7~GFY=N4`h1<$WBx<+$Bt$w$s~u}2b4P8$?z3zBAu8Z@^}{W^ zm&`TAx`Scu!@|_Z2{76sn`I71C+={*B%5{nrR>$o=Upnnw^?*&?h2X_NZ|h_zT-o0 zj9T%v-Ja8O^ToUgNXvgi|7#6>Y4#$OPW+aYn`mEsw(rheVMf`DFLwmop5dhm->6!v zVopUayC`8+ZHmk#V0Q@f?N{=KR?3SZr-a^RLcJcBMf?frH!CXv#vC}Zr5F83I$WJR z1Bs>VKj7PmR5&w{>2ldJhJ@)QSY`8C-p$pVkkzKJ2n!SkTQTWDXM{5nCrRV>Z~XFO zks3*A+i^5i-QAb|)oqWn(@oKtpkf#{gBXrIw}m8ZxGKpqk{XiFU7Vjh;?tiN)=cr} zAdTCs>><#Vy*}CN%wOmmbgP}^m!f2JD91fLRQ)yQWj-=y`(s<=O=>D-FrHe8KaHZ&27z#?jnNz6IiwdY?dR*t3N$eDc5Eul<5;X5iJo^O*b1k&i5bR0$ z+U3$bcDgL|Gjp%>u*^A1w0gkU>8o1$BV5LMo|c*aFp^2ikgwNBk}2G-!4kbV$-xB% zY<221Bw330hG|@cQZyRR9hEY#8E74&$E^nnokp5J=_Kh0 zd^OyR`V`=%@1ZTV&@tICA1n9C*gITsQDTmk;*u^d>r)Gl&om7Yx!l7L&7aq9md;cF zCL;>_Q;)8`oC}vv@g^@-JG|zMD_RcMZ=sYApVawJvN*|eKGepGs=qrKx?0>|AwrGT zPXI)PAuaLvOvtE&$bdok!c*lwg-+AjL5|@^jCx!~fNlCnt4U3}oc6H+I!;(B{ZJP8pCkf6Lm^Fg`0KMubH`PfWYEl5& zgjxIZT5+)`a{Jt`vx6_K2(J0OFqv5P_G8m*Zy&7!9c;F^ZEv}47j+35x6=h2(AK6M zefBmw2*cvd_Y@c!1-P?HEG0k!@`+Bpo6DZT3&Liat-2~lWK^_r(?8P%Lt~O8I)P0b zs3WvP+fIP)ZXYzO4%NC$WEl>ihsr}aB&H-8Pmyn?dFCdS@<=AU^I}-zH05sZO{TuN zwp@$2MiCs`$lFsvXJ;qXx&LavRP7x=oUix`qP}23Z2MLC>mv>$uSYPl^Ghxe)aFw{ zRTyIggtD4F2t$GR0A)OM$z~?qF&j?PxjXcSx9vnic=k7)LA^IcuYC8mb7V5N550uj zHaCF63hCFjd)~b)X5p}BvPu+rVcXh-CK1Wradrd?Iu(44&=~A2n^~7;bTbx6V-PV? zei!4stKkx~kbqf_3u{`Jrce z;qk?h5-|L6w`O^tA&3AY#!K{_b?%76sNT2)8~P0sV1g+Q@!9I2AFaA<3UV7aq606M z!U*0kjZYuBRx@ASg5jTD>bmu!4^a#soo4z=Xzwc6nTD;dtXnYZ=GH($9u&8h-`jTG zeyKm%C}Qfn{|G{Fhn$p*G8oeJ z(KZ?Hy3$uAX!N0wWo!Qfr0;upL?B=RhW|YJ`{seb<6j`Lg^0^N0PNt7aRM75VdGvy zoAUr)tbq1p^OOm5uAx{jT&n4XOxjRCFk&7^;8rrEm&e@~mei~YsX`gjiPbr2cQn%Z z3$988-tw4WZFP;Hz9!6uN5jQlygkx_(@IqhqJ%taK>XfFN2A_O;b|yES<^Oh#a?;0 z(A&hbQ#xH?vd7DY@b|mAOL47bidQsPeq3V)!^@BNJW?UjqNO^_$!z(a0eUXd>jsi@FR z-LJw3R{jEqM*zW+d6nPMvNr^v^=X}f?Y1{3#aLS9xi)52>K+k zILkER)CZ^7>=!VsyC1j7qU@=0dNT7=+ex>P#&NGl?~VOpttlay<_b1%?7g&G4mScN zkd>?HK!6SZ#7$wE9<dx5C5a| zWpA@FMJ?NWi7&5gHE0ry`>35T4H2Wk(88)MN@Zk*yPQ4?zcACbYzn4C#r_X)Zfha# zfQGj!5};xCZyOLaRRT-eOT#EA8V321++bclP|fSyZ8pfDb$B&1 z$ss#KS^ISs@rbKPZkqXHc`}Nhtgd3qGTOBUIr9K&1m!PGdxreM^2EX7?BOa=LH5p~V<8RP$m87HMAxYq~g;+JtLzxo#D682Gm*Y2Y#(6Dkk*Z|aqh)3-BfZKV})?P)iGptBBqGehJU=drv}vt~TtI`NB2$yL|T(fROr z$yF>b_=Q)tvulcHc(Mt*(Vk!SMtE5vQ2Dlb8`B6>k(-aePYz>ma*}#zwb5po!~4=R zDzOR06;vAL%q>-(rOg^B?^3?3$}yUMF#bQeyawMqiE$T=>P0KOIdju*2{g z)%wwe9RO_q2k_-(B>HZnj38*nDxgypxJz1E8)0GRvGujhs1O*kR~TR(D?$g&DmDI{ zy78iUy{)D-w~IZoU$FA4MbAKPDq}dt81p^KirVq6U@VmaXWIS0%@cW%<&zLv6{vfJ z;OYiLgLepnYK`jt=1B~S0`iJ`uzowG-*UE$nJupLNT$vw7t(jRg}Y_t-0tD0-I=8I zD{Dg|(Eh{)g_RkZl2f3VZ^YlVNpcAZbzA}KDm0=f=HotD##aLqiMrU_*je{~lhy+4 z*Dkf{+)Pn^mBL*ud}sQ-@?vS?!z11uBsGH4bZXZAltN3o{T+oquy5r+7I14C=gVxq z>ap@Nu;tJobeSKK1UEX${97=#Y<@a!Ja0X}@jehZIRdlOBFw@TH?-+fTG+ zulrJF)$@098)MZ(+cMCf#*KNTlK3=q&JkKqMqjjuV$=jTOoZW8tQRe5UUuL&iO`*) z;^ChP+og`~F8)weRiR=p07~Jp%>yZyb2X3kXz2l}&eh3lrC03_i`%agAUG8o1`!14 zzH!>)`)aWj?w-g2WO)x~tyKpO1ga)cIK|z0)TS_HMU~0aLe)Rjg410Jott?wcVqCv zW#HR-1%huTHSHD%%D9lmCq=UN^}?xBB}m;<=wsFFlddXRO$W%2j}Y+N>8l z^H!cXs4U{B11MReF6p?E*Lij#rCj|qR2-vxhM>SsELSmH zP?MxXa|e|ZY$F@u>Fs2~A1d_ke+EHnc$Wg1NNuqS8k{`OF;tGKY;`*7`oj(fO|j}M za1VQ7ho0n~6Ap*;A-D>qBh8l&T|puWa15Gh~!r zJoXxu*5(h>`RqDgCc6K3D`Gqoi)z5hyZovnMMsH&DFGkd*5d+8gCV{iz0iJX$2_t% zZKd;n07}1l1+C^bYygF?HIqEmI%D_0Ka){pPo|Zd77KHmVIe@(Jh{$eJE0c+WGwP~ z+l%(Oi7KjGL!^_9MCrcLCLi)t3e>sWaqlH{(qe%N2fyND){!33L5w&(3a+M6*o9Uk ze6S)Qc)>R<_Fq9m{m^0vRt zuMqC!fIAyiSSG3dL#c0&cT2Bt-AajZ~?EVt@siF8Ae6zEp*>{C^>cDQ6LKgsyYC+O3iMVYQ_0lRj6+L{D zp^&6Xkr_6Ih0vnILrrjaQW5VtFMCL;CjX^kpxMK zH*hVqD>8p5>N>~_ptA^^$UZ5Q2>Bt;-m6zJlD4}@z4QD^k7ad~)Rhvc5iRA-AD4w+ zpcgAREb9(Q?&}knsIy6fvZ#-PBRvtGDD`Xo5eLgG^kG5=*b^T3BI-@nHyP=_wi+LD zq=bO(Fq^+ht}c3ta5pEpH^yOs;7{0Q(uf$E0vZp0yd3&@Zz~9h;?|K#n^QDb7z);} z-jW_}(B)Tm<I_l#Q9)MuAM1 zLkjC+c5ZV1Vp`M>XS}ZvKS(h}x~&iU$=~5^ZSSS}XR87a#xGLdNyeg4YxbRe*xli$ zGCzioaE$z%%bNCC4|Eq_r*9|%yup6TBC}KG|8Dv`&#wH1G~09-Ko(TgKP_(pJ67Ss zpq}p^S7TO%);zN9nW`(0!OUSs4KG}P+=5DA2F0WMKf3JikwQ|kxg?k6X#Tnp@;>UO zGcrq_tL20@GylmZG@k2SJYNOhcQ*yx|T2bXC#E$q_vUz&^v1Mb{MpgRf8JnS5IA^!DN zl$5V3G?4Q)7Ts7Jh#EDpO`sB*FZyh}^nCElC3^i`;kslI_6JAhMYB5E*M*9eC@gdw z3aM<7iVc;<-MS|q@sq@V6yL1TSEyv&KMW|OW9r-Owc~OdD=#N&x!1KdIf-;}>z%Lt zebB@W;M@`!QAZj~F^HI=cQ-#8J_uo8^JIj9`5$LprgSg7(_0GpT$4+`dKHyCIZWI2 zM<@8*13hBcmwu=pU=5DdsQcqd_UfP$e>oVm(&Oe}2i8;c%o>SyHjeU!2++H}(<0-n z=^lc;ZtSd|>$LQ((CknW!rM1nl~${nfgB^?l2j?w&Y;0tZJ8GH%;ilRpT~7 zN(XgK#qa-QBpF9%F}66fx}+cMRH)|LgV~lWv}eKHbG>PD_wDHx)whJ)7vFl<=C|ca zyyo_!SPykvcTc;p-VT)iIn9N=w|CENB0(zS_v>sr+j%-jrmfzOO1zP!mp!u*Jv$;|-Qb7X3Mr9c8V)ZFgi4R9 z^ZTzdt}Xxc-%I}^mDYS??dKgIEX@e9ZvX0LJfG5Wj z)1*^L5z*y7b`ec030p6!Zj zWi>uLu8Aq#S*=X?L)%8J& z1&rx%vMG=+7K~S?J}l zhymKUxG@d|J9M!KVFJQjS3lud>-zcpFU@8EWHE7I`9 zbJ1d%sgM{TkM)~pw-KM?$qAA#9`O3h%8FI6i+Z8~S1LMUONGCp0%}vZePLu_K--3a zKt(!`vDsVW2D4k-YCZD{y#%k!twhBt@0dA@ugNd%Z^+u2??vIl7k)IDiKD{_n&F!n z8D?7)MEO}Alpw+}$_CB3vi2H$W;;+;eTZtvHB{DJb;C2a=O^nWb9G)uw;S0Kz>#UEDOWV@w_qC~B?MogSeMtL| z01q#1W2_9uA;@tUuLX?1z^L*Np( zA0O+4@baTkk~6{Mte{ zTFfcr2u<3nvLDGE5K}I1B3AFUY;##Fb~r%GB3p9DrNf1=HIe*mI>Yy>ac7Hn@uQ={ zu~WYl2VcEfs@n|84*`w*)zp_crWHbm&pT9=|erp4|x8C|Q)-7$Y@Ka5e3w)}x9W?{|AAt8x9o)yI&MPRcd9keH zl0~9C1QJlVQ%>n#`Q*5|Gl{Q`lbB1y9n6R$2;U~Yw+-+~x$JOFnVvb~oi>vDb0dHJ zS}pD-1$$y5k`R6Ub0<6Klb^Rcp4IC?2bmg~hl@06?;KduAEeo{*Uzh9@r z%=url9y?7fp!Xe={lccMZmlc{1_@^i=7}Z&$fE9@`P-cz3+sWl#{vA8es|1Al~!C< zPn9wRnd+y1zJe&SIpQh*YN@sOj?9WC%p;|7B<-seezwOi1HB!tq(pkgxB5ek*stim zHF+wIEtOCy7%_Mnh7xRCi#qk%`xc#JaR_Oto0!h#4otXh;g{~In^P=)K|A*lZHshr z9nw7x7-NEz@W>&;=ea|IcUffc)FF_rRG&PW$)=<{2ds*l?%$io-JF`@gSOD%9fr%m zAfwCjYQdQB+ibem{{y_+FU<^i%n6}|-rb3uY*G1ASJ1B?a)m8~jr#Ts9uInw59v`C z;p-%z*aZ6WNllV&j*eBDt-*cYXZa_wKf}#ZoS?rq7`vK)IfZu)?Ld)(jubt;Ch1$m ztDj0R@s`Wm-$eUYd~-9}Jp5-u$iT#j_Fs=z*j>g$BKX`nI<_|sDZ@)KQ~UdFt_H%~ zUEq!<=)Cia{{f6E_U+Q_oI<&Bf%Tj+WFR(rAa*_6MK@@H$0#Eu_VBR7mEX>7iP+pu z6r@C;CUUhT9!8E7V8ZeeZZ(nWd2!-)wYf77b1m6Ri=@uoqkS}1(UY8TR4TH}3p;MW zk?)mYr1{l8ROZsRn?qBI!d}QyE;v5^WABG|d@&?neyZ^5*_ zNYI_%3c7dF%{3UOyT{uIKCUBs$clnTZbHFURtXBSt7?0usYL4Ac6GhkM_Z#BG+EtwiLV?L%R+C z1H5|h#4E@(y{h6}$9T-zSt7w#q^siDZKGZ20EZhxl#0tFkqh(wZ*#v{(aj5^b4GQ% z6M^bc(~st;cPbVUvUb~2PcoJzhN=DpKJwKKdBZNGux-9D5hBp-CygKKp!&1lkoF^_ z<9bz(jf;WSb2~Wp=P{f$_KodlW02hSxd|obHeY4apmnOMBd0O zGU%8d2nKlLf$Ww17wh&cu}ipaf%zf=?yvR$JpCjo)h{ zw|r+je57)#H1f0N#)?i{=2FcUdIBjPO2Of@*Vij*0RWH< zN{tG#pas-(yKg6@5sO+jJES@*g9HLQRt^vFmBAvNHP90&m~3Ti5>5syK5-I{J`~yn zgn_pJAEjT4Nbgau6%nOga_RtVVw*{;1^OVG=0%zI`DIQBWEd_bj+yf8jd#(OAoDB z^DUyY`C@6rYxj5IzbcGbKw@@Xu98R@#&8!ECEHm;2b&?2U>H*pf-2iWd)X0NO&eSN zq9!mC&@`2)X1}~iE<}#ZM%3M&eSVdYETL!ws*LIV#vMNj=)64lO>>5EvLrk*@!q23%m{rcFM1O6NHMu?H0cPCSx8-UvK*HE&`H42bs(?B+FJ zn>`MFsCZ}yNao1w1$BG74ekrD$KfjI0>JXcN*SGyb4mF5Mc8@XBy=hrnG+Ou8G@=V03Qj4C0jOL~vE(8-GkrZV~$jKg`N~-Qu zObZvAxnYA|d$ZKW#5Tvwa8OIM@JnR!2TIVANX@juSx~{WqQ{>2^r)6sva}5GmLD^y z#tAq*sD#IR63QaOU;Xov*4GZ!={DC;st6q9iiRldbjz6{jzJUpEu zk_R#lr3N`2s=1I)(a{V0fBlk=!r< zkhsWGNM~VjF?91r(a1YNKT4rJ#1?jIA(Z*Du3?DuAMGDnhTrWo#xr=X8FBK0Gx*kS z?PyDjz0C=(3aY$KlB6iX&N@`Sd@~>tjK~)XtHT4%Gg&cOEEj6<+=+Ut@thy!S{E|H zj3bS_>6wARTmkj}0PEI?t2Dw%XiewIaeEWSJi{2nv+^){Q;{T;kjk(iQhesXV*bOw zwN$p9r1Qy;Dc^$_IKck^3UszI+$tfyaAF&<2w-#Csxr2$dMyjK7WUTg$kE)*D3P~G zCj5fKy)0Ki!*GQeIA%!#;IRJyJ*tVfnVL}>fdonp;&Kjq`+X^~%F)KC)T!v2r9(eqh^x^cbrdHx^J_oGD;c<9-is z=T#pzep<$xB`!9Qz^PN8!-^Yj%@Es8#jzt@&iO7B#k=JjN_y4Eqqu8GqqhOj0%UNT zi+@_lj7=2N$#!JO$s}>t9AtXc%OYjFjz6;~K%|F5fTQ}=#_eiLmr|^*B7$EgGakYN zrotSaJ!&X*1f7+cB!ygo)8;+8der)@+)T1X<~yh)F6^DT_x}Lv)hkGfW;ZNEWlN5P z=bDLL_UJfkseInNE@WAoJQR$t%%o@2)}ebfdB6rR(gP|n7+`%f)9|ZO+gqg4T*p1a zi7-nlfcbSpG=!%Q5x9r+aA7oy#|I6BJ7`5Do!oh#5R`KD7*y7_WE8 zo?&$$sNsP6(L9rcNMVxI4aR_==@*;q7zNCd730;~DVvPOZTXj!qj#twhk^ruaF(YnVOD9e=tIo#g= z09q4!+-rAY`$QJRZHOsi8H$Vod;YaU%G|)?r*49^V1?t-;x{=m5>E$;ghjqUHuOEa ziuCE72 zeGMqoS80jp?p14KSxt#qf?#c6bCo@T^u;8rbsJ8xd0`^L`JGPRuk;nK68ZMXq(hZl zrW*v2+OuBz1(eMvnq!a^WF1uc`%uzKSmfZn%l*Yp#bH^QO(6O(*$-&xl)7G&P?Hf>gc=A*Mw1L3-)|9JSO5m@1 zot3PiPScUdJF&)VWL&qFu9+krLG4_{mBfwmvjPblv(0Oyaw!9F2*;**73n5bjUVlE zY}g>-f%$tH;rvN$6#A=9w-BqkA;> z>7VefTuO3-K_skm5d@ADc^A#Y4D91`cKn4$X=CL=)2qtBu_G^?$KUBhfh2bZVV%u_8b{%j(AFWx9Cy_16E-lMB zkyvg3k4)5QawCOsif$Z|1c&AdF_L{Ll4|k3n{hF4?-Yv6TY!6hLY!kL?ma9;FrLk# ziESe{Ww4EsTOIlBQyFB9u{@Sq4{Xsb+&!wSR<4n=5M+{)$1xQ$@|k^*NHk7#af zB@xSPk+A`gsmNdHT88c~G(_;oS+ab{S0H^wG@NyNl-Vq%Ihe^Fe9?fuVfkv5_wJu( zS9SA{=PG_tQ(6_AvWUd)4=SBk@c#e`VYY>(ea2Kd83c1pK4z`4DoDCxxp+6naEcs! z%6U`tsy7iy-{)dq=vJTof}x#F9n{{-2F$C)Ea> zSs04u2@o`y0Gt4LALCjm@kFH+*zOqG2PZWYCS{o9kt+$ zh5=FjQ}|$1sVQiOZpC<@R*P!8D@XHz)9}qX{>d6N+oV)Ie5?=;^r)q?mfj(2E3?R0 z7HE$_^{EY~C}59rk%P6zBlDpxE)HGFsS{7I$i$#xcOk&X`$ndg-X?aBAG&qJ6mA7M z*KFx+G-Slf_i!+ufT!E(w&(B58pKHmF@O)fRr(OyDy51u4YL*r%zWbq`qxdM+%)e# z<&7U^30?~Td9Gdb`>3usf^%-YUw_Kn@GGJ4@)W$gcH6`p1v$Vbs*EKWU6%&i)V*+m z;lOMIk&5)_(^v6430C~wLCZvkyONmGX05M%@qozsNW&8!44B$6D z`K;^K3n!LFM_Sr)uNLM5vS_AzQReNzQ~&}1{`N@v(ov9Upxb!hwOFm~?d2C2Gvp(E)z1JQZcTbj=%z&& zJ5_KybmF)#3&7HNip~wjPbq^JkmQvaKTg%(T!LgIbUpp+uL&P z2`8t$UXn8nyRhH|Cj?eS&h)moEMh)m7Q~%6tftzCJ+vy@MGxD{#N?I=NFzPJ!l$*3 zCA^)HSb)H(X5;+vRN|4=X+Z%uo&xen#{)l)>rS#4Hmxt7qBx`_(Mjk4>@--V~$rDI5@z@NX=v$m}LzRjzU>W7D7lM4u+L(By~-; z#KJJmDHy>1wA0jXI#{~tQC}R$#C*Ll4N%mCkK_`bTB~K-7%%vD6(UP^^GvZs z0N=wrGPdthgbZLczc?t5_(e@U?@DJuXinBD4Kpr+2E=K&X zuccA5x_8ssDYn6K?d2AH7kqY6+cj=wj!2{o6iA?Q!=1yXDO=sssIFI%OPQ`Fb(MDQ z2%x7CulVN_t8Hz7Ae@VX$xQ7S^~bFja9c!GDgvs#`atZ0PXb^RV~@&^5T_N zISB}5U|4g`N4V)y&tMkS43R)2Rw5)AEV$?ENhw+OK%|PZT;E;ad5LTq5=s5isE_fyz#Olszy+Pr7#IU<5leyE-cz(F^nM0lNLDye+UDuSu#$`1hB;Jp#fk>AW<0g z2GY{S4BW*O5{rK^BTyZfWkBgtG{G7qxP`=wGJ+x%NdEvDxgG7*q+!w`L1NCB!kp9E z_*^ncHKUUv;TS1WI`Nt!qNCg!cDpD`9ITfLu`qI?NRedE$`3=+lSvSj*vLsqf|6zX zrvv%%#a)1GChj!{E>*sKupRpPb*5fKVB00zyX2Ks2uLBj`eUJ_qkUF_PVUTtKd`V^ z7_(fTxaa1^K9wmgXn_oiDh3UJK_Z!^wgHAqav4_)@hB`uxA@Z~2#ay{X#BFQ3aN5f zo_RUoQk~R!kU2*C7anBT*~FO15k@wE53j9Ny+b65k-TzlKsRFmjy-)Sw^KYRF<_FG z{qpBI82vNrNFsq3e3D9$hQNuGjiCGc(Kh*6JgvKlaAvo1(lb4{hZzGrgOA6Gj^j>y z$SrLYgpGFuXZyr92tR@Tbv!zh(#Fxo?6JP!hj<4g90Gl5UsW^RC=%QZ$CYMKRC{qv zM(XTuO$b;`3dIkSp@ABrKUAaC(3Bs)EX8h2?f)(HA+!L94cx zD0C!OP8L>{MUl3UIL1AwQM?nE>ARW_ZS_D1ZC&9xY1fc9*E zI($HzH`s z-Xq|9lbU>P=KA6!hEQU{`I3Foymb2gY4cBf&Pt>!2tryo(LgXfN(HR`R0W0pyQ`PE7_XY<&Y5kZwr7jdVVz> zw5uejBXu`sV7tl4AN_OfOg2|An~dGGs>)1ieq~H&?^i+LojEQUZ)AkB+n`}CI4{#5 zhgzyobtFM|QW6Q4B0(ykann3h7N|_%5t7;CH9f(3E{hK2jGm-amPic15y;6Tbq2j; zis(OU2=0kI1S2N6=MKe- zM1TMsgN%M3g*sb%a}?pwp<)cBUptQj@W-WcB@~!DA`*7Oe5^TAHyGCp3D4t?rAcF_ zUr1$>?C%t67%OfBpK(=Uwt^@;!FeP3L;@8z9DW_mUJrC3o#AW}4#if>1@*;LX~*p& zHijZiEx5vfu^*H%=~=S1#AnO}aLiZBJP>&ARy_A{BZiZF3ZkyM)fohdKMadsWD#wT$x%q|=-{h%iF(Mt}O$$?VC!Luqp2 zNY3U~0PZIoWAv>PP^bluV~b#q)MP^_6G4i9?6@gfJawkqrdZgLmz5w!3tYSf9DLCGLv zBB^-7Cxc9mc<>vQ&p=KpYeMjCf@cXhg7p2M4UQIQn$2JFWU7IWEPATQM_5 zaTGzoDr8*n4m$ocixgH-tbb(2|5Ku@Q^&oo;cB)*9sGM)PSVqyHSl(Fz z@sxZJKOP9HnrO68((IcBisgp@)`Zh2wT{_T?;|aR-Glyn)*RQjP)=fv7GMZCLIW?k z2DElVcIatOV{G3va`|#A1uY*3ZC6VXZ!}FM>1gC8J-f0`p{UYEl4LspRwQ|Yff z^iIO!M0FCXkVE--B>wes)a6ptwPq9jcC&huF>~M$@I z2}LBg`Sq&}AeB+(xgr%Dd3**Qp7k>%&K6lN(8j^G#~cb653NHkp9D4_&g{;_GDyTJ zU!_+pVzgshZV3+1@(CH^IRkf3rgKDIe~_%|h=&8p>$G}|bg2?eJ#3|BkP^faGr+(D zw|a(Fxs4PgkgQLG7} zj5O>cLyx={IV15urD6Gq@T6AzhsV4D;#I zqF(e`4qXXlzKU4fTh5M4Vani)lgQ$j7UyOSCDcVnKM=e2NJ%Em83Pa&Ls?k%q1H z#9wGhHw$>AMkHXjLDSm0sb>!=w*h`->w})vh2fd}&$qgW*z8cUuU6;1Xv={hkQ{@_ z&pg+49Uy3zam5dvxG^LQ@yI5VU}>_s2aKpW{&dK4RgN%5NGG7Dg($@E+lM*i)|-@9 z)ntDlMxZYC3czw}FT^I)n@dSq(k+DLPBWZwS~sj@zJeiw*$uPTrDN+e!6nKuc%>0k!~WB>Rh@=gwjNk86#b!u>dl+ zwt1>T;_AZgMOk4A>yfzh{{VO%X(YUodrvkvqk!&H7tMk()k&##rSdfLM zkF7M9L34QxwYP%f&A67}sC?!a@Nh>%l4SERnTC7d-dj;cF=KMiqJ@HZe|gqMA8t&=8TmG__o%NgiU7h zSu+V0u*5jY1dc{})gyd9%X2l{%PL0@WE=8H&d@6=&i)vd7Lpm7-b}{NA0{^GyVoAo zDl)q45?zj|Z)SUFRi|Btg;j7j5PJIl6$G+NJYimGfQTY|g6<$?+acYKL0@2fITY^NX{`mS)SlYndo`Zj?RNs(&R?#2kZJHq zb8uCd%3yUdjr@{5IHpf&6zMI*Er8f!vacYXgFUK@GTlRS6l-k76mMXp5`_A8Z^Dy> z!Mp5UxLItkqbV~+#B&%^mHfZ0K&)=sN1A0N0425+X+J8_R=2%qV=9?t$R&yu_B~Hp z$GfzdMp8$J83^3b;HVwH+ZpdwC|%e&o6(&JOLVp5A;AUEDJPDZ;SeLAjvstd`b)ea?#l^_+{k1bFY8E{&| zGZ|9iU_yYH8Oc7R^Pi=4+E;+}4WfCke3o(vR~y?Nhp?@^KT6Rw+1~OCc;uEdm!BL4 z{{Uok6)o<*Bf|NcSmcjd?}i>R_=e=(g(!RlrKYF$iSFc;l}faKbP#<_ZA%rUrJQy) z5h6sr6$SvVYA+J4)}XdgBg-@LRXYz;>0JQ3hB#b8ps?YXoSNPdgnkrDO&F8Jkz5Rp z4?|Cg1-On_*C@bb;;QP59a=_xyLrexDViptv*s}2!u2PbSFEOjSOA-6g$ZTV^b-p#dgvlMH+%d@+0!Y-KnEeM8&f7~AFe{vbKtmW% zaxysduPYfRD#XP*fwpOEUmB3Pxq|{iW7^CM><|Ggr16gDiip|!&2BdI8E2^5B1+N48;#p$ z3UI*ka8G}8?M$|tO0&32xKecU0S@TKMG8;3tIJgv={2n>BXdYZQ* z#%FhuNX|f1O}HcvLTNiE%7r9}f3u0akU;W+pe9&uE$H7-QY*AKOwt5-CsP|c5(Y=0 z>`g>(%c{?97GR9+ok7X<{3x@(v$?pMcZxqukNUj_?83P`_Us};AiF1_=!6UUUw4QF0c^ryLm>BWb zfNNTc@MviB>(a%N2v!3;3c#5gEIG~r`g2*=nwy(&E|I1nk+cjI{6_|()2^+XPKw|b zkjhIFwBxCdz`JPt8U zr*7eEB1>qJ+@r#QAdnJdB!(mTb?r$N>#fDvRfqDpZ3mv*hR}2o}fq*ksExydLkjR4_9Zy{+N6p-a}wYeXG_oz~gQ0=aS*4LKsp%!O+ zig3erNI%HbkiE6+tS)SPxq-oxJ5-PUy-gh^_sCk}2=IP*)O_RctM-maO(Z>u~Q=V&!E3YM`Pc*RrFgg1F0G~=~-#SMcHN?=VepsgE0CXgdMPF$y?xYsh(z8!D z5frr&htEU1u%@7%l3JPayPXe%tQzZ5lJ(B$ZLq|$<9vj1`BzChJd9*+q>?$}ujzVt z(llr+Sci37%HyLqdZ#&4A?OqymERhCk+H0FGN$t#+i4>MuVGxqwkAbl?gD^8$*#HX zBSvhn3`Tvcit%=(3dE5TGb3?~WQtSLH!VF*CiXpJSJdMcWD8h<&U5n-+*Pd?!{XX= zGswo`V9o{y%6nr6jMe*AYiJr}i{^kG<#@>K2fwWrvfbPO@}h^7e2fMUUZ16K&ZN3* zCpT@$?N&J1Ose=*JLGUTWd2<$pljH*2S&qNjzcPek@(WwY8PlC9$67e*p@dtNcF)y zRmd%t4c>c2AlbKkqaL{L{&QKo+fzPkmMyN}khHVFWVmoacT!Z0bl~7p$>z?18stXd zM#u8T(g$;zuMw8j;!tGrpDaX#IUMxJ;*#R?WO7IbRImjICy!5Rl_z_2DK4j5;hPt; zvAUJE=0+Iu+aQ5gqzxp>6lE9#gT+JO;pX^kutK?&hzGYy)s1+;UU?w*uR;;D5-l>c zaR~N?I2br1ttkr5R4zgOAUUE|b>MKh>PJ6Xmf6%PJ2F88G2j}swXia!y`qg|IUIm^ z&o#|ziySxi(oB+=orv53JqWJm7}c-5z;XyY_O1(6x{fI>!!q0{n3Y8bavj~PEJC~z z^CDNclEG;d2+_HFiEzNMz)(GN)~Wfhn}faNi;RbiXC(b=CjQ<=f?I{2c$DDzK#)l% zjzQ^Gtss_X1!RXC22w+SNbj2Alx?`vS{7%x-o9KakUQh|pdXv-S@)&_4YM*e+u$oQ zVB@I()`iO0*~1b|H<&{ah5NY2@cY#jj>h)krCVP(fHt9%Y2BU)=Yd+sl}5~l-R?Hp zC~oolvLrwT;<+SqSCD;%JVNan5&?Bhr@u}rBOT-ox+raIN)|koF5P-(IqgjslU_&z zOEj1O#kk`XeV~O^VnZaSXg;-FqO{6MD!dXh;5GR&lT+RuuJ`*e8`h_V0s#< zJ1eFn?bNvXTu{3C203I^1gPL+KK*@hRkbZDf3u{v61BLCIgGXjJx8``$J*A(1eY#h zk(Y;FRO5~@QCjNo7}@2WS~gZG!2tT;1DbaJVlKq7)E3&#NN0DNc^3f}EQ~$LtE+Py zrYR$^KveQcWn6Z~4hPb*V!g61F79k(XqfB{%H4f(ImK$pb!~KH`#hUuRa3PW3 z&S}<3IgsU9w&BOrW~)II@!SEaMC)(&xsaQ^by@bhh#Xk#)znJlp~uIQDyX6K)% zJt;<#cSQOU&#BEaNfM$lYTT`)L5@sl^+7^lhKPX?DAY-3;z}9uLT12;emQ16V&IM&$UD?_owS}1& zk|R`d%aiCm>E3nRt!oshs*)>aATR+Pi5|5}6)m?PI!|Lq!#cILn`|M6dAX8I%W*CP zNx>vzvev42mL-{rz9CjSm~;Mlu1@0G($?BBZ30AEU}TqZUPgR9Us;vx4tIJTKC!Munj_4N$W&%_Zjtrl>T8tJyhVBT_%19OKeDubV#eJL z4mR=m)v4uz{{YLkp2}&XWK)4C0Dd^f80%Nywz0R3?d+|iGD12@hEfk;MlqVTRnE_- zMYYp&GgH)N*7Y`;d`eg4IFAadFn(O{MMG}$vq>~;uOhGrjetn!&~&Rd+Ix%Ul0XqR z6Nr9bbJyjhkXl>I9GZF&9^x6DaDV#WjZL|mV54?hxPLZlSs+0s)e*F0aHpEeOUP%O zL30%GK-))~9s?=Q^sDK7$da^)BC?bqUD3Vd$iN6W3n29$_yb=V7QV+$sV1$b5htSomOVNS(L6eGVbrt{#8p@yR_4-H0V{) z?kyreJI9teJp%O4)}U2o2`aRTw+=>63LB0w!5OMb(!Sw#^&Zx;t4TC>5apGn+RcEd zGspC!Ox8`+xXic;0Y%-CLFzHwiqMX0J6nk?WVI3!fmMkN6pU^MJ-)S9Q?OXp3y|R% z_XyEY1Gn(>^`WI}qHgbEXR$Yz!s^xGaOzf7^DqG&mpwj}RqSpOR!L@8f=J6q>iKBM z`c;iK)dM*WVnDcLJc7rJ`i_-kn>$OHR!d;{K;V*i>6&$|h4aN6>2>8oiWL-RAoJHek&k+P zrNqT#H)^|)@JM`;quaO8)bhCOQ%iDh48-#4>2g#ZrK4;}Nf}&>gI(>a<*=ub&2WAg zm(16iGruu3fp;zospU`Q>0RtVtE+*5kQf7+?!w>Mro|OwRx}b6C3D6Js~52OmNG^# zOGe5w(hy=Kd)g!3M3M+A-Kh2%T2Jap^DKg+tel#7`jP_ihElJeO0>_=l){?&Ko$K(-h7QwYR1pA+IYF#^Mo`8~inD@4t zaagVvRX;G=AzUcvaz|QZH#$Ycj&9NgDEU_1hAr(@8apjQ7K#)g?l>iR2l`UkXm$`t zsU4h8a6vn!VaN2ytx`?s>=D%wrP)bjj>hChbH>ygkJmkF?T({sYRxpLl3apElv8Ms zpW^nYg@Z$4ZD#Vo5{V@wjmNJ_&bH96Hnh%Ccb$$MM$i}AJ^N7}WbT-@)fetAY;G=7 z$a!F!s6zPw_vF;E-B`z}Jknb@KrrGsLmu7u>shd9*9CvlE?+B^OelUm>eIz7ovuVe z;G&SFxhj5?uXU)T@40?E`!n{0iLL>cE*XGQ0PWubp1Pa~2|)62z>ugJzbeFn-dJb1 zyOJ1^IZo15xd*VvYU#WY;W;(i=`QarnhSUv0#$A$9>e<8(~Fz5M!T{dD@D}&PpG?F zK{>Vq278HiB71FKze?S2gtN;qxt8+G+;k)(;QqDgijZmcfg+47QH%vqj0%uVmcoqs z5zyD68WFama=zyaVery8oc*^^d7@#13a$t1T^^0$O(VnZWwC*Tgl*jJSb^>7TYI*T zn2L%^h|qP(^s7|k9gwss-LpoF#kuDk*E4%70L_939S1{OmriBLEyiM6WYRC+%x;k*NtIaS zIRL7Yo&fcy&tx4fl*mL1NtJP&x2OWCPTxP2vB%{N#U4e=D8_Ne^Q)GS&9W(k@vDHq zL;+S}N3ZgxpD|Bwbeaf!_|Qiqqa$T^4DMcsBi6CzN4LL!vdYDY-qD6Tco^h%HEEhz zBvR451(*V|@sc~_Q&Ky-%j;+gB*J-wX_w0`cpQpxidWFg^v?uGbUYbpCBkizLnCeL zfxC>=&o!esEII;<2NkCxlWUFu>H#2;`qU{|6xO3{Z*#fb(PIj7 z-7#KYCZlmCrsSe|VafAQ9Opm8E3=*Aw6?iPL3LojsTt}zezoVzbS;hq3Z2;QeTcFhMDXDo0OBSROAfa3ofXZz#Z{gYBA%L~D%AZmAvaD3@4QZeO)-5}#FkM8l$(w`Kl(!cK+cPJd- zV0wLhsh5Fgc+8N?9@HLRmPt^3bB?_!^380P3YYh2Ba%>6=Zt*E`=_3~QnJ|7mfcH* zqBjCbBgu#oo554p9FFxD+L{Zd-3vy{cV{>XIP@F}Nr1Li5=k(2izpwI3;xeuztX8e zs`+XRtLCWa#EhS)qR5+A(Gn%ZPSCMXWI041D`UQT(_c_Yt({mOl}FA-;oM-4(xSSa zE7-R@L2$sMM%xL<=zT>)aw3XXjzx{t@WGoWx7}hX-6oyH$Re?VaLnXLsn44mj(xHD z@li>rLvJ7wOvR~Eg~n)oR33Rn$`)Rlzo&oc`SAf8_u_`EC<(;v& zJ&sQ&6x5Ad%vX^c4oSjw9@UzQ*^S=DbP}0Eq*nk+5PYg~z=NM&D#1yamPQi*nR6tL zxs&Ps6g{4kYj&muC|C{0v*(;QQ^&PPrs2H8Z9>8bWip+ouWV4@leMmaCQB{Axv&sM zVGj;>d8$5L$2b7-_||N2>Q=WZ8r4O%N`j;2Cmw^Kt+a|%V52oq!l4qK|5**YQTH3c)-RWGne)d-W~J_UTi~;;hQ$xn}ax z?$pb3WiW6C3Ie;q=m$I=r2Z8I7I$)5n5MWcVU3*!_O`c*jv|wdtk@?$$C@P{cTq)m84i&$tS0v6W>=3avhYyjJxSw=eDm7D zYvl;`za$mN9la_Az0{{^+W<*9DH(O#5m%0%Ow~#4C-POJlX+epi{L5eqVv|K>u83L zLMN48?<&n1T&sD5Cx55^0A8=e_glZ7hvuh=M{32HrxBq~oPhi%UVc*qCZQTO-dP!Hg>*j1XI#;Etb-QG->E z-K3J}6_P!+R4|?I61_-AHW`G$5#u zCO%Fx_|=$Dd0td%@FZXugB*ztI{kg9t0M%Kd*63zYDmYRZatk(p&w9~~5H!-lG8YY!yS+2{Qg@TG8+sY9 zEZ4I<2?>md@~((7dhoCBL~hbBwFxMQ5X6+m!@$ma=8{QJO47ze-;CH6n zME?Lf$u#6Mh5#=Ldi_lbD)%PIGekoql6{%uSBnIyhB+XE$sy!mnur=(P^qRQVHUoIo?oQu?njBBoW5ys+TTMUI}A)Hwbn=0nc%c z_^BYap1?q|E2M#o46!OtYMxX_AZ<{`k{n|Lp)1@IRFd71DQjY})h!ZOmgVDFNXO28 zMZx5HW}PIK*Ag4sW`<^q_(Zv7IpC=2M7|ra)Fy!@wsu$!_@ys`K=tTOao(R{px8k_ zn-WazBeNupCgf5{>FJN99Fb7EWTugFSQYJ7Td9IBa-wzzCytorqrhwdJHXT^D zN9*bBP(78ivx)EnDO6y@I3u_R(wN#o@>V-%AsNXVzlf3QDf_QhhT3`(>XuQP=e!Ex zt~NZM-6t3$*aK26$y&%HR*F9Fa#fu2-Qg!InFFbTBD|DQAs_ z0UfsW#Y5+-hGha6RY7RU%O6aiZ+fhk%J;rpj-pu`Y;5h71HYy}3cR*0eLQxEj?UjG z2^c$vspguGKB!Al--Us=){KLd5w|0t>A>$@szbEr9I)i|#WzFnwZ4f9Id9)fkC!76 zk3m{bZHY3ZkZaPTOOgo!7SUB4Ab!KIt!pCSW3jA+w?1HFledmLR;(oK0fq<_w#Y7d z&yDOLw%1netx&s0WJ$K*sT@}v+Mn4b5tt=IF@y7jcY@8C9(4+vW*=;2OqmeInM*v@}~6CML1Z3<4oA8c;9QeV0IYO z8@L1feJWVqC?SqHA8`skSyfq+{gO}l=CA4Z5=`wFMh?nk;PHW;e-0`(ytCfl>DJKm z_Z>z@*F7nzH?kKy>QtCX6E(oKL?wYQ4=Von_4cVPN(dB6iUSaLsml@QbI7W&GNYD) zSskCMfwbqIDht_UfrN#cJA%Y{U}Ha8dTySjG@iztU zA-s4*WFI7QSXOggwZK*ik(Iy$84c^~DmS-=7H#ETb|IAszz6#MD%6{HWUE}$6I`nq zW@lK!5>swe4_~cNitf^DONkksHjotw!xQiL)o_N`t;CmCFCk#&6yZTWqngvSxzxO- zd93X~Y=vxuVf}wfrx?C~;H=o1X={n#jtxDEOvQFHZrph8P6d00izU2zLcCKZjb>(L7}~l{!y2l2~qu-mCKFLEH|NbSD~Hk9DEy zSDK8CJ4N<(g@W!@cKhFe&q}qXF%i2;3Q3HFC-{3;19ReE?HQv{3#aibXC+UsdZ#yv zE-m$F=4+OcWnHpH%s_QG`=hb;HPY)vJM<1ldM(B2S&z+(4u-Sl)S^}ZT!eV0+Bb8x;#Y3)< zc1Av{;(4`8l(!RvP|UmfowciHa{yUxO|~rKfEe9z$mx%%^{kcf;=_9oSf$+H@ZIC} zKU&kCJDBZK#@UuTD6(XgxI2&dS2Ws_yhSZurAwQqxo@_j$VUS_C{gK?jz0?3O_#RK zB1QzVBB*jm$Jf@XSS^e;v-wCaW^f0YBNrWgPZctogLQ58P*|`#`QUTY)1@gWy%W@# zBr@VV`*fBoWEV)lf=rkr(;20LYlX42dBNR}FKGumm(==mR$6tmXJtVn1~>tR2OM{* z(O!$owce>C1M;eI^J1q>f;tuu-nQ5#iW3a4lQJBcKl}1mQfar13x@NkL zEaz?(TP(0f+A?@QC_bP4dgt{o5V84H$WRF+_x}JI>vb}V3uF7I1lN;mcX3M6s4k<- z5w^|_)8F&rxoSyALscZTWVaq6n^S^oaSIsu4KGoj@+uo+YqUMm#3bL1shk3T$*j07 zq_Sy?-LWOW${%i+9jffs(+DGY?L%PmBxgOk;<@TJ^hOewG_Gj1%fSE`LFc|003mJp z8%_=@GTvu`?q;}DEgZW+Z%&~7DIVSIZjyPfB9I0r1m}1?2iC6<_NdHq?=h>19dXZ2 zm5RQ?cSUKB?qp*#c>rx)kA+6hYDu85h6v-HCV63FAYt?+LU=jZS>#U;EdT2~`Cbf~0Aq1-&gK>TMDn}!)zqM&f&@9t=R?M6JV%Qr=>@(Dl zrDbW(5hj*vdxu_fh~RUBj(-ZXb*M)H^8^c`^#Dg_Fq{VA$)= zY*j0(yN0)hO~T&eb&P>ru*v@B0Oym*G$OiV@=LGu{j{{XMo zIi+dtZKjSX9bI;)c?keBA5qsH^;+jo(yw&olI7ie)dDt8k@s>&J9j^wIz2sYt@CUi zON1Ckk@5iJj&oG?*Re?_rNm2Ot2BlA=A65Gu*D-x0s zWNZWd-^;x)KB0U(wt?AL5~Vp%zQ_92DKu*xRyF%g+>wpIk+aM$53fD_YHLW6-KCAL zC3u41fgTCQG4l4nqDs-%?kT(K%OdUkwRB84LW=mv2jA<8n%W|_3346OnD7)f6t}O} z+M-LCP2R{6%EkiiQOW7o-`rMQsViTZBo_#=xj1#eIr>spiWcV7(n`#)I;WUhle=&t zu3CJts>u{{MkB`M=Pl{$pL)$Ri0vnlf;_<$N2-o<^r&FHm&-yC8N(}XQ=DLqpIR={ zeq=kX%QC}vsBW3&lmqiG&w>YFe+qw;?Fly5Cwi`SGh-*vpGs}y?`x|nM2bwUnEb$5 zNF3mflQYwo1$$iuoUbesLSrukQF(~X_t1hrkp5Kc-KNod0=>(S7X+`$dSvn2oK*3Z zvyGt(vUymm5}-6-~h;8h(fTCNWd}`@6YR2 zZO!BWtA|+5Hi+;@>-moLr(n!sb#mf0U>#(Sj8Syhos5_+W&1*#yU4|`!3!Q(7WJk} z1cos)fYS?u8vNvd*eBofHKToP0$oIrxdZ-M0Uj44xcw@H#qf`6Bxvqf4eYr9cKl5a zNjA3>>_u&LboTMG7Mup#DwQFS=da;gbLueMLYDGBn>1h#F-BC6PkeA|5({yB;mp!` zjN43TxY&JiI*M!>9G`64Gc=I|1h<+Lf)8AF#VFHh6UxoGZY^)rM$?t@Sy+DL5uW^> zo|Woe6w__A?Hy)Fg3mbfC2|Jilfe37ytY3N!7iZ>mvW}&NfZ_HODhgDz~FkL@oo5+UReZf+d``j-u2Yz0+unOH_Np3=hX6RgA+T- z$gW*3Rd{tcucz|mklRSb;xe{T`(TsM`&7$qs9Kj7OA^MQkT-CB@Oo9LOTv$~cPTD# zySHQ?Ufgu2ptljgbqHH33Pm(aR#E=9bGPWvTqIfHznNQ7&E8;&#URAtq# zmd*x}MYs_HSsx&hdS@MLqgYzse1+q9LXjC!f`8AoXWU!;v14Rwb8#NmOK?Kxj)Ok5 z)NHNPrryaEMzLu;%A!dv$rzH{EYLiRXq(?ytF2; zFKuz;mx@VtU~b4O?~a)tN+TxCX0}+wq_}jE@wFTK!2bYBN#nUo1&N_j@+lHXB*5Fx zDr&rTQ$2#($!cV`I08nHD}oPW{VJ1`UWRf>y$cZOhSq5(ySS28U`UZi$Uh%{da)x! z@HBSF9uh%)?UKKZFxm*?xLG2QI6)e&{{WD#eL8Vf?`2@(6}JWxgAh_dY=1v`B-(bl zai2nXA!M3Xh)mG20g^RR20vO`?J^xo7K(2%7AD-T9IFC72kDBZ9E`E7moN5dN%>wW zFmc#pzt){Tr83VX23BW8(u zPhu~!-A3;zjGJ}=jD@nNuO6nOFKZ&SVippC#Nte(V2?qbbH!;(50;W!O**BxR#x)Q zAvhpl^dR&#ibSh!V77tcj&ch$gDu8+BYCSoN3P!D zte90L^Hx2C47l2NXV~_p7t~>j-Uwtn+<_y&p$m1#1Nzovv0k;&X^f@7Cv?2289o01 z#Iy0)D~%!to7-7FsgK+iQ=VRt)^EeN*+ec`!J9X~3qZ>PK~B3(3SuJ|k% zl_2x_`%!likk4tTOpuS4%2-KOBRq4>N`t=TMcw%fhR)hZ8W57eTju$jYJ<-J`cyDk z>JT#)cnl;HmXLr~+Z<$%TDfVdh#EUnZ1c+MPQWn43hX>9;d{RnM93B>wBaCPnnFetrgob86HoBIji*`1 zZw{Sq%NW2f7-D}q(zl8j7i2IbE9n{q6~EPcvKzU{_!GhsXc5hOdxLpDMC;{`O#0Vg z>=bk)o&fc!YB#_mJqfO;MMYT&l%_&AF#GXE#TAi`5B6alPU-}82G{?=taec(i6n35aI6{ZJrH$gfk~o>i4DGr$&$nYvSVRm26BLbDg2U$kXR!p$$Vu8vc2cXS>Vhyt6O zhWUxW8OPyS!pSuo>!~ER%z$BKjmo!9d-`!xYrR7Jtfysjc9wEnIvtG}W>e6Cf%xN* zik{b67bYurfl5fbNF#8S;S+dxR-8Qf$BM`Yo|qStYo-S{jnATkJMAq z14ALRU9(OtOlC$b;el>5&}8sFm89&j+FV`6vI(w8!j{MYFC-4fHC-OoRW68B9;)ne z{@`@M{HZQ%Bc6MCfsiy`>XgV755yjnuAby;Z3~u{4Dh3ejY%7c-GIQ4@@SC6B=c5Q zZU74r7(e4xS^})ZqNHU>N9sW5aQqK6>tQ8}F^vqRh&Th1eF5w-N)MMoUiy;7cLT*T zI+M9ioDK$f9<<=%+`{p^ep;w16>*IE=Av7hH3)7YSI#&pN~!fH{AiV8apnh^5C9FR zUBLT-dsVHia+>N|N3wDx4dfut0APd9r}L`GWoaeT-G8OXNx}%ZE)H{^I(Mpe_Q76A z!EU!F^i%tmp} z4nLP_itAPrzm+_NP_O_-fGHlmsqFBX+UDnT?Ls`iGjgA=rDw+z%H%~VlIy}sIT;?J zlatskSpLyTcHId!E(j_|TFm&y}smuR=RXp621;S8dU9sF=<=9=uiH@+(M}(6EQRk)TjO z>7H@VuN5tYsycN&s^e(rw)9q{k9H}GLs7t3pj;85uBbiA}igLZVC#5#_{woQi zx4CIVOrR?cPY0>Tr@d5U?`Acwz+r~Pq_&DHqzVIk-A+B~<-7-Uq`nlXZOSXJwAr4 zU)n3%$ub~F=L*bJW9J?Hv&CcqE4!KI3n2t5D`b!UwIJ1nqMJ+UrEnC4Wd8t`gMs;d zYFleSYRG(1K(ncM!<15R20t2ET&?U^bGraks8u*pdH(?Q(hHeX#yq1GK!6LJ=b^=G zHR9Y`vd0WzHmb(tZqHrcFJtdh3paMiTI$NtSYKSUV$oTxhJ3^6x;Vpic&5m(p+A%lfAM?0PO>T+L;W__YG|i znDRQ6kZr>D&P_JbNDBS0IE_>;^&FP>&q6&ZF7-V+)%q}$fL|Dzr@)Dw)TxJxz4B-YGWiZ+AY>20)Ar%&Cv&Ym&8+%JNi|CHo-+=L@ue ze!lg6r-sT0B)!5ZRr45g&pmsAT{Nn=tu`txrDM^w&l0AiWb$ttnOT4b4UXSR!`1aW zE9-T=jh;7svAk``1Y)^WltUqx?JinTor-pXzN6FWQ3gpPV5rwHNw~CRVZikVq2{)D zhQ}Ansn^+f!b>)mE0nlogyaG;>ZhEZ-<2)T#A{pVqiazlZiEyLFg@7%fIppbI+vSt zWe$>tm<3ME7pOkisD8k*K#`IJ!G=PB2=w~YJWTCjbH-0&M^ly?t63+Fo^Ojx@k0vw)yFIvvAmr#P`U}+GSawU;pB~zX~a%)mpEhU8myp1$uu_|&A zzP_TARIhX`tcD#-EtFR9*^_PvC?v@}`})&Fnr!Hd^RzAo89dT3X(v$<$0A=xFe-N` zL&-G+O=kp397iL!%uc{?Pij4)S}>H9MnQI3*UXU_703Zuc>sF)Rl|D{2;>-njzMwr z1CMIPH!;rgK^%vsK_ex)=QRnTmmB0rjgm?4j@68p#uqWGmWhIr4n{MB$nQ+Qc{J2nmM!90 zOk>NJ=PlRn!s zd3gkTdRBB6g;cz9ByPov@tlMH2AyfF%c?h%G#Oa|Ql-k1jCA9zSh*0+BDAwR3jESW z_ytJnymzMm0GMv^(5h8#*{v86I)GWi?F4q_me$oR?xcnwT0jXTFxovmDe^RzPaB6Y zNyrK^yMX?9tLjS5@y3ZO#+*obVtevEJ?iPjHK>xgmSt(@U}cER$^c>s+y-(p&*x5t z(oqu$Z`=}aWfV?at=Aq2OVnNyb?mB3x^Pv z+?$C!5_t8+SNjxF6C{^0L{>soKxXMZp%5(2Q=OJ}tr+E~L7 z5Vh1&GZuDLQVAW6N{}#xgGB_c?oawTBo_Ywd)Bpdi+De@qqFkj0hly}+a##<{#;e1 zcVtq3%5L~C;q;n{{{U<2GqAQS3@{{N8G!E1*9Xv7tJ}i`ji8PQ)+i%VLV1R2Kr$(C#sya!D_@llavu%hp(Al}Rfb0^Wpw7wOF{hjP8lvb?~q zMtLK)f5wvH@Wo>(f4V^go-x6tp6=cYP-Kt-tCDlnde@!!tHw%n`C+qUaQ^^#$-;&8 z1lCiUin|+kM**s(=A)`y+|8GcNjN+37<BN;KnXsN5%*M9jzK z1e|fx7^mubG?3@)mt+|mX^q*GjQ!)sUf31G7VmcT6J;x*AeuC~W|cCoyA*PVJm;Xn zt8hjw;4n(><~Pcz)SP$v)2wZpCoq|3n`-U|<2!)u?m4TGPA{Snu^wW}^C=s51JtX2 z1By~hp-ZtYs?j~~+B~@xjA`aM1A&YWZ1Gg3GszmfEf6@uyPh`ZJ-(lvT6rW(GaI3n zWPS?(cMNe^UP7v@GDsp)4nX;XWP_hU^fZ@tOk;C#FPR`QPKu65Ts9mx9dq1$X_wk# zO!76XY*5GYnEvU{s`bd{kyY$%E;S3dS&}kU#BsI?{Jo0-#(%=DY7kt9mf}4?Jg7h= zVk?oJpmC9cc=oEhxcO4ltEWLMcGKFcKbVSKEUp8B4?~{BW~(fwYkk(?mN0&1JrDl? zUaVfBv|Doxvq>z9jAV*3c*R?S)u5GbouNg4^{6={9)km+r)1WmR&0hFnQiR=IWsck zAbenF)|~`CPnR}&Ic=;u&p6}!DQqF#G_qY8mKRgO2j)5S>N`~x z{?U6idA7!Fk)%dd%K^c`>T}+&+Ft66k+jmsbLSvsTws&j9FIy(H1;WK>?BDtC`Ay< z7|BS(?J@e`pX6%$+Zd+b3W05aBS{Eo{{X}s`c`U*rIs@giXL+(ml-BAli#P~QV3s9 z)ow1<_1)!QHh>*M;BXhG<4rrgM(k*WW%7nl$Q8a-ZOC(+f%H67GhJ_X+Z(A2r82k} z&-zmXC)!9; z8pY(X`g7~vsXmCL`H`UZe`B~`G9?kHC6pYNeo4hu@)kRxCA#JjmJ%P7l6dL*RK95Q zBU#-DjkmBx$2{@%q%mByM&{WGg^mib+5sP5dR*7i!g1KBEj)+m1g>_ov0BY7c81&|4RcC1~6; z&A8Jfi&%zRLf0u2WCcf2gXnworr?&H0WB_z5XEt6C~3_5 z<7Ov(e<}gL zNTe}gf&u(6e>$aUV`wA}E!`(v;z-XU@H}FzUqud_v)yTuyEVK7$d?JX4fuYX*Enu( zukJv)e<9ox14ef+ep`)1?7~UcmEzUUO1k@oRy@ph{jU{6! zWl#xVM{`zeBrphGNeVfUTWK4d0DJn(;r%)FrY8N%ZMt22Hy`!!CRA*XvWo zsL0a@!Ih*S0u@PI^Z0Q}&N9&rtuAbRnh& z=TbQbCqL7grE6KC#!YTkl`PskJY}iGFra zaG)MIs-(_bDT~ZNpcTLceZd`n8g!FjN&p3$gkSNi(9SL411~9&gls2VD-8Ae)7$LP`I1J3 z3X(S!%DErNR%O-fS1lVCbb&GqfPBh(kMO3hn?t7^Y-~zx>?M0;h_XmYc?if{=Yfu= z+uEHc*`;9^3i9&4XG{P;O3S&rk_S{1Dnm4CsE#tlhB?ov6)b5aZudq)j7|^*{z=ABeWfsTdg%?I5uOA%-!Yf2~qFd@{`-Gs2S>E+ilY2Oj>Ss@+9A*^b@XIdQew z&n0-z9qUCzQpT?B<^{lYljJ9-ang-%a#Gg9q!C%&M4=h(9S-JU!2=yL$KzJ6O3LEX zlEHF}!yq31#)8pF317V!=W{kNDe~W&Z9*7ih{9JnW=?W@{RrcVd+ZvrtA}fmb?0W^ z3UF7c0A~Zx^FfDv$uw^&M&KBXCk3(JJ&iK*T~5YjkXwTQVpN34UWf3_YFpihjZMmV zrGXC?+y)+=+)}e{%&un@sgrG}Z@iu)n7&B!)IQ@G=aPH=6?Nn(cC$!XA>ID56D)Dl z@TgQ-*xeZ6vRiDa%80{cpHtl8tpvA`SIk*(2r9^McAx&WD|^;Nt2dy?E?~I3cu$+T2;Yyjb$)B#>Q6BLw?#PQ8uo;ECq_ z^dU*y4{qH*EE*lrQde4&xJW}Z_RDb$qbneC5BEL+<| z?j0I6WkVUuDLK#5h~<}O%eGPd;GF(E9%|)i>dBV94n8|e>nWCNxT5{wz>#IaOy{Ed znysfvd^Hnl(7dn~!?6USkb3_B`m3*>NsKbzqc@WwD(dnU0E}RdL+w;$yNgdUTOvt@ zxMyZ4O5*^5{Oa4b#VJ^p$#!m7{%%O~3C?=DQ`eIIvZeHblzyxm^{E3X7 zNcF2aew<-~;uCPvsEjKanbGdB19+nfd}*Br7<9xN1qUcZo?TQYz9BiH8NXVhn_fIS5blr z%hw*qy+uBqE&aT6T}H@YkcC%}gU&E^`um!1UBYhV%UxA2Eu@ZAj&NKE;{cDCa>Sp~ zsogr2p5kdEww6`_VoWF?bOiRMK+zjX{%FHMtP7u%hHP{7qT5C?#~iW+l_MG7w{o8S z1uIHVLSnA<92WL_uk?1rYZ%KG+C9Pd`crOWSnULJIo!-cIl(yowK0`T8RPSsKz8mt zAmgSz>PcE<}m8fge=L6bW`KD82k5S`w_X|~WbSsU^r45YXKWBl}|%8|~@!dSuq zmdVLn=k=x~kcm~*B`R_Y$@f?5?Mn=okVwug<5!V3Wnv{rJaBq`yjIDpS(2l35-lS3 z9Yv;9K2kXVMJ5#RdwTTz>ckeB%vh2Yd$?5jW;n=K0CoG_M|#z_j^ai!vOw;qV(+tWa1cqi}Reu3W&rJ(y^hFtU2Giz8SC}%e-lUMgOT|~Oo=#ZI< z8*6yPqT?AX8z6o@-lDmyScT-!OC;q&kY#Ab(*D@#T5lmsb8ON$w-Rj)A?uH&DAb*- zhpJX=u_uaT)Pq|_o0&%kI8(tnC)bL{(>zlysbTgVvPCdpP=oUD!MgOtN{tPhO7Ytx z+!)zcE=s6w{kv9c-ODH>EHRC^LmGg&srJ%WxcRiP>G#_8wdR{Nvp<$j(DCoj103>5&yF#X$9l)O)e<;dNdt+)0e~<*m?y3=Sw>R5*(zEYD;#iK#LyKn?IJ*+ zsB8>=6{Ih1ZsT`!Q6wnFJIv_mIVYz*<(=Cu+VYlxmn z>=hRUWR6Y`OmUtKWcj4aTC)z^yImxpq7k*eRyZr%{YRxv5=DmynE{!(0|e)v{{U59 zjyPdPir&p*h~ZK}g(UamrYgmkk{dHJ%D!O)BXji67(YrHL)n`1qn_Gn*UFA~(YEBu;1Y9Nchcy(j4{O>UpgwAX{jj z(XCys7%KURf)P3A-_oS}5VnG7on&h%nRb)Njlj<3Z%%q*l3h@z?;?`iwcM=>-0{Fs zx9^N#59QXWwY;}6%+fjtfdC8uN4GVi)-lA=T*#^ve;j8cp!taU(hEyzEld{|S!sb~ zWQdRe&%YEcQ`FCz$f<9sF@oL)w3p8)5lXMj_#^{^+nQ{)vqx%P;#NWC4ip02e(4?0 z(x*#%xTB8bT;@HXAy1T(>`zLQ>_+-vA-Pq+F3!!8PaXYxW{FD0H+>4CSbotEEXf=2 z>{A>cdeUhvE++Gfk+@+9Ahrh?1MsMAB7o`yUBi?ZDBJUpamO6>sV?HRhWdsQ{E3%l(sKukzHxX$7|2k>r zWaD?ztlS8E(|ZNH%A^)4oS4pg5A*ufh!9P08^EA2A&Lbm0psw_1thLfnpN{6WjPx- z2b0g!v8Bql8%>L4WD@2Z!qTj7i84k4k^awI=CowBj@s4Q&goGlk&C*ic?fQp^c{Lr zZEj9fO}vq`DQNP^p5Xh`iF1Exa8g@kVUVdQjHt)A;aX_41wLI0H*hALA)UfY7$huZ za?kF^ijX>LI%`QPgjn1Vk@BQ-`3%y#CDU9@D&4tx2;H{`pz(lxD>gXR7-U;_*ra)8 zJY)>$W^sGTGt;9Z1mO~=Q zNO?#k5BF-D(!@qhXkLV`*#6Mvg}WRF-;AH}ra@@x?C^{vid2wy5=j-3mKPt|F|VAh z0RuL|UPNt4>FrXY$dc5QY3lKj7-S^0jt9!esLlr% z$E6n5Nj0;JtyoS=6-+2c!0g%U$m%M+zm!keX4Ki5bs{j!l1?}y90OQZ&R+CGZos5K zmXv}{eLW~uB%Rr+Z6myvYe8ciE@PQTDhb_yKR$WyR8lKcc9tYUBO!{B^Ap=9u$>E7 zhAp#WlA$Ksl7FAAZCT7!%ElKh84ZFMHy!Gn?5@L6!)3xlHL|F1rCH-#uwl>4Jvr^^ zNqFmVVw)yU6$Y)yneF6Xs+#LcuNOb@QTeozgkp%)_zgKZR^tT-wj4$u+aZ zxnx9UUzC7zp1+qhns5YL z+|jk6q8;@)V=|1As@PN4KI1f{E2VLCv)Fyj#j>5SKi(vqwgR5u_ohX24D4D&SBwF= zMh75v%`PRHA@+C{A`<}PchetQPqVeWD+Fq>9m9Ar;GVzdG^HiI#L`a4boVFDmNt$u zr9lc;c76VpY9-T_z(=}Ea)5y7{$lfTk0(5#yto1sV%hoxI`k_QMKbC*pHcb9ewFFB<@pJWG!z5u=%cr zNmCKVg@Wz;dm5*7j;n3t&SF(0z7Gn&&bDu-pUNus?#cqI$&kB^P`gW&w*o9qS@OgT zvFq}I%}kSeAmqByl#R^OOasJPAi47xU;)SD%|x1vF@mPx0>|dv^8!6T`t=>m4ld=e z7ShQeQSwGe19$ho`o^KR&?G=o=3tW{BqrcMUw`naah1&7Hyblr-ts%>OmQqg#-ttN zs-ez&zCf~L4lS0Gxflzu5InoQTCnEv6Pa!;YFE-jU0wSwklkS5*5 zGC=kB=BfRz3whB3IF3ErWIT{gK3`tdF6bpzhG?1NW+bsVDt(WoS3b5xN?!HQ+lzS- zUJFwd+?Wf3arlw+tDj_4+ifF2WeOQYZ@Xc@;F>`{kmA6D5`l>3>KTCGKhxHtfJ^8$AKPMhtzhY)%FLu7~LeVV?1C2r0s9GN$PhFYbN(hh6}kB?BY;xR}wf3xau%@ z`qx1v=AUmYa~HWX#tKG^ICIW7cjCG0x$f`oq?%c6<$Ue=m+s^EX-d}IQN8TVD}a-v zjteL6}(`m5FY~G_p%+ zwkv4X=HRYKjWN&H{6%eeerNvETS4U*fyj1MBxHM4br-ib^T{(RVF`q)6$c$dcB^vS zTEh%(Ax4pt06>`9#BypClk+QPUX@Mmr!}>u-Msr-=WyU{2N)dS{xuGvr3oxWmE6eW z$Xa3#%NhYq-Pfo zxpJ{|BP8-if2CNxnW4Id(A@2|jPCPG4jA?W(uY36T4*Zj0`gcuk}NubEK#XY7-yV+ zjT4}bIMiFHW;pW$eeS2|0H#VUE$*=z!s^&LjNp^>{{R}WqwO~F#Pcv#W-5#Z2eAjz zv)@K16?>wm+S>913P|##coHBO$@R(O@T<1QNM~@c+=zJ3IoXbL_|-#g6lcnid1|^XGtxf5Uh-MxqLGW=kE9Qt%)AmTc)**$~-I@L}7s;$8V)&G|dDe76gT2>Af;T zbIls0INM(2@xYeXQcdO!sfAd|;dgre9ckv>OkzWF&bj^Da0mJN)pRE62^h?<$gCG; z2t70S)mbNz)gqLD7;~No0C(r~tlPEBZW>nWrXx$RU}Tpd*ZA6aGadz0tkCXf2{ps(`PX9_|Nh(vrMYky5p+iQ=?i%M=WO zqz#k1jkxBrUgXH+z|o+Of+KI4vE1XnD%jNmPa;Nnh>%(|z##fmqEwV9nkRQWf+E^S zUcCY1y(OwLX*6WpHOJZymGd;)alQD-_cb-qGfxXnkTHxg5;0xM*~M5(a*``WB0|ux z+^o0+bL-xZ?8poGD%<>jDc%iqIBUTN=$vQ?kPvl30WQ9GrK~dsbcTyQR?0 ztEwc8v4F`OpVF<_X^A!3Y9?le68>O>;Pn0+`c<0{7TOq+;B&z3S7!Gz zaFiI$&gpFkx{=ydAOfUGjsfQ!W8SA}>=w$(AqBw@N#%tS7cvrgE%iUjlq{i|G=g}M z&;^oC3XzUF^y0JaaR%Wv${7|wRFpeIWas>vEYjSjor}X%wk% zKxeqCOn^_`$mo8R6Gs}Et|2k0k$`DI`E%TL>-4MZJ*ZoIZ!_%^2-{<9 zzz2hY-|;mPZ4st|Ti@PD(_6Gk>?8*=N3|qD>C{sl>z zM+DzIS;833Kqoy`X6#XI+pvFDW#)Cwmy1YxOOQNXJe}k!2ACI_365eqok9`8D~=%DszAo z_a39I7QoD)E5Dr?4Y5}n$4}0*BDqF|11klCITEfn1K0AU89s!!riV2T+n|akfmtJo zy0V>*p1*+=p=+!rp*(PeklGxP+N#e8i%ykfN4bo!I2^j4?u)460Ws(L)+N9*h>PG|9(y0FcW^P-lo_Az~fT2{Q3}lf}>B{$aEgT|9zzw={ zk&Jadoc{niw|A+mgj-nL!o(fCI43>wJxyaPz0op#8H+uzF_xX9F{=zZf=aJ!`_`_T zZ)CU42hyp`#QBMA?1e~oe2z~XX0xD?Cz#69+(|G(gvTb>z(1BxwNxjmD_v23 z{{Td_MTSV*?6NB(FYhuAGlkFV%|j@K^+qvTf+S%gDNv`Lx%I*IqH^hN3^wvaM-n+J zk>C5-=}!_{Tw9@qVe=r$q=50Bc|O!mJ<7g=!a+RIM)Nt1TP({EbNO`qYHQm|O*jJu z&d5=`!;^+~Askx`;#xSTKsG_t*sz_9jNs+M4y=7^ z4qMr6p|?*cY?XH`+sd$~i==g6U7fHPGrj{F$?2m))o9K6or3+MGC?3TAT~x0Dv_Rf zt{7ZP<{hEI=){VFE@YKrV=IM5RSDB*Jk*zStc4OVttLp=9}EZbHCDd82Pr1*23vnTTgu=NBNDTFU`1|^V*f|SzuXJ z;+cL@V*WH9x$jb4T$Ol&WdlS?y1o(D>FPVx$z9#CG}F|~n&RdH7_z*9a_sBN0iKm! z(*8)*g8)`RkYxEvAO5vypp9+E+9Zs|r*k1vN#nD3BBDrbWM(kJh6c;z91wWyY8(>Q z~FeDNqAu>4(1GWKgMjzI$S-qbCNTd;+yKsNQ@v0Zv zoOaQxKFGr{+|0m`J#kgcmWZa>*lncZ+8d}YqPX5kV9G&IJ*z-i+RXk`j~I!g4AGS9 z{Qm$tqXptzGL?PS%gLCo0UpD@(x)@RvF?s_fzu4VvyW5Lr)oLH&ei^RgP#5CLeg38Ap+9f!9Y~x;fVD);()f3BS&oBTEw|3MS>u-tDyPa19=NI(^2>0*ub(Iw0z`Zq{{Xe^ z?^o7nt!H5OvH>#T3-YJGxIUE~tf4g{Y-7ZduKxfqrWB~>jQ;>iQ<`in_aAF%ER(mG zS>%nBeC?1Fj@hjnO)BU_-dbCrF2`x@&%H9aLH__FB%k%f{{ZMTQB^PEBD$K_zwzS# z0PdwypZ#^W{dY7`S=9K~shixt2m0HOqkl0~Z$H$_{{RD`ieG6KwqsnqQ`$?8{Dk!O*>(}%(X2idE{{YuRS9E{%{{V>p0MJboR;&1o!aEqZzw4*} z0Iu!*DXI5g{<*ajRK3xnBY(Po-XGGSzW)GMb^icepZ@?sqKcQig+0dDf5e$@D5VXNFxtz1`aleqFiqc;LHR#TbKjX>|_eFBA_b2}Ve1GUdiYmIVnQ{0L*H8ZdAdA2H_^|#;k)hME>Um0@Q6nszr03Ol*0M}~M)Bgb0+JDPP)^{{RZev;P2)JfHR4f9PV0DNA-#%CUdP z%%AUnT9W(zKAZi?Kaiq|M6J`q4!QzqY<+g^%5}{{V1* zO2oPU0FTN40PC0j%qXIXb@d_#O>_SMAj|&%`r{Att~#o1+r}$AtX)lZJGJBmw zy$Acv{VN&|^><(O-2VU}MHP&;`^b@QPx%N-{{UX;{LNZl`18~M02(N%e*opOGb}&U z-u*}OsPwP-2ao#sAJ&R0sqBXjgj@dr`3ATBa(a)#{{Zp4)?$CiLhJq1qKb;2 z0Q!+=Uw(hb`qS?J0MtMIauiWlPvS&N;u)p?0LT&#_Yd==lm0y${<={`WiN@B4~&j4 z`1>F3{{T^1hy7gt0Pj&n6#f=rEz9x$0LUWi{l)%t*uUf|pY_atIw+&DQP_i7GS!a(Q2?(0`sM!st|F?~f2y16-`0vM-^7BaL6U#SAb-Ap zty$Fn0OR{d{{V6S0Q4Fttg3$z`hSp?!r1=+sF(5nRnOXWx&HvJ@BHYZl)sDeE ResidualBlock { let bn2 = BatchNormConfig::new(out_channels).init(); let downsample = { - if stride != 1 || in_channels != out_channels { + if in_channels != out_channels { Some(Downsample::new(in_channels, out_channels, stride)) } else { None @@ -102,6 +102,7 @@ impl Downsample { // conv1x1 (default padding = valid) let conv = Conv2dConfig::new([in_channels, out_channels], [1, 1]) .with_stride([stride, stride]) + .with_padding(PaddingConfig2d::Explicit(0, 0)) .with_bias(false) .with_initializer(Initializer::KaimingNormal { gain: f64::sqrt(2.0), // recommended value for ReLU @@ -128,13 +129,15 @@ pub struct LayerBlock { } impl LayerBlock { - pub fn new(num_blocks: usize, in_channels: usize, out_channels: usize) -> Self { + pub fn new(num_blocks: usize, in_channels: usize, out_channels: usize, stride: usize) -> Self { let blocks = (0..num_blocks) .map(|b| { if b == 0 { - ResidualBlock::new(in_channels, out_channels, 2) + // First block uses the specified stride + ResidualBlock::new(in_channels, out_channels, stride) } else { - ResidualBlock::new(out_channels, out_channels, 2) + // Other blocks use a stride of 1 + ResidualBlock::new(out_channels, out_channels, 1) } }) .collect(); diff --git a/resnet-burn/src/model/imagenet.rs b/resnet-burn/src/model/imagenet.rs new file mode 100644 index 0000000..2240f63 --- /dev/null +++ b/resnet-burn/src/model/imagenet.rs @@ -0,0 +1,1049 @@ +use burn::tensor::{backend::Backend, Tensor}; + +// ResNet18_Weights.DEFAULT.transforms() +// NOTE: we simply resize to 256 instead of resize + center crop +// ImageClassification( +// crop_size=[224] +// resize_size=[256] +// interpolation=InterpolationMode.BILINEAR +// mean=[0.485, 0.456, 0.406] +// std=[0.229, 0.224, 0.225] +// ) +const MEAN: [f32; 3] = [0.485, 0.456, 0.406]; +const STD: [f32; 3] = [0.229, 0.224, 0.225]; + +/// Normalizer for the ImageNet dataset. +pub struct Normalizer { + pub mean: Tensor, + pub std: Tensor, +} + +impl Normalizer { + /// Creates a new normalizer. + pub fn new() -> Self { + let mean = Tensor::from_floats(MEAN).reshape([1, 3, 1, 1]); + let std = Tensor::from_floats(STD).reshape([1, 3, 1, 1]); + Self { mean, std } + } + + /// Normalizes the input image according to the ImageNet dataset. + /// + /// The input image should be in the range [0, 1]. + /// The output image will be in the range [-1, 1]. + /// + /// The normalization is done according to the following formula: + /// `input = (input - mean) / std` + pub fn normalize(&self, input: Tensor) -> Tensor { + (input - self.mean.clone()) / self.std.clone() + } +} + +impl Default for Normalizer { + fn default() -> Self { + Self::new() + } +} + +// ImageNet categories +pub const CLASSES: &'static [&'static str; 1000] = &[ + "tench", + "goldfish", + "great white shark", + "tiger shark", + "hammerhead", + "electric ray", + "stingray", + "cock", + "hen", + "ostrich", + "brambling", + "goldfinch", + "house finch", + "junco", + "indigo bunting", + "robin", + "bulbul", + "jay", + "magpie", + "chickadee", + "water ouzel", + "kite", + "bald eagle", + "vulture", + "great grey owl", + "European fire salamander", + "common newt", + "eft", + "spotted salamander", + "axolotl", + "bullfrog", + "tree frog", + "tailed frog", + "loggerhead", + "leatherback turtle", + "mud turtle", + "terrapin", + "box turtle", + "banded gecko", + "common iguana", + "American chameleon", + "whiptail", + "agama", + "frilled lizard", + "alligator lizard", + "Gila monster", + "green lizard", + "African chameleon", + "Komodo dragon", + "African crocodile", + "American alligator", + "triceratops", + "thunder snake", + "ringneck snake", + "hognose snake", + "green snake", + "king snake", + "garter snake", + "water snake", + "vine snake", + "night snake", + "boa constrictor", + "rock python", + "Indian cobra", + "green mamba", + "sea snake", + "horned viper", + "diamondback", + "sidewinder", + "trilobite", + "harvestman", + "scorpion", + "black and gold garden spider", + "barn spider", + "garden spider", + "black widow", + "tarantula", + "wolf spider", + "tick", + "centipede", + "black grouse", + "ptarmigan", + "ruffed grouse", + "prairie chicken", + "peacock", + "quail", + "partridge", + "African grey", + "macaw", + "sulphur-crested cockatoo", + "lorikeet", + "coucal", + "bee eater", + "hornbill", + "hummingbird", + "jacamar", + "toucan", + "drake", + "red-breasted merganser", + "goose", + "black swan", + "tusker", + "echidna", + "platypus", + "wallaby", + "koala", + "wombat", + "jellyfish", + "sea anemone", + "brain coral", + "flatworm", + "nematode", + "conch", + "snail", + "slug", + "sea slug", + "chiton", + "chambered nautilus", + "Dungeness crab", + "rock crab", + "fiddler crab", + "king crab", + "American lobster", + "spiny lobster", + "crayfish", + "hermit crab", + "isopod", + "white stork", + "black stork", + "spoonbill", + "flamingo", + "little blue heron", + "American egret", + "bittern", + "crane bird", + "limpkin", + "European gallinule", + "American coot", + "bustard", + "ruddy turnstone", + "red-backed sandpiper", + "redshank", + "dowitcher", + "oystercatcher", + "pelican", + "king penguin", + "albatross", + "grey whale", + "killer whale", + "dugong", + "sea lion", + "Chihuahua", + "Japanese spaniel", + "Maltese dog", + "Pekinese", + "Shih-Tzu", + "Blenheim spaniel", + "papillon", + "toy terrier", + "Rhodesian ridgeback", + "Afghan hound", + "basset", + "beagle", + "bloodhound", + "bluetick", + "black-and-tan coonhound", + "Walker hound", + "English foxhound", + "redbone", + "borzoi", + "Irish wolfhound", + "Italian greyhound", + "whippet", + "Ibizan hound", + "Norwegian elkhound", + "otterhound", + "Saluki", + "Scottish deerhound", + "Weimaraner", + "Staffordshire bullterrier", + "American Staffordshire terrier", + "Bedlington terrier", + "Border terrier", + "Kerry blue terrier", + "Irish terrier", + "Norfolk terrier", + "Norwich terrier", + "Yorkshire terrier", + "wire-haired fox terrier", + "Lakeland terrier", + "Sealyham terrier", + "Airedale", + "cairn", + "Australian terrier", + "Dandie Dinmont", + "Boston bull", + "miniature schnauzer", + "giant schnauzer", + "standard schnauzer", + "Scotch terrier", + "Tibetan terrier", + "silky terrier", + "soft-coated wheaten terrier", + "West Highland white terrier", + "Lhasa", + "flat-coated retriever", + "curly-coated retriever", + "golden retriever", + "Labrador retriever", + "Chesapeake Bay retriever", + "German short-haired pointer", + "vizsla", + "English setter", + "Irish setter", + "Gordon setter", + "Brittany spaniel", + "clumber", + "English springer", + "Welsh springer spaniel", + "cocker spaniel", + "Sussex spaniel", + "Irish water spaniel", + "kuvasz", + "schipperke", + "groenendael", + "malinois", + "briard", + "kelpie", + "komondor", + "Old English sheepdog", + "Shetland sheepdog", + "collie", + "Border collie", + "Bouvier des Flandres", + "Rottweiler", + "German shepherd", + "Doberman", + "miniature pinscher", + "Greater Swiss Mountain dog", + "Bernese mountain dog", + "Appenzeller", + "EntleBucher", + "boxer", + "bull mastiff", + "Tibetan mastiff", + "French bulldog", + "Great Dane", + "Saint Bernard", + "Eskimo dog", + "malamute", + "Siberian husky", + "dalmatian", + "affenpinscher", + "basenji", + "pug", + "Leonberg", + "Newfoundland", + "Great Pyrenees", + "Samoyed", + "Pomeranian", + "chow", + "keeshond", + "Brabancon griffon", + "Pembroke", + "Cardigan", + "toy poodle", + "miniature poodle", + "standard poodle", + "Mexican hairless", + "timber wolf", + "white wolf", + "red wolf", + "coyote", + "dingo", + "dhole", + "African hunting dog", + "hyena", + "red fox", + "kit fox", + "Arctic fox", + "grey fox", + "tabby", + "tiger cat", + "Persian cat", + "Siamese cat", + "Egyptian cat", + "cougar", + "lynx", + "leopard", + "snow leopard", + "jaguar", + "lion", + "tiger", + "cheetah", + "brown bear", + "American black bear", + "ice bear", + "sloth bear", + "mongoose", + "meerkat", + "tiger beetle", + "ladybug", + "ground beetle", + "long-horned beetle", + "leaf beetle", + "dung beetle", + "rhinoceros beetle", + "weevil", + "fly", + "bee", + "ant", + "grasshopper", + "cricket", + "walking stick", + "cockroach", + "mantis", + "cicada", + "leafhopper", + "lacewing", + "dragonfly", + "damselfly", + "admiral", + "ringlet", + "monarch", + "cabbage butterfly", + "sulphur butterfly", + "lycaenid", + "starfish", + "sea urchin", + "sea cucumber", + "wood rabbit", + "hare", + "Angora", + "hamster", + "porcupine", + "fox squirrel", + "marmot", + "beaver", + "guinea pig", + "sorrel", + "zebra", + "hog", + "wild boar", + "warthog", + "hippopotamus", + "ox", + "water buffalo", + "bison", + "ram", + "bighorn", + "ibex", + "hartebeest", + "impala", + "gazelle", + "Arabian camel", + "llama", + "weasel", + "mink", + "polecat", + "black-footed ferret", + "otter", + "skunk", + "badger", + "armadillo", + "three-toed sloth", + "orangutan", + "gorilla", + "chimpanzee", + "gibbon", + "siamang", + "guenon", + "patas", + "baboon", + "macaque", + "langur", + "colobus", + "proboscis monkey", + "marmoset", + "capuchin", + "howler monkey", + "titi", + "spider monkey", + "squirrel monkey", + "Madagascar cat", + "indri", + "Indian elephant", + "African elephant", + "lesser panda", + "giant panda", + "barracouta", + "eel", + "coho", + "rock beauty", + "anemone fish", + "sturgeon", + "gar", + "lionfish", + "puffer", + "abacus", + "abaya", + "academic gown", + "accordion", + "acoustic guitar", + "aircraft carrier", + "airliner", + "airship", + "altar", + "ambulance", + "amphibian", + "analog clock", + "apiary", + "apron", + "ashcan", + "assault rifle", + "backpack", + "bakery", + "balance beam", + "balloon", + "ballpoint", + "Band Aid", + "banjo", + "bannister", + "barbell", + "barber chair", + "barbershop", + "barn", + "barometer", + "barrel", + "barrow", + "baseball", + "basketball", + "bassinet", + "bassoon", + "bathing cap", + "bath towel", + "bathtub", + "beach wagon", + "beacon", + "beaker", + "bearskin", + "beer bottle", + "beer glass", + "bell cote", + "bib", + "bicycle-built-for-two", + "bikini", + "binder", + "binoculars", + "birdhouse", + "boathouse", + "bobsled", + "bolo tie", + "bonnet", + "bookcase", + "bookshop", + "bottlecap", + "bow", + "bow tie", + "brass", + "brassiere", + "breakwater", + "breastplate", + "broom", + "bucket", + "buckle", + "bulletproof vest", + "bullet train", + "butcher shop", + "cab", + "caldron", + "candle", + "cannon", + "canoe", + "can opener", + "cardigan", + "car mirror", + "carousel", + "carpenter's kit", + "carton", + "car wheel", + "cash machine", + "cassette", + "cassette player", + "castle", + "catamaran", + "CD player", + "cello", + "cellular telephone", + "chain", + "chainlink fence", + "chain mail", + "chain saw", + "chest", + "chiffonier", + "chime", + "china cabinet", + "Christmas stocking", + "church", + "cinema", + "cleaver", + "cliff dwelling", + "cloak", + "clog", + "cocktail shaker", + "coffee mug", + "coffeepot", + "coil", + "combination lock", + "computer keyboard", + "confectionery", + "container ship", + "convertible", + "corkscrew", + "cornet", + "cowboy boot", + "cowboy hat", + "cradle", + "crane", + "crash helmet", + "crate", + "crib", + "Crock Pot", + "croquet ball", + "crutch", + "cuirass", + "dam", + "desk", + "desktop computer", + "dial telephone", + "diaper", + "digital clock", + "digital watch", + "dining table", + "dishrag", + "dishwasher", + "disk brake", + "dock", + "dogsled", + "dome", + "doormat", + "drilling platform", + "drum", + "drumstick", + "dumbbell", + "Dutch oven", + "electric fan", + "electric guitar", + "electric locomotive", + "entertainment center", + "envelope", + "espresso maker", + "face powder", + "feather boa", + "file", + "fireboat", + "fire engine", + "fire screen", + "flagpole", + "flute", + "folding chair", + "football helmet", + "forklift", + "fountain", + "fountain pen", + "four-poster", + "freight car", + "French horn", + "frying pan", + "fur coat", + "garbage truck", + "gasmask", + "gas pump", + "goblet", + "go-kart", + "golf ball", + "golfcart", + "gondola", + "gong", + "gown", + "grand piano", + "greenhouse", + "grille", + "grocery store", + "guillotine", + "hair slide", + "hair spray", + "half track", + "hammer", + "hamper", + "hand blower", + "hand-held computer", + "handkerchief", + "hard disc", + "harmonica", + "harp", + "harvester", + "hatchet", + "holster", + "home theater", + "honeycomb", + "hook", + "hoopskirt", + "horizontal bar", + "horse cart", + "hourglass", + "iPod", + "iron", + "jack-o'-lantern", + "jean", + "jeep", + "jersey", + "jigsaw puzzle", + "jinrikisha", + "joystick", + "kimono", + "knee pad", + "knot", + "lab coat", + "ladle", + "lampshade", + "laptop", + "lawn mower", + "lens cap", + "letter opener", + "library", + "lifeboat", + "lighter", + "limousine", + "liner", + "lipstick", + "Loafer", + "lotion", + "loudspeaker", + "loupe", + "lumbermill", + "magnetic compass", + "mailbag", + "mailbox", + "maillot", + "maillot tank suit", + "manhole cover", + "maraca", + "marimba", + "mask", + "matchstick", + "maypole", + "maze", + "measuring cup", + "medicine chest", + "megalith", + "microphone", + "microwave", + "military uniform", + "milk can", + "minibus", + "miniskirt", + "minivan", + "missile", + "mitten", + "mixing bowl", + "mobile home", + "Model T", + "modem", + "monastery", + "monitor", + "moped", + "mortar", + "mortarboard", + "mosque", + "mosquito net", + "motor scooter", + "mountain bike", + "mountain tent", + "mouse", + "mousetrap", + "moving van", + "muzzle", + "nail", + "neck brace", + "necklace", + "nipple", + "notebook", + "obelisk", + "oboe", + "ocarina", + "odometer", + "oil filter", + "organ", + "oscilloscope", + "overskirt", + "oxcart", + "oxygen mask", + "packet", + "paddle", + "paddlewheel", + "padlock", + "paintbrush", + "pajama", + "palace", + "panpipe", + "paper towel", + "parachute", + "parallel bars", + "park bench", + "parking meter", + "passenger car", + "patio", + "pay-phone", + "pedestal", + "pencil box", + "pencil sharpener", + "perfume", + "Petri dish", + "photocopier", + "pick", + "pickelhaube", + "picket fence", + "pickup", + "pier", + "piggy bank", + "pill bottle", + "pillow", + "ping-pong ball", + "pinwheel", + "pirate", + "pitcher", + "plane", + "planetarium", + "plastic bag", + "plate rack", + "plow", + "plunger", + "Polaroid camera", + "pole", + "police van", + "poncho", + "pool table", + "pop bottle", + "pot", + "potter's wheel", + "power drill", + "prayer rug", + "printer", + "prison", + "projectile", + "projector", + "puck", + "punching bag", + "purse", + "quill", + "quilt", + "racer", + "racket", + "radiator", + "radio", + "radio telescope", + "rain barrel", + "recreational vehicle", + "reel", + "reflex camera", + "refrigerator", + "remote control", + "restaurant", + "revolver", + "rifle", + "rocking chair", + "rotisserie", + "rubber eraser", + "rugby ball", + "rule", + "running shoe", + "safe", + "safety pin", + "saltshaker", + "sandal", + "sarong", + "sax", + "scabbard", + "scale", + "school bus", + "schooner", + "scoreboard", + "screen", + "screw", + "screwdriver", + "seat belt", + "sewing machine", + "shield", + "shoe shop", + "shoji", + "shopping basket", + "shopping cart", + "shovel", + "shower cap", + "shower curtain", + "ski", + "ski mask", + "sleeping bag", + "slide rule", + "sliding door", + "slot", + "snorkel", + "snowmobile", + "snowplow", + "soap dispenser", + "soccer ball", + "sock", + "solar dish", + "sombrero", + "soup bowl", + "space bar", + "space heater", + "space shuttle", + "spatula", + "speedboat", + "spider web", + "spindle", + "sports car", + "spotlight", + "stage", + "steam locomotive", + "steel arch bridge", + "steel drum", + "stethoscope", + "stole", + "stone wall", + "stopwatch", + "stove", + "strainer", + "streetcar", + "stretcher", + "studio couch", + "stupa", + "submarine", + "suit", + "sundial", + "sunglass", + "sunglasses", + "sunscreen", + "suspension bridge", + "swab", + "sweatshirt", + "swimming trunks", + "swing", + "switch", + "syringe", + "table lamp", + "tank", + "tape player", + "teapot", + "teddy", + "television", + "tennis ball", + "thatch", + "theater curtain", + "thimble", + "thresher", + "throne", + "tile roof", + "toaster", + "tobacco shop", + "toilet seat", + "torch", + "totem pole", + "tow truck", + "toyshop", + "tractor", + "trailer truck", + "tray", + "trench coat", + "tricycle", + "trimaran", + "tripod", + "triumphal arch", + "trolleybus", + "trombone", + "tub", + "turnstile", + "typewriter keyboard", + "umbrella", + "unicycle", + "upright", + "vacuum", + "vase", + "vault", + "velvet", + "vending machine", + "vestment", + "viaduct", + "violin", + "volleyball", + "waffle iron", + "wall clock", + "wallet", + "wardrobe", + "warplane", + "washbasin", + "washer", + "water bottle", + "water jug", + "water tower", + "whiskey jug", + "whistle", + "wig", + "window screen", + "window shade", + "Windsor tie", + "wine bottle", + "wing", + "wok", + "wooden spoon", + "wool", + "worm fence", + "wreck", + "yawl", + "yurt", + "web site", + "comic book", + "crossword puzzle", + "street sign", + "traffic light", + "book jacket", + "menu", + "plate", + "guacamole", + "consomme", + "hot pot", + "trifle", + "ice cream", + "ice lolly", + "French loaf", + "bagel", + "pretzel", + "cheeseburger", + "hotdog", + "mashed potato", + "head cabbage", + "broccoli", + "cauliflower", + "zucchini", + "spaghetti squash", + "acorn squash", + "butternut squash", + "cucumber", + "artichoke", + "bell pepper", + "cardoon", + "mushroom", + "Granny Smith", + "strawberry", + "orange", + "lemon", + "fig", + "pineapple", + "banana", + "jackfruit", + "custard apple", + "pomegranate", + "hay", + "carbonara", + "chocolate sauce", + "dough", + "meat loaf", + "pizza", + "potpie", + "burrito", + "red wine", + "espresso", + "cup", + "eggnog", + "alp", + "bubble", + "cliff", + "coral reef", + "geyser", + "lakeside", + "promontory", + "sandbar", + "seashore", + "valley", + "volcano", + "ballplayer", + "groom", + "scuba diver", + "rapeseed", + "daisy", + "yellow lady's slipper", + "corn", + "acorn", + "hip", + "buckeye", + "coral fungus", + "agaric", + "gyromitra", + "stinkhorn", + "earthstar", + "hen-of-the-woods", + "bolete", + "ear", + "toilet tissue", +]; diff --git a/resnet-burn/src/model/mod.rs b/resnet-burn/src/model/mod.rs index dcbbb7a..3a220b8 100644 --- a/resnet-burn/src/model/mod.rs +++ b/resnet-burn/src/model/mod.rs @@ -1,2 +1,3 @@ mod block; +pub mod imagenet; pub mod resnet; diff --git a/resnet-burn/src/model/resnet.rs b/resnet-burn/src/model/resnet.rs index f06665b..6996d4c 100644 --- a/resnet-burn/src/model/resnet.rs +++ b/resnet-burn/src/model/resnet.rs @@ -47,10 +47,10 @@ impl ResNet { .init(); // Residual blocks - let layer1 = LayerBlock::new(blocks[0], 64, 64); - let layer2 = LayerBlock::new(blocks[1], 64, 128); - let layer3 = LayerBlock::new(blocks[2], 128, 256); - let layer4 = LayerBlock::new(blocks[2], 256, 512); + let layer1 = LayerBlock::new(blocks[0], 64, 64, 1); + let layer2 = LayerBlock::new(blocks[1], 64, 128, 2); + let layer3 = LayerBlock::new(blocks[2], 128, 256, 2); + let layer4 = LayerBlock::new(blocks[3], 256, 512, 2); // Average pooling [B, 512, H, W] -> [B, 512, 1, 1] let avgpool = AdaptiveAvgPool2dConfig::new([1, 1]).init(); @@ -97,7 +97,6 @@ impl ResNet { let out = self.avgpool.forward(out); // Reshape [B, C, 1, 1] -> [B, C] - // println!("Flatten in: {:?}", out.shape()); // let out = out.flatten(2, 3); let out: Tensor = out.squeeze(3); let out: Tensor = out.squeeze(2); From 3440d3de6852c31bb9eb342a60936d9755ccef32 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 18 Jan 2024 13:12:47 -0500 Subject: [PATCH 3/9] Add resnet-{50, 101, 152} implementations w/ generic LayerBlock --- resnet-burn/Cargo.toml | 4 +- resnet-burn/examples/inference.rs | 28 +++-- resnet-burn/src/model/block.rs | 178 +++++++++++++++++++++++++----- resnet-burn/src/model/imagenet.rs | 14 +-- resnet-burn/src/model/resnet.rs | 128 ++++++++++++++++----- 5 files changed, 278 insertions(+), 74 deletions(-) diff --git a/resnet-burn/Cargo.toml b/resnet-burn/Cargo.toml index b2bd614..7644a25 100644 --- a/resnet-burn/Cargo.toml +++ b/resnet-burn/Cargo.toml @@ -11,12 +11,12 @@ default = ["burn/default"] [dependencies] # Note: default-features = false is needed to disable std -burn = { version = "0.11.1", default-features = false } +burn = { path = "../../burn/burn", default-features = false } serde = { version = "1.0.192", default-features = false, features = [ "derive", "alloc", ] } # alloc is for no_std, derive is needed [dev-dependencies] -burn-ndarray = { version = "0.11.1", package = "burn-ndarray" } +burn-ndarray = { path = "../../burn/burn-ndarray", package = "burn-ndarray" } image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs index 4b209a4..75ebfaf 100644 --- a/resnet-burn/examples/inference.rs +++ b/resnet-burn/examples/inference.rs @@ -4,17 +4,16 @@ use resnet_burn::model::{imagenet, resnet::ResNet}; use burn::{ module::Module, - record::{FullPrecisionSettings, NamedMpkGzFileRecorder}, + record::{FullPrecisionSettings, NamedMpkFileRecorder}, tensor::Tensor, }; use burn_ndarray::NdArray; use image::{self, GenericImageView, Pixel}; +const NUM_CLASSES: usize = 1000; const HEIGHT: usize = 224; const WIDTH: usize = 224; -type Backend = NdArray; - pub fn main() { // Load image let img_path = std::env::args().nth(1).expect("No image path provided"); @@ -42,21 +41,25 @@ pub fn main() { } } + let device = &Default::default(); + // Create a tensor from the array - let image_input = Tensor::::from_data(img_array).reshape([1, 3, HEIGHT, WIDTH]); - let recorder = NamedMpkGzFileRecorder::::new(); + let image_input = + Tensor::, 3>::from_data(img_array, device).reshape([1, 3, HEIGHT, WIDTH]); + let recorder = NamedMpkFileRecorder::::new(); // Normalize the image - let x = imagenet::Normalizer::new().normalize(image_input); + let x = imagenet::Normalizer::new(device).normalize(image_input); - // Load pre-trained ResNet-18 + // Load pre-trained ResNet let model_path = Path::new(file!()) .parent() .unwrap() .parent() .unwrap() .join("model/resnet18-ImageNet1k"); - let model = ResNet::::resnet18(1000) + let model = ResNet::resnet18(NUM_CLASSES, device); + let model = model .load_file(model_path.clone(), &recorder) .unwrap_or_else(|_| panic!("Failed to load model file: {}", model_path.display())); @@ -73,4 +76,13 @@ pub fn main() { idx, score.into_scalar() ); + + // let recorder = NamedMpkFileRecorder::::new(); + // // let recorder = PrettyJsonFileRecorder::::new(); + // let now = Instant::now(); + // model + // .save_file("resnet152.mpk", &recorder) + // .expect("Could not save model to file with full precision settings."); + // let elapsed = now.elapsed(); + // println!("Model save took {} seconds.", elapsed.as_millis()); } diff --git a/resnet-burn/src/model/block.rs b/resnet-burn/src/model/block.rs index 16dafe8..ffcbbe3 100644 --- a/resnet-burn/src/model/block.rs +++ b/resnet-burn/src/model/block.rs @@ -1,3 +1,5 @@ +use core::marker::PhantomData; + use alloc::vec::Vec; use burn::{ @@ -6,13 +8,18 @@ use burn::{ conv::{Conv2d, Conv2dConfig}, BatchNorm, BatchNormConfig, Initializer, PaddingConfig2d, ReLU, }, - tensor::{backend::Backend, Tensor}, + tensor::{backend::Backend, Device, Tensor}, }; -/// ResNet basic residual block implementation. -/// Derived from [torchivision.models.resnet.BasicBlock](https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py) +pub trait ResidualBlock { + fn new(in_channels: usize, out_channels: usize, stride: usize, device: &Device) -> Self; + fn forward(&self, input: Tensor) -> Tensor; +} + +/// ResNet [basic residual block](https://paperswithcode.com/method/residual-block) implementation. +/// Derived from [torchivision.models.resnet.BasicBlock](https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py). #[derive(Module, Debug)] -pub struct ResidualBlock { +pub struct BasicBlock { conv1: Conv2d, bn1: BatchNorm, relu: ReLU, @@ -21,8 +28,8 @@ pub struct ResidualBlock { downsample: Option>, } -impl ResidualBlock { - pub fn new(in_channels: usize, out_channels: usize, stride: usize) -> Self { +impl ResidualBlock for BasicBlock { + fn new(in_channels: usize, out_channels: usize, stride: usize, device: &Device) -> Self { // conv3x3 let conv1 = Conv2dConfig::new([in_channels, out_channels], [3, 3]) .with_stride([stride, stride]) @@ -30,10 +37,10 @@ impl ResidualBlock { .with_bias(false) .with_initializer(Initializer::KaimingNormal { gain: f64::sqrt(2.0), // recommended value for ReLU - fan_out_only: false, // TODO: switch to true when fixed in burn + fan_out_only: true, }) - .init(); - let bn1 = BatchNormConfig::new(out_channels).init(); + .init(device); + let bn1 = BatchNormConfig::new(out_channels).init(device); let relu = ReLU::new(); // conv3x3 let conv2 = Conv2dConfig::new([out_channels, out_channels], [3, 3]) @@ -42,14 +49,14 @@ impl ResidualBlock { .with_bias(false) .with_initializer(Initializer::KaimingNormal { gain: f64::sqrt(2.0), // recommended value for ReLU - fan_out_only: false, // TODO: switch to true when fixed in burn + fan_out_only: true, }) - .init(); - let bn2 = BatchNormConfig::new(out_channels).init(); + .init(device); + let bn2 = BatchNormConfig::new(out_channels).init(device); let downsample = { if in_channels != out_channels { - Some(Downsample::new(in_channels, out_channels, stride)) + Some(Downsample::new(in_channels, out_channels, stride, device)) } else { None } @@ -64,8 +71,7 @@ impl ResidualBlock { downsample, } } - - pub fn forward(&self, input: Tensor) -> Tensor { + fn forward(&self, input: Tensor) -> Tensor { let identity = input.clone(); // Conv block @@ -90,7 +96,7 @@ impl ResidualBlock { } } -/// Downsample layer applies a 1x1 conv to reduce the resolution [H, W] and adjust the number of channels. +/// Downsample layer applies a 1x1 conv to reduce the resolution (H, W) and adjust the number of channels. #[derive(Module, Debug)] pub struct Downsample { conv: Conv2d, @@ -98,18 +104,18 @@ pub struct Downsample { } impl Downsample { - pub fn new(in_channels: usize, out_channels: usize, stride: usize) -> Self { - // conv1x1 (default padding = valid) + pub fn new(in_channels: usize, out_channels: usize, stride: usize, device: &Device) -> Self { + // conv1x1 let conv = Conv2dConfig::new([in_channels, out_channels], [1, 1]) .with_stride([stride, stride]) .with_padding(PaddingConfig2d::Explicit(0, 0)) .with_bias(false) .with_initializer(Initializer::KaimingNormal { gain: f64::sqrt(2.0), // recommended value for ReLU - fan_out_only: false, // TODO: switch to true when fixed in burn + fan_out_only: true, }) - .init(); - let bn = BatchNormConfig::new(out_channels).init(); + .init(device); + let bn = BatchNormConfig::new(out_channels).init(device); Self { conv, bn } } @@ -122,27 +128,143 @@ impl Downsample { } } +/// ResNet [bottleneck residual block](https://paperswithcode.com/method/bottleneck-residual-block) +/// implementation. +/// Derived from [torchivision.models.resnet.Bottleneck](https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py). +/// +/// **NOTE:** Following common practice, this bottleneck block places the stride for downsampling +/// to the second 3x3 convolution while the original paper places it to the first 1x1 convolution. +/// This variant improves the accuracy and is known as [ResNet V1.5](https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch). +#[derive(Module, Debug)] +pub struct Bottleneck { + conv1: Conv2d, + bn1: BatchNorm, + relu: ReLU, + conv2: Conv2d, + bn2: BatchNorm, + conv3: Conv2d, + bn3: BatchNorm, + downsample: Option>, +} + +impl ResidualBlock for Bottleneck { + fn new(in_channels: usize, out_channels: usize, stride: usize, device: &Device) -> Self { + // Intermediate output channels w/ expansion = 4 + let int_out_channels = out_channels / 4; + // conv1x1 + let conv1 = Conv2dConfig::new([in_channels, int_out_channels], [1, 1]) + .with_stride([1, 1]) + .with_padding(PaddingConfig2d::Explicit(0, 0)) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: true, + }) + .init(device); + let bn1 = BatchNormConfig::new(int_out_channels).init(device); + let relu = ReLU::new(); + // conv3x3 + let conv2 = Conv2dConfig::new([int_out_channels, int_out_channels], [3, 3]) + .with_stride([stride, stride]) + .with_padding(PaddingConfig2d::Explicit(1, 1)) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: true, + }) + .init(device); + let bn2 = BatchNormConfig::new(int_out_channels).init(device); + // conv1x1 + let conv3 = Conv2dConfig::new([int_out_channels, out_channels], [1, 1]) + .with_stride([1, 1]) + .with_padding(PaddingConfig2d::Explicit(0, 0)) + .with_bias(false) + .with_initializer(Initializer::KaimingNormal { + gain: f64::sqrt(2.0), // recommended value for ReLU + fan_out_only: true, + }) + .init(device); + let bn3 = BatchNormConfig::new(out_channels).init(device); + + let downsample = { + if in_channels != out_channels { + Some(Downsample::new(in_channels, out_channels, stride, device)) + } else { + None + } + }; + + Self { + conv1, + bn1, + relu, + conv2, + bn2, + conv3, + bn3, + downsample, + } + } + + fn forward(&self, input: Tensor) -> Tensor { + let identity = input.clone(); + + // Conv block + let out = self.conv1.forward(input); + let out = self.bn1.forward(out); + let out = self.relu.forward(out); + let out = self.conv2.forward(out); + let out = self.bn2.forward(out); + let out = self.relu.forward(out); + let out = self.conv3.forward(out); + let out = self.bn3.forward(out); + + // Skip connection + let out = { + match &self.downsample { + Some(downsample) => out + downsample.forward(identity), + None => out + identity, + } + }; + + // Activation + let out = self.relu.forward(out); + + out + } +} + /// Collection of sequential residual blocks. #[derive(Module, Debug)] -pub struct LayerBlock { - blocks: Vec>, +pub struct LayerBlock { + blocks: Vec, + _backend: PhantomData, } -impl LayerBlock { - pub fn new(num_blocks: usize, in_channels: usize, out_channels: usize, stride: usize) -> Self { +impl> LayerBlock { + pub fn new( + num_blocks: usize, + in_channels: usize, + out_channels: usize, + stride: usize, + device: &Device, + ) -> Self { let blocks = (0..num_blocks) .map(|b| { if b == 0 { // First block uses the specified stride - ResidualBlock::new(in_channels, out_channels, stride) + M::new(in_channels, out_channels, stride, device) } else { // Other blocks use a stride of 1 - ResidualBlock::new(out_channels, out_channels, 1) + M::new(out_channels, out_channels, 1, device) } }) .collect(); - Self { blocks } + Self { + blocks, + _backend: PhantomData, + } } pub fn forward(&self, input: Tensor) -> Tensor { let mut out = input; diff --git a/resnet-burn/src/model/imagenet.rs b/resnet-burn/src/model/imagenet.rs index 2240f63..3dd569b 100644 --- a/resnet-burn/src/model/imagenet.rs +++ b/resnet-burn/src/model/imagenet.rs @@ -1,4 +1,4 @@ -use burn::tensor::{backend::Backend, Tensor}; +use burn::tensor::{backend::Backend, Device, Tensor}; // ResNet18_Weights.DEFAULT.transforms() // NOTE: we simply resize to 256 instead of resize + center crop @@ -20,9 +20,9 @@ pub struct Normalizer { impl Normalizer { /// Creates a new normalizer. - pub fn new() -> Self { - let mean = Tensor::from_floats(MEAN).reshape([1, 3, 1, 1]); - let std = Tensor::from_floats(STD).reshape([1, 3, 1, 1]); + pub fn new(device: &Device) -> Self { + let mean = Tensor::from_floats(MEAN, device).reshape([1, 3, 1, 1]); + let std = Tensor::from_floats(STD, device).reshape([1, 3, 1, 1]); Self { mean, std } } @@ -38,12 +38,6 @@ impl Normalizer { } } -impl Default for Normalizer { - fn default() -> Self { - Self::new() - } -} - // ImageNet categories pub const CLASSES: &'static [&'static str; 1000] = &[ "tench", diff --git a/resnet-burn/src/model/resnet.rs b/resnet-burn/src/model/resnet.rs index 6996d4c..f007391 100644 --- a/resnet-burn/src/model/resnet.rs +++ b/resnet-burn/src/model/resnet.rs @@ -5,29 +5,35 @@ use burn::{ pool::{AdaptiveAvgPool2d, AdaptiveAvgPool2dConfig, MaxPool2d, MaxPool2dConfig}, BatchNorm, BatchNormConfig, Initializer, Linear, LinearConfig, PaddingConfig2d, ReLU, }, - tensor::{backend::Backend, Tensor}, + tensor::{backend::Backend, Device, Tensor}, }; -use super::block::LayerBlock; +use super::block::{BasicBlock, Bottleneck, LayerBlock, ResidualBlock}; /// ResNet implementation. /// Derived from [torchivision.models.resnet.ResNet](https://github.com/pytorch/vision/blob/main/torchvision/models/resnet.py) #[derive(Module, Debug)] -pub struct ResNet { +pub struct ResNet { conv1: Conv2d, bn1: BatchNorm, relu: ReLU, maxpool: MaxPool2d, - layer1: LayerBlock, - layer2: LayerBlock, - layer3: LayerBlock, - layer4: LayerBlock, + layer1: LayerBlock, + layer2: LayerBlock, + layer3: LayerBlock, + layer4: LayerBlock, avgpool: AdaptiveAvgPool2d, fc: Linear, } -impl ResNet { - fn new(blocks: [usize; 4], num_classes: usize) -> Self { +impl> ResNet { + fn new(blocks: [usize; 4], num_classes: usize, expansion: usize, device: &Device) -> Self { + // `new()` is private but still check just in case... + assert!( + expansion == 1 || expansion == 4, + "ResNet module only supports expansion values [1, 4] for residual blocks" + ); + // 7x7 conv, 64, /2 let conv1 = Conv2dConfig::new([3, 64], [7, 7]) .with_stride([2, 2]) @@ -37,8 +43,8 @@ impl ResNet { gain: f64::sqrt(2.0), // recommended value for ReLU fan_out_only: false, // TODO: switch to true when fixed in burn }) - .init(); - let bn1 = BatchNormConfig::new(64).init(); + .init(device); + let bn1 = BatchNormConfig::new(64).init(device); let relu = ReLU::new(); // 3x3 maxpool, /2 let maxpool = MaxPool2dConfig::new([3, 3]) @@ -47,16 +53,16 @@ impl ResNet { .init(); // Residual blocks - let layer1 = LayerBlock::new(blocks[0], 64, 64, 1); - let layer2 = LayerBlock::new(blocks[1], 64, 128, 2); - let layer3 = LayerBlock::new(blocks[2], 128, 256, 2); - let layer4 = LayerBlock::new(blocks[3], 256, 512, 2); + let layer1 = LayerBlock::new(blocks[0], 64, 64 * expansion, 1, device); + let layer2 = LayerBlock::new(blocks[1], 64 * expansion, 128 * expansion, 2, device); + let layer3 = LayerBlock::new(blocks[2], 128 * expansion, 256 * expansion, 2, device); + let layer4 = LayerBlock::new(blocks[3], 256 * expansion, 512 * expansion, 2, device); // Average pooling [B, 512, H, W] -> [B, 512, 1, 1] let avgpool = AdaptiveAvgPool2dConfig::new([1, 1]).init(); // Output layer - let fc = LinearConfig::new(512, num_classes).init(); + let fc = LinearConfig::new(512 * expansion, num_classes).init(device); Self { conv1, @@ -72,16 +78,6 @@ impl ResNet { } } - pub fn resnet18(num_classes: usize) -> Self { - Self::new([2, 2, 2, 2], num_classes) - } - - pub fn resnet34(num_classes: usize) -> Self { - Self::new([3, 4, 6, 3], num_classes) - } - - // TODO: resnet{50, 101, 152} use different blocks - pub fn forward(&self, input: Tensor) -> Tensor { // First block let out = self.conv1.forward(input); @@ -105,3 +101,83 @@ impl ResNet { out } } + +impl ResNet> { + /// ResNet-18 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). + /// + /// # Arguments + /// + /// * `num_classes`: Number of output classes of the model. + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A ResNet-18 module. + pub fn resnet18(num_classes: usize, device: &Device) -> Self { + Self::new([2, 2, 2, 2], num_classes, 1, device) + } +} + +impl ResNet> { + /// ResNet-34 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). + /// + /// # Arguments + /// + /// * `num_classes`: Number of output classes of the model. + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A ResNet-34 module. + pub fn resnet34(num_classes: usize, device: &Device) -> Self { + Self::new([3, 4, 6, 3], num_classes, 1, device) + } +} + +impl ResNet> { + /// ResNet-50 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). + /// + /// # Arguments + /// + /// * `num_classes`: Number of output classes of the model. + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A ResNet-50 module. + pub fn resnet50(num_classes: usize, device: &Device) -> Self { + Self::new([3, 4, 6, 3], num_classes, 4, device) + } +} + +impl ResNet> { + /// ResNet-101 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). + /// + /// # Arguments + /// + /// * `num_classes`: Number of output classes of the model. + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A ResNet-101 module. + pub fn resnet101(num_classes: usize, device: &Device) -> Self { + Self::new([3, 4, 23, 3], num_classes, 4, device) + } +} + +impl ResNet> { + /// ResNet-152 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). + /// + /// # Arguments + /// + /// * `num_classes`: Number of output classes of the model. + /// * `device` - Device to create the module on. + /// + /// # Returns + /// + /// A ResNet-152 module. + pub fn resnet152(num_classes: usize, device: &Device) -> Self { + Self::new([3, 8, 36, 3], num_classes, 4, device) + } +} From 509288d01bbd3fdcf014eac9b906b307b5514a40 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 18 Jan 2024 13:16:51 -0500 Subject: [PATCH 4/9] Remove comment block --- resnet-burn/examples/inference.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs index 75ebfaf..afc8325 100644 --- a/resnet-burn/examples/inference.rs +++ b/resnet-burn/examples/inference.rs @@ -76,13 +76,4 @@ pub fn main() { idx, score.into_scalar() ); - - // let recorder = NamedMpkFileRecorder::::new(); - // // let recorder = PrettyJsonFileRecorder::::new(); - // let now = Instant::now(); - // model - // .save_file("resnet152.mpk", &recorder) - // .expect("Could not save model to file with full precision settings."); - // let elapsed = now.elapsed(); - // println!("Model save took {} seconds.", elapsed.as_millis()); } From c3f99c8690622b4c363aaaefb41b11a14b25be28 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 18 Jan 2024 13:30:30 -0500 Subject: [PATCH 5/9] Switch remaining fan_out_only initializer option --- resnet-burn/src/model/resnet.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resnet-burn/src/model/resnet.rs b/resnet-burn/src/model/resnet.rs index f007391..dc5feec 100644 --- a/resnet-burn/src/model/resnet.rs +++ b/resnet-burn/src/model/resnet.rs @@ -41,7 +41,7 @@ impl> ResNet { .with_bias(false) .with_initializer(Initializer::KaimingNormal { gain: f64::sqrt(2.0), // recommended value for ReLU - fan_out_only: false, // TODO: switch to true when fixed in burn + fan_out_only: true, }) .init(device); let bn1 = BatchNormConfig::new(64).init(device); From 7401d4bc2279421872b5dc6b28a7f1e8407799a0 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Tue, 30 Jan 2024 14:34:47 -0500 Subject: [PATCH 6/9] Add device parameter to load_file() --- resnet-burn/examples/inference.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs index afc8325..d3c801c 100644 --- a/resnet-burn/examples/inference.rs +++ b/resnet-burn/examples/inference.rs @@ -26,7 +26,7 @@ pub fn main() { image::imageops::FilterType::Triangle, // also known as bilinear in 2D ); - // 3d array of 224x224x3 floats + // 3d array of 3x224x224 floats let mut img_array = [[[0.0; WIDTH]; HEIGHT]; 3]; // Iterate over the pixels and populate the array @@ -60,7 +60,7 @@ pub fn main() { .join("model/resnet18-ImageNet1k"); let model = ResNet::resnet18(NUM_CLASSES, device); let model = model - .load_file(model_path.clone(), &recorder) + .load_file(model_path.clone(), &recorder, device) .unwrap_or_else(|_| panic!("Failed to load model file: {}", model_path.display())); // Forward pass From f7ed5c5bcec48af6ef4bf3714e65079499ed2bdc Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Tue, 6 Feb 2024 15:59:49 -0500 Subject: [PATCH 7/9] WIP PyTorchFileRecorder weights import --- resnet-burn/Cargo.toml | 10 ++- resnet-burn/README.md | 10 ++- resnet-burn/examples/inference.rs | 112 ++++++++++++------------------ resnet-burn/src/model/block.rs | 14 ++-- resnet-burn/src/model/resnet.rs | 17 ++--- 5 files changed, 67 insertions(+), 96 deletions(-) diff --git a/resnet-burn/Cargo.toml b/resnet-burn/Cargo.toml index 7644a25..d50e9e3 100644 --- a/resnet-burn/Cargo.toml +++ b/resnet-burn/Cargo.toml @@ -11,12 +11,16 @@ default = ["burn/default"] [dependencies] # Note: default-features = false is needed to disable std -burn = { path = "../../burn/burn", default-features = false } +burn = { version = "0.12.1", default-features = false} +# burn = { path = "../../burn/burn", default-features = false} serde = { version = "1.0.192", default-features = false, features = [ "derive", "alloc", ] } # alloc is for no_std, derive is needed [dev-dependencies] -burn-ndarray = { path = "../../burn/burn-ndarray", package = "burn-ndarray" } -image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file +burn = { version = "0.12.1", default-features = false, features = ["ndarray"]} +burn-import = "0.12.1" +# burn = { path = "../../burn/burn", default-features = false} +# burn-import = {path = "../../burn/burn-import"} +# image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file diff --git a/resnet-burn/README.md b/resnet-burn/README.md index ff1e141..9641205 100644 --- a/resnet-burn/README.md +++ b/resnet-burn/README.md @@ -1,6 +1,8 @@ # ResNet Burn -To this day, [ResNet](https://arxiv.org/abs/1512.03385)s are still a strong baseline for your image classification tasks. You can find the [Burn](https://github.com/tracel-ai/burn) implementation for the ResNet variants in [src/model/resnet.rs](src/model/resnet.rs). +To this day, [ResNet](https://arxiv.org/abs/1512.03385)s are still a strong baseline for your image +classification tasks. You can find the [Burn](https://github.com/tracel-ai/burn) implementation for +the ResNet variants in [src/model/resnet.rs](src/model/resnet.rs). The model is [no_std compatible](https://docs.rust-embedded.org/book/intro/no-std.html). @@ -17,8 +19,10 @@ resnet-burn = { git = "https://github.com/burn-rs/models", package = "resnet-bur ### Example Usage -The [inference example](examples/inference.rs) initializes a ResNet-18 with the `NdArray` backend, loads the ImageNet [pre-trained weights](model/) and performs inference on the provided input image. You can also run it yourself with the following command: +The [inference example](examples/inference.rs) initializes a ResNet-18 with the `NdArray` backend, +loads the ImageNet [pre-trained weights](model/) and performs inference on the provided input image. +You can also run it yourself with the following command: ```sh cargo run --release --example inference samples/dog.jpg -``` \ No newline at end of file +``` diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs index d3c801c..541c372 100644 --- a/resnet-burn/examples/inference.rs +++ b/resnet-burn/examples/inference.rs @@ -1,79 +1,53 @@ -use std::path::Path; - -use resnet_burn::model::{imagenet, resnet::ResNet}; +use resnet_burn::model::resnet::ResNet; use burn::{ + backend::{ndarray::NdArrayDevice, NdArray}, module::Module, - record::{FullPrecisionSettings, NamedMpkFileRecorder}, - tensor::Tensor, + record::{FullPrecisionSettings, NamedMpkFileRecorder, Recorder}, }; -use burn_ndarray::NdArray; -use image::{self, GenericImageView, Pixel}; +use burn_import::pytorch::{LoadArgs, PyTorchFileRecorder}; +const MODEL_PATH: &str = "resnet18-ImageNet1k"; const NUM_CLASSES: usize = 1000; -const HEIGHT: usize = 224; -const WIDTH: usize = 224; pub fn main() { - // Load image - let img_path = std::env::args().nth(1).expect("No image path provided"); - let img = image::open(&img_path).unwrap_or_else(|_| panic!("Failed to load image: {img_path}")); - - // Resize to 224x224 - let resized_img = img.resize_exact( - WIDTH as u32, - HEIGHT as u32, - image::imageops::FilterType::Triangle, // also known as bilinear in 2D - ); - - // 3d array of 3x224x224 floats - let mut img_array = [[[0.0; WIDTH]; HEIGHT]; 3]; - - // Iterate over the pixels and populate the array - for y in 0..HEIGHT { - for x in 0..WIDTH { - let pixel = resized_img.get_pixel(x as u32, y as u32); - let rgb = pixel.to_rgb(); - - img_array[0][y][x] = rgb[0] as f32 / 255.0; - img_array[1][y][x] = rgb[1] as f32 / 255.0; - img_array[2][y][x] = rgb[2] as f32 / 255.0; - } - } - - let device = &Default::default(); - - // Create a tensor from the array - let image_input = - Tensor::, 3>::from_data(img_array, device).reshape([1, 3, HEIGHT, WIDTH]); + // Create ResNet-18 + // let device = Default::default(); + let device = NdArrayDevice::default(); + let model: ResNet = ResNet::resnet18(NUM_CLASSES, &device); + + // Load weights from torch state_dict + // NOTE: the remap chain below only works if you remove the `break` in `remap` + // https://github.com/tracel-ai/burn/blob/main/burn-core/src/record/serde/data.rs#L180 + // Download URL: https://download.pytorch.org/models/resnet18-f37072fd.pth + let load_args = LoadArgs::new("resnet18-f37072fd.pth".into()) + // Map *.downsample.0.* -> *.downsample.conv.* + .with_key_remap("(.+)\\.downsample\\.0\\.(.+)", "$1.downsample.conv.$2") + // Map *.downsample.1.* -> *.downsample.bn.* + .with_key_remap("(.+)\\.downsample\\.1\\.(.+)", "$1.downsample.bn.$2") + // Map layer[i].[j].* -> layer[i].blocks.[j].* + .with_key_remap("layer[1-4]\\.([0-9])\\.(.+)", "layer$1.blocks.$2.$3"); + println!("Loading record w/ key remap"); + let record = PyTorchFileRecorder::::new() + .load(load_args, &device) + .map_err(|err| format!("Failed to load weights.\nError: {err}")) + .unwrap(); + // .expect("Should load PyTorch model weights correctly"); + + println!("Loading record into model"); + let model = model.load_record(record); + + // Save the model to a supported format and load it back + println!("Saving the model to Named MessagePack"); let recorder = NamedMpkFileRecorder::::new(); - - // Normalize the image - let x = imagenet::Normalizer::new(device).normalize(image_input); - - // Load pre-trained ResNet - let model_path = Path::new(file!()) - .parent() - .unwrap() - .parent() - .unwrap() - .join("model/resnet18-ImageNet1k"); - let model = ResNet::resnet18(NUM_CLASSES, device); - let model = model - .load_file(model_path.clone(), &recorder, device) - .unwrap_or_else(|_| panic!("Failed to load model file: {}", model_path.display())); - - // Forward pass - let out = model.forward(x); - - // Output class index w/ score (raw) - let (score, idx) = out.max_dim_with_indices(1); - let idx = idx.into_scalar() as usize; - - println!( - "Predicted: {}\nCategory Id: {}\nScore: {:.4}", - imagenet::CLASSES[idx], - idx, - score.into_scalar() - ); + model + .clone() // `save_file` takes ownership but we want to load the file after + .save_file(MODEL_PATH, &recorder) + .expect(&format!( + "Should be able to save weights to file {MODEL_PATH}" + )); + let _ = model + .load_file(MODEL_PATH, &recorder, &device) + .map_err(|err| format!("Failed to load weights from file {MODEL_PATH}.\nError: {err}")) + .unwrap(); } diff --git a/resnet-burn/src/model/block.rs b/resnet-burn/src/model/block.rs index ffcbbe3..1d238cb 100644 --- a/resnet-burn/src/model/block.rs +++ b/resnet-burn/src/model/block.rs @@ -71,6 +71,7 @@ impl ResidualBlock for BasicBlock { downsample, } } + fn forward(&self, input: Tensor) -> Tensor { let identity = input.clone(); @@ -90,9 +91,7 @@ impl ResidualBlock for BasicBlock { }; // Activation - let out = self.relu.forward(out); - - out + self.relu.forward(out) } } @@ -122,9 +121,7 @@ impl Downsample { pub fn forward(&self, input: Tensor) -> Tensor { let out = self.conv.forward(input); - let out = self.bn.forward(out); - - out + self.bn.forward(out) } } @@ -228,9 +225,7 @@ impl ResidualBlock for Bottleneck { }; // Activation - let out = self.relu.forward(out); - - out + self.relu.forward(out) } } @@ -266,6 +261,7 @@ impl> LayerBlock { _backend: PhantomData, } } + pub fn forward(&self, input: Tensor) -> Tensor { let mut out = input; for block in &self.blocks { diff --git a/resnet-burn/src/model/resnet.rs b/resnet-burn/src/model/resnet.rs index dc5feec..d65f22c 100644 --- a/resnet-burn/src/model/resnet.rs +++ b/resnet-burn/src/model/resnet.rs @@ -58,7 +58,7 @@ impl> ResNet { let layer3 = LayerBlock::new(blocks[2], 128 * expansion, 256 * expansion, 2, device); let layer4 = LayerBlock::new(blocks[3], 256 * expansion, 512 * expansion, 2, device); - // Average pooling [B, 512, H, W] -> [B, 512, 1, 1] + // Average pooling [B, 512 * expansion, H, W] -> [B, 512 * expansion, 1, 1] let avgpool = AdaptiveAvgPool2dConfig::new([1, 1]).init(); // Output layer @@ -93,12 +93,11 @@ impl> ResNet { let out = self.avgpool.forward(out); // Reshape [B, C, 1, 1] -> [B, C] - // let out = out.flatten(2, 3); - let out: Tensor = out.squeeze(3); - let out: Tensor = out.squeeze(2); - let out = self.fc.forward(out); + let out = out.flatten(1, 3); + // let out: Tensor = out.squeeze(3); + // let out: Tensor = out.squeeze(2); - out + self.fc.forward(out) } } @@ -116,9 +115,7 @@ impl ResNet> { pub fn resnet18(num_classes: usize, device: &Device) -> Self { Self::new([2, 2, 2, 2], num_classes, 1, device) } -} -impl ResNet> { /// ResNet-34 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). /// /// # Arguments @@ -148,9 +145,7 @@ impl ResNet> { pub fn resnet50(num_classes: usize, device: &Device) -> Self { Self::new([3, 4, 6, 3], num_classes, 4, device) } -} -impl ResNet> { /// ResNet-101 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). /// /// # Arguments @@ -164,9 +159,7 @@ impl ResNet> { pub fn resnet101(num_classes: usize, device: &Device) -> Self { Self::new([3, 4, 23, 3], num_classes, 4, device) } -} -impl ResNet> { /// ResNet-152 from [`Deep Residual Learning for Image Recognition`](https://arxiv.org/abs/1512.03385). /// /// # Arguments From c63c3782af6ff1187d3ccae6c735b1fd553e8b69 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 8 Feb 2024 09:40:30 -0500 Subject: [PATCH 8/9] Finalize inference example --- resnet-burn/Cargo.toml | 11 ++--- resnet-burn/README.md | 9 +++- resnet-burn/examples/inference.rs | 81 ++++++++++++++++++++++++------- resnet-burn/src/model/imagenet.rs | 2 +- 4 files changed, 76 insertions(+), 27 deletions(-) diff --git a/resnet-burn/Cargo.toml b/resnet-burn/Cargo.toml index d50e9e3..8d97bc5 100644 --- a/resnet-burn/Cargo.toml +++ b/resnet-burn/Cargo.toml @@ -11,16 +11,13 @@ default = ["burn/default"] [dependencies] # Note: default-features = false is needed to disable std -burn = { version = "0.12.1", default-features = false} -# burn = { path = "../../burn/burn", default-features = false} +burn = { path = "../../burn/burn", default-features = false} serde = { version = "1.0.192", default-features = false, features = [ "derive", "alloc", ] } # alloc is for no_std, derive is needed [dev-dependencies] -burn = { version = "0.12.1", default-features = false, features = ["ndarray"]} -burn-import = "0.12.1" -# burn = { path = "../../burn/burn", default-features = false} -# burn-import = {path = "../../burn/burn-import"} -# image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file +burn = { path = "../../burn/burn", default-features = false} # local burn to include the with_key_remap chain fix +burn-import = {path = "../../burn/burn-import"} # local burn-import depends on the git version of candle-core +image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file diff --git a/resnet-burn/README.md b/resnet-burn/README.md index 9641205..3cdcdfc 100644 --- a/resnet-burn/README.md +++ b/resnet-burn/README.md @@ -20,8 +20,13 @@ resnet-burn = { git = "https://github.com/burn-rs/models", package = "resnet-bur ### Example Usage The [inference example](examples/inference.rs) initializes a ResNet-18 with the `NdArray` backend, -loads the ImageNet [pre-trained weights](model/) and performs inference on the provided input image. -You can also run it yourself with the following command: +imports the ImageNet pre-trained weights from +[`torchvision`](https://download.pytorch.org/models/resnet18-f37072fd.pth) and performs inference on +the provided input image. + +After downloading the +[pre-trained weights](https://download.pytorch.org/models/resnet18-f37072fd.pth) to the current +directory, you can run the example with the following command: ```sh cargo run --release --example inference samples/dog.jpg diff --git a/resnet-burn/examples/inference.rs b/resnet-burn/examples/inference.rs index 541c372..4562a39 100644 --- a/resnet-burn/examples/inference.rs +++ b/resnet-burn/examples/inference.rs @@ -1,53 +1,100 @@ -use resnet_burn::model::resnet::ResNet; +use resnet_burn::model::{imagenet, resnet::ResNet}; use burn::{ - backend::{ndarray::NdArrayDevice, NdArray}, + backend::NdArray, module::Module, record::{FullPrecisionSettings, NamedMpkFileRecorder, Recorder}, + tensor::{backend::Backend, Data, Device, Element, Shape, Tensor}, }; use burn_import::pytorch::{LoadArgs, PyTorchFileRecorder}; +const TORCH_WEIGHTS: &str = "resnet18-f37072fd.pth"; const MODEL_PATH: &str = "resnet18-ImageNet1k"; const NUM_CLASSES: usize = 1000; +const HEIGHT: usize = 224; +const WIDTH: usize = 224; + +fn to_tensor( + data: Vec, + shape: [usize; 3], + device: &Device, +) -> Tensor { + Tensor::::from_data(Data::new(data, Shape::new(shape)).convert(), device) + // permute(2, 0, 1) + .swap_dims(2, 1) // [H, C, W] + .swap_dims(1, 0) // [C, H, W] + / 255 // normalize between [0, 1] +} pub fn main() { + // Parse arguments + let img_path = std::env::args().nth(1).expect("No image path provided"); + // Create ResNet-18 - // let device = Default::default(); - let device = NdArrayDevice::default(); + let device = Default::default(); let model: ResNet = ResNet::resnet18(NUM_CLASSES, &device); // Load weights from torch state_dict - // NOTE: the remap chain below only works if you remove the `break` in `remap` - // https://github.com/tracel-ai/burn/blob/main/burn-core/src/record/serde/data.rs#L180 - // Download URL: https://download.pytorch.org/models/resnet18-f37072fd.pth - let load_args = LoadArgs::new("resnet18-f37072fd.pth".into()) + let load_args = LoadArgs::new(TORCH_WEIGHTS.into()) // Map *.downsample.0.* -> *.downsample.conv.* .with_key_remap("(.+)\\.downsample\\.0\\.(.+)", "$1.downsample.conv.$2") // Map *.downsample.1.* -> *.downsample.bn.* .with_key_remap("(.+)\\.downsample\\.1\\.(.+)", "$1.downsample.bn.$2") // Map layer[i].[j].* -> layer[i].blocks.[j].* - .with_key_remap("layer[1-4]\\.([0-9])\\.(.+)", "layer$1.blocks.$2.$3"); - println!("Loading record w/ key remap"); + .with_key_remap("(layer[1-4])\\.([0-9])\\.(.+)", "$1.blocks.$2.$3"); let record = PyTorchFileRecorder::::new() .load(load_args, &device) .map_err(|err| format!("Failed to load weights.\nError: {err}")) .unwrap(); - // .expect("Should load PyTorch model weights correctly"); - println!("Loading record into model"); let model = model.load_record(record); // Save the model to a supported format and load it back - println!("Saving the model to Named MessagePack"); let recorder = NamedMpkFileRecorder::::new(); model .clone() // `save_file` takes ownership but we want to load the file after .save_file(MODEL_PATH, &recorder) - .expect(&format!( - "Should be able to save weights to file {MODEL_PATH}" - )); - let _ = model + .map_err(|err| format!("Failed to save weights to file {MODEL_PATH}.\nError: {err}")) + .unwrap(); + let model = model .load_file(MODEL_PATH, &recorder, &device) .map_err(|err| format!("Failed to load weights from file {MODEL_PATH}.\nError: {err}")) .unwrap(); + + // Load image + let img = image::open(&img_path) + .map_err(|err| format!("Failed to load image {img_path}.\nError: {err}")) + .unwrap(); + + // Resize to 224x224 + let resized_img = img.resize_exact( + WIDTH as u32, + HEIGHT as u32, + image::imageops::FilterType::Triangle, // also known as bilinear in 2D + ); + + // Create tensor from image data + let img_tensor = to_tensor( + resized_img.into_rgb8().into_raw(), + [HEIGHT, WIDTH, 3], + &device, + ) + .unsqueeze::<4>(); // [B, C, H, W] + + // Normalize the image + let x = imagenet::Normalizer::new(&device).normalize(img_tensor); + + // Forward pass + let out = model.forward(x); + + // Output class index w/ score (raw) + let (score, idx) = out.max_dim_with_indices(1); + let idx = idx.into_scalar() as usize; + + println!( + "Predicted: {}\nCategory Id: {}\nScore: {:.4}", + imagenet::CLASSES[idx], + idx, + score.into_scalar() + ); } diff --git a/resnet-burn/src/model/imagenet.rs b/resnet-burn/src/model/imagenet.rs index 3dd569b..e1dbf1c 100644 --- a/resnet-burn/src/model/imagenet.rs +++ b/resnet-burn/src/model/imagenet.rs @@ -39,7 +39,7 @@ impl Normalizer { } // ImageNet categories -pub const CLASSES: &'static [&'static str; 1000] = &[ +pub const CLASSES: [&str; 1000] = [ "tench", "goldfish", "great white shark", From ec5fe2bf1ab5d8eb9ac66eb426f191c4f1309177 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 8 Feb 2024 13:07:16 -0500 Subject: [PATCH 9/9] Pin burn dependencies to a specific hash in the meantime --- resnet-burn/Cargo.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/resnet-burn/Cargo.toml b/resnet-burn/Cargo.toml index 8d97bc5..2e1a9fc 100644 --- a/resnet-burn/Cargo.toml +++ b/resnet-burn/Cargo.toml @@ -11,13 +11,12 @@ default = ["burn/default"] [dependencies] # Note: default-features = false is needed to disable std -burn = { path = "../../burn/burn", default-features = false} +burn = { git = "https://github.com/tracel-ai/burn.git", rev = "75cb5b6d5633c1c6092cf5046419da75e7f74b11", default-features = false } serde = { version = "1.0.192", default-features = false, features = [ "derive", "alloc", ] } # alloc is for no_std, derive is needed [dev-dependencies] -burn = { path = "../../burn/burn", default-features = false} # local burn to include the with_key_remap chain fix -burn-import = {path = "../../burn/burn-import"} # local burn-import depends on the git version of candle-core -image = { version = "0.24.7", features = ["png", "jpeg"] } \ No newline at end of file +burn-import = { git = "https://github.com/tracel-ai/burn.git", rev = "75cb5b6d5633c1c6092cf5046419da75e7f74b11"} +image = { version = "0.24.7", features = ["png", "jpeg"] }