+
+#### ! Important: The example provided can be tested also by using Amazon SageMaker Notebook Instances
+
+### Prerequisites
+
+1. Required NVIDIA NGC Account. Follow the instruction https://docs.nvidia.com/ngc/ngc-catalog-user-guide/index.html#registering-activating-ngc-account
+
+## Step 1: Clone this repository
+
+## Step 2: Build Studio image
+
+In this example, we provide a [Dokerfile](./studio-image/image_tensorrt/Dockerfile) example to build a custom image for SageMaker Studio.
+
+To build the image, push it and make it available in your Amazon SageMaker Studio environment, edit [sagemaker-studio-config](./studio-image/studio-domain-config.json) by replacing `$DOMAIN_ID` with your Studio domain ID.
+
+We also provide automation scripts in order to [build and push](./studio-image/build_image.sh) your docker image to an ECR repository
+and [create](./studio-image/create_studio_image.sh) or [update](./studio-image/update_studio_image.sh) an Amazon SageMaker Image. Please follow the instructions in the [README](./studio-image/README.md) for additional info on the usage of this script.
+
+## Step 3: Compile model, create an Amazon SageMaker Real-Time Endpoint with NVIDIA Triton Inference Server
+
+Clone this repository into your Amazon SageMaker Studio environment and execute the cells in the [notebook](./examples/triton_sentence_embeddings.ipynb)
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/bert-trt/config.pbtxt b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/bert-trt/config.pbtxt
new file mode 100644
index 0000000000..3bf605dfc4
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/bert-trt/config.pbtxt
@@ -0,0 +1,32 @@
+name: "bert-trt"
+platform: "tensorrt_plan"
+max_batch_size: 16
+input [
+ {
+ name: "token_ids"
+ data_type: TYPE_INT32
+ dims: [128]
+ },
+ {
+ name: "attn_mask"
+ data_type: TYPE_INT32
+ dims: [128]
+ }
+]
+output [
+ {
+ name: "output"
+ data_type: TYPE_FP32
+ dims: [128, 384]
+ },
+ {
+ name: "854"
+ data_type: TYPE_FP32
+ dims: [384]
+ }
+]
+instance_group [
+ {
+ kind: KIND_GPU
+ }
+ ]
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/ensemble/1/README.md b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/ensemble/1/README.md
new file mode 100644
index 0000000000..a8e639ff9d
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/ensemble/1/README.md
@@ -0,0 +1 @@
+Do not delete me!
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/ensemble/config.pbtxt b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/ensemble/config.pbtxt
new file mode 100644
index 0000000000..aa36dd1ded
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/ensemble/config.pbtxt
@@ -0,0 +1,70 @@
+name: "ensemble"
+platform: "ensemble"
+max_batch_size: 16
+input [
+ {
+ name: "INPUT0"
+ data_type: TYPE_STRING
+ dims: [ 1 ]
+ }
+]
+output [
+ {
+ name: "finaloutput"
+ data_type: TYPE_FP32
+ dims: [384]
+ }
+]
+ensemble_scheduling {
+ step [
+ {
+ model_name: "preprocess"
+ model_version: -1
+ input_map {
+ key: "INPUT0"
+ value: "INPUT0"
+ }
+ output_map {
+ key: "OUTPUT0"
+ value: "token_ids"
+ }
+ output_map {
+ key: "OUTPUT1"
+ value: "attn_mask"
+ }
+ },
+ {
+ model_name: "bert-trt"
+ model_version: -1
+ input_map {
+ key: "token_ids"
+ value: "token_ids"
+ }
+ input_map {
+ key: "attn_mask"
+ value: "attn_mask"
+ }
+ output_map {
+ key: "output"
+ value: "output"
+ }
+ },
+ {
+ model_name: "postprocess"
+ model_version: -1
+ input_map {
+ key: "TOKEN_EMBEDS_POST"
+ value: "output"
+ }
+ input_map {
+ key: "ATTENTION_POST"
+ value: "attn_mask"
+ }
+ output_map {
+ key: "SENT_EMBED"
+ value: "finaloutput"
+ }
+
+ }
+ ]
+}
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/1/model.py b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/1/model.py
new file mode 100644
index 0000000000..5373e90dbb
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/1/model.py
@@ -0,0 +1,78 @@
+import json
+import logging
+import numpy as np
+import subprocess
+import sys
+import os
+
+import triton_python_backend_utils as pb_utils
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class TritonPythonModel:
+ """This model loops through different dtypes to make sure that
+ serialize_byte_tensor works correctly in the Python backend.
+ """
+
+ def __mean_pooling(self, token_embeddings, attention_mask):
+ logger.info("token_embeddings: {}".format(token_embeddings))
+ logger.info("attention_mask: {}".format(attention_mask))
+
+ input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+ return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+ def initialize(self, args):
+ self.model_dir = args['model_repository']
+ subprocess.check_call([sys.executable, "-m", "pip", "install", '-r', f'{self.model_dir}/requirements.txt'])
+ global torch
+ import torch
+
+ self.device_id = args['model_instance_device_id']
+ self.model_config = model_config = json.loads(args['model_config'])
+ self.device = torch.device(f'cuda:{self.device_id}') if torch.cuda.is_available() else torch.device('cpu')
+
+ output0_config = pb_utils.get_output_config_by_name(
+ model_config, "SENT_EMBED")
+
+ self.output0_dtype = pb_utils.triton_string_to_numpy(
+ output0_config["data_type"])
+
+ def execute(self, requests):
+
+ responses = []
+
+ for request in requests:
+ tok_embeds = pb_utils.get_input_tensor_by_name(request, "TOKEN_EMBEDS_POST")
+ attn_mask = pb_utils.get_input_tensor_by_name(request, "ATTENTION_POST")
+
+ tok_embeds = tok_embeds.as_numpy()
+
+ logger.info("tok_embeds: {}".format(tok_embeds))
+ logger.info("tok_embeds shape: {}".format(tok_embeds.shape))
+
+ tok_embeds = torch.tensor(tok_embeds,device=self.device)
+
+ logger.info("tok_embeds_tensor: {}".format(tok_embeds))
+
+ attn_mask = attn_mask.as_numpy()
+
+ logger.info("attn_mask: {}".format(attn_mask))
+ logger.info("attn_mask shape: {}".format(attn_mask.shape))
+
+ attn_mask = torch.tensor(attn_mask,device=self.device)
+
+ logger.info("attn_mask_tensor: {}".format(attn_mask))
+
+ sentence_embeddings = self.__mean_pooling(tok_embeds, attn_mask)
+ sentence_embeddings = torch.nn.functional.normalize(sentence_embeddings, p=2, dim=1)
+
+ out_0 = np.array(sentence_embeddings.cpu(),dtype=self.output0_dtype)
+ logger.info("out_0: {}".format(out_0))
+
+ out_tensor_0 = pb_utils.Tensor("SENT_EMBED", out_0)
+ logger.info("out_tensor_0: {}".format(out_tensor_0))
+
+ responses.append(pb_utils.InferenceResponse([out_tensor_0]))
+
+ return responses
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/config.pbtxt b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/config.pbtxt
new file mode 100644
index 0000000000..de573d924d
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/config.pbtxt
@@ -0,0 +1,26 @@
+name: "postprocess"
+backend: "python"
+max_batch_size: 16
+
+input [
+ {
+ name: "TOKEN_EMBEDS_POST"
+ data_type: TYPE_FP32
+ dims: [128, 384]
+
+ },
+ {
+ name: "ATTENTION_POST"
+ data_type: TYPE_INT32
+ dims: [128]
+ }
+]
+output [
+ {
+ name: "SENT_EMBED"
+ data_type: TYPE_FP32
+ dims: [ 384 ]
+ }
+]
+
+instance_group [{ kind: KIND_GPU }]
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/requirements.txt b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/requirements.txt
new file mode 100644
index 0000000000..08ed5eeb4b
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/postprocess/requirements.txt
@@ -0,0 +1 @@
+torch
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/1/model.py b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/1/model.py
new file mode 100644
index 0000000000..47b1f1befc
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/1/model.py
@@ -0,0 +1,74 @@
+import json
+import logging
+import numpy as np
+import subprocess
+import sys
+
+import triton_python_backend_utils as pb_utils
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+class TritonPythonModel:
+ """This model loops through different dtypes to make sure that
+ serialize_byte_tensor works correctly in the Python backend.
+ """
+
+ def initialize(self, args):
+ self.model_dir = args['model_repository']
+ subprocess.check_call([sys.executable, "-m", "pip", "install", '-r', f'{self.model_dir}/requirements.txt'])
+ global transformers
+ import transformers
+
+ self.tokenizer = transformers.AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+ self.model_config = model_config = json.loads(args['model_config'])
+
+ output0_config = pb_utils.get_output_config_by_name(
+ model_config, "OUTPUT0")
+ output1_config = pb_utils.get_output_config_by_name(
+ model_config, "OUTPUT1")
+
+ self.output0_dtype = pb_utils.triton_string_to_numpy(
+ output0_config['data_type'])
+ self.output1_dtype = pb_utils.triton_string_to_numpy(
+ output0_config['data_type'])
+
+ def execute(self, requests):
+
+ file = open("logs.txt", "w")
+
+ responses = []
+ for request in requests:
+ logger.info("Request: {}".format(request))
+
+ in_0 = pb_utils.get_input_tensor_by_name(request, "INPUT0")
+ in_0 = in_0.as_numpy()
+
+ logger.info("in_0: {}".format(in_0))
+
+ tok_batch = []
+
+ for i in range(in_0.shape[0]):
+ decoded_object = in_0[i,0].decode()
+
+ logger.info("decoded_object: {}".format(decoded_object))
+
+ tok_batch.append(decoded_object)
+
+ logger.info("tok_batch: {}".format(tok_batch))
+
+ tok_sent = self.tokenizer(tok_batch,
+ padding='max_length',
+ max_length=128,
+ )
+
+
+ logger.info("Tokens: {}".format(tok_sent))
+
+ out_0 = np.array(tok_sent['input_ids'],dtype=self.output0_dtype)
+ out_1 = np.array(tok_sent['attention_mask'],dtype=self.output1_dtype)
+ out_tensor_0 = pb_utils.Tensor("OUTPUT0", out_0)
+ out_tensor_1 = pb_utils.Tensor("OUTPUT1", out_1)
+
+ responses.append(pb_utils.InferenceResponse([out_tensor_0,out_tensor_1]))
+ return responses
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/config.pbtxt b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/config.pbtxt
new file mode 100644
index 0000000000..c3f70b03dd
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/config.pbtxt
@@ -0,0 +1,28 @@
+name: "preprocess"
+backend: "python"
+max_batch_size: 16
+
+input [
+ {
+ name: "INPUT0"
+ data_type: TYPE_STRING
+ dims: [ 1 ]
+
+ }
+]
+output [
+ {
+ name: "OUTPUT0"
+ data_type: TYPE_INT32
+ dims: [ 128 ]
+ },
+
+ {
+ name: "OUTPUT1"
+ data_type: TYPE_INT32
+ dims: [ 128 ]
+ }
+
+]
+
+instance_group [{ kind: KIND_CPU }]
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/requirements.txt b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/requirements.txt
new file mode 100644
index 0000000000..747b7aa97a
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/ensemble_hf/preprocess/requirements.txt
@@ -0,0 +1 @@
+transformers
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/triton_sentence_embeddings.ipynb b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/triton_sentence_embeddings.ipynb
new file mode 100644
index 0000000000..5fef5c4405
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/triton_sentence_embeddings.ipynb
@@ -0,0 +1,1015 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "ce5723a5",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "# Prerequisites\n",
+ "\n",
+ "Install the necessary Python modules to use and interact with [NVIDIA Triton Inference Server](https://github.com/triton-inference-server/server/)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f5995424",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "! pip install torch==1.10.0 sagemaker transformers==4.9.1 tritonclient[all]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4d9d12fa",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "# Part 1 - Setup"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3aef44c4",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import argparse\n",
+ "import boto3\n",
+ "import copy\n",
+ "import datetime\n",
+ "import json\n",
+ "import numpy as np\n",
+ "import os\n",
+ "import pandas as pd\n",
+ "import pprint\n",
+ "import re\n",
+ "import sagemaker\n",
+ "import sys\n",
+ "import time\n",
+ "from time import gmtime, strftime\n",
+ "import tritonclient.http as http_client"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fe9a1636",
+ "metadata": {
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "session = boto3.Session()\n",
+ "role = sagemaker.get_execution_role()\n",
+ "\n",
+ "sm_client = session.client(\"sagemaker\")\n",
+ "sagemaker_session = sagemaker.Session(boto_session=session)\n",
+ "sm_runtime_client = boto3.client(\"sagemaker-runtime\")\n",
+ "\n",
+ "region = boto3.Session().region_name"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a4996fe6",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "account_id_map = {\n",
+ " \"us-east-1\": \"785573368785\",\n",
+ " \"us-east-2\": \"007439368137\",\n",
+ " \"us-west-1\": \"710691900526\",\n",
+ " \"us-west-2\": \"301217895009\",\n",
+ " \"eu-west-1\": \"802834080501\",\n",
+ " \"eu-west-2\": \"205493899709\",\n",
+ " \"eu-west-3\": \"254080097072\",\n",
+ " \"eu-north-1\": \"601324751636\",\n",
+ " \"eu-south-1\": \"966458181534\",\n",
+ " \"eu-central-1\": \"746233611703\",\n",
+ " \"ap-east-1\": \"110948597952\",\n",
+ " \"ap-south-1\": \"763008648453\",\n",
+ " \"ap-northeast-1\": \"941853720454\",\n",
+ " \"ap-northeast-2\": \"151534178276\",\n",
+ " \"ap-southeast-1\": \"324986816169\",\n",
+ " \"ap-southeast-2\": \"355873309152\",\n",
+ " \"cn-northwest-1\": \"474822919863\",\n",
+ " \"cn-north-1\": \"472730292857\",\n",
+ " \"sa-east-1\": \"756306329178\",\n",
+ " \"ca-central-1\": \"464438896020\",\n",
+ " \"me-south-1\": \"836785723513\",\n",
+ " \"af-south-1\": \"774647643957\",\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "d31659f5",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "***"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "14a0ba73",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "# Part 2 - Generate TensorRT Model\n",
+ "\n",
+ "In the following cells, we are using [HuggingFace Auto Classes](https://huggingface.co/docs/transformers/model_doc/auto) to load a pre-trained model from the [HuggingFace Model Hub](https://huggingface.co/models). We then convert the model to the ONNX format, and compile it using NVIDIA TensorRT - namely its command-line wrapper tool, `trtexec` -, using the scripts provided in the official AWS Sample for [SageMaker Triton](https://github.com/aws/amazon-sagemaker-examples/tree/main/sagemaker-triton).\n",
+ "\n",
+ "NVIDIA TensorRT is an SDK that facilitates high-performance machine learning inference. You can use it to create `engines` from models that have already been trained, \n",
+ "optimizing for a selected GPU architecture. Triton natively supports the TensorRT runtime, which enables you to easily deploy a TensorRT engine and pair it with the rich features that Triton provides.\n",
+ "\n",
+ "### Parameters:\n",
+ "\n",
+ "* `model_name`: Model identifier from the Hugging Face model hub library"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5e3f1883",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "model_id = \"sentence-transformers/all-MiniLM-L6-v2\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "436d115f",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Option 1 - TensorRT Model with Amazon SageMaker Studio"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "629bfade-8f14-44b6-9be7-88a2fdb84ba9",
+ "metadata": {},
+ "source": [
+ "> **WARNING**: The next cell will only work if you have first created a custom Studio image, described in Step 2 of this repository's README. Change the `RUNNING_IN_STUDIO` to `True` if this is the case."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6df2bf7d-a595-46b1-8d1a-43ab946bc858",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "RUNNING_IN_STUDIO = False"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7c0eb376",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "if RUNNING_IN_STUDIO:\n",
+ " !/bin/bash ./workspace/generate_model_trt.sh $model_id && rm -rf ensemble_hf/bert-trt/1 && mkdir -p ensemble_hf/bert-trt/1 && cp ./model.plan ensemble_hf/bert-trt/1/model.plan && rm -rf ./model.plan ./conversion_bs16_dy.txt ./model.onnx"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b34cf8a7",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Option 2 - TensorRT Model with SageMaker Notebook Instances \n",
+ "\n",
+ "To make sure we use TensorRT version and dependencies that are compatible with the ones in our Triton container, we compile the model using the corresponding version of NVIDIA's PyTorch container image.\n",
+ "\n",
+ "If you take a look at the python files within the `workspace` folder, you will see that we are first convert the model into ONNX format, specifying dynamic axis indexes so that inputs with a different batch size and sequence length can be passed to the model. TensorRT will treat other input dimensions as fixed, and optimize for those.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4f1e86f3",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "! docker run --gpus=all --rm -it -v `pwd`/workspace:/workspace nvcr.io/nvidia/pytorch:21.08-py3 /bin/bash generate_model_trt.sh $model_id"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "dd0bf459",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "! rm -rf ensemble_hf/bert-trt && mkdir -p ensemble_hf/bert-trt/1 && cp workspace/model.plan ensemble_hf/bert-trt/1/model.plan && rm -rf workspace/model.onnx workspace/core*"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fe095659",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "Explore the output logs of the compilation process; at the very end, we get a section headlined \"=== Performance summary ===\" which gives us a series of metrics on the obtained engine's performance (latency, throughput, etc...). "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8455c0ca",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "# Part 3 - Run Local Triton Inference Server"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ed0cd1c9-c8ab-4f75-a368-a719dd165c04",
+ "metadata": {},
+ "source": [
+ "> **WARNING**: The cells under part 3 will only work if run within a SageMaker Notebook Instance!\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9c22df98",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "\n",
+ "\n",
+ "The following cells run the Triton Inference Server container in the background and load all the models within the folder `/ensemble_hf`. The docker won't fail if one or more of the model fails because of `--exit-on-error=false`, which is useful for iterative code and model repository building. Remove `-d` to see the logs."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "35cac085",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "!sudo docker system prune -f"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d6965857",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "!docker run --gpus=all -d --shm-size=4G --rm -p8000:8000 -p8001:8001 -p8002:8002 -v$(pwd)/ensemble_hf:/model_repository nvcr.io/nvidia/tritonserver:21.08-py3 tritonserver --model-repository=/model_repository --exit-on-error=false --strict-model-config=false\n",
+ "time.sleep(20)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b9950b9c",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "CONTAINER_ID=!docker container ls -q\n",
+ "FIRST_CONTAINER_ID = CONTAINER_ID[0]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3f903432-6e87-449c-84ea-4c3ed2fa445b",
+ "metadata": {},
+ "source": [
+ "Uncomment the next cell and run it to view the container logs and understand Triton model loading."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9ca9f7dc",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# !docker logs $FIRST_CONTAINER_ID -f"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f5946837",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Test TensorRT model by invoking the local Triton Server"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7b5775f1",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Start a local Triton client\n",
+ "try:\n",
+ " triton_client = http_client.InferenceServerClient(url=\"localhost:8000\", verbose=True)\n",
+ "except Exception as e:\n",
+ " print(\"context creation failed: \" + str(e))\n",
+ " sys.exit()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "36b46556",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Create inputs to send to Triton\n",
+ "model_name = \"ensemble\"\n",
+ "\n",
+ "text_inputs = [\"Sentence 1\", \"Sentence 2\"]\n",
+ "\n",
+ "# Text is passed to Trtion as BYTES\n",
+ "inputs = []\n",
+ "inputs.append(http_client.InferInput(\"INPUT0\", [len(text_inputs), 1], \"BYTES\"))\n",
+ "\n",
+ "# We need to structure batch inputs as such\n",
+ "batch_request = [[text_inputs[i]] for i in range(len(text_inputs))]\n",
+ "input0_real = np.array(batch_request, dtype=np.object_)\n",
+ "\n",
+ "inputs[0].set_data_from_numpy(input0_real, binary_data=False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b47da0cb",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "outputs = []\n",
+ "\n",
+ "outputs.append(http_client.InferRequestedOutput(\"finaloutput\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2588e261",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a405f8ee",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "outputs_data = results.as_numpy(\"finaloutput\")\n",
+ "\n",
+ "for idx, output in enumerate(outputs_data):\n",
+ " print(text_inputs[idx])\n",
+ " print(output)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1d88a95c",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Use this to stop the container that was started in detached mode\n",
+ "!docker kill $FIRST_CONTAINER_ID"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dc75efff",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "***"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ef0e365f",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "# Part 4 - Deploy Triton to SageMaker Real-Time Endpoint"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "706db9cb",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Deploy with SageMaker Triton container"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bdceed91-9fbc-4ea3-ab9e-0e2599ba7281",
+ "metadata": {},
+ "source": [
+ "First we get the URI for the Sagemaker Triton container image that matches the one we used for TensorRT model compilation."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e0dd85b6",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "if region not in account_id_map.keys():\n",
+ " raise (\"UNSUPPORTED REGION\")\n",
+ "\n",
+ "base = \"amazonaws.com.cn\" if region.startswith(\"cn-\") else \"amazonaws.com\"\n",
+ "\n",
+ "triton_image_uri = \"{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:21.08-py3\".format(\n",
+ " account_id=account_id_map[region], region=region, base=base\n",
+ ")\n",
+ "\n",
+ "triton_image_uri"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "081d1204",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "print(sagemaker_session.default_bucket())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0adf2f45",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "ensemble_prefix = \"mme_gpu_tests/ensemble-singlemodel\"\n",
+ "!tar -C ensemble_hf/ -czf ensemble-sentencetrans.tar.gz .\n",
+ "model_uri_tf = sagemaker_session.upload_data(\n",
+ " path=\"ensemble-sentencetrans.tar.gz\", key_prefix=ensemble_prefix\n",
+ ")\n",
+ "\n",
+ "print(\"S3 model uri: {}\".format(model_uri_tf))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "da4eb8ef",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Important to define what which one of the models loaded by Triton is the default to be served by SM\n",
+ "# That is, SAGEMAKER_TRITON_DEFAULT_MODEL_NAME\n",
+ "container_model = {\n",
+ " \"Image\": triton_image_uri,\n",
+ " \"ModelDataUrl\": model_uri_tf,\n",
+ " \"Mode\": \"SingleModel\",\n",
+ " \"Environment\": {\"SAGEMAKER_TRITON_DEFAULT_MODEL_NAME\": \"ensemble\"},\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16b0ce98-b921-476d-8921-5e3ffb5cc7d4",
+ "metadata": {},
+ "source": [
+ "Register the model with Sagemaker."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e15ae6ac",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "sm_model_name = \"triton-sentence-ensemble\" + time.strftime(\"%Y-%m-%d-%H-%M-%S\", time.gmtime())\n",
+ "\n",
+ "create_model_response = sm_client.create_model(\n",
+ " ModelName=sm_model_name, ExecutionRoleArn=role, PrimaryContainer=container_model\n",
+ ")\n",
+ "\n",
+ "print(\"Model Arn: \" + create_model_response[\"ModelArn\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "695e71ab-c842-4001-8674-0d5c67ffd79f",
+ "metadata": {},
+ "source": [
+ "Create an endpoint configuration."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c6096778",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "endpoint_config_name = \"triton-sentence-ensemble\" + time.strftime(\n",
+ " \"%Y-%m-%d-%H-%M-%S\", time.gmtime()\n",
+ ")\n",
+ "\n",
+ "create_endpoint_config_response = sm_client.create_endpoint_config(\n",
+ " EndpointConfigName=endpoint_config_name,\n",
+ " ProductionVariants=[\n",
+ " {\n",
+ " \"InstanceType\": \"ml.g4dn.xlarge\",\n",
+ " \"InitialVariantWeight\": 1,\n",
+ " \"InitialInstanceCount\": 1,\n",
+ " \"ModelName\": sm_model_name,\n",
+ " \"VariantName\": \"AllTraffic\",\n",
+ " }\n",
+ " ],\n",
+ ")\n",
+ "\n",
+ "print(\"Endpoint Config Arn: \" + create_endpoint_config_response[\"EndpointConfigArn\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c4893f46-dd98-4e60-9490-36026f128446",
+ "metadata": {},
+ "source": [
+ "Deploy the endpoint."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e9932ae0",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "endpoint_name = \"triton-sentence-ensemble\" + time.strftime(\"%Y-%m-%d-%H-%M-%S\", time.gmtime())\n",
+ "\n",
+ "create_endpoint_response = sm_client.create_endpoint(\n",
+ " EndpointName=endpoint_name, EndpointConfigName=endpoint_config_name\n",
+ ")\n",
+ "\n",
+ "print(\"Endpoint Arn: \" + create_endpoint_response[\"EndpointArn\"])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a3ec6a48-5fc5-4495-9b1d-10f94ef95277",
+ "metadata": {},
+ "source": [
+ "Wait for the endpoint to be up and running."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f257b3e7",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
+ "status = resp[\"EndpointStatus\"]\n",
+ "print(\"Status: \" + status)\n",
+ "\n",
+ "while status == \"Creating\":\n",
+ " time.sleep(60)\n",
+ " resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
+ " status = resp[\"EndpointStatus\"]\n",
+ " print(\"Status: \" + status)\n",
+ "\n",
+ "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
+ "print(\"Status: \" + status)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7aee2ad4",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "***"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "59448028",
+ "metadata": {
+ "pycharm": {
+ "name": "#%% md\n"
+ }
+ },
+ "source": [
+ "## Test the SageMaker Triton Endpoint"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d55e6ea4",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "text_inputs = [\"Sentence 1\", \"Sentence 2\"]\n",
+ "\n",
+ "inputs = []\n",
+ "inputs.append(http_client.InferInput(\"INPUT0\", [len(text_inputs), 1], \"BYTES\"))\n",
+ "\n",
+ "batch_request = [[text_inputs[i]] for i in range(len(text_inputs))]\n",
+ "\n",
+ "input0_real = np.array(batch_request, dtype=np.object_)\n",
+ "\n",
+ "inputs[0].set_data_from_numpy(input0_real, binary_data=False)\n",
+ "\n",
+ "len(input0_real)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "71acf686",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "outputs = []\n",
+ "\n",
+ "outputs.append(http_client.InferRequestedOutput(\"finaloutput\"))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8cc9a386",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "request_body, header_length = http_client.InferenceServerClient.generate_request_body(\n",
+ " inputs, outputs=outputs\n",
+ ")\n",
+ "\n",
+ "print(request_body)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2782361a",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "response = sm_runtime_client.invoke_endpoint(\n",
+ " EndpointName=endpoint_name,\n",
+ " ContentType=\"application/vnd.sagemaker-triton.binary+json;json-header-size={}\".format(\n",
+ " header_length\n",
+ " ),\n",
+ " Body=request_body,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "65724cd1",
+ "metadata": {
+ "collapsed": false,
+ "jupyter": {
+ "outputs_hidden": false
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "## json.loads fails\n",
+ "# a = json.loads(response[\"Body\"].read().decode(\"utf8\"))\n",
+ "\n",
+ "header_length_prefix = \"application/vnd.sagemaker-triton.binary+json;json-header-size=\"\n",
+ "header_length_str = response[\"ContentType\"][len(header_length_prefix) :]\n",
+ "\n",
+ "# Read response body\n",
+ "result = http_client.InferenceServerClient.parse_response_body(\n",
+ " response[\"Body\"].read(), header_length=int(header_length_str)\n",
+ ")\n",
+ "\n",
+ "outputs_data = result.as_numpy(\"finaloutput\")\n",
+ "\n",
+ "for idx, output in enumerate(outputs_data):\n",
+ " print(text_inputs[idx])\n",
+ " print(output)"
+ ]
+ }
+ ],
+ "metadata": {
+ "instance_type": "ml.g4dn.xlarge",
+ "kernelspec": {
+ "display_name": "conda_pytorch_p38",
+ "language": "python",
+ "name": "conda_pytorch_p38"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/workspace/generate_model_trt.sh b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/workspace/generate_model_trt.sh
new file mode 100755
index 0000000000..8c68d40c60
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/workspace/generate_model_trt.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+MODEL_NAME=$1
+python -m pip install transformers==4.9.1
+python onnx_exporter.py --model $MODEL_NAME
+
+trtexec \
+ --onnx=model.onnx \
+ --saveEngine=model.plan \
+ --minShapes=token_ids:1x128,attn_mask:1x128 \
+ --optShapes=token_ids:16x128,attn_mask:16x128 \
+ --maxShapes=token_ids:32x128,attn_mask:32x128 \
+ --verbose \
+ --workspace=14000 \
+| tee conversion.txt
\ No newline at end of file
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/workspace/onnx_exporter.py b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/workspace/onnx_exporter.py
new file mode 100644
index 0000000000..b7e0907e52
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/examples/workspace/onnx_exporter.py
@@ -0,0 +1,30 @@
+import torch
+from transformers import AutoModel
+import argparse
+import os
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--save", default="model.onnx")
+ parser.add_argument("--model", required=True)
+
+ args = parser.parse_args()
+
+ model = AutoModel.from_pretrained(args.model, torchscript=True)
+
+ bs = 1
+ seq_len = 128
+ dummy_inputs = (torch.randint(1000, (bs, seq_len),dtype=torch.int), torch.zeros(bs, seq_len, dtype=torch.int))
+
+ torch.onnx.export(
+ model,
+ dummy_inputs,
+ args.save,
+ export_params=True,
+ opset_version=10,
+ input_names=["token_ids", "attn_mask"],
+ output_names=["output"],
+ dynamic_axes={"token_ids": [0, 1], "attn_mask": [0, 1], "output": [0]},
+ )
+
+ print("Saved {}".format(args.save))
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/images/triton-ensemble.png b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/images/triton-ensemble.png
new file mode 100644
index 0000000000..32f5c1a6aa
Binary files /dev/null and b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/images/triton-ensemble.png differ
diff --git a/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/studio-image/README.md b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/studio-image/README.md
new file mode 100644
index 0000000000..76083a1c26
--- /dev/null
+++ b/inference/nlp/realtime/huggingface/sentence-transformers-triton-ensemble/studio-image/README.md
@@ -0,0 +1,67 @@
+## build_image.sh
+
+This script allows you to create a custom docker image and push on ECR
+
+Parameters:
+* IMAGE_NAME: *Mandatory* - Name of the image you want to build
+* REGISTRY_NAME: *Mandatory* - Name of the ECR repository you want to use for pushing the image
+* IMAGE_TAG: *Mandatory* - Tag to apply to the ECR image
+* DOCKER_FILE: *Mandatory* - Dockerfile to build
+* PLATFORM: *Optional* - Target architecture chip where the image is executed
+```
+./build_image.sh \n", + " -- `data.csv`
\n", + "-- `validation`
\n", + " -- `data.csv` \n", + "-- `cat_idx.json`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fee4296f", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "\n", + "train, val_n_test = train_test_split(\n", + " churn, test_size=0.3, random_state=42, stratify=churn[\"target\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48080aca", + "metadata": {}, + "outputs": [], + "source": [ + "val, test = train_test_split(\n", + " val_n_test, test_size=0.3, random_state=42, stratify=val_n_test[\"target\"]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1771b769", + "metadata": {}, + "outputs": [], + "source": [ + "train.to_csv(\"train.csv\", header=False, index=False)\n", + "val.to_csv(\"validation.csv\", header=False, index=False)\n", + "test.to_csv(\"test.csv\", header=False, index=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c26e7053", + "metadata": {}, + "outputs": [], + "source": [ + "boto3.Session().resource(\"s3\").Bucket(bucket).Object(\n", + " os.path.join(prefix, \"train/data.csv\")\n", + ").upload_file(\"train.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c297dff2", + "metadata": {}, + "outputs": [], + "source": [ + "boto3.Session().resource(\"s3\").Bucket(bucket).Object(\n", + " os.path.join(prefix, \"validation/data.csv\")\n", + ").upload_file(\"validation.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3cb55d7a", + "metadata": {}, + "outputs": [], + "source": [ + "boto3.Session().resource(\"s3\").Bucket(bucket).Object(\n", + " os.path.join(prefix, \"test/data.csv\")\n", + ").upload_file(\"test.csv\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "042b6f55", + "metadata": {}, + "outputs": [], + "source": [ + "boto3.Session().resource(\"s3\").Bucket(bucket).Object(\n", + " os.path.join(prefix, \"cat_idx.json\")\n", + ").upload_file(\"cat_idx.json\")" + ] + }, + { + "cell_type": "markdown", + "id": "b278de2a", + "metadata": {}, + "source": [ + "## 3. Train A LightGBM Model with AMT" + ] + }, + { + "cell_type": "markdown", + "id": "26d18ad9", + "metadata": {}, + "source": [ + "### 3.1. Retrieve Training Artifacts\n", + "\n", + "___\n", + "\n", + "Here, we retrieve the training docker container, the training algorithm source, and the tabular algorithm. Note that model_version=\"*\" fetches the latest model.\n", + "\n", + "For the training algorithm, we have four choices in this demonstration for classification task.\n", + "* [LightGBM](https://lightgbm.readthedocs.io/en/latest/): To use this algorithm, specify `train_model_id` as `lightgbm-classification-model` in the cell below.\n", + "* [CatBoost](https://catboost.ai/en/docs/): To use this algorithm, specify `train_model_id` as `catboost-classification-model` in the cell below.\n", + "* [TabTransformer](https://arxiv.org/abs/2012.06678): To use this algorithm, specify `train_model_id` as `pytorch-tabtransformerclassification-model` in the cell below.\n", + "* [AutoGluon Tabular](https://auto.gluon.ai/stable/tutorials/tabular_prediction/index.html): To use this algorithm, specify `train_model_id` as `autogluon-classification-ensemble` in the cell below.\n", + "\n", + "Note. [XGBoost](https://xgboost.readthedocs.io/en/latest/) (`train_model_id: xgboost-classification-model`) and [Linear Learner](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression) (`train_model_id: sklearn-classification-linear`) are the other choices in the tabular classification category. Since they have different input-format requirements, please check separate notebooks `xgboost_linear_learner_tabular/Amazon_Tabular_Classification_XGBoost_LinearLearner.ipynb`, `tabtransformer_tabular/Amazon_Tabular_Classification_TabTransformer.ipynb`, and `autogluon_tabular/Amazon_Tabular_Classification_AutoGluon.ipynb` for details.\n", + "\n", + "For regression task, you just need replace `classification` in the `train_model_id` with `regression`.\n", + "\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0ad11b96", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import image_uris, model_uris, script_uris\n", + "\n", + "train_model_id, train_model_version, train_scope = \"lightgbm-classification-model\", \"*\", \"training\"\n", + "training_instance_type = \"ml.m5.4xlarge\"\n", + "\n", + "# Retrieve the docker image\n", + "train_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " image_scope=train_scope,\n", + " instance_type=training_instance_type,\n", + ")\n", + "# Retrieve the training script\n", + "train_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=train_scope\n", + ")\n", + "# Retrieve the pre-trained model tarball to further fine-tune\n", + "train_model_uri = model_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, model_scope=train_scope\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e8a4d3d3", + "metadata": {}, + "source": [ + "### 3.2. Set Training Parameters\n", + "\n", + "---\n", + "\n", + "Now that we are done with all the setup that is needed, we are ready to train our tabular algorithm. To begin, let us create a [``sageMaker.estimator.Estimator``](https://sagemaker.readthedocs.io/en/stable/api/training/estimators.html) object. This estimator will launch the training job. \n", + "\n", + "There are two kinds of parameters that need to be set for training. The first one are the parameters for the training job. These include: (i) Training data path. This is S3 folder in which the input data is stored, (ii) Output path: This the s3 folder in which the training output is stored. (iii) Training instance type: This indicates the type of machine on which to run the training.\n", + "\n", + "The second set of parameters are algorithm specific training hyper-parameters. \n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a1f8559", + "metadata": {}, + "outputs": [], + "source": [ + "training_dataset_s3_path = f\"s3://{bucket}/{prefix}\"\n", + "\n", + "output_prefix = \"jumpstart-example-tabular-training\"\n", + "s3_output_location = f\"s3://{bucket}/{output_prefix}/output_lgb\"" + ] + }, + { + "cell_type": "markdown", + "id": "8828563c", + "metadata": {}, + "source": [ + "---\n", + "For algorithm specific hyper-parameters, we start by fetching python dictionary of the training hyper-parameters that the algorithm accepts with their default values. This can then be overridden to custom values. For the evaluation metric that is used by early stopping and automatic model tuning, we choose `auc` score. Note. LightGBM does not have built-in F1 score supported. See [LightGBM documentation](https://lightgbm.readthedocs.io/en/latest/Parameters.html#metric-parameters).\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8cd5d2fd", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import hyperparameters\n", + "\n", + "# Retrieve the default hyper-parameters for fine-tuning the model\n", + "hyperparameters = hyperparameters.retrieve_default(\n", + " model_id=train_model_id, model_version=train_model_version\n", + ")\n", + "\n", + "# [Optional] Override default hyperparameters with custom values\n", + "hyperparameters[\n", + " \"num_boost_round\"\n", + "] = \"500\" # The same hyperparameter is named as \"iterations\" for CatBoost\n", + "\n", + "\n", + "hyperparameters[\"metric\"] = \"auc\"\n", + "print(hyperparameters)" + ] + }, + { + "cell_type": "markdown", + "id": "f43ec07c", + "metadata": {}, + "source": [ + "### 3.3. Train with Automatic Model Tuning \n", + "\n", + "\n", + "Amazon SageMaker automatic model tuning, also known as hyperparameter tuning, finds the best version of a model by running many training jobs on your dataset using the algorithm and ranges of hyperparameters that you specify. It then chooses the hyperparameter values that result in a model that performs the best, as measured by a metric that you choose. We will use a HyperparameterTuner object to interact with Amazon SageMaker hyperparameter tuning APIs.\n", + "\n", + "* Note. In this notebook, we set AMT budget (total tuning jobs) as 10 for each of the tabular algorithm except AutoGluon-Tabular. For [AutoGluon-Tabular](https://arxiv.org/abs/2003.06505), it succeeds by ensembling multiple models and stacking them in multiple layers. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b136b897", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker.tuner import ContinuousParameter, IntegerParameter, HyperparameterTuner\n", + "\n", + "use_amt = True\n", + "\n", + "hyperparameter_ranges_lgb = {\n", + " \"learning_rate\": ContinuousParameter(1e-4, 1, scaling_type=\"Logarithmic\"),\n", + " \"num_boost_round\": IntegerParameter(2, 30),\n", + " \"num_leaves\": IntegerParameter(10, 50),\n", + " \"feature_fraction\": ContinuousParameter(0, 1),\n", + " \"bagging_fraction\": ContinuousParameter(0, 1),\n", + " \"bagging_freq\": IntegerParameter(1, 10),\n", + " \"max_depth\": IntegerParameter(5, 30),\n", + " \"min_data_in_leaf\": IntegerParameter(5, 50),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "f209be30", + "metadata": {}, + "source": [ + "### 3.4. Start Training" + ] + }, + { + "cell_type": "markdown", + "id": "caf86ae9", + "metadata": {}, + "source": [ + "---\n", + "We start by creating the estimator object with all the required assets and then launch the training job.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c6d9bab", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker.estimator import Estimator\n", + "from sagemaker.utils import name_from_base\n", + "\n", + "training_job_name = name_from_base(f\"jumpstart-{train_model_id}-train\")\n", + "\n", + "# Create SageMaker Estimator instance\n", + "tabular_estimator = Estimator(\n", + " role=aws_role,\n", + " image_uri=train_image_uri,\n", + " source_dir=train_source_uri,\n", + " model_uri=train_model_uri,\n", + " entry_point=\"transfer_learning.py\",\n", + " instance_count=1,\n", + " instance_type=training_instance_type,\n", + " max_run=360000,\n", + " hyperparameters=hyperparameters,\n", + " output_path=s3_output_location,\n", + ")\n", + "\n", + "if use_amt:\n", + "\n", + " tuner = HyperparameterTuner(\n", + " tabular_estimator,\n", + " \"auc\",\n", + " hyperparameter_ranges_lgb,\n", + " [{\"Name\": \"auc\", \"Regex\": \"auc: ([0-9\\\\.]+)\"}],\n", + " max_jobs=10,\n", + " max_parallel_jobs=5,\n", + " objective_type=\"Maximize\",\n", + " base_tuning_job_name=training_job_name,\n", + " )\n", + "\n", + " tuner.fit({\"training\": training_dataset_s3_path}, logs=True)\n", + "else:\n", + " # Launch a SageMaker Training job by passing s3 path of the training data\n", + " tabular_estimator.fit(\n", + " {\"training\": training_dataset_s3_path}, logs=True, job_name=training_job_name\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "1f1c8f37", + "metadata": {}, + "source": [ + "### 3.5. Deploy and Run Inference on the Trained Tabular Model\n", + "\n", + "---\n", + "\n", + "In this section, you learn how to query an existing endpoint and make predictions of the examples you input. For each example, the model will output the probability of the sample for each class in the model. \n", + "Next, the predicted class label is obtained by taking the class label with the maximum probability over others.\n", + "\n", + "\n", + "We start by retrieving the artifacts and deploy the `tabular_estimator` that we trained.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0d18d65", + "metadata": {}, + "outputs": [], + "source": [ + "inference_instance_type = \"ml.m5.large\"\n", + "\n", + "# Retrieve the inference docker container uri\n", + "deploy_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " image_scope=\"inference\",\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " instance_type=inference_instance_type,\n", + ")\n", + "# Retrieve the inference script uri\n", + "deploy_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=\"inference\"\n", + ")\n", + "\n", + "endpoint_name = name_from_base(f\"jumpstart-lgb-churn-{train_model_id}-\")\n", + "\n", + "# Use the estimator from the previous step to deploy to a SageMaker endpoint\n", + "predictor = (tuner if use_amt else tabular_estimator).deploy(\n", + " initial_instance_count=1,\n", + " instance_type=inference_instance_type,\n", + " entry_point=\"inference.py\",\n", + " image_uri=deploy_image_uri,\n", + " source_dir=deploy_source_uri,\n", + " endpoint_name=endpoint_name,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "57a3c147", + "metadata": {}, + "source": [ + "---\n", + "Next, we read the customer churn test data into pandas data frame, prepare the ground truth target and predicting features to send into the endpoint. \n", + "\n", + "Below is the screenshot of the first 5 examples in the test set.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0f8fdb7", + "metadata": {}, + "outputs": [], + "source": [ + "newline, bold, unbold = \"\\n\", \"\\033[1m\", \"\\033[0m\"\n", + "\n", + "import numpy as np\n", + "import pandas as pd\n", + "from sklearn.metrics import accuracy_score, f1_score, roc_auc_score\n", + "from sklearn.metrics import confusion_matrix\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# read the data\n", + "test_data_file_name = \"test.csv\"\n", + "test_data = pd.read_csv(test_data_file_name, header=None)\n", + "test_data.columns = [\"Target\"] + [f\"Feature_{i}\" for i in range(1, test_data.shape[1])]\n", + "\n", + "num_examples, num_columns = test_data.shape\n", + "print(\n", + " f\"{bold}The test dataset contains {num_examples} examples and {num_columns} columns.{unbold}\\n\"\n", + ")\n", + "\n", + "# prepare the ground truth target and predicting features to send into the endpoint.\n", + "ground_truth_label, features = test_data.iloc[:, :1], test_data.iloc[:, 1:]\n", + "\n", + "print(f\"{bold}The first 5 observations of the data: {unbold} \\n\")\n", + "test_data.head(5)" + ] + }, + { + "cell_type": "markdown", + "id": "4f628562", + "metadata": {}, + "source": [ + "---\n", + "The following code queries the endpoint you have created to get the prediction for each test example. \n", + "The `query_endpoint()` function returns an array-like of shape (num_examples, num_classes), where each row indicates \n", + "the probability of the example for each class in the model. The num_classes is 2 in above test data. \n", + "Next, the predicted class label is obtained by taking the class label with the maximum probability over others for each example. \n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "da19a629", + "metadata": {}, + "outputs": [], + "source": [ + "content_type = \"text/csv\"\n", + "\n", + "\n", + "def query_endpoint(encoded_tabular_data, endpoint_name):\n", + " client = boto3.client(\"runtime.sagemaker\")\n", + " response = client.invoke_endpoint(\n", + " EndpointName=endpoint_name,\n", + " ContentType=content_type,\n", + " Body=encoded_tabular_data,\n", + " )\n", + " return response\n", + "\n", + "\n", + "def parse_response(query_response):\n", + " model_predictions = json.loads(query_response[\"Body\"].read())\n", + " predicted_probabilities = model_predictions[\"probabilities\"]\n", + " return np.array(predicted_probabilities)\n", + "\n", + "\n", + "# split the test data into smaller size of batches to query the endpoint if test data has large size.\n", + "batch_size = 1500\n", + "predict_prob = []\n", + "for i in np.arange(0, num_examples, step=batch_size):\n", + " query_response_batch = query_endpoint(\n", + " features.iloc[i : (i + batch_size), :].to_csv(header=False, index=False).encode(\"utf-8\"),\n", + " endpoint_name,\n", + " )\n", + " predict_prob_batch = parse_response(query_response_batch) # prediction probability per batch\n", + " predict_prob.append(predict_prob_batch)\n", + "\n", + "\n", + "predict_prob = np.concatenate(predict_prob, axis=0)\n", + "predict_label = np.argmax(predict_prob, axis=1)" + ] + }, + { + "cell_type": "markdown", + "id": "aabdee3e", + "metadata": {}, + "source": [ + "## 3.6. Evaluate the Prediction Results Returned from the Endpoint\n", + "\n", + "---\n", + "We evaluate the predictions results returned from the endpoint by following two ways.\n", + "\n", + "* Visualize the predictions results by plotting the confusion matrix.\n", + "\n", + "* Measure the prediction results quantitatively.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1f3610bc", + "metadata": {}, + "outputs": [], + "source": [ + "# Visualize the predictions results by plotting the confusion matrix.\n", + "conf_matrix = confusion_matrix(y_true=ground_truth_label.values, y_pred=predict_label)\n", + "fig, ax = plt.subplots(figsize=(7.5, 7.5))\n", + "ax.matshow(conf_matrix, cmap=plt.cm.Blues, alpha=0.3)\n", + "for i in range(conf_matrix.shape[0]):\n", + " for j in range(conf_matrix.shape[1]):\n", + " ax.text(x=j, y=i, s=conf_matrix[i, j], va=\"center\", ha=\"center\", size=\"xx-large\")\n", + "\n", + "plt.xlabel(\"Predictions\", fontsize=18)\n", + "plt.ylabel(\"Actuals\", fontsize=18)\n", + "plt.title(\"Confusion Matrix\", fontsize=18)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a59c801e", + "metadata": {}, + "outputs": [], + "source": [ + "# Measure the prediction results quantitatively.\n", + "eval_accuracy = accuracy_score(ground_truth_label.values, predict_label)\n", + "eval_f1 = f1_score(ground_truth_label.values, predict_label)\n", + "eval_auc = roc_auc_score(ground_truth_label.values, predict_prob[:, 1])\n", + "\n", + "lgb_results = pd.DataFrame.from_dict(\n", + " {\n", + " \"Accuracy\": eval_accuracy,\n", + " \"F1\": eval_f1,\n", + " \"AUC\": eval_auc,\n", + " },\n", + " orient=\"index\",\n", + " columns=[\"LightGBM with AMT\"],\n", + ")\n", + "\n", + "lgb_results" + ] + }, + { + "cell_type": "markdown", + "id": "ab7d2f6d", + "metadata": {}, + "source": [ + "## 4. Train A CatBoost model with AMT\n" + ] + }, + { + "cell_type": "markdown", + "id": "c49487ca", + "metadata": {}, + "source": [ + "### 4.1. Train with Automatic Model Tuning\n" + ] + }, + { + "cell_type": "markdown", + "id": "0e3350a3", + "metadata": {}, + "source": [ + "Retrieve Training Artifacts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a67cce3a", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import image_uris, model_uris, script_uris\n", + "\n", + "train_model_id, train_model_version, train_scope = \"catboost-classification-model\", \"*\", \"training\"\n", + "training_instance_type = \"ml.m5.4xlarge\"\n", + "\n", + "# Retrieve the docker image\n", + "train_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " image_scope=train_scope,\n", + " instance_type=training_instance_type,\n", + ")\n", + "# Retrieve the training script\n", + "train_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=train_scope\n", + ")\n", + "# Retrieve the pre-trained model tarball to further fine-tune\n", + "train_model_uri = model_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, model_scope=train_scope\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "5798369b", + "metadata": {}, + "source": [ + "Set training parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b7ada7ad", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import hyperparameters\n", + "\n", + "# Retrieve the default hyper-parameters for fine-tuning the model\n", + "hyperparameters = hyperparameters.retrieve_default(\n", + " model_id=train_model_id, model_version=train_model_version\n", + ")\n", + "\n", + "# [Optional] Override default hyperparameters with custom values\n", + "hyperparameters[\"iterations\"] = \"500\"\n", + "\n", + "\n", + "hyperparameters[\"eval_metric\"] = \"AUC\"\n", + "print(hyperparameters)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c02b4aa", + "metadata": {}, + "outputs": [], + "source": [ + "s3_output_location_cat = f\"s3://{bucket}/{output_prefix}/output_cat\"" + ] + }, + { + "cell_type": "markdown", + "id": "c69d3f7a", + "metadata": {}, + "source": [ + "Train with Automatic Model Tuning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0cd0ac41", + "metadata": {}, + "outputs": [], + "source": [ + "hyperparameter_ranges_cat = {\n", + " \"learning_rate\": ContinuousParameter(0.00001, 0.1, scaling_type=\"Logarithmic\"),\n", + " \"iterations\": IntegerParameter(50, 1000),\n", + " \"depth\": IntegerParameter(1, 10),\n", + " \"l2_leaf_reg\": IntegerParameter(1, 10),\n", + " \"random_strength\": ContinuousParameter(0.01, 10, scaling_type=\"Logarithmic\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "17053327", + "metadata": {}, + "source": [ + "Start training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb34ccbe", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker.estimator import Estimator\n", + "from sagemaker.utils import name_from_base\n", + "\n", + "training_job_name = name_from_base(f\"jumpstart-{train_model_id}-training\")\n", + "\n", + "# Create SageMaker Estimator instance\n", + "tabular_estimator_cat = Estimator(\n", + " role=aws_role,\n", + " image_uri=train_image_uri,\n", + " source_dir=train_source_uri,\n", + " model_uri=train_model_uri,\n", + " entry_point=\"transfer_learning.py\",\n", + " instance_count=1,\n", + " instance_type=training_instance_type,\n", + " max_run=360000,\n", + " hyperparameters=hyperparameters,\n", + " output_path=s3_output_location_cat,\n", + ")\n", + "\n", + "if use_amt:\n", + "\n", + " tuner_cat = HyperparameterTuner(\n", + " tabular_estimator_cat,\n", + " \"AUC\",\n", + " hyperparameter_ranges_cat,\n", + " [{\"Name\": \"AUC\", \"Regex\": \"bestTest = ([0-9\\\\.]+)\"}],\n", + " max_jobs=10,\n", + " max_parallel_jobs=5,\n", + " objective_type=\"Maximize\",\n", + " base_tuning_job_name=training_job_name,\n", + " )\n", + "\n", + " tuner_cat.fit({\"training\": training_dataset_s3_path}, logs=True)\n", + "else:\n", + " # Launch a SageMaker Training job by passing s3 path of the training data\n", + " tabular_estimator_cat.fit(\n", + " {\"training\": training_dataset_s3_path}, logs=True, job_name=training_job_name\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "33ad5e7a", + "metadata": {}, + "source": [ + "### 4.2. Deploy and Run Inference on the Trained Tabular Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2159fc95", + "metadata": {}, + "outputs": [], + "source": [ + "inference_instance_type = \"ml.m5.large\"\n", + "\n", + "# Retrieve the inference docker container uri\n", + "deploy_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " image_scope=\"inference\",\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " instance_type=inference_instance_type,\n", + ")\n", + "# Retrieve the inference script uri\n", + "deploy_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=\"inference\"\n", + ")\n", + "\n", + "endpoint_name_cat = name_from_base(f\"jumpstart-cat-churn-{train_model_id}-\")\n", + "\n", + "# Use the estimator from the previous step to deploy to a SageMaker endpoint\n", + "predictor_cat = (tuner_cat if use_amt else tabular_estimator_cat).deploy(\n", + " initial_instance_count=1,\n", + " instance_type=inference_instance_type,\n", + " entry_point=\"inference.py\",\n", + " image_uri=deploy_image_uri,\n", + " source_dir=deploy_source_uri,\n", + " endpoint_name=endpoint_name_cat,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "fd36650b", + "metadata": {}, + "source": [ + "Query the endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa560463", + "metadata": {}, + "outputs": [], + "source": [ + "# split the test data into smaller size of batches to query the endpoint if the test data has large size.\n", + "batch_size = 1500\n", + "predict_prob_cat = []\n", + "for i in np.arange(0, num_examples, step=batch_size):\n", + " query_response_batch = query_endpoint(\n", + " features.iloc[i : (i + batch_size), :].to_csv(header=False, index=False).encode(\"utf-8\"),\n", + " endpoint_name_cat,\n", + " )\n", + " predict_prob_batch = parse_response(query_response_batch) # prediction probability per batch\n", + " predict_prob_cat.append(predict_prob_batch)\n", + "\n", + "\n", + "predict_prob_cat = np.concatenate(predict_prob_cat, axis=0)\n", + "predict_label_cat = np.argmax(predict_prob_cat, axis=1)" + ] + }, + { + "cell_type": "markdown", + "id": "3c62c458", + "metadata": {}, + "source": [ + "Evaluate the prediction results returned from the endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b012badc", + "metadata": {}, + "outputs": [], + "source": [ + "# Visualize the predictions results by plotting the confusion matrix.\n", + "conf_matrix = confusion_matrix(y_true=ground_truth_label.values, y_pred=predict_label_cat)\n", + "fig, ax = plt.subplots(figsize=(7.5, 7.5))\n", + "ax.matshow(conf_matrix, cmap=plt.cm.Blues, alpha=0.3)\n", + "for i in range(conf_matrix.shape[0]):\n", + " for j in range(conf_matrix.shape[1]):\n", + " ax.text(x=j, y=i, s=conf_matrix[i, j], va=\"center\", ha=\"center\", size=\"xx-large\")\n", + "\n", + "plt.xlabel(\"Predictions\", fontsize=18)\n", + "plt.ylabel(\"Actuals\", fontsize=18)\n", + "plt.title(\"Confusion Matrix\", fontsize=18)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e1e6c3a0", + "metadata": {}, + "outputs": [], + "source": [ + "# Measure the prediction results quantitatively.\n", + "eval_accuracy_cat = accuracy_score(ground_truth_label.values, predict_label_cat)\n", + "eval_f1_cat = f1_score(ground_truth_label.values, predict_label_cat)\n", + "eval_auc_cat = roc_auc_score(ground_truth_label.values, predict_prob_cat[:, 1])\n", + "\n", + "cat_results = pd.DataFrame.from_dict(\n", + " {\n", + " \"Accuracy\": eval_accuracy_cat,\n", + " \"F1\": eval_f1_cat,\n", + " \"AUC\": eval_auc_cat,\n", + " },\n", + " orient=\"index\",\n", + " columns=[\"CatBoost with AMT\"],\n", + ")\n", + "\n", + "results_lab_cat = pd.concat([lgb_results, cat_results], axis=1)\n", + "results_lab_cat" + ] + }, + { + "cell_type": "markdown", + "id": "026fc463", + "metadata": {}, + "source": [ + "## 5. Train A TabTransformer model with AMT" + ] + }, + { + "cell_type": "markdown", + "id": "a618e4af", + "metadata": {}, + "source": [ + "### 5.1. Train with Automatic Model Tuning" + ] + }, + { + "cell_type": "markdown", + "id": "f20e80bc", + "metadata": {}, + "source": [ + "Retrieve Training Artifacts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f420b4d1", + "metadata": {}, + "outputs": [], + "source": [ + "train_model_id, train_model_version, train_scope = (\n", + " \"pytorch-tabtransformerclassification-model\",\n", + " \"*\",\n", + " \"training\",\n", + ")\n", + "training_instance_type = \"ml.p3.2xlarge\"\n", + "\n", + "# Retrieve the docker image\n", + "train_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " image_scope=train_scope,\n", + " instance_type=training_instance_type,\n", + ")\n", + "# Retrieve the training script\n", + "train_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=train_scope\n", + ")\n", + "# Retrieve the pre-trained model tarball to further fine-tune\n", + "train_model_uri = model_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, model_scope=train_scope\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "e133b1ed", + "metadata": {}, + "source": [ + "Set training parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e4348e5", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import hyperparameters\n", + "\n", + "# Retrieve the default hyper-parameters for fine-tuning the model\n", + "hyperparameters = hyperparameters.retrieve_default(\n", + " model_id=train_model_id, model_version=train_model_version\n", + ")\n", + "\n", + "# [Optional] Override default hyperparameters with custom values\n", + "hyperparameters[\"n_epochs\"] = 40 # The same hyperparameter is named as \"iterations\" for CatBoost\n", + "hyperparameters[\"patience\"] = 10\n", + "\n", + "print(hyperparameters)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0079c15c", + "metadata": {}, + "outputs": [], + "source": [ + "s3_output_location_tab = f\"s3://{bucket}/{output_prefix}/output_tab\"" + ] + }, + { + "cell_type": "markdown", + "id": "7bcce249", + "metadata": {}, + "source": [ + "Train with Automatic Model Tuning" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d9baa338", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker.tuner import (\n", + " ContinuousParameter,\n", + " IntegerParameter,\n", + " HyperparameterTuner,\n", + " CategoricalParameter,\n", + ")\n", + "\n", + "hyperparameter_ranges_tab = {\n", + " \"learning_rate\": ContinuousParameter(0.001, 0.01, scaling_type=\"Auto\"),\n", + " \"batch_size\": CategoricalParameter([64, 128, 256, 512]),\n", + " \"attn_dropout\": ContinuousParameter(0.0, 0.8, scaling_type=\"Auto\"),\n", + " \"mlp_dropout\": ContinuousParameter(0.0, 0.8, scaling_type=\"Auto\"),\n", + " \"input_dim\": CategoricalParameter([\"16\", \"32\", \"64\", \"128\", \"256\"]),\n", + " \"frac_shared_embed\": ContinuousParameter(0.0, 0.5, scaling_type=\"Auto\"),\n", + "}" + ] + }, + { + "cell_type": "markdown", + "id": "edba0682", + "metadata": {}, + "source": [ + "Start training" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c1b2be2", + "metadata": {}, + "outputs": [], + "source": [ + "training_job_name = name_from_base(f\"jumpstart-{train_model_id}-training\")\n", + "\n", + "# Create SageMaker Estimator instance\n", + "tabular_estimator_tab = Estimator(\n", + " role=aws_role,\n", + " image_uri=train_image_uri,\n", + " source_dir=train_source_uri,\n", + " model_uri=train_model_uri,\n", + " entry_point=\"transfer_learning.py\",\n", + " instance_count=1,\n", + " instance_type=training_instance_type,\n", + " max_run=360000,\n", + " hyperparameters=hyperparameters,\n", + " output_path=s3_output_location_tab,\n", + ")\n", + "\n", + "if use_amt:\n", + "\n", + " tuner_tab = HyperparameterTuner(\n", + " tabular_estimator_tab,\n", + " \"f1_score\", # Note, TabTransformer currently does not support AUC score, thus we use its default setting F1 score as an alternative evaluation metric.\n", + " hyperparameter_ranges_tab,\n", + " [{\"Name\": \"f1_score\", \"Regex\": \"metrics={'f1': (\\\\S+)}\"}],\n", + " max_jobs=10,\n", + " max_parallel_jobs=5, # reduce max_parallel_jobs number if the instance type is limited in your account\n", + " objective_type=\"Maximize\",\n", + " base_tuning_job_name=training_job_name,\n", + " )\n", + "\n", + " tuner_tab.fit({\"training\": training_dataset_s3_path}, logs=True)\n", + "else:\n", + " # Launch a SageMaker Training job by passing s3 path of the training data\n", + " tabular_estimator_tab.fit(\n", + " {\"training\": training_dataset_s3_path}, logs=True, job_name=training_job_name\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "4f5a8b89", + "metadata": {}, + "source": [ + " \n", + "### 5.2. Deploy and Run Inference on the Trained Tabular Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5d1d6afb", + "metadata": {}, + "outputs": [], + "source": [ + "inference_instance_type = \"ml.m5.2xlarge\"\n", + "\n", + "# Retrieve the inference docker container uri\n", + "deploy_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " image_scope=\"inference\",\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " instance_type=inference_instance_type,\n", + ")\n", + "# Retrieve the inference script uri\n", + "deploy_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=\"inference\"\n", + ")\n", + "\n", + "endpoint_name_tab = name_from_base(f\"jumpstart-tabtransformer-churn-{train_model_id}-\")\n", + "\n", + "# Use the estimator from the previous step to deploy to a SageMaker endpoint\n", + "predictor_tab = (tuner_tab if use_amt else tabular_estimator_tab).deploy(\n", + " initial_instance_count=1,\n", + " instance_type=inference_instance_type,\n", + " entry_point=\"inference.py\",\n", + " image_uri=deploy_image_uri,\n", + " source_dir=deploy_source_uri,\n", + " endpoint_name=endpoint_name_tab,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e6d70ad", + "metadata": {}, + "outputs": [], + "source": [ + "# split the test data into smaller size of batches to query the endpoint if the test data has large size.\n", + "batch_size = 1500\n", + "predict_prob_tab = []\n", + "for i in np.arange(0, num_examples, step=batch_size):\n", + " query_response_batch = query_endpoint(\n", + " features.iloc[i : (i + batch_size), :].to_csv(header=False, index=False).encode(\"utf-8\"),\n", + " endpoint_name_tab,\n", + " )\n", + " predict_prob_batch = parse_response(query_response_batch) # prediction probability per batch\n", + " predict_prob_tab.append(predict_prob_batch)\n", + "\n", + "\n", + "predict_prob_tab = np.concatenate(predict_prob_tab, axis=0)\n", + "predict_label_tab = np.argmax(predict_prob_tab, axis=1)" + ] + }, + { + "cell_type": "markdown", + "id": "c7533d36", + "metadata": {}, + "source": [ + "Evaluate the prediction results returned from the endpoint" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "641f8234", + "metadata": {}, + "outputs": [], + "source": [ + "# Visualize the predictions results by plotting the confusion matrix.\n", + "conf_matrix = confusion_matrix(y_true=ground_truth_label.values, y_pred=predict_label_tab)\n", + "fig, ax = plt.subplots(figsize=(7.5, 7.5))\n", + "ax.matshow(conf_matrix, cmap=plt.cm.Blues, alpha=0.3)\n", + "for i in range(conf_matrix.shape[0]):\n", + " for j in range(conf_matrix.shape[1]):\n", + " ax.text(x=j, y=i, s=conf_matrix[i, j], va=\"center\", ha=\"center\", size=\"xx-large\")\n", + "\n", + "plt.xlabel(\"Predictions\", fontsize=18)\n", + "plt.ylabel(\"Actuals\", fontsize=18)\n", + "plt.title(\"Confusion Matrix\", fontsize=18)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17e29efd", + "metadata": {}, + "outputs": [], + "source": [ + "# Measure the prediction results quantitatively.\n", + "eval_accuracy_tab = accuracy_score(ground_truth_label.values, predict_label_tab)\n", + "eval_f1_tab = f1_score(ground_truth_label.values, predict_label_tab)\n", + "eval_auc_tab = roc_auc_score(ground_truth_label.values, predict_prob_tab[:, 1])\n", + "\n", + "tab_results = pd.DataFrame.from_dict(\n", + " {\n", + " \"Accuracy\": eval_accuracy_tab,\n", + " \"F1\": eval_f1_tab,\n", + " \"AUC\": eval_auc_tab,\n", + " },\n", + " orient=\"index\",\n", + " columns=[\"TabTransformer with AMT\"],\n", + ")\n", + "\n", + "results_lab_cat_tab = pd.concat([results_lab_cat, tab_results], axis=1)\n", + "results_lab_cat_tab" + ] + }, + { + "cell_type": "markdown", + "id": "ea964d81", + "metadata": {}, + "source": [ + "## 6. Train An AutoGluon-Tabular model" + ] + }, + { + "cell_type": "markdown", + "id": "2c1fd4df", + "metadata": {}, + "source": [ + "### 6.1. Train with AutoGluon-Tabular model\n" + ] + }, + { + "cell_type": "markdown", + "id": "d9c6393b", + "metadata": {}, + "source": [ + "Retrieve Training Artifacts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b247833f", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import image_uris, model_uris, script_uris\n", + "\n", + "# Currently, not all the object detection models in jumpstart support finetuning. Thus, we manually select a model\n", + "# which supports finetuning.\n", + "train_model_id, train_model_version, train_scope = (\n", + " \"autogluon-classification-ensemble\",\n", + " \"*\",\n", + " \"training\",\n", + ")\n", + "training_instance_type = \"ml.g4dn.2xlarge\" # set a different GPU type to avoid instance insufficiency for p3 instance that is used by TabTransformer\n", + "\n", + "# Retrieve the docker image\n", + "train_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " image_scope=train_scope,\n", + " instance_type=training_instance_type,\n", + ")\n", + "# Retrieve the training script\n", + "train_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=train_scope\n", + ")\n", + "# Retrieve the pre-trained model tarball to further fine-tune\n", + "train_model_uri = model_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, model_scope=train_scope\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c288f5a8", + "metadata": {}, + "source": [ + "Set training parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "577586e1", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker import hyperparameters\n", + "\n", + "# Retrieve the default hyper-parameters for fine-tuning the model\n", + "hyperparameters = hyperparameters.retrieve_default(\n", + " model_id=train_model_id, model_version=train_model_version\n", + ")\n", + "\n", + "hyperparameters[\"eval_metric\"] = \"roc_auc\"\n", + "print(hyperparameters)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "55b4b386", + "metadata": {}, + "outputs": [], + "source": [ + "s3_output_location_ag = f\"s3://{bucket}/{output_prefix}/output_ag\"" + ] + }, + { + "cell_type": "markdown", + "id": "278f7178", + "metadata": {}, + "source": [ + "Start training\n", + "\n", + "Note. We do not perform automatic model tuning as AutoGluon-Tabular do not focus on hyperparameter selections. Instead, it ensembles multiple models and stacks them in multiple layers. For details, see [paper](https://arxiv.org/abs/2003.06505)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c07b6103", + "metadata": {}, + "outputs": [], + "source": [ + "from sagemaker.estimator import Estimator\n", + "from sagemaker.utils import name_from_base\n", + "\n", + "training_job_name = name_from_base(f\"jumpstart-{train_model_id}-training\")\n", + "\n", + "# Create SageMaker Estimator instance\n", + "tabular_estimator_ag = Estimator(\n", + " role=aws_role,\n", + " image_uri=train_image_uri,\n", + " source_dir=train_source_uri,\n", + " model_uri=train_model_uri,\n", + " entry_point=\"transfer_learning.py\",\n", + " instance_count=1,\n", + " instance_type=training_instance_type,\n", + " max_run=360000,\n", + " hyperparameters=hyperparameters,\n", + " output_path=s3_output_location_ag,\n", + ")\n", + "\n", + "\n", + "# Launch a SageMaker Training job by passing s3 path of the training data\n", + "tabular_estimator_ag.fit(\n", + " {\"training\": training_dataset_s3_path}, logs=True, job_name=training_job_name\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "d6b8361d", + "metadata": {}, + "source": [ + "### 6.2. Deploy and Run Inference on the Trained Tabular Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f6dc44a3", + "metadata": {}, + "outputs": [], + "source": [ + "inference_instance_type = \"ml.m5.2xlarge\"\n", + "\n", + "# Retrieve the inference docker container uri\n", + "deploy_image_uri = image_uris.retrieve(\n", + " region=None,\n", + " framework=None,\n", + " image_scope=\"inference\",\n", + " model_id=train_model_id,\n", + " model_version=train_model_version,\n", + " instance_type=inference_instance_type,\n", + ")\n", + "# Retrieve the inference script uri\n", + "deploy_source_uri = script_uris.retrieve(\n", + " model_id=train_model_id, model_version=train_model_version, script_scope=\"inference\"\n", + ")\n", + "\n", + "endpoint_name_ag = name_from_base(f\"jumpstart-ag-churn-{train_model_id}-\")\n", + "\n", + "# Use the estimator from the previous step to deploy to a SageMaker endpoint\n", + "predictor_ag = tabular_estimator_ag.deploy(\n", + " initial_instance_count=1,\n", + " instance_type=inference_instance_type,\n", + " entry_point=\"inference.py\",\n", + " image_uri=deploy_image_uri,\n", + " source_dir=deploy_source_uri,\n", + " endpoint_name=endpoint_name_ag,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c5cf7b37", + "metadata": {}, + "outputs": [], + "source": [ + "# split the test data into smaller size of batches to query the endpoint if the test data has large size.\n", + "batch_size = 1500\n", + "predict_prob_ag = []\n", + "for i in np.arange(0, num_examples, step=batch_size):\n", + " query_response_batch = query_endpoint(\n", + " features.iloc[i : (i + batch_size), :].to_csv(header=False, index=False).encode(\"utf-8\"),\n", + " endpoint_name_ag,\n", + " )\n", + " predict_prob_batch = parse_response(query_response_batch) # prediction probability per batch\n", + " predict_prob_ag.append(predict_prob_batch)\n", + "\n", + "\n", + "predict_prob_ag = np.concatenate(predict_prob_ag, axis=0)\n", + "predict_label_ag = np.argmax(predict_prob_ag, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d86ccb2", + "metadata": {}, + "outputs": [], + "source": [ + "# Visualize the predictions results by plotting the confusion matrix.\n", + "conf_matrix = confusion_matrix(y_true=ground_truth_label.values, y_pred=predict_label_ag)\n", + "fig, ax = plt.subplots(figsize=(7.5, 7.5))\n", + "ax.matshow(conf_matrix, cmap=plt.cm.Blues, alpha=0.3)\n", + "for i in range(conf_matrix.shape[0]):\n", + " for j in range(conf_matrix.shape[1]):\n", + " ax.text(x=j, y=i, s=conf_matrix[i, j], va=\"center\", ha=\"center\", size=\"xx-large\")\n", + "\n", + "plt.xlabel(\"Predictions\", fontsize=18)\n", + "plt.ylabel(\"Actuals\", fontsize=18)\n", + "plt.title(\"Confusion Matrix\", fontsize=18)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2bfbab51", + "metadata": {}, + "outputs": [], + "source": [ + "# Measure the prediction results quantitatively.\n", + "eval_accuracy_ag = accuracy_score(ground_truth_label.values, predict_label_ag)\n", + "eval_f1_ag = f1_score(ground_truth_label.values, predict_label_ag)\n", + "eval_auc_ag = roc_auc_score(ground_truth_label.values, predict_prob_ag[:, 1])\n", + "\n", + "ag_results = pd.DataFrame.from_dict(\n", + " {\n", + " \"Accuracy\": eval_accuracy_ag,\n", + " \"F1\": eval_f1_ag,\n", + " \"AUC\": eval_auc_ag,\n", + " },\n", + " orient=\"index\",\n", + " columns=[\"AutoGluon-Tabular\"],\n", + ")\n", + "\n", + "results_lab_cat_tab_ag = pd.concat([results_lab_cat_tab, ag_results], axis=1)\n", + "results_lab_cat_tab_ag" + ] + }, + { + "cell_type": "markdown", + "id": "cbbfc102", + "metadata": {}, + "source": [ + "## 7. Compare Prediction Results of Four Trained Models on the Same Test Data\n", + "\n", + "For the three evaluation metrics accuracy, f1 score, and roc_auc, larger value indicates better results. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25ebee1c", + "metadata": {}, + "outputs": [], + "source": [ + "results_lab_cat_tab_ag" + ] + }, + { + "cell_type": "markdown", + "id": "b7f3a1eb", + "metadata": {}, + "source": [ + "Now you can use this template to evaluate the performance of LightGBM, CatBoost, TabTransformer, and AutoGluon-Tabular on your own dataset." + ] + }, + { + "cell_type": "markdown", + "id": "95194916", + "metadata": {}, + "source": [ + "---\n", + "Next, we delete the endpoint corresponding to the trained model.\n", + "\n", + "---" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8491a547", + "metadata": {}, + "outputs": [], + "source": [ + "# Delete the SageMaker endpoint and the attached resources\n", + "predictor.delete_model()\n", + "predictor.delete_endpoint()\n", + "predictor_cat.delete_model()\n", + "predictor_cat.delete_endpoint()\n", + "predictor_tab.delete_model()\n", + "predictor_tab.delete_endpoint()\n", + "predictor_ag.delete_model()\n", + "predictor_ag.delete_endpoint()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "conda_python3", + "language": "python", + "name": "conda_python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/introduction_to_applying_machine_learning/visual_object_detection/README.md b/introduction_to_applying_machine_learning/visual_object_detection/README.md new file mode 100644 index 0000000000..d592972b0e --- /dev/null +++ b/introduction_to_applying_machine_learning/visual_object_detection/README.md @@ -0,0 +1,145 @@ +# Visual Inspection Automation with Pre-trained Amazon SageMaker Models + +This solution detects product defects with an end-to-end Deep Learning workflow for quality control in manufacturing process. The solution takes input of product images and identifies defect regions with bounding boxes. In particular, this solution uses a pre-trained Sagemaker object detection model and fine-tune on the target dataset. + +This solution will demonstrate the immense advantage of fine-tuning a high-quality pre-trained model on the target dataset, both visually and numerically. + +### Contents +1. [Overview](#overview) + 1. [What Does the Input Data Look Like?](#input) + 2. [How to Prepare Your Data to Feed into the Model?](#preparedata) + 3. [What are the Outputs?](#output) + 4. [What is the Estimated Cost?](#cost) + 5. [What Algorithms & Models are Used?](#algorithms) + 6. [What Does the Data Flow Look Like?](#dataflow) +2. [Solution Details](#solution) + 1. [Background](#background) + 2. [What is Visual Inspection?](#inspection) + 3. [What are the Problems?](#problems) + 4. [What Does this Solution Offer?](#offer) +3. [Architecture Overview](#architecture) +4. [Cleaning up](#cleaning-up) +5. [Customization](#customization) + + +## 1. Overview + +### 1.1. What Does the Input Data Look Like? + +Input is an image of a defective / non-defective product. The training data should have relatively balanced classes, with annotations for ground truth defects (locations and defect types) per image. Here are examples of annotations used in the demo, they show some "inclusion" defects on the surface: + + + +The NEU surface defect database (see [references](#references)) is a *balanced* dataset which contains + +> Six kinds of typical surface defects of the hot-rolled steel strip are collected, i.e., rolled-in scale (RS), patches (Pa), crazing (Cr), pitted surface (PS), inclusion (In) and scratches (Sc). The database includes 1,800 grayscale images: 300 samples each of six different kinds of typical surface defects + +Here is a sample image of the six classes + + + +### 1.2. How to Prepare Your Data to Feed into the Model? + +There are data preparation and preprocessing steps and should be followed in the notebooks. It's critical to prepare your image annotations beforehand. +For finetuning pretrained Sagemaker models, you need to prepare either a single `annotation.json` for all data, or a `RecordIO` file for both all images and all annotations. Check the notebook for details. + +### 1.3. What are the Outputs? + +* For each image, the trained model will produce bounding boxes of detected visual defects (if any), the predicted defect type, and prediction confidence score (0~1). +* If you have a labeled test dataset, you could obtain the mean Average Precision (mAP) score for each model and compare among all the models. + * For example, the mAP scores on a test set of the NEU dataset + + | | Type1 | Type1+HPO | Type2 | Type2+HPO| + | --- | --- | --- | --- | ---| + | mAP | 0.067 | 0.226 | 0.371 | 0.375| + + +### 1.4. What is the Estimated Cost? + +* Running the notebook costs around $130~140 USD, assuming using p3.2xlarge EC2 instance in the notebook, and $3.06 on-demand hourly rate in US East. This notebook provides advanced materials, including finetuning two types of pretrained Sagemaker models **till convergence**, with and without hyperparameter optimization (HPO), and result in four models for inference. You could choose to train either one model, or all four models according to your budget and requirements. The cost and runtime for training each model are: + + | Model | Cost (USD) | Runtime (Hours) | Billable time (Hours)| + |:----------:|:---------------:|:----:|:-----:| + |Type 1| 1.5 | 0.5 | 0.5| + |Type 1 with HPO (20 jobs)| 30.6 | 1* | 10| + |Type 2| 4.6 | 1.5 | 1.5| + |Type 2 with HPO (20 jobs)| 92 | 3* | 30| + (*) HPO tasks in this solution consider 20 jobs in total and 10 jobs in parallel. So 1 actual runtime hour amounts to 10 billable cost hours. +* Please make sure you have read the cleaning up part in [Section 4](#cleaning-up) after training to avoid incurred cost from deployed models. + + + +### 1.5. What Algorithms & Models are Used? + +* The pretrained Sagemaker models include SSD models and FasterRCNN model, using either VGG, ResNet, or MobileNet as backbone, pretrained on either ImageNet, COCO, VOC, or FPN dataset. + +### 1.6. How Does the Data Flow Look Like? + + + +## 2. Solution Details + +### 2.1. Background + +According to the [Gartner study on the top 10 strategic tech trends for 2020](https://www.gartner.com/smarterwithgartner/gartner-top-10-strategic-technology-trends-for-2020/), hyper-automation is the number one trend in 2020 and will continue advancing in future. When it comes to manufacturing, one of the main barriers to hyper-automation is in areas where Human involvements is still struggling to be reduced and intelligent systems have hard times to become on-par with Human visual recognition abilities and become mainstream, despite great advancement of Deep Learning in Computer Vision. This is mainly due to lack of enough annotated data (or when data is sparse) in areas such as _Quality Control_ sections where trained Human eyes still dominates. + + +### 2.2. What is Visual Inspection? + +The **analysis of products on the production line for the purpose of Quality Control**. According to [Everything you need to know about Visual Inspection with AI](https://nanonets.com/blog/ai-visual-inspection/), visual inspection can also be used for internal and external assessment of the various equipment in a production facility such as storage tanks, pressure vessels, piping, and other equipment which expands to many industries from Electronics, Medical, Food and Raw Materials. + +### 2.3. What are the Problems? + +* *Human visual inspection error* is a major factor in this area. According to the report [The Role of Visual Inspection in the 21st Century](https://www.osti.gov/servlets/purl/1476816) + + > Most inspection tasks are much more complex and typically exhibit error rates of 20% to 30% (Drury & Fox, 1975) + +which directly translates to *cost*. +* Cost: according to [glassdoor estimate](https://www.glassdoor.co.in/Salaries/us-quality-control-inspector-salary-SRCH_IL.0,2_IN1_KO3,28.htm), a trained quality inspector salary varies between 29K (US) - 64K per year. + +### 2.4. What Does this Solution Offer? + +This solution offers a complete solution using high-quality pretrained Sagemaker models to finetune on the target dataset with and without hyperparameter optimization (HPO). + +The **most important** information this solution delivers, is that training a deep learning model from scratch on a small dataset can be both time-consuming and less effective, whereas finetuning a high-quality pretrained model, which was trained on large-scale dataset, could be both cost- and runtime-efficient and highly performant. Here are the sample detection results + +
+
+## 3. Architecture Overview
+
+The following illustration is the architecture for the end-to-end training and deployment process
+
+
+
+1. The input data located in an [Amazon S3](https://aws.amazon.com/s3/) bucket
+2. The provided [SageMaker notebook](source/deep_demand_forecast.ipynb) that gets the input data and launches the later stages below
+3. **Training Classifier and Detector models** and evaluating its results using Amazon SageMaker. If desired, one can deploy the trained models and create SageMaker endpoints
+4. **SageMaker endpoint** created from the previous step is an [HTTPS endpoint](https://docs.aws.amazon.com/sagemaker/latest/dg/how-it-works-hosting.html) and is capable of producing predictions
+5. Monitoring the training and deployed model via [Amazon CloudWatch](https://aws.amazon.com/cloudwatch/)
+
+## 4. Cleaning up
+
+If you run the notebook end-to-end, the Cleaning up section in the notebook will delete all the checkpoints and models automatically for you. If you choose to only train some of the four models in the notebook, please make sure to run corresponding code in the Cleaning up section to delete all the artifacts.
+
+**Caution:** You need to manually delete any extra resources that you may have created in this notebook. For examples extra Amazon S3 bucketis.
+
+## 5. Customization
+
+For using your own data, make sure it is labeled and is a *relatively* balanced dataset. Also make sure the image annotations follow the required format.
+
+
+
+### Useful Links
+
+* [Amazon SageMaker Getting Started](https://aws.amazon.com/sagemaker/getting-started/)
+* [Amazon SageMaker Developer Guide](https://docs.aws.amazon.com/sagemaker/latest/dg/whatis.html)
+* [Amazon SageMaker Python SDK Documentation](https://sagemaker.readthedocs.io/en/stable/)
+* [AWS CloudFormation User Guide](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html)
+
+### References
+
+* K. Song and Y. Yan, “A noise robust method based on completed local binary patterns for hot-rolled steel strip surface defects,” Applied Surface Science, vol. 285, pp. 858-864, Nov. 2013.
+
+* Yu He, Kechen Song, Qinggang Meng, Yunhui Yan, “An End-to-end Steel Surface Defect Detection Approach via Fusing Multiple Hierarchical Features,” IEEE Transactions on Instrumentation and Measuremente, 2020,69(4),1493-1504.
+
+* Hongwen Dong, Kechen Song, Yu He, Jing Xu, Yunhui Yan, Qinggang Meng, “PGA-Net: Pyramid Feature Fusion and Global Context Attention Network for Automated Surface Defect Detection,” IEEE Transactions on Industrial Informatics, 2020.
diff --git a/introduction_to_applying_machine_learning/visual_object_detection/numerical.png b/introduction_to_applying_machine_learning/visual_object_detection/numerical.png
new file mode 100644
index 0000000000..427d7583d3
Binary files /dev/null and b/introduction_to_applying_machine_learning/visual_object_detection/numerical.png differ
diff --git a/introduction_to_applying_machine_learning/visual_object_detection/patches_116.jpeg b/introduction_to_applying_machine_learning/visual_object_detection/patches_116.jpeg
new file mode 100644
index 0000000000..fc2443f9df
Binary files /dev/null and b/introduction_to_applying_machine_learning/visual_object_detection/patches_116.jpeg differ
diff --git a/introduction_to_applying_machine_learning/visual_object_detection/src/im2rec.py b/introduction_to_applying_machine_learning/visual_object_detection/src/im2rec.py
new file mode 100644
index 0000000000..16693351fc
--- /dev/null
+++ b/introduction_to_applying_machine_learning/visual_object_detection/src/im2rec.py
@@ -0,0 +1,409 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import print_function
+
+import argparse
+import os
+import random
+import sys
+import time
+import traceback
+
+import cv2
+import mxnet as mx
+
+
+curr_path = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(curr_path, "../python"))
+
+
+
+def list_image(root, recursive, exts):
+ """Traverses the root of directory that contains images and generates image list iterator.
+
+ Args:
+ root (str): the directory to explore
+ recursive (bool): whether subdirectories should be explored recursively
+ exts (collection[str]): the file extensions to look for
+
+ Returns:
+ image iterator that contains all the image under the specified path
+ """
+
+ i = 0
+ if recursive:
+ cat = {}
+ for path, dirs, files in os.walk(root, followlinks=True):
+ dirs.sort()
+ files.sort()
+ for fname in files:
+ fpath = os.path.join(path, fname)
+ suffix = os.path.splitext(fname)[1].lower()
+ if os.path.isfile(fpath) and (suffix in exts):
+ if path not in cat:
+ cat[path] = len(cat)
+ yield (i, os.path.relpath(fpath, root), cat[path])
+ i += 1
+ for k, v in sorted(cat.items(), key=lambda x: x[1]):
+ print(os.path.relpath(k, root), v)
+ else:
+ for fname in sorted(os.listdir(root)):
+ fpath = os.path.join(root, fname)
+ suffix = os.path.splitext(fname)[1].lower()
+ if os.path.isfile(fpath) and (suffix in exts):
+ yield (i, os.path.relpath(fpath, root), 0)
+ i += 1
+
+
+def write_list(path_out, image_list):
+ r"""Helper function to write image list into the file.
+
+ The format is as below, integer_image_index \t float_label_index \t path_to_image
+ Note that the blank between number and tab is only used for readability.
+
+ Args:
+ path_out (str): the path of the file to write to
+ image_list (list): objects in the images
+ """
+ with open(path_out, "w") as fout:
+ for i, item in enumerate(image_list):
+ line = "%d\t" % item[0]
+ for j in item[2:]:
+ line += "%f\t" % j
+ line += "%s\n" % item[1]
+ fout.write(line)
+
+
+def make_list(args):
+ """Generates .lst file.
+
+ Args:
+ args: object that contains all the arguments
+ """
+ image_list = list_image(args.root, args.recursive, args.exts)
+ image_list = list(image_list)
+ if args.shuffle is True:
+ random.seed(100)
+ random.shuffle(image_list)
+ N = len(image_list)
+ chunk_size = (N + args.chunks - 1) // args.chunks
+ for i in range(args.chunks):
+ chunk = image_list[i * chunk_size : (i + 1) * chunk_size]
+ if args.chunks > 1:
+ str_chunk = "_%d" % i
+ else:
+ str_chunk = ""
+ sep = int(chunk_size * args.train_ratio)
+ sep_test = int(chunk_size * args.test_ratio)
+ if args.train_ratio == 1.0:
+ write_list(args.prefix + str_chunk + ".lst", chunk)
+ else:
+ if args.test_ratio:
+ write_list(args.prefix + str_chunk + "_test.lst", chunk[:sep_test])
+ if args.train_ratio + args.test_ratio < 1.0:
+ write_list(args.prefix + str_chunk + "_val.lst", chunk[sep_test + sep :])
+ write_list(args.prefix + str_chunk + "_train.lst", chunk[sep_test : sep_test + sep])
+
+
+def read_list(path_in):
+ """Reads the .lst file and generates corresponding iterator.
+
+ Args:
+ path_in (str)
+
+ Returns:
+ item iterator that contains information in .lst file
+ """
+ with open(path_in) as fin:
+ while True:
+ line = fin.readline()
+ if not line:
+ break
+ line = [i.strip() for i in line.strip().split("\t")]
+ line_len = len(line)
+ # check the data format of .lst file
+ if line_len < 3:
+ print("lst should have at least has three parts, but only has %s parts for %s" % (line_len, line))
+ continue
+ try:
+ item = [int(line[0])] + [line[-1]] + [float(i) for i in line[1:-1]]
+ except Exception as e:
+ print("Parsing lst met error for %s, detail: %s" % (line, e))
+ continue
+ yield item
+
+
+def image_encode(args, i, item, q_out):
+ """Reads, preprocesses, packs the image and put it back in output queue.
+
+ Args"
+ args (object): image
+ i (int): image index
+ item (list): labels
+ q_out (queue): collection to store the image to
+ """
+ fullpath = os.path.join(args.root, item[1])
+
+ if len(item) > 3 and args.pack_label:
+ header = mx.recordio.IRHeader(0, item[2:], item[0], 0)
+ else:
+ header = mx.recordio.IRHeader(0, item[2], item[0], 0)
+
+ if args.pass_through:
+ try:
+ with open(fullpath, "rb") as fin:
+ img = fin.read()
+ s = mx.recordio.pack(header, img)
+ q_out.put((i, s, item))
+ except Exception as e:
+ traceback.print_exc()
+ print("pack_img error:", item[1], e)
+ q_out.put((i, None, item))
+ return
+
+ try:
+ img = cv2.imread(fullpath, args.color)
+ except Exception:
+ traceback.print_exc()
+ print("imread error trying to load file: %s " % fullpath)
+ q_out.put((i, None, item))
+ return
+ if img is None:
+ print("imread read blank (None) image for file: %s" % fullpath)
+ q_out.put((i, None, item))
+ return
+ if args.center_crop:
+ if img.shape[0] > img.shape[1]:
+ margin = (img.shape[0] - img.shape[1]) // 2
+ img = img[margin : margin + img.shape[1], :]
+ else:
+ margin = (img.shape[1] - img.shape[0]) // 2
+ img = img[:, margin : margin + img.shape[0]]
+ if args.resize:
+ if img.shape[0] > img.shape[1]:
+ newsize = (args.resize, img.shape[0] * args.resize // img.shape[1])
+ else:
+ newsize = (img.shape[1] * args.resize // img.shape[0], args.resize)
+ img = cv2.resize(img, newsize)
+
+ try:
+ s = mx.recordio.pack_img(header, img, quality=args.quality, img_fmt=args.encoding)
+ q_out.put((i, s, item))
+ except Exception as e:
+ traceback.print_exc()
+ print("pack_img error on file: %s" % fullpath, e)
+ q_out.put((i, None, item))
+ return
+
+
+def read_worker(args, q_in, q_out):
+ """Function that will be spawned to fetch the image from the input queue and put it back to output queue.
+
+ Args:
+ args (object): image
+ q_in (queue): input queue
+ q_out (queue): output queue
+ """
+ while True:
+ deq = q_in.get()
+ if deq is None:
+ break
+ i, item = deq
+ image_encode(args, i, item, q_out)
+
+
+def write_worker(q_out, fname, working_dir):
+ """Function that will be spawned to fetch processed image from the output queue and write to the .rec file.
+
+ Args:
+ q_out (queue): output queue
+ fname (str): name of the files
+ working_dir (str): name of directory to write to
+ """
+ pre_time = time.time()
+ count = 0
+ fname = os.path.basename(fname)
+ fname_rec = os.path.splitext(fname)[0] + ".rec"
+ fname_idx = os.path.splitext(fname)[0] + ".idx"
+ record = mx.recordio.MXIndexedRecordIO(
+ os.path.join(working_dir, fname_idx), os.path.join(working_dir, fname_rec), "w"
+ )
+ buf = {}
+ more = True
+ while more:
+ deq = q_out.get()
+ if deq is not None:
+ i, s, item = deq
+ buf[i] = (s, item)
+ else:
+ more = False
+ while count in buf:
+ s, item = buf[count]
+ del buf[count]
+ if s is not None:
+ record.write_idx(item[0], s)
+
+ if count % 1000 == 0:
+ cur_time = time.time()
+ print("time:", cur_time - pre_time, " count:", count)
+ pre_time = cur_time
+ count += 1
+
+
+def parse_args():
+ """Defines all arguments.
+
+ Returns:
+ args object that contains all the params
+ """
+ parser = argparse.ArgumentParser(
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+ description="Create an image list or \
+ make a record database by reading from an image list",
+ )
+ parser.add_argument("prefix", help="prefix of input/output lst and rec files.")
+ parser.add_argument("root", help="path to folder containing images.")
+
+ cgroup = parser.add_argument_group("Options for creating image lists")
+ cgroup.add_argument(
+ "--list",
+ action="store_true",
+ help="If this is set im2rec will create image list(s) by traversing root folder\
+ and output to ![\"drawing\"](\"./numerical.png\")
\n",
+ "\n",
+ "If you predict all test images using all endpoints, you end up with this table. The pycocotools package reports more metric values. We wil focus on row 1 - the mAP averaged over all IoU thresholds, all recall thresholds, all region sizes (small, medium, large), and all numbers of predicted bbox (1, 10, and 100), and all object categories. It's the [standard practice](https://cocodataset.org/#detection-eval) to use this metric for evaluating object detection algorithms."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "papermill": {
+ "duration": 2.403376,
+ "end_time": "2022-08-05T14:36:28.020761",
+ "exception": false,
+ "start_time": "2022-08-05T14:36:25.617385",
+ "status": "completed"
+ },
+ "pycharm": {
+ "name": "#%% md\n"
+ },
+ "tags": []
+ },
+ "source": [
+ "### 7. Clean Up the Endpoints\n",
+ "\n",
+ "When you are done with the endpoint, you should clean it up.\n",
+ "\n",
+ "All of the training jobs, models and endpoints we created can be viewed through the SageMaker console of your AWS account."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {
+ "execution": {
+ "iopub.execute_input": "2022-08-05T14:36:32.833309Z",
+ "iopub.status.busy": "2022-08-05T14:36:32.832838Z",
+ "iopub.status.idle": "2022-08-05T14:36:39.752304Z",
+ "shell.execute_reply": "2022-08-05T14:36:39.751877Z"
+ },
+ "papermill": {
+ "duration": 9.328678,
+ "end_time": "2022-08-05T14:36:39.752413",
+ "exception": false,
+ "start_time": "2022-08-05T14:36:30.423735",
+ "status": "completed"
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# Delete the SageMaker endpoint\n",
+ "od_type1_predictor.delete_model()\n",
+ "od_type1_predictor.delete_endpoint()\n",
+ "\n",
+ "od_type1_hpo_predictor.delete_model()\n",
+ "od_type1_hpo_predictor.delete_endpoint()\n",
+ "\n",
+ "od_type2_predictor.delete_model()\n",
+ "od_type2_predictor.delete_endpoint()\n",
+ "\n",
+ "od_type2_hpo_predictor.delete_model()\n",
+ "od_type2_hpo_predictor.delete_endpoint()\n",
+ "\n",
+ "# You should go to the Sagemaker console and manually delete the DDN model endpoint"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "papermill": {
+ "duration": 2.415178,
+ "end_time": "2022-08-05T14:36:44.577598",
+ "exception": false,
+ "start_time": "2022-08-05T14:36:42.162420",
+ "status": "completed"
+ },
+ "pycharm": {
+ "name": "#%% md\n"
+ },
+ "tags": []
+ },
+ "source": [
+ "## 8. Conclusion\n",
+ "\n",
+ "Both visual and numerical comparison confirm that the Type 2 (latest) OD model or Type 2 (latest) OD + HPO performs the best. \n",
+ "\n",
+ "1. Training models from scratch can be very time-consuming and less effective. In this example, the target dataset is very small, consisting of only 1,800 images in 6 categories, and the training data is only 64% of this small dataset.\n",
+ "2. The built-in Sagemaker OD models were pre-trained on large-scale dataset, e.g., the ImageNet dataset includes 14,197,122 images for 21,841 categories, and the PASCAL VOC dataset includes 11,530 images for 20 categories. The pre-trained models have learned rich and diverse low level features, and can efficiently transfer knowledge to finetuned models and focus on learning high-level semantic features for the target dataset.\n",
+ "3. HPO is extremely effective, especially for models with large hyperparameter search spaces. Since we finetuned on three hyperparameters (learning rate, momentum, and weight decay) for the Type 1 (legacy) OD models and only one hyperparameter (adam learning rate) for the Type 2 (latest) OD model, there is relatively larger room for improvement for the Type 1 (legacy) OD model and we do observe larger performance enhancement. Of course, we need to trade off model performance with budget (compute resource and training time) when running HPO.\n",
+ "4. In terms of training time, for the steel surface dataset, training the Type 1 (legacy) OD model took 34 min, Type 2 (latest) OD model took 1 hour, and the model trained from scratch took 8+ hours. It indicates finetuning a pre-trained model is much more efficient.\n",
+ "5. In summary, finetuning a pretrained model is both more efficient and more performant, we suggest taking advantage of the pre-trained Sagemaker built-in models and finetune on your target datasets.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "papermill": {
+ "duration": 2.382722,
+ "end_time": "2022-08-05T14:36:49.342790",
+ "exception": false,
+ "start_time": "2022-08-05T14:36:46.960068",
+ "status": "completed"
+ },
+ "pycharm": {
+ "name": "#%%\n"
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "instance_type": "ml.g4dn.xlarge",
+ "kernelspec": {
+ "display_name": "Python 3 (PyTorch 1.8 Python 3.6 CPU Optimized)",
+ "language": "python",
+ "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-west-2:236514542706:image/1.8.1-cpu-py36"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.13"
+ },
+ "papermill": {
+ "default_parameters": {},
+ "duration": 23169.556233,
+ "end_time": "2022-08-05T14:36:52.361701",
+ "environment_variables": {},
+ "exception": null,
+ "input_path": "4_finetune-processed.ipynb",
+ "output_path": "4_finetune-processed-output.ipynb",
+ "parameters": {},
+ "start_time": "2022-08-05T08:10:42.805468",
+ "version": "2.3.4"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/licenses/2-CLAUSE-BSD b/licenses/2-CLAUSE-BSD
new file mode 100644
index 0000000000..b79ea4a15c
--- /dev/null
+++ b/licenses/2-CLAUSE-BSD
@@ -0,0 +1,28 @@
+2-Clause BSD License
+=====================
+
+Copyright (c) 2007-2020 by the Sphinx team (see AUTHORS file).
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
diff --git a/ml-lifecycle/index.rst b/ml-lifecycle/index.rst
new file mode 100644
index 0000000000..8f7e17178a
--- /dev/null
+++ b/ml-lifecycle/index.rst
@@ -0,0 +1,7 @@
+Feature Store
+---------------------------
+
+.. toctree::
+ :maxdepth: 1
+
+ feature_store/FS_demo
diff --git a/model-governance/index.rst b/model-governance/index.rst
new file mode 100644
index 0000000000..838392f015
--- /dev/null
+++ b/model-governance/index.rst
@@ -0,0 +1,7 @@
+Model Governance
+---------------------
+
+.. toctree::
+ :maxdepth: 0
+
+ model_governance_graph
diff --git a/r_examples/index.rst b/r_examples/index.rst
index 8609592361..ab927f7fda 100644
--- a/r_examples/index.rst
+++ b/r_examples/index.rst
@@ -10,3 +10,7 @@ R Examples
r_batch_transform/r_xgboost_batch_transform
r_xgboost_hpo_batch_transform/r_xgboost_hpo_batch_transform
r_byo_r_algo_hpo/tune_r_bring_your_own
+ r_serving_with_plumber/Plumber_Example
+ r_sagemaker_binary_classification_algorithms/R_binary_classification_algorithms_comparison
+ r_serving_with_fastapi/FastAPI_Example
+ r_serving_with_restrserve/RestRServe_Example
diff --git a/r_examples/r_api_serving_examples/API Serving Examples.ipynb b/r_examples/r_api_serving_examples/API Serving Examples.ipynb
deleted file mode 100644
index cb85c5bac6..0000000000
--- a/r_examples/r_api_serving_examples/API Serving Examples.ipynb
+++ /dev/null
@@ -1,610 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# R API Serving Examples\n",
- "\n",
- "In this example, we demonstrate how to quickly compare the runtimes of three methods for serving a model from an R hosted REST API. The following SageMaker examples discuss each method in detail:\n",
- "\n",
- "* **Plumber**\n",
- " * Website: [https://www.rplumber.io/](https://www.rplumber.io)\n",
- " * SageMaker Example: [r_serving_with_plumber](../r_serving_with_plumber)\n",
- "* **RestRServe**\n",
- " * Website: [https://restrserve.org](https://restrserve.org)\n",
- " * SageMaker Example: [r_serving_with_restrserve](../r_serving_with_restrserve)\n",
- "* **FastAPI** (reticulated from Python)\n",
- " * Website: [https://fastapi.tiangolo.com](https://fastapi.tiangolo.com)\n",
- " * SageMaker Example: [r_serving_with_fastapi](../r_serving_with_fastapi)\n",
- " \n",
- "We will reuse the docker images from each of these examples. Each one is configured to serve a small XGBoost model which has already been trained on the classical Iris dataset."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Building Docker Images for Serving\n",
- "\n",
- "First, we will build each docker image from the provided SageMaker Examples."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Plumber Serving Image"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "scrolled": true
- },
- "outputs": [],
- "source": [
- "!cd .. && docker build -t r-plumber -f r_serving_with_plumber/Dockerfile r_serving_with_plumber"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### RestRServe Serving Image"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "scrolled": true
- },
- "outputs": [],
- "source": [
- "!cd .. && docker build -t r-restrserve -f r_serving_with_restrserve/Dockerfile r_serving_with_restrserve"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### FastAPI Serving Image"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "scrolled": true
- },
- "outputs": [],
- "source": [
- "!cd .. && docker build -t r-fastapi -f r_serving_with_fastapi/Dockerfile r_serving_with_fastapi"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Launch Serving Containers"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Next, we will launch each search container. The containers will be launch on the following ports to avoid port collisions on your local machine or SageMaker Notebook instance:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "ports = {\n",
- " \"plumber\": 5000,\n",
- " \"restrserve\": 5001,\n",
- " \"fastapi\": 5002,\n",
- "}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!bash launch.sh"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!docker container list"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Define Simple Client"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import requests\n",
- "from tqdm import tqdm\n",
- "import pandas as pd"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_predictions(examples, instance=requests, port=5000):\n",
- " payload = {\"features\": examples}\n",
- " return instance.post(f\"http://127.0.0.1:{port}/invocations\", json=payload)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_health(instance=requests, port=5000):\n",
- " instance.get(f\"http://127.0.0.1:{port}/ping\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Define Example Inputs"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Next, we define a example inputs from the classical [Iris](https://archive.ics.uci.edu/ml/datasets/iris) dataset.\n",
- "* Dua, D. and Graff, C. (2019). UCI Machine Learning Repository [http://archive.ics.uci.edu/ml]. Irvine, CA: University of California, School of Information and Computer Science."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "column_names = [\"Sepal.Length\", \"Sepal.Width\", \"Petal.Length\", \"Petal.Width\", \"Label\"]\n",
- "iris = pd.read_csv(\n",
- " \"s3://sagemaker-sample-files/datasets/tabular/iris/iris.data\", names=column_names\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "iris_features = iris[[\"Sepal.Length\", \"Sepal.Width\", \"Petal.Length\", \"Petal.Width\"]]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "example = iris_features.values[:1].tolist()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "many_examples = iris_features.values[:100].tolist()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Testing\n",
- "\n",
- "Now it's time to test how each API server performs under stress."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We will test two use cases:\n",
- "* **New Requests**: In this scenario, we test how quickly the server can respond with predictions when each client request establishes a new connection with the server. This simulates the server's ability to handle real-time requests. We could make this more realistic by creating an asynchronous environment that tests the server's ability to fulfill concurrent rather than sequential requests.\n",
- "* **Keep Alive / Reuse Session**: In this scenario, we test how quickly the server can respond with predictions when each client request uses a session to keep its connection to the server alive between requests. This simulates the server's ability to handle sequential batch requests from the same client."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "For each of the two use cases, we will test the performance on following situations:\n",
- "\n",
- "* 1000 requests of a single example\n",
- "* 1000 requests of 100 examples\n",
- "* 1000 pings for health status"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## New Requests"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Plumber"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# verify the prediction output\n",
- "get_predictions(example, port=ports[\"plumber\"]).json()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(example, port=ports[\"plumber\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(many_examples, port=ports[\"plumber\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " get_health(port=ports[\"plumber\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### RestRserve"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# verify the prediction output\n",
- "get_predictions(example, port=ports[\"restrserve\"]).json()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(example, port=ports[\"restrserve\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(many_examples, port=ports[\"restrserve\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " get_health(port=ports[\"restrserve\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### FastAPI"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# verify the prediction output\n",
- "get_predictions(example, port=ports[\"fastapi\"]).json()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(example, port=ports[\"fastapi\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(many_examples, port=ports[\"fastapi\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " get_health(port=ports[\"fastapi\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Keep Alive (Reuse Session)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now, let's test how each one performs when each request reuses a session connection. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# reuse the session for each post and get request\n",
- "instance = requests.Session()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Plumber"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(example, instance=instance, port=ports[\"plumber\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(many_examples, instance=instance, port=ports[\"plumber\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " get_health(instance=instance, port=ports[\"plumber\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### RestRserve"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(example, instance=instance, port=ports[\"restrserve\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(many_examples, instance=instance, port=ports[\"restrserve\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " get_health(instance=instance, port=ports[\"restrserve\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### FastAPI"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(example, instance=instance, port=ports[\"fastapi\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " _ = get_predictions(many_examples, instance=instance, port=ports[\"fastapi\"])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for i in tqdm(range(1000)):\n",
- " get_health(instance=instance, port=ports[\"fastapi\"])"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Stop All Serving Containers"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Finally, we will shut down the serving containers we launched for the tests."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!docker kill $(docker ps -q)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Conclusion"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "In this example, we demonstrated how to conduct a simple performance benchmark across three R model serving solutions. We leave the choice of serving solution up to the reader since in some cases it might be appropriate to customize the benchmark in the following ways:\n",
- "\n",
- "* Update the serving example to serve a specific model\n",
- "* Perform the tests across multiple instances types\n",
- "* Modify the serving example and client to test asynchronous requests.\n",
- "* Deploy the serving examples to SageMaker Endpoints to test within an autoscaling environment.\n",
- "\n",
- "For more information on serving your models in custom containers on SageMaker, please see our [support documentation](https://docs.aws.amazon.com/sagemaker/latest/dg/your-algorithms-inference-main.html) for the latest updates and best practices."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "conda_python3",
- "language": "python",
- "name": "conda_python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.13"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/r_examples/r_api_serving_examples/iris.csv b/r_examples/r_api_serving_examples/iris.csv
deleted file mode 100644
index 8b6393099a..0000000000
--- a/r_examples/r_api_serving_examples/iris.csv
+++ /dev/null
@@ -1,151 +0,0 @@
-Sepal.Length,Sepal.Width,Petal.Length,Petal.Width,Species
-5.1,3.5,1.4,0.2,setosa
-4.9,3,1.4,0.2,setosa
-4.7,3.2,1.3,0.2,setosa
-4.6,3.1,1.5,0.2,setosa
-5,3.6,1.4,0.2,setosa
-5.4,3.9,1.7,0.4,setosa
-4.6,3.4,1.4,0.3,setosa
-5,3.4,1.5,0.2,setosa
-4.4,2.9,1.4,0.2,setosa
-4.9,3.1,1.5,0.1,setosa
-5.4,3.7,1.5,0.2,setosa
-4.8,3.4,1.6,0.2,setosa
-4.8,3,1.4,0.1,setosa
-4.3,3,1.1,0.1,setosa
-5.8,4,1.2,0.2,setosa
-5.7,4.4,1.5,0.4,setosa
-5.4,3.9,1.3,0.4,setosa
-5.1,3.5,1.4,0.3,setosa
-5.7,3.8,1.7,0.3,setosa
-5.1,3.8,1.5,0.3,setosa
-5.4,3.4,1.7,0.2,setosa
-5.1,3.7,1.5,0.4,setosa
-4.6,3.6,1,0.2,setosa
-5.1,3.3,1.7,0.5,setosa
-4.8,3.4,1.9,0.2,setosa
-5,3,1.6,0.2,setosa
-5,3.4,1.6,0.4,setosa
-5.2,3.5,1.5,0.2,setosa
-5.2,3.4,1.4,0.2,setosa
-4.7,3.2,1.6,0.2,setosa
-4.8,3.1,1.6,0.2,setosa
-5.4,3.4,1.5,0.4,setosa
-5.2,4.1,1.5,0.1,setosa
-5.5,4.2,1.4,0.2,setosa
-4.9,3.1,1.5,0.2,setosa
-5,3.2,1.2,0.2,setosa
-5.5,3.5,1.3,0.2,setosa
-4.9,3.6,1.4,0.1,setosa
-4.4,3,1.3,0.2,setosa
-5.1,3.4,1.5,0.2,setosa
-5,3.5,1.3,0.3,setosa
-4.5,2.3,1.3,0.3,setosa
-4.4,3.2,1.3,0.2,setosa
-5,3.5,1.6,0.6,setosa
-5.1,3.8,1.9,0.4,setosa
-4.8,3,1.4,0.3,setosa
-5.1,3.8,1.6,0.2,setosa
-4.6,3.2,1.4,0.2,setosa
-5.3,3.7,1.5,0.2,setosa
-5,3.3,1.4,0.2,setosa
-7,3.2,4.7,1.4,versicolor
-6.4,3.2,4.5,1.5,versicolor
-6.9,3.1,4.9,1.5,versicolor
-5.5,2.3,4,1.3,versicolor
-6.5,2.8,4.6,1.5,versicolor
-5.7,2.8,4.5,1.3,versicolor
-6.3,3.3,4.7,1.6,versicolor
-4.9,2.4,3.3,1,versicolor
-6.6,2.9,4.6,1.3,versicolor
-5.2,2.7,3.9,1.4,versicolor
-5,2,3.5,1,versicolor
-5.9,3,4.2,1.5,versicolor
-6,2.2,4,1,versicolor
-6.1,2.9,4.7,1.4,versicolor
-5.6,2.9,3.6,1.3,versicolor
-6.7,3.1,4.4,1.4,versicolor
-5.6,3,4.5,1.5,versicolor
-5.8,2.7,4.1,1,versicolor
-6.2,2.2,4.5,1.5,versicolor
-5.6,2.5,3.9,1.1,versicolor
-5.9,3.2,4.8,1.8,versicolor
-6.1,2.8,4,1.3,versicolor
-6.3,2.5,4.9,1.5,versicolor
-6.1,2.8,4.7,1.2,versicolor
-6.4,2.9,4.3,1.3,versicolor
-6.6,3,4.4,1.4,versicolor
-6.8,2.8,4.8,1.4,versicolor
-6.7,3,5,1.7,versicolor
-6,2.9,4.5,1.5,versicolor
-5.7,2.6,3.5,1,versicolor
-5.5,2.4,3.8,1.1,versicolor
-5.5,2.4,3.7,1,versicolor
-5.8,2.7,3.9,1.2,versicolor
-6,2.7,5.1,1.6,versicolor
-5.4,3,4.5,1.5,versicolor
-6,3.4,4.5,1.6,versicolor
-6.7,3.1,4.7,1.5,versicolor
-6.3,2.3,4.4,1.3,versicolor
-5.6,3,4.1,1.3,versicolor
-5.5,2.5,4,1.3,versicolor
-5.5,2.6,4.4,1.2,versicolor
-6.1,3,4.6,1.4,versicolor
-5.8,2.6,4,1.2,versicolor
-5,2.3,3.3,1,versicolor
-5.6,2.7,4.2,1.3,versicolor
-5.7,3,4.2,1.2,versicolor
-5.7,2.9,4.2,1.3,versicolor
-6.2,2.9,4.3,1.3,versicolor
-5.1,2.5,3,1.1,versicolor
-5.7,2.8,4.1,1.3,versicolor
-6.3,3.3,6,2.5,virginica
-5.8,2.7,5.1,1.9,virginica
-7.1,3,5.9,2.1,virginica
-6.3,2.9,5.6,1.8,virginica
-6.5,3,5.8,2.2,virginica
-7.6,3,6.6,2.1,virginica
-4.9,2.5,4.5,1.7,virginica
-7.3,2.9,6.3,1.8,virginica
-6.7,2.5,5.8,1.8,virginica
-7.2,3.6,6.1,2.5,virginica
-6.5,3.2,5.1,2,virginica
-6.4,2.7,5.3,1.9,virginica
-6.8,3,5.5,2.1,virginica
-5.7,2.5,5,2,virginica
-5.8,2.8,5.1,2.4,virginica
-6.4,3.2,5.3,2.3,virginica
-6.5,3,5.5,1.8,virginica
-7.7,3.8,6.7,2.2,virginica
-7.7,2.6,6.9,2.3,virginica
-6,2.2,5,1.5,virginica
-6.9,3.2,5.7,2.3,virginica
-5.6,2.8,4.9,2,virginica
-7.7,2.8,6.7,2,virginica
-6.3,2.7,4.9,1.8,virginica
-6.7,3.3,5.7,2.1,virginica
-7.2,3.2,6,1.8,virginica
-6.2,2.8,4.8,1.8,virginica
-6.1,3,4.9,1.8,virginica
-6.4,2.8,5.6,2.1,virginica
-7.2,3,5.8,1.6,virginica
-7.4,2.8,6.1,1.9,virginica
-7.9,3.8,6.4,2,virginica
-6.4,2.8,5.6,2.2,virginica
-6.3,2.8,5.1,1.5,virginica
-6.1,2.6,5.6,1.4,virginica
-7.7,3,6.1,2.3,virginica
-6.3,3.4,5.6,2.4,virginica
-6.4,3.1,5.5,1.8,virginica
-6,3,4.8,1.8,virginica
-6.9,3.1,5.4,2.1,virginica
-6.7,3.1,5.6,2.4,virginica
-6.9,3.1,5.1,2.3,virginica
-5.8,2.7,5.1,1.9,virginica
-6.8,3.2,5.9,2.3,virginica
-6.7,3.3,5.7,2.5,virginica
-6.7,3,5.2,2.3,virginica
-6.3,2.5,5,1.9,virginica
-6.5,3,5.2,2,virginica
-6.2,3.4,5.4,2.3,virginica
-5.9,3,5.1,1.8,virginica
diff --git a/r_examples/r_api_serving_examples/launch.sh b/r_examples/r_api_serving_examples/launch.sh
deleted file mode 100644
index e456602d35..0000000000
--- a/r_examples/r_api_serving_examples/launch.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-echo "Launching Plumber"
-docker run -d --rm -p 5000:8080 r-plumber
-
-echo "Launching RestRServer"
-docker run -d --rm -p 5001:8080 r-restrserve
-
-echo "Launching FastAPI"
-docker run -d --rm -p 5002:8080 r-fastapi
-
diff --git a/reinforcement_learning/index.rst b/reinforcement_learning/index.rst
index 0dddd0b6af..e9ce08ade2 100644
--- a/reinforcement_learning/index.rst
+++ b/reinforcement_learning/index.rst
@@ -177,6 +177,17 @@ Play global thermonuclear war with a computer.
rl_tic_tac_toe_coach_customEnv/rl_tic_tac_toe_coach_customEnv
+Stock Trading
+------------
+
+Try stock trading with SageMaker RL.
+
+.. toctree::
+ :maxdepth: 0
+
+ rl_stock_trading_coach_customEnv/rl_stock_trading_coach_customEnv
+
+
Traveling salesman problem
--------------------------
diff --git a/sagemaker-clarify/index.rst b/sagemaker-clarify/index.rst
index fc5d79b652..7bca215dcf 100644
--- a/sagemaker-clarify/index.rst
+++ b/sagemaker-clarify/index.rst
@@ -26,3 +26,12 @@ SageMaker Clarify Model Monitoring
:maxdepth: 1
../sagemaker_model_monitor/fairness_and_explainability/SageMaker-Model-Monitor-Fairness-and-Explainability
+
+SageMaker Clarify Online Explainability
+---------------------------------------
+
+.. toctree::
+ :maxdepth: 1
+
+ online_explainability/tabular/tabular_online_explainability_with_sagemaker_clarify
+ online_explainability/natural_language_processing/nlp_online_explainability_with_sagemaker_clarify
diff --git a/sagemaker-clarify/online_explainability/natural_language_processing/code/inference.py b/sagemaker-clarify/online_explainability/natural_language_processing/code/inference.py
new file mode 100644
index 0000000000..79d4b72789
--- /dev/null
+++ b/sagemaker-clarify/online_explainability/natural_language_processing/code/inference.py
@@ -0,0 +1,57 @@
+from io import StringIO
+import numpy as np
+import os
+import pandas as pd
+import json
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import torch
+from typing import Any, Dict, List
+
+
+def model_fn(model_dir: str) -> Dict[str, Any]:
+ """
+ Load the model for inference
+ """
+ model_path = os.path.join(model_dir, "model")
+
+ # Load HuggingFace tokenizer.
+ tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+
+ # Load HuggingFace model from disk.
+ model = AutoModelForSequenceClassification.from_pretrained(model_path, local_files_only=True)
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ model.to(device)
+ model.eval()
+ model_dict = {"model": model, "tokenizer": tokenizer}
+ return model_dict
+
+
+def predict_fn(input_data: List, model: Dict) -> np.ndarray:
+ """
+ Apply model to the incoming request
+ """
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ tokenizer = model["tokenizer"]
+ huggingface_model = model["model"]
+
+ encoded_input = tokenizer(input_data, truncation=True, padding=True, max_length=128, return_tensors="pt")
+ encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+ with torch.no_grad():
+ output = huggingface_model(input_ids=encoded_input["input_ids"], attention_mask=encoded_input["attention_mask"])
+ res = torch.nn.Softmax(dim=1)(output.logits).detach().cpu().numpy()[:, 1]
+ return res
+
+
+def input_fn(request_body: str, request_content_type: str) -> List[str]:
+ """
+ Deserialize and prepare the prediction input
+ """
+ if request_content_type == "application/json":
+ sentences = [json.loads(request_body)]
+
+ elif request_content_type == "text/csv":
+ # We have a single column with the text.
+ sentences = list(pd.read_csv(StringIO(request_body), header=None).values[:, 0].astype(str))
+ else:
+ sentences = request_body
+ return sentences
diff --git a/sagemaker-clarify/online_explainability/natural_language_processing/code/requirements.txt b/sagemaker-clarify/online_explainability/natural_language_processing/code/requirements.txt
new file mode 100644
index 0000000000..721806cd03
--- /dev/null
+++ b/sagemaker-clarify/online_explainability/natural_language_processing/code/requirements.txt
@@ -0,0 +1,3 @@
+transformers==4.2.1
+torch==1.7.1
+pandas
\ No newline at end of file
diff --git a/sagemaker-clarify/online_explainability/natural_language_processing/nlp_online_explainability_with_sagemaker_clarify.ipynb b/sagemaker-clarify/online_explainability/natural_language_processing/nlp_online_explainability_with_sagemaker_clarify.ipynb
new file mode 100644
index 0000000000..ac77aa7e77
--- /dev/null
+++ b/sagemaker-clarify/online_explainability/natural_language_processing/nlp_online_explainability_with_sagemaker_clarify.ipynb
@@ -0,0 +1,1097 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "# NLP Online Explainability with SageMaker Clarify"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "* [Introduction](#Introduction)\n",
+ "* [General Setup](#General-Setup)\n",
+ " * [Install dependencies](#Install-dependencies)\n",
+ " * [Import libraries](#Import-libraries)\n",
+ " * [Set configurations](#Set-configurations)\n",
+ " * [Create serializer and deserializer](#Create-serializer-and-deserializer)\n",
+ " * [For visualization](#For-visualization)\n",
+ "* [Prepare data](#Prepare-data)\n",
+ " * [Download data](#Download-data)\n",
+ " * [Loading the data](#Loading-the-data)\n",
+ " * [Data preparation for model training](#Data-preparation-for-model-training)\n",
+ " * [Upload the dataset](#Upload-the-dataset)\n",
+ "* [Train and Deploy Hugging Face Model](#Train-and-Deploy-Hugging-Face-Model)\n",
+ " * [Train model with Hugging Face estimator](#Train-model-with-Hugging-Face-estimator)\n",
+ " * [Download the trained model files](#Download-the-trained-model-files)\n",
+ " * [Prepare model container definition](#Prepare-model-container-definition)\n",
+ "* [Create endpoint](#Create-endpoint)\n",
+ " * [Create model](#Create-model)\n",
+ " * [Create endpoint config](#Create-endpoint-config)\n",
+ " * [Create endpoint](#Create-endpoint)\n",
+ "* [Invoke endpoint](#Invoke-endpoint)\n",
+ " * [Single record request](#Single-record-request)\n",
+ " * [Single record request, no explanation](#Single-record-request,-no-explanation)\n",
+ " * [Batch request, explain both](#Batch-request,-explain-both)\n",
+ " * [Batch request, explain none](#Batch-request,-explain-none)\n",
+ " * [Batch request with more records, explain some of the records](#Batch-request-with-more-records,-explain-some-of-the-records)\n",
+ "* [Cleanup](#Cleanup)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Introduction\n",
+ "\n",
+ "Amazon SageMaker Clarify helps improve your machine learning models by detecting potential bias and helping explain how these models make predictions. The fairness and explainability functionality provided by SageMaker Clarify takes a step towards enabling AWS customers to build trustworthy and understandable machine learning models. \n",
+ "\n",
+ "SageMaker Clarify currently supports explainability for SageMaker models as an offline processing job. This example notebook showcases a new feature for explainability on a [SageMaker real-time inference](https://docs.aws.amazon.com/sagemaker/latest/dg/realtime-endpoints.html) endpoint, a.k.a. online explainability.\n",
+ "\n",
+ "This example notebook walks you through: \n",
+ "1. Key terms and concepts needed to understand SageMaker Clarify\n",
+ "1. Trained the model on the Women's ecommerce clothing reviews dataset.\n",
+ "1. Create a model from trained model artifacts, create an endpoint configuration with the new SageMaker Clarify explainer configuration, and create an endpoint using the same explainer configuration.\n",
+ "1. Invoke the endpoint with single and batch request with different `EnableExplanations` query.\n",
+ "1. Explaining the importance of the various input features on the model's decision.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## General Setup\n",
+ "\n",
+ "We recommend you use `Python 3 (Data Science)` kernel on SageMaker Studio or `conda_python3` kernel on SageMaker Notebook Instance."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Install dependencies\n",
+ "\n",
+ "The following packages are required by data preparation and training."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install \"datasets[s3]==1.6.2\" \"transformers==4.6.1\" --upgrade"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Upgrade the SageMaker Python SDK, and captum is used to visualize the feature attributions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install sagemaker --upgrade\n",
+ "!pip install boto3 --upgrade\n",
+ "!pip install botocore --upgrade"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install captum --upgrade"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Import libraries"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import boto3\n",
+ "import csv\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import pprint\n",
+ "import tarfile\n",
+ "\n",
+ "from sagemaker.huggingface import HuggingFace\n",
+ "from datasets import Dataset\n",
+ "from datasets.filesystems import S3FileSystem\n",
+ "from captum.attr import visualization\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sagemaker import get_execution_role, Session\n",
+ "from sagemaker.serializers import CSVSerializer\n",
+ "from sagemaker.deserializers import JSONDeserializer\n",
+ "from sagemaker.utils import unique_name_from_base"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Set configurations"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "boto3_session = boto3.session.Session()\n",
+ "sagemaker_client = boto3.client(\"sagemaker\")\n",
+ "sagemaker_runtime_client = boto3.client(\"sagemaker-runtime\")\n",
+ "\n",
+ "# Initialize sagemaker session\n",
+ "sagemaker_session = Session(\n",
+ " boto_session=boto3_session,\n",
+ " sagemaker_client=sagemaker_client,\n",
+ " sagemaker_runtime_client=sagemaker_runtime_client,\n",
+ ")\n",
+ "\n",
+ "region = sagemaker_session.boto_region_name\n",
+ "print(f\"Region: {region}\")\n",
+ "\n",
+ "role = get_execution_role()\n",
+ "print(f\"Role: {role}\")\n",
+ "\n",
+ "prefix = unique_name_from_base(\"DEMO-NLP-Women-Clothing\")\n",
+ "\n",
+ "s3_bucket = sagemaker_session.default_bucket()\n",
+ "s3_prefix = f\"sagemaker/{prefix}\"\n",
+ "s3_key = f\"s3://{s3_bucket}/{s3_prefix}\"\n",
+ "print(f\"Demo S3 key: {s3_key}\")\n",
+ "\n",
+ "model_name = f\"{prefix}-model\"\n",
+ "print(f\"Demo model name: {model_name}\")\n",
+ "endpoint_config_name = f\"{prefix}-endpoint-config\"\n",
+ "print(f\"Demo endpoint config name: {endpoint_config_name}\")\n",
+ "endpoint_name = f\"{prefix}-endpoint\"\n",
+ "print(f\"Demo endpoint name: {endpoint_name}\")\n",
+ "\n",
+ "# SageMaker Clarify model directory name\n",
+ "model_path = \"model/\"\n",
+ "\n",
+ "# Instance type for training and hosting\n",
+ "instance_type = \"ml.m5.xlarge\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Create serializer and deserializer\n",
+ "\n",
+ "CSV serializer to serialize test data to string"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "csv_serializer = CSVSerializer()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "JSON deserializer to deserialize invoke endpoint response"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "json_deserializer = JSONDeserializer()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### For visualization"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# This method is a wrapper around the captum that helps produce visualizations for local explanations. It will\n",
+ "# visualize the attributions for the tokens with red or green colors for negative and positive attributions.\n",
+ "def visualization_record(\n",
+ " attributions, # list of attributions for the tokens\n",
+ " text, # list of tokens\n",
+ " pred, # the prediction value obtained from the endpoint\n",
+ " delta,\n",
+ " true_label, # the true label from the dataset\n",
+ " normalize=True, # normalizes the attributions so that the max absolute value is 1. Yields stronger colors.\n",
+ " max_frac_to_show=0.05, # what fraction of tokens to highlight, set to 1 for all.\n",
+ " match_to_pred=False, # whether to limit highlights to red for negative predictions and green for positive ones.\n",
+ " # By enabling `match_to_pred` you show what tokens contribute to a high/low prediction not those that oppose it.\n",
+ "):\n",
+ " if normalize:\n",
+ " attributions = attributions / max(max(attributions), max(-attributions))\n",
+ " if max_frac_to_show is not None and max_frac_to_show < 1:\n",
+ " num_show = int(max_frac_to_show * attributions.shape[0])\n",
+ " sal = attributions\n",
+ " if pred < 0.5:\n",
+ " sal = -sal\n",
+ " if not match_to_pred:\n",
+ " sal = np.abs(sal)\n",
+ " top_idxs = np.argsort(-sal)[:num_show]\n",
+ " mask = np.zeros_like(attributions)\n",
+ " mask[top_idxs] = 1\n",
+ " attributions = attributions * mask\n",
+ " return visualization.VisualizationDataRecord(\n",
+ " attributions,\n",
+ " pred,\n",
+ " int(pred > 0.5),\n",
+ " true_label,\n",
+ " attributions.sum() > 0,\n",
+ " attributions.sum(),\n",
+ " text,\n",
+ " delta,\n",
+ " )\n",
+ "\n",
+ "\n",
+ "def visualize_result(result, all_labels):\n",
+ " if not result[\"explanations\"]:\n",
+ " print(f\"No Clarify explanations for the record(s)\")\n",
+ " return\n",
+ " all_explanations = result[\"explanations\"][\"kernel_shap\"]\n",
+ " all_predictions = list(csv.reader(result[\"predictions\"][\"data\"].splitlines()))\n",
+ "\n",
+ " labels = []\n",
+ " predictions = []\n",
+ " explanations = []\n",
+ "\n",
+ " for i, expl in enumerate(all_explanations):\n",
+ " if expl:\n",
+ " labels.append(all_labels[i])\n",
+ " predictions.append(all_predictions[i])\n",
+ " explanations.append(all_explanations[i])\n",
+ "\n",
+ " attributions_dataset = [\n",
+ " np.array([attr[\"attribution\"][0] for attr in expl[0][\"attributions\"]])\n",
+ " for expl in explanations\n",
+ " ]\n",
+ " tokens_dataset = [\n",
+ " np.array([attr[\"description\"][\"partial_text\"] for attr in expl[0][\"attributions\"]])\n",
+ " for expl in explanations\n",
+ " ]\n",
+ "\n",
+ " # You can customize the following display settings\n",
+ " normalize = True\n",
+ " max_frac_to_show = 1\n",
+ " match_to_pred = False\n",
+ " vis = []\n",
+ " for attr, token, pred, label in zip(attributions_dataset, tokens_dataset, predictions, labels):\n",
+ " vis.append(\n",
+ " visualization_record(\n",
+ " attr, token, float(pred[0]), 0.0, label, normalize, max_frac_to_show, match_to_pred\n",
+ " )\n",
+ " )\n",
+ " _ = visualization.visualize_text(vis)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Prepare data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Download data\n",
+ "Data Source: `https://www.kaggle.com/nicapotato/womens-ecommerce-clothing-reviews/`\n",
+ "\n",
+ "The Women’s E-Commerce Clothing Reviews dataset has been made available under a Creative Commons Public Domain license. A copy of the dataset has been saved in a sample data Amazon S3 bucket. In the first section of the notebook, we’ll walk through how to download the data and get started with building the ML workflow as a SageMaker pipeline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "! curl https://sagemaker-sample-files.s3.amazonaws.com/datasets/tabular/womens_clothing_ecommerce/Womens_Clothing_E-Commerce_Reviews.csv > womens_clothing_reviews_dataset.csv"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Load the dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "df = pd.read_csv(\"womens_clothing_reviews_dataset.csv\", index_col=[0])\n",
+ "df.head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Context**\n",
+ "\n",
+ "The Women’s Clothing E-Commerce dataset contains reviews written by customers. Because the dataset contains real commercial data, it has been anonymized, and any references to the company in the review text and body have been replaced with “retailer”.\n",
+ "\n",
+ "\n",
+ "\n",
+ "**Content**\n",
+ "\n",
+ "The dataset contains 23486 rows and 10 columns. Each row corresponds to a customer review.\n",
+ "\n",
+ "The columns include:\n",
+ "\n",
+ "* Clothing ID: Integer Categorical variable that refers to the specific piece being reviewed.\n",
+ "* Age: Positive Integer variable of the reviewer's age.\n",
+ "* Title: String variable for the title of the review.\n",
+ "* Review Text: String variable for the review body.\n",
+ "* Rating: Positive Ordinal Integer variable for the product score granted by the customer from 1 Worst, to 5 Best.\n",
+ "* Recommended IND: Binary variable stating where the customer recommends the product where 1 is recommended, 0 is not recommended.\n",
+ "* Positive Feedback Count: Positive Integer documenting the number of other customers who found this review positive.\n",
+ "* Division Name: Categorical name of the product high level division.\n",
+ "* Department Name: Categorical name of the product department name.\n",
+ "* Class Name: Categorical name of the product class name.\n",
+ "\n",
+ "**Goal**\n",
+ "\n",
+ "To predict the sentiment of a review based on the text, and then explain the predictions using SageMaker Clarify."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Data preparation for model training"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Target Variable Creation\n",
+ "Since the dataset does not contain a column that indicates the sentiment of the customer reviews, lets create one. To do this, let's assume that reviews with a `Rating` of 4 or higher indicate positive sentiment and reviews with a `Rating` of 2 or lower indicate negative sentiment. Let's also assume that a `Rating` of 3 indicates neutral sentiment and exclude these rows from the dataset. Additionally, to predict the sentiment of a review, we are going to use the `Review Text` column; therefore let's remove rows that are empty in the `Review Text` column of the dataset\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def create_target_column(df, min_positive_score, max_negative_score):\n",
+ " neutral_values = [i for i in range(max_negative_score + 1, min_positive_score)]\n",
+ " for neutral_value in neutral_values:\n",
+ " df = df[df[\"Rating\"] != neutral_value]\n",
+ " df[\"Sentiment\"] = df[\"Rating\"] >= min_positive_score\n",
+ " return df.replace({\"Sentiment\": {True: 1, False: 0}})\n",
+ "\n",
+ "\n",
+ "df = create_target_column(df, 4, 2)\n",
+ "df = df[~df[\"Review Text\"].isna()]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Train-Validation-Test splits"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The most common approach for model evaluation is using the train/validation/test split. Although this approach can be very effective in general, it can result in misleading results and potentially fail when used on classification problems with a severe class imbalance. Instead, the technique must be modified to stratify the sampling by the class label as below. Stratification ensures that all classes are well represented across the train, validation and test datasets.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "target = \"Sentiment\"\n",
+ "cols = \"Review Text\"\n",
+ "\n",
+ "X = df[cols]\n",
+ "y = df[target]\n",
+ "\n",
+ "# Data split: 11%(val) of the 90% (train and test) of the dataset ~ 10%; resulting in 80:10:10split\n",
+ "test_dataset_size = 0.10\n",
+ "val_dataset_size = 0.11\n",
+ "RANDOM_STATE = 42\n",
+ "\n",
+ "# Stratified train-val-test split\n",
+ "X_train, X_test, y_train, y_test = train_test_split(\n",
+ " X, y, test_size=test_dataset_size, stratify=y, random_state=RANDOM_STATE\n",
+ ")\n",
+ "X_train, X_val, y_train, y_val = train_test_split(\n",
+ " X_train, y_train, test_size=val_dataset_size, stratify=y_train, random_state=RANDOM_STATE\n",
+ ")\n",
+ "\n",
+ "print(\n",
+ " \"Dataset: train \",\n",
+ " X_train.shape,\n",
+ " y_train.shape,\n",
+ " y_train.value_counts(dropna=False, normalize=True).to_dict(),\n",
+ ")\n",
+ "print(\n",
+ " \"Dataset: validation \",\n",
+ " X_val.shape,\n",
+ " y_val.shape,\n",
+ " y_val.value_counts(dropna=False, normalize=True).to_dict(),\n",
+ ")\n",
+ "print(\n",
+ " \"Dataset: test \",\n",
+ " X_test.shape,\n",
+ " y_test.shape,\n",
+ " y_test.value_counts(dropna=False, normalize=True).to_dict(),\n",
+ ")\n",
+ "\n",
+ "# Combine the independent columns with the label\n",
+ "df_train = pd.concat([X_train, y_train], axis=1).reset_index(drop=True)\n",
+ "df_test = pd.concat([X_test, y_test], axis=1).reset_index(drop=True)\n",
+ "df_val = pd.concat([X_val, y_val], axis=1).reset_index(drop=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "headers = df_test.columns.to_list()\n",
+ "feature_headers = headers[0]\n",
+ "label_header = headers[1]\n",
+ "print(f\"Feature names: {feature_headers}\")\n",
+ "print(f\"Label name: {label_header}\")\n",
+ "print(f\"Test data (without label column):\")\n",
+ "test_data = df_test.iloc[:, :1]\n",
+ "test_data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We have split the dataset into train, test, and validation datasets. We use the train and validation datasets during training process, and run Clarify on the test dataset.\n",
+ "\n",
+ "In the cell below, we convert the Pandas DataFrames into Hugging Face Datasets for downstream modeling"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "train_dataset = Dataset.from_pandas(df_train)\n",
+ "val_dataset = Dataset.from_pandas(df_val)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Upload the dataset\n",
+ "Here, we upload the prepared datasets to S3 buckets so that we can train the model with the Hugging Face Estimator."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# S3 key prefix for the datasets\n",
+ "s3 = S3FileSystem()\n",
+ "\n",
+ "# save train_dataset to s3\n",
+ "training_input_path = f\"{s3_key}/train\"\n",
+ "print(f\"training input path: {training_input_path}\")\n",
+ "train_dataset.save_to_disk(training_input_path, fs=s3)\n",
+ "\n",
+ "# save val_dataset to s3\n",
+ "val_input_path = f\"{s3_key}/test\"\n",
+ "print(f\"validation input path: {val_input_path}\")\n",
+ "val_dataset.save_to_disk(val_input_path, fs=s3)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Train and Deploy Hugging Face Model"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In this step of the workflow, we use the [Hugging Face Estimator](https://sagemaker.readthedocs.io/en/stable/frameworks/huggingface/sagemaker.huggingface.html) to load the pre-trained `distilbert-base-uncased` model and fine-tune the model on our dataset."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Train model with Hugging Face estimator\n",
+ "The hyperparameters defined below are parameters that are passed to the custom PyTorch code in [`scripts/train.py`](./scripts/train.py). The only required parameter is `model_name`. The other parameters like `epoch`, `train_batch_size` all have default values which can be overridden by setting their values here.\n",
+ "\n",
+ "The training job requires GPU instance type. Here, we use `ml.g4dn.xlarge`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Hyperparameters passed into the training job\n",
+ "hyperparameters = {\"epochs\": 1, \"model_name\": \"distilbert-base-uncased\"}\n",
+ "\n",
+ "huggingface_estimator = HuggingFace(\n",
+ " entry_point=\"train.py\",\n",
+ " source_dir=\"scripts\",\n",
+ " instance_type=\"ml.g4dn.xlarge\",\n",
+ " instance_count=1,\n",
+ " transformers_version=\"4.6.1\",\n",
+ " pytorch_version=\"1.7.1\",\n",
+ " py_version=\"py36\",\n",
+ " role=role,\n",
+ " hyperparameters=hyperparameters,\n",
+ " disable_profiler=True,\n",
+ " debugger_hook_config=False,\n",
+ ")\n",
+ "\n",
+ "# starting the train job with our uploaded datasets as input\n",
+ "huggingface_estimator.fit({\"train\": training_input_path, \"test\": val_input_path}, logs=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Download the trained model files"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "! aws s3 cp {huggingface_estimator.model_data} model.tar.gz\n",
+ "! mkdir -p {model_path}\n",
+ "! tar -xvf model.tar.gz -C {model_path}/"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Prepare model container definition\n",
+ "\n",
+ "We are going to use the trained model files along with the HuggingFace Inference container to deploy the model to a SageMaker endpoint."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with tarfile.open(\"hf_model.tar.gz\", mode=\"w:gz\") as archive:\n",
+ " archive.add(model_path, recursive=True)\n",
+ " archive.add(\"code/\")\n",
+ "directory_name = s3_prefix.split(\"/\")[-1]\n",
+ "zipped_model_path = sagemaker_session.upload_data(\n",
+ " path=\"hf_model.tar.gz\", key_prefix=directory_name + \"/hf-model-sm\"\n",
+ ")\n",
+ "zipped_model_path"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Create a new model object and then update its model artifact and inference script. The model object will be used to create the SageMaker model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model = huggingface_estimator.create_model(name=model_name)\n",
+ "container_def = model.prepare_container_def(instance_type=instance_type)\n",
+ "container_def[\"ModelDataUrl\"] = zipped_model_path\n",
+ "container_def[\"Environment\"][\"SAGEMAKER_PROGRAM\"] = \"inference.py\"\n",
+ "pprint.pprint(container_def)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "## Create endpoint"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Create model\n",
+ "\n",
+ "The following parameters are required to create a SageMaker model:\n",
+ "\n",
+ "* `ExecutionRoleArn`: The ARN of the IAM role that Amazon SageMaker can assume to access the model artifacts/ docker images for deployment\n",
+ "\n",
+ "* `ModelName`: name of the SageMaker model.\n",
+ "\n",
+ "* `PrimaryContainer`: The location of the primary docker image containing inference code, associated artifacts, and custom environment map that the inference code uses when the model is deployed for predictions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sagemaker_client.create_model(\n",
+ " ExecutionRoleArn=role,\n",
+ " ModelName=model_name,\n",
+ " PrimaryContainer=container_def,\n",
+ ")\n",
+ "print(f\"Model created: {model_name}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Create endpoint config\n",
+ "\n",
+ "Create an endpoint configuration by calling the `create_endpoint_config` API. Here, supply the same `model_name` used in the `create_model` API call. The `create_endpoint_config` now supports the additional parameter `ClarifyExplainerConfig` to enable the Clarify explainer. The SHAP baseline is mandatory, it can be provided either as inline baseline data (the `ShapBaseline` parameter) or by a S3 baseline file (the `ShapBaselineUri` parameter). Please see the developer guide for the optional parameters."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Here we use a special token as the baseline."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "baseline = [[\"
+
+ Note
+
+ Files can also be concatenated similar to join operations
+
+ .. raw:: html
+
+
+
+Hit configure.
+|image24|
+
+Here, choose a name for the resulting join file and choose the
+type of join and columns on to join (Please refer to the image
+below).
+|image25|
+
+Hit **Apply** (*Preview*) . You can see a preview of the Joined
+dataset as shown in the image below.
+|image26|
+
+.. raw:: html
+
+
+
+Note
+
+Depending on the version of SageMaker it might be Preview and not
+Add
+
+.. raw:: html
+
+
+
+Hit *Add* at the upper right corner to add this Join transform to
+the original data flow.
+
+At the end of this step, the data flow looks as shown below.
+|image27|
+
+Next step, let’s see how to use Data Wrangler to add custom
+transform to perform more advanced feature engineering. Here, we
+want to use pyspark to calculate the average values of 5-star
+ratings for different columns and use them as new features.
+
+Select on the jointrackid.csv block and click the + icon, under
+which click on Add transform.
+|image28|
+
+Click the custom transform at this step.
+|image29|
+
+Give a name to this custom transform step and copy and paste below
+custom script to the window.
+
+.. code:: python
+
+ # Table is available as variable `df`
+ import pyspark.sql.functions as F
+ # numeric features
+ num_feat_cols = ['energy', 'acousticness', 'valence', 'speechiness', 'instrumentalness', 'liveness', 'tempo', 'danceability', 'genre_Latin', 'genre_Folk', 'genre_Blues', 'genre_Rap', 'genre_Reggae', 'genre_Jazz', 'genre_RnB', 'genre_Country', 'genre_Electronic', 'genre_Pop_Rock']
+ # calculate averages for 5-star ratings
+ agg_obj = [F.avg(feat_col).alias(feat_col+"_5star") for feat_col in num_feat_cols]
+ df = df.filter(F.col('Rating')==5).groupBy('userId').agg(*agg_obj)
+
+- .. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-25.png
+
+
+
+ Click *Preview* and the *Add* this step to the flow.
+
+We want to join the generated new features back to the original data.
+Before joining back, we notice some columns in the joint dataset are
+not needed for the model training, such as the id related columns.
+Let’s see now how to add a simple transform using Data Wrangler to
+drop the columns after the JOIN operation we did previously.
+
+Select the jointrackid.csv block and select *Add transform*.
+|image30|
+
+Let us apply the manage columns transform to drop some columns
+listed as below
+
+*trackId_0*
+*trackId_1*
+*ts*
+*sessionId*
+*itemInSession*
+*ratingEventId*
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-27.png
+
+
+image
+
+we can drop multiple columns by selecting each column from the
+drop down manual.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-28.png
+
+
+
+Once all the columns are selected, hit **Preview** first and then
+**Add**.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-29.png
+
+
+
+
+Now go back to data flow.
+
+You should now be able to see the 2 transforms (custom transform
+and dropping the columns) as shown below in the Data Flow
+interface.
+|image31|
+
+Next step is to join the two dataset back together. Similarly as
+the first join, we select one block and choose *Join*.
+|image32|
+
+Select the other file transform block and get a Join preview.
+|image33|
+
+Fill in the step Name, Join Type and columns to join on
+(*UserId*).
+|image34|
+
+Preview and Add this step to the flow file. When we go back to the
+data flow, this is how the flow looks like now.
+|image35|
+
+After joining the two data source, we also want to drop the
+*userId* columns and move the target column *Rating* to the first
+column.
+
+Similar to the previous manage columns transform instructions, we
+add two transform steps to drop the *userId_0* and *userId_1*
+columns, and then move the *Rating* step to the start of the
+table.
+
+|image36|
+
+|image37|
+
+Once all the transform steps are finished, we will export the
+transformed data. SageMaker Data Wrangler also allow you to split
+your dataset into train and test based on the ratio you set.
+
+To split the dataset, add another transform step and choose
+**Split data**.
+|image38|
+
+We choose *Randomized split* and get 80% for training and 20% data
+for testing.
+|image39|
+
+The data flow now looks as below:
+|image40|
+
+
+Export transformed features to S3
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+To export the transformed dataset, first click on the + symbol and
+choose Add Destination, followed by Amazon S3 as pointed out by the
+screen shot below.
+
+|image41|
+
+A new window is opened, Click Export data, choose the S3 location
+where you want to save the transformed dataset.
+
+|image42|
+
+Follow the same step to set the S3 location for the test data.
+
+A job is needed to export the data to Amazon S3, to do this press the
+Create Job button on the top right, this will open a window.
+
+Set the Job name to something like generate-train-test-data
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-42.png
+
+
+
+Hit the **Configure Job** button at the bottom
+
+Leave the default instance type, and press the Run button at the
+bottom.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-43.png
+
+
+
+
+Note that your job has been created successfully and if you want to
+see the progress of the job you can do so by following the link to
+the generate-train-test-data process.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-44.png
+
+
+
+
+Follow the link to see the status of your job. This processing job
+takes around 5-10 mins.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-45.png
+
+
+
+
+When the job is complete the train and test output files will be
+available in the corresponding S3 output folders. You can find the
+output location from the processing job configurations.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/joined-dataflow/image-processing-job-output.png
+
+
+
+
+Other ways to export the transformations and analysis
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The join.flow file that we created initially captures all of the
+transformations, joins and analysis.
+
+In a way, this file allows us to capture and persist every step of
+our feature engineering journey into a static file.
+
+The flow file can then be used to re-create the analysis and feature
+engineering steps via Data Wrangler. All you need to do is import the
+flow file to SageMaker Studio and click on it.
+
+We saw previously, how to export transformed dataset into S3.
+Additionally, we can also export the analysis and transformations in
+many other formats.
+
+To start exporting, click on the train Dataset data block and click
+on the + icon and select **Export to**.
+
+|image43|
+
+You can export the analysis and transforms in 4 different ways in
+addition to direct export to S3 which we saw previously.
+
+* Save to S3 as a SageMaker Processing job notebook.
+* Export as a SageMaker Pipeline notebook.
+* Export as a Python script.
+* Export to SageMaker Feature Store as a notebook.
+
+
+So far, we have demonstrated how to use Amazon SageMaker Data Wrangler
+to preprocess the data and perform feature engineering to prepare for
+the train and test data set. After the data preparation step, data
+scientists can work on training a machine learning model using these
+datasets. In the next section, we will show you how to directly start a
+training job with the train data by leveraging Amazon SageMaker
+Autopilot from the SageMaker Data Wrangler data flow.
+
+
+Import Dataflow
+----------------------------
+
+Here is the final flow file
+`Flow file
+Data Source
+
+For this experiment the Data Source will be [Amazon S3](https://aws.amazon.com/s3/)
+
+
+
+## Experiment steps
+
+### Downloading the dataset, and notebooks
+
+* Ensure that you have a working [Amazon SageMaker Studio](https://aws.amazon.com/sagemaker/studio/) environment and that it has been updated.
+
+* Follow the steps below to download the dataset.
+ * Start with the `explore-dataset.ipynb` SageMaker Studio notebook.
+ * Explore the dataset locally by running the cells in the notebook.
+ * Upload the datasets (CSV files) to an S3 location for consumption by SageMaker Data Wrangler later.
+ * Copy the S3 URLs of the tracks and ratings data files to your clipboard. We will use these URLs later to import these part files into Data Wrangler and join them.
+ * Let's run though each of the steps above.
+
+* Double click on the file called explore-data.ipynb. Amazon SageMaker may prompt you to select a kernel and image. If it does select Data Science as the image and Python 3 as the Kernel, as shown here:
+
+
+* You have now successfully downloaded the data and opened a notebook, we will now upload the data to your S3 bucket. Note: An Amazon S3 bucket was created for you when the Amazon SageMaker Studio environment was started.
+
+
+* From the menu select **Run / Run All Cells** to execute all the cells in the notebook.
+
+
+* Examine the results, as you can see the samples of the data are printed to the screen.
+
+
+* The data structure is defined in the next section. However, before we continue, note the path to the various datasets. Your paths will be different to the ones in the image below. Please copy these paths as you will use them later. An example of a path is s3://sagemaker-eu-west-1-112233445566/music-recommendation-demo/input/tracks.csv
+
+
+
+ Note
+
+ Please also check the S3 location to make sure the files are uploaded successfully before moving to the next section.
+
+
+
+
+In the next section we will import the datasets into Data Wrangler via the SageMaker Studio User Interface (UI).
+
+
+## Importing datasets from a data source (S3) to Data Wrangler
+
+* Initialize SageMaker Data Wrangler via SageMaker Studio UI.
+ * There are two ways that you can do this, either from the Launcher screen as depicted here:
+ 
+ * Or from the SageMaker resources menu on the left, selecting Data Wrangler, and new flow
+ 
+ 
+* It takes a few minutes to load.
+
+* Once Data Wrangler is loaded, you should be able to see it under running instances and apps as shown below.
+
+* Next, make sure you have copied the data paths when running the `explore_data.ipynb` notebook from the previous section (see section: **Downloading the dataset, and notebooks**), as you will need them in this section.
+* Once Data Wrangler is up and running, you can see the following data flow interface with options for import, creating data flows and export as shown below.
+
+* Make sure to rename the untitled.flow to your preference (for e.g., join.flow)
+* Paste the S3 URL for the tracks.csv file into the search box below and hit go.
+
+* Select the CSV file from the drop down results. On the right pane, make sure COMMA is chosen as the delimiter and Sampling is *None*. Hit *import* to import this dataset to Data Wrangler.
+
+* Once the dataset is imported, the Data flow interface looks as shown below.
+
+* Since currently you are in the data flow tab, hit the import tab (left of data flow tab) as seen in the above image.
+* Import the second part file (ratings.csv) following the same set of instructions as noted previously.
+
+
+### Transform tracks dataset
+* We firstly want to perform some data transformation using Data Wrangler. Let us walkthough how to perform different transformations using built-in and custom formula functionality in Data Wrangler.
+ * As the *genre* column in the tracks dataset is a categorical feature, we need to perform one-hot encoding to trasform this feature.
+ * Click on the tracks file transform block as show in the image below and select **Add transform**:
+ 
+ * This takes us to the Data Wrangler transformations interface where there are over 300+ transformations you can apply to your dataset. Select **Add step** as shown below.
+ 
+ * In the **ADD TRANFORM** window, double click the option **Encode categorical**.
+ 
+ * Then on the **ENCODE CATEGORICAL** window, choose *One-hot encode* as the Transform type, *genre* as the input columns, and *Columns* as the output style. Click *Preview* and the output is shown as below:
+ 
+ * Click **Add** to add the tranform step to the flow. If you go back to the *Data Flow*, you can see the step has been added.
+ 
+
+* We also want to generate a new feature based on the danceability of the track. Danceability describes how suitable a track is for dancing based on a combination of musical elements including tempo, rhythm stability, beat strength, and overall regularity.
+ * Click on the newly added **One-hot encode** step and select *Add transformation*:
+ 
+ * Select **Add step** and choose **Custom formula**.
+ 
+ * Copy and paste below formula and put **danceability** to the *Output Column*.
+
+ 0.3\*valence + 0.1\*liveness + 0.1\*energy
+
+ 
+
+ * Click **Preview** and **Add** the step to the flow.
+ 
+
+### Joining datasets - first join
+* Given, we have imported both the tracks and ratings CSV files in the beginning steps. Let us walk through on how to join these CSV files based on a common unique identifier column, *trackId*. Then we will perform some feature engineering to generate a new set of features that can help to enrich the trainig data.
+
+* Click on either the tracks or ratings transform block as shown in the image blow:
+ * Here, we have selected tracks transform flow block and hit **Join**
+ 
+ * Select the other rating file transform block and it automatically maps (converges) both the files into a Join preview as shown below.
+
+ Note
+
+ Files can also be concatenated similar to join operations
+
+
+
+ * Hit configure.
+ 
+ * Here, choose a name for the resulting join file and choose the type of join and columns on to join (Please refer to the image below).
+ 
+ * Hit **Apply** (*Preview*) . You can see a preview of the Joined dataset as shown in the image below.
+ 
+
+ Note
+
+ Depending on the version of SageMaker it might be Preview and not Add
+
+
+
+ * Hit *Add* at the upper right corner to add this Join transform to the original data flow.
+ * At the end of this step, the data flow looks as shown below.
+ 
+
+ * Next step, let's see how to use Data Wrangler to add custom transform to perform more advanced feature engineering. Here, we want to use pyspark to calculate the average values of 5-star ratings for different columns and use them as new features.
+ * Select on the jointrackid.csv block and click the + icon, under which click on Add transform.
+ 
+ * Click the custom transform at this step.
+ 
+ * Give a name to this custom transform step and copy and paste below custom script to the window.
+
+```python
+# Table is available as variable `df`
+import pyspark.sql.functions as F
+# numeric features
+num_feat_cols = ['energy', 'acousticness', 'valence', 'speechiness', 'instrumentalness', 'liveness', 'tempo', 'danceability', 'genre_Latin', 'genre_Folk', 'genre_Blues', 'genre_Rap', 'genre_Reggae', 'genre_Jazz', 'genre_RnB', 'genre_Country', 'genre_Electronic', 'genre_Pop_Rock']
+# calculate averages for 5-star ratings
+agg_obj = [F.avg(feat_col).alias(feat_col+"_5star") for feat_col in num_feat_cols]
+df = df.filter(F.col('Rating')==5).groupBy('userId').agg(*agg_obj)
+```
+*
+ 
+ * Click *Preview* and the *Add* this step to the flow.
+
+* We want to join the generated new features back to the original data. Before joining back, we notice some columns in the joint dataset are not needed for the model training, such as the id related columns. Let’s see now how to add a simple transform using Data Wrangler to drop the columns after the JOIN operation we did previously.
+ * Select the jointrackid.csv block and select *Add transform*.
+ 
+ * Let us apply the manage columns transform to drop some columns listed as below
+ * *trackId_0*
+ * *trackId_1*
+ * *ts*
+ * *sessionId*
+ * *itemInSession*
+ * *ratingEventId*
+
+ 
+
+ * we can drop multiple columns by selecting each column from the drop down manual.
+
+ 
+
+ * Once all the columns are selected, hit **Preview** first and then **Add**.
+
+ 
+
+ * Now go back to data flow.
+ * You should now be able to see the 2 transforms (custom transform and dropping the columns) as shown below in the Data Flow interface.
+ 
+
+ * Next step is to join the two dataset back together. Similarly as the first join, we select one block and choose *Join*.
+ 
+ * Select the other file transform block and get a Join preview.
+ 
+ * Fill in the step Name, Join Type and columns to join on (*UserId*).
+ 
+ * Preview and Add this step to the flow file. When we go back to the data flow, this is how the flow looks like now.
+ 
+ * After joining the two data source, we also want to drop the *userId* columns and move the target column *Rating* to the first column.
+ * Similar to the previous manage columns transform instructions, we add two transform steps to drop the *userId_0* and *userId_1* columns, and then move the *Rating* step to the start of the table.
+
+ 
+ 
+
+* Once all the transform steps are finished, we will export the transformed data. SageMaker Data Wrangler also allow you to split your dataset into train and test based on the ratio you set.
+ * To split the dataset, add another transform step and choose **Split data**.
+ 
+ * We choose *Randomized split* and get 80% for training and 20% data for testing.
+ 
+ * The data flow now looks as below:
+ 
+
+
+## Export transformed features to S3 (will be consumed by SageMaker Autopilot)
+
+* To export the transformed dataset, first click on the + symbol and choose Add Destination, followed by Amazon S3 as pointed out by the screen shot below.
+
+
+* A new window is opened, Click Export data, choose the S3 location where you want to save the transformed dataset.
+
+* Follow the same step to set the S3 location for the test data.
+
+* A job is needed to export the data to Amazon S3, to do this press the Create Job button on the top right, this will open a window.
+
+* Set the Job name to something like generate-train-test-data
+
+
+
+* Hit the **Configure Job** button at the bottom
+
+* Leave the default instance type, and press the Run button at the bottom.
+
+
+
+* Note that your job has been created successfully and if you want to see the progress of the job you can do so by following the link to the generate-train-test-data process.
+
+
+
+* Follow the link to see the status of your job. This processing job takes around 5-10 mins.
+
+
+
+* When the job is complete the train and test output files will be available in the corresponding S3 output folders. You can find the output location from the processing job configurations.
+
+
+
+### *Other ways to export the transformations and analysis*
+
+* The join.flow file that we created initially captures all of the transformations, joins and analysis.
+* In a way, this file allows us to capture and persist every step of our feature engineering journey into a static file.
+* The flow file can then be used to re-create the analysis and feature engineering steps via Data Wrangler. All you need to do is import the flow file to SageMaker Studio and click on it.
+* We saw previously, how to export transformed dataset into S3. Additionally, we can also export the analysis and transformations in many other formats.
+* To start exporting, click on the train Dataset data block and click on the + icon and select **Export to**.
+
+* You can export the analysis and transforms in 4 different ways in addition to direct export to S3 which we saw previously.
+ * Save to S3 as a SageMaker Processing job notebook.
+ * Export as a SageMaker Pipeline notebook.
+ * Export as a Python script.
+ * Export to SageMaker Feature Store as a notebook.
+
+:bulb:**NOTE** - Also, you can import the [flow file](./join.flow) by following the steps [here](../import-flow.md)
+
+So far, we have demonstrated how to use Amazon SageMaker Data Wrangler to preprocess the data and perform feature engineering to prepare for the train and test data set. After the data preparation step, data scientists can work on training a machine learning model using these datasets. In the next section, we will show you how to directly start a training job with the train data by leveraging Amazon SageMaker Autopilot from the SageMaker Data Wrangler data flow.
+
+### *Run Autopilot training directly from Data Wrangler flow* (**Optional**)
+* SageMaker Data Wragler now allow you to directly run an [Autopilot](https://docs.aws.amazon.com/sagemaker/latest/dg/autopilot-automate-model-development.html) job to automatically train a model.
+ * To set up a SageMaker Autopilot job, click the train data block, select **Train model**.
+ 
+ * On the new window, select the S3 location you want the training dataset and the Autopilot job output to be saved.
+ 
+ * Select **Export and train**. This will take about one minute to export the train data to S3.
+ 
+ * When data exported successfully, we can configure the Autopilot job. Select the *Target* training column (Rating).
+ 
+ * Under the **Advanced settings**, choose the machine learning problem type as *Regression*. By default, SageMaker autopilot will run 250 training jobs to find the best model, this will take a few hours for the job to finish. To reduce runtime, you can set the *Max candidates* to a smaller number.
+ 
+ * After click **Create Experiment**, an autopilot job will be started. You can come back to SageMaker Studio later to check the job output.
+ 
diff --git a/sagemaker-datawrangler/readme.md b/sagemaker-datawrangler/readme.md
new file mode 100644
index 0000000000..d6c963f90d
--- /dev/null
+++ b/sagemaker-datawrangler/readme.md
@@ -0,0 +1,41 @@
+
+
+# Amazon SageMaker Data Wrangler Examples
+
+Example flows that demonstrate how to aggregate and prepare data for Machine Learning using Amazon SageMaker Data Wrangler.
+
+## :books: Background
+
+[Amazon SageMaker Data Wrangler](https://aws.amazon.com/sagemaker/data-wrangler/) reduces the time it takes to aggregate and prepare data for ML. From a single interface in SageMaker Studio, you can import data from Amazon S3, Amazon Athena, Amazon Redshift, AWS Lake Formation, and Amazon SageMaker Feature Store, and in just a few clicks SageMaker Data Wrangler will automatically load, aggregate, and display the raw data. It will then make conversion recommendations based on the source data, transform the data into new features, validate the features, and provide visualizations with recommendations on how to remove common sources of error such as incorrect labels. Once your data is prepared, you can build fully automated ML workflows with Amazon SageMaker Pipelines or import that data into Amazon SageMaker Feature Store.
+
+
+
+The [SageMaker example notebooks](https://sagemaker-examples.readthedocs.io/en/latest/) are Jupyter notebooks that demonstrate the usage of Amazon SageMaker.
+
+## :hammer_and_wrench: Setup
+
+Amazon SageMaker Data Wrangler is a feature in Amazon SageMaker Studio. Use this section to learn how to access and get started using Data Wrangler. Do the following:
+
+* Complete each step in [Prerequisites](https://docs.aws.amazon.com/sagemaker/latest/dg/data-wrangler-getting-started.html#data-wrangler-getting-started-prerequisite).
+
+* Follow the procedure in [Access Data Wrangler](https://docs.aws.amazon.com/sagemaker/latest/dg/data-wrangler-getting-started.html#data-wrangler-getting-started-access) to start using Data Wrangler.
+
+
+
+
+## :notebook: Examples
+
+### **[Tabular DataFlow](tabular-dataflow/README.md)**
+
+This example provide quick walkthrough of how to aggregate and prepare data for Machine Learning using Amazon SageMaker Data Wrangler for Tabular dataset.
+
+### **[Timeseries DataFlow](timeseries-dataflow/readme.md)**
+
+This example provide quick walkthrough of how to aggregate and prepare data for Machine Learning using Amazon SageMaker Data Wrangler for Timeseries dataset.
+
+### **[Joined DataFlow](joined-dataflow/readme.md)**
+
+This example provide quick walkthrough of how to aggregate and prepare data for Machine Learning using Amazon SageMaker Data Wrangler for Joined dataset.
+
+
+
diff --git a/sagemaker-datawrangler/tabular-dataflow/Hotel-Bookings-Classification.flow b/sagemaker-datawrangler/tabular-dataflow/Hotel-Bookings-Classification.flow
new file mode 100644
index 0000000000..f6eb64de42
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/Hotel-Bookings-Classification.flow
@@ -0,0 +1,242231 @@
+{
+ "metadata": {
+ "version": 1,
+ "disable_limits": false,
+ "instance_type": "ml.m5.4xlarge"
+ },
+ "nodes": [
+ {
+ "node_id": "1f7fb244-77d9-4d90-9dba-8d9956ba5f79",
+ "type": "SOURCE",
+ "operator": "sagemaker.s3_source_0.1",
+ "parameters": {
+ "dataset_definition": {
+ "__typename": "S3CreateDatasetDefinitionOutput",
+ "datasetSourceType": "S3",
+ "name": "hotel_bookings.csv",
+ "description": null,
+ "s3ExecutionContext": {
+ "__typename": "S3ExecutionContext",
+ "s3Uri": "s3://hotelbookings/hotel_bookings.csv",
+ "s3ContentType": "csv",
+ "s3HasHeader": true,
+ "s3FieldDelimiter": ",",
+ "s3DirIncludesNested": false,
+ "s3AddsFilenameColumn": false
+ }
+ }
+ },
+ "inputs": [],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "9ed99534-7891-44ba-972c-849614c96e20",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.infer_and_cast_type_0.1",
+ "parameters": {},
+ "trained_parameters": {
+ "schema": {
+ "hotel": "string",
+ "is_canceled": "long",
+ "lead_time": "long",
+ "arrival_date_year": "long",
+ "arrival_date_month": "string",
+ "arrival_date_week_number": "long",
+ "arrival_date_day_of_month": "long",
+ "stays_in_weekend_nights": "long",
+ "stays_in_week_nights": "long",
+ "adults": "long",
+ "children": "long",
+ "babies": "long",
+ "meal": "string",
+ "country": "string",
+ "market_segment": "string",
+ "distribution_channel": "string",
+ "is_repeated_guest": "long",
+ "previous_cancellations": "long",
+ "previous_bookings_not_canceled": "long",
+ "reserved_room_type": "string",
+ "assigned_room_type": "string",
+ "booking_changes": "long",
+ "deposit_type": "string",
+ "agent": "long",
+ "company": "string",
+ "days_in_waiting_list": "long",
+ "customer_type": "string",
+ "adr": "float",
+ "required_car_parking_spaces": "long",
+ "total_of_special_requests": "long",
+ "reservation_status": "string",
+ "reservation_status_date": "date"
+ }
+ },
+ "inputs": [
+ {
+ "name": "default",
+ "node_id": "1f7fb244-77d9-4d90-9dba-8d9956ba5f79",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "fd325a95-b061-46bb-a9bc-f0d4b1b10b85",
+ "type": "VISUALIZATION",
+ "operator": "sagemaker.visualizations.data_insights_report_0.1",
+ "parameters": {
+ "report_content": "Warning summary",
+ "insights_report_parameters": {
+ "target_column": "is_canceled",
+ "problem_type": "Classification"
+ }
+ },
+ "trained_parameters": {
+ "insights_report_parameters": {
+ "duplicate_ratio": 0.3364184605075802,
+ "high warnings": [
+ {
+ "title": "Duplicate rows",
+ "warningText": "We found that 33.6% of the data are duplicate. Some data sources could include valid duplicates and in other cases these duplicates could point to problems in data collection. Duplicate samples resulting from faulty data collection, could derail machine learning processes that rely on splitting to independent training and validation folds. For example quick model scores, prediction power estimation and automatic hyper parameter tuning. Duplicate samples could be removed from the dataset using the Drop duplicates transform under Manage rows.
", + "severity": "high_sev" + }, + { + "title": "Target leakage", + "warningText": "The feature reservation_status predicts the target extremely well on it's own. A feature this predictive often indicates an error called target leakage. The cause is typically data that is not available at time of prediction. For example, a duplicate of the target column in the dataset can result in target leakage.
Alternatively, if the machine learning task is \"easy\", then a single feature can have legitimately high prediction power. If you think that a single feature is very highly predictive, you don't need to do anything further. However, if you think there's target leakage, we recommended that you remove the highly predictive column from the dataset using the Drop column transform under Manage columns.
+Dataset
+
+We will be using the [Hotel Booking Demand dataset](https://www.kaggle.com/jessemostipak/hotel-booking-demand) that is publically available. This data set contains booking information for a city hotel and a resort hotel, and includes information such as when the booking was made, length of stay, the number of adults, children, and/or babies, and the number of available parking spaces, among other things.
+ +The data needs to be downloaded from the locations specified, and uploaded to S3 bucket before we start the Data Preprocessing phase. Please follow the Experiment Steps outlined in later sections, to download the data and notebooks. + + +## Description of the Columns + + + +| Column Name | Description | +|---|---| +| `hotel` | Type of the hotel (`H1` = Resort Hotel or `H2` = City Hotel) | +| `is_canceled` | Value indicating if the booking was canceled (1) or not (0) | +| `lead_time` | Number of days that elapsed between the entering date of the booking into the PMS and the arrival date | +| `arrival_date_year` | Year of arrival date | +| `arrival_date_month` | Month of arrival date | +| `arrival_date_week_number` | Week number of year for arrival date | +| `arrival_date_day_of_month` | Day of arrival date | +| `stays_in_weekend_nights` | Number of weekend nights (Saturday or Sunday) the guest stayed or booked to stay at the hotel | +| `stays_in_week_nights` | Number of week nights (Monday to Friday) the guest stayed or booked to stay at the hotel | +| `adults` | Number of adults | +| `children` | Number of children | +| `babies` | Number of babies | +| `meal` | Type of meal booked. Categories are presented in standard hospitality meal packages: `Undefined/SC` – no meal package; `BB` – Bed & Breakfast; `HB` – Half board (breakfast and one other meal – usually dinner); `FB` – Full board (breakfast, lunch and dinner) | +| `country`| Country of origin. Categories are represented in the `ISO 3155–3:2013` format | +|`market_segment`|Market segment designation. In categories, the term `TA` means “Travel Agents” and `TO` means “Tour Operators”| +|`distribution_channel`|Booking distribution channel. The term `TA` means “Travel Agents” and `TO` means “Tour Operators”| +|`is_repeated_guest`|Value indicating if the booking name was from a repeated guest (1) or not (0)| +|`previous_cancellations`|Number of previous bookings that were cancelled by the customer prior to the current booking| +|`previous_bookings_not_canceled`|Number of previous bookings not cancelled by the customer prior to the current booking| +|`reserved_room_type`|Code of room type reserved. Code is presented instead of designation for anonymity reasons.| +|`assigned_room_type`|Code for the type of room assigned to the booking. Sometimes the assigned room type differs from the reserved room type due to hotel operation reasons (e.g. overbooking) or by customer request. Code is presented instead of designation for anonymity reasons.| +|`booking_changes`|Number of changes/amendments made to the booking from the moment the booking was entered on the PMS until the moment of check-in or cancellation| +|`deposit_type`|Indication on if the customer made a deposit to guarantee the booking. This variable can assume three categories: No Deposit – no deposit was made; `Non Refund` – a deposit was made in the value of the total stay cost; `Refundable` – a deposit was made with a value under the total cost of stay.| +|`agent`|ID of the travel agency that made the booking| +|`company`|ID of the company/entity that made the booking or responsible for paying the booking. ID is presented instead of designation for anonymity reasons| +|`days_in_waiting_list`|Number of days the booking was in the waiting list before it was confirmed to the customer| +|`customer_type`|Type of booking, assuming one of four categories: `Contract` - when the booking has an allotment or other type of contract associated to it; `Group` – when the booking is associated to a group; `Transient` – when the booking is not part of a group or contract, and is not associated to other transient booking; `Transient-party` – when the booking is transient, but is associated to at least other transient booking| +|`adr`|Average Daily Rate as defined by dividing the sum of all lodging transactions by the total number of staying nights| +|`required_car_parking_spaces`|Number of car parking spaces required by the customer| +|`total_of_special_requests`|Number of special requests made by the customer (e.g. twin bed or high floor)| +|`reservation_status`|Reservation last status, assuming one of three categories: `Canceled` – booking was canceled by the customer; `Check-Out` – customer has checked in but already departed; `No-Show` – customer did not check-in and did inform the hotel of the reason why| +|`reservation_status_date`|Date at which the last status was set. This variable can be used in conjunction with the ReservationStatus to understand when was the booking canceled or when did the customer checked-out of the hotel| + +--- + +## Pre-requisites: + + * We need to ensure dataset (tracks and ratings dataset) for ML is uploaded to a data source (instructions to download the dataset to Amazon S3 is available in the following section). + * Data source can be any one of the following options: + * S3 + * Athena + * RedShift + * SnowFlake + + +
+ +The data needs to be downloaded from the locations specified, and uploaded to S3 bucket before we start the Data Preprocessing phase. Please follow the Experiment Steps outlined in later sections, to download the data and notebooks. + + +## Description of the Columns + + + +| Column Name | Description | +|---|---| +| `hotel` | Type of the hotel (`H1` = Resort Hotel or `H2` = City Hotel) | +| `is_canceled` | Value indicating if the booking was canceled (1) or not (0) | +| `lead_time` | Number of days that elapsed between the entering date of the booking into the PMS and the arrival date | +| `arrival_date_year` | Year of arrival date | +| `arrival_date_month` | Month of arrival date | +| `arrival_date_week_number` | Week number of year for arrival date | +| `arrival_date_day_of_month` | Day of arrival date | +| `stays_in_weekend_nights` | Number of weekend nights (Saturday or Sunday) the guest stayed or booked to stay at the hotel | +| `stays_in_week_nights` | Number of week nights (Monday to Friday) the guest stayed or booked to stay at the hotel | +| `adults` | Number of adults | +| `children` | Number of children | +| `babies` | Number of babies | +| `meal` | Type of meal booked. Categories are presented in standard hospitality meal packages: `Undefined/SC` – no meal package; `BB` – Bed & Breakfast; `HB` – Half board (breakfast and one other meal – usually dinner); `FB` – Full board (breakfast, lunch and dinner) | +| `country`| Country of origin. Categories are represented in the `ISO 3155–3:2013` format | +|`market_segment`|Market segment designation. In categories, the term `TA` means “Travel Agents” and `TO` means “Tour Operators”| +|`distribution_channel`|Booking distribution channel. The term `TA` means “Travel Agents” and `TO` means “Tour Operators”| +|`is_repeated_guest`|Value indicating if the booking name was from a repeated guest (1) or not (0)| +|`previous_cancellations`|Number of previous bookings that were cancelled by the customer prior to the current booking| +|`previous_bookings_not_canceled`|Number of previous bookings not cancelled by the customer prior to the current booking| +|`reserved_room_type`|Code of room type reserved. Code is presented instead of designation for anonymity reasons.| +|`assigned_room_type`|Code for the type of room assigned to the booking. Sometimes the assigned room type differs from the reserved room type due to hotel operation reasons (e.g. overbooking) or by customer request. Code is presented instead of designation for anonymity reasons.| +|`booking_changes`|Number of changes/amendments made to the booking from the moment the booking was entered on the PMS until the moment of check-in or cancellation| +|`deposit_type`|Indication on if the customer made a deposit to guarantee the booking. This variable can assume three categories: No Deposit – no deposit was made; `Non Refund` – a deposit was made in the value of the total stay cost; `Refundable` – a deposit was made with a value under the total cost of stay.| +|`agent`|ID of the travel agency that made the booking| +|`company`|ID of the company/entity that made the booking or responsible for paying the booking. ID is presented instead of designation for anonymity reasons| +|`days_in_waiting_list`|Number of days the booking was in the waiting list before it was confirmed to the customer| +|`customer_type`|Type of booking, assuming one of four categories: `Contract` - when the booking has an allotment or other type of contract associated to it; `Group` – when the booking is associated to a group; `Transient` – when the booking is not part of a group or contract, and is not associated to other transient booking; `Transient-party` – when the booking is transient, but is associated to at least other transient booking| +|`adr`|Average Daily Rate as defined by dividing the sum of all lodging transactions by the total number of staying nights| +|`required_car_parking_spaces`|Number of car parking spaces required by the customer| +|`total_of_special_requests`|Number of special requests made by the customer (e.g. twin bed or high floor)| +|`reservation_status`|Reservation last status, assuming one of three categories: `Canceled` – booking was canceled by the customer; `Check-Out` – customer has checked in but already departed; `No-Show` – customer did not check-in and did inform the hotel of the reason why| +|`reservation_status_date`|Date at which the last status was set. This variable can be used in conjunction with the ReservationStatus to understand when was the booking canceled or when did the customer checked-out of the hotel| + +--- + +## Pre-requisites: + + * We need to ensure dataset (tracks and ratings dataset) for ML is uploaded to a data source (instructions to download the dataset to Amazon S3 is available in the following section). + * Data source can be any one of the following options: + * S3 + * Athena + * RedShift + * SnowFlake + + +
+Data Source
+
+For this experiment the Data Source will be [Amazon S3](https://aws.amazon.com/s3/)
+
+
+
+## Experiment steps
+
+### Downloading the dataset
+
+* Ensure that you have a working [Amazon SageMaker Studio](https://aws.amazon.com/sagemaker/studio/) environment and that it has been updated.
+
+* Follow the steps below to download the dataset.
+1. Download the [Hotel Booking Demand dataset](https://www.kaggle.com/jessemostipak/hotel-booking-demand) from the specified location.
+2. Create a private S3 bucket to upload the dataset in. You can reference the instructions for bucket creaiton [here] https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html
+3. Upload the data in step 1 to the bucket created in step 2. Steps to upload the data can be found [here] https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html
+4. Note the S3 URL for the file uploaded in Step 3 before moving to the next sextion. This data will be used as input to the Datawrangler. The S3 URL will be used in the next step.
+
+### Data Import from S3 to Data Wrangler
+The hotel-bookings.csv file uploaded in previous section needs to be imported in Data Wrangler as input. Please refer to **[Data Import from S3](data-import/Data-Import.md)** and follow steps for importing the data.
+
+### Exploratory Data Analysis
+Before applying various data transformations, we need to explore the data to find correlations, duplicate rows as well as target leakage. Please refer to **[Exploratory Data Analysis](data-exploraion/Data-Exploration.md)** and follow steps for Data exploration.
+
+### Data Transformation
+Based on the Data explorations carried out in previous step, we are now ready to apply transformations to the data. Please refer to **[Data Transformations](data-transformation/Data-Transformations.md)** and follow steps for Data Transformation.
+
+### Data Export
+Data Wrangler UI can also be used to export the transformed data to Amazon S3. To get started with this process, first let's create a destination node. Right click on the final transform on your data and choose `Add destination` → `Amazon S3`. Assign a name for your output data and choose the S3 location where you want the data to be stored and click Add destination button at the bottom as shown below.
+
+
+
+This adds a destination node to our data flow. The destination node acts as a sink to your data flow.
+
+
+
+Next, click on the Create job button in the upper right corner of the page. In the configuration page for the SageMaker Processing job we are about to create, choose the Instance type and Instance count for our processing cluster. Advance configuration is optional, where you can assign tags as needed and choose the appropriate Volume Size.
+
+
+
+Select `Run` to start the export job. The job created will have Job Name and Job ARN which can be used to search for the job status.
+
+
+
+The job created in the previous step will be available in the monitoring page for `SageMaker Processing job` as shown in the figure below.
+
+
+
+After a certain time, the job will be complete. The image below shows the completed job. Exported data should be available in the output S3 bucket.
+
+
+
+
+ This exported data can now be used for running the ML Models
+
+ :bulb:**NOTE** - Also, you can import the [flow file](./Hotel-Bookings-Classification.flow) by following the steps [here](../import-flow.md)
diff --git a/sagemaker-datawrangler/tabular-dataflow/data-exploration/Data-Exploration.md b/sagemaker-datawrangler/tabular-dataflow/data-exploration/Data-Exploration.md
new file mode 100644
index 0000000000..4b4d6cec78
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/data-exploration/Data-Exploration.md
@@ -0,0 +1,242 @@
+## Exploratory Data Analysis
+
+### Analyze and Visualize
+
+Before we transform our raw features to make it ML ready for model building, lets analyze and visualize the booking cancellations dataset to detect features that are important to our problem and ones that are not. This can be achieved through Exploratory Data Analysis (EDA). Amazon SageMaker Data Wrangler includes built-in analyses that help you generate visualizations and data analyses in a few clicks. You can also create custom analyses using your own code.
+
+In order to apply an action on the imported data, select the **Add Analysis** option on right clicking the the **Data Types** block. As depicted in the figure below, you can see options to add a transform, perform an analysis, add a destination sink or export the steps as Jupyter notebook. You can also join and concatenate on the imported dataset with other datasets.
+
+
+
+On selecting Add Analysis, the Analysis pane os shown, where you can choose the analysis you want to perform.
+
+All analyses are generated using 100,000 rows of your dataset.
+
+You can add the following analysis to a dataframe:
+
+* Data visualizations, including histograms and scatter plots.
+* A quick summary of your dataset, including number of entries, minimum and maximum values (for numeric data), and most and least frequent categories (for categorical data).
+* A quick model of the dataset, which can be used to generate an importance score for each feature.
+* A target leakage report, which you can use to determine if one or more features are strongly correlated with your target feature.
+* A custom visualization using your own code.
+
+
+Following sections showcase few of the analysis techniques for the Hotel-bookings data.
+
+### Get Insights
+
+You can get the Data Insights report by selecting **Get Insights** option for the **Data Types** block as shown in the figure.
+
+
+
+
+Please select the following parameters and hit **Create**.
+ - `Target column`: `is-cancelled`
+ - `Problem type`: `Classification`
+
+After the report is generated, it outlines findings about statistics, duplicate rows, warnings, confusion matrix and feature summary. This can be a useful report before we start our detailed analysis.
+
+
+
+
+
+### Table Summary
+
+Select **Table Summary** Analysis in the **Add Analysis** window.
+
+Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Table Summary`
+
+Overall details of the data for various columsn is displayed as depicted in figure below.
+
+
+
+Select **Save** to save the analysis.
+
+
+### Target Leakage
+
+Target leakage occurs when there is data in a machine learning training dataset that is strongly correlated with the target label, but is not available in real-world data. For example, you may have a column in your dataset that serves as a proxy for the column you want to predict with your model.
+
+When you use the Target Leakage analysis, you specify the following:
+* **Target**: This is the feature about which you want your ML model to be able to make predictions.
+* **Problem type**: This is the ML problem type on which you are working. Problem type can either be classification or regression.
+* (Optional) **Max features**: This is the maximum number of features to present in the visualization, which shows features ranked by their risk of being target leakage.
+
+For classification, the target leakage analysis uses the area under the receiver operating characteristic, or AUC - ROC curve for each column, up to Max features. For regression, it uses a coefficient of determination, or R2 metric.
+
+The AUC - ROC curve provides a predictive metric, computed individually for each column using cross-validation, on a sample of up to around 1000 rows. A score of 1 indicates perfect predictive abilities, which often indicates target leakage. A score of 0.5 or lower indicates that the information on the column could not provide, on its own, any useful information towards predicting the target. Although it can happen that a column is uninformative on its own but is useful in predicting the target when used in tandem with other features, a low score could indicate the feature is redundant.
+
+To create a target leakage analysis, Select **Target Leakage** Analysis in the **Add Analysis** window.
+Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Target Leakage`
+- `Max features` : `40`
+- `Target column`: `is-cancelled`
+- `Problem type`: `Classification`
+
+
+
+
+
+For our example dataset, the image below shows a target leakage report for a hotel booking cancellation problem, that is, predicting if a person will cancel his hotel reservation or not. An AUC - ROC curve is used to calculate the predictive ability of 31 raw features, out of which `reservation_status` was determined to a target leakage. Also, features - `arrival_day_of_month`, `babies`, `reservation_status_date`, `arrival_date_month`, `reserved_room_type`, `hotel` and `days_in_waiting_list` were identified as redundant.
+
+The identified features can be fairly omitted as part of the transformations we will apply post this initial analysis.
+
+
+
+Select **Save** to save the analysis.
+
+
+
+
+Next, with SageMaker Data Wrangler’s feature correlation visualization you can easily calculate the correlation of features in your data set and visualize them as a correlation matrix. We will look into 2 types of feature correlations and how to use them on our example dataset in hand.
+
+### Feature Correlation (Linear)
+
+Linear feature correlation is based on Pearson's correlation. Numeric to categorical correlation is calculated by encoding the categorical features as the floating point numbers that best predict the numeric feature before calculating Pearson's correlation. Linear categorical to categorical correlation is not supported.
+
+Numeric to numeric correlation is in the range [-1, 1] where 0 implies no correlation, 1 implies perfect correlation and -1 implies perfect inverse correlation. Numeric to categorical and categorical to categrical correlations are in the range [0, 1] where 0 implies no correlation and 1 implies perfect correlation.
+To create the analysis, choose **Feature Correlation** for the Analysis type and choose **linear** for Correlation type. Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Linear Correlation`
+
+This analysis will take a few minutes to complete.
+
+Features that are not either numeric or categorical are ignored. The table below lists for each feature what is the most correlated feature to it.
+
+
+
+Based on the correlation values, we can see the top 6 feature pairs (as listed below) are strongly correlating with one another. Also, some of these features also showed up in the target analysis we did previously.
+
+
+
+P.S.: A limit of 100,000 rows is used for this analysis.
+
+Select **Save** to save the analysis.
+
+### Feature Correlation (Non-Linear)
+
+Non-linear feature correlation is based on Spearman's rank correlation. Numeric to categorical correlation is calculated by encoding the categorical features as the floating point numbers that best predict the numeric feature before calculating Spearman's rank correlation. Categorical to categorical correlation is based on the normalized Cramer's V test.
+
+Numeric to numeric correlation is in the range [-1, 1] where 0 implies no correlation, 1 implies perfect correlation and -1 implies perfect inverse correlation. Numeric to categorical and categorical to categrical correlations are in the range [0, 1] where 0 implies no correlation and 1 implies perfect correlation
+
+Features that are not either numeric or categorical are ignored.
+
+To create the analysis, choose **Feature Correlation** for the Analysis type and **non-linear** for Correlation type. Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Non-Linear Correlation`
+
+This analysis will take a few minutes to complete.
+
+The table below lists for each feature what is the most correlated feature to it. You can see most of the top correlated feature pairs overlap with the previous two analyses.
+
+
+
+Select **Save** to save the analysis.
+
+### Multicolinearity (Variance Inflation Factors)
+
+Variance Inflation Factor (VIF) is a measure of colinearity among variables. It is calculated by solving a regression problem to predict one variable given the rest. A VIF score is a positive number that is greater or equal than 1, and a score of 1 means the variable is completely independent of the others. The larger the score, the more dependent it is. Since it is an inverse, it is possible for the VIF score to be infinite. Note that we cap the VIF score at 50. As a rule of thumb for cases where the number of samples is not abnormally small, a score of up to 5 means the variable is only moderatly correlated, and beyond 5 it is highly correlated.
+
+To create the analysis for VIF, choose **Multicollinearity** for Analysis type and choose **Variance inflation factors** for Analysis. Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Variance Inflation Factors`
+
+This analysis will take a few minutes to complete.
+
+As per the above rule, we can eliminate the following feature columns from our feature set since they will not contribute effectively towards the prediction capability of the model that gets trained using these features.
+
+* `arrival_date_year`
+* `adults`, `agents`
+* `arrival_date_week_number`
+* `stays_in_week_nights`
+
+
+
+Select **Save** to save the analysis.
+
+
+### Multicolinearity - Principal Component Analysis (PCA)
+
+Principal Component Analysis (PCA) measures the variance of the data along different directions in the feature space. The ordered list of variances, also known as the singular values, can inform about multicolinearity in our data. This list contains non-negative numbers. When the numbers are roughly uniform, the data has very few multicolinearities. However, when the opposite is true, the magnitude of the top values will dominate the rest. In order to avoid issues related to different scales, the individual features are standardized to have mean 0 and standard deviation 1 before applying PCA.
+
+To create the analysis for PCA, choose **Multicollinearity** for Analysis type and choose **Principal component analysis** for Analysis. Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Principal Component Analysis`
+
+This analysis will take a few minutes to complete.
+
+As per the above rule, it is evident the numbers (variances) are not uniform hence confirming that the data has multicolinearies to fix. This has already been confirmed by our previous analysis.
+
+
+
+
+
+Select **Save** to save the analysis.
+
+
+### Multicolinearity Lasso Feature Selection
+
+
+Lasso feature selection trains a linear classifier with L1 regularization (you can control the strength of L1 penalty by adjusting "L1 magnitude") that induces a sparse solution. The regressor provides a coefficient for each feature, and the absolute value of this coefficient could be interpreted as an importance score for that feature.
+
+To create the analysis for Lasso Feature Selection, choose **Multicollinearity** for Analysis type and choose **Lasso feature selection** for Analysis. Please select the following parameters and hit **Preview**.
+- `Analysis name`: `Non-Linear Correlation`
+- `L1 Magnitude`: `1`
+- `Problem Type`: `Classification`
+- `Label Column` : `is_cancelled`
+
+This analysis will take a few minutes to complete.
+
+
+The plot below provides features' importance scores (absolute coefficients) after training a classifier on a sample of the dataset (10k for large dataset). The training process includes a standardization of the features to have mean 0 and standard deviation 1 in order to avoid a skewed importance score due to different scales.
+
+The classifier obtained a roc_auc score: `0.639269142214666`.
+
+
+
+Select **Save** to save the analysis.
+
+
+
+### Detect Duplicate Rows
+Next, with the new duplicate row detection visualization, you can quickly detect if your data set has any duplicate rows. To apply this analysis, choose **Duplicate rows** for Analysis type.
+
+From the figure bwlow, we can see almost ~33% of the rows in the dataset are duplicates.
+
+
+
+
+
+### Quick Model
+
+We can create a quick model using the raw features to determine how good our features are, prior to applying transformations.
+
+Use the Quick Model visualization to quickly evaluate your data and produce importance scores for each feature. A feature importance score indicates how useful a feature is at predicting a target label. The feature importance score is between [0, 1] and a higher number indicates that the feature is more important to the whole dataset. On the top of the quick model chart, there is a model score. A classification problem shows an F1 score. A regression problem has a mean squared error (MSE) score.
+
+When you create a quick model chart, you select a dataset you want evaluated, and a target label against which you want feature importance to be compared. Data Wrangler does the following:
+* Infers the data types for the target label and each feature in the dataset selected.
+* Determines the problem type. Based on the number of distinct values in the label column, Data Wrangler determines if this is a regression or classification problem type. Data Wrangler sets a categorical threshold to 100. If there are more than 100 distinct values in the label column, Data Wrangler classifies it as a regression problem; otherwise, it is classified as a classification problem.
+* Pre-processes features and label data for training. The algorithm used requires encoding features to vector type and encoding labels to double type.
+* Trains a random forest algorithm with 70% of data. Spark’s RandomForestRegressor is used to train a model for regression problems. The RandomForestClassifier is used to train a model for classification problems.
+* Evaluates a random forest model with the remaining 30% of data. Data Wrangler evaluates classification models using an F1 score and evaluates regression models using an MSE (mean squared error) score.
+* Calculates feature importance for each feature using the Gini importance method.
+
+Let us create a prediction model on the fly for the problem for the booking cancellation problem using the raw crude features we started with in Data Wrangler's Quick Model option.
+
+Please choose **Quick Model** for Analysis type. Select the following parameters and hit **Preview**.
+- `Analysis name`: `Model pre-transform`
+
+This analysis will take a few minutes to complete.
+
+
+
+A limit of 100,000 rows is used for this analysis. You can use the Quick Model feature to provide a rough estimate of the expected predicted quality and the predictive power of the features in your dataset.
+
+We can from the results below, Quick model was able to predict with an F1 score of 82% on the test set. But, this is misleading, given we haven't eliminated most of the feature columns that are a target leakage or redundant based on high colinearity. This is justified in the results below where the column `reservation_status` which is a target leakage ranked as the most important feature.
+
+
+
+
+Select **Save** to save the model.
+
+### Next Steps
+
+In the next section, we will apply post analysis transformations to fix the data of the various colinearity and other issues and re-generate a quick model and compare the differences. Please refer to **[Data Transformations](./Data-Transformations.md)** and follow steps for Data Transformation.
+
diff --git a/sagemaker-datawrangler/tabular-dataflow/data-exploration/index.rst b/sagemaker-datawrangler/tabular-dataflow/data-exploration/index.rst
new file mode 100644
index 0000000000..e3dc6e001d
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/data-exploration/index.rst
@@ -0,0 +1,408 @@
+Exploratory Data Analysis
+=========================
+
+Analyze and Visualize
+---------------------
+
+Before we transform our raw features to make it ML ready for model
+building, lets analyze and visualize the booking cancellations dataset
+to detect features that are important to our problem and ones that are
+not. This can be achieved through Exploratory Data Analysis (EDA).
+Amazon SageMaker Data Wrangler includes built-in analyses that help you
+generate visualizations and data analyses in a few clicks. You can also
+create custom analyses using your own code.
+
+In order to apply an action on the imported data, select the **Add
+Analysis** option on right clicking the the **Data Types** block. As
+depicted in the figure below, you can see options to add a transform,
+perform an analysis, add a destination sink or export the steps as
+Jupyter notebook. You can also join and concatenate on the imported
+dataset with other datasets.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-8.png
+
+
+
+
+On selecting Add Analysis, the Analysis pane os shown, where you can
+choose the analysis you want to perform.
+
+All analyses are generated using 100,000 rows of your dataset.
+
+You can add the following analysis to a dataframe:
+
+- Data visualizations, including histograms and scatter plots.
+- A quick summary of your dataset, including number of entries, minimum
+ and maximum values (for numeric data), and most and least frequent
+ categories (for categorical data).
+- A quick model of the dataset, which can be used to generate an
+ importance score for each feature.
+- A target leakage report, which you can use to determine if one or
+ more features are strongly correlated with your target feature.
+- A custom visualization using your own code.
+
+Following sections showcase few of the analysis techniques for the
+Hotel-bookings data.
+
+Get Insights
+------------
+
+You can get the Data Insights report by selecting **Get Insights**
+option for the **Data Types** block as shown in the figure.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/get-insights.png
+
+
+
+
+| Please select the following parameters and hit **Create**.
+| - ``Target column``: ``is-cancelled`` - ``Problem type``:
+ ``Classification``
+
+After the report is generated, it outlines findings about statistics,
+duplicate rows, warnings, confusion matrix and feature summary. This can
+be a useful report before we start our detailed analysis.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/get-insights-report.png
+
+
+
+Table Summary
+-------------
+
+Select **Table Summary** Analysis in the **Add Analysis** window.
+
+| Please select the following parameters and hit **Preview**.
+| - ``Analysis name``: ``Table Summary``
+
+Overall details of the data for various columsn is displayed as depicted
+in figure below.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/table-information.png
+
+
+
+Select **Save** to save the analysis.
+
+Target Leakage
+--------------
+
+Target leakage occurs when there is data in a machine learning training
+dataset that is strongly correlated with the target label, but is not
+available in real-world data. For example, you may have a column in your
+dataset that serves as a proxy for the column you want to predict with
+your model.
+
+When you use the Target Leakage analysis, you specify the following: \*
+**Target**: This is the feature about which you want your ML model to be
+able to make predictions. \* **Problem type**: This is the ML problem
+type on which you are working. Problem type can either be classification
+or regression. \* (Optional) **Max features**: This is the maximum
+number of features to present in the visualization, which shows features
+ranked by their risk of being target leakage.
+
+For classification, the target leakage analysis uses the area under the
+receiver operating characteristic, or AUC - ROC curve for each column,
+up to Max features. For regression, it uses a coefficient of
+determination, or R2 metric.
+
+The AUC - ROC curve provides a predictive metric, computed individually
+for each column using cross-validation, on a sample of up to around 1000
+rows. A score of 1 indicates perfect predictive abilities, which often
+indicates target leakage. A score of 0.5 or lower indicates that the
+information on the column could not provide, on its own, any useful
+information towards predicting the target. Although it can happen that a
+column is uninformative on its own but is useful in predicting the
+target when used in tandem with other features, a low score could
+indicate the feature is redundant.
+
+| To create a target leakage analysis, Select **Target Leakage**
+ Analysis in the **Add Analysis** window. Please select the following
+ parameters and hit **Preview**.
+| - ``Analysis name``: ``Target Leakage`` - ``Max features`` : ``40`` -
+ ``Target column``: ``is-cancelled`` - ``Problem type``:
+ ``Classification``
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/target-leakage-pre.png
+
+
+ image
+
+For our example dataset, the image below shows a target leakage report
+for a hotel booking cancellation problem, that is, predicting if a
+person will cancel his hotel reservation or not. An AUC - ROC curve is
+used to calculate the predictive ability of 31 raw features, out of
+which ``reservation_status`` was determined to a target leakage. Also,
+features - ``arrival_day_of_month``, ``babies``,
+``reservation_status_date``, ``arrival_date_month``,
+``reserved_room_type``, ``hotel`` and ``days_in_waiting_list`` were
+identified as redundant.
+
+The identified features can be fairly omitted as part of the
+transformations we will apply post this initial analysis.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/target-leakage.png
+
+
+Select **Save** to save the analysis.
+
+Next, with SageMaker Data Wrangler’s feature correlation visualization
+you can easily calculate the correlation of features in your data set
+and visualize them as a correlation matrix. We will look into 2 types of
+feature correlations and how to use them on our example dataset in hand.
+
+Feature Correlation (Linear)
+----------------------------
+
+Linear feature correlation is based on Pearson’s correlation. Numeric to
+categorical correlation is calculated by encoding the categorical
+features as the floating point numbers that best predict the numeric
+feature before calculating Pearson’s correlation. Linear categorical to
+categorical correlation is not supported.
+
+| Numeric to numeric correlation is in the range [-1, 1] where 0 implies
+ no correlation, 1 implies perfect correlation and -1 implies perfect
+ inverse correlation. Numeric to categorical and categorical to
+ categrical correlations are in the range [0, 1] where 0 implies no
+ correlation and 1 implies perfect correlation. To create the analysis,
+ choose **Feature Correlation** for the Analysis type and choose
+ **linear** for Correlation type. Please select the following
+ parameters and hit **Preview**.
+| - ``Analysis name``: ``Linear Correlation``
+
+This analysis will take a few minutes to complete.
+
+Features that are not either numeric or categorical are ignored. The
+table below lists for each feature what is the most correlated feature
+to it.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/linear-pre.png
+
+Based on the correlation values, we can see the top 6 feature pairs (as
+listed below) are strongly correlating with one another. Also, some of
+these features also showed up in the target analysis we did previously.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/linear-strongly-correlated.png
+
+
+P.S.: A limit of 100,000 rows is used for this analysis.
+
+Select **Save** to save the analysis.
+
+Feature Correlation (Non-Linear)
+--------------------------------
+
+Non-linear feature correlation is based on Spearman’s rank correlation.
+Numeric to categorical correlation is calculated by encoding the
+categorical features as the floating point numbers that best predict the
+numeric feature before calculating Spearman’s rank correlation.
+Categorical to categorical correlation is based on the normalized
+Cramer’s V test.
+
+Numeric to numeric correlation is in the range [-1, 1] where 0 implies
+no correlation, 1 implies perfect correlation and -1 implies perfect
+inverse correlation. Numeric to categorical and categorical to
+categrical correlations are in the range [0, 1] where 0 implies no
+correlation and 1 implies perfect correlation
+
+Features that are not either numeric or categorical are ignored.
+
+| To create the analysis, choose **Feature Correlation** for the
+ Analysis type and **non-linear** for Correlation type. Please select
+ the following parameters and hit **Preview**.
+| - ``Analysis name``: ``Non-Linear Correlation``
+
+This analysis will take a few minutes to complete.
+
+The table below lists for each feature what is the most correlated
+feature to it. You can see most of the top correlated feature pairs
+overlap with the previous two analyses.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/non-linear-pre.png
+
+
+Select **Save** to save the analysis.
+
+Multicolinearity (Variance Inflation Factors)
+---------------------------------------------
+
+Variance Inflation Factor (VIF) is a measure of colinearity among
+variables. It is calculated by solving a regression problem to predict
+one variable given the rest. A VIF score is a positive number that is
+greater or equal than 1, and a score of 1 means the variable is
+completely independent of the others. The larger the score, the more
+dependent it is. Since it is an inverse, it is possible for the VIF
+score to be infinite. Note that we cap the VIF score at 50. As a rule of
+thumb for cases where the number of samples is not abnormally small, a
+score of up to 5 means the variable is only moderatly correlated, and
+beyond 5 it is highly correlated.
+
+| To create the analysis for VIF, choose **Multicollinearity** for
+ Analysis type and choose **Variance inflation factors** for Analysis.
+ Please select the following parameters and hit **Preview**.
+| - ``Analysis name``: ``Variance Inflation Factors``
+
+This analysis will take a few minutes to complete.
+
+As per the above rule, we can eliminate the following feature columns
+from our feature set since they will not contribute effectively towards
+the prediction capability of the model that gets trained using these
+features.
+
+- ``arrival_date_year``
+- ``adults``, ``agents``
+- ``arrival_date_week_number``
+- ``stays_in_week_nights``
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/vif-pre.png
+
+
+Select **Save** to save the analysis.
+
+Multicolinearity - Principal Component Analysis (PCA)
+-----------------------------------------------------
+
+Principal Component Analysis (PCA) measures the variance of the data
+along different directions in the feature space. The ordered list of
+variances, also known as the singular values, can inform about
+multicolinearity in our data. This list contains non-negative numbers.
+When the numbers are roughly uniform, the data has very few
+multicolinearities. However, when the opposite is true, the magnitude of
+the top values will dominate the rest. In order to avoid issues related
+to different scales, the individual features are standardized to have
+mean 0 and standard deviation 1 before applying PCA.
+
+| To create the analysis for PCA, choose **Multicollinearity** for
+ Analysis type and choose **Principal component analysis** for
+ Analysis. Please select the following parameters and hit **Preview**.
+| - ``Analysis name``: ``Principal Component Analysis``
+
+This analysis will take a few minutes to complete.
+
+As per the above rule, it is evident the numbers (variances) are not
+uniform hence confirming that the data has multicolinearies to fix. This
+has already been confirmed by our previous analysis.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/pca-pre.png
+
+Select **Save** to save the analysis.
+
+Multicolinearity Lasso Feature Selection
+----------------------------------------
+
+Lasso feature selection trains a linear classifier with L1
+regularization (you can control the strength of L1 penalty by adjusting
+“L1 magnitude”) that induces a sparse solution. The regressor provides a
+coefficient for each feature, and the absolute value of this coefficient
+could be interpreted as an importance score for that feature.
+
+| To create the analysis for Lasso Feature Selection, choose
+ **Multicollinearity** for Analysis type and choose **Lasso feature
+ selection** for Analysis. Please select the following parameters and
+ hit **Preview**.
+| - ``Analysis name``: ``Non-Linear Correlation`` - ``L1 Magnitude``:
+ ``1`` - ``Problem Type``: ``Classification`` - ``Label Column`` :
+ ``is_cancelled``
+
+This analysis will take a few minutes to complete.
+
+The plot below provides features’ importance scores (absolute
+coefficients) after training a classifier on a sample of the dataset
+(10k for large dataset). The training process includes a standardization
+of the features to have mean 0 and standard deviation 1 in order to
+avoid a skewed importance score due to different scales.
+
+The classifier obtained a roc_auc score: ``0.639269142214666``.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/lasso-pre.png
+
+
+Select **Save** to save the analysis.
+
+Detect Duplicate Rows
+---------------------
+
+Next, with the new duplicate row detection visualization, you can
+quickly detect if your data set has any duplicate rows. To apply this
+analysis, choose **Duplicate rows** for Analysis type.
+
+From the figure bwlow, we can see almost ~33% of the rows in the dataset
+are duplicates.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/duplicate-rows.png
+
+
+Quick Model
+-----------
+
+We can create a quick model using the raw features to determine how good
+our features are, prior to applying transformations.
+
+Use the Quick Model visualization to quickly evaluate your data and
+produce importance scores for each feature. A feature importance score
+indicates how useful a feature is at predicting a target label. The
+feature importance score is between [0, 1] and a higher number indicates
+that the feature is more important to the whole dataset. On the top of
+the quick model chart, there is a model score. A classification problem
+shows an F1 score. A regression problem has a mean squared error (MSE)
+score.
+
+When you create a quick model chart, you select a dataset you want
+evaluated, and a target label against which you want feature importance
+to be compared. Data Wrangler does the following: \* Infers the data
+types for the target label and each feature in the dataset selected. \*
+Determines the problem type. Based on the number of distinct values in
+the label column, Data Wrangler determines if this is a regression or
+classification problem type. Data Wrangler sets a categorical threshold
+to 100. If there are more than 100 distinct values in the label column,
+Data Wrangler classifies it as a regression problem; otherwise, it is
+classified as a classification problem. \* Pre-processes features and
+label data for training. The algorithm used requires encoding features
+to vector type and encoding labels to double type. \* Trains a random
+forest algorithm with 70% of data. Spark’s RandomForestRegressor is used
+to train a model for regression problems. The RandomForestClassifier is
+used to train a model for classification problems. \* Evaluates a random
+forest model with the remaining 30% of data. Data Wrangler evaluates
+classification models using an F1 score and evaluates regression models
+using an MSE (mean squared error) score. \* Calculates feature
+importance for each feature using the Gini importance method.
+
+Let us create a prediction model on the fly for the problem for the
+booking cancellation problem using the raw crude features we started
+with in Data Wrangler’s Quick Model option.
+
+| Please choose **Quick Model** for Analysis type. Select the following
+ parameters and hit **Preview**.
+| - ``Analysis name``: ``Model pre-transform``
+
+This analysis will take a few minutes to complete.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/create-quick-model.png
+
+
+
+
+A limit of 100,000 rows is used for this analysis. You can use the Quick
+Model feature to provide a rough estimate of the expected predicted
+quality and the predictive power of the features in your dataset.
+
+We can from the results below, Quick model was able to predict with an
+F1 score of 82% on the test set. But, this is misleading, given we
+haven’t eliminated most of the feature columns that are a target leakage
+or redundant based on high colinearity. This is justified in the results
+below where the column ``reservation_status`` which is a target leakage
+ranked as the most important feature.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/quick-model-pre.png
+
+
+
+
+Select **Save** to save the model.
+
+Next Steps
+----------
+.. toctree::
+ :maxdepth: 1
+
+ ./data-transdormations/index
diff --git a/sagemaker-datawrangler/tabular-dataflow/data-import/Data-Import.md b/sagemaker-datawrangler/tabular-dataflow/data-import/Data-Import.md
new file mode 100644
index 0000000000..a71a73153d
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/data-import/Data-Import.md
@@ -0,0 +1,49 @@
+# Importing Dataset into Data Wrangler using SageMaker Studio
+
+Following steps outline how to import data into Sagemaker to be consumed by Data Wrangler
+
+
+Steps to import data
+1. Initialize SageMaker Data Wrangler via SageMaker Studio UI. You can use any one of the options specified below.
+
+
+
+ - Option 1. Use the Sage Maker Launcher screen as depicted here:
+
+ 
+
+ - Option 2. You can use the SageMaker resources menu on the left, selecting Data Wrangler, and new flow
+
+ 
+ 
+ - Option 3. You can also use the File -> New -> DataWrangler option as shown here
+
+ 
+2. Data Wrangler takes a few minutes to load.
+
+
+3. Once Data Wrangler is loaded, you should be able to see it under running instances and apps as shown below.
+
+
+
+4. Once Data Wrangler is up and running, you can see the following data flow interface with options for import, creating data flows and export as shown below.
+
+
+
+5. Make sure to rename the untitled.flow to your preference (for e.g., hotel-bookings.flow)
+
+6. Now you will have the option to select your data source. Because the data is in Amazon S3, select Amazon S3 to import data. Paste the S3 URL for the hotel-bookings.csv file into the search box below and hit go.
+
+
+
+7. Data Wrangler will show you a preview of the data. Select the CSV file from the drop down results. On the right pane, make sure COMMA is chosen as the delimiter and Sampling is *None*. Our data set is small enough to run Data Wrangler transformations on the full data set. If you have a large data set, consider using sampling. Finally select *Import* to import this dataset to Data Wrangler.
+
+
+
+8. Once the dataset is imported, Data Wrangler automatically validates the dataset and detects the data types. The flow editor now shows 2 blocks showcasing that the data was imported from a source and data types recognized. You are also allowed to edit the data types if needed.
+
+
+
+### Next Steps
+
+As a next step, we will explore the data that we uploaded. Please refer to **[Exploratory Data Analysis](./Data-Exploration.md)** and follow steps for Data exploration.
diff --git a/sagemaker-datawrangler/tabular-dataflow/data-import/index.rst b/sagemaker-datawrangler/tabular-dataflow/data-import/index.rst
new file mode 100644
index 0000000000..76ce7141f4
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/data-import/index.rst
@@ -0,0 +1,87 @@
+Importing Dataset into Data Wrangler using SageMaker Studio
+===========================================================
+
+Following steps outline how to import data into Sagemaker to be consumed
+by Data Wrangler
+
+| Steps to import data
+| 1. Initialize SageMaker Data Wrangler via SageMaker Studio UI. You can
+ use any one of the options specified below.
+
+
+
+Option 1 : Use the Sage Maker Launcher screen
+--------------------------------------------
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-1.png
+
+
+Option 2 : You can use the SageMaker resources menu on the left, selecting Data Wrangler, and new flow
+----------------------------------------------------------------------------
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-1-1.png
+
+
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-1-2.png
+
+
+Option 3. : You can also use the File -> New -> DataWrangler option as shown here
+--------------------------------------------------------------------------------
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-1-3.png
+
+
+2. Data Wrangler takes a few minutes to load.
+
+|image|
+
+3. Once Data Wrangler is loaded, you should be able to see it
+under running instances and apps as shown below.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-3.png
+
+
+4. Once Data Wrangler is up and running, you can see the following data
+ flow interface with options for import, creating data flows and
+ export as shown below.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-4.png
+
+
+5. Make sure to rename the untitled.flow to your preference (for e.g.,
+ hotel-bookings.flow)
+
+6. Now you will have the option to select your data source. Because the
+ data is in Amazon S3, select Amazon S3 to import data. Paste the S3
+ URL for the hotel-bookings.csv file into the search box below and hit
+ go.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-5.png
+
+
+7. Data Wrangler will show you a preview of the data. Select the CSV
+ file from the drop down results. On the right pane, make sure COMMA
+ is chosen as the delimiter and Sampling is *None*. Our data set is
+ small enough to run Data Wrangler transformations on the full data
+ set. If you have a large data set, consider using sampling. Finally
+ select *Import* to import this dataset to Data Wrangler.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-6.png
+
+
+8. Once the dataset is imported, Data Wrangler automatically validates
+ the dataset and detects the data types. The flow editor now shows 2
+ blocks showcasing that the data was imported from a source and data
+ types recognized. You are also allowed to edit the data types if
+ needed.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/image-7.png
+
+
+Next Steps
+----------
+.. toctree::
+ :maxdepth: 1
+
+ ./data-exploration/index
+
diff --git a/sagemaker-datawrangler/tabular-dataflow/data-transformations/Data-Transformations.md b/sagemaker-datawrangler/tabular-dataflow/data-transformations/Data-Transformations.md
new file mode 100644
index 0000000000..d2a782f538
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/data-transformations/Data-Transformations.md
@@ -0,0 +1,195 @@
+# Data Transformations
+
+
+Based on the Data explorations carried out in previous step, we are now ready to apply transformations to the data.
+Amazon SageMaker Data Wrangler provides numerous ML data transforms to streamline cleaning, transforming, and featurizing your data. When you add a transform, it adds a step to the data flow. Each transform you add modifies your dataset and produces a new dataframe. All subsequent transforms apply to the resulting dataframe.
+
+Data Wrangler includes built-in transforms, which you can use to transform columns without any code. You can also add custom transformations using PySpark, Pandas, and PySpark SQL. Some transforms operate in place, while others create a new output column in your dataset.
+
+In order to apply an action on the imported data, select the **Add Transform** option on right clicking the the **Data Types** block. In the displayed window, select **Add Steps** to display **Add Transform** window.
+
+ 
+
+
+
+### Drop Columns
+ Now we will drop columns based on the analyses we performed in the previous section.
+
+- based on target leakage : drop `reservation_status`
+
+- redundant columns : drop columns that are redundant - `days_in_waiting_list`, `hotel`, `reserved_room_type`, `arrival_date_month`, `reservation_status_date`, `babies` and `arrival_date_day_of_month`
+
+- based on linear correlation results : drop columns `arrival_date_week_number`, `arrival_date_year` as correlation values for these feature (column) pairs are greater than the recommended threshold of 0.90.
+
+- based on non-linear correlation results: drop `reservation_status`. This column was already marked to be dropped based on Target leakage analysis.
+
+ we can drop all these columns in one go. To drop columns, choose **Manage columns** transform from the **Add Transform** window. Then select the **Drop column** option from **Manage columns** transform.
+
+Please select the the column names we want to drop as shown in the image below and hit **Preview**.
+
+
+ 
+
+If the Preview is OK, click **Add** to add the transform in the data flow.
+
+Further, based on the multi-colinearity analysis results, we can also drop the columns `adults` and `agent` for whom the variance inflation factor scores are greater than 5. Please select the the column names we want to drop and hit **Preview**.
+
+
+
+ 
+
+ If the Preview is OK, click **Add** to add the transform in the data flow.
+
+
+### Drop Duplicate Rows
+To drop the duplicate rows that we identified based on the analysis we did in the previous section. To drop columns, choose **Manage rows** transform from the **Add Transform** window. Then select the **Drop duplicates** option from **Manage rows** transform and hit **Preview**.
+
+
+ 
+
+ If the Preview is OK, click **Add** to add the transform in the data flow.
+
+
+### Handle Outliers
+An outlier can cause serious problems in statistical analysis. Machine learning models are sensitive to the distribution and range of feature values. Outliers, or rare values, can negatively impact model accuracy and lead to longer training times. When you define a Handle outliers transform step, the statistics used to detect outliers are generated on the data available in Data Wrangler when defining this step. These same statistics are used when running a Data Wrangler job.
+
+To handle outliers, choose **Handle outliers** transform from the **Add Transform** window.
+Please select the following parameters and hit **Preview**.
+
+- `Transform`: `Standard deviation numeric outliers`
+- `Fix method` : `Remove`
+- `Standard deviations`: `4`
+- `Input columns`: `lead_time`,`stays_in_weekend_nights`, `stays_in_weekday_nights`, `is_repeated_guest`, `prev_cancellations`, `prev_bookings_not_canceled`, `booking_changes`, `adr`, `total_of_specical_requests`, `required_car_parking_spaces`,
+
+
+
+
+
+
+ If the Preview is OK, click **Add** to add the transform in the data flow.
+
+
+### Handle Missing Values
+To handle outliers, choose **Handle missing values** transform from the **Add Transform** window. We can do the following to handle missing values in our feature columns using Data Wrangler.
+
+
+ - Missing values in **Children** column : Majority of the visitors were not accompanied by children and hence missing data can be replaced by number of children = 0.
+
+
+
+ Please hit **Preview** to look at the transform preview, and if it is OK, click **Add** to add the transform in the data flow.
+
+- Missing values in **Country** column
+Iterating through the country column reveals that most of the clients are from Europe. Therefore, all the missing values in the country column are replaced with the country of maximum occurrence - Portugal (PRT). Fill missing country column with `PRT` based on value counts
+
+
+
+Please hit **Preview** to look at the transform preview, and if it is OK, click **Add** to add the transform in the data flow.
+
+- Custom Transform - Meal type has Undefined category, changing the Undefined value to the most used which is BB by implementing a custom pyspark transform with two simple lines of code. This can be done by choosing **Custom transform** transform from the **Add Transform** window. Specify the name of the transform, and paste following code in the transform as shown in figure below.
+
+ 
+
+
+```python
+from pyspark.sql.functions import when
+
+df = df.withColumn('meal', when(df.meal == 'Undefined', 'BB').otherwise(df.meal))
+```
+
+Please hit **Preview** to look at the transform preview, and if it is OK, click **Add** to add the transform in the data flow.
+
+ ### Numeric Normalization
+Normalization is a scaling technique in which values are shifted and rescaled so that they end up ranging between 0 and 1. It is also known as Min-Max scaling. Standardization is another scaling technique where the values are centered around the mean with a unit standard deviation. This means that the mean of the attribute becomes zero and the resultant distribution has a unit standard deviation.
+
+For our example use case, let's normalize the numeric feature columns to a standard scale [0,1].
+
+From Data Wrangler's list of pre-built transforms, choose **Process numeric**. Please select the following parameters and hit **Preview**.
+
+- `Transform`: `Scale values`
+- `Scalar` : `Min-max scalar`
+- `Min`: `0`
+- `Max`: `1`
+- `Input columns`: `lead_time`,`stays_in_weekend_nights`, `stays_in_weekday_nights`, `is_repeated_guest`, `prev_cancellations`, `prev_bookings_not_canceled`, `booking_changes`, `adr`, `total_of_specical_requests`, `required_car_parking_spaces`
+
+
+ 
+
+
+ If the Preview is OK, click **Add** to add the transform in the data flow.
+
+### Handle Categorical Data
+
+Categorical data is usually composed of a finite number of categories, where each category is represented with a string. Encoding categorical data is the process of creating a numerical representation for categories. With Data Wrangler, we can select Ordinal encode to encode categories into an integer between 0 and the total number of categories in the Input column you select. Select one-hot encoding or use similarity encoding when you have a large number of categorical variables and Noisy data.
+
+
+From Data Wrangler's list of pre-built transforms, choose **Encode Categorical**. Please select the following parameters and hit **Preview**.
+- `Transform`: `One-hot encode`
+- `Invalid handling strategy` : `Keep`
+- `Output style`: `Columns`
+- `Max`: `1`
+- `Input columns`: `meal`, `is_repeated_guest`, `market_segment`, `assigned_room_type`, `deposit_type`, `customer_type`
+
+
+
+ 
+
+
+ If the Preview is OK, click **Add** to add the transform in the data flow.
+
+
+### Balancing the target variable
+
+DataWrangler also helps to balance the target variable (column) for class imbalance. Let's presume the following for the negative and positive cases.
+
+ is_canceled = 0 (negative case)
+ is_canceled = 1 (positive case)
+
+In Data Wrangler, we can handle class imbalance using 3 different techniques.
+
+ - Random Undersample
+ - Random Oversample
+ - SMOTE
+
+From the Data Wrangler's transform pane, choose **Balance Data** as the transform. Please select the following parameters as shown in image below and hit **Preview**.
+- `Target column`: `is_canceled`
+- `Desiered ratio` : `1`
+- `Transform`: `Random oversample`
+
+ 
+
+
+ If the Preview is OK, click **Add** to add the transform in the data flow.
+
+The state of the classes before and after balancing is as follows:
+
+The ratio of positive to negative case = ~0.38
+
+
+
+
+After balancing, the ratio is 1
+
+
+### Quick Model
+Given, we have applied most of the needed transformations on our feature columns, we can now create a Quick Model again using the transformed features to identify the predictive ability of our features and take a look at their attribution towards prediction.
+
+It is a good practice to run a Quick Model everytime we make a set of feature transforms. Previously, we ran a Quick Model analysis using the raw features. The results of this previous run was mostly incorrect and misleading, given, we haven't fixed most of the correlation and other issues with our dataset.
+
+The below figure shows the results of the newly run Quick Model created using the transformed features. As you can see, the Quick Model achieved an F1 score of 62% on the test data. The top 5 most contributing features towards this score are as follows which is different from what we see previously.
+
+ lead_time
+ country
+ customer_type_Transient
+ required_car_parking_spaces
+ booking_changes
+
+Craete a quick model, similar to one we created in the **[Exploratory Data Analysis](./Data-Exploration.md)** step.
+
+
+
+Compare the model with the one created in Exploratory Data Analysis step.
+
+### Next Steps
+
+As a next step, we will export the transformed data for further use. Please refer to **[Readme](./README.md)** and follow steps for Data Export.
diff --git a/sagemaker-datawrangler/tabular-dataflow/data-transformations/index.rst b/sagemaker-datawrangler/tabular-dataflow/data-transformations/index.rst
new file mode 100644
index 0000000000..942304a959
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/data-transformations/index.rst
@@ -0,0 +1,301 @@
+Data Transformations
+====================
+
+Based on the Data explorations carried out in previous step, we are now
+ready to apply transformations to the data. Amazon SageMaker Data
+Wrangler provides numerous ML data transforms to streamline cleaning,
+transforming, and featurizing your data. When you add a transform, it
+adds a step to the data flow. Each transform you add modifies your
+dataset and produces a new dataframe. All subsequent transforms apply to
+the resulting dataframe.
+
+Data Wrangler includes built-in transforms, which you can use to
+transform columns without any code. You can also add custom
+transformations using PySpark, Pandas, and PySpark SQL. Some transforms
+operate in place, while others create a new output column in your
+dataset.
+
+In order to apply an action on the imported data, select the **Add
+Transform** option on right clicking the the **Data Types** block. In
+the displayed window, select **Add Steps** to display **Add Transform**
+window.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/add-transform.png
+
+
+Drop Columns
+------------
+
+Now we will drop columns based on the analyses we performed in the
+previous section.
+
+- based on target leakage : drop ``reservation_status``
+
+- redundant columns : drop columns that are redundant -
+ ``days_in_waiting_list``, ``hotel``, ``reserved_room_type``,
+ ``arrival_date_month``, ``reservation_status_date``, ``babies`` and
+ ``arrival_date_day_of_month``
+
+- based on linear correlation results : drop columns
+ ``arrival_date_week_number``, ``arrival_date_year`` as correlation
+ values for these feature (column) pairs are greater than the
+ recommended threshold of 0.90.
+
+- based on non-linear correlation results: drop ``reservation_status``.
+ This column was already marked to be dropped based on Target leakage
+ analysis.
+
+we can drop all these columns in one go. To drop columns, choose
+**Manage columns** transform from the **Add Transform** window. Then
+select the **Drop column** option from **Manage columns** transform.
+
+Please select the the column names we want to drop as shown in the image
+below and hit **Preview**.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/drop-columns.png
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+Further, based on the multi-colinearity analysis results, we can also
+drop the columns ``adults`` and ``agent`` for whom the variance
+inflation factor scores are greater than 5. Please select the the column
+names we want to drop and hit **Preview**.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/drop-more-cols.png
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+Drop Duplicate Rows
+-------------------
+
+We should drop the duplicate rows that we identified based on the analysis we
+did in the previous section. To drop columns, choose **Manage rows**
+transform from the **Add Transform** window. Then select the **Drop
+duplicates** option from **Manage rows** transform and hit **Preview**.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/drop-duplicates.png
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+Handle Outliers
+---------------
+
+An outlier can cause serious problems in statistical analysis. Machine
+learning models are sensitive to the distribution and range of feature
+values. Outliers, or rare values, can negatively impact model accuracy
+and lead to longer training times. When you define a Handle outliers
+transform step, the statistics used to detect outliers are generated on
+the data available in Data Wrangler when defining this step. These same
+statistics are used when running a Data Wrangler job.
+
+| To handle outliers, choose **Handle outliers** transform from the
+ **Add Transform** window.
+| Please select the following parameters and hit **Preview**.
+
+- ``Transform``: ``Standard deviation numeric outliers``
+- ``Fix method`` : ``Remove``
+- ``Standard deviations``: ``4``
+- ``Input columns``: ``lead_time``,\ ``stays_in_weekend_nights``,
+ ``stays_in_weekday_nights``, ``is_repeated_guest``,
+ ``prev_cancellations``, ``prev_bookings_not_canceled``,
+ ``booking_changes``, ``adr``, ``total_of_specical_requests``,
+ ``required_car_parking_spaces``,
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/handle-outliers.png
+
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+Handle Missing Values
+---------------------
+
+To handle outliers, choose **Handle missing values** transform from the
+**Add Transform** window. We can do the following to handle missing
+values in our feature columns using Data Wrangler.
+
+- Missing values in **Children** column : Majority of the visitors were
+ not accompanied by children and hence missing data can be replaced by
+ number of children = 0.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/fill-missing-children-val.png
+
+
+Please hit **Preview** to look at the transform preview, and if it is
+OK, click **Add** to add the transform in the data flow.
+
+- Missing values in **Country** column Iterating through the country
+ column reveals that most of the clients are from Europe. Therefore,
+ all the missing values in the country column are replaced with the
+ country of maximum occurrence - Portugal (PRT). Fill missing country
+ column with ``PRT`` based on value counts
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/fill-missing-country-val.png
+
+
+Please hit **Preview** to look at the transform preview, and if it is
+OK, click **Add** to add the transform in the data flow.
+
+- Custom Transform - Meal type has Undefined category, changing the
+ Undefined value to the most used which is BB by implementing a custom
+ pyspark transform with two simple lines of code. This can be done by
+ choosing **Custom transform** transform from the **Add Transform**
+ window. Specify the name of the transform, and paste following code
+ in the transform as shown in figure below.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/custom-pyspark-code.png
+
+.. code:: python
+
+ from pyspark.sql.functions import when
+
+ df = df.withColumn('meal', when(df.meal == 'Undefined', 'BB').otherwise(df.meal))
+
+Please hit **Preview** to look at the transform preview, and if it is
+OK, click **Add** to add the transform in the data flow.
+
+### Numeric Normalization Normalization is a scaling technique in which
+values are shifted and rescaled so that they end up ranging between 0
+and 1. It is also known as Min-Max scaling. Standardization is another
+scaling technique where the values are centered around the mean with a
+unit standard deviation. This means that the mean of the attribute
+becomes zero and the resultant distribution has a unit standard
+deviation.
+
+For our example use case, let’s normalize the numeric feature columns to
+a standard scale [0,1].
+
+From Data Wrangler’s list of pre-built transforms, choose **Process
+numeric**. Please select the following parameters and hit **Preview**.
+
+- ``Transform``: ``Scale values``
+- ``Scalar`` : ``Min-max scalar``
+- ``Min``: ``0``
+- ``Max``: ``1``
+- ``Input columns``: ``lead_time``,\ ``stays_in_weekend_nights``,
+ ``stays_in_weekday_nights``, ``is_repeated_guest``,
+ ``prev_cancellations``, ``prev_bookings_not_canceled``,
+ ``booking_changes``, ``adr``, ``total_of_specical_requests``,
+ ``required_car_parking_spaces``
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/min-max.png
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+Handle Categorical Data
+-----------------------
+
+Categorical data is usually composed of a finite number of categories,
+where each category is represented with a string. Encoding categorical
+data is the process of creating a numerical representation for
+categories. With Data Wrangler, we can select Ordinal encode to encode
+categories into an integer between 0 and the total number of categories
+in the Input column you select. Select one-hot encoding or use
+similarity encoding when you have a large number of categorical
+variables and Noisy data.
+
+| From Data Wrangler’s list of pre-built transforms, choose **Encode
+ Categorical**. Please select the following parameters and hit
+ **Preview**.
+| - ``Transform``: ``One-hot encode`` - ``Invalid handling strategy`` :
+ ``Keep`` - ``Output style``: ``Columns`` - ``Max``: ``1``
+| - ``Input columns``: ``meal``, ``is_repeated_guest``,
+ ``market_segment``, ``assigned_room_type``, ``deposit_type``,
+ ``customer_type``
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/categorical-one-hot.png
+
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+Balancing the target variable
+-----------------------------
+
+DataWrangler also helps to balance the target variable (column) for
+class imbalance. Let’s presume the following for the negative and
+positive cases.
+
+::
+
+ is_canceled = 0 (negative case)
+ is_canceled = 1 (positive case)
+
+In Data Wrangler, we can handle class imbalance using 3 different
+techniques.
+
+::
+
+ - Random Undersample
+ - Random Oversample
+ - SMOTE
+
+| From the Data Wrangler’s transform pane, choose **Balance Data** as
+ the transform. Please select the following parameters as shown in
+ image below and hit **Preview**.
+| - ``Target column``: ``is_canceled`` - ``Desiered ratio`` : ``1`` -
+ ``Transform``: ``Random oversample``
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/random-oversample.png
+
+
+If the Preview is OK, click **Add** to add the transform in the data
+flow.
+
+The state of the classes before and after balancing is as follows:
+
+The ratio of positive to negative case = ~0.38
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/class-before-balance.png
+
+
+After balancing, the ratio is 1 |post-balance|
+
+Quick Model
+-----------
+
+Given, we have applied most of the needed transformations on our feature
+columns, we can now create a Quick Model again using the transformed
+features to identify the predictive ability of our features and take a
+look at their attribution towards prediction.
+
+It is a good practice to run a Quick Model everytime we make a set of
+feature transforms. Previously, we ran a Quick Model analysis using the
+raw features. The results of this previous run was mostly incorrect and
+misleading, given, we haven’t fixed most of the correlation and other
+issues with our dataset.
+
+The below figure shows the results of the newly run Quick Model created
+using the transformed features. As you can see, the Quick Model achieved
+an F1 score of 62% on the test data. The top 5 most contributing
+features towards this score are as follows which is different from what
+we see previously.
+
+::
+
+ lead_time
+ country
+ customer_type_Transient
+ required_car_parking_spaces
+ booking_changes
+
+Craete a quick model, similar to one we created in the `Exploratory Data
+Analysis <./Data-Exploration.md>`__ step.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/quick-model-post.png
+
+Compare the model with the one created in Exploratory Data Analysis
+step.
+
+Next Steps
+----------
+
+As a next step, we will export the transformed data for further use.
+Please follow steps for Data
+Export.
+
+.. |post-balance| image:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/class-after-balance.png
diff --git a/sagemaker-datawrangler/tabular-dataflow/index.rst b/sagemaker-datawrangler/tabular-dataflow/index.rst
new file mode 100644
index 0000000000..8fc5c1a3d3
--- /dev/null
+++ b/sagemaker-datawrangler/tabular-dataflow/index.rst
@@ -0,0 +1,361 @@
+
+Tabular Dataflow Example
+========================
+This example provide quick walkthrough of how to aggregate and prepare
+data for Machine Learning using Amazon SageMaker Data Wrangler for
+Tabular dataset.
+
+Hotel Booking Demand Example
+----------------------------
+
+Background
+----------
+
+Amazon SageMaker helps data scientists and developers to prepare, build,
+train, and deploy machine learning models quickly by bringing together a
+broad set of purpose-built capabilities. This example shows how
+SageMaker can accelerate machine learning development during the data
+preprocessing stage to help process the hotel demand data and find
+relevant features for the training model to predict hotel cancellations.
+
+Dataset
+~~~~~~~
+
+.. raw:: html
+
+
+
+Dataset
+
+We are using the `Hotel Booking Demand
+dataset `__
+that is publically available. This data set contains booking information
+for a city hotel and a resort hotel, and includes information such as
+when the booking was made, length of stay, the number of adults,
+children, and/or babies, and the number of available parking spaces,
+among other things.
+
+The data needs to be downloaded from the locations specified, and
+uploaded to S3 bucket before we start the Data Preprocessing phase.
+Please follow the Experiment Steps outlined in later sections, to
+download the data and notebooks.
+
+Description of the Columns
+--------------------------
+
++-----------------------------------+-----------------------------------+
+| Column Name | Description |
++===================================+===================================+
+| ``hotel`` | Type of the hotel (``H1`` = |
+| | Resort Hotel or ``H2`` = City |
+| | Hotel) |
++-----------------------------------+-----------------------------------+
+| ``is_canceled`` | Value indicating if the booking |
+| | was canceled (1) or not (0) |
++-----------------------------------+-----------------------------------+
+| ``lead_time`` | Number of days that elapsed |
+| | between the entering date of the |
+| | booking into the PMS and the |
+| | arrival date |
++-----------------------------------+-----------------------------------+
+| ``arrival_date_year`` | Year of arrival date |
++-----------------------------------+-----------------------------------+
+| ``arrival_date_month`` | Month of arrival date |
++-----------------------------------+-----------------------------------+
+| ``arrival_date_week_number`` | Week number of year for arrival |
+| | date |
++-----------------------------------+-----------------------------------+
+| ``arrival_date_day_of_month`` | Day of arrival date |
++-----------------------------------+-----------------------------------+
+| ``stays_in_weekend_nights`` | Number of weekend nights |
+| | (Saturday or Sunday) the guest |
+| | stayed or booked to stay at the |
+| | hotel |
++-----------------------------------+-----------------------------------+
+| ``stays_in_week_nights`` | Number of week nights (Monday to |
+| | Friday) the guest stayed or |
+| | booked to stay at the hotel |
++-----------------------------------+-----------------------------------+
+| ``adults`` | Number of adults |
++-----------------------------------+-----------------------------------+
+| ``children`` | Number of children |
++-----------------------------------+-----------------------------------+
+| ``babies`` | Number of babies |
++-----------------------------------+-----------------------------------+
+| ``meal`` | Type of meal booked. Categories |
+| | are presented in standard |
+| | hospitality meal packages: |
+| | ``Undefined/SC`` – no meal |
+| | package; ``BB`` – Bed & |
+| | Breakfast; ``HB`` – Half board |
+| | (breakfast and one other meal – |
+| | usually dinner); ``FB`` – Full |
+| | board (breakfast, lunch and |
+| | dinner) |
++-----------------------------------+-----------------------------------+
+| ``country`` | Country of origin. Categories are |
+| | represented in the |
+| | ``ISO 3155–3:2013`` format |
++-----------------------------------+-----------------------------------+
+| ``market_segment`` | Market segment designation. In |
+| | categories, the term ``TA`` means |
+| | “Travel Agents” and ``TO`` means |
+| | “Tour Operators” |
++-----------------------------------+-----------------------------------+
+| ``distribution_channel`` | Booking distribution channel. The |
+| | term ``TA`` means “Travel Agents” |
+| | and ``TO`` means “Tour Operators” |
++-----------------------------------+-----------------------------------+
+| ``is_repeated_guest`` | Value indicating if the booking |
+| | name was from a repeated guest |
+| | (1) or not (0) |
++-----------------------------------+-----------------------------------+
+| ``previous_cancellations`` | Number of previous bookings that |
+| | were cancelled by the customer |
+| | prior to the current booking |
++-----------------------------------+-----------------------------------+
+| ` | Number of previous bookings not |
+| `previous_bookings_not_canceled`` | cancelled by the customer prior |
+| | to the current booking |
++-----------------------------------+-----------------------------------+
+| ``reserved_room_type`` | Code of room type reserved. Code |
+| | is presented instead of |
+| | designation for anonymity |
+| | reasons. |
++-----------------------------------+-----------------------------------+
+| ``assigned_room_type`` | Code for the type of room |
+| | assigned to the booking. |
+| | Sometimes the assigned room type |
+| | differs from the reserved room |
+| | type due to hotel operation |
+| | reasons (e.g. overbooking) or by |
+| | customer request. Code is |
+| | presented instead of designation |
+| | for anonymity reasons. |
++-----------------------------------+-----------------------------------+
+| ``booking_changes`` | Number of changes/amendments made |
+| | to the booking from the moment |
+| | the booking was entered on the |
+| | PMS until the moment of check-in |
+| | or cancellation |
++-----------------------------------+-----------------------------------+
+| ``deposit_type`` | Indication on if the customer |
+| | made a deposit to guarantee the |
+| | booking. This variable can assume |
+| | three categories: No Deposit – no |
+| | deposit was made; ``Non Refund`` |
+| | – a deposit was made in the value |
+| | of the total stay cost; |
+| | ``Refundable`` – a deposit was |
+| | made with a value under the total |
+| | cost of stay. |
++-----------------------------------+-----------------------------------+
+| ``agent`` | ID of the travel agency that made |
+| | the booking |
++-----------------------------------+-----------------------------------+
+| ``company`` | ID of the company/entity that |
+| | made the booking or responsible |
+| | for paying the booking. ID is |
+| | presented instead of designation |
+| | for anonymity reasons |
++-----------------------------------+-----------------------------------+
+| ``days_in_waiting_list`` | Number of days the booking was in |
+| | the waiting list before it was |
+| | confirmed to the customer |
++-----------------------------------+-----------------------------------+
+| ``customer_type`` | Type of booking, assuming one of |
+| | four categories: ``Contract`` - |
+| | when the booking has an allotment |
+| | or other type of contract |
+| | associated to it; ``Group`` – |
+| | when the booking is associated to |
+| | a group; ``Transient`` – when the |
+| | booking is not part of a group or |
+| | contract, and is not associated |
+| | to other transient booking; |
+| | ``Transient-party`` – when the |
+| | booking is transient, but is |
+| | associated to at least other |
+| | transient booking |
++-----------------------------------+-----------------------------------+
+| ``adr`` | Average Daily Rate as defined by |
+| | dividing the sum of all lodging |
+| | transactions by the total number |
+| | of staying nights |
++-----------------------------------+-----------------------------------+
+| ``required_car_parking_spaces`` | Number of car parking spaces |
+| | required by the customer |
++-----------------------------------+-----------------------------------+
+| ``total_of_special_requests`` | Number of special requests made |
+| | by the customer (e.g. twin bed or |
+| | high floor) |
++-----------------------------------+-----------------------------------+
+| ``reservation_status`` | Reservation last status, assuming |
+| | one of three categories: |
+| | ``Canceled`` – booking was |
+| | canceled by the customer; |
+| | ``Check-Out`` – customer has |
+| | checked in but already departed; |
+| | ``No-Show`` – customer did not |
+| | check-in and did inform the hotel |
+| | of the reason why |
++-----------------------------------+-----------------------------------+
+| ``reservation_status_date`` | Date at which the last status was |
+| | set. This variable can be used in |
+| | conjunction with the |
+| | ReservationStatus to understand |
+| | when was the booking canceled or |
+| | when did the customer checked-out |
+| | of the hotel |
++-----------------------------------+-----------------------------------+
+
+--------------
+
+Pre-requisites:
+---------------
+
+- We need to ensure dataset (tracks and ratings dataset) for ML is
+ uploaded to a data source (instructions to download the dataset to
+ Amazon S3 is available in the following section).
+- Data source can be any one of the following options:
+
+ - S3
+ - Athena
+ - RedShift
+ - SnowFlake
+
+.. container:: alert alert-block alert-info
+
+ Data Source
+
+ For this experiment the Data Source will be `Amazon
+ S3 `__
+ from the specified location.
+2. Create a private S3 bucket to upload the dataset in. You can
+ reference the instructions for bucket creaiton [here]
+ https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html
+3. Upload the data in step 1 to the bucket created in step 2. Steps to
+ upload the data can be found [here]
+ https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html
+4. Note the S3 URL for the file uploaded in Step 3 before moving to the
+ next sextion. This data will be used as input to the Datawrangler.
+ The S3 URL will be used in the next step.
+
+Data Import from S3 to Data Wrangler
+------------------------------------
+
+
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Data Import from S3 to Data Wrangler
+
+ data-import/index
+
+
+Exploratory Data Analysis
+--------------------------
+
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Data Exploration from S3 to Data Wrangler
+
+ data-exploration/index
+
+Data Transformation
+-------------------
+
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Data transformations from S3 to Data Wrangler
+
+ data-transformations/index
+
+Data Export
+~~~~~~~~~~~
+
+Data Wrangler UI can also be used to export the transformed data to
+Amazon S3. To get started with this process, first let’s create a
+destination node. Right click on the final transform on your data and
+choose ``Add destination`` → ``Amazon S3``. Assign a name for your
+output data and choose the S3 location where you want the data to be
+stored and click Add destination button at the bottom as shown below.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/add-destination.png
+
+
+
+
+This adds a destination node to our data flow. The destination node acts
+as a sink to your data flow.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/destination.png
+
+
+
+Next, click on the Create job button in the upper right corner of the
+page. In the configuration page for the SageMaker Processing job we are
+about to create, choose the Instance type and Instance count for our
+processing cluster. Advance configuration is optional, where you can
+assign tags as needed and choose the appropriate Volume Size.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/configure-job.png
+
+
+
+
+Select ``Run`` to start the export job. The job created will have Job
+Name and Job ARN which can be used to search for the job status.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/create-job.png
+
+
+
+
+The job created in the previous step will be available in the monitoring
+page for ``SageMaker Processing job`` as shown in the figure below.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/processing-job.png
+
+
+
+
+After a certain time, the job will be complete. The image below shows
+the completed job. Exported data should be available in the output S3
+bucket.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/tabular-dataflow/complete-job.png
+
+
+
+This exported data can now be used for running the ML Models
+
+
+Import Dataflow
+----------------------------
+
+Here is the final `Flow file
+`__ available which you can directly import to expediate the process or validate the flow.
+
+Here are the steps to import the flow
+
+* Download the flow file
+
+* In Sagemaker Studio, drag and drop the flow file or use the upload button to browse the flow and upload
diff --git a/sagemaker-datawrangler/timeseries-dataflow/TS-Workshop-DataPreparation.flow b/sagemaker-datawrangler/timeseries-dataflow/TS-Workshop-DataPreparation.flow
new file mode 100644
index 0000000000..458de87e01
--- /dev/null
+++ b/sagemaker-datawrangler/timeseries-dataflow/TS-Workshop-DataPreparation.flow
@@ -0,0 +1,585 @@
+{
+ "metadata": {
+ "version": 1,
+ "disable_limits": false,
+ "instance_type": "ml.m5.4xlarge"
+ },
+ "parameters": [],
+ "nodes": [
+ {
+ "node_id": "8e286fbf-3ad5-4683-92fe-7c2e6f5c69b6",
+ "type": "SOURCE",
+ "operator": "sagemaker.s3_source_0.1",
+ "parameters": {
+ "dataset_definition": {
+ "__typename": "S3CreateDatasetDefinitionOutput",
+ "datasetSourceType": "S3",
+ "name": "trip_data",
+ "description": null,
+ "s3ExecutionContext": {
+ "__typename": "S3ExecutionContext",
+ "s3Uri": "s3://768746145684-us-east-1-dw-ts-lab/trip data/",
+ "s3ContentType": "parquet",
+ "s3HasHeader": true,
+ "s3FieldDelimiter": ",",
+ "s3DirIncludesNested": false,
+ "s3AddsFilenameColumn": false
+ }
+ }
+ },
+ "inputs": [],
+ "outputs": [
+ {
+ "name": "default",
+ "sampling": {
+ "sampling_method": "sample_by_limit",
+ "limit_rows": 50000
+ }
+ }
+ ]
+ },
+ {
+ "node_id": "bf119511-56d6-4b73-9bc2-d9026ee3a10d",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.infer_and_cast_type_0.1",
+ "parameters": {},
+ "trained_parameters": {
+ "schema": {
+ "store_and_fwd_flag": "string",
+ "VendorID": "long",
+ "tpep_pickup_datetime": "datetime",
+ "tpep_dropoff_datetime": "datetime",
+ "passenger_count": "long",
+ "trip_distance": "float",
+ "RatecodeID": "long",
+ "PULocationID": "long",
+ "DOLocationID": "long",
+ "payment_type": "long",
+ "fare_amount": "float",
+ "extra": "float",
+ "mta_tax": "float",
+ "tip_amount": "float",
+ "tolls_amount": "float",
+ "improvement_surcharge": "float",
+ "total_amount": "float",
+ "congestion_surcharge": "float",
+ "airport_fee": "float"
+ }
+ },
+ "inputs": [
+ {
+ "name": "default",
+ "node_id": "8e286fbf-3ad5-4683-92fe-7c2e6f5c69b6",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "ff490248-9beb-4bd9-8764-40409b93e59e",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.manage_columns_0.1",
+ "parameters": {
+ "operator": "Drop column",
+ "drop_column_parameters": {
+ "column_to_drop": [
+ "VendorID",
+ "RatecodeID",
+ "store_and_fwd_flag",
+ "DOLocationID",
+ "payment_type",
+ "fare_amount",
+ "extra",
+ "mta_tax",
+ "tolls_amount",
+ "improvement_surcharge",
+ "passenger_count",
+ "congestion_surcharge",
+ "airport_fee"
+ ]
+ }
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "bf119511-56d6-4b73-9bc2-d9026ee3a10d",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "27cabb74-7b37-4bc8-8611-e3f36e3bbcda",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Validate timestamps_parameters": {
+ "timestamp_column": "tpep_pickup_datetime",
+ "policy": "drop"
+ },
+ "operator": "Validate timestamps"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "ff490248-9beb-4bd9-8764-40409b93e59e",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "33b52060-f318-4a5d-8fe3-895d703a025b",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Validate timestamps_parameters": {
+ "timestamp_column": "tpep_dropoff_datetime",
+ "policy": "drop"
+ },
+ "operator": "Validate timestamps"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "27cabb74-7b37-4bc8-8611-e3f36e3bbcda",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "3fc0ab7a-94b8-46ad-be39-7b9786206261",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.custom_code_0.1",
+ "parameters": {
+ "operator": "Python (PySpark)",
+ "pyspark_parameters": {
+ "code": "# Table is available as variable `df`\nfrom pyspark.sql.functions import col, round\ndf = df.withColumn('duration', round((col(\"tpep_dropoff_datetime\").cast(\"long\")-col(\"tpep_pickup_datetime\").cast(\"long\"))/60,2))\ndf = df.drop(\"tpep_dropoff_datetime\")"
+ },
+ "name": "Duration_Transformation"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "33b52060-f318-4a5d-8fe3-895d703a025b",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "6f1de2ca-cf98-48c3-9bde-026fbf66399a",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.handle_missing_0.1",
+ "parameters": {
+ "operator": "Fill missing",
+ "fill_missing_parameters": {
+ "input_column": [
+ "PULocationID",
+ "tip_amount",
+ "total_amount"
+ ],
+ "fill_value": "0"
+ },
+ "impute_parameters": {
+ "column_type": "Numeric",
+ "numeric_parameters": {
+ "strategy": "Approximate Median"
+ }
+ }
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "3fc0ab7a-94b8-46ad-be39-7b9786206261",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "5eb611f2-2cd0-4a04-bdd3-221ead204bbc",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Handle missing_parameters": {
+ "hm_input_type_Along column_parameters": {
+ "hmac_output_column": "",
+ "hmac_id_column": "PULocationID",
+ "hmac_strategy": "Constant Value",
+ "hmac_leftovernans": "Fill with Forward/Backward feed",
+ "hmac_strategy_Constant Value_parameters": {
+ "hmac_custom_value": "0.0"
+ },
+ "hmac_sequence_column": "trip_distance",
+ "hmac_timestamp_column": "tpep_pickup_datetime"
+ },
+ "hm_input_type": "Along column"
+ },
+ "operator": "Handle missing"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "6f1de2ca-cf98-48c3-9bde-026fbf66399a",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "4d6ac70b-97d9-4688-930d-988d63238d07",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.custom_code_0.1",
+ "parameters": {
+ "operator": "Python (PySpark)",
+ "pyspark_parameters": {
+ "code": "# Table is available as variable `df`\ndf = df.filter(df.trip_distance >= 0)\ndf = df.filter(df.tip_amount >= 0)\ndf = df.filter(df.total_amount >= 0)\ndf = df.filter(df.duration >= 1)\ndf = df.filter((1 <= df.PULocationID) & (df.PULocationID <= 263))\ndf = df.filter((df.tpep_pickup_datetime >= \"2019-01-01 00:00:00\") & (df.tpep_pickup_datetime < \"2020-03-01 00:00:00\"))"
+ }
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "5eb611f2-2cd0-4a04-bdd3-221ead204bbc",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "22d3a3bc-1d23-451d-8b2c-8be6e0e9fca0",
+ "type": "VISUALIZATION",
+ "operator": "sagemaker.visualizations.describe_0.1",
+ "parameters": {
+ "name": "Cleaned Dataset Summary"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "4d6ac70b-97d9-4688-930d-988d63238d07",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "6681625c-0323-475a-8fac-443fe96da56a",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.handle_outliers_0.1",
+ "parameters": {
+ "operator": "Standard deviation numeric outliers",
+ "standard_deviation_numeric_outliers_parameters": {
+ "standard_deviations": 4,
+ "input_column": [
+ "trip_distance",
+ "tip_amount",
+ "total_amount",
+ "duration"
+ ],
+ "fix_method": "Remove"
+ }
+ },
+ "trained_parameters": {
+ "standard_deviation_numeric_outliers_parameters": [
+ {
+ "_hash": "10b2dff28011403357a5a7dd4a37e9bea91721a8",
+ "lower_threshold": -12.393631771918914,
+ "upper_threshold": 18.423848932384466,
+ "input_column": "trip_distance"
+ },
+ {
+ "_hash": "10b2dff28011403357a5a7dd4a37e9bea91721a8",
+ "lower_threshold": -8.711892476401351,
+ "upper_threshold": 12.165641974849178,
+ "input_column": "tip_amount"
+ },
+ {
+ "_hash": "10b2dff28011403357a5a7dd4a37e9bea91721a8",
+ "lower_threshold": -29.770794210538003,
+ "upper_threshold": 58.2552896798414,
+ "input_column": "total_amount"
+ },
+ {
+ "_hash": "10b2dff28011403357a5a7dd4a37e9bea91721a8",
+ "lower_threshold": -255.48697194000522,
+ "upper_threshold": 284.4440844902989,
+ "input_column": "duration"
+ }
+ ]
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "4d6ac70b-97d9-4688-930d-988d63238d07",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "31009b84-d667-427a-8649-a846f4370e99",
+ "type": "VISUALIZATION",
+ "operator": "sagemaker.visualizations.describe_0.1",
+ "parameters": {
+ "name": "Updated Table Summary"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "6681625c-0323-475a-8fac-443fe96da56a",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "a16da785-dc80-44c4-b426-fdd68972a317",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.custom_code_0.1",
+ "parameters": {
+ "operator": "Python (PySpark)",
+ "pyspark_parameters": {
+ "code": "# Table is available as variable `df`\nfrom pyspark.sql.functions import col, date_trunc\ndf = df.withColumn('pickup_time', date_trunc(\"hour\",col(\"tpep_pickup_datetime\")))\ndf = df.drop(\"tpep_pickup_datetime\")"
+ }
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "6681625c-0323-475a-8fac-443fe96da56a",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "2d851ed9-6e82-409c-a10f-f2869d53b9f9",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.custom_code_0.1",
+ "parameters": {
+ "operator": "Python (PySpark)",
+ "pyspark_parameters": {
+ "code": "# Table is available as variable `df`\nfrom pyspark.sql import functions as f\nfrom pyspark.sql import Window\ndf = df.withColumn('count', f.count('duration').over(Window.partitionBy([f.col(\"pickup_time\"), f.col(\"PULocationID\")])))"
+ }
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "a16da785-dc80-44c4-b426-fdd68972a317",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "ce8c8a4a-8161-4615-b222-a5d1cda8d674",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Resample_parameters": {
+ "frequency": {
+ "quantity": 1,
+ "offset_description": "Hourly"
+ },
+ "downsample": {
+ "non_numeric": "most common",
+ "numeric": "mean"
+ },
+ "upsample": {
+ "non_numeric": "ffill",
+ "numeric": "linear"
+ },
+ "timestamp_column": "pickup_time",
+ "id_column": "PULocationID"
+ },
+ "operator": "Resample"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "2d851ed9-6e82-409c-a10f-f2869d53b9f9",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "a7405baf-56f2-4ddc-8a7b-688f66e487da",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Featurize datetime_parameters": {
+ "output_column": "date",
+ "output_mode": "Ordinal",
+ "output_format": "Columns",
+ "infer_datetime_format": false,
+ "date_time_format": "",
+ "year": true,
+ "month": true,
+ "day": true,
+ "hour": true,
+ "minute": false,
+ "second": false,
+ "week_of_year": true,
+ "day_of_year": true,
+ "quarter": true,
+ "input_column": "pickup_time"
+ },
+ "operator": "Featurize datetime"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "ce8c8a4a-8161-4615-b222-a5d1cda8d674",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "51c5993e-56c5-43c6-8378-95c26037becd",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Lag features_parameters": {
+ "lag": 8,
+ "entire_window": true,
+ "drop_rows": false,
+ "entire_window_True_parameters": {
+ "flatten": true
+ },
+ "sequence_column": "count",
+ "id_column": "PULocationID",
+ "timestamp_column": "pickup_time"
+ },
+ "operator": "Lag features"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "a7405baf-56f2-4ddc-8a7b-688f66e487da",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "85f7158d-64d9-4cbe-b965-b18364eca536",
+ "type": "TRANSFORM",
+ "operator": "sagemaker.spark.time_series_0.1",
+ "parameters": {
+ "Rolling window features_parameters": {
+ "window_size": 8,
+ "flatten": true,
+ "strategy": "Minimal subset",
+ "sequence_column": "count",
+ "timestamp_column": "pickup_time",
+ "id_column": "PULocationID"
+ },
+ "operator": "Rolling window features"
+ },
+ "inputs": [
+ {
+ "name": "df",
+ "node_id": "51c5993e-56c5-43c6-8378-95c26037becd",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ },
+ {
+ "node_id": "f157eb91-0cd7-4c04-a023-e1be3f343b2e",
+ "type": "DESTINATION",
+ "operator": "sagemaker.spark.s3_destination_0.1",
+ "name": "S3: NYC_Export",
+ "parameters": {
+ "output_config": {
+ "compression": "none",
+ "output_path": "s3://768746145684-us-east-1-dw-ts-lab/",
+ "output_content_type": "CSV",
+ "delimiter": ","
+ }
+ },
+ "inputs": [
+ {
+ "name": "default",
+ "node_id": "85f7158d-64d9-4cbe-b965-b18364eca536",
+ "output_name": "default"
+ }
+ ],
+ "outputs": [
+ {
+ "name": "default"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
diff --git a/sagemaker-datawrangler/timeseries-dataflow/index.rst b/sagemaker-datawrangler/timeseries-dataflow/index.rst
new file mode 100644
index 0000000000..05cb0a9340
--- /dev/null
+++ b/sagemaker-datawrangler/timeseries-dataflow/index.rst
@@ -0,0 +1,1050 @@
+Timeseries Data Flow example
+===================================================
+This example provide quick walkthrough of how to aggregate and prepare data for Machine Learning using Amazon SageMaker Data Wrangler for Timeseries dataset.
+
+
+
+New York city(NYC) yellow cab time-series example
+-------------------------------------------------
+
+Our end goal for this lab is to prepare a time-series dataset and get it
+to a ready state for ML modeling. We will start with the New York city
+(NYC) yellow cab time-series dataset and work towards exploring,
+preparing and transforming the dataset to help us design a ML model that
+will predict the number of NYC yellow taxi pickups for any hour of the
+day and location. As part of the exercise, we will learn how to derive
+various insights about the trip like average tip value, average distance
+for the trip, etc.
+
+Data used in this demo: - Original data source for all open data from
+2008 to Current can be accessed here:
+https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page -
+AWS-hosted location:
+https://registry.opendata.aws/nyc-tlc-trip-records-pds/
+
+The taxi trip records include fields capturing pick-up and drop-off
+dates/times, pick-up and drop-off locations, trip distances, itemized
+fares, rate types, payment types, and driver-reported passenger counts.
+The raw data is split 1 month per file, per Yellow, Green, or ForHire
+from 2008 through 2020, with each file around 600 MB. The entire raw
+dataset is huge. For our lab, we will use 13 of these files of around a
+year’s worth of trip data and focus only on the iconic yellow cab trips.
+We picked trip data from Feb 2019 to Feb 2020 to avoid COVID effects.
+The data dictionary below describes the Yellow taxi trip data with raw
+feature column names and their respective descriptions. The picked
+dataset covers 13 months and encapsulates approximately 90 million
+trips.
+
+Instructions to download the dateset
+------------------------------------
+
+You can use SageMaker Data Wrangler to import data from the following
+data sources: Amazon Simple Storage Service (Amazon S3), Amazon Athena,
+Amazon Redshift, and Snowflake. The dataset that you import can include
+up to 1000 columns. For this lab, we will be using Amazon S3 as the
+preferred data source. Before we import the dataset into SageMaker Data
+Wrangler, let’s ensure we copy the dataset first from the publicly
+hosted location to our local S3 bucket in our own account.
+
+To copy the dataset, copy and execute the Python code below within
+SageMaker Studio. It is recommended to execute this code in a notebook
+setting. We also recommend to have your S3 bucket in the same region as
+SageMaker Data Wrangler.
+
+To create a SageMaker Studio notebook, from the launcher page, click on
+the **Notebook Python 3** options under **Notebooks and compute
+resources** as show in the figure below.
+
+Copy and paste the shared code snippet below into the launched
+notebook’s cell (shown below) and execute it by clicking on the play
+icon on the top bar.
+
+.. code:: python
+
+ import boto3
+ import json
+
+
+ # Setup
+ REGION = 'us-east-1'
+ account_id = boto3.client('sts').get_caller_identity().get('Account')
+ bucket_name = f'{account_id}-{REGION}-dw-ts-lab'
+
+
+ # Create S3 bucket to download dataset in your account
+ s3 = boto3.resource('s3')
+
+ if REGION == 'us-east-1':
+ s3.create_bucket(Bucket=bucket_name)
+ else:
+ s3.create_bucket(Bucket=bucket_name,CreateBucketConfiguration={'LocationConstraint': REGION})
+ # Copy dataset from public hosted location to your S3 bucket
+ trips = [
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-02.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-03.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-04.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-05.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-06.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-07.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-08.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-09.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-10.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-11.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-12.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2020-01.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2020-02.parquet'}
+ ]
+ for trip in trips:
+ s3.meta.client.copy(trip, bucket_name, trip['Key'])
+
+Now we have raw data in our S3 bucket and ready to explore it and build
+a training dataset
+
+Dataset Import
+--------------
+
+Our first step is to launch a new SageMaker Data Wrangler session and
+there are multiple ways how to do that. For example, use the following:
+
+Click File -> New -> Data Wrangler Flow
+
+Amazon SageMaker will start to provision a resources for you and you a
+could find a new Data Wrangler Flow file in a File Browser section
+
+Lets rename our new workflow: Right click on file -> Rename Data
+Wrangler Flow
+
+Put a new name, for example: ``TS-Workshop-DataPreparation.flow``
+
+In few minutes Data Wrangler will finish to provision resources and you
+could see “Import Data” screen. SageMaker Data Wrangler supports many
+data sources: Amazon S3, Amazon Athena, Amazon Redshift, Snowflake,
+Databricks. Our data already in S3, let’s import it by clicking “Amazon
+S3” button.
+
+You will see all your S3 buckets so please search for your bucket (if
+you used provided code the bucket will have a suffix dw-ts-lab)
+
+All the files required for this lab are in “trip data” folder, so let’s
+select it. SageMaker Data Wrangler will import all files from a folder
+and sample up to 100 MB of data for an interactive preview. On a right
+side menu you could customize import job settings like Name, File type,
+Delimiter, etc. More information about import process could be found
+`here `__.
+
+To finish setting up import step select “parquet” in “File type” drop
+down menu and press the orange button “Import”
+
+It will take a few minutes to import data and validate it. SageMaker
+Data Wrangler will automatically recognize data types. You should see
+“Validation complete 0 errors message”
+
+Change data types
+------------------
+
+First we will check the data types were correctly recognized. This might
+be necessary as Data Wrangler selects data types based on a sampled data
+which is limited to 50000 rows. Sampled data might potentially miss some
+variations.
+
+To add a data transformation step use the plus sign next to Data types
+and choose Edit data types as shown below.
+
+In our case several columns were incorrectly recognized: -
+``passenger_count`` (must be ``long`` instead of ``float``) -
+``RatecodeID`` (must be ``long`` instead of ``float``) - ``airport_fee``
+(must be ``float`` instead of ``long``)
+
+I know correct data types from dataset description. In real life you
+could also easily find such information. Let’s correct data types by
+selecting a correct type from a drop down menu.
+
+Click Preview and then Apply button.
+
+Click Back to data flow.
+
+Dataset preparation
+-------------------
+
+Drop columns
+------------
+
+Before we analyze data and do feature engineering we have to clean
+dataset and below steps show how to remove unwanted data.
+
+To re-iterate our business goal: Predict the number of NY City yellow
+taxi pickups in the next 24 hour for each pickup per hour zones and
+provide some insights for drivers like average tips, average distance,
+etc.
+
+As we are interested in per hour forecast we have to aggregate some
+features and remove features which are impossible to aggregate. For this
+purpose we don’t need the following columns:
+
+1. ``VendorID`` (A code indicating the TPEP provider that provided the
+ record)
+2. ``RatecodeID`` (The final rate code in effect at the end of the
+ trip)
+3. ``Store_and_fwd_flag`` (This flag indicates whether the trip record
+ was held in vehicle memory before sending to the vendor, aka “store
+ and forward,”because the vehicle did not have a connection to the
+ server)
+4. ``DOLocationID`` (TLC Taxi Zone in which the taximeter was
+ disengaged)
+5. ``Payment_type`` (A numeric code signifying how the passenger paid
+ for the trip)
+6. ``Fare_amount`` (The time-and-distance fare calculated by the meter)
+ - we will use total amount feature
+7. ``Extra`` (Miscellaneous extras and surcharges)
+8. ``MTA_tax`` (0.50 MTA tax that is automatically triggered based on
+ the metered rate in use)
+9. ``Tolls_amount`` (Total amount of all tolls paid in trip)
+10. ``Improvement_surcharge`` (improvement surcharge assessed trips at
+ the flag drop. The improvement surcharge began being levied in 2015)
+11. ``Passenger_count`` (This is a driver-entered value)
+12. ``congestion_surcharge`` (Total amount collected in trip for NYS
+ congestion surcharge)
+13. ``Airport_fee`` (Only at LaGuardia and John F. Kennedy Airports)
+
+To remove those columns: 1. Click the plus sign next to “Data types”
+element and choose Add transform.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/SelectAddTransform.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. | Choose Manage columns.
+
+4. For Transform, choose Drop column and for Column to drop, choose all
+ mentioned above.
+
+5. Choose Preview
+
+6. Choose Add to save the step.
+
+Once transformation is applied on a sampled data you should see all
+current steps and a preview of a resulted dataset like show here.
+
+Click Back to data flow.
+
+Handle missing and invalid data in timestamps
+---------------------------------------------
+
+Missing data is a common problem in real life, it could be a result of
+data corruption, data loss or issues in data ingestion. The best
+practice is to verify the presence of any missing or invalid values and
+handle them appropriately.
+
+There are many different strategies how missing or invalid data could be
+handled, for example dropping rows with missing values or filling the
+missing values with static or calculated values. Depending on dataset
+size you could choose what to do: fix values or just drop them. The
+**Time Series - Handle missing** transform allows you to choose from
+multiple strategies.
+
+All future aggregations will be based on time stamps, so we have to make
+sure that we don’t have any rows with missing time stamps (
+``tpep_pickup_datetime`` and ``tpep_dropoff_datetime`` features).
+SageMaker Data Wrangler has several time series specific
+transformations, including **Validate timestamps** which checks for
+scenarios: 1. Checking timestamp column for any missing values. 2.
+Validate the timestamp columns for the desired timestamp format.
+
+To validate timestamps in ``tpep_dropoff_datetime`` and
+``tpep_pickup_datetime`` columns: 1. Click the plus sign next to “Drop
+columns” element and choose Add transform.
+
+
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddDateValidationTransform.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Time Series.
+
+4. For Transform choose Validate Timestamps, For TimeStamp columns
+ choose ``tpep_pickup_datetime``, for Policy select drop.
+
+5. Choose Preview
+
+6. Choose Add to save the step.
+
+7. Repeat same steps again for ``tpep_dropoff_datetime`` column
+
+When you apply a transformation a sampled data you should see all
+current steps and a preview of a resulted dataset.
+
+Click Back to data flow.
+
+Feature engineering based on a timestamp with a custom transformation.
+----------------------------------------------------------------------
+
+At this stage we have pickup and drop-off timestamps, but we are more
+interested in pickup timestamp and ride duration. We have to create a
+new feature ``ride duration`` as a difference between pick up and drop
+off time in minutes. There is no built-in date difference transformation
+in a Data Wrangler, but we could create it with a custom transformation.
+The **Custom Transforms** allows you to use Pyspark, Pandas, or Pyspark
+(SQL) to define your own transformations. For all three options, you use
+the variable ``df`` to access a dataframe to which you want to apply the
+transform. You do not need to include a return statement.
+
+To create a custom transformation you have to: 1. Click the plus sign
+next to a collection of transformation elements and choose Add
+transform.
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Custom transform.
+
+4. Name the transformation as “Duration_Transformation” - (naming is
+ optional but good to have a structure)
+
+5. In drop down menu select Python (PySpark) and use code below. This
+ code will import functions, calculate difference between two
+ timestamps by converting them to unix format (real number) and round
+ result and drop tpep_dropoff_datetime column
+
+ .. code:: python
+
+ from pyspark.sql.functions import col, round
+ df = df.withColumn('duration', round((col("tpep_dropoff_datetime").cast("long")-col("tpep_pickup_datetime").cast("long"))/60,2))
+ df = df.drop("tpep_dropoff_datetime")
+
+6. Choose Preview
+
+7. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all
+current steps and a preview of a resulted dataset with a new column
+duration and without column tpep_dropoff_datetime
+
+Click Back to data flow.
+
+Handling missing data in numeric attributes
+-------------------------------------------
+
+We already discussed what are missing values and why it is important to
+handle them. So far, we have been working with timestamps only. Now, we
+are going to handle missing values in the rest of attributes. We can
+exclude ``duration`` feature from this operation as it was calculated
+from timestamps in the previous step. As we discussed before, there are
+several ways to handle missing data: fill a static number or calculate a
+value (for example: median or mean for last 7 days). It might make sense
+to calculate a value if your time-series represents a continuous process
+like sensor reading or product sale quantity. In our case, all trips are
+independent from each other and we cannot calculate values based on
+previous trips as it might bring data bias and increase error. We can
+replace missing values with zeros or sometimes it might make sense to
+drop the entire row with missing values.
+
+Amazon Data Wrangler has two types of transformations to handle missing
+data: i) generic and ii) specifically designed for time series data.
+Here, we demonstrate how to use both of them and describe when to use
+each of these transformations.
+
+Handle missing data with the generic “Handle missing values” transformations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This transformation can be used if you want to: 1. Replace missing
+values with a same static value for all time series 2. Replace missing
+values with a calculated value and you have only one time series (for
+example: one sensor or one product in a shop)
+
+To create this transformation you have to: 1. Click the plus sign next
+to a collection of transformation elements and choose Add transform.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddTransformMissingGeneral.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Handle Missing
+
+4. For “Transform” choose Fill missing
+
+5. For “inputs columns” choose ``PULocationID``, ``tip_amount``, and
+ ``total_amount``
+
+6. For “Fill value” put 0
+
+7. Choose Preview
+
+8. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all
+current steps and a preview of a resulted dataset.
+
+Handle missing data with special Time Series transformation
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In real life datasets, we have many time-series in the same dataset and
+to separate them, we use some form of IDs. For example, sensor ID or
+item SKU. If we want to replace missing values with calculated values,
+for example mean for last 10 sensor observations, we must calculate it
+based on data for each time series independently. Instead of writing
+code, you could use the special Time Series transformation in Data
+Wrangler and get this easily done!.
+
+To create this transformation you have to: 1. Click “+ Add step” orange
+button in the TRANSFORMS menu.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddStep.png
+
+
+2. Choose Time Series
+
+3. For “Transform” choose Handle missing
+
+4. For “Time series input type” choose Along column
+
+5. For “Impute missing values for this column” choose ``trip_distance``
+
+6. For “Timestamp column” choose tpep_pickup_datetime
+
+7. For “ID column” choose PULocationID
+
+8. For “Method for imputing values” choose Constant value
+
+9. For “Custom value” put 0.0
+
+10. Choose Preview
+
+11. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all
+current steps until this point in time and get a preview of the
+resulting dataset.
+
+Filter rows with invalid data
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Based on our understanding of the dataset until this point, we could
+also apply several filters to remove invalid or corrupt data from a
+business point of view. This will improve data quality even further and
+ensure we feed only correct data to our model training process.
+
+We can filter data based on following rules: 1. ``tpep_pickup_datetime``
+- have to be in range from 1 Jan 2019 (included) till 1 March 2020
+(excluded) 2. ``trip_distance`` - have to be greater than or equal to 0
+(only positive numbers) 3. ``tip_amount`` - have to be greater than or
+equal to 0 (only positive numbers) 4. ``total_amount`` - have to be
+greater than or equal to 0 (only positive numbers) 5. ``duration`` -
+have to be greater than or equal to 1 (we are not interested in super
+short trips). 6. ``PULocationID`` - have to be in the range (1 to 263).
+These are the assigned zones. For the sake of brevity, let’s use only
+the 1st ten location IDs for this workshop (see image below).
+
+There is no built-in filter transformation in Data Wrangler to handle
+these various constraints. Hence, we will create a custom
+transformation.
+
+To create a custom transformation, follow the steps below: 1. Click the
+plus sign next to a collection of transformation elements and choose Add
+transform.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddTransformFilter.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Custom Transform.
+
+4. In drop down menu select Python (PySpark) and use code below. This
+ code will filter rows based on the specified conditions.
+
+ .. code:: python
+
+ df = df.filter(df.trip_distance >= 0)
+ df = df.filter(df.tip_amount >= 0)
+ df = df.filter(df.total_amount >= 0)
+ df = df.filter(df.duration >= 1)
+ df = df.filter((1 <= df.PULocationID) & (df.PULocationID <= 263))
+ df = df.filter((df.tpep_pickup_datetime >= "2019-01-01 00:00:00") & (df.tpep_pickup_datetime < "2020-03-01 00:00:00"))
+
+5. Choose Preview
+
+6. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all
+current steps until this point in time and get a preview of the
+resulting dataset.
+
+Quick analysis of dataset
+-------------------------
+
+Amazon SageMaker Data Wrangler includes built-in analysis that help you
+generate visualizations and data insights in a few clicks. You can
+either leverage the built-in analyses types we offer out of the box with
+the product or create your own custom analysis using your own code if
+needed. SageMaker Data Wrangler also provides automated insights by
+automatically performing exploratory and descriptive analyses behind the
+scenes on your data. It identifies hidden anomalies and red flags within
+your dataset and proposes prescriptive actions in the form of what
+transforms can be applied on what columns of your data to fix these
+issues.
+
+For this lab, let’s use the Table Summary built-in analysis type to
+quickly summarize our existing dataset in its current form. For the
+numeric columns, including long and float data, table summary reports
+the number of entries (``count``), minimum (``min``), maximum (``max``),
+mean, and standard deviation (``stddev``) for each column. For columns
+with non-numerical data, including columns with String, Boolean, or
+DateTime data, table summary reports the number of entries (``count``),
+least frequent value (``min``), and most frequent value (``max``).
+
+To create this analysis, follow the steps below: 1. Click the plus sign
+next to a collection of transformation elements and choose “Add
+analyses”.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/addFirstAnalyses.png
+
+
+2. In a “analyses type” drop down menu select “Table Summary” and
+ provide a name for “Analysis name”, for example: “Cleaned dataset
+ summary”
+
+3. Choose Preview
+
+4. Choose Add to save the analyses.
+
+5. You could find your first analyses on a “Analysis” tab. All future
+ visualizations will could be also found here.
+
+6. Click on analyses icon to open it.
+
+Let’s take a look at our results. The most interesting part is the
+summary for duration column: maximum value is 1439 and this is in
+minutes! 1439 minutes = almost 24 hours and this is definitely an issue
+which will reduce the quality of our model if this dataset is used in
+its current form. This looks more like an issue due to the prevalence of
+outliers in our dataset. Next, let’s see how to issue this issue using a
+built-in transform Data Wrangler offers.
+
+Handling outliers in numeric attributes
+---------------------------------------
+
+In statistics, an outlier is a data point that differs significantly
+from other observations in the same dataset. An outlier may be due to
+variability in the measurement or it may indicate experimental error.
+The latter are sometimes excluded from the dataset. For example, in our
+dataset we have the ``tip_amount`` feature and usually it is less than
+10 dollars, but due to an error in a data collection, some values can
+show thousands of dollar as a tip. Such data errors will skew statistics
+and aggregated values which will lead to a lower model accuracy.
+
+An outlier can cause serious problems in statistical analysis. Machine
+learning models are sensitive to the distribution and range of feature
+values. Outliers, or rare values, can negatively impact model accuracy
+and lead to longer training times. When you define a Handle outliers
+transform step, the statistics used to detect outliers are generated on
+the data available in Data Wrangler when defining this step. These same
+statistics are used when running a Data Wrangler job.
+
+SageMaker Data Wrangler supports several outliers detection and handle
+methods. We are going to use **Standard Deviation Numeric Outliers** and
+we remove all outliers as our dataset is big enough. This transform
+detects and fixes outliers in numeric features using the mean and
+standard deviation. You specify the number of standard deviations a
+value must vary from the mean to be considered an outlier. For example,
+if you specify 3 for standard deviations, a value falling more than 3
+standard deviations from the mean is considered an outlier.
+
+To create this transformation, follow the steps below: 1. Click the plus
+sign next to a collection of transformation elements and choose “Add
+transform”.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddTransformOutliers.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Handle Outliers.
+
+4. For “Transform” choose “Standard deviation numeric outliers”
+
+5. For “Inputs columns” choose ``tip_amount``, ``total_amount``,
+ ``duration``, and ``trip_distance``
+
+6. For “Fix method” choose “Remove”
+
+7. For “Standard deviations” put 4
+
+8. Choose Preview
+
+9. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all
+current steps and a preview of resulted dataset.
+
+Optional: If you want, you could repeat the steps from our previous
+analysis (“Quick analysis of a current dataset”) to create a new table
+summary and check for the new maximum for the ``duration`` column. You
+can see, the new max value for duration is 243 minutes = just over an
+hour. This is more realistic for long trips than what we previously had.
+
+Grouping/Aggregating data
+-------------------------
+
+At this moment we have cleaned dataset by removing outliers, invalid
+values, and added new features. There are few more steps before we start
+training our forecasting model.
+
+As we are interested in a hourly forecast we have to count number of
+trips per hour per station and also aggregate (with mean) all metrics
+such as distance, duration, tip, total amount.
+
+Truncating timestamp
+~~~~~~~~~~~~~~~~~~~~
+
+We don’t need minutes and seconds in out timestamp, so we remove them.
+There is no built-in filter transformation in SageMaker Data Wrangler,
+so we create a custom transformation.
+
+To create a custom transformation, follow the steps below:: 1. Click the
+plus sign next to a collection of transformation elements and choose
+“Add transform”.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/addTrandformDate.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Custom Transform.
+
+4. In drop down menu select Python (PySpark) and use code below. This
+ code will create a new column with a truncated timestamp and then
+ drop original pickup column.
+
+ .. code:: python
+
+ from pyspark.sql.functions import col, date_trunc
+ df = df.withColumn('pickup_time', date_trunc("hour",col("tpep_pickup_datetime")))
+ df = df.drop("tpep_pickup_datetime")
+
+5. Choose Preview
+
+6. Choose Add to save the step
+
+When you apply the transformation on sampled data, you can see all the
+current steps until this point in time and get a preview of the
+resulting dataset with a new column ``pickup_time`` and without the old
+column ``tpep_pickup_datetime``
+
+Count number of trips per hour per station
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Currently, we have only piece of information about each trip, but we
+don’t know how many trips were made from each station per hour. The
+simplest way to do that is count number of records per stationID per
+hourly timestamp. While Amazon Data Wrangler provides GroupBy
+transformation. The built-in transformation doesn’t support grouping by
+multiple columns, so we use a custom transformation.
+
+To create a custom transformation you have to: 1. Click the plus sign
+next to a collection of transformation elements and choose “Add
+transform”.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/addTrandformDate.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Custom Transform.
+
+4. In drop down menu select Python (PySpark) and use code below. This
+ code will create a new column with a number of trips from each
+ location for each timestamp.
+
+ .. code:: python
+
+ from pyspark.sql import functions as f
+ from pyspark.sql import Window
+ df = df.withColumn('count', f.count('duration').over(Window.partitionBy([f.col("pickup_time"), f.col("PULocationID")])))
+
+5. Choose Preview
+
+6. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all
+current steps and a preview of a resulted dataset with a new column
+count.
+
+Resample time series
+--------------------
+
+Now, we are ready to make a final aggregation! We want to aggregate all
+rows by a combination of ``PULocationID`` and ``pickup_time`` columns,
+while features should be replaced by mean value for each combination.
+
+We use special built-in Time Series transformation **Resample**. The
+Resample transformation changes the frequency of the time series
+observations to a specified granularity. It also comes with both
+upsampling and downsampling options. Applying upsampling increases the
+frequency of the observations, for example from daily to hourly, whereas
+downsampling decreases the frequency of the observations, for example
+from hourly to daily.
+
+To create this transformation, follow the steps below: 1. Click the plus
+sign next to a collection of transformation elements and choose Add
+transform.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddResample.png
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Time Series.
+
+4. For “Transform” choose “Resample”
+
+5. For “Timestamp” choose ``pickup_time``
+
+6. For “ID column” choose ``PULocationID``
+
+7. For “Frequency unit” choose “Hourly”
+
+8. For “Frequency quantity” put 1
+
+9. For “Method to aggregate numeric values” choose “mean”
+
+10. Use default values for the rest of parameters
+
+11. Choose Preview
+
+12. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all
+current steps and a preview of a resulted dataset.
+
+.. _resample-time-series-1:
+
+Resample time series
+--------------------
+
+Now we are ready to make a final aggregation! We aggregate all rows by
+combination of ``PULocationID`` and ``pickup_time`` timestamp while
+features should be replaced by mean value for each combination.
+
+We use special built-in Time Series transformation **Resample**. The
+Resample transformation changes the frequency of the time series
+observations to a specified granularity. It also comes with both
+upsampling and downsampling options. Applying upsampling increases the
+frequency of the observations, for example from daily to hourly, whereas
+downsampling decreases the frequency of the observations, for example
+from hourly to daily.
+
+To create this transformation you have to: 1. Click the plus sign next
+to a collection of transformation elements and choose Add transform.
+
+.. figure:: https://s3.amazonaws.com/sagemaker-sample-files/images/sagemaker-datawrangler/timeseries-dataflow/AddResample.png
+
+
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Time Series.
+
+4. For “Transform” choose “Resample”
+
+5. For “Timestamp” choose pickup_time
+
+6. For “ID column” choose “PULocationID”
+
+7. For “Frequency unit” choose “Hourly”
+
+8. For “Frequency quantity” put 1
+
+9. For “Method to aggregate numeric values” choose “mean”
+
+10. Use default values for the rest of parameters
+
+11. Choose Preview
+
+12. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all
+current steps until this point in time and get a preview of the
+resulting dataset.
+
+`here `__.
+
+Featurize Date Time
+-------------------
+
+“Featurize datetime” time series transformation will add the month, day
+of the month, day of the year, week of the year, hour and quarter
+features to our dataset. Because we’re providing the date/time
+components as separate features, we enable ML algorithms to detect
+signals and patterns for improving prediction accuracy.
+
+To create this transformation you have to: 1. Click the plus sign next
+to a collection of transformation elements and choose Add transform
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu
+
+3. Choose Time Series
+
+ - For “Transform” choose “Featurize date/time”
+
+ - For “Input Column” choose ``pickup_time``
+
+ - For “Output Column” enter “date”
+
+ - For “Output mode” choose “Ordinal”
+
+ - For “Output format” choose “Columns”
+
+ - For date/time features to extract, select Year, Month, Day, Hour,
+ Week of year, Day of year, and Quarter.
+
+4. Choose Preview
+
+5. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all
+current steps until this point in time and get a preview of the
+resulting dataset.
+
+Click “Back to data flow” to head back to the block diagram editor
+window.
+
+Lag feature
+-----------
+
+Next let’s create lag features for the target column count. Lag features
+in time-series analysis are values at prior timestamps that are
+considered helpful in inferring future values. They also help identify
+**autocorrelation**, also known as serial correlation, patterns in the
+residual series by quantifying the relationship of the observation with
+observations at previous time steps. Autocorrelation is similar to
+regular correlation but between the values in a series and its past
+values. It forms the basis for the **autoregressive forecasting** models
+in the **ARIMA** series.
+
+With SageMaker Data Wrangler’s Lag feature transform, you can easily
+create lag features n periods apart. Additionally, we often want to
+create multiple lag features at different lags and let the model decide
+the most meaningful features. For such a scenario, the **Lag features**
+transform helps create multiple lag columns over a specified window
+size.
+
+To create this transformation, follow the steps below: 1. Click the plus
+sign next to a collection of transformation elements and choose Add
+transform.
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Time Series
+
+ - For “Transform” choose “Lag features”
+
+ - For “Generate lag features for this column” choose “count”
+
+ - For “ID column” enter “PULocationID”
+
+ - For “Timestamp Column” choose “pickup_time”
+
+ - For Lag, enter 8. You could try to use different values, maybe 24
+ hours in our case makes more sense.
+
+ - Because we’re interested in observing up to the previous 8 lag
+ values, let’s select Include the entire lag window.
+
+ - To create a new column for each lag value, select Flatten the
+ output
+
+4. Choose Preview
+
+5. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all
+current steps and a preview of a resulted dataset.
+
+Rolling window features
+-----------------------
+
+We can also calculate meaningful statistical summaries across a range of
+values and include them as input features. Let’s extract common
+statistical time series features.
+
+Data Wrangler implements automatic time series feature extraction
+capabilities using the open source ``tsfresh`` package. With the time
+series feature extraction transforms, you can automate the feature
+extraction process. This eliminates the time and effort otherwise spent
+manually implementing signal processing libraries. We will extract
+features using the **Rolling window** features transform. This method
+computes statistical properties across a set of observations defined by
+the window size.
+
+To create this transformation you have to: 1. Click the plus sign next
+to a collection of transformation elements and choose Add transform
+
+2. Click “+ Add step” orange button in the TRANSFORMS menu.
+
+3. Choose Time Series
+
+ - For “Transform” choose “Rolling window features”
+
+ - For “Generate rolling window features for this column” choose
+ “count”
+
+ - For “Timestamp Column” choose “pickup_time”
+
+ - For “ID column” enter ``PULocationID``
+
+ - For “Window size”, enter 8. You could try to use different values,
+ maybe 24 hours in our case makes more sense.
+
+ - Select Flatten to create a new column for each computed feature.
+
+ - Choose “Strategy” as “Minimal subset”. This strategy extracts
+ eight features that are useful in downstream analyses. Other
+ strategies include Efficient Subset, Custom subset, and All
+ features.
+
+4. Choose Preview
+
+5. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all
+current steps until this point in time and get a preview of the
+resulting dataset.
+
+Click “Back to data flow” to head back to the block diagram editor
+window.
+
+Export Data
+-----------
+
+At this stage, we have a new dataset that is cleaned and transformed
+with newly engineered features. This dataset can be used for forecasting
+either using open source libraries/frameworks or AWS services like
+`Amazon SageMaker
+Autopilot `__ available which you can directly import to expediate the process or validate the flow.
+
+Here are the steps to import the flow
+
+* Download the flow file
+
+* In Sagemaker Studio, drag and drop the flow file or use the upload button to browse the flow and upload
+
+
+Clean up
+-----------------------------------------------------
+
+- Delete artifacts in S3.
+- Delete data flow file in SageMaker Studio.
+- Stop active SageMaker Data Wrangler instance.
+- Delete SageMaker user profile and domain (optional).
diff --git a/sagemaker-datawrangler/timeseries-dataflow/readme.md b/sagemaker-datawrangler/timeseries-dataflow/readme.md
new file mode 100644
index 0000000000..21ad358746
--- /dev/null
+++ b/sagemaker-datawrangler/timeseries-dataflow/readme.md
@@ -0,0 +1,780 @@
+# Amazon SageMaker Data Wrangler time series workshop
+
+Our end goal for this lab is to prepare a time-series dataset and get it to a ready state for ML modeling. We will start with the New York city (NYC) yellow cab time-series dataset and work towards exploring, preparing and transforming the dataset to help us design a ML model that will predict the number of NYC yellow taxi pickups for any hour of the day and location. As part of the exercise, we will learn how to derive various insights about the trip like average tip value, average distance for the trip, etc.
+
+Data used in this demo:
+- Original data source for all open data from 2008 to Current can be accessed here: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page
+- AWS-hosted location: https://registry.opendata.aws/nyc-tlc-trip-records-pds/
+
+The taxi trip records include fields capturing pick-up and drop-off dates/times, pick-up and drop-off locations, trip distances, itemized fares, rate types, payment types, and driver-reported passenger counts. The raw data is split 1 month per file, per Yellow, Green, or ForHire from 2008 through 2020, with each file around 600 MB. The entire raw dataset is huge. For our lab, we will use 13 of these files of around a year's worth of trip data and focus only on the iconic yellow cab trips. We picked trip data from Feb 2019 to Feb 2020 to avoid COVID effects. The data dictionary below describes the Yellow taxi trip data with raw feature column names and their respective descriptions. The picked dataset covers 13 months and encapsulates approximately 90 million trips.
+
+
+
+
+## Instructions to download the dateset
+You can use SageMaker Data Wrangler to import data from the following data sources: Amazon Simple Storage Service (Amazon S3), Amazon Athena, Amazon Redshift, and Snowflake. The dataset that you import can include up to 1000 columns. For this lab, we will be using Amazon S3 as the preferred data source. Before we import the dataset into SageMaker Data Wrangler, let's ensure we copy the dataset first from the publicly hosted location to our local S3 bucket in our own account.
+
+To copy the dataset, copy and execute the Python code below within SageMaker Studio. It is recommended to execute this code in a notebook setting. We also recommend to have your S3 bucket in the same region as SageMaker Data Wrangler.
+
+To create a SageMaker Studio notebook, from the launcher page, click on the ***Notebook Python 3*** options under ***Notebooks and compute resources*** as show in the figure below.
+
+
+
+Copy and paste the shared code snippet below into the launched notebook's cell (shown below) and execute it by clicking on the play icon on the top bar.
+
+
+
+
+```python
+import boto3
+import json
+
+
+# Setup
+REGION = 'us-east-1'
+account_id = boto3.client('sts').get_caller_identity().get('Account')
+bucket_name = f'{account_id}-{REGION}-dw-ts-lab'
+
+
+# Create S3 bucket to download dataset in your account
+s3 = boto3.resource('s3')
+
+if REGION == 'us-east-1':
+ s3.create_bucket(Bucket=bucket_name)
+else:
+ s3.create_bucket(Bucket=bucket_name,CreateBucketConfiguration={'LocationConstraint': REGION})
+ # Copy dataset from public hosted location to your S3 bucket
+trips = [
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-02.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-03.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-04.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-05.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-06.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-07.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-08.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-09.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-10.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-11.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-12.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2020-01.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2020-02.parquet'}
+]
+for trip in trips:
+ s3.meta.client.copy(trip, bucket_name, trip['Key'])
+```
+
+Now we have raw data in our S3 bucket and ready to explore it and build a training dataset
+
+## Dataset Import
+
+Our first step is to launch a new SageMaker Data Wrangler session and there are multiple ways how to do that. For example, use the following:
+
+Click File -> New -> Data Wrangler Flow
+
+ 
+
+Amazon SageMaker will start to provision a resources for you and you a could find a new Data Wrangler Flow file in a File Browser section
+
+ 
+
+Lets rename our new workflow: Right click on file -> Rename Data Wrangler Flow
+
+
+
+Put a new name, for example: `TS-Workshop-DataPreparation.flow`
+
+
+
+In few minutes Data Wrangler will finish to provision resources and you could see "Import Data" screen.
+SageMaker Data Wrangler supports many data sources: Amazon S3, Amazon Athena, Amazon Redshift, Snowflake, Databricks.
+Our data already in S3, let's import it by clicking "Amazon S3" button.
+
+
+
+
+You will see all your S3 buckets so please search for your bucket (if you used provided code the bucket will have a suffix dw-ts-lab)
+
+
+
+All the files required for this lab are in "trip data" folder, so let's select it. SageMaker Data Wrangler will import all files from a folder and sample up to 100 MB of data for an interactive preview. On a right side menu you could customize import job settings like Name, File type, Delimiter, etc. More information about import process could be found [here](https://docs.aws.amazon.com/sagemaker/latest/dg/data-wrangler-import.html).
+
+
+To finish setting up import step select "parquet" in "File type" drop down menu and press the orange button "Import"
+
+
+
+
+It will take a few minutes to import data and validate it. SageMaker Data Wrangler will automatically recognize data types. You should see "Validation complete 0 errors message"
+
+
+
+
+# Change data types
+
+First we will check the data types were correctly recognized. This might be necessary as Data Wrangler selects data types based on a sampled data which is limited to 50000 rows. Sampled data might potentially miss some variations.
+
+
+To add a data transformation step use the plus sign next to Data types and choose Edit data types as shown below.
+
+
+
+
+In our case several columns were incorrectly recognized:
+- `passenger_count` (must be `long` instead of `float`)
+- `RatecodeID` (must be `long` instead of `float`)
+- `airport_fee` (must be `float` instead of `long`)
+
+I know correct data types from dataset description. In real life you could also easily find such information. Let's correct data types by selecting a correct type from a drop down menu.
+
+
+
+
+Click Preview and then Apply button.
+
+Click Back to data flow.
+
+# Dataset preparation
+## Drop columns
+
+Before we analyze data and do feature engineering we have to clean dataset and below steps show how to remove unwanted data.
+
+To re-iterate our business goal:
+Predict the number of NY City yellow taxi pickups in the next 24 hour for each pickup per hour zones and provide some insights for drivers like average tips, average distance, etc.
+
+As we are interested in per hour forecast we have to aggregate some features and remove features which are impossible to aggregate. For this purpose we don't need the following columns:
+
+1. `VendorID` (A code indicating the TPEP provider that provided the record)
+2. `RatecodeID` (The final rate code in effect at the end of the trip)
+3. `Store_and_fwd_flag` (This flag indicates whether the trip record was held in vehicle memory before sending to the vendor, aka “store and forward,”because the vehicle did not have a connection to the server)
+4. `DOLocationID` (TLC Taxi Zone in which the taximeter was disengaged)
+5. `Payment_type` (A numeric code signifying how the passenger paid for the trip)
+6. `Fare_amount` (The time-and-distance fare calculated by the meter) - we will use total amount feature
+7. `Extra` (Miscellaneous extras and surcharges)
+8. `MTA_tax` (0.50 MTA tax that is automatically triggered based on the metered rate in use)
+9. `Tolls_amount` (Total amount of all tolls paid in trip)
+10. `Improvement_surcharge` (improvement surcharge assessed trips at the flag drop. The improvement surcharge began being levied in 2015)
+11. `Passenger_count` (This is a driver-entered value)
+12. `congestion_surcharge` (Total amount collected in trip for NYS congestion surcharge)
+13. `Airport_fee` (Only at LaGuardia and John F. Kennedy Airports)
+
+To remove those columns:
+1. Click the plus sign next to "Data types" element and choose Add transform.
+
+ 
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+ 
+
+
+3. Choose Manage columns. \
+
+ 
+
+
+4. For Transform, choose Drop column and for Column to drop, choose all mentioned above.
+
+ 
+
+
+5. Choose Preview
+6. Choose Add to save the step.
+
+Once transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset like show here.
+
+ 
+
+Click Back to data flow.
+
+## Handle missing and invalid data in timestamps
+Missing data is a common problem in real life, it could be a result of data corruption, data loss or issues in data ingestion. The best practice is to verify the presence of any missing or invalid values and handle them appropriately.
+
+There are many different strategies how missing or invalid data could be handled, for example dropping rows with missing values or filling the missing values with static or calculated values. Depending on dataset size you could choose what to do: fix values or just drop them. The **Time Series - Handle missing** transform allows you to choose from multiple strategies.
+
+All future aggregations will be based on time stamps, so we have to make sure that we don't have any rows with missing time stamps ( `tpep_pickup_datetime` and `tpep_dropoff_datetime` features). SageMaker Data Wrangler has several time series specific transformations, including **Validate timestamps** which checks for scenarios:
+1. Checking timestamp column for any missing values.
+2. Validate the timestamp columns for the desired timestamp format.
+
+To validate timestamps in `tpep_dropoff_datetime` and `tpep_pickup_datetime` columns:
+1. Click the plus sign next to "Drop columns" element and choose Add transform.
+
+ 
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series.
+
+ 
+
+4. For Transform choose Validate Timestamps, For TimeStamp columns choose `tpep_pickup_datetime`, for Policy select drop.
+
+ 
+
+5. Choose Preview
+6. Choose Add to save the step.
+7. Repeat same steps again for `tpep_dropoff_datetime` column
+
+When you apply a transformation a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+Click Back to data flow.
+
+
+## Feature engineering based on a timestamp with a custom transformation.
+
+At this stage we have pickup and drop-off timestamps, but we are more interested in pickup timestamp and ride duration. We have to create a new feature `ride duration` as a difference between pick up and drop off time in minutes. There is no built-in date difference transformation in a Data Wrangler, but we could create it with a custom transformation. The **Custom Transforms** allows you to use Pyspark, Pandas, or Pyspark (SQL) to define your own transformations. For all three options, you use the variable `df` to access a dataframe to which you want to apply the transform. You do not need to include a return statement.
+
+To create a custom transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+ 
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom transform.
+
+ 
+
+4. Name the transformation as "Duration_Transformation" - (naming is optional but good to have a structure)
+5. In drop down menu select Python (PySpark) and use code below. This code will import functions, calculate difference between two timestamps by converting them to unix format (real number) and round result and drop tpep_dropoff_datetime column
+
+ ```Python
+ from pyspark.sql.functions import col, round
+ df = df.withColumn('duration', round((col("tpep_dropoff_datetime").cast("long")-col("tpep_pickup_datetime").cast("long"))/60,2))
+ df = df.drop("tpep_dropoff_datetime")
+ ```
+ 
+
+6. Choose Preview
+7. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset with a new column duration and without column tpep_dropoff_datetime
+
+ 
+
+Click Back to data flow.
+
+## Handling missing data in numeric attributes
+
+We already discussed what are missing values and why it is important to handle them. So far, we have been working with timestamps only. Now, we are going to handle missing values in the rest of attributes. We can exclude `duration` feature from this operation as it was calculated from timestamps in the previous step. As we discussed before, there are several ways to handle missing data: fill a static number or calculate a value (for example: median or mean for last 7 days). It might make sense to calculate a value if your time-series represents a continuous process like sensor reading or product sale quantity. In our case, all trips are independent from each other and we cannot calculate values based on previous trips as it might bring data bias and increase error. We can replace missing values with zeros or sometimes it might make sense to drop the entire row with missing values.
+
+Amazon Data Wrangler has two types of transformations to handle missing data: i) generic and ii) specifically designed for time series data. Here, we demonstrate how to use both of them and describe when to use each of these transformations.
+
+### Handle missing data with the generic "Handle missing values" transformations
+
+This transformation can be used if you want to:
+1. Replace missing values with a same static value for all time series
+2. Replace missing values with a calculated value and you have only one time series (for example: one sensor or one product in a shop)
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+ 
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Handle Missing
+
+ 
+
+4. For "Transform" choose Fill missing
+
+5. For "inputs columns" choose `PULocationID`, `tip_amount`, and `total_amount`
+
+6. For "Fill value" put 0
+
+ 
+
+7. Choose Preview
+8. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+### Handle missing data with special Time Series transformation
+
+In real life datasets, we have many time-series in the same dataset and to separate them, we use some form of IDs. For example, sensor ID or item SKU. If we want to replace missing values with calculated values, for example mean for last 10 sensor observations, we must calculate it based on data for each time series independently. Instead of writing code, you could use the special Time Series transformation in Data Wrangler and get this easily done!.
+
+To create this transformation you have to:
+1. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+
+2. Choose Time Series
+
+
+
+3. For "Transform" choose Handle missing
+
+4. For "Time series input type" choose Along column
+
+5. For "Impute missing values for this column" choose `trip_distance`
+
+6. For "Timestamp column" choose tpep_pickup_datetime
+
+7. For "ID column" choose PULocationID
+
+8. For "Method for imputing values" choose Constant value
+
+9. For "Custom value" put 0.0
+
+ 
+
+10. Choose Preview
+11. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+### Filter rows with invalid data
+
+Based on our understanding of the dataset until this point, we could also apply several filters to remove invalid or corrupt data from a business point of view. This will improve data quality even further and ensure we feed only correct data to our model training process.
+
+We can filter data based on following rules:
+1. `tpep_pickup_datetime` - have to be in range from 1 Jan 2019 (included) till 1 March 2020 (excluded)
+2. `trip_distance` - have to be greater than or equal to 0 (only positive numbers)
+3. `tip_amount` - have to be greater than or equal to 0 (only positive numbers)
+4. `total_amount` - have to be greater than or equal to 0 (only positive numbers)
+5. `duration` - have to be greater than or equal to 1 (we are not interested in super short trips).
+6. `PULocationID` - have to be in the range (1 to 263). These are the assigned zones. For the sake of brevity, let's use only the 1st ten location IDs for this workshop (see image below).
+
+
+
+There is no built-in filter transformation in Data Wrangler to handle these various constraints. Hence, we will create a custom transformation.
+
+To create a custom transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+ 
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom Transform.
+
+
+
+4. In drop down menu select Python (PySpark) and use code below. This code will filter rows based on the specified conditions.
+
+ ```Python
+ df = df.filter(df.trip_distance >= 0)
+ df = df.filter(df.tip_amount >= 0)
+ df = df.filter(df.total_amount >= 0)
+ df = df.filter(df.duration >= 1)
+ df = df.filter((1 <= df.PULocationID) & (df.PULocationID <= 263))
+ df = df.filter((df.tpep_pickup_datetime >= "2019-01-01 00:00:00") & (df.tpep_pickup_datetime < "2020-03-01 00:00:00"))
+ ```
+
+ 
+
+
+5. Choose Preview
+6. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+## Quick analysis of dataset
+
+Amazon SageMaker Data Wrangler includes built-in analysis that help you generate visualizations and data insights in a few clicks. You can either leverage the built-in analyses types we offer out of the box with the product or create your own custom analysis using your own code if needed. SageMaker Data Wrangler also provides automated insights by automatically performing exploratory and descriptive analyses behind the scenes on your data. It identifies hidden anomalies and red flags within your dataset and proposes prescriptive actions in the form of what transforms can be applied on what columns of your data to fix these issues.
+
+For this lab, let's use the Table Summary built-in analysis type to quickly summarize our existing dataset in its current form. For the numeric columns, including long and float data, table summary reports the number of entries (`count`), minimum (`min`), maximum (`max`), mean, and standard deviation (`stddev`) for each column. For columns with non-numerical data, including columns with String, Boolean, or DateTime data, table summary reports the number of entries (`count`), least frequent value (`min`), and most frequent value (`max`).
+
+To create this analysis, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose "Add analyses".
+
+ 
+
+2. In a "analyses type" drop down menu select "Table Summary" and provide a name for "Analysis name", for example: "Cleaned dataset summary"
+
+ 
+
+3. Choose Preview
+
+4. Choose Add to save the analyses.
+
+5. You could find your first analyses on a "Analysis" tab. All future visualizations will could be also found here.
+
+ 
+
+6. Click on analyses icon to open it.
+
+Let's take a look at our results. The most interesting part is the summary for duration column: maximum value is 1439 and this is in minutes! 1439 minutes = almost 24 hours and this is definitely an issue which will reduce the quality of our model if this dataset is used in its current form. This looks more like an issue due to the prevalence of outliers in our dataset. Next, let's see how to issue this issue using a built-in transform Data Wrangler offers.
+
+ 
+
+## Handling outliers in numeric attributes
+
+In statistics, an outlier is a data point that differs significantly from other observations in the same dataset. An outlier may be due to variability in the measurement or it may indicate experimental error. The latter are sometimes excluded from the dataset. For example, in our dataset we have the `tip_amount` feature and usually it is less than 10 dollars, but due to an error in a data collection, some values can show thousands of dollar as a tip. Such data errors will skew statistics and aggregated values which will lead to a lower model accuracy.
+
+An outlier can cause serious problems in statistical analysis. Machine learning models are sensitive to the distribution and range of feature values. Outliers, or rare values, can negatively impact model accuracy and lead to longer training times. When you define a Handle outliers transform step, the statistics used to detect outliers are generated on the data available in Data Wrangler when defining this step. These same statistics are used when running a Data Wrangler job.
+
+SageMaker Data Wrangler supports several outliers detection and handle methods. We are going to use **Standard Deviation Numeric Outliers** and we remove all outliers as our dataset is big enough. This transform detects and fixes outliers in numeric features using the mean and standard deviation. You specify the number of standard deviations a value must vary from the mean to be considered an outlier. For example, if you specify 3 for standard deviations, a value falling more than 3 standard deviations from the mean is considered an outlier.
+
+To create this transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose "Add transform".
+
+ 
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Handle Outliers.
+
+ 
+
+4. For "Transform" choose "Standard deviation numeric outliers"
+5. For "Inputs columns" choose `tip_amount`, `total_amount`, `duration`, and `trip_distance`
+6. For "Fix method" choose "Remove"
+7. For "Standard deviations" put 4
+
+ 
+
+8. Choose Preview
+9. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of resulted dataset.
+
+
+
+
+Optional: If you want, you could repeat the steps from our previous analysis ("Quick analysis of a current dataset") to create a new table summary and check for the new maximum for the `duration` column. You can see, the new max value for duration is 243 minutes = just over an hour. This is more realistic for long trips than what we previously had.
+
+
+
+## Grouping/Aggregating data
+At this moment we have cleaned dataset by removing outliers, invalid values, and added new features. There are few more steps before we start training our forecasting model.
+
+As we are interested in a hourly forecast we have to count number of trips per hour per station and also aggregate (with mean) all metrics such as distance, duration, tip, total amount.
+
+### Truncating timestamp
+We don't need minutes and seconds in out timestamp, so we remove them. There is no built-in filter transformation in SageMaker Data Wrangler, so we create a custom transformation.
+
+To create a custom transformation, follow the steps below::
+1. Click the plus sign next to a collection of transformation elements and choose "Add transform".
+
+ 
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom Transform.
+
+
+
+4. In drop down menu select Python (PySpark) and use code below. This code will create a new column with a truncated timestamp and then drop original pickup column.
+
+ ```Python
+ from pyspark.sql.functions import col, date_trunc
+ df = df.withColumn('pickup_time', date_trunc("hour",col("tpep_pickup_datetime")))
+ df = df.drop("tpep_pickup_datetime")
+ ```
+
+ 
+
+5. Choose Preview
+
+6. Choose Add to save the step
+
+When you apply the transformation on sampled data, you can see all the current steps until this point in time and get a preview of the resulting dataset with a new column `pickup_time` and without the old column `tpep_pickup_datetime`
+
+
+
+### Count number of trips per hour per station
+Currently, we have only piece of information about each trip, but we don't know how many trips were made from each station per hour. The simplest way to do that is count number of records per stationID per hourly timestamp. While Amazon Data Wrangler provides GroupBy transformation. The built-in transformation doesn't support grouping by multiple columns, so we use a custom transformation.
+
+To create a custom transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose "Add transform".
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom Transform.
+
+
+
+4. In drop down menu select Python (PySpark) and use code below. This code will create a new column with a number of trips from each location for each timestamp.
+
+ ```Python
+ from pyspark.sql import functions as f
+ from pyspark.sql import Window
+ df = df.withColumn('count', f.count('duration').over(Window.partitionBy([f.col("pickup_time"), f.col("PULocationID")])))
+ ```
+
+ 
+
+5. Choose Preview
+6. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset with a new column count.
+
+
+
+## Resample time series
+Now, we are ready to make a final aggregation! We want to aggregate all rows by a combination of `PULocationID` and `pickup_time` columns, while features should be replaced by mean value for each combination.
+
+We use special built-in Time Series transformation **Resample**. The Resample transformation changes the frequency of the time series observations to a specified granularity. It also comes with both upsampling and downsampling options. Applying upsampling increases the frequency of the observations, for example from daily to hourly, whereas downsampling decreases the frequency of the observations, for example from hourly to daily.
+
+To create this transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+ 
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+
+3. Choose Time Series.
+
+
+
+4. For "Transform" choose "Resample"
+5. For "Timestamp" choose `pickup_time`
+6. For "ID column" choose `PULocationID`
+7. For "Frequency unit" choose "Hourly"
+8. For "Frequency quantity" put 1
+9. For "Method to aggregate numeric values" choose "mean"
+10. Use default values for the rest of parameters
+
+ 
+
+11. Choose Preview
+12. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+## Resample time series
+Now we are ready to make a final aggregation! We aggregate all rows by combination of `PULocationID` and `pickup_time` timestamp while features should be replaced by mean value for each combination.
+
+We use special built-in Time Series transformation **Resample**. The Resample transformation changes the frequency of the time series observations to a specified granularity. It also comes with both upsampling and downsampling options. Applying upsampling increases the frequency of the observations, for example from daily to hourly, whereas downsampling decreases the frequency of the observations, for example from hourly to daily.
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series.
+
+
+
+4. For "Transform" choose "Resample"
+5. For "Timestamp" choose pickup_time
+6. For "ID column" choose "PULocationID"
+7. For "Frequency unit" choose "Hourly"
+8. For "Frequency quantity" put 1
+9. For "Method to aggregate numeric values" choose "mean"
+10. Use default values for the rest of parameters
+
+
+
+11. Choose Preview
+12. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+[here](https://docs.aws.amazon.com/sagemaker/latest/dg/data-wrangler-import.html).
+
+## Featurize Date Time
+
+"Featurize datetime" time series transformation will add the month, day of the month, day of the year, week of the year, hour and quarter features to our dataset. Because we’re providing the date/time components as separate features, we enable ML algorithms to detect signals and patterns for improving prediction accuracy.
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu
+
+
+
+3. Choose Time Series
+
+
+
+ * For "Transform" choose "Featurize date/time"
+ * For "Input Column" choose `pickup_time`
+ * For "Output Column" enter "date"
+ * For "Output mode" choose "Ordinal"
+ * For "Output format" choose "Columns"
+ * For date/time features to extract, select Year, Month, Day, Hour, Week of year, Day of year, and Quarter.
+
+ 
+
+4. Choose Preview
+5. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+Click "Back to data flow" to head back to the block diagram editor window.
+
+## Lag feature
+Next let’s create lag features for the target column count. Lag features in time-series analysis are values at prior timestamps that are considered helpful in inferring future values. They also help identify **autocorrelation**, also known as serial correlation, patterns in the residual series by quantifying the relationship of the observation with observations at previous time steps. Autocorrelation is similar to regular correlation but between the values in a series and its past values. It forms the basis for the **autoregressive forecasting** models in the **ARIMA** series.
+
+With SageMaker Data Wrangler's Lag feature transform, you can easily create lag features n periods apart. Additionally, we often want to create multiple lag features at different lags and let the model decide the most meaningful features. For such a scenario, the **Lag features** transform helps create multiple lag columns over a specified window size.
+
+To create this transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series
+
+
+
+ * For "Transform" choose "Lag features"
+ * For "Generate lag features for this column" choose "count"
+ * For "ID column" enter "PULocationID"
+ * For "Timestamp Column" choose "pickup_time"
+ * For Lag, enter 8. You could try to use different values, maybe 24 hours in our case makes more sense.
+ * Because we’re interested in observing up to the previous 8 lag values, let’s select Include the entire lag window.
+ * To create a new column for each lag value, select Flatten the output
+
+ 
+
+4. Choose Preview
+5. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset.
+
+ 
+
+## Rolling window features
+We can also calculate meaningful statistical summaries across a range of values and include them as input features. Let’s extract common statistical time series features.
+
+Data Wrangler implements automatic time series feature extraction capabilities using the open source `tsfresh` package. With the time series feature extraction transforms, you can automate the feature extraction process. This eliminates the time and effort otherwise spent manually implementing signal processing libraries. We will extract features using the **Rolling window** features transform. This method computes statistical properties across a set of observations defined by the window size.
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series
+
+
+
+ * For "Transform" choose "Rolling window features"
+ * For "Generate rolling window features for this column" choose "count"
+ * For "Timestamp Column" choose "pickup_time"
+ * For "ID column" enter `PULocationID`
+ * For "Window size", enter 8. You could try to use different values, maybe 24 hours in our case makes more sense.
+ * Select Flatten to create a new column for each computed feature.
+ * Choose "Strategy" as "Minimal subset". This strategy extracts eight features that are useful in downstream analyses. Other strategies include Efficient Subset, Custom subset, and All features.
+
+ 
+
+4. Choose Preview
+5. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+Click "Back to data flow" to head back to the block diagram editor window.
+
+## Export Data
+At this stage, we have a new dataset that is cleaned and transformed with newly engineered features. This dataset can be used for forecasting either using open source libraries/frameworks or AWS services like [Amazon SageMaker Autopilot](https://aws.amazon.com/sagemaker/autopilot/), [Amazon SageMaker Canvas](https://aws.amazon.com/sagemaker/canvas/) or [Amazon Forecast](https://aws.amazon.com/forecast/).
+
+Given, we had only used a sample of the dataset for creating our data preparation and transformation recipe so far, what need to do next is to apply the same recipe (data flow) on our entire dataset and scale the whole process in a distributed fashion. Amazon Data Wrangler let's you do this in multiple ways. You can export your data flow: 1/ as a processing job, 2/ as a SageMaker pipeline step, or 3/ as a Python script. You can also kick-off these distributed jobs via the UI without writing any code using Data Wrangler's destination node option. The export options are also facilitated via SageMaker Studio notebooks (Jupyter). Additionally, the transformed features can also be ingested directly to SageMaker Feature Store.
+
+For this lab, we will see how to use the destination nodes option to export the transformed features to S3 via a distributed PySpark job powered by SageMaker Processing.
+
+### Exporting to S3 using Destination Nodes
+This option creates a SageMaker processing job which uses the data flow (recipe) we have created previously to kick-off a distributed processing job on the "entire" dataset saving the results to a specified S3 bucket.
+
+Additionally, you can also drop columns if needed right before the export step. For the sake of brevity and to simplify the prediction problem statement, let's drop all columns except three columns `pickup_time`, `count`, `PULocationID`. Here count is the target variable we will try to predict. `pickup_time` and `PULocationID` will be our feature columns used for modeling. To create the model, we will be using SageMaker Autopilot. This will be covered in the next 2 sections.
+
+Follow the next steps to setup export to S3.
+1. Click the plus sign next to a collection of transformation elements and choose **"Add destination" -> "Amazon S3"**
+
+ 
+
+2. Provide parameters for S3 destination:
+ * Dataset name - name for new dataset, for example used "NYC_export"
+ * File type - CSV
+ * Delimiter - Comma
+ * Compression - none
+ * Amazon S3 location - You can use the same bucket name which we created at the beginning
+
+3. Click "Add destination" orange button
+
+ 
+
+4. Now your dataflow has a final step and you see a new "Create job" orange button. Click it.
+
+ 
+
+5. Provide a "Job name" or keep autogenerated option and select "destination". We have only one "S3:NYC_export", but you might have multiple destinations from different steps in your workflow. Leave a "KMS key ARN" field empty and click "Next" orange button.
+
+ 
+
+6. Now your have to provide configuration for a compute capacity for a job. You can keep all defaults values:
+ * For Instance type use "ml.m5.4xlarge"
+ * For Instance count use "2"
+ * You can explore "Additional configuration", but keep them without change.
+ * Click "Run" orange button
+
+ 
+
+7. Now your job is started and it takes about 1 hour to process 6 GB of data according to our Data Wrangler processing flow. Cost for this job will be around 2 USD as "ml.m5.4xlarge" cost 0.922 USD per hour and we are using two of them.
+
+ 
+
+8. If you click on the job name you will be redirected to a new window with the job details. On the job details page you see all parameters from a previous steps.
+
+ 
+
+Approximately in one hour you should see that job status changed to "Completed" and you could also check "Processing time (seconds)" value.
+
+
+
+Now you could close job details page.
+
+## Check Processed output
+After the SageMaker Data Wrangler processing job is completed, we can check the results saved in our destination S3 bucket.
+
+At this stage, you have designed a data flow for data processing and feature engineering and successfully launched it. Of course it is not mandatory to always run a job by clicking on the "Run" button. You could also automate it, but this is a topic of another workshop in this series!
+
+')['Role']['Arn']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if role is None:\n",
+ " role = sagemaker.get_execution_role()\n",
+ "\n",
+ "print(role)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, upload the data into the default bucket. You can select our own data set for the `input_data_uri` as is appropriate."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!mkdir -p data"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Pull the dataset from SageMaker's public S3 bucket and upload it to your own S3 bucket\n",
+ "\n",
+ "local_path = \"data/abalone-dataset.csv\"\n",
+ "\n",
+ "s3 = boto3.resource(\"s3\")\n",
+ "s3.Bucket(f\"sagemaker-sample-files\").download_file(\n",
+ " \"datasets/tabular/uci_abalone/abalone.csv\", local_path\n",
+ ")\n",
+ "\n",
+ "base_uri = f\"s3://{default_bucket}/{prefix}/abalone-data-set\"\n",
+ "input_data_uri = sagemaker.s3.S3Uploader.upload(\n",
+ " local_path=local_path,\n",
+ " desired_s3_uri=base_uri,\n",
+ ")\n",
+ "print(input_data_uri)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.parameters import ParameterString, ParameterFloat\n",
+ "\n",
+ "processing_instance_count = 1\n",
+ "training_instance_count = 1\n",
+ "transform_instance_count = 1\n",
+ "instance_type = \"ml.m5.xlarge\"\n",
+ "\n",
+ "input_data = ParameterString(\n",
+ " name=\"InputData\",\n",
+ " default_value=input_data_uri,\n",
+ ")\n",
+ "\n",
+ "mse_threshold = ParameterFloat(name=\"MseThreshold\", default_value=7.0)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a Processing Step for Feature Engineering\n",
+ "\n",
+ "First, develop a preprocessing script that is specified in the Processing step.\n",
+ "\n",
+ "This notebook cell writes a file `preprocessing_abalone.py`, which contains the preprocessing script. You can update the script, and rerun this cell to overwrite. The preprocessing script uses `scikit-learn` to do the following:\n",
+ "\n",
+ "* Fill in missing sex category data and encode it so that it is suitable for training.\n",
+ "* Scale and normalize all numerical fields, aside from sex and rings numerical data.\n",
+ "* Split the data into training, validation, and test datasets.\n",
+ "\n",
+ "The Processing step executes the script on the input data. The Training step uses the preprocessed training features and labels to train a model. The Evaluation step uses the trained model and preprocessed test features and labels to evaluate the model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!mkdir -p code"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%writefile code/preprocessing.py\n",
+ "import argparse\n",
+ "import os\n",
+ "import requests\n",
+ "import tempfile\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "\n",
+ "from sklearn.compose import ColumnTransformer\n",
+ "from sklearn.impute import SimpleImputer\n",
+ "from sklearn.pipeline import Pipeline\n",
+ "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
+ "\n",
+ "\n",
+ "# Since we get a headerless CSV file, we specify the column names here.\n",
+ "feature_columns_names = [\n",
+ " \"sex\",\n",
+ " \"length\",\n",
+ " \"diameter\",\n",
+ " \"height\",\n",
+ " \"whole_weight\",\n",
+ " \"shucked_weight\",\n",
+ " \"viscera_weight\",\n",
+ " \"shell_weight\",\n",
+ "]\n",
+ "label_column = \"rings\"\n",
+ "\n",
+ "feature_columns_dtype = {\n",
+ " \"sex\": str,\n",
+ " \"length\": np.float64,\n",
+ " \"diameter\": np.float64,\n",
+ " \"height\": np.float64,\n",
+ " \"whole_weight\": np.float64,\n",
+ " \"shucked_weight\": np.float64,\n",
+ " \"viscera_weight\": np.float64,\n",
+ " \"shell_weight\": np.float64,\n",
+ "}\n",
+ "label_column_dtype = {\"rings\": np.float64}\n",
+ "\n",
+ "\n",
+ "def merge_two_dicts(x, y):\n",
+ " z = x.copy()\n",
+ " z.update(y)\n",
+ " return z\n",
+ "\n",
+ "\n",
+ "if __name__ == \"__main__\":\n",
+ " base_dir = \"/opt/ml/processing\"\n",
+ "\n",
+ " df = pd.read_csv(\n",
+ " f\"{base_dir}/input/abalone-dataset.csv\",\n",
+ " header=None,\n",
+ " names=feature_columns_names + [label_column],\n",
+ " dtype=merge_two_dicts(feature_columns_dtype, label_column_dtype),\n",
+ " )\n",
+ " numeric_features = list(feature_columns_names)\n",
+ " numeric_features.remove(\"sex\")\n",
+ " numeric_transformer = Pipeline(\n",
+ " steps=[\n",
+ " (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
+ " (\"scaler\", StandardScaler()),\n",
+ " ]\n",
+ " )\n",
+ "\n",
+ " categorical_features = [\"sex\"]\n",
+ " categorical_transformer = Pipeline(\n",
+ " steps=[\n",
+ " (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"missing\")),\n",
+ " (\"onehot\", OneHotEncoder(handle_unknown=\"ignore\")),\n",
+ " ]\n",
+ " )\n",
+ "\n",
+ " preprocess = ColumnTransformer(\n",
+ " transformers=[\n",
+ " (\"num\", numeric_transformer, numeric_features),\n",
+ " (\"cat\", categorical_transformer, categorical_features),\n",
+ " ]\n",
+ " )\n",
+ "\n",
+ " y = df.pop(\"rings\")\n",
+ " X_pre = preprocess.fit_transform(df)\n",
+ " y_pre = y.to_numpy().reshape(len(y), 1)\n",
+ "\n",
+ " X = np.concatenate((y_pre, X_pre), axis=1)\n",
+ "\n",
+ " np.random.shuffle(X)\n",
+ " train, validation, test = np.split(X, [int(0.7 * len(X)), int(0.85 * len(X))])\n",
+ "\n",
+ " pd.DataFrame(train).to_csv(f\"{base_dir}/train/train.csv\", header=False, index=False)\n",
+ " pd.DataFrame(validation).to_csv(\n",
+ " f\"{base_dir}/validation/validation.csv\", header=False, index=False\n",
+ " )\n",
+ " pd.DataFrame(test).to_csv(f\"{base_dir}/test/test.csv\", header=False, index=False)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next, create an instance of a `SKLearnProcessor` processor and use that in our `ProcessingStep`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.sklearn.processing import SKLearnProcessor\n",
+ "\n",
+ "framework_version = \"1.0-1\"\n",
+ "\n",
+ "sklearn_processor = SKLearnProcessor(\n",
+ " framework_version=framework_version,\n",
+ " instance_type=instance_type,\n",
+ " instance_count=processing_instance_count,\n",
+ " base_job_name=\"sklearn-abalone-process\",\n",
+ " role=role,\n",
+ " sagemaker_session=local_pipeline_session,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Finally, we take the output of the processor's `run` method and pass that as arguments to the `ProcessingStep`. By passing the `local_pipeline_session` to the `sagemaker_session`, calling `.run()` does not launch the processing job, it returns the arguments needed to run the job as a step in the pipeline.\n",
+ "\n",
+ "Note the `\"train_data\"` and `\"test_data\"` named channels specified in the output configuration for the processing job. Step `Properties` can be used in subsequent steps and resolve to their runtime values at execution. Specifically, this usage is called out when you define the training step."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.processing import ProcessingInput, ProcessingOutput\n",
+ "from sagemaker.workflow.steps import ProcessingStep\n",
+ "\n",
+ "processor_args = sklearn_processor.run(\n",
+ " inputs=[\n",
+ " ProcessingInput(source=input_data, destination=\"/opt/ml/processing/input\"),\n",
+ " ],\n",
+ " outputs=[\n",
+ " ProcessingOutput(output_name=\"train\", source=\"/opt/ml/processing/train\"),\n",
+ " ProcessingOutput(output_name=\"validation\", source=\"/opt/ml/processing/validation\"),\n",
+ " ProcessingOutput(output_name=\"test\", source=\"/opt/ml/processing/test\"),\n",
+ " ],\n",
+ " code=\"code/preprocessing.py\",\n",
+ ")\n",
+ "\n",
+ "step_process = ProcessingStep(name=\"AbaloneProcess\", step_args=processor_args)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%writefile code/abalone.py\n",
+ "\n",
+ "import argparse\n",
+ "import json\n",
+ "import logging\n",
+ "import os\n",
+ "import pathlib\n",
+ "import pickle as pkl\n",
+ "import tarfile\n",
+ "\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import xgboost as xgb\n",
+ "\n",
+ "logging.basicConfig(level=logging.INFO)\n",
+ "\n",
+ "TRAIN_VALIDATION_FRACTION = 0.2\n",
+ "RANDOM_STATE_SAMPLING = 200\n",
+ "\n",
+ "logging.basicConfig(level=logging.INFO)\n",
+ "\n",
+ "\n",
+ "def prepare_data(train_dir, validation_dir):\n",
+ " \"\"\"Read data from train and validation channel, and return predicting features and target variables.\n",
+ "\n",
+ " Args:\n",
+ " data_dir (str): directory which saves the training data.\n",
+ "\n",
+ " Returns:\n",
+ " Tuple of training features, training target, validation features, validation target.\n",
+ " \"\"\"\n",
+ " df_train = pd.read_csv(\n",
+ " os.path.join(train_dir, \"train.csv\"),\n",
+ " header=None,\n",
+ " )\n",
+ " df_train = df_train.iloc[np.random.permutation(len(df_train))]\n",
+ " df_train.columns = [\"target\"] + [f\"feature_{x}\" for x in range(df_train.shape[1] - 1)]\n",
+ "\n",
+ " try:\n",
+ " df_validation = pd.read_csv(\n",
+ " os.path.join(validation_dir, \"validation.csv\"),\n",
+ " header=None,\n",
+ " )\n",
+ " df_validation.columns = [\"target\"] + [\n",
+ " f\"feature_{x}\" for x in range(df_validation.shape[1] - 1)\n",
+ " ]\n",
+ "\n",
+ " except FileNotFoundError: # when validation data is not available in the directory\n",
+ " logging.info(\n",
+ " f\"Validation data is not found. {TRAIN_VALIDATION_FRACTION * 100}% of training data is \"\n",
+ " f\"randomly selected as validation data. The seed for random sampling is {RANDOM_STATE_SAMPLING}.\"\n",
+ " )\n",
+ " df_validation = df_train.sample(\n",
+ " frac=TRAIN_VALIDATION_FRACTION,\n",
+ " random_state=RANDOM_STATE_SAMPLING,\n",
+ " )\n",
+ " df_train.drop(df_validation.index, inplace=True)\n",
+ " df_validation.reset_index(drop=True, inplace=True)\n",
+ " df_train.reset_index(drop=True, inplace=True)\n",
+ "\n",
+ " X_train, y_train = df_train.iloc[:, 1:], df_train.iloc[:, :1]\n",
+ " X_val, y_val = df_validation.iloc[:, 1:], df_validation.iloc[:, :1]\n",
+ "\n",
+ " return X_train.values, y_train.values, X_val.values, y_val.values\n",
+ "\n",
+ "\n",
+ "def main():\n",
+ " \"\"\"Run training.\"\"\"\n",
+ " parser = argparse.ArgumentParser()\n",
+ "\n",
+ " parser.add_argument(\n",
+ " \"--max_depth\",\n",
+ " type=int,\n",
+ " )\n",
+ " parser.add_argument(\"--eta\", type=float)\n",
+ " parser.add_argument(\"--gamma\", type=int)\n",
+ " parser.add_argument(\"--min_child_weight\", type=int)\n",
+ " parser.add_argument(\"--subsample\", type=float)\n",
+ " parser.add_argument(\"--verbosity\", type=int)\n",
+ " parser.add_argument(\"--objective\", type=str)\n",
+ " parser.add_argument(\"--num_round\", type=int)\n",
+ " parser.add_argument(\"--tree_method\", type=str, default=\"auto\")\n",
+ " parser.add_argument(\"--predictor\", type=str, default=\"auto\")\n",
+ " parser.add_argument(\"--learning_rate\", type=str, default=\"auto\")\n",
+ " parser.add_argument(\"--output_data_dir\", type=str, default=os.environ.get(\"SM_OUTPUT_DATA_DIR\"))\n",
+ " parser.add_argument(\"--model_dir\", type=str, default=os.environ.get(\"SM_MODEL_DIR\"))\n",
+ " parser.add_argument(\"--train\", type=str, default=os.environ.get(\"SM_CHANNEL_TRAIN\"))\n",
+ " parser.add_argument(\"--validation\", type=str, default=os.environ.get(\"SM_CHANNEL_VALIDATION\"))\n",
+ " parser.add_argument(\"--sm_hosts\", type=str, default=os.environ.get(\"SM_HOSTS\"))\n",
+ " parser.add_argument(\"--sm_current_host\", type=str, default=os.environ.get(\"SM_CURRENT_HOST\"))\n",
+ "\n",
+ " args, _ = parser.parse_known_args()\n",
+ "\n",
+ " X_train, y_train, X_val, y_val = prepare_data(args.train, args.validation)\n",
+ "\n",
+ " # create dataset for lightgbm\n",
+ " dtrain = xgb.DMatrix(data=X_train, label=y_train)\n",
+ " dval = xgb.DMatrix(data=X_val, label=y_val)\n",
+ " watchlist = [(dtrain, \"train\"), (dval, \"validation\")]\n",
+ "\n",
+ " # specify your configurations as a dict\n",
+ " params = {\n",
+ " \"booster\": \"gbtree\",\n",
+ " \"objective\": args.objective,\n",
+ " \"learning_rate\": args.learning_rate,\n",
+ " \"gamma\": args.gamma,\n",
+ " \"min_child_weight\": args.min_child_weight,\n",
+ " \"max_depth\": args.max_depth,\n",
+ " \"subsample\": args.subsample,\n",
+ " \"colsample_bytree\": 1,\n",
+ " \"reg_lambda\": 1,\n",
+ " \"reg_alpha\": 0,\n",
+ " \"eval_metric\": \"rmse\",\n",
+ " }\n",
+ "\n",
+ " bst = xgb.train(\n",
+ " params=params,\n",
+ " dtrain=dtrain,\n",
+ " num_boost_round=args.num_round,\n",
+ " evals=watchlist,\n",
+ " xgb_model=None,\n",
+ " )\n",
+ "\n",
+ " model_location = args.model_dir + \"/xgboost-model\"\n",
+ " pkl.dump(bst, open(model_location, \"wb\"))\n",
+ " logging.info(\"Stored trained model at {}\".format(model_location))\n",
+ "\n",
+ "\n",
+ "if __name__ == \"__main__\":\n",
+ " main()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.estimator import Estimator\n",
+ "from sagemaker.inputs import TrainingInput\n",
+ "\n",
+ "model_path = f\"s3://{default_bucket}/{prefix}/model\"\n",
+ "image_uri = sagemaker.image_uris.retrieve(\n",
+ " framework=\"xgboost\",\n",
+ " region=region,\n",
+ " version=\"1.5-1\",\n",
+ " py_version=\"py3\",\n",
+ " instance_type=instance_type,\n",
+ ")\n",
+ "\n",
+ "xgb_train = Estimator(\n",
+ " image_uri=image_uri,\n",
+ " entry_point=\"code/abalone.py\",\n",
+ " instance_type=instance_type,\n",
+ " instance_count=training_instance_count,\n",
+ " output_path=model_path,\n",
+ " role=role,\n",
+ " sagemaker_session=local_pipeline_session,\n",
+ ")\n",
+ "\n",
+ "xgb_train.set_hyperparameters(\n",
+ " objective=\"reg:squarederror\",\n",
+ " learning_rate=0.01,\n",
+ " num_round=50,\n",
+ " max_depth=5,\n",
+ " eta=0.2,\n",
+ " gamma=4,\n",
+ " min_child_weight=6,\n",
+ " subsample=0.7,\n",
+ ")\n",
+ "\n",
+ "train_args = xgb_train.fit(\n",
+ " inputs={\n",
+ " \"train\": TrainingInput(\n",
+ " s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\"train\"].S3Output.S3Uri,\n",
+ " content_type=\"text/csv\",\n",
+ " ),\n",
+ " \"validation\": TrainingInput(\n",
+ " s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\n",
+ " \"validation\"\n",
+ " ].S3Output.S3Uri,\n",
+ " content_type=\"text/csv\",\n",
+ " ),\n",
+ " }\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Finally, we use the output of the estimator's `.fit()` method as arguments to the `TrainingStep`. By passing the `local_pipeline_session` to the `sagemaker_session`, calling `.fit()` does not launch the training job, it returns the arguments needed to run the job as a step in the pipeline.\n",
+ "\n",
+ "Pass in the `S3Uri` of the `\"train_data\"` output channel to the `.fit()` method. Also, use the other `\"test_data\"` output channel for model evaluation in the pipeline. The `properties` attribute of a Pipeline step matches the object model of the corresponding response of a describe call. These properties can be referenced as placeholder values and are resolved at runtime. For example, the `ProcessingStep` `properties` attribute matches the object model of the [DescribeProcessingJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeProcessingJob.html) response object."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.inputs import TrainingInput\n",
+ "from sagemaker.workflow.steps import TrainingStep\n",
+ "\n",
+ "step_train = TrainingStep(\n",
+ " name=\"AbaloneTrain\",\n",
+ " step_args=train_args,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a Model Evaluation Step to Evaluate the Trained Model\n",
+ "\n",
+ "First, develop an evaluation script that is specified in a Processing step that performs the model evaluation.\n",
+ "\n",
+ "After pipeline execution, you can examine the resulting `evaluation.json` for analysis.\n",
+ "\n",
+ "The evaluation script uses `xgboost` to do the following:\n",
+ "\n",
+ "* Load the model.\n",
+ "* Read the test data.\n",
+ "* Issue predictions against the test data.\n",
+ "* Build a classification report, including accuracy and ROC curve.\n",
+ "* Save the evaluation report to the evaluation directory."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%writefile code/evaluation.py\n",
+ "import json\n",
+ "import pathlib\n",
+ "import pickle\n",
+ "import tarfile\n",
+ "\n",
+ "import joblib\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import xgboost\n",
+ "import math\n",
+ "\n",
+ "from sklearn.metrics import mean_squared_error\n",
+ "\n",
+ "if __name__ == \"__main__\":\n",
+ " model_path = f\"/opt/ml/processing/model/model.tar.gz\"\n",
+ " with tarfile.open(model_path) as tar:\n",
+ " tar.extractall(path=\".\")\n",
+ "\n",
+ " model = pickle.load(open(\"xgboost-model\", \"rb\"))\n",
+ "\n",
+ " test_path = \"/opt/ml/processing/test/test.csv\"\n",
+ " df = pd.read_csv(test_path, header=None)\n",
+ " df.columns = [\"target\"] + [f\"feature_{x}\" for x in range(df.shape[1] - 1)]\n",
+ "\n",
+ " y_test = df.iloc[:, 0].to_numpy()\n",
+ " df.drop(df.columns[0], axis=1, inplace=True)\n",
+ "\n",
+ " X_test = xgboost.DMatrix(df.values)\n",
+ "\n",
+ " predictions = model.predict(X_test)\n",
+ "\n",
+ " mse = mean_squared_error(y_test, predictions)\n",
+ " std = np.std(y_test - predictions)\n",
+ " report_dict = {\n",
+ " \"regression_metrics\": {\n",
+ " \"mse\": {\"value\": math.sqrt(mse), \"standard_deviation\": std},\n",
+ " },\n",
+ " }\n",
+ "\n",
+ " output_dir = \"/opt/ml/processing/evaluation\"\n",
+ " pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ " evaluation_path = f\"{output_dir}/evaluation.json\"\n",
+ " with open(evaluation_path, \"w\") as f:\n",
+ " f.write(json.dumps(report_dict))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next, create an instance of a `ScriptProcessor` processor and use it in the `ProcessingStep`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.processing import ScriptProcessor\n",
+ "\n",
+ "script_eval = ScriptProcessor(\n",
+ " image_uri=image_uri,\n",
+ " command=[\"python3\"],\n",
+ " instance_type=instance_type,\n",
+ " instance_count=processing_instance_count,\n",
+ " base_job_name=\"script-abalone-eval\",\n",
+ " role=role,\n",
+ " sagemaker_session=local_pipeline_session,\n",
+ ")\n",
+ "\n",
+ "eval_args = script_eval.run(\n",
+ " inputs=[\n",
+ " ProcessingInput(\n",
+ " source=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
+ " destination=\"/opt/ml/processing/model\",\n",
+ " ),\n",
+ " ProcessingInput(\n",
+ " source=step_process.properties.ProcessingOutputConfig.Outputs[\"test\"].S3Output.S3Uri,\n",
+ " destination=\"/opt/ml/processing/test\",\n",
+ " ),\n",
+ " ],\n",
+ " outputs=[\n",
+ " ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\"),\n",
+ " ],\n",
+ " code=\"code/evaluation.py\",\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Use the processor's arguments returned by `.run()` to construct a `ProcessingStep`, along with the input and output channels and the code that will be executed when the pipeline invokes pipeline execution. \n",
+ "\n",
+ "Specifically, the `S3ModelArtifacts` from the `step_train` `properties` and the `S3Uri` of the `\"test_data\"` output channel of the `step_process` `properties` are passed as inputs. The `TrainingStep` and `ProcessingStep` `properties` attribute matches the object model of the [DescribeTrainingJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTrainingJob.html) and [DescribeProcessingJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeProcessingJob.html) response objects, respectively."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.properties import PropertyFile\n",
+ "\n",
+ "evaluation_report = PropertyFile(\n",
+ " name=\"EvaluationReport\", output_name=\"evaluation\", path=\"evaluation.json\"\n",
+ ")\n",
+ "step_eval = ProcessingStep(\n",
+ " name=\"AbaloneEval\",\n",
+ " step_args=eval_args,\n",
+ " property_files=[evaluation_report],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a Create Model Step to Create a Model\n",
+ "\n",
+ "In order to perform batch transformation using the example model, create a SageMaker model. \n",
+ "\n",
+ "Specifically, pass in the `S3ModelArtifacts` from the `TrainingStep`, `step_train` properties. The `TrainingStep` `properties` attribute matches the object model of the [DescribeTrainingJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeTrainingJob.html) response object.\n",
+ "\n",
+ "We provide a custom inference script that defines the logic for the batch transform job"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%writefile code/inference.py\n",
+ "\n",
+ "import json\n",
+ "import os\n",
+ "import pickle as pkl\n",
+ "\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import sagemaker_xgboost_container.encoder as xgb_encoders\n",
+ "import xgboost as xgb\n",
+ "import io\n",
+ "import logging\n",
+ "\n",
+ "logging.basicConfig(level=logging.INFO)\n",
+ "\n",
+ "\n",
+ "def model_fn(model_dir):\n",
+ " \"\"\"\n",
+ " Deserialize and return fitted model.\n",
+ " \"\"\"\n",
+ " model_file = \"xgboost-model\"\n",
+ " booster = pkl.load(open(os.path.join(model_dir, model_file), \"rb\"))\n",
+ " return booster\n",
+ "\n",
+ "\n",
+ "def transform_fn(model, request_body, request_content_type, accept):\n",
+ " \"\"\" \"\"\"\n",
+ " if request_content_type == \"text/libsvm\":\n",
+ " input_data = xgb_encoders.libsvm_to_dmatrix(request_body)\n",
+ " if request_content_type == \"text/csv\":\n",
+ " df = pd.read_csv(io.StringIO(request_body.strip(\"\\n\")), header=None)\n",
+ " df.drop(0, axis=1, inplace=True)\n",
+ " input_data = xgb.DMatrix(data=df)\n",
+ "\n",
+ " else:\n",
+ " raise ValueError(\"Content type {} is not supported.\".format(request_content_type))\n",
+ "\n",
+ " prediction = model.predict(input_data)\n",
+ " feature_contribs = model.predict(input_data, pred_contribs=True, validate_features=False)\n",
+ " output = np.hstack((prediction[:, np.newaxis], feature_contribs))\n",
+ "\n",
+ " logging.info(\"Successfully completed transform job!\")\n",
+ "\n",
+ " return \",\".join(str(x) for x in output[0])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.model import Model\n",
+ "\n",
+ "model = Model(\n",
+ " image_uri=image_uri,\n",
+ " model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
+ " source_dir=\"code\",\n",
+ " entry_point=\"inference.py\",\n",
+ " role=role,\n",
+ " sagemaker_session=local_pipeline_session,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Define the `ModelStep` by providing the return values from `model.create()` as the step arguments. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.model_step import ModelStep\n",
+ "\n",
+ "step_create_model = ModelStep(\n",
+ " name=\"AbaloneCreateModel\", step_args=model.create(instance_type=instance_type)\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a Transform Step to Perform Batch Transformation\n",
+ "\n",
+ "Now that a model instance is defined, create a `Transformer` instance with the appropriate model type, compute instance type, and desired output S3 URI.\n",
+ "\n",
+ "Specifically, pass in the `ModelName` from the `CreateModelStep`, `step_create_model` properties. The `CreateModelStep` `properties` attribute matches the object model of the [DescribeModel](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_DescribeModel.html) response object."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.transformer import Transformer\n",
+ "\n",
+ "\n",
+ "transformer = Transformer(\n",
+ " model_name=step_create_model.properties.ModelName,\n",
+ " instance_type=instance_type,\n",
+ " instance_count=transform_instance_count,\n",
+ " output_path=f\"s3://{default_bucket}/{prefix}/transform\",\n",
+ " sagemaker_session=local_pipeline_session,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Pass in the transformer instance and the `TransformInput` with the `batch_data` pipeline parameter defined earlier."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.inputs import TransformInput\n",
+ "from sagemaker.workflow.steps import TransformStep\n",
+ "from sagemaker.workflow.functions import Join\n",
+ "\n",
+ "transform_data = Join(\n",
+ " on=\"/\",\n",
+ " values=[\n",
+ " step_process.properties.ProcessingOutputConfig.Outputs[\"test\"].S3Output.S3Uri,\n",
+ " \"test.csv\",\n",
+ " ],\n",
+ ")\n",
+ "\n",
+ "transform_args = transformer.transform(transform_data, content_type=\"text/csv\")\n",
+ "\n",
+ "step_transform = TransformStep(name=\"AbaloneTransform\", step_args=transform_args)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.fail_step import FailStep\n",
+ "\n",
+ "step_fail = FailStep(\n",
+ " name=\"AbaloneMSEFail\",\n",
+ " error_message=Join(on=\" \", values=[\"Execution failed due to MSE >\", mse_threshold]),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a Condition Step to Check Accuracy and Conditionally Create a Model and Run a Batch Transformation Or Terminate the Execution in Failed State\n",
+ "\n",
+ "In this step, the model is registered only if the accuracy of the model, as determined by the evaluation step `step_eval`, exceeded a specified value. Otherwise, the pipeline execution fails and terminates. A `ConditionStep` enables pipelines to support conditional execution in the pipeline DAG based on the conditions of the step properties.\n",
+ "\n",
+ "In the following section, you:\n",
+ "\n",
+ "* Define a `ConditionLessThanOrEqualTo` on the accuracy value found in the output of the evaluation step, `step_eval`.\n",
+ "* Use the condition in the list of conditions in a `ConditionStep`.\n",
+ "* Pass the `CreateModelStep` and `TransformStep` steps into the `if_steps` of the `ConditionStep`, which are only executed if the condition evaluates to `True`.\n",
+ "* Pass the `FailStep` step into the `else_steps`of the `ConditionStep`, which is only executed if the condition evaluates to `False`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.conditions import ConditionLessThanOrEqualTo\n",
+ "from sagemaker.workflow.condition_step import ConditionStep\n",
+ "from sagemaker.workflow.functions import JsonGet\n",
+ "\n",
+ "cond_lte = ConditionLessThanOrEqualTo(\n",
+ " left=JsonGet(\n",
+ " step_name=step_eval.name,\n",
+ " property_file=evaluation_report,\n",
+ " json_path=\"regression_metrics.mse.value\",\n",
+ " ),\n",
+ " right=mse_threshold,\n",
+ ")\n",
+ "\n",
+ "step_cond = ConditionStep(\n",
+ " name=\"AbaloneMSECond\",\n",
+ " conditions=[cond_lte],\n",
+ " if_steps=[step_create_model, step_transform],\n",
+ " else_steps=[step_fail],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a Pipeline using `LocalPipelineSession`\n",
+ "\n",
+ "In this section, combine the steps into a Pipeline so it can be executed. We provide a `LocalPipelineSession` object to the `Pipeline` so that when executed, all the steps in the pipeline will run locally on the machine. By switching the `LocalPipelineSession` to a `sagemaker.session.Session` object, you can switch the execution to run in the cloud on SageMaker instances."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.pipeline import Pipeline\n",
+ "\n",
+ "pipeline_name = f\"LocalModelPipeline\"\n",
+ "pipeline = Pipeline(\n",
+ " name=pipeline_name,\n",
+ " parameters=[\n",
+ " input_data,\n",
+ " mse_threshold,\n",
+ " ],\n",
+ " steps=[step_process, step_train, step_eval, step_cond],\n",
+ " sagemaker_session=local_pipeline_session,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### (Optional) Examining the pipeline definition\n",
+ "\n",
+ "The JSON of the pipeline definition can be examined to confirm the pipeline is well-defined and the parameters and step properties resolve correctly."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import json\n",
+ "\n",
+ "definition = json.loads(pipeline.definition())\n",
+ "definition"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Submit the pipeline to SageMaker and start execution\n",
+ "\n",
+ "Submit the pipeline definition to the Pipeline service. The Pipeline service uses the role that is passed in to create all the jobs defined in the steps."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pipeline.upsert(role_arn=role)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Start the pipeline and accept all the default parameters."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "execution = pipeline.start()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "steps = execution.list_steps()\n",
+ "steps"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Get the step outputs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get output files from processing job\n",
+ "\n",
+ "processing_job_name = steps[\"PipelineExecutionSteps\"][0][\"Metadata\"][\"ProcessingJob\"][\"Arn\"]\n",
+ "outputs = local_pipeline_session.sagemaker_client.describe_processing_job(\n",
+ " ProcessingJobName=processing_job_name\n",
+ ")[\"ProcessingOutputConfig\"][\"Outputs\"]\n",
+ "for key in outputs:\n",
+ " print(outputs[key][\"S3Output\"][\"S3Uri\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get output from training job\n",
+ "\n",
+ "training_job_name = steps[\"PipelineExecutionSteps\"][1][\"Metadata\"][\"TrainingJob\"][\"Arn\"]\n",
+ "outputs = local_pipeline_session.sagemaker_client.describe_training_job(\n",
+ " TrainingJobName=training_job_name\n",
+ ")\n",
+ "print(\"Model location : \", outputs[\"ModelArtifacts\"][\"S3ModelArtifacts\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get output from model evaluation step (processing job)\n",
+ "\n",
+ "processing_job_name = steps[\"PipelineExecutionSteps\"][2][\"Metadata\"][\"ProcessingJob\"][\"Arn\"]\n",
+ "outputs = local_pipeline_session.sagemaker_client.describe_processing_job(\n",
+ " ProcessingJobName=processing_job_name\n",
+ ")[\"ProcessingOutputConfig\"][\"Outputs\"]\n",
+ "for key in outputs:\n",
+ " print(outputs[key][\"S3Output\"][\"S3Uri\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get output of ModelStep\n",
+ "import json\n",
+ "\n",
+ "model_name = steps[\"PipelineExecutionSteps\"][-1][\"Metadata\"][\"Model\"][\"Arn\"]\n",
+ "outputs = local_pipeline_session.sagemaker_client.describe_model(ModelName=model_name)\n",
+ "print(outputs)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Get output from the TransformStep\n",
+ "\n",
+ "transform_job_name = steps[\"PipelineExecutionSteps\"][4][\"Metadata\"][\"TransformJob\"][\"Arn\"]\n",
+ "outputs = local_pipeline_session.sagemaker_client.describe_transform_job(\n",
+ " TransformJobName=transform_job_name\n",
+ ")\n",
+ "print(outputs)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Transition to running pipeline as SageMaker Managed Pipeline\n",
+ "\n",
+ "We will now use a non-local PipelineSession object to re-run the Pipeline steps via SageMaker as a managed service. This will run all pipeline steps as SageMaker-managed processes. This will also allow us to view and track the results directly in the SageMaker Studio UI."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.workflow.pipeline_context import PipelineSession\n",
+ "\n",
+ "pipeline_session = PipelineSession()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Recreate the SKLearnProcessor with non-local session\n",
+ "\n",
+ "framework_version = \"1.0-1\"\n",
+ "\n",
+ "sklearn_processor = SKLearnProcessor(\n",
+ " framework_version=framework_version,\n",
+ " instance_type=instance_type,\n",
+ " instance_count=processing_instance_count,\n",
+ " base_job_name=\"sklearn-abalone-process\",\n",
+ " role=role,\n",
+ " sagemaker_session=pipeline_session, # use non-local session\n",
+ ")\n",
+ "\n",
+ "processor_args = sklearn_processor.run(\n",
+ " inputs=[\n",
+ " ProcessingInput(source=input_data, destination=\"/opt/ml/processing/input\"),\n",
+ " ],\n",
+ " outputs=[\n",
+ " ProcessingOutput(output_name=\"train\", source=\"/opt/ml/processing/train\"),\n",
+ " ProcessingOutput(output_name=\"validation\", source=\"/opt/ml/processing/validation\"),\n",
+ " ProcessingOutput(output_name=\"test\", source=\"/opt/ml/processing/test\"),\n",
+ " ],\n",
+ " code=\"code/preprocessing.py\",\n",
+ ")\n",
+ "\n",
+ "step_process = ProcessingStep(name=\"AbaloneProcess\", step_args=processor_args)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"image_uri: {image_uri}\")\n",
+ "print(f\"model_path: {model_path}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Recreate the Estimator instance with non-local session\n",
+ "\n",
+ "xgb_train = Estimator(\n",
+ " image_uri=image_uri,\n",
+ " entry_point=\"code/abalone.py\",\n",
+ " instance_type=instance_type,\n",
+ " instance_count=training_instance_count,\n",
+ " output_path=model_path,\n",
+ " role=role,\n",
+ " sagemaker_session=pipeline_session, # use non-local session\n",
+ ")\n",
+ "\n",
+ "xgb_train.set_hyperparameters(\n",
+ " objective=\"reg:squarederror\",\n",
+ " learning_rate=0.01,\n",
+ " num_round=50,\n",
+ " max_depth=5,\n",
+ " eta=0.2,\n",
+ " gamma=4,\n",
+ " min_child_weight=6,\n",
+ " subsample=0.7,\n",
+ ")\n",
+ "\n",
+ "train_args = xgb_train.fit(\n",
+ " inputs={\n",
+ " \"train\": TrainingInput(\n",
+ " s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\"train\"].S3Output.S3Uri,\n",
+ " content_type=\"text/csv\",\n",
+ " ),\n",
+ " \"validation\": TrainingInput(\n",
+ " s3_data=step_process.properties.ProcessingOutputConfig.Outputs[\n",
+ " \"validation\"\n",
+ " ].S3Output.S3Uri,\n",
+ " content_type=\"text/csv\",\n",
+ " ),\n",
+ " }\n",
+ ")\n",
+ "\n",
+ "step_train = TrainingStep(\n",
+ " name=\"AbaloneTrain\",\n",
+ " step_args=train_args,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Recreate the Script Processor instance with non-local session\n",
+ "\n",
+ "script_eval = ScriptProcessor(\n",
+ " image_uri=image_uri,\n",
+ " command=[\"python3\"],\n",
+ " instance_type=instance_type,\n",
+ " instance_count=processing_instance_count,\n",
+ " base_job_name=\"script-abalone-eval\",\n",
+ " role=role,\n",
+ " sagemaker_session=pipeline_session, # use non-local session\n",
+ ")\n",
+ "\n",
+ "eval_args = script_eval.run(\n",
+ " inputs=[\n",
+ " ProcessingInput(\n",
+ " source=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
+ " destination=\"/opt/ml/processing/model\",\n",
+ " ),\n",
+ " ProcessingInput(\n",
+ " source=step_process.properties.ProcessingOutputConfig.Outputs[\"test\"].S3Output.S3Uri,\n",
+ " destination=\"/opt/ml/processing/test\",\n",
+ " ),\n",
+ " ],\n",
+ " outputs=[\n",
+ " ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\"),\n",
+ " ],\n",
+ " code=\"code/evaluation.py\",\n",
+ ")\n",
+ "\n",
+ "evaluation_report = PropertyFile(\n",
+ " name=\"EvaluationReport\", output_name=\"evaluation\", path=\"evaluation.json\"\n",
+ ")\n",
+ "\n",
+ "step_eval = ProcessingStep(\n",
+ " name=\"AbaloneEval\",\n",
+ " step_args=eval_args,\n",
+ " property_files=[evaluation_report],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Recreate the Model instance with non-local session\n",
+ "\n",
+ "model = Model(\n",
+ " image_uri=image_uri,\n",
+ " model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n",
+ " source_dir=\"code\",\n",
+ " entry_point=\"inference.py\",\n",
+ " role=role,\n",
+ " sagemaker_session=pipeline_session, # use non-local session\n",
+ ")\n",
+ "\n",
+ "step_create_model = ModelStep(\n",
+ " name=\"AbaloneCreateModel\", step_args=model.create(instance_type=instance_type)\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Recreate the Transformer instance with non-local session\n",
+ "\n",
+ "transformer = Transformer(\n",
+ " model_name=step_create_model.properties.ModelName,\n",
+ " instance_type=instance_type,\n",
+ " instance_count=transform_instance_count,\n",
+ " output_path=f\"s3://{default_bucket}/{prefix}/transform\",\n",
+ " sagemaker_session=pipeline_session, # use non-local session\n",
+ ")\n",
+ "\n",
+ "transform_data = Join(\n",
+ " on=\"/\",\n",
+ " values=[\n",
+ " step_process.properties.ProcessingOutputConfig.Outputs[\"test\"].S3Output.S3Uri,\n",
+ " \"test.csv\",\n",
+ " ],\n",
+ ")\n",
+ "\n",
+ "transform_args = transformer.transform(transform_data, content_type=\"text/csv\")\n",
+ "\n",
+ "step_transform = TransformStep(name=\"AbaloneTransform\", step_args=transform_args)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Recreate the Step condition with new step instances\n",
+ "\n",
+ "step_cond = ConditionStep(\n",
+ " name=\"AbaloneMSECond\",\n",
+ " conditions=[cond_lte],\n",
+ " if_steps=[step_create_model, step_transform],\n",
+ " else_steps=[step_fail],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Now that all the Steps are re-defined, we create a new Managed Pipeline\n",
+ "\n",
+ "We add each of the recreated steps to a new Pipeline instance that we will run as a managed (in-the-cloud) pipeline."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Re-define the Pipeline using non-local session\n",
+ "\n",
+ "pipeline_name = f\"SM-Managed-Pipeline\"\n",
+ "\n",
+ "sm_pipeline = Pipeline(\n",
+ " name=pipeline_name,\n",
+ " parameters=[\n",
+ " input_data,\n",
+ " mse_threshold,\n",
+ " ],\n",
+ " steps=[step_process, step_train, step_eval, step_cond],\n",
+ " sagemaker_session=pipeline_session, # non-local session\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sm_pipeline.upsert(role_arn=role)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# start execution of SageMaker-managed pipeline\n",
+ "sm_execution = sm_pipeline.start()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sm_execution.wait(delay=60, max_attempts=60)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sm_execution.list_steps()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### (Optional) After a Pipeline Step completes, you can view the CloudWatch Log output \n",
+ "\n",
+ "Using SageMaker Studio and navigating to the Pipelines components, find the specific execution that just completed. Under the 'Graph' tab on the left panel, select a particular step, like Training (AbaloneTrain in this example), then click the 'Logs' tab on the right panel, and click the 'view logs in CloudWatch console' link. This will open a new tab/window showing the log output from the Training job."
+ ]
+ },
+ {
+ "attachments": {
+ "blog-pipeline-local-mode-AbaloneTrain-logs-link.png": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAABnUAAAQRCAYAAAD/mw2gAAABQWlDQ1BJQ0MgUHJvZmlsZQAAKJFjYGASSCwoyGFhYGDIzSspCnJ3UoiIjFJgf8bAwSAAhGIM5onJxQWOAQE+QCUMMBoVfLvGwAiiL+uCzDr7PJkrpSlF8TrHs12XXpb9wFSPArhSUouTgfQfIE5KLigqYWBgTACylctLCkDsFiBbpAjoKCB7BoidDmGvAbGTIOwDYDUhQc5A9hUgWyA5IzEFyH4CZOskIYmnI7Gh9oIAR4CjkblFYBgBp5IOSlIrSkC0c35BZVFmekaJgiMwhFIVPPOS9XQUjAyMjBgYQOENUf35BjgcGcU4EGLFdgwMlkC/MHUixOL/MjBsL2VgkF6LEFOexcDAr87AsOVbQWJRItwBjN9YitOMjSBs7u0MDKzT/v//HM7AwK7JwPD3+v//v7f///93GQMD8y0GhgPfAMxwYJ+R36TtAAAAYmVYSWZNTQAqAAAACAACARIAAwAAAAEAAQAAh2kABAAAAAEAAAAmAAAAAAADkoYABwAAABIAAABQoAIABAAAAAEAAAZ1oAMABAAAAAEAAAQRAAAAAEFTQ0lJAAAAU2NyZWVuc2hvdEZnRy0AAAI/aVRYdFhNTDpjb20uYWRvYmUueG1wAAAAAAA8eDp4bXBtZXRhIHhtbG5zOng9ImFkb2JlOm5zOm1ldGEvIiB4OnhtcHRrPSJYTVAgQ29yZSA2LjAuMCI+CiAgIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgICAgIDxyZGY6RGVzY3JpcHRpb24gcmRmOmFib3V0PSIiCiAgICAgICAgICAgIHhtbG5zOmV4aWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vZXhpZi8xLjAvIgogICAgICAgICAgICB4bWxuczp0aWZmPSJodHRwOi8vbnMuYWRvYmUuY29tL3RpZmYvMS4wLyI+CiAgICAgICAgIDxleGlmOlBpeGVsWURpbWVuc2lvbj4xMDQxPC9leGlmOlBpeGVsWURpbWVuc2lvbj4KICAgICAgICAgPGV4aWY6VXNlckNvbW1lbnQ+U2NyZWVuc2hvdDwvZXhpZjpVc2VyQ29tbWVudD4KICAgICAgICAgPGV4aWY6UGl4ZWxYRGltZW5zaW9uPjE2NTM8L2V4aWY6UGl4ZWxYRGltZW5zaW9uPgogICAgICAgICA8dGlmZjpPcmllbnRhdGlvbj4xPC90aWZmOk9yaWVudGF0aW9uPgogICAgICA8L3JkZjpEZXNjcmlwdGlvbj4KICAgPC9yZGY6UkRGPgo8L3g6eG1wbWV0YT4K+3kStwAAQABJREFUeAHsnQec1NQWxo/03nvvHekdaSoIiiCIgoiIgCAgRanio0gHRRFBVBRFKaKgCEpVEKRJ773D0nuvvvtlSTaZSabszg477Hfeb0xyW27+yczy7pdzzmOFStf4T2gkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIxmkCcGD07To4ESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESEAjQFGHDwIJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJhAABijohcJM4RRIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARKgqMNngARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARCgABFnRC4SZwiCZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACVDU4TNAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiFAgKJOCNwkTpEESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEKOrwGSABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiCBECBAUScEbhKnSAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIUdfgMkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkEAIEKCoEwI3iVMkARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgAYo6fAZIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIIAQIUNQJgZvEKZIACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAARR0+AyRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiQQAgQo6oTATeIUSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESICiDp8BEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEggBAvFCYI5+TbFo4QJa+5caPSfbd+41+s6YNcfYd91Bn+0797gW85gESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEYgyBxwqVrvFfjJlNFCbyUqP6AiHHk82YNVdcxR30K1o4v/QfMtpTV7/q4sSJIwXy55Hr167LkWNhfvW1a5w8WVLJmjWznD17Xs6eO+/WpHiRQvJ48cKya/c+2bhlu9y/f9+tDQtIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARCm0DIizrwsoGYU7RwQZ/vhC7u6ELQ9p27AyrqpEqZQiaOGyX37t2Tl1t28HleTg1fqP+MNH/5BTkWdkK69hxgNEuSJLF8NXaEJEyY0Ch7p/fAgAhJxoBR2MmVM7v07NpeNm/dIV98MyUKI7ErCZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZBASIdf00UZ/TZCnEHINWxhCKmGNjCzF0+4CJTfLyFIGySG/afu0zU1Qefq1Wvyy5z5Ag+hE6fOxJhZPlm9imRIn05qVqscraJO7lzZpfVrTeWvv1eqz4oYc/2cCAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAkEkkDIijqugo7ufeMKRw+3hq25jz+ePa5jxpTjSuVLa1OZv3ipzP59YUyZljGPWb/NkzRpUmmeOkZhNOwg/FyhAvnk9JlzFHWigS+HJAESIAESIAESIAESIAESIAESIAESIAESIAESIAESiBkEQljUicif4yTouCKGsIP8OcEWdNKkTiWPFyssjz0m8vc/a9xy3iCMWjEVPi5TpvTymPrfkuWr5PLlK67TdztOkDCBVvaYGjhd2jTavl3OHb1jntw5JXfObJIqZUo5dfqMrF2/WW7dvq1X224RSg7jX7h4SZInTyalHi8qyVSOn9X/bpDzFy5qfXJkyyLFixaSvQcOyZ69B4xxrl67JpO+n2E5R9KkSSRxokRy7vwFbVzkHiqQL7esW79Fwk6eMvomTJBAO9+NmzflmspNpJve/8qVq8a4mTNl0KoTJ0pocMD4//0Xni4qfvx4UqRQAcmTK4cWwm6Tyjt0585dfUjbLbyeihUpKFkzZ1LXm0TWbtgihw4fdWsLNgUL5FWiUl45eOiobN2+SzDHBPHja9do7oByCFAZM6ST3fsOaKyY/8hMiPskQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAKeCIRkTh2zx42vgg4gmPuZoURXTh2IChACcisxQTfk2Rk7YZL8s2qtVtT29VekzlPV9Wpju0LVfzxuonbsmlOnlgpr9labFpooYnR4sPPiq+1ciyRu3Ljyw8RPBeKG2TCXt7v3Ux4uZ83Flv0pX3+qhXjbtmO3JnLolbi2OX8skhoqtFoKJfbodvHSJWnTsad2WOOJStKp3eua+PPm2720siH9e0rB/HlloxJWiivRJF68iDmZcwa90eJlqVenluzeu1/6DhypDy/DBvaW/Hlzy2/q3Av/XCZjP/zAlkPbTj01IapqpXLaHMznuXXrlvQb/JHsP3jYGNe8U7b049KrWwe3ccGpQ7e+RlMIXiMH9xWIdrpBhIKgA1GoSYv2erG81qyx1K/3tGVMtB066jPZuXuv0Y47JEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJOBEII5TRUwuN+fH0cOreZuvk6DjrV9U6uHFkS1rFlm5Zp3MnbdY8ziBwAKRRjctD87J0/Lv+k0CgQpiB6xyxbJuIozeJ+zESYG3ie5lA7Fh4+ZtyvNmk97EbXvt+jXNM2TewiXyx4K/BHl4MJeXG4fnHHLr4FJQtHABzctlweK/tevAtT3/bG1JpLxjIPgsW7FG84yBF1D5MiVdersfwuMn7MQpjQvmDsuWJbPkzJHNvbFDyfUbNzQOV65e1VpcUt5NGGud8qq5qrx70qZJLV06tNaEo337D2p8T546rYlUPbpGCC6uw8NLCJ4+W7btFISQw7VBAEN+oNIlixnNe7/TQRN0IHCt37hF/lj4l+YBBK5mq161osYKzNZu2KyNCU8peCz1691V3ef45ubcJwESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAFbAhFuErbVMbsQIogv9jAEHcwLQkDz1m8b4db2KmGhW6e2kjVLJmPaX3zzg7Gv73w5doQmFkAMWLzkH73Y2O7as1+GjBornyovlSyZMsr8RUs1zxWjgcsO5qF7z+hVO3btle5d2llECr3ObttnwHDZt/+QVnXk2HGBh9HNm7fk1TadjeYIw5YrZ3apogQpiFSebNFfy8V87RPHjdTCwlWvUkEmHznmqatRhxB14NC3x9tSqkQxLXfPp59/Y9S3btlU84w5oDxyevcfrpXPnrtAflDeR3q4OqOxaWfF6nWCj9nixYsrlSuUlbq1a8mGTdsEIfPyKY8h2KARYzQBCPvfTP5Rfv7hC+wa1qJZI20fgthX307V9qfO+FUmTfhIkidLJs1efF4mT5tptOcOCZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACdgRCDlRBwKNvxYeXm23v90C0t6cMwVCCgweG65WuGB+TWA5HnZSeZlc00SdJIkTuzaL0jGEiKdqVpW4ceLKmbPntLHMYck8Df7f/fD8NGhz8EFumf8kogzlp9WYEHXgfeTN7ty5Y2ly+vQ5TdTxdT6Wzg4H2bJm1mrAu0Pb14xWELlwHuQIQm4eJ0Pum5oqvNylS1c0AQvtEFoNli9PLm0LLx149HgyCDewX+bMtzRbv3GrIEQdch3RSIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESMAbgZATdcwX5Gvote0795i7PbR9CACuVrNaJWnbqrkhFrjWB+p49PB+kiNb1kANF/BxXAWiQJwgWdKk2jDIaQSxyWy4F06CDsKsjRz0niRLFt7f3E/fRz4d2G0XcUqv17cQlPRwbAjpZraLFy9ph6lTpTQXc58ESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEbAmEnKhTtHB+40KQ5yWmCDbGpPzYwYJ/m5bNNEFn+cp/ZebsPySByufSrWMbyZwpgx8jeW7auEE9TdBBuLTPJ04WhIErVCCfdH7rDc8dY0DtY+Lu1eTrtCDapFDeOL8v+FMmfT/D127StWNrTdA5FnZCvpw0VS5euCRNGj0nT1Qub4yhe+cg/w5y97gKNnpDiEfwDIKwkzFDekFOH93Spk2j7Z6/cFEv4pYESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEHAl4j5Pl2PXhVPiaR8eX2UEUwudhWYb0aSVhwoSChX/kgjl2/IQg/8uNGzdspxTHJmybbUOXwgplS2olGzdv03LFnD5zTnbv3e/SKmYd3rp9W5tQxozpLROL/yD8mbnwzt272iEEHLMdPR6mHSIXjj+me/X8OHOO7FBeXmEnT8mp02csQ1y8dFnu3Ak/b1clwqVJnUqrr/NUdUs7HFx44JHzkhKGdIOgV7ZUce0wpt8Lfc7ckgAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJPFwCIeepY8ZVtHDBSHvqIDePvsjef8hHkR7HPB9/90+dPivXrl+XpEmSyLABveTAoSNS8vGigvBfZtu974B2mDlTRhnSv6cMGjHGyPFibue0v2TZKi1vS9nSJeStNi0kRYrkUrpEMafmMaJ88ZLl8kL9ZySlmuuwgb2Vh8sZKV60kMq7Ex72zDxJeDmVL1NSShQvIkP791LeTvFl5Cefy9ffTZcKZUsJwptNnfSZJtBcu3ZdcufKLolVvqK2nXqahzH2d+/Zr52r9WtNpYjKdZQ3T04pkC+PUa/vwOsJ3k6FC+aTL8eOMDxy9Hp9+/Xk6dKrWwepVqWCurdptWvBfcYcIF4FUqjUz8ktCZAACZAACZAACZAACZAACZAACZAACZAACZAACZDAo0cg5Dx1zOHWzKHY/L01uqCDfuYx/R3Hrv29+/ftikUvN+fWmf7Tb1pelnx5c0vtJ6urcGHJNY8dDHD/wTjwFkEoMHh3FMyfVwvRZj6B3s5cZt5f8OffcjzspMSPH0+erFFVyilxZ8fuvZqHkPqPuanbvl210/kQZgx2/0EnvZ2+1eruez7f3QdjQPDauXufNl5+xQahz3D9u/aEl91/0A4NVq1ZL4ePHNPaFsifR7Jny6IxgocMxJ1bt25pIe4gpFSpVE4Lg6bns9E6ufxn8rSf5bIK3QYBqW7tmoLzb92xS2tlvnfLVqyR8V9NlkOHj2rnOHvuvEz4+nuX0UTWrt8sGBN9EfauxhOVtLEvXb4i7w0YYdxnt44sIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAETgccKla7heZXd1Dim7Jq9bODlMGPWHL+mFtX+fp3Mx8bFihSU8+cvaqG+nLokSpRQbt++E2kRILkKT5Yvd07ZpkSiO3fuOJ0mRpVjziWVB86OXXsd89boE0ZotsSJE8llJZa4Whbl5ZQvby5NgDlyLDwsm2sb1+NsWTNLsqRJVKi6A+ECmGsDm2OcY/jAPnLj5k1p0aaLpQVEqXx5cmneOshrhDB4NBIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARLwlUBIijq4uIF931H5cApq1+mPsGMWdND5xVfbaWPwPyTgLwGEe4NQs33nbi2/Do4R3i5d2jSybcduGTB0tL9Dsj0JkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJOBII2Zw6EHIG9g0XdfRQat48dlwFHeTSoZFAZAm82LCeyqeTX+uO0HNx48bV9u/cuSujP/sqssOyHwmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAnYEoibLnOuAbY1MbzwzFmErnpMeesU0GaKLUSb8LKCRp4clCM5fad2LaVmtcrGVUEUWrp8lXHMHRLwl8CefQfk1k2VrydBfEmiwr4hR86W7Tvl/Q9GyrVr1/0dju1JgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIwCOBkA2/pl+Vq/eNXu60RagsCDrbVV4ZGgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmECoGQF3V00L6IO/7k3tHH5ZYESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEYgKBR0bU0WEi3FrRwuG5dooWzq955aCOnjk6IW5JgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARCkcAjJ+qE4k3gnEmABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEjAG4E43hqwngRIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARI4OEToKjz8O8BZ0ACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACXglQ1PGKiA1IgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARI4OEToKjz8O8BZ0ACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACXgnE89qCDUiABEiABEiABEiABEiABEiABGIVgTu3bwk+9+/fi1XXzYslARIgARIgARIgARIggWAQiBMnrsRPkFD7+Hs+ijr+EmN7EiABEiABEiABEiABEiABEniECUDMuXXz+iN8hbw0EiABEiABEiABEiABEni4BPDylPZv7sdE4sdP6NdkGH7NL1xsTAIkQAIkQAIkQAIkQAIkQAKPNgGIOjQSIAESIAESIAESIAESIIHoJ3Dnlv//9qaoE/33hWcgARIgARIgARIgARIgARIggZAhwJBrIXOrOFESIAESIAESIAESIIEQJxCZf3tT1Anxm87pkwAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJxA4CFHVix33mVZIACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACYQ4AYo6IX4DOX0SIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIIHYQYCiTuy4z7xKEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiCBECdAUSfEbyCnTwIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkEDsIUNSJHfeZV0kCJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJBDiBCjqhPgN5PRJgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARiBwGKOrHjPvMqSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEQpwARZ0Qv4GcPgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQQOwgQFEndtxnXiUJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkECIE6CoE+I3kNMnARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARKIHQQo6sSO+8yrJAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESCHECFHVC/AZy+iRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAArGDAEWd2HGfeZUkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIhToCiTojfQE6fBEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEggdhCgqBM77jOvkgRIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIIMQJUNQJ8RvI6ZMACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACcQOAvFix2XyKkmABEiABEiABEiABEiABEiABEjATwJZsoV3KFM+fKsf4yjsWHjZieNqH58Hx+Gl/C8JkAAJkAAJkAAJkAAJRAsBijrRgpWDkgAJkAAJkAAJkAAJkAAJkAAJhCyBshVEMmcVMYs4rhej12Fb5kHl+jUi69SHFlQC5cqUlMceCw9Esn3HLrl2/XpQzx+okz0q1xEoHhyHBEiABEiABEjAngBFHXsuLCUBEiABEiABEiABEiABEiABEohtBCDmlFGfyBr64kNxJ7IE/e5XpVJ5mT75S6PfR2M+l08++8I4DpWdR+U6QoU350kCJEACJEACoUyAok4o3z3OnQRIgARIgARIgARIgARIgARIIDAE6jfy7Jnjz1kg7MDTZ/2/DMv2gFv9Z+tIg2efkTy5c0ratGnk8uUrcuTYcVmxco1MmPid3L9/3x/CRtsuHd809v/77z+Z+O0PxrF5p0a1KlKrelUpVqSQ5MyZXTvf0aPHZcOmLTLy43Fy+/Ztc3O3/Tdee0USJ07kVm5XsH7jZln973q7KseyYF2HPoGSjxeTOk/XlEoVykrOHNnlzJmzsk7Ne/Fff8vSZSt9uh9x48aRJo0aCDyMChcqIJkzZZQbN27IwUNHZP6iJfL91Bn66Txu06ROJU0aN5BSJYpJgfx5JW2aNHLu/HnZv/+QfPvDdFmxSn2PHpKBEwQ3GJ6RrybZP1++Tq9ggXzyfL06UuLxouq7kEsSJUooJ0+dll2798lHY8bJ8bCTPg2VPHkyeebpWlJTPdMl1VgJEiSQnbv2yLJ/VskfCxb7PI7dyUqVKC6VK5azq9LKrl67Jus3bJYd6nyevrfp0qWVl9V91W3t+k3y77oN+qFP24rly0iZUiWMtr/PXySHDh81jh/lnfz58kjtJ2sYlzj1x5ly4eIl49huJ7p/p+zOGYiyQP2WmOcSJ04cadqkoaROlUorPnDwsMxb+Ke5CfdJIKQJPFaodI3/QvoKOHkSIAESIAESIAESIAESIAESIIGAEbh6+ULAxgqJgRA+DYJOdFks99rBIvbE8R9LLiWkOBkWiXu/P1hmz53n1MS2HAvZe7euFizewbap0Gt1GzS1tIWI88VnH0qO7A/yI1lqww9u3bolvd4fJDN/nWtTG150ZM9GFeLtMcd6c8XSZSukReuO5iKP+8G8Dkzkh2/GS/UnKjvO6djxE1K7fhO5cuWqY5vGDZ+TIQPfk6RJkji2gVjxWutOsnP3Hts2uG/93ntXWrVoZtxDu4Zr123UeAY7rF7KlClkw8rFmmCCeUE0zFGglN0UvZalTpVSxo0ZIU9UrujYFuN/98OP8r8Phju2QUW5sqXkR+WdFj9+fNt2GKffoBHy7ffTbeu9FS5Z8Ivky5PbWzOtHt+5N9p1kRMnT7m1H9Svt7zeIuL7iO9ZwRKV5N493wRcLPTv3rxKEiZMaIz9wdAPoyysGYPF0B3c3yH9+2hCqXmKjZu94VUUi87fKfNcArkfiN8S83zwvPTp0Vlebfqi5dmB2Fzt6efNTblPAjGKQLIUqf2aT/i/fPzqwsYkQAIkQAIkQAIkQAIkQAIkQAIk8AgQ8EfQCTsW7nWjb329fHjtIKxbLDQsis/5+Qc3QQeLzmZLljSpjB09VPBWvj/W6jWrGDD+y0mW7u/36ibzZk/3KOigAxYBPx45SIoXK2Lprx9gcdlXQUfv4882WNeB6/xr/iyPgg7mnS1rZlmzbIEjt7kzp8gnowZ7FHQwTqaMGWTWj5MMUQRlusHTZPuG5dK6ZXOPgg7aY5F7yref612DtkVYPwhuUbVaNarKhlV/eRR0cA48YxBB3n6rteMpGzV4Vn6e8rWjoKOPA0EFn+g2iKar/54n8MTzZnj+3uvR1Vszo75vr3csi/JGxSO6U7f2k7Jyye8ya9okN0HHl0uO7t8pX+bgb5tA/Jbo58TfG/wu7dq0UvtdwfNGI4FHmQDDrz3Kd5fXRgIkQAIkQAIkQAIkQAIkQAIk4EzAm4cOBByEUINh3858ycMDYSfsuPMYduM+AmWTvvjUErJs/sK/lEfMB3L+wkVNEHivV1dp0ayJtpiNBe3JE8dJgcedPRlckbRs/rJRBC+AOb8vMI6xU8EkEiFM1P6Dh2TT5u2yZdt2yZs7l7zyciNj0R7n/+7LT6V05acsY+Agfbp0ljJv4cAW/vm3pb23g2Bdx+fKUyR/3jzGdG7cuCkLFi+Rf1QIvMeLF5G6KpxX+vTh15o8WVKZOe0bKVe1ttFe30GINN3AfefuvbJh4xY5fOSYYl5asDiti2C6YNeuU3e9i7ZNmSKFoE63CxcvqvBhe2XN2vVy/foNebbu0yqsWDG9WgvB1eC5un57cxkD+LnT4c1WWqg+P7vZNi9RvJjEixfXqIMn1I6du2WNCkWWQolbzV5qJBkecEej7l07ypTpM7XvidFJ7cDbB4vWOlvUbd22Q/C8XblyRWrVrKaEowpGPQSif1atkQUqFF5k7c6dO7JsxWqje1zlXQWxDmEUdcELHlefKFEUHmqevLswSKuWr2jhDvHceDIsyJu9fDy1DfU6eLst/3OO+p1JG6VLie7fqShNzqFzIH5LMHT3rh2UGNrGq0DsMA0Wk0BIEqCoE5K3jZMmARIgARIgARIgARIgARIgARKIEgFvgs6cWb6JMOvWiODjTdwpo/JyOAlDUbqQmNsZuT50W75ytbTt+I5+KAil1bf/UG0RuGO7N7Ry5KwpXLCAY7guo7PayZ4tq/pkMYo8CSl/L18pnd99z22RfOTHn8mm1X8Zi9MQNLBQ7ZpfB4vYusHLqOlrEXl89PLIboN1HVgkr1n9CWOaEHRqPtPQyL3y48+/Sv9BI2XZotkaWzTEdSNfDnKhuNrdu/dk0vfTZPDw0Za8Kt9MnqoJC1O/+8LoUr5saWPfdefK1Wvyv4HD3ELfIc/SB//rJfBi0q1B/WeCIuogVGCvd9/WTxuwLXJ6dHqnjybEmAf98JPxmgeVLrhBJHmy5hPy06w55mbSV3memQWdH6b9LH36DTbafP3dVGlYv67yehtmlOE6oiLqIIfL623dWeB7MmXS54Z3HY7HKMHpjfaePXHix4snQ1XYvnd79zfmaLeDNmgbGyxp0iRugs5RlW8Mz4unMImubKLzd8r1XIE8DsRvSVUlZuphODE3iJF/Lf1HqlWtZHmxIJDz5lgk8LAJMPzaw74DPD8JkAAJkAAJkAAJkAAJkAAJkEBwCUCAQeg1O4Pw4qugY+4PYUd5pjgKN/6EejOPG6L7eJPfnPMDuULsbJQSVsxW7YlK5kPH/Xe7vGWpGz3GPTzXmHFfSa26jeTVNzq4CTroDK+CX+fMs4xTQYX6crUMGSI8de7eu+daHaXjYF1Ht7fbWbxFILAdDztpmfvdu3flyXqNtdwxesX/+ryr7xrbgUNHSfFy1QT5TeAB5WrLlecPFqV1S5M6PFG5foztiZMnNRGpSKkqboKO3g55YczjF1L5mYJhP0/9xrJAHNVz/vLb7/Jmx3eleu0GboKOPvaAwaP0XW1boZw1FCEWrF94vp7R5uixMIugo1fgeZ7zx0L9UPPM8jU/jtHJhx0In81bvaUtnuvNS5g8q/Qyu+2LL9QXhMpyMtShTWwzeC/9Nne+VKn1nFSuqcLs/WIV9bzxiM7fKf3c+C7jt92bZcuaxTF8o7lvIH5LzONBCIM3aL5iFaRNh26CfG2RNVwDXjLQPdIiOw77kUB0EYgdsnd00eO4JEACJEACJEACJEACJEACJEACoUcA4dDsTBd07Op8LYMgBC8gO9EIZfjEAo+duHEjwk15Qof0OvB+0T0QEiSwT/7uOgZCfOl28tRp2XfgoH5obBf/5T0M2i21OG22xEkSmw+1fXNYpDu377jVR6UgWNeBEF+6YUEe3kt2Bg+effsPSv584WHaEAItnvKYgOCjG0KDebM7dyLa6/fW3OfevfsCrx5vdv/+f0pgCW/l6zPlbUxP9aOG9peMGdJrTfBcfjFxsrRv29JTF5WDKIv8+P2XmocTxKyXW7wpx46HGX0OHT4q+HiyGzdvWqpd84Egmbx5cXn2XKsYae48YeK3Ur9ebaOoe7cO0v7tHsZxoHbwHO0/cEgKFcyvDZnaRrzTz3VHPT/x1G8CngUIVB8NG6gtuuv15u3IIf0MUQ1eHKGYK8Z8Pd72L166LG917iFz5y3y1tRjfSB+p9q3aSktX22qnWfmr3MEXmQIU9mpfRvluZfeEOpxPw8dOiKNm7USeHPheR0yoI/Uql5VUqdObQjI+A6tWrNOWr/VVa4qrzxXC8RvCcYc9fE47VnE34KoGEI89n/vXUmnQuGZf7fwHO7eu0++/naK8qD7LSqnYF8SCBgBeuoEDCUHIgESIAESIAESIAESIAESIAESiPEE4KXjZBBkAmF6Hh67sRCGLRYYFnvN1rVTO/Ohsf+MyuNiXjzbvGW7Uee081St6pLEJL74sjDoNFZ1FZ7HbFu27jAfavtp06Qxym7dvmXsR3UnmNeRInlyY7rrNmw29u124FmiG+5N3jy59EOftli0Rwgz3SAURcaqVCpvLA6jf1QXbL3NoaLKwdS0yQtGs0mTp/kUCrBP986aVwJY5cieTXDsr73U6HlLlz379luOzaEMUTFx0g+WevMBnmHkJdItTy7vnhV6W3+3l1UuH908hUuDGLpw8VK9qdR+qoYW3s8oeLCD3EJmoXPqjJkWzzHX9o/CMcSxqAo64BCI36kihQsqkTKz9mn9enPZsOpPGTG4nxbq0ux5iXsN4ffPP2bK+yos4M6N/8jLLzbUcnKZ80fhO1G5Yjn545dpkbpVvv6WIM9ZVH8fxn8yQj77eJh2Dea/SZg4rqmoYvNBv16Rug52IoHoIEBRJzqockwSIAESIAESIAESIAESIAESIIGYScDJS2e9Cp8WKPPk8WPnwROo88agcRA2a9fuvcaMihUpJDN+mKi90a0XYsEOXgS6Ib/Ksn9W6YeO27ffam3U4Tyff/WtcezPDt4uR04b3XB+u4VB8xvw169HTqDQz2HeBus64OlgXmjds9cqGJjnhP0Nm7ZaiszJzC0VDgd1nqppeFqgyaYt2xxaei5u/nJjSwOEpoouw7Pw7VdjjeGPHT8h/QePNI497WQz5XZCO9djT331uhrVqui72vanmVZvgKzKG0g3eA2cO39BP7Tdnj13zihPmzZClDQKA7STJXNmYySED/Nk7/bpb4TTw6L5mA+HuDX/5MPBhsgLsWPgkA/d2rDAnkCgf6eSJU1q5PrB7+xF5ZGDe2I25CFrp7x7dMEHz+b5CxeN+6y3zZ0rh9SqUVU/9HkbqN8SbydE7p36z9axNMP14uWEqIRwswzIAxIIMAGKOgEGyuFIgARIgARIgARIgARIgARIgARiKAEnQQUiDHLiBNIwplOYNad5BPL8MWCsdm93t+TcqFShrOzctEIQXiltmtTy+y9TtZwfmCoWAzt06el11okTJxKEBNMNgoG3xWS9ret2wthRxgIy6qb/9ItrE+04VaqURnnWLJlk9+ZVcnj3Bjm6d5Mc2LFW1q9crCWNNycqNzo47ATzOvLkzmWZxekzZyzHrgfHw05YinLnzGE59nYwfPD/LE1Gjo4QSywVHg4Q0uzZuk8bLZD4/NsfphvHgd75Tgk6SZMk0YbFAvZLr7bx+RQTvvrO0tb12FJpcwDvIPOzs3vPPjdx0VzvurBuM6RcunzFKE6eLJmxH8idUiWKax4d+pgQwjzZJRVmbMbM2UYTeHCY8/3Au+uJyhWNeoi1vlyr0SGW70TH71TYiZPSpXtfyVmwtMqjVV3yFi0vH9nkL4OA/0rLdpK7cBkpUb6GFHi8kluIRwgn/logfkt8OWfbVq9amiGvEa63Rp2GUrhkFe36n3+xhbR5q5ulHQ9I4GESoKjzMOnz3CRAAiRAAiRAAiRAAiRAAiRAAsEjkCXCK8Ny0hPHLYcBO3AKwxZLQrAhaXWd51+WM2cjvAYQtgf5XTatWSLw3oFdu35datdvIkuXrfCKvl3rlhYvkE/HfeW1j10DhH96qmY1owpeOkNGjDaOzTvx48czH2qh3+BlBMMb6ggZhQXLNcvmy/96v2Np63QQzOvInzePZRphJ05Zjl0PXOtz+SHqfDhsgCCZum7Ip7F+4xb90OctctTojNFp2Kgxgjw80WFNVOgzhHrTbfDw0YLcOL7avIV/Svkn6sgHQz/Utjj21ZC7Y+jA94zmyEHylo24mc7kbXP9RkRoNaOjyw7ynOiWKFFCfTcg2+TJk0nrlq/IzGnfWMabOsN7+Mr3Bw63CL2fjBpsjPHpR0ONffwmjP50gnHMHe8EAv07BXGzQrVnZNbsiHCMmMWn47+yeOKcOn1Gnn6uiSxfGfFiBIT2EaM/s0w6Z47slmNvB4H6LfF2HtSnTJnCaAZPoyNH1UsZJgOLjZu3CsK80UggphCw/sskpsyK8yABEiABEiABEiABEiABEiABEiCBQBPI7CDqBNpLJ9DzDuHxypUpKSlTRORzsbuU69euSwq1UOyLNW8aEZILeUP+XLrcl26WNgi5NmGsNaxTe+VV5CQa6CHXsOB+9949uY/P/f9UKLkEFuEBIsSbrV+TXcrTwlsy7WBeh2v4LW/hhFy9I1Kl9Hz/dLiNGjyr5dXQjzEOEqT7a+PHjNRy0+j99h04KF95yCGjt4vMFqLKCOU5ptvWbTsida4TJ0/53Q/Py7xfpxmhqzCH7374UfbuO6BPx9gmUuHhdLurktR7sxumnDqu+UG89TXXQ7CEV5rZzGKbXo45f/n1ZP3QcYvF/i+/+V46tntDa1Pi8aKa5x2uCZ4/uo386DOLcKCXc+tMIDp+p+zOBoHjxMnTAq9FT4bvEjww9dCP8eLG9dTcUheo3xLLoB4Olv+z2nj+IEojpNwXE60eeB66s4oEHgqB8FdLHsqpeVISIAESIAESIAESIAESIAESIAESeIQJOIVgiyXh15AzBkm2EyRIoN1khPJp16m79sYzBBLdkJfh1xmTtYTbepndFqGazGGofp+/2K6ZxzKEPZs7a4rAY0i3zyZ87TGXT8++A1U4oYqSo0ApyVO4rOQrVkE7RkgihOlZvnK1PpS2HaHCjyVLltRSZj4I9nUcPnLUfHrJkC6d5dj1AJ4YZjtx6rT50HYfScQ/HjnIqMP9bd7qLbly5apR5stOG5WcvX692kZTCHcIexRd9vOUr41nASJUs9fbR9ep3MadMulzy/O8bccu+d8Hw93aoeDylYhwamaBx7axKkxhElKxsB4Vg4hj/riOBQ+Gxs1auRY7Ho9UHhy4r7p9PGqQ5dmBp8Q3k6fq1dz6SCDQv1OeTnvz5k1P1Ubd/fv+P3uB+i0xJuHDzvfTfrK0er9XNy1U6KB+vS2eh5ZGPCCBh0yAos5DvgE8PQmQAAmQAAmQAAmQAAmQAAmQQJAI2IkpTnlvgjSlR/U0+fPlkZ7vvG1c3tr1m6RyzXryx4LF2iI9wvq4etng7ej26uNk73btYKka/ennlmNvB1iYXjjnJ8si3T8qZNCIjzznfIEXwY0b9ouYCNPzSsv2FlEIIdmerFHNcTrBvo49e/db5pIxY3rLsetBtixZLEWHDh2xHLseIP/N7J8mW7yWBg37SFb/u961qcfj51QOnf59exht4BHQqOnrfgtDxgBednp06yR58+QyWr39znuCvC/BsFFD+0vVyhWMU124eFEavuT87JtDGEKY9GapTXmgbt6yf3a9jaHXQxQyf5DfCCG31q7bKG06dNO+z+Zwb3o/py3u64efjDOqIXIWKpjfOH5/4DBjnzu+Ewj075TvZw5cy0D9lvg7o5NKuB4weKSYXzZIljSpvN6iqRYqdN7s6Ua4UH/HZnsSiC4CEa+mRNcZOC4JkAAJkAAJkAAJkAAJkAAJkAAJxAQC6yNi/hvTCfM9d4bRhzteCYwc0t9ogwVhLM6bDeGqXm/7tpQvW1qmq/wpuudMdyXcTHAIe/N0rQih5OixMDl2PMw8pNf9X2d8J0jGrtvOXXukmUruHQjrO2CoLF88xxiqfLlSMnvuPOPYvBPs68CCpdkQUsuTZcmc0VK9/+Ahy7H5ALkoFs79SYWiiwgPhlBp/oZLw3Mw7pMRxtBYXG3VrrNs37nbKAvkDkRHeJLphpB0uXPlMMKC6eXFixXWdwVhzPSwYWiPUGmRsXc6t5emTV4wuiJ/zFP1XhSEJnMy5DkqXTK8FqKhN0uePCJk3sWLkReqTp85K2UqP+XtdH7X4/no3KGtpDKJTxjkeNhJmfP7Ar/HYwffCPjzO+XbiIFrFajfksjO6OvvpsqW7Tvli09HCbxHdcP3HvnfIOwgJNvgER/rVdySwEMlQFHnoeLnyUmABEiABEiABEiABEiABEiABIJGgLlzgoa6cKGIN++3bt/heN5/122QseMnCha6YRAHkOfk7Nlzlj71n61jEQ6+m+LfgvqkLz81ciZgYHjY1G3Y1HKOqBwcOmwNcZY/bx7b4R7WdSC0mB4Gr6yuDtjOUORJk3iGJnv2uud4QTk8Rv6aN0uSm0LNzZj5m3ww1JqvCG09WeGCBTRhT8/VAkEHXjN/Lf3HU7co1VWuWE4TafRB8FZ+7+6d9UPHrd7mjvLeioyo0/LVl6Xb2xEh3nBfnn62iUA88WTmEHrgVLxYEUHOEjtDfcYMEd5YZ89Zv0t2fR5GWX/lGTHmwyGWU/foEyEGWyp4EBACvv5OBeRkfgwSqN8SP05p2xTeZ6WViAkR550ub0mNalWMFw7QAd6kE1TuKNe/T7aDsZAEopkARZ1oBszhSYAESIAESIAESIAESIAESIAEYjEBu5BvsQBHksSJjavcu++gsW+3s3nrNktxPhUSy3XRrMObETk7EL7p62+nWPp4OhivPECeqhnh5YPcPrXqNpZ79+576uZXHYQos8GTyM4e1nUcOXZcEOYKli9vbk3ggaBgZ888VdMoRqgtO48oCER/zftFzF4/v89fJO/27mf09WUHHjO//2LNcdSn32BHLydfxoypbV5+saEM7t/HmB7Y1nvhFTmq7o03+3PJMsNLCG3ffKOFEr4ixjL3f1aFsdOT06N8jZ9h8MxjRef+rNm/S+OGzwlCbsF27dmr8lOtic5Txvqxff2dCiaoQP2WBHLOyG/1Rrsu2osEQwe+Jy81bmAMj7xfwz/81DiO6TsNs16XZjmvSq6kd2P6VIM2v4PX4sm0w8lk9vEkQTtndJyIok50UOWYJEACJEACJEACJEACJEACJEACJBCLCVy/cUOSJglfMClX5kHcKAceNZ6oYqlxXeROnjyZIHm2bmvWblA5PnxboPpo+ECBd4xux46fkBp1GnoMdaW39Wfb+a02lubLV6yyHOPgYV7HqNHj5IvPwj1oEE6oc4c2Kq/JeLc5Pl68iCX00G82obDixYsnf/4xUy3GZzb6I2RWh669jGNfdnJkzybzfp0m5nBiPd4bKNN/+sWX7lFqs3nLdrecTnYDZs2cyZLvRc8DdeDgYbfmWJx2EsoaPFdXkEdHN7R7pkFT2bvP3gtKb6dvkZMKeWx0D5zaT9bQPKXscj11ah8RVg4C6Ed+5p7SzxmMbfNWbwXjNDzHAwK+/E4FE1agfkuia84Iifhu7/5Ss3pVSf9AuH+yxhMhI+pA0GmV67pkuJJf0l9OFV2YQm7c+3EuSutc4bnmQlnYoagTco8eJ0wCJEACJEACJEACJEACJEACJBASBMpGJEK3zNcut4+lQegfYNG7eNHC2oUgVwnCVtm93VyubClBSCrdkKgeeTXMhjwmECJ0++SzL/Rdj9shA/pY3rCGWFSjzguOC+92g1WpVF7q1XlSfpw5W7ZstQ939cZrr2gJtfX+WEjXF//1Mmwf5nX8sWCxXL58RVKkCM+10qXjm3Lw0BGZ+etcY4rwmvn1x++MY4RBc80fgdBeC+fOsOQmgsdFl+59jX6+7GTNkkkWmXLx4Fzw8vlpVkReIl/G+frzT9SCaxW5rbxePh77hZbzwpd+m7Zs03I6eWvbqMGzRogwzBF5oFwtT+6c8vOUrzUx7IwKo/Zi89ZiFn3qPF1Txo4eajzDWCh+5vmmsu+AZw821/N8NuFrGdSvt1acJEliJazNkmpPP28ROD9X+UCKFCpgdF32zyqxE36MBtx5JAgE8ncqWEAC9VsSiPnC4/DsufOC325Xw8sJadOkNorxwkKoGDx0IOhkvE9Bx3zPwOO/K3mVB9PekPbWoahjvqvcJwESIAESIAESIAESIAESIAESIIFAESjjIOoEavwYPA6EFyy46wZBA+GnVq1eK4dVPhu89Vy65OMCIcFsE23CqplD31y5ek1WqjG8GXIivNY8QixC+0yZMsquTSu8dZVWKuzO38tXau2eVGHbMA4+CJd18tRpOaPy/SB0WwKVsD6PChVnzimDTgOHfChXrlx1O8/DvA5M5tPPJ8r7vboZ8/pk1GDp37e7HFOh4pAYHF4gZvFsxap/3cLg9eneRVzzBdWvV1vw8WR3796TAo9XNJpM+uJTgTChG847YnA/7aOX2W337j8odeq/ZFSVLvW45ukDbx+EJEMi82Bbr3ffNrybwBHH7Tp1N6bx+ZiRFq7w6IEw5s2mzZglfQcMM5p9+/106fnO28bzlj1bFtmzdbUg387dO3clZ47smveO3gEiFJ5F2qNPIJC/U8GiFajfkkDMF56HELyPh52QTZu3CcJVnr9wUSqWLyMVypYWCFC6zVACfygYXoPIqUKuZaCHju3tyqSEnTghHpKOoo7trWUhCZAACZAACZAACZAACZAACZAACUSBgJOXDoZc9+jnjVi4eKmMGfelwCNEt3Rp01hCoenl+nbBoiXi6oUDcUYPe4N2s+fM05t73CZ5EPrN3Ci+Chvmi5nzAZnbQzjIni2r9jGXm/cRhuybyVPNRdp+TLgOCB4JlaDQo1tHY36pU6USfFxt+crV8krL9q7FkixZUrcyc/g0t8oHBQizZLZEiRKZD7V9X8ZJk9o6V/Mb9L6GMnM7cRQL0qe15lNyPY4bN67lDBCwfLlWu/sCz5yFc2YY3wk803quJPNJIEA2fa2d395A5jG4H5oEovI7FcwrDtRvSSDmDNEGH2+/7/DAmzYj+sNDBuKaOMajTyBCanz0r5VXSAIkQAIkQAIkQAIkQAIkQAIkQALBIeDkpRMLQq/pgJGz5dU3Osihw0f1ItstvF/adOimfVwbvNPFmnPj47ETXJsE/NgcXuenmb8JPFaccqXoJ0eunkbNWjnmlXnY16HP89PxX0mnbn3k+nX7EEJ3VK6iyVN+tBV09DEe5vaO8kjRrXDBAhYPmF99FPz0/r5sr12/bjSD54udwQNKr8MWx4GwGzdvuQ1zVnmJVaxeV5ATSD+na6PTKgRcrbqN5N91G1yrHtrxPZuwVpGZjPm7GZn+odTHn2sN1O/UzZs3DUSBuGfw0HuYBnHTF5s8dYZcvHjJsSnCss2dt0hqKy9BuxBtjh1ZQQLRSOCxQqVr2P9VisaTcmgSIAESIAESIAESIAESIAESIIGYSeDq5Qsxc2KhNKv6jUSyZLOfsQo7FRsNb0Aj3Fqhgvkka5bMEnbipOzctVc2b93mKPrgzel929YYXg14S7p67QYPDV8ZFeqrSKGCKsxVNsmgQpUhVA/y7KzfuFmwkO5kMe069HnC6+WZ2rWkmMp9hBBsyLvjTYDT+8aEbYc3W0mfHl20qUCMyle0/ENbcE2ePJlUrVRB/lm1xjb0XnTxQk6q2rVqSNx4cWX1v+tk6bKVXgXI6JoLx40ZBCL7OxUzZv9wZ1GwQD4pW7qE5MqRQzJmTC+nTp2R7Tt3aeE4L3gQfR7urO3PjvBrq54OkwwnKtk3YKmczrxKKi7KEmNIJEsRkbvJl0lZ/V996cE2JBBFAqUeL6oSYTaRtOofkPfv/ycn1FtZE77+IaT+8RhFBOxOAiRAAiRAAiRAAiRAAiTwKBKAkFOmvLOgE4u8dFxv71GVowCf2XNda5yPGzd8zhB00MourJlz78DXrN+4RQk4W/weOKZdh34ByBkx9cdZ+mHIbSFo6Pbv2g0PTdDBHJBDad7CP/XpBG27dt1GwYdGAjqByP5O6f1j83b3nn2CD40EQoEARZ1QuEuP0Bwrlist3bu0s1xRPpVYctTgvtKt90CByzot5hNAHOY8uXNKnlzZBS65uG+HjxwTJC0NlCFJXX71bGTNmkn7B/JRdY6jR8Pk1u3bgTrFIzFOdN+LTBkzqLcR80viRAllz76Dsl+9HUl34+A/Onjzr2D+PJI1cyY5rUIu7FX34uy58wGbSNKkSbQ3TrNnzawtnCD574kTp1SCyLCAnQOxw/FWa4F8uSVunLiy/9BhOXjoiEo4fDdg5+BAJEACJEACJPBQCUDQgYeOJ4sFuXQ8Xb6/de1av2Z0gSfGD9N+Mo5DaedRuY6YxrxQgfzGlMYGKOSZMSB3SIAESIAESCAGE6CoE4NvzqM4tddeedH2srDY1/q1pjJw2Me29Sz0jQAWflsqxjmzZ3XrsFO9bfDN5B/dyn0tyJUzu7R7o7kScnKIa6JHfQzE9D13/oL8/c9q+e2PRXLtWkT8Yb2Np22VimXl1aaNBAlU8UzY2b179wQCzwKVePXPpSsiJTAg/EHjBnWlfJmSbqe4fPWqDB31meA8gbIK5UrJiw3quQ136fIVGfrhZ35fQ3TdC9zXJi88K+XLlhQkHEWCWLv7AGHt33WbZPxX3wV0QT4qnDJmSCfdO1sFYzfgXgpw7wcNH+OllX11pQplpH7dp8Q1+S6+E2M+/0aF5jhp39FDafWqFaVFs0aSMkUK2/uAsRFned7CJfLjzDmOMbWdToF73KxJA8HcIQ7a2V21eLJ3/yGZ+N00Tbi1a+OpDL9JXTu01sKseDrHUSUejZ0wKaAikqd5sY4ESIAESIAEAkZAD7HmyTtHP9mc0PWI0C8hmFuEBiuQP69xylWr16p/o983jkNl51G5jpjIG+GRYMh7g7xHNBIgARIgARKILQQo6sSWOx1DrjN+fOdHLtODf5DFkKmG1DSyqbfr27RsJkULWxNFmi8iW9YskRJ1sFjeq1sHyWEjFJnHxz4EAAgyjZWAgc/AoR/L1h27XJu5HZcrU0Lat26hFq+Tu9W5FkB4yKXe9m/3xquaENj09Y6uTRyP4Y3wevMm8kTl8hIvnvOzmDF9Ogk7ecpxHH8qkidLKt06tnE8X+pUKTUhzJcxo/NeQFSt/WQ1R8HOPD8szoMhRJhvJk+XxUv+MVdHaj+qnPB85laCY1TMXyEPz3vD5+pIg2drSzJ1n52sWpUKMu2n2U7VbuUFlEdO1w5tJEP6tG515gKcP2mSJPJiw2c1QWnKjF9VHPS/zE0c91s0ayzP13vaViwyd8L3pLCKe//R0P/J8pX/ypjxX5urHfchnLZp2VSerlXNp3Pg3o0e3l927t4nn4yb6PN3wnECrCABEiABEiCB6CRQtoJI5gcvUemijrfzIexa2DFvrVhvIgDPnM+/+tYo+fHnX439UNp5VK4jJjLfs3e/JEgQX3757Y+YOD3OiQRIgARIgASijYDzqma0nZIDhzqBDm1f0xKHuV5Hu8691Vv7d1yLLcfxVPI+J7tx46ZTFcsdCJQuWUzg/ZRNJRqNDsuaJZOMGvK+JIgfP1LDQ0TxZjWeqCQd32zpdeHXbhxPwoy5Pa4DokXxooUidR7zWP7u9+vd1VHQ8Wes6LoXuVUIvb49OkuqlCn8mY7WFs8FxDh8d1esXud3f3OHqHLyRRA0ny8q+0mSJJYWTRsLnl1PQnVkzgFBZ0i/nn4/pwkTJpQ3Wrys/QorMFwAAEAASURBVJ/aX+cscDw1hCCEu4S3l78GIS9N6pTSf8hor12H9O8p+fPm9trOtQEEpNHD+8mbnXox1KIrHB6TAAmQAAnELAK+ijmYNTx0KOj4ff+Qo2TYqMh5Uft9smjs8KhcRzQiivTQzzz/cqT7siMJkAAJkAAJhDIBijqhfPce0twfL1ZYUqiQOq4WN24cJeq4llqPN23Zob3hby0NP0LILppvBOrVqSWNnq8bqYV4384gkiNbFhkxqG/AF63N53+qZlVNFDCXBXIfzyo8c3zxMgrkefWx4MERVe8RjBWd96Jn17ei/By93b6VbNm2M9I5lQLBCSHKotvgKdXm9VekZPEifosuvswN3kr9+3SL0tjNX3pB9u8/7Ogh927nNyMl6OjzL1q4oLzVpoV8PvF7vchtC6E5MoKOPhC8j4YO6CXvvjdIL+KWBEiABEiABEKTAISc9SokFAWd0Lx/nDUJkAAJkAAJkAAJxFACcWLovDitGEwgKounyJmwW7lIuxrC+sz+faFrMY9tCLzU6DntjfzIeFbYDOdYNKhfj2gVdLBAjhBq0WUZVAg1eH88LEEH14f8QIGw6L4XUZ0jPKa6d4lcPptAcUqRwl1ojup1ufZHeLBSjxeNkujiOqb5uJMSx5zyzqDdxUuX5FjYCbms3lp1Mnji9HznLdtq5KyqWK60bZ0/hbWqVxHcNzuDFxPCujkZcvScPHVaTp856zFvVU4VYvE5laOIRgIkQAIkQAIhSwDh1uihE7K3jxMnARIgARIgARIggZhMgJ46MfnuxMC5YcExKuGG7t+/L30HjhS8kZ45c0YtLNWBg4fl5s1bMfBqY+aUkKsiug3CEd6Wd7IbKjk7hLjtO/fIlu27tIVohAfDp3SJYpI/X26P/TEu8sxgAdrJTpw8LWvWbpBt6hwQApFgNIt6ZnKq3CllSj2u8upkV8+Pczi/YHBymjvKB/Z91+P1eeprrgvGvTCfT9+/evWa7DtwSE6ePqPd3xLKOwX3wMnyRDKfTaA4JU9mL+rcuXNXE0Oc5m0uP3j4qPnQbd/T8+rWOBIFRQrlt+0FIWTAsI9ll8o3oxvEpV7vdLAN7Zc4USJJrrwpEerDbM1ffsF8aNlHPqGZs+fJyjXrJHXKlFL9iYpaeDlLowcH4ADvrPc/GOVW/VSNqm5lesH6jVtk5CcTDDEHf0t6v9NR8GzZWbnSj8vceYvtqlhGAiRAAiRAAjGXALxyIObQSIAESIAESIAESIAESCCaCFDUiSawj+qwBQvkDcilXVELxlf2HgjIWBwksATiqzwpCO3mZLv27JOBwz6x5E+6ohqfPXdeNm/dYSRqx6Kyk1hXSOXNyOch38ZvfyySyVN/tkzh+I2TcjzspKxdv1l+/jU8EWbaNKktbWLKQVsVoitd2jRRnk4w7oXrJLft2C3jv5qseVK41r3zdlupXKGsa7F2nDhxYoGQBuHWVwsUJ5zPKX/Ths1bZZQSEmK6ZcqYQSDG2Nn302ZaBB202bhlu0z8bppj+MIKZUvK4iX/GMPBcw0fO/vvv//k7e79jHt+7PgJLXzbwUNHpJXK02NnBfPnlbhx4xoCjd6msvIGsrNr16/LsI/GWaoguA0aMUYmTfhICf3uotzD8rKzTJIHJEACJEACJOCJgDmsGsOseSLFOhIgARIgARIgARIggQASiDGiTrasmaV4kUJSQL3hnylTBjl/4aIcOHhEe0N/t1r8v+MtWUsAoXgbCguXmCdyC+TLm0uQoBteBfuVx8muPfsFC2FYJIusYXyMi7e2M6pFuLPnLmghd/YfOKwtnEd23ED0K6iSeMcUy5A+rRRTzwzmlD1rFpXP46rmWbB9117Zre4D3jwPtCG0UPGihaRooQKC8EBYwD585Jh231crr5KHaRBV/l2/Seo+XTNKHiJNX3ze9u1/XNu+/Qdt3863u25XLwFzm7Ytm5kPLfszZs2RGbPmWsqcDs6dv+BU5ViOxeUtW3dK0SIFbXNDOXb0saKQEj5rP1nNx9aemwXjXugzOHLsuIwZ/432POtlrtuPP5soFcqW0hbzXetwXEB9F83eJHZt9LJAcsKYyZLae5ZdvnxVP2W0bOFFA08yeDF6Eiq9nbxwQWfBfKlDvjGINghhaOdBlCNbVsspa1WvbDk2H8DrDuHQXO33BX9JTRVqLZf6rXM1nPPJGlVk4Z/LLFVOwtGZM+cs7cwHEGoR0s3VkiihkEYCJEACJEACMZLAOhVaDR8aCZAACZAACZAACTziBDaoF/MXXbgkm69elxtqHbSEir5UTK2PlkyWRPIltn859RFHEiMu76GKOokSJZQ+73bSxAu7RSksHsIgkMxRIVhc39xHHd7wTZY0KXbdDG/74w1nV8uTO6eM+KCPa7F2jEX6l1t2sK2D8PR+z862b+AXyJdHqletqPW7dfu2jP/yO1mxep3tOHaFKZQw1KNLe8mTK7skTJjQrolWdunyFflbLfD9/Ovvcv36DaPdhDHDxM5rAcJG09c7Gu3MO6+81FBeqP+MucjYH//Vd7Jk2Srt+InK5aVcmRJqbjkkfbq0RhvXne+/GuNapDw1bkqLtl2NciTQru+QJ2HM+K/ln1VrjbZ2O3gzHIm+y5UuYbuQibBcuoHVqE8+1wQXvcxuC/Zfj3MPI7RXCRjvDRihdUGOiJca1Rc8s64GkQe5H5DvYsTozwX9gmH4Xpw4eUoLgzZ/0VIjSX3tWtUcRRlf5oV77WQTvpniVOVXOb5LdnZbibc//fK7XZXfZf/9F+Exgnuzdv0WmTt/sebtg8FGD+sXcFEHz2ffHm/bPpt+X4DqEIx7MW/REjlyNEw2Kc8Pb4Zn7saNm5JM/QG3M0/5YMztA80JYzt5uVy6fNl86oDs31C/a/BomrdwiWzZtlMb87VmjaMk6jz2WBzHuaVMnlyuXbtuW3/37j37kJgukQ0RutDJVnj43V26bKW8/upLtl0RAs5V1LFt6KXwwsVLXlqwmgRIgARIgARIgARIgARIgARIgARIINgEfjh1VnoftIaqX3vlmjaNaimTy9TC+Ywp7VHrRSOOhEm9tKmkcbqoR68xBuaOLYGHJuogWXPnDm9IAhXqyZtB8MGiOkST/kM+EoSG0Q1vKjsJE+gzRwk7Fy9ZF/U85fL4Zc58fWjLtoVasMN4duKTpaE6wMJmt05tNfGir8o54M1jpF6dWtJSiR1Y6PRm8ArCPOAFYZ4rcovYzc2uTD8Hkps71Zvn0r5NC+2a9H5OW7ux4rhcU3wfz2l3jseLFZYeXds7Lt669gGrwf16yrIVa2TshEkevafs5p47Zw4pqRYtO7V7XVKlTOE6vNtxKpWHYuiAXtLp3ffl1Gn3t97dOkSyAG+1YxH0r79XKA+2u5Ecxb4bOGRWYaDs7MjR43LIS84Ru36uZfDIMz9f5vr5apE8Kl5u5rFwD8Z9+a1s3Lzd7TfA3C6Q+z3xfNp4F+A3AB4dngRb13kE417gnL/9vsj11B6PE3t4CwM5eHyxQHLSz+c0r0uXEBwwMPbRmC/knPIiDcT3wHVGR4+FuRYZxy1eaawE4/HGsb4Dz0GnHGfw7DSbJ0H++IlT5qaW/UV/LZeWzZvY/q2wC4+Gv00pVOhFV4NnI148sPOug2eqnV11ELLs2rKMBEiABEiABEiABEiABEiABEiABEggcARO3b5jCDotM6aTxunTSLaECeTgjVsCASe+WkM0235VtkB59BRIQu8dM5fo2nd+NTi6zqjGbdXiJenepZ1Pgo55Glikh6eM2ab8+IucOWsf1gWLoj27vmVuLlVUvP/MKrybnZ08dVqm//ybW9VHQ/8nDZ6tbbuo5dbYVIBQPB3fbGkqcd8dNbivvKFyFjgtcrv3iJ0lSBbfr3dXnwUdM6VqVSrIROWJ4y9jLJbiefNF0NHPh2euT/dO+mG0bBHmb8HivwMu6GCypUsUc3zOAxVerqaHMFDzFy8NKDN4m7mKugE9gWmwShXKiNlTzFQlX307VeCF5I8F4174Mx+0xTU6fY8gXDl5k5jPE2hO+tgJlJhuZ4H0Alm/aWu0CDqY9yEVxtFJ0IRnYuMG9dwur7K6H3aGcRCK0VfLnDG9Y1N4nt5RgqSdpUqV0q34WFjESxfmSvw2jhj0nu3vKbxA7Qw5umgkQAIkQAIkQAIkQAIkQAIkQAIkQALBJ7DyQTj7dGp9tH/ObFJaRW3JoJwzKqRIJi2UyNM0gzWa0ykf173OqBfUEdItTK03+JK85Ipab9qkQr+hPS2CQNBFHcTbr1e7VsQM/NxD8nF4qpht0PAxjothyPEAbwvd3nyjub5r2WIRbMDQjy1lOKjzVHUtd4pbhY8FCF2WNUsm29YIf5bbYTHLtkMsLYSo0uSF56J09RAE32z1SpTG8LVztiyZffJs8nW8YLYrr5KrO9nefRFh5bBAmyNbFqlZrZI883QNyZUzu1M3t3KnvEz4Dp425d2Ir/5Q4PuL8Ws8Uck27KHb4A+pIKnK59K5/Ru2Z9+5e68lYb1tI5vCYNwLm9N6LGqufrOcDOHIvFl0cNLP6RT67fqNG4L8PU/XekKFUHxOE6b8EWr18aN7i7xxnjydmjVpICOVKKKHLoSXTiv1QoCdrVyzXguTZ67z5AlUpnRE2EpzH33/jno7x87iuXhios3UGb/aNdXKwB2hQnEf8BsCQ9i65MncPXvwezDxu2laG/6HBEiABEiABEiABEiABEiABEiABEgguAQSxgn//+037/8n9z3IL+tUOLa6W3fL+wePaRMce/yUZFu9UfvMOHPemDRy8lTbtENKrd8qz2/bI+U3bNf2FyrvHrNNPX1O6/vB4ePSZs9BKbx2izy3bbfWvtbmnbJfeQrRRIIefq2H8tDRF3Ncb0DYiZOCxSgswmXOmFElTq5s+1Z4U7W4hXw5uoWpvCIIRdbo+bp6kWXbtUNreb39O4Kk40mT2CfTRu4dJJo3G95IRw4YJ9u6fZds3rZDHlP/K1K4gCC/gKvhWruo8/d8f4ilCnlc4P3jiyFsE0KlxVaDV5fTMwMmWPy7ohRe5LvxFM4Pibhnzv7DIhz4wxSeCAgHhITsTt4K+ngF1SKynmtDLwuFbZrUqRynie8ZkrJXrVzO1mMK9wE5lOBF9IPyoHMyLEbbGb73EIdeffkFKVakoO0zj3MgR8rEb6dLoDyH7Obib1n/Pt1sw2DBy2HwiE/9HU5rH4x74c/EkDcqk0NoPowzxcNivn6e6OCkj40QlHYGDz87w7MELx54USGkYUywcSoXGzxDnX5fkA/ukxED5OChI5JchTizE7LgmfbZF9+6XQ5yfeE30M6ervmEwOsV+ZLsDDmEIMi5Wpw47u+FHA87qUIebpNSyuvPzvC3DPnJGjxXR7YoT5xyZUraNZOvJ0+35I2zbcRCEiABEiABEiABEiABEiABEiABEiCBaCFQ/UEqiqtqPfSF7Xuld44sgjw61qBr4aeGN0/mBPHlhHopFNtCD9b+0jxYq9l9/aY8q4QZWLnkSaW8WtNACLdFStB5Y/cBmVhAvdSdJjwayH8PBKQvT5wWjIvQbwnU+sPccxe0Po137JFVpYpKYps1ifDZxI7/BlUpKFOyuKNnys+//u4W+gz5cMaMGui2oI+F++TK5QsL+brh7WB4xdjlDUBS7zavNxMsXNkZcoWYRSK9TfvWr9oummExsP/Q0bJj5x69qSYqPffMk7YJpe3CvfXo0t5x4Q6DHlaheMZ9NVlbvMP5sHiWJ3cO9cZ5PsmZPassXvqPce7o3un0zvvGXN/r0Ul5aGS1PWXXXgPUor5VLfWWT8h2IFMh8q/gmp0Moho8tXSDV1afdzsa89XLsYUwBO49XAQ2cxu7feTPwBvju/bs16pxL+Ch0r51C7dnU++fL0+ukBR18L1yss8+Gux4vegDvsgn07D+M1JD8RmucoDs23/IbTinMFkQXD8c8r5be3MBzoHcRRD69uw7IMM+GqfllzK3CfY+PAedwkd9+MkEgbATGQvGvfB1XsiF0qJpI8fme/YekAMqLKAniy5O+jk9Cb96G/MW7SGc9erWQZv7iI8/t833Yu4T3fvIFzdqzARtTp6ux8nD87LKZ9Ot90AVmtHds2bHrr2O04eIhBCVX02aKqvUixV4ZiG+IuwbPOVw/+3MaY5DRo3VxCnk0XEyCFJOgg7+PTB/0VKnriwnARIgARIgARIgARIgARIgARIgARKIZgJJ48aRYbmzS5+DR2Wresm9+c59kke9UP+KElleU58kD0SVskqk+b5QXhl6JEzGh52SF1XunV7Zs1hmN/TIce24cbo08nG+nKK/IvqZ8uoZfjRMBimvnKdSp5B4aq1GtwpK+JlSOK8kenCerlkzSfXNO+SsCt8GDyCIPbHZdIZBYVDfJWyaftITJ+1z2cAzYPfe8IV0va2+hSeEqw0c9oljGLZnnqphu9AP0WHAMPewaxi7YrlSrqfQjhcvWW4RdPRGc+f/6SZqoC5RwoR6E22LBbTCBZ2Fitm/L5R33xukLTRC0IHdv39fWyCfO2+xSv7+XVAXsvE2O7yY8Ll79542H7v/6G3M26jms2j4fB27U2lleFvdLOigcNOW7TJw+CeOfbAY6rQQadcJb7d37zvYEHTQBvfiz6UrtHPZ9UFZQpU4LBQNAqiT+cMNwsuwAb1tPTs8eVM5nduuvEC+PPKpjehr1za6yhBOsoUKH2VnK1atlY3qeYysBeNe+DI3/F6NGNTH9vcT/fEbOmz0OI9DRScnjyf2sRIeMONGD9ZeFvCxS7Q1W7dhiybM6L/9vp7ohvJ0a9Oxh+PfBnjQnL9w0XE4iCyd2r0uU74ZKz//8IVM/vITebt9K8mvcsP5893XT4C/YUuWrdQPfd6OVy8z2OW283kANiQBEiABEiABEiABEiABEiABEiABEggIAeTOWVKisLykhBrYAfUy/2AlwFRXYdT+uXTFp3PcU2vbf168rLVtnyWDIeigoFWm8By/h2/dkuO3rC9FZ1AeP7qgg7YplddPywc5gTeokG+x3YIq6mBhz87+WPCXXbFWdkAt3NsZFppc7eSp0zLrt3muxR6PEXrn8mX7hzBRokS2fb+fPsu2HIVnz1tDuKEMC2JZMmXErmZOb/WjEgtzCAVHCyeQKUP4l9uOh1OIL3hQXbx0ya6LVpZDeTr5ap4WVvd78Uzw9RwxqR08bQJleO7tQl9FZoHYaU7IxdGh7WtO1dFe/sH779oueMNj4pPxX0fp/MG4F75MENcIkc7Jvv3hJ0chQe8TnZz0c0R1i7Bgds9rVMf1tz/yxr3Tqa3tc+VpLDwvkyZ8JGU95MeB91wgzdPvY8P6daRalQp+nw7f597vdFThDOP73ZcdSIAESIAESIAESIAESIAESIAESIAEAksgf+JEMjpvTtlR9nEZlCubJFMv/yLMWlPluXPJw8v/+ixOmaKJ5FKePmaDN1DOB84QR1xEHXM7fT9novCX6I8oESi2W7xgAkip4u7ZWfWqFaRMqeJ2VepNf/tFfafF/mk/zVZh2CpIhvRpbcczFyJ80+Il/5iLjH2Em3FafMaCm5OlTmW/+Ink1vA8gtl5GenjLVm2St/lVhFwSmh+W/0gbFb5GJxs9dqNAu8sOyuYP48W3s6uzp+yK2rh3ldDLpoaT1TytbnRbsasOVpoP6MgmnfsEp/bnRKLuU7fD3N7fA9fbFhPfv71D604voqF6avpC8bezlOzWmX5XQnDCJMXTGv9WlPBAryrYd4fKG8xff6u9b4eR/e98GUebV9/RQrmd/eK1PuuXLNO5i1aoh/abqObk+1JVSH4w7Pw7r27WhN4iDnlqtHHgCcfcgfBI/JhGP5+jBk5QHn6Wf+Rg7ngerx9FxDCEILIt1N+sr0GhMiDB1mVSuUCcnnwWrSzN1q8LPXq1LKr8uk6IEyN/3iIdOnZn3l1bCmykARIgARIgARIgARIgARIgARIgASCSyCF8pSBZ81zaVNLqfVbtZMvVXl9G6jjYFl4PCvRcuwE65wx9Ty+r7BG8QqwGGWX1BnD5rPxuvF2uusOCZ3RDwuqYz/8wOMC2B0Vf2/wyE8dT1PIJryb3hh5W/y1q9evG108eeocUrl0aOEEtGfGZnETtdev3/CIKexEuIBm1yivCrUUbEulBE1/BA19fkiGHky7c9c9F4d+foiSfy9fLQv/WmZ4ZkB0Q24hT94BVVWuK13UwffOybBojXB3Cxb/LStVXg89L0juXNmlW8c2kiVzJqeuUksJO998/6NjfaAr4CmIXCN29uvcBQERmKL7XtjN3VzW6Pm6Uuep6uYiy37YiZMyeuxXljLXg2Bw0s/53oARytPxplxQXnrY2hnEeghVeInASSCpV7umrSBiN14gyyA4jR7ez1bQQa6yDu/01XIAtX7tZS3PmNP8MafXmzeRa9euqfBn7i8JfDxuohxX967JC885MvD1uu7ZiDrILeck6PymQotOUfnvGj5XR31qazm4nM4FgQvhFdt07OnUhOUkQAIkQAIkQAIkQAIkQAIkQAIkQAJBJpBevbBdPGkSLc/OPof1F/OUMpoicRxW6xuFVA5f3a6rdQWEXoPl8CGVxd4H58vlsF6sjxsbtkELv5Y1i/OCbGRAX7nq7CWBMGy/zJnvcdg58xZ5FAZy58zusb+/lZeUcqkbvHac7OBh+3BzTu0f5XJPz8xVtWDpyU6dOuNY7Wlcx06xpOK2cp90ss7d+8nM2X8Ygg7aXVTP9XCVT+W3PxY5dZN0aazeLE4eLMi/hIX5v/9ZbQg6GPTgoaPSuUd/OXI0PKma3YmcEsfbtY1qGRbf3+/V2XZBHPnBpvz4S1RPofUPxr1wmuiTNarIKy81dKoWhJfr1W+YYz0qgsVJnwTCIUJ4dBJ00O7c+Qva8woPOCdLnSqVU1W0ljdXvBFO0NWQs6jn+0O0MKHwRvvfoA+lfZc+snmbs6cixoC3jJPNmDVX3ur6nuw7cEhuObgswxsS+X3wvXf6zl5TiRLNFkclL2z+ciNzkbG/Zt1GmaxCi+J68DvSom1X+eKbH2zz0OmdEPYPnlM0EiABEiABEiABEiABEiABEiABEiCB4BK4eu++nDaFTtPPvkKlMdn6YD2ghBJ3dEuh1stgh25ac+PEVY4eNVOl0Oq+PHFadG8bFHx7Mnz9NrPKn5PVRdRBiLWLpvBuJ9Wa5bcnz2rjVE9lHw1Mq4wl/wmap04cdQMDaUePhXkcLr1NWCRzhydrVBWEanNarHpMLU4F0syeI/GUu5qT3TM9rE5tWO6dwH+Wnwjv7dkinMBNhwVe1KZIkdwx/9TkqT9rnisIceVqiVS8TCz26qGasLULg5XESz6fER+PV8nsh7gOrx1nzpTBtjw6Cp99ppYgzJWdYR4/fT/BrspWBELDLz4dbvwOXbp82fBMCMa9sJtouTIlNO8ruzqUYSG/S49+HsUTtAsWJ5zLX/vpl9/l+XpP23qKwKMuiXprxJs3oL/n9NQeXjfPKA8hO9u6fZcRulOvhzg1aPgYKV2ymPTq1sH2+4QcOxXLlZbVazfo3Szbs+fOS+8Hwhy+jwXy5ZGMGdLJEfW3FeKR/n1FmZNX0PGwk5Yxm774vKNH4rgvv7O0xcGiv5bL0uWrpH+fbpr3kVsDVYD79LDC4dnNh2UkQAIkQAIkQAIkQAIkQAIkQAIkEBsI/Klevu6495CWQ6d0siSSXXnHrFUv+e554C1TQpVVUxF8dHsydQoZfjRM5py7oHLt3JWEai2wbppU8lL6NPJejiyy5OJlmXHmvBxVuXNKJUsqB5TXzvzzF7XuA1Wunngu2sHmq9el4sbt8qQShBKr3Dtzz12Uq+pFUXgJPaPGje0WWOXCA82jx0841t66fVuuXr3m12fHrr2O4yF02hNeEjSnVAvUb7Z6xXEM5B5wMn/nijBFZvHo9OlzTkNLMD0OHCcRQypcFwzN00pmUoLN5fp+pozOi/yextX7x9btqdPhirfd9WfJnNGu2CjDIrGTpU8X4a1z1eXtfr2Pt/B0mBve8rczLMIHy+yEK/O5sQBu9zG3cd3X28ePFyGKBeNeuM6jeJFC0rPrW46L+Ddu3tTynFxRv9feLFicvM3Dqf7MWefntXCBfE7doqU8X55c4sTLSZTBRDZs2iZfT57uOKcC+XI71pkr8L3auXuvJrDgb58u6KBN0cIFzU0t+/tUuESzVShbynxo7MOz1kkkQ0hGeB/h76qdpQhyCEq7ObCMBEiABEiABEiABEiABEiABEiABGIbgTzqJe06qVNKojiPybJLV2SKWpfTBZ0WGdPJ5IJ5VW6bxwwshdXaXMcsGSWdelkW7Vddvir3VaoFGOp+K1ZAciphCOXjw05pgk4y9ZLpVwVySz0bkQbiTVWVzmK2Eommq7V0CDoQc6YXzidBEzSMq4t5O0Hz1IGoAfHGLq/OD9NmeU227Ss6LI4iUbQv9lTNJ2S+yt9x2CaPDXJ7OBlyGzgtUDn1MZcfPnrMMQdJjuxZzU1j9b72zCjPEbuk4UkcPCV0YJk9iDoHDgU/xN3mbTsjJdgdD3MWQ/VrDeR2z94DUq50CdshK5UvLbt277OtQ6F5Idi10Vn1A6wbwiNCVHU1eAsg3xHCaPlryDnyqFkw7oWZWaGC+aRfn66Ogg4Yd+s1UAu5Z+4XqvuePCY95WyLjustmD+P47AbN293rEMFvF3ebNXctk0gQk065cfBCVf9a/UCSqP+sWdnngRKtMdvPUSlcmVKunWPFy+e9kyaX4xwa8QCEiABEiABEiABEiABEiABEiABEiCBgBKAqPJ1wfD1ihMq9Bny56RW/x8duW9SOESh6qM8cnqpzxUViSqlS5vSyjtnRakickqNdVxpBOlVtB+EXHMSaHIpUenz/Lnkhor4gz4ZVYi2xAGOrBVQYEEeLGiiDq7rinLRSmgTFg1J1uctWhKQS+/4ZktJph4SXwwC0Ps9O0vbTj3dmp8+c05baEIbV6taqZws/HOZa7HPx55EhZoq4fv3Ku9AoMwuzFWgxnYdJ776MgZ6cf2SitOYIX1C11Npb7UXK1JQtu3Y7VaHgopKgHCy3Uq4CLbNX7RU8InptmnrdpUT4wXbaVapWE4mfT/Dtg6FqVX+CzuDF4DZw2bvvoNSMH9eu6bSuGE9Gfnx57Z1COPm9DxfvnLFtk8oFwbjXuh88ufNLQPfe8dR0IEXRddeAx4ZQQe/6+nSpdUv320b7NxmGTOkd5uDXpAndw4tF5B+7Lr19PfunvqHT1QsR7YskitHNtshbty44SbAJkiQwLZtKpM7tm0DVehJqKeg40SN5SRAAiRAAiRAAiRAAiRAAiRAAiQQ/QSQ8wYfXwwijaugY+4HcQYfXw1CDgQempWAkxhmbRWgI6e3dUsULyII+xNVw8JkjScq+TVM6lQppVWLl2z7OOW0eP3Vl7ScC7adfCjcvWe/YyuEmmnwbG3H+shU5FHeD4Ey8+K865h1n67hWhTlY6dnBgM7iQ94DnBfnczOM8upbWwrP3joqGPidCzMQoC1M+TuSOoQEu+yEnPN9sdCZwG3dIlikia1fVzMt1q3MA9j2T+qYnY+CnbXFF4uGPcCzHLnyi6D+/VwFMwuXrok8E68eOlyjEFs5oR8TTWrVdJyPvk6wQ5tX3MMd4bfuECL097mdeCQs3faSy8857F7M5XHxsnOuoSYQ5jCtGlSOzW3lKNt73c7WcrMB/+u32w+1PavXb/hVoaCdOpljgIevJGSqxcxnLyVPP3NsT0ZC0mABEiABEiABEiABEiABEiABEiABEjgEScQVE+dv/9ZreLzF7BF2rfn2zLrt/ny86+/O4ZxypA+nVSuWEYKF8wvwz78zDIO3rx+r4f9AhQWhd7/YJQMHdDL9k30erVraZ43rrlWkLenTMnilvPgALkPxn44SCZ+N01WrVnvVq8XQFyAxwhyhXwz+Ue9WFscxUI3BBw7a9GssZQsUVQ+Hf+NXFBJqXTLljWzuvZ8gvwLS5ev1sLV6HXIS+BkXTu2lnf7fCB6G8ynWZOGUvfpmk5dHMtPnjojuC47A8fVKhwPEm3DsFCHZN2nz5y1a+5T2Zx5i6R4UXvBD/Po3qWdfDjmC2MshJCCx4GTHTl63JLfyKldbC7fsHmbVCpfxhYBkrJ/9sW3gu+ybvBc6/jm6/qh23bRX1avNjwPeK7thDeEWhr74QfSSyVwP2bKw/V68ybqu1/WbWy9YPrM3/TdaN8uW/GvnL8Q8b309YStX3vZNpQgPPP0HDV79lm9yKL7XsATY9iA3o6CDq5tpfqNa9ygnq+XKWvWbRSEjgsWJ+R60p+/a9evy779h2TF6nWyeesONw8X/A3p/FYrKeQhZ86W7bt8vtZANYRA4hQ0FHnWunVqq33v7ty5YzllkxeeldpPVreUmQ9cvQPhyYq8N/CA3LBpqyz482/Zf+Cw5TcRf0vLly2pOL1hGy4V48Nz5ofps8yn0vYhmDv9Xvfr1VVGjZmg3RdzR4SI+1+vLoLvvp1RhLejwjISIAESIAESIAESIAESIAESIAESeDQJxJXHtAtLHCd8+2heZdSvyn4VJerj2o7w198r1OJgXbELNYMFnZcaPafVY8EXb4XfUvlU/s/eXcDJUaR9HH+ycXd3FyBAQoAQAsHd3R1yhxwcdxwc8MIdLoe7EzQ4wQkBggS3QIS4EXd33vrXpiczuzOzM7szm5Ff3WczPS3V1d/unubq6aqqXauW705NAZBYlT7a2V/OPsWvG23Hr7/1vmmMnBGuoq+fq4AumlSJpUqlgX+7ImLR3Q88bk8++L+oFZ4aE+TSC8+11eescRVkS315N27802rXrmm1atb0Y4YoXyUFcMKDOpp374NP2JWu67dYSS2XHr3vVl95ttZV5CmQFOSnbWShMQiCpBYtjWN0J9SiWVMb9MjdpoHs1ZpCwZbwvII8wj9jLVdQJFZSN0B33HxNqIJQeajy75hTBsbapMT5Ggh8wqQpPpAVbWW1EBn89AO20nUFVMmNyaIgUqykstx610OxFjN/k4Aqa+Ua7RrQvAsHnmEXnHe6D0RUr1bNBy1j4SmgqvuvaHr1zXft7NNOKDrbf9cYSnfdcq0PQq5es9pquHMaq9s1baAgUdGAbNSMUzRT+/t4ePKBylNOODJqUOeLr74rFnwIipruc3H4IfvH/V1VORSsTSYpaKygTnk6BeWr6cbaUstP/QVp/fr1ts796Tc03nUUrP/Us7G7GAzWSfXnihUrfesgdTEYLfVzAc2d+2xv8+Yv8M8aPWMaN24YM+iiPJYtX24zZ8+Jlp1/PqmrT/0pKeCv39Aqrvmz7umSkrofDX/hIFj/l99Gxwzq6Nj0nFVAaeHCRabWVgrsqhVPvPRhkaBwvHVZhgACCCCAAAIIIIAAAggggAAC2S1wfJOGpj9SfIFy7X5NRbktrFVFtKKp0k2VPGqNslX3rtamdUvfHVO8gI4GVw8qp4rmqbEgXnyl8C3++x55yhQgiZa0zxOPPTxi0UrXlYwqn+MlVVQpSKUxQtSKplWL5qZuqqJViIfn89PIUT5YET4v2rTyqerGKSgpv2kzYgdblK8qWps3a+JbB5WUV7RyBPNG/jYmmIz5qfzLso+iGaslTrwxFXTNKPgXL6CjPL8Y8a3NnjO3aPZ8LyKgAGF4S5wii/1XnV8FWnVdxUtPugryoIVY+HpqQaAK6nhJeeu8xquI13VxY5FWe/HyzLZl5XEuss0k2fLq2aFARbzrKMhTLx6UZ4Aw2K8+X3797fCvxaZV/mZNm/hWRmq1qedCvPTqG/GfXeHb6l7TiwqJBHQUlHl80Ivhm4emdV/rmRsvaT9qfaSWliUFdJTXx8NHxMuOZQgggAACCCCAAAIIIIAAAggggEDeCZR7UGfK1OmuO5/vUwatyuV///PCmEGEBx4dFNqXKpefiFEZpZWOcG+tN23SKLS+Jl567W3X1dLiiHmp+qJgRbQK79Lk/7rrui5e4KM0eUbbZuLkqa57o8nRFqVtnloYvfXu0DLlr26ZHnhs87VQpszyYOMHH3vG5sydV6YjVdeERbt/Cs/wPzfdFTPIGr5evOn7H3k6opu2eOtm67LyOBfZapPKcqs1Z/jzIpV5J5LXm+986LrV/CqRVUtcR0HZt98fVuJ6ya6g39GLL7smZhepGovosqtvMLWOKmtSHspr48aNZc2K7RFAAAEEEEAAAQQQQAABBBBAAIGcEij3oI707rj3UXvimcEpqfg594yT/BvG0c6Kugr79oefIxZ99MkXNmt29NYaChD93+UXR6yvL+qWTZVkqQ6aKFhx5l8vtd9G/15sn8nOUFc478UZgL5ofuoW650PhpVqvBt1YZaKSruiZYr3fZAbd0TjMaxZuzbealGXff/jSDv7/H+mLIAWdSc5NlPXx8X/utZmzpqd9JHpPtHYWP+795G426rV1GVX3eC7WYy7YpSFanGncbVSVQkeZRcZM6s8zkXGHGwpCrJ+/YZSbLV5E12v+u284pqbN8/cQlMar+o798wq7bNG2+meuPehJ6MewaxZ0btji7pykZlqwXTxZdeGxn8qsjj0de68BXbtjXfYKtedW2mTnmdXX3e7ez7Fb81X2vzZDgEEEEAAAQQQQAABBBBAAAEEEMhmgS0S1BHYux98bGe5ivZRY8Yl7KcKKw2a/PATz/ptCgoKbO89do26vdZVS5ho6ZY7H4hZaaau1Hptt3XEZnpTWJVk/3QV0ArEJJpUqaUBu6+75a6Ym6xatdpXgN1654N+rIGSKvNUmf2TG8j+pdfeKpanAmWvDXkv5rFpA+WvAcQV5HjymZfcdPTu1DSmUayklktn/OVSn0+88irwU3Tgd43zUNr0zXc/2ZkDL7VfRyc2kLn8b7j1Hrv5jvvjBnTWliJQVNpjyKbt1Irson9eYxpjRG/gl5R0LWgslX9ceX2oy8OStpnxxyw79dxLfNBUwYuSkq7/r779wV+/P7iB3vMllce5SJXlMjeGWHkmBQcVeFeQOpnfZ11vGpfsksv/E7M7sfI8jmBft7hnwXkXXW7f/fhL3N/yYH196t7TfXGm+11WYChWenbw6/bXS6706yYamFd3a488+Zz9zbXQiTaOTrR9jR030U4552J70LWOTHQb5aN177zvUTvngsv8OHjR8mYeAggggAACCCCAAAIIIIAAAgggkO8CFbr1GvDnlkZQC5kOro/9bl06Wgc3Pk5rN15AgRs/YLGr4FngBlSePWeeTZ0+wwUzRiVcyZXOY9JYBl06dbCuXTr4sQGauUCQBuJWhZTGCZnrxiQZNXacTZ4yvVTFUBdwPbp19mMOaJwYDTiu1kVTXEBL3deVlJo0bmQD+u9sGmtIg2kvXLjYfh8/0X4dNdZ9Tkq5ocZ32Hbr7v4Nbo2zMs8FvvRWtyrs05WCa2arHl28U726dX03PVOmTbcxYyf4wI8GHyelTkD35l6797Mm7vqs58bF0BgfCv7pmp/srsthn34RN3iWSEl27L2d7eQGhG/QoJ7VcePpKKCqIKLe2Nf1W7TlXSJ55uI65XEustWtdq2aLjC/jR/rTL+lDRvU9+M/KSC4ePFSm79wofuNGO8D7pl+jBqzTc8CjS3Xsnkza+J+z+vWqeNeAFjqn4vT3W/s1KkzbPykyQkFXsOPV7+hXfXMdc/eti7/Zk0b+7zVxZruuZmuVc/Qjz9PKlAWnn/4dJtWLayTG0OnVYtmfmy3YCyd4FmhVrWTpkzbYuMZhZeVaQQQQACBQoHlSxdBgQACCCCAAAIIIJClAhVcub/aZ6Y1mdU3S48g/cWe2/wr23loi/TvKME91KpTP8E1C1fLiKBOUiVmZQQQQAABBBBAAAEEEEAAgbQJENRJGy0ZI4AAAggggAACaRcgqFMycbYHdbZY92sl07IGAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAIFApmOATAQQQQAABBBBAAAEEEEAAgWwWqFmzhq1du851C7wumw+DsiNQTEDd59apXdt3xVtsYY7POOPUE6xmjRo28tdR9tmXX5fpaKtXr2YdO7S338dN4HeiTJJsnE6BU0861t/vGgd22CefpXNXWyzvnlv3sN127WsbXLf7Dz7yZJnL0d51Lb5mzVrXlfjsMudFBghkgwBBnWw4S5QRAQQQQAABBBBAAAEEEEAgqkDzZk3tgH33stpufM+KFQs7o1i3br2v/NbYbZ9+9mXU7ZiJQDYI6Npu376NVata1Rf3Tzcq8spVq2yBG8v2y6++dePvzsmGwyhTGRXM0r1dx40vWdZ09uknW+XKlazvjr3t0SefLWt2bI9AQgKHH3KAtWoZf+yO2XPm2iuvv+Xzq1u3jlWuVMmNe1o7ofyzcaX69epalSqVU1L0nfr0sn59d/R5vTHkPTdm69SU5JvNmbhHhU1dUck2Fiy2ZhvrZfOhpKXsswsW2TTnk80pu0ufzfKUHQEEEEAAAQQQQAABBBBAoEwC3bp29gEd14ghIqnStlHDBla/Xr1iQR1VrrVr29p+Hjmq2LKITBL4sv2229ju/fvafFfB/uwLrySwBasgkJhAQUGBnXrisdagQWRlnK71mjWqu7+WNntOu4igTnlfj+W9v8TkzM4761RTi5xXXn/bZvwxM2IzuSpVchXmJATKS6C2C0yWFMBQkIMUWyDeszsIemtr3fukQoEXptayM9tNNFvWkcBO2EUx2wW65teeZC9MqRU2N/smeYpl3zmjxAgggAACCCCAAAIIIIAAAk5gj913NVVyq/XCDz/9YqNGj3WtGFZb2zatrGWL5lEr0Ro2qG+q2K26qeVDWSAbN2ro86pWjUqksjiybXGBHt27hgI6ixcvta+//d7GTZhkuubatG5pzZo2KRawKO/rsbz3V1wp+pwarqs2/S5UqlSx2ApqCdGjexf7+Zffii1jBgLpFtiwYaO98dZ7UXejFnik2ALxnt1qtbh+wwbf/eroseNiZ5JnS974o4apxc4JbcdbxZrr8+zoYx+uWugooPOm88nmRFAnm88eZUcAAQQQQAABBBBAAAEE8lSgevXqVr1aYZdUehv/sy++CkmMdeMQ6C9aqrppm2jLgnkK+uit6bp169ry5ct9S5yNrt//okldvqU6aewUVZYrUKTjirbfVO+T/DJPoLtrhRakl18fYsuWLfdf1d1arC7XEr0ede+ogrRq1Sp+/IlVLhBampTo/pS3Ws3VrVvbZs2aY2vWro27u3ru3tMYOhob409FbJNIlSpW9AGdWJv8MXOW6a+kVLlyZRcYbmZz5823lSsjK9truJZSTZs0tnnzF7jfhxVxs0rmuONmxMKcENjgAg9Tp00v87Ho+mxQv57VrlXLFi5a5P4WJ5Snrls932L9hiiTZJ5BavHWonkzW+UCUrof4iXl26Z1K1uwcGGJ9020fOI9uxXQUWCnpKQyqLyrV6925VgUsbqORS+D6JlfdFnEiu6LfkObNW3s/9sg+G0uuk6mfFfgItuDF5limWnlIKiTaWeE8iCAAAIIIIAAAggggAACCJQoEIyfoxVVUVZSOubIQ61xYxcs2dRCZyv3tn5QcT5l6jT/9rQGbu63y06hYFGQp+qVx42fYO+8/5Gftbsb3Ll7ty6+Ykcz6rjgzsUXnOeXqXLr4ccHmcb6Oe7ow/28F156zebMneen9U/rVi3tqMMP9t+fef6lUAXSHrv3s+16bhNRKb1q9Rr74cef7dvvfwptz0TuCxQUbO5TcP36+G9YJ3I9qjLzxOOOdAHDRq5Sd3PeklSF8AdDP4mo6B149mk+sPjeB8OstWsZ1L1bZz/Gx8RJU2zx4iUlXv/KV/s8cP+9rUunjhHXtAIhrw95t1glsFrebeVaKIV3UzXjj1m+Elr5xUvqcunE446yGq6yNUhHHHpQKCj0yfAv7JdfR9mZp57oxuap7QOmwfglvbbr6QdsV2Xu967FX7+dd/QBryAfVdo+8/zL1n/Xna1Ht66hsbu0XOOgPD/4tWBV/5nscUdszBcE4gjsu/cA69qlk78Xw1dT4HHEN9/bSHeNB+nIww7yQZRJk6f6caQUUHG3pE96po0e87t98NEnwer+M9FnkIIfhx68f8SzUnn+PPI3070WnhR80roKcAb7X7dunRv3bln4ajGnE3l2d+nc0Q7cb2+fx5ODXvBj6tWsWcPOOeMUN+9P/3zfa0B/F1jePC7X+vUbTM9mjXW0y859Iu751e65+/Rzg23FipUR5dJvRf9+O0f8Bqxzv89Dhw2P+SJJRAZ8QSCFAoWdiaYwQ7JCAAEEEEAAAQQQQAABBBBAIN0CqhhesekN+rZtWvuKlmC8jGj7btasSSigEyxX5bb+9NatkiqGgtY/6iZHFU9KqohSRZrGEFFSXnpbP6ig0rwgryqu9UPh94LQvKLds6lbqGD9oBu49u3a+PyVp1rnqJJOnyqPKuNI+SXw8y+bK2dPOv5oa9K4UUyARK7Hiq4FS+Fb+hV8d4Vr167zn8pUla6qAA5Pld0A5rpGD9hvT+u5dfdQJbJaCCSyP+V12snHWVd3T+maVoXvunWFwalatWra8ccc7uZvqmF26/buta27/rf2AR2tq/Jt3Pinq3BtHnGfhZcxfFr3vgaV13haQVL2wX0W3OMKGGleENzVujomzVOwZ08XWFILJpVV5VBSi6S/nneGbbNVd7deQeh3QcvUDZ7+wlMyxx2+HdMIlCTQuWOH0L2oa1TPKSU9j4oGLfRs0XXdqWM73yWpWr3pmabrWvfGVj262kEH7BPaZaLPII1Jd9zRh4WelcFvifLUPbxTn16hPPW7c/yxR/jfGC1XIEXl1j2nce8SSYk8u9VCL7jXK7hjVipwOyycV+B/3xTQ0f71u6Kk5/ApJx7junHtV3jPu+BMcM9Xc8/dHXfYfBxaX74DdtslFNBZs6awxWFl18LngH33Svh4lBcJgVQIbH7apSI38kAAAQQQQAABBBBAAAEEEECgnAR++PEX94b9zr6Cqk/v7az39j1t2vQ/7Cv3xnLR7mXufeAxXyq1qFFFz6gx41zrhI8jSvq9axEz8rfRNm3aDFu9Zo1fVttVQJ/t3vZVhZQqqH/65Vcb/MqbftkZp57gu2lb6t7kf+zJZyPySvbLgN36+U3UFdaDjz4V2lwtEILAT2gmEzkv8LtrGbbPXrv7IIdagp18wtGuC7YV/vrU+FHhrXcSuR7Vmk1dFE6fMTPUakxBlSMOPdBUSatAhgIfamkWnhTEmD1nnnsLfZxv0bPItdJZsmSpXyXe9a+WbAoWKek+/XzENz5IqcDpwa4iWZW6CpL++PNI0xv1u/Xr69dd7t6MH/Ts4ND916f39raraz2n+y9e0hv1d9zzkO8+SRXOSq+9+Y5NmZpcV1e/uXG5hn82wncRp2DseWed6itxVdn77Xc/2ldubCMFW1UhfepJx/r97OB+e95+90M/ncxx+w34J68EdN0rkFA0TZ4yzb5w90hJafjnI0zPG3UhGLRQ7bnNVrb3Hv39PaJWcd/9ENmqU9fu8C9G2I8/jfTZ6z4/zV27CgRp/U9qfOFfIkj0GbT/Pnv6fPSMfPm1t2ye66JQx6U8FRjVPfuNu1eUDjlw31AA9ZPhX/rnp+YrwHL0EYf4QKy+x0uJPLvjba9l6hrunfeG+t8w/e4p8Br8Pk2YOMWGfjzc//ZVqVLF3/MKDnfp1CHU6khWclZS67zXh7zn169bp46dcerx7r8pCvwYfy+/NsSvwz8IlIcALXXKQ5l9IIAAAggggAACCCCAAAIIpFxAQZgPXbcnelNYSRUrqqA+wb0ZfNZpJ/qxaZLZqSqAx42fGKpQ1rbLXIugoBJbA7CnK63ZFETSm80KJAVJQR51d0XKPwEFCsPHf6ldu6b169vHLhh4lq9ATEZEb+l/74Ir4d0Aal54t34aZ6doUpD0+cGvuuDLrz5AEtwLRdcr+l0BViW1JBjugknB2FC6v4IWO81dizelDu3bhoI2Q95+P+L+UwV1sK1fOc3/qBulYMyfwnE3Fvo9Koj25dffhsoyf8HC0O9OUDmsFZM57jQfCtlnoICCkxozreifuuRMJCnoOG36jFBAR9v86l5EUOBGKbx7scI5Znq2BAEdzVPg9v1NLzSoPGqBppTIM0jXugIcSr+MHOUDOppWC6BfR43RpA/w+An3T3M3fo2S7he9EBEk/Y5MmDAp+Jr2T5U1GHdIv3vjw/Y97NPPQsHstW68LwVtlMJb2KrbtSCwrK4qg+D3kqVLQ7+pGoePhEB5CtBSpzy12RcCCCCAAAIIIIAAAggggEBKBX5zFUn66+HG4tjBdeEUdOmiyi11W/Xw40+7CpjEB4LXYMnqbqqRq3hThfT8EgZ/TtXB/DZqrO/GSd1DqWXQjD9m2oivv4uo1E/VvsgnOwT0Jrxa4aj7r13cOC96c1xvkKulmbo5quFacQXjPCV6RBokXEEU5alg4fIVK+JuWtqASq2ahYHJjRs3FOvaLdhhUGmq8TmUtK+gQjVYZ0t/xvvtUEV2+Pg/Kmsyx72lj439l7+Auv76ybVOK5pmbQokFJ0f7btah3Tr2sm1NKnvWtistLnuGaV7J3ycuWjbhc/TCwwKBClQ0bBBA78okWdQMzdWXJD0WxLebaNa+ykpT72coJZEQTeHCg5nUlL3rbFStHs++O8Kme3ef5eITYN7vlJY148RK/AFgTQJENRJEyzZIoAAAggggAACCCCAAAIIlJ+ABn3Wn96A1qDMGl9Dld+9tu3p37BPpCQau+bIww70LX4SWT+V66jbN3X9okGYVSnWulULP27BXNe1zatvvBN6MziV+ySv7BBY5rpbUleBH7pBzXV99N5+W3+NdOzQPqkDOO7ow133ZIVvzie1YSlWDroMVLdMaj0XLamrNaWghZDGu8j2lMxxZ/uxUv7kBdTiSy3XSpvURZq6LQxajZQ2H20XBIKCljeJPIPCW6XpGRUtKfChgI5erAjKuWz58mirZs08jQOmpOOJ9XsWtBjOmoOioFkvQFAn608hB4AAAggggAACCCCAAAIIIBAIqO/8p5550S766zm+AqZLl44JB3UO2HdPH9BRC52Jk6b4VjIaI0ADSgdvHAf7ScdnMKbPjjtsb9ts3cMPRN2kcSPX4uioMo/Zk47ykmf5CqjbII2LU92N9aJrUgN9q5VLeBdtsUrUtXOnUEBHXSFNmDjZV7w2dteXWgClOq11rViqV6zqx+MZ9slnUbNfumyZn7906TJr2qSxP56oK2bRzGSOO4sOi6JmgIC6aOu13Ta+JGpNMm7CRFu0aIkfG0bPiyCAkmhR1V2pkoLGQSrpGaTuxoL04stv+N+Q4HvwGbTu030dpKA1S/A92z5Xriwca0yt8156Nfq4OatXF47Dl23HRnmzV4CgTvaeO0qOAAIIIIAAAggggAACCCAQRUBvCeuNaHVVVXFTxVXR1YpWgOntWw3YrqRBnr92A6IHqbnrcqZF883dzgTzE/2MNs5BrG3Vp78GzNbfvnsNsK236mZ1XFdZVV0rnmCsj1jbMj8/BBYv2TzGkq7xRFL/fjv51da5+2LQcy+FNtE1n46gzgrXrVv1alX9+FDh4/iEdhw2MeOPWdbZBZZUyazgTknrh20ad1IDopd3Sua4y7ts7C+7BdTlWpAUUFm0eHHw1bp36+Kfd6EZJUzoeRfcHnoRIjzFewbNnjMvtGrtWrXs9/ETQt+LTigIvWbNWlO3bG3btCq6uNTfg3KXOoNSbLjABcJlVqlSZd9tJc/iUiCyScoFCsOyKc+WDBFAAAEEEEAAAQQQQAABBBBIn4AqgNUVTeeOHXzAI9iTxhvoH4MFAABAAElEQVQ4zHW/FlR2T5k2I1jkPxXsUWrRLHY3VOq2LUiqZA760w/mBZ9Bv/x6C1ljCISnhYsWhb5267K5Mk7jiKj7rKJJXVBpPJ/wpDFVglTRtcog5Y9Ah/btrLcbI0rdCQZJQQq1uNmpT28/S+NzTA27vuNdj0ENbgXbfG0rv5133CHIPunPePv7fVxhZa+CRkXHoNCOmjVtEmr9NnHylNC+DzvkgIgBynfq0yup7hDDK7o7dWwfyre8JpI57vIqE/vJPoE/3b2tVN2NmxWkgoLNz4DwwMZWbjy5os+OYBt9qktA/ZYESd2tHbDvXv6rfkPU7ZpSIs8gjTG3bl3hM3SfvXa3evXq+m2Df/R8070dJLUKVNIzNGhlpO/169XzgVxNJ5oSeXYnmley641yXbsqyf2Yow4t5q0XN3RMJATKUyDyvxjLc8/sCwEEEEAAAQQQQAABBBBAAIFSCiiIokqioKJIXaap25cgmKNs1VXKF19+HbGHyVOmWVfXJVu9enXskgsHusGiN7o3b5fasy++Eho4WpXmqijTPoLxBiIy2fTl519+82PfKAik7t6Cbmfuvv8RNwbOalvqurVRK5tWLZvbhX8525enevXqQf16RJYnui7WKrugjlpSrFi+0ipXqWw1XeWb0syZs92A2IXdv0RsxJecFei9fU9/bekANUaFrmWNTxNemfvdDz+5ZYWVv1ov3vU43Q1UHnTZpmtRA6zXcm/aJzO4uvYRnuLtT63derngpVrr6Fh6btPDX9d6a18VvzoOtUb79vufTN00/fDTSL9eLRcE+ss5p/vjVUV1eIA1fN+xpnWfKNikMTC22aq79ejW1QeFRv46yoZ9+nmszVI2P5njTtlOyShrBKq43/W/XzQwankXLlrsuw7VQrVW05g1eqng4gvOtSFvf2DjXKuYrbp38dueetJx/r7R80l5xku613bfta/132UnU/eAVatUDf2O/PjzSN+qVdsn+gzSmEB779Hf7/eMU06wlatW+TwKX24ocPfuerv3wcd8kYa8876dfcbJ/tmmlzB23WVn/5sV/pyOV/bwZbGe3U89+2L4ammZVnBqytTpvrWOukS9YOBZtty1RixwuDVq1PC/UzNnzbEXX349LfsnUwSiCdBSJ5oK8xBAAAEEEEAAAQQQQAABBDJaQF2sLQnrs1+V0+EVReomZtBzL1t4axcd0IivvzNV0KguXJVdFSoU2ApXwa23gIcO+9QFZv7082u7YIwqzLQPVeYo/en+F57GT5xk01xluQJKyktlUKVZkL7+9ge/H31X2ZSfKuc/HDY8ND9Y9w/XBZUqwxTYUcApCOhovIM33/kgWI3PPBGYOm26uyY3+KPVtaWKW30qaf7nX35jX371beGMTf/Gux6HfjzcFi4s7K5J16LeLFdrNwVTdM0rBUHJTdmV+BFvf9r4yUEv2KzZc30+wXWtlgc6Dg0qPmfu/NA+hn8+wsb+PiF0LxUebwV/7y1ZUjiOR6LlG/75Vz6wo8yDoNWKTUHR4A7e4ALAQUok3/DgWbBdrM9kjjtWHszPPwHdI0HSc0ovBijpPtWfghqjRhe2GFGwU88J3Sd6BukFAiW9pFA06UWBefMX+jw0NpzuPz3/vvrmez9GV7B+os8gBUj1DAuee3pW1XVjz+leU76zZs8JsvTH8NIrb9qKFSv9PI0Dpt8f3f/ffPujnxcWlw5tF20i1rNb64bfw+HT0fIpOi/R9V8f8q5r1TTGH6P89cKGgseaVh76zSYhUJ4CFbr1GhA808pzv+wLAQQQQAABBBBAAAEEEEAgAwWWL12UgaWKXSS99d/KDRZf11VwFbgAjQZynjlrdqhSN9aWlVwrHHUdE3QPE6yn1hAakLqKq3hS122rVxdWrAXLY32qmyy9aa1gU3hS91Pt27bxFWrqKit8oOnw9YJpHY+6jVLF2yz35m/R8gXr8ZkfAurKSF0XFQYE19vsuXNttguUlBRkiHU9qosljRGl62r2nMKASyokY+1PeVdx40FpTCodx3JXuasunOJd12qdoPvwj5mzSjzOeGWXmVr7hA/YHm/9VC9L9rhTvX/yy34BBQ0UEAm/3/VM0TNqjeues/ClgshnTnDUJxx7pLvXm7hn2Bp74JEn/b2gVqMKFsUbtyqZZ5B+T1q6569atarrQ7VW0Zg80ZLu6batW9msOXNCQZ5o65U0L9azu6TtUrVcAbamTRtb86ZNXcvadT5Yrv/mCD9HqdoX+eSXQK069ZM6YII6SXGxMgIIIIAAAggggAACCCCQ2wLZFtTJ7bPB0SGAAAIIIJC8QNGgTvI5sAUCCJSnQLJBHbpfK8+zw74QQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAgVIKENQpJRybIYAAAggggAACCCCAAAIIIIAAAggggAACCCCAAALlKbB5FK7y3Cv7QgABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEi5wIivv/NjWS1btjzleZMhAghseQGCOlv+HFACBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAgJQJTp003/ZEQQCA3Beh+LTfPK0eFAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACOSZAUCfHTiiHgwACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAArkpQFAnN88rR4UAAggggAACCCCAAAIIIIAAAggggAACCCCAAAI5JkBQJ8dOKIeDAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACuSlAUCc3zytHhQACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAjkmQFAnx04oh4MAAggggAACCCCAAAIIIIAAAggggAACCCCAAAK5KUBQJzfPK0eFAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACOSZAUCfHTiiHgwACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAArkpQFAnN88rR4UAAggggAACCCCAAAIIIIAAAggggAACCCCAAAI5JkBQJ8dOKIeDAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACuSlAUCc3zytHhQACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAjkmQFAnx04oh4MAAggggAACCCCAAAIIIIAAAggggAACCCCAAAK5KUBQJzfPK0eFAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACOSZAUCfHTiiHgwACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAArkpQFAnN88rR4UAAggggAACCCCAAAIIIIAAAggggAACCCCAAAI5JkBQJ8dOKIeDAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACuSlAUCc3zytHhQACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAjkmQFAnx04oh4MAAggggAACCCCAAAIIIIAAAggggAACCCCAAAK5KUBQJzfPK0eFAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACOSZAUCfHTiiHgwACCCCAAAIIIIAAAgiURaCgoGJZNmdbBBBAAAEEEEAAAQQQSFCgNP/tTVAnQVxWQwABBBBAAAEEEEAAAQTyQaBylar5cJgcIwIIIIAAAggggAACW1ygctXk/9u70hYvNQVAAAEEEEAAAQQQQAABBBDIGIEgqLNu7RrbuHFDxpSLgiCAAAIIIIAAAgggkCsCaqGjgE7lygR1cuWcchwIIIAAAggggAACCCCAwBYTUGAnCO5ssUKwYwQQQAABBBBAAAEEECgmQPdrxUiYgQACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAghkngBBncw7J5QIAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECgmQFCnGAkzEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAIHMEyCok3nnhBIhgAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAsUECOoUI2EGAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIJB5AgR1Mu+cUCIEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAoJgAQZ1iJMxAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDJPgKBO5p0TSoQAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIFBMgqFOMhBkIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQOYJENTJvHNCiRBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQACBYgIEdYqRMAMBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQyDwBgjqZd04oEQIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCBQTICgTjESZiCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACmSdAUCfzzgklQgABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQSKCRDUKUbCDAQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAg8wQI6mTeOaFECCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAxAYI6xUiYgQACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAghkngBBncw7J5QIAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECgmQFCnGAkzEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAIHMEyCok3nnhBIhgAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAsUECOoUI2EGAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIJB5AgR1Mu+cUCIEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAoJgAQZ1iJMxAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDJPgKBO5p0TSoQAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIFBMgqFOMhBkIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQOYJENTJvHNCiRBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQACBYgIEdYqRMAMBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQyDwBgjqZd04oEQIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCBQTICgTjESZiCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggAACmSdAUCfzzgklQgABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQSKCRDUKUbCDAQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAg8wQI6mTeOaFECCCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAxAYI6xUiYgQACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAghkngBBncw7J5QIAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECgmQFCnGAkzEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAIHMEyCok3nnhBIhgAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAsUECOoUI2EGAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIJB5AgR1Mu+cUCIEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAoJgAQZ1iJMxAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBDJPgKBO5p2TlJeoZs0a9vTDd9ozj95lO/XZPuX5kyECCCCQ6QIFBQXWpHGjjClmxYoVbUD/vtaoYYOMKRMFQQABBBBAAAEEEEAAAQQQQAABBBDIfIFKmV/EzCxh5cqV7a/nnGpdO3e0evXqWBX3/c8//7R169fbrFlz7INhw+3DYZ9lROGrVa1qCuwo1aldKyPKRCESEzj4gL3tuKMOKbbyunXrbf78BfbOBx/b8C++LracGQggUChw0V/OtN7bbRP6DdTv9JKlS236jFn2/Etv2PiJk7cI1XVX/cO6dO7gnxsnnnmhrVu3bouUg50igAACCCCAAAIIIIAAAggggAACCGSXAC11SnG+du7Ty556+A7rv8uO7s3vhj6go2wqVKjgp9u2aWVnnXp8KXJmEwQiBerXq2vVq1Ur9qfgXIf2be3CgWfYNVdcErlRHn579L5bbfDTD1iP7l3y8Og55GgCaglz163X2m79dgoFdLSefqfr1a1r22zVzfr03jbapgnNq1e3jr/mnnv8npjrX3Hp+X6dM045ttg61apVDc2rUqVyaJoJBBBAAAEEEEAAAQQQQAABBBBAAAEE4gnQUieeTpRlTZs0sksvOtdXDGrx2HET7IuvvrPxEyZbbVfR3nPr7q71TgfXIqZ2lK2ZhUDpBa667ja/ceWKlax/vx2t7069fbBHldP77rVbxrQMK/0Rln5LVbCrsr6qazFHQkACA/rvbK1aNPcYs+fMtZdff8e+/vZHa9O6pQ/odO7U3kaPHV9qrNYtW5gCR+rWLVZq1bK5X6dmjcKWkuHr3XzH/XbEIfvbT7+MshUrVoYvYhoBBBBAAAEEEEAAAQQQQAABBBBAAIGYAgR1YtJEX/DPvw0MBXRuvfNB+/aHnyNW/HnkqIjvyX7R29stmzezhg3q2x+zZtsfM2cnlEVH12pDlYvJdCXUplULq1+/nt9m5cpVCe2HlbacwNjfJ4R2/uvosfbKG+/aA3fe4Ofts0f/YkGdOnVqmyqVa9ao7oKPE23ZsuWh7ZOdKG1e2n/tWrV88FPdXgVJARhVqhe4z/ETp9iGDRuCRcU+q1apYl27dHRdZi2zqdNmFFuurhCVXyJJ63Xq0M4qVqroA7Hx9hsvP5VJLaXWrl1rEydPjbcqy7aQQP9ddgrt+dob77T5Cxb67/qNTOR3Utd8pw5tbdr0maFtQxm6iRYtmoZ/jTpdq2bNqPM1c+68BfbwE8/FXB6+QPeRngkqd7zfapW5Y7s2Ns6tV1KgiGs4XJhpBBBAAAEEEEAAAQQQQAABBBBAIHsECOokca6aNW1i7dq29luMcRXsRQM68bK66rKL/NvhatWj1j0nHnu4qcJPYzucff5lfnyefn37mCrawpMqsge/OiSiwj7I64eff/Xrq3VQUKmtivNPP//K7n/k6fBsQtMaKPzm/1xuHV3FdrCNFn7y2YiY24Q2ZiKjBObOm+8rbjVeUuNGDX3ZFNjT+VUXgGpFEJ5muiDhvQ89FVGh/dj9t7mgS02758EnbOse3Wy3XXfy19R3Llh5290PlyovjVOy/bZb21auK7TgGtN1+figF23ajJl2wbmnRQxYr8DK7fc8bN/98Et4ca1b1072r0v+4oNCwQLl897QT+yJQYN9y7hb/nuF1XUV2UH69z8vtI0bN/qvTzwz2D74aLifVkue/7viYlPriqBMWjDm9/F24+332apVq/16B++/l51ywlE2z41XdM9DT9pFrnu7pk0a+2XHn36+H//kuqv/4cfS8jPdPyrTosVL7M77HvP5BfP53LICFStubkGzxgXfEk26Bk467kirXHnz41HbP/TYM/b5iG9tpz7b++41g+tO15O6/gvSORf+yy698Bxr16Z1qNu3Af37+u46tY5a5qiVjrpO3NX95q9ctcrOGHip31zdLT50901++rKrb/T3ip454ddstN9qBfV1fYe3CNKzQy8ZaB8LFi6yv15ypc9XvxFcw56CfxBAAAEEEEAAAQQQQAABBBBAAIGsFNhc65WVxS/fQndzrQWC9Ppb7wWTCX2q4l2V7KpgO+/Mk31FtSrqKlcq7C5KFYVBQGfNmjW2fv16n68qDs85/URXsdwotJ8grx17b2fbbtPDV2KvXr3GVy4rzz1228X+7ioVoyV199OpY3u/zdqwgblV6RjsP9p2zMtMgWBcjnWbrpfKlSr5FiS61hRsWOUqjPWp1MK1ALvqXxdFHEj16tX8danB5PfZs3/oGqhataq7NkuXl4IiW/foGnGN6bo8+7QT7L9XXuoDOrq+g1YyKutJxx0RUa7te25l11/9z1BAZ9Xq1aHr+8B997QjDz3AKrntFKRUWYOk/Sg//WncISUd44N33WRtWrX0lePr1m3ed/eune3qy/4WbG7y1LYK5Nzwf5eZArnKU38FBRVs4FknhwI6quhftryw9VMD1+KtbZuWoXyY2PIC7w/9NFSIW6/7t7VvVxiQD82MMnH26SfY6ScfGwrorFhZ2C2afht1j6h1o36Ldb51nQQpuOb0Wa1qFf8bq9/p8BSsU7duYRCyhmtBp3nh168CLsF6/7vxalfmNhH3kfIr+lut8tzsgptBQEfXpX7b9ezYfdedfX66B4LENRxI8IkAAggggAACCCCAAAIIIIAAAghkp8DmV5Gzs/zlWuoOroItSBMmTQ0mk/pUhd3iJUt8C4JJk6fZkmXL/PZPP/eK7+JHLQdU6aykcVLOPeMkX6Gs8VPeeOsDPz/4R5X1Tz//ir393kd+lrreufPma3xlXt8de7vBwAe7fS0NVvefCxcttgceHeTf4NaMow8/0I4/+jC/jwP329Nef+v9iPX5krkChx+yX6hiedr0P3xBFdwZ9MIr9tvocTZpU7dgCkhc6VqwbOcCJar41XWy1L3FH550XU6YNMU+//Ibmzlrjs2cPcdKm5e6fVLLH7UkUzr3jBPdtby7n9bYJne4Fi0qm8p1i6ts132lLgdVcR60qDj/vNP9+srrmhvvsClTp/uAyx03XeMCOQ3dWCT72WtD3rOjTz7Pt+hRAEjphlvvsZ+KdIGoFhFqdaH75X/3PGJff/ej3/fF559t/XbewbpoDKwiJiqb7sOhn3xmo8eMt9Uu0KrvqiRX0rHd5Fr4BEnBnyAAEMzjc8sKfPn19z4IV716dd+S7bbrr/K/sR998rm99e5HoWstKKVac+236Tqd4Lovu/F/9/v7RIHDe2//r7/Xzjz1eLvWXY9D3hkauq51XR1zysAgG/950pkX+k+14NG9pdaT9z38VMQ6JX1J9Lf6gvPO8NezynHX/Y+ZjltJLyFc6VqIVq+2OaCj+VzDUiAhgAACCCCAAAIIIIAAAggggAAC2StAUCeJc9e8eeEYCqo8C68UV/c4t11f2LVNeHbvfvixPfnMS+GzfMWvulsrmj4e/mXRWTb04899Kx1VMDdtXNgFVPhKqvAOAjqarzLd67qMUvds2mavAbvaq2++G76JGyz87VBARwvedJWTxx11qF8/PGgVsRFfMkLgJtetmpIqidWNWNA9lK7HYGwOdT2mCufwpOWvuWCdgjpKGp9jdJGgjsbo+Y8bd6RoKk1eGvcjCOgov7ffGxYK6nw8fEQo2KRyfeaCSLrudL3q/lLwpmWLZi4gWccX5f2PPvXz9EWt0YZ9+oWdcMxhLsATWVHtV47xz/Y9t/ZLlLcCOkra93ODX/NBHX3v1rljse4UTz/vkmIV/2phJP8Grpss+QcBWAWrSJknMPDif9vlf/+rqUWWUqOGDXwQ+5gjDrYPhg333fgFpVZQW9eh0n2u+8rgN17dHGrcpC6dOrjWbiWPoxPkV9bPRH+ru7ixqZR03wUBHX3XOFoag6pbl076GkpcwyEKJhBAAAEEEEAAAQQQQAABBBBAAIGsFCCok8RpCyr5VPEX3qpgXVg3ZsouqBisVLE4758bNw8WX3TXeiO8/y47+kpttbBRJfT69RtClfdF14/2XWMoqMJaZVDlfUlJZVc3WJVcV1ukzBbo7LrNK5rULdldruWLKp7Dk97S7719T2vkBlefNWeeLVy0KHxxsekN7jqLlZLNq2g+c4qULXy5xvoIkq5bpfDjDFobBOvUdF1WKen6Dg+qBMuLfqo7qyD4VdONYaWWC+EpuFfq168bPttPh3dPGCz8+dfRttMO25u6xXrm0bttxDc/2IuvvOkHvQ/W4TNzBBT4vvq62zcFcw61vjv28t2dKTCnbvzq1natG13rFiV1z6eka+K0k47x08E/6m5NaUt2URntt7py5co+yKiyjRozTh8lJq7hEolYAQEEEEAAAQQQQAABBBBAAAEEEMhoAWrykzg9U6fPCK3dvn0bG/v7BP/9j5mzI7rfCbrcCa2cwMQZpxzrKxmDgFACm8RcJQjS1KtX2Noh5oosyCqBdz4YFirv0qXL7ffxE2302PF+zI3QAjdxneuKrHvXTuGzSj2dyrwSLUR4a4ituneNupkq3oNWMlFX2DSzebMmocXqtk1/0dLcuZFBsWjraJ66b7vgvNNtt347+UCoPvWn7rXUraFaSpEyT2D+goW++7P7XQucU44/0g45cB8fGOyzw3ahwjZsUBi40W+wxnSKlla6MaoyKTV0QdsgLV++IpiM+8k1HJeHhQgggAACCCCAAAIIIIAAAggggEDGCxDUSeIUhY+js9fu/UJBnSSyiLqqBpU/aL+9/LKly5bbV+7t/1luTBN1Q7X3Hv1DLX+ibhxlpt5CV5o/f2GUpczKVoGiXflFOw6NERMEdDTOzrc//Gxr165zLUtau1YKvaNtEnNeKvOKuZMoC8JbHV3131ujBm/Wu9ZliaTwvIa882FE91Th20+bUTgmUfi8aNMK2mi8oJdee9t1W3iIb/mhVm4avF4p2XFTou2DeekTUDBw0AuvWu3atWyP3XaxKq6li7pm01hmizd1Sahu/q654X9RC7EswcBJ1I3TMFPd/gWtzTSuUyKJazgRJdZBAAEEEEAAAQQQQAABBBBAAAEEMleAoE4S50Ytc1ThV61aVV+J++IrQyy8+6gksopYVV2uBemq/9zqB6kPvqsVQNWqVYOvJX7qDfOgtY/GUyDll8BJxx3pD3jN2rX29yv+Gzr4+m4MmGSDOqnMK1SQBCbCg6dqiRA+TkhJm1coqBCxilrzaAwRBV4au1Y6GhslFUmV6Xc/8LhrnfO0/e+m/7MWzZq67u62SUXW5FEOArNdl4RBqlq1ip+cPmOmb6Gj77Pc+V25MnWtcoLf5GCfqfxc4oJRGoNql516u7G1no3IuqBCQcT38C9cw+EaTCOAAAIIIIAAAggggAACCCCAAALZIxC7xid7jqHcSqo3nB8f9ILfnyrpHrjzBjv2yEOsxqZxPkpbkKBljbYPr5TeY7e+VqVKYYVjtLxr1qxhh7puhIKkir0L/3Km/6ou2IZ+/HmwiM88EXCXpU/hlcgaV+aYIw5KWiCVeSWzcwUj16xZ4zcZeNbJVrQFQu1aNa1Tx3ahLGfPnhua1ng3RdOkKdP8rJ379LLe20UGXjTejlrKJZrU4inc1geNXOBIKV4FeqL5s17qBHbo1dP/PrZt0yqUqe4FtUA76vAD/Tz9Tv7ixklS+vSzEf5T5/c///57sfFzmjZp5IN3fiX3j7rdVNL6sa6htS64qtS1cwf/mY5/vv7uR5+tngf33P5f1wKpr3/pQF0ndomyX67hdJwF8kQAAQQQQAABBBBAAAEEEEAAAQTKT4CWOklaf/LZV26g9F6mCkMFY4498mD/p8pBpfAATaJZj/jm+1D3TXe4t/7nzV9gdevUturVCweFj5WPKhNPPfFoO+m4I3wLIgWXggrnd94fZmqtQcovgd9G/x7qVuq5x++xxUuWmlq7qKVKsimVeSWzb3Un9fTzr9i5Z5zk74F7XUX1kqVLfTdy9evVMwViFPQ56ayLfLY6xoWLFpsGs99rwK62+647+/vwg2HD7bGnXrC77n/M7r/jBn9vXPGPC0xjj6gbLY05Vb1aNZ/H2ef/01vFK2ejhg3stuuv8quscmOrLHD7bOj2GdynP48cFW9zlpWzwMH77x0KtuiaWr16tWtlWS30G6nivPH2B777Mk1Pcy11fnLnUK0d27drY4MevctfV/pN1++xPjWO1ZWuNaXShy5ofvrJx/r8rnVBIAX4KlWqaFdff3uoa84ffxnlg0gKTL78zEN+zCW1Arr4smt9Hqn4R9d4l04drIMrs1qMnX/u6TGz5RqOScMCBBBAAAEEEEAAAQQQQAABBBBAIGsECrKmpBlU0JvvuN9uvP1eV9G8LFQhqAq/IKCjAI/GaBj6yeaWMkHQJ9ph/Pjzb/bJprfElYcqAFVR/OuosT7Ao22iDcCuoI1aNWgbvaWtgI4qLzXeh8aNSDYlOk5JsvmyfukF1HVYMumhx5+1GTNn+U3UbV/TJo399fH2ex9ZcA1u3BSALCnfVOYV7Gv9hsSO58Nhn9mDjw3yXafpuq5Xt641adzIB3R0jf8+flKQpf98+rmXfQW8vgQBrCUu2KM0d94Cu/z/bgoFbWq5lj7Nm7l7bFNARwGh1ZtaBsW7B1a5oMBMN9aVzonuz1YtmocCOgqA3eW6YyNljoBa4Kxdt84XSNeQzlkQ9Nb8Zwe/Zi+8/GZEgW+87V7fwlHXmH5XGzdq6IOFmtb988uvY0Lrr3N5DHl3qCnAp6Rgo9bRGFZBGuy66NR4TcpP+1ZLocWLC6/L4Dddy5JNRa/TK/9ziw1+dYhNmTrdVqxcaRpP6/GnX7SglVpw73MNJyvN+ggggAACCCCAAAIIIIAAAggggEDmCVTo1mtA8jVKmXccW7RErVo2929JL1223DQuQ2nH2dG4J+rGR5VyCujoze9o6ab/XG6dO7b3LQ5OH1jYTVCPbp1N+0/VmCHR9su87BHQNam391WhPGHilDIVPJV5laYg2n/3rp1cpXllmzlrjm8tsWrV6qhZqQvCKlUq+0BOtBV0j+leUeulOfPm2/gJk0PBoGjrx5qnwGvv7bb2gd3RY8eXKo9YeTM/tQLqqq9Nq5a+tY0C4RMmTfH3RBBUibY3BXE6dmhrXdzvrAJ+M1xXa7+PmxgK4hfdRi1klixbZitWrCy6yH/Xtdu8aWPfGijqCmma+dj9t/qA6ISJk+3ya26O2AvXcAQHXxBAAAEEEEAAAQQQQAABBBBAAIGsESCokzWnanNBiwZ1Ni9hCgEEEEAAAbNtt+lhV//rb57i3Q8/ticGDYYFAQQQQAABBBBAAAEEEEAAAQQQQCAHBJIfaCMHDppDQAABBBBAIBcEdu3bx/7217N894LqenHu3PnWonkz69aloz88dTX3TCm648wFG44BAQQQQAABBBBAAAEEEEAAAQQQyEUBgjq5eFY5JgQQQACBvBDQGD4ar0ddC+rPemw+bI33c8udD8bsynPzmkwhgAACCCCAAAIIIIAAAggggAACCGSLAN2vZcuZCiunutXp2rmDzV+wyD4e/mXYEiYRQAABBPJNoE6d2nbQfnu6sYNamKbVWme8Gzvo/aGfWryxg/LNieNFAAEEEEAAAQQQQAABBBBAAAEEckGAoE4unEWOAQEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBHJeoCDnj5ADRAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQRyQICgTg6cRA4BAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEcl+AoE7un2OOEAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBHJAgKBODpxEDgEBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQRyX4CgTu6fY44QAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEckCAoE4OnEQOAQEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBHJfgKBO7p9jjhABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQRyQICgTg6cRA4BAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEcl+AoE7un2OOEAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBHJAoFJ5HMO6fzxTHrthHwiEBCrffkpomgkEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBHJBoEK3XgP+TPeB9OzUJN27IH8EEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAIKsERk6Ym1R5y6WlTrKFSuoIWBkBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQyAMBxtTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gsQ1Mn+c8gRIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQB4IENTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gsQ1Mn+c8gRIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQB4IENTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gsQ1Mn+c8gRIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQB4IENTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gsQ1Mn+c8gRIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQB4IENTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gsQ1Mn+c8gRIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQB4IENTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gsQ1Mn+c8gRIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQB4IENTJg5PMISKAAAIIIIAAAggggAACCCCAAAIIIIAAAggggED2CxDUyf5zyBEggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAHggQ1MmDk8whIoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPYLENTJ/nPIESCAAAIIIIAAAggggAACCCCAAAIIIIAAAggggEAeCBDUyYOTzCEigAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA9gtUKo9DWL50UXnshn0ggAACCCCAAAIIIIAAAjkhUKtO/ZQfB/+/LOWkZIgAAggggAACCCCAQJkFkv1vf1rqlJmcDBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQACB9AsQ1Em/MXtAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBMosQFCnzIRkgAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgikX4CgTvqN2QMCCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgggUGYBgjplJiQDBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQCD9AgR10m/MHhBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQACBMgsQ1CkzIRkggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAukXIKiTfmP2gAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAgiUWYCgTpkJyQABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQSL8AQZ30G7MHBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQQKDMAgR1ykxIBggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBA+gUI6qTfmD0ggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIAAAmUWIKhTZkIyQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQTSL0BQJ/3G7AEBBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQQQKLMAQZ0yE5IBAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIJB+AYI66TdmDwgggAACCCCAAAIIIIAAAggggAACCCCAAAIIIIBAmQUI6pSZkAwQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAgfQLENRJvzF7QAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAAQTKLEBQp8yEZIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIpF+AoE76jdkDAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIIFBmAYI6ZSYkAwQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAg/QIEddJvzB4QQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAAgTILENQpMyEZIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAALpFyCok35j9oAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAIIlFmAoE6ZCckAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEi/AEGd9BuzBwQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEECgzAIEdcpMSAYIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQPoFCOqk35g9IIAAAggggAACCCCAAAIIIIAAAggggAACCCCAAAJlFiCoU2ZCMkAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEE0i9AUCf9xuwBAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEECizAEGdMhOSAQIIIIAAAggggAACCCCAAAIIIIAAAggggAACCCCQfgGCOuk3Zg8IIIAAAggggAACCCCAAAIIIIAAAggggAACCCCAQJkFCOqUmZAMEEAAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAIH0CxDUSb8xe0AAAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEyixQqcw5kAECCCCAAAIIZJxA00Z1rUWTBla9WpWMKxsFQgABBHJRYNXqtTZz7kKbM39JLh4ex4QAAggggAACCCCAQFSBm6+72p4f/KqN/G101OWpnNlz6x524nFH2eVXX5fKbLMurwrdeg34M92lXr50Ubp3Qf4IIIAAAgggsElAAZ3mjevbpOlzbOnyVbgggAACCJSDQJ1a1a1D66Y2a96ilAR2atWpn/JS8//LUk5KhggggAACCCCAQF4LKKDTc+vu3uDyq69Pa2BHAZ2br7vK72vkb2NyKrCT7H/70/1aXt92HDwCCCCAQC4KqIUOAZ1cPLMcEwIIZLKAguj67dVvMAkBBBBAAAEEEEAAgVwXCA/o6FgVcFHgJR0pPKCj/BVI0v7zNRHUydczz3EjgAACCOSsgLpco4VOzp5eDgwBBDJYQL+9dHuZwSeIoiGAAAIIIIAAAgikRKBoQCfINB2BnaIBnWBf+RzYIagTXAV8IoAAAggggAACCCCAAAIIIIAAAggggAACCCCAQFwBjaETK6UysBMroBPsO145gnVy8ZOgTi6eVY4JAQQQQAABBBBAAAEEEEAAAQQQQAABBBBAAIE0CIz8bbQb0+b6mDmnIrBTUkAn3WP4xDy4DFhAUCcDTgJFQAABBBBAAAEEEEAAAQQQQAABBBBAAAEEEEAgWwTSGdghoBP/KqgUfzFLEUAAAQQQQAABBBBAAAEEECh/gSbdzZpuZdagvVmtJmZVarky/Gm2doXZ8rlmCyebzRllNndM+ZeNPSKAAAIIIIAAAgiYBYEdtcyJljQ/2RY1BHSiSUbOq9ioebtrI2el/tvaNatTnyk5IoAAAggggEBUgdbNG9n0WQuiLmMmAggggEB6BVL1G1ylavWUFzRb/n9Zh93NdjjDrPPeZg07mNVoYFapqlmFCoV/mtY8LWuzs1mrHRzVRrNFU1NORoYIIIAAAggggAACJQjMmTvPfh01xvbec7eoa2q+lmu9klK+BnSS/W//Ct16DXDvOqU3LV+6KL07IHcEEEAAAQQQCAns0qurjfjx99B3JhBAAAEEyk8gVb/BterUT3mhM/3/l9Vva9bzWBes6Vi6Q18w0eyXwWaLp5Vu+3Rs9d//+5d1bN/OZz123AS77qb/pWM3JeaZKeUosaB5vEKlihVt/YYNeSzAoSOAAAIIZLtAWQMyZd0+m/2S/W9/ul/L5rNN2RFAAAEEEEAAAQQQQACBHBBQa5s+ZxW2xCnt4SgYtMflZt89bjbjh9LmEn27rl06mSoagvTt9z/Z1GnTg68xP9u2bmX169X1yzW9pVKmlKM8jv+oww+2goKCUu9q+fIV9t6Hw0q9fWk2VNCt93bbuqDOent+8Gs2+JU3SpMN2yCAAAIIILBFBcrSFVs+B3RKc9II6pRGjW0QQAABBBBAAAEEEEAAAQRSIqCAzo5npyQrq+Dq8nc8pzCvVAZ2Lv/HRda0caNQIXfrt7NdeOm/Q9+ZyByBs047sUyFWbtuXbkGdZo2bWI7bL+tL3PlSpXsuKMPI6hTpjPIxggggAACW1KgNIEdAjrJn7HSv76S/L7YAgEEEEAAAQQQQAABBBBAAIGQgLpcUwudVCflWa9NanJt2qRxREBHuXZo386qV6uWmh2QS14LbFi/PuL4N250A0SREEAAAQQQyGKBILAT6xBuvu6qUAtoAjqxlOLPp6VOfB+WIoAAAgggkPcCnTq2twP23cs7aFrp/Q8/9p/l3T2J3yn/IIAAAgjkjEDP48rW5VosCLXY2dblPfy2WGskPv/E444stnKFCmZHH3GIPfPCy8WWMWPLCowdN9EqViz+/mqD+vWsYYPNY1X9MXO2rVy1qlhhFyxcWGxeOmfMX7DQPvrkM9tt17623gV4nnj6hXTujrwRQAABBBAoF4EgsKMATrSk+c+5LkdPivLfWcH6l199vSkfUnEBgjrFTZiDAAIIIIAAApsELvzL2RYEcsJR9t93T/9VyxTYmTBxcvhiphFAAAEEEChRoMPuZg07lLhaqVfQGDvtdzOb/Fmps/Ab9uu7UyiDDRs2hgIG++w9gKBOSCZzJv7+r6ujFubwQw+0c884ObTsrvsetlFjfg9935ITd9zzkOmPhAACCCCAQC4JlBTYIaBT+rNd/PWV0ufFlggggAACCCCQQwJ3335DKKCjoI1a5wR/QRBHQZ1YgZ8cokjqUKpXr26HHLivHez+ypKqV69mhx60v53jKqD0d+B+e5clO7ZFAAEEMk6gY+H7AWktV6cy7qPnNj2shvs9DtK9Dz4WTFoj1+qjRfOmoe/JTlStWtXat2tjFdTspxSpYcMG1qVzR2vmxmRJZ2rTuqXVqV0rZbto1bKFqeylSXXr1DGVJ9NTlcqV/bnt0L6t1ahRPe3FLYtp2gvHDhBAAAEE8logCOwkg0ALnZK1aKlTshFrIIAAAgggkHcCCtQESYGcaN2sqUu2oMWOpsMruoJty/KpSq7+u+xkbdq0tvr16vpKr9WrV9vESVPsixHf2Jq1a8uSfdq27dypg+lPb3OXNlV2lUFnn36yVa68+T/VOnZoV9rs2A4BBBDIOIEmPcxqlz4ekvDx1G5m1qS72dwxCW8SseIJxxwR+j5z1hz7cNindv7AM00D2iuddPzRdtud94fWKWminnueXX7pRT4YU61qFb/6n3+aLV22zN4f+rE9/ezgqFnUqVPbTjz2SNtxh14uIFI/tP9g5Y0uk1mufHfc86CN+X18MLtUn7Vq1bQrL7vYOnZobzVr1HDP38Js1rmuwRYvXmIvv/aWvf3eh1Hz1vPv325bpekz/rD/++8tpjGJLr7gXD8OUa2aNUP5qcy/jBxlN9x6p61cuSpqfpp5+CEH2JGHH2z16taxShUr+vVktsr9N8GQd963Qc+9FHPb0i44yu0veDljqDvnz7vuYRSAO/fMU621CypVcwG5MWPH2dX/vdnvYped+/gXOjp36mjVqlW1ggBtUwFWrV5jnwz/wh545EmLNmbO7TddGwp0vf3uh/bqG2+Hip4O01DmTCCAAAIIIFAOAkFgJ1ZXbOFFIKATrhF7enNNQex1WIIAAggggAACeSSgAE3Q5VqsgI44FOgZP3FSqKWOtgla8JSVq0qVKnayqyirV69ORFaqaNq251Y+oKPATiam1q1a+GItX7681MXbvf8uPqCjih+dgwWLFvsKpFJnmMINVbnW1L0VPviVN3zlXgqzJisEEMgjgWZbld/BNnX7Kk1Qp6CgwLbu4SJCm5LGPVH6ddQY67XtNn667047+M9E/mnSpJE9+8QDxSr8Vf9f1wVtjjvqMD9osCoz1q1bF5Hlrdf/X9wWKgoitGzRzP5383/s0suvKXVgRy9T/OPi8yNeKggKokBW40YN7a/nnm57DtjVrvrPTcWCMRq3pmnjRn4TrasAT7+ddwwFcoK89Kkyb7/t1nbfHTfbmQP/Fr7ITyugdP21V1hX1xqpaJKZWlAdf/ThNqB/PzvrLxfbn4r0pCiphU1wHH133MEa1K/vWszuFZF7q03Pe/33z1X/uiRiWdEv1V2gR9vrWC689N9FF/uAVxDk26HXdhFBnVSaFtsxMxBAAAEEECgnAQV2ShpDR8u1HqlkgYKSV2ENBBBAAAEEEMhHAQVoorXQCbfQOkEgR8GgVKXDDt7fB3TU2uWHH3+xp5550e6892F76dUhNmnKNBs3fmKqdpXyfJpsqsyaO29+qfPu6CqTlH53xzl23ASb5/LSG89bOqn1VPt2risZ18XckiVLt3Rx2D8CCGSxQP325Vf4BqXcl7q9rFix8P8yK16gViFKr73xTqjwarHRe/ttQ9/jTaiVSdCCY60L2ixdttzU+iU8de/a2bVqOS98VuG0ohib0rp1623J0mWmlkPz5i8o1jL0ysv/Hqya1Gdb1zL2in/+LSKgo9Y0i93v/eo1ayLy6talk916wzUR84p+0bHu2ndzQEfHqnIXTc2aNja1dCmabr7+6oiAzuo1a/1/A6jFrgyCpO3PPfOU4GvKP9VStmhAJ3wn4d3n6TpZuWq1zV+4yJ+fZctXhK/qWj+1swG7ggnvhQAAQABJREFU9YuYl8yXspomsy/WRQABBBBAIJUCPbfuYfHG0NG+tFzrkUoWoKVOyUasgQACCCCAQF4JBF2qBcGakg5egZ9Uj6vTvFlhnzwfD//cfv1tc585M/6YafqLlVq3amm1XWseBUKidW+i7Wq7cQHU9/x6VyE0e84cK1rhUjRvdYW23lVEhb8BrLe3NYaBusBZtGhJRJnq1K7ts5i+qZxqXdSqRXMbN2FSzDIV3WcN93ay0oQJk4su8t9VgaTubBq5t6Dnuwq9OXPnRZQv2kYqs/50LPGSglKN3FgHk6dOs1WuYio8qbJR9Yqq3Av3CF9H0xpXSN3FzHCBqIWulVG8pP3Vr1fPVdRNLfZmerztWIYAAtktUCu9w8BE4JR2X4cctHlstBkzZ4Zapfz480j3O7jWtaAs7D7t+GMOtx9++iVin7G+KFh/+10P2B8zZ4VWUfDo/PPODLVm2b1/X3vw0SdteVhA4PMvv/K/4UOHDXfPrrmhbTWhsXkeuPsWa75pbJ0Gm7osjfc7HZHBpi8K6ISnz7742m7+3z2hWep+7LYbrw2NMdTBfe/Tezv77oefQ+sUnVCQQ1533/+IzV+w0C9WV3JX/OMi23abzc21em3X00Z8/V1o897b97TgBQfN1LIbb7s79BzVmDX333WLb52k5QcfsK89/PggTaYtrVy1yrXSGuteZplkq113akGAapF7zn0x4lvTOfrqm+9t/YYNEWXYbde+rsu9C0Pzdtmpj3362Zeh78lOlNY02f2wPgIIIIAAAqkSUKAmka7XtD+tRxdsJcsT1CnZiDUQQAABBBBAoBwFFLCoVKmi32OliiX/p4rWP+LQA01vGCvgoLTfPnvaB25sgtGuv/sgqRuYQw/az+q6PvnD0xpXMffO+0NtytTprh/8ar5bGfV9P/SjT23vPXfzAxzPmj3HXnjpdb+ZutrRmAbB29uauWzZCnv0yWf89sH8pUuW2cCzTwsNkHyAq9hSpdQ33/0QvvuI6fPOOtUHRILjCCoUVYFz130P+0BKj+5dba89+keMp7B27Tp77c133FvBs0P5qYKwqqtw1PgMh6rlU926PqATa+yjrp072QH77ekrDYNM1P2PWkfJbo/dd3XnpfB86M30v1800K/2vWtJ9dkXX/npNq1b+TEFtN8gLV681J4Y9Lz/Gu773gcf2SHufATjUqhVltZb5t5cJyGAQO4LVK1ZfsdYpRT7UuChtXsBIEgfumdCePp55K+2c5/eflb3bl38WC9FK/PD19e0WtZcctnVRWfbu+73UC8JBOP3qDXGmaeeaPc88Gho3edefDU0XXRijQu0P+ICGtf8+x+hRQrATJo8NfS9pIntXDdobTZ1J6Z1R435PSKgo3mTXUvZc8//uz3z+AOh5+2Ffz3HTj3rfC2Omv570+3uufdjxLKlrrXOtTfcbq+/+GRofvAyRzDjgoFnBZO20I3jc/0td4a+a0Itna5y49k8+dBdfr6evXrZomjAK2KjMnx56bUhvtVwtCwUrLrxtsJyRFuuZ+RZp53on6Va3qpl82irJTyvtKYJ74AVEUAAAQQQSKFAMgGdYLcEdgKJ2J90vxbbhiUIIIAAAgggkIBA544dElgr8VX0ZrG6pFHS28rhb/IWzaWqG3vnrNNPsnZtW7s3Y9f7bln05qyCIru4Ll+CpJYjJ7gBphXQUYBEXaOpck1JAYig6xRVgikpaKGAQ40a1f36M2cWBkv69N7eFNRR5ZGCEMpHQYj5Cxb47fTWcpAOc2PPaL/qGkfBEZVJbx7HStqnK5pt2Fj4hq/KuXzFSv83b/58H9BRC6P99t7DB0LU9YwqkrRelSqV7cjDDgodh1rkBIGVE487yrWEqet3u8hVjEVLqrw86IC9fUBH3arpTXJ/zK7QS5ct862b1qxdG9pUxxOUbeq0GX7+9ttuYxpYWvtV65xx4yf5smlcpMBV50lJx3rEoQe5bogKLCiTTOOda78h/yCAQM4I6Pcuk1MQYFEZ9Tv71jsfRBT3ldffCn1XEOZg98woS3rx5TciulFTN13JpPGuNWh40ng0yaSd3MsK4enBR58K/xqa1u/77+MnhL5rvJd4adXqyBafwboKRIV3oabnVnhq6FqMBuntdz8MJiM+57gWS+Hd1/VwwbV0JHVBp25gy5KCVkrKQ+MGliWV1rQs+2RbBBBAAAEESiNQUkBHY+jESgrs0BVbLB2zkl9/jb0tSxBAAAEEEEAgBwXU7ZoG/dXflkrDPxvhggz7+CCDWqWor/3h7k3X0e7N4fB0gBt0uI7rTk0tax5/6jlb6wIPW7mWLPvts4dpUGIlBWxOPv5o3/pHgYgnnn7et1hRF2OnnnSsX2fa9MLxatq0arlpG7MFrj/89z/82HdtppkKSOzqBpBWCm+d4mds+qe1a6kSJAVDBj33kg/oaFsFXaq5Mqk80brEUZdmetNa62n9BQsX+u2D/GrWrOGDJm5zH7x6Y8i7fpFaxxx9xME+sKPt9Ca1uqHbnP60N99+37+xHW2/Wi/4j2WV4XHnUzSpOxn9XXJhYeucd94f5vKbElpNZdMYASrb519+47ri+ckv+8s5p7vAVjXXnVtDX662m3y03uw581zrp9e8xUnHH+W7k1P5vxjxTShfJhBAIHcF1rqhRqrWKp/j076STeHjnkydNt23DAnPY/SYcX7slBruN07p4P33seB3OXy9RKcVLF+ydKkFQZKGDerH3FTdb3Z1Y9po3SmubONdEF0tV8qSWrfe/NxQi6N4rXx++uVX05g6Sgpo1XMvDiyO8dJAvDLpJYbKUaokqrtWsxp/KEhdO3d0LV1OCr7G/KxfQoAp5oYpXKBu4Tp1au+6jmvnuzAdN2GiGxNvZoldn6aqCLFMU5U/+SCAAAIIIJCoQEkBnaCLtV9/Gx2zazZa7MTWJqgT24YlCCCAAAII5KVA+Bg5CuzEG1tHy4MxeBQASVUa7/qrf/ypZ+3oIw/1rUwUGNjfBWr0Fq66GdN4OXXrqAVIW7/LZ1942Qd0VNG154Bd/bxJLrihtM9eu7uWJjX9Ni8MfjVUsdKhfeG26rpsw6b+75u6wZaVFNBRt2Xh6SBXYadghCpngu7GwpdrummTRn6WWp/ord4giKJWPUp62zuY52dE+UfBJqU5LugRnvq6PvjVmkXlHeKCNEGaNn2GO7Y/XQBM4+w08cGTdm0Kg0t6g/kxF+wqOjZOsG3wuXLlKj9ZtUpVbxzebV2wjmx1/ErTXCVieDpg3738shl/zPIBHb1xvc+eu/uAjo555K+j/OqB78KFi+15dy6CJG/ln+yb5cH2fCKAQPYJLHfDwpRXUGd5YcPMhJHUSqaua8EYpIWLF9u+ew0IvoY+FXyvsamLthbNm/ogS0njiIU2jjKhbsmCoI7GYyua1BryJPeSwv+zdx5wehTlH5+7yyWXS7v03gkJhBASOoKAitKboAiCiAoKNvCvgIJIR0EEpUmTIgIiiFIU6UrvJCENUkjvl3aXXJK7+z+/2Xf23vfuvbv3aq58h8/ezM7Ozs5+d9/dML99nifE8kneLmuShqQB9v4IqTpLkLB9sT3rk5PirUn4b6w0ZsyolK723nOys1dgrWnFitT3Zq07NGIDuX772Xnfj8Wuyl038PJU7o51CEAAAhCAQIsmkKmgo5OYYqKOBB4JOOkSwk46KljqpKdCLQQgAAEIQKAdE5CIE6x1fvC9b3trFQk9lZMEHU3mh5SuTdhWn3yDBYj+030POlmiHGlWO7JyGWZfEu86YWf3wYfT3B6TJ3ohocisb+QCbvdJE714o2PJzcnT/37OH3aExdpRmvrRDKc+Q5IrM6W16ypckkkoUpqRFItH63LzFtyZPfvCy6pKm8L++odpsngzNBGnQJZEtaX8hMucT02sSU6aMFRS7B+JWskpuI9bU1joqwcOHODzlStX1yroqKHGu8/eu3u3aBLp9rDA14889oTtG4k9ahNEMH1NXjluRGA5/9MF7hiL3yOxTSKTJrF0HYLrtpjPR9PVZZzCJKbEHRIEINA+CBTOc6536tx9k534mvl161rWg8lpsrmX1FJbkrvLm267q7Zm1W4PHxiogcT65KR4ORI3qkuymGlIys2t+N6zLPEhQnX9hWd62C5Xo42Z8jvXzXVcOPbipLhyoa45cr0Db73xNymx9ioft4GXp3J3rEMAAhCAAARaLIG6CDrhJBB2AonM84p/uWW+Dy0hAAEIQAACEGjjBIK1jk4zWOKoLAsaJYk5ye7ZarLm8Ts04I8sUeTb/3tnnu5Fh53G7uhFnb59evteI9df+/myLE7kpi2IMhI7unSJvnb+ZM78lFH0Sey/bJl9Lm4p11ymhEktuTBLTmPM9YuSxJTqXMyk7F8pOPWghMiieDU1JcX8kRiiNL/SGAp6RHFxlixN/UJaQkmYLArucoJIsiTDCa5tZtFz+133e9dvCuAsa6EzzzjVWxvJHZDS4EFRYOe1lc5BcYfCmIN7Ook5spR65tkXYlEplU8qX7nuUdK1JkEAAu2DwLJpzu1Q8V1Ak570cjtWXdLuu02sS/O47QH779MgUUfxzULaYC48Q9rThPZkQWdzyRb3z6f+7aZOm+Fdbw63jxdGmZi+/357hV3qnCv+W3iv5tci0oR3WjjIbIvD1pgp/Fsj9PnzX13l3/thvaXll19yQYqgI2tXfdCwyiy5hgwa5D9IOfjA/V23NNZXLe1cGA8EIAABCECgIQTqI+iE4yHsBBKZ5Yg6mXGiFQQgAAEIQKBdEZBI84db74zFmyDsHOo+l8IhWPVo+43XXen3aQqBR1YvEmzyOnXybtY0iDD5tWFDkXv/wylOkyjBjVgYpISIIHgkCwaKVRBcfQWLmGDRI0FCk1vJKcTaqSlmgeLBKGn/yu53QsDnpctr9gE0OuESbtu2Uqf4NskpJxFfQNuSU5jok1s2iTMSsjoZJyVZ9WSatO9fH/2HF29OPP4oP0Eli53nX/yv7yLEd1ixYlVKl4MGRFZBqpwydboJf/Oc4k9UToGvvj4vNFdGIY0ds4O/rlpvinsnHIccAhBoWQRWzHBugz0Su0VGiE02uA3LnFsxM/PuDzxgv1jg116y8qz8TkjubccdRjuJ4Urd7CMCuSKbMevj5CYZlfXsDgK3dpDFaUhHH3FoKPp3zNk/+pnFJYs+SNAGuT6Tu7aGiDqLFi/x7jfVnz5w0DuyqLhYq1VS8kcdeudp38ZMq+3c1W94f8v1qix0W2IS9/59I9erGt8HUz5yP7/kynioei8qjRwxzE0Yv1NcTwECEIAABCDQ1gg0RNAJLBB2Aona8+zam9ACAhCAAAQgAIH2SCAIO4qVk26yXaKPluQvauWuLXmyp67c9JVwEA+S9915p7EWW6fAV300Y5bPJWIoyff/O+99GAs63bp19fFZ/LYk92FBBOprky9f/9qJ8WRRED6GJeLQFCft4w9gfzZs3OiLEpVkcRKSLFrkHk5peCIvTjMJFr56XrBgcdg1bT5kcBSoOhwvuZFEFyUFxw5JotUu48f51Tfffs/n+oI6TIQtXFTz8UI/yfniJUtj65oQAFzb8yxwtdI6i/mQnJLHquDZyYKO4lKEFPNN4iMro8O+FAmFsqSSGKYJsiMP/6L7zD4VX5ynq9N5Kr6ERCESBCDQOgnMeaHpx/1JHY9xgj1XQlKcmgt/eaW77oZbql0uu/q3obnP5YKtPunoI77kcjtUfHM5+5PIMlZ9KV5LSCVbSlIEnVDf0Hz2xxXHU1/fOv2UtF0qZto+e+0eb6v8AUK8oYGF5Lg+Rx3+pQb21nS7h3d/OILegyQIQAACEIBAeyPQGIJOYBaEnbBeOVeMHR2vvaeKfzW2dxKcPwQgAAEIQAACaQnUFitHgo+En2DNI2FHYk86ISjtAZIqP7v/vm64iSsl5lpmvbme2bp1m5NLsi7m4ktJk/6zZn/iy58uXOh69Spw/fr2dqee/BX7knqVU6Bn1W202Dm3332/U4yWrSaGaKLstJNPdEVFm3x/Id6NtilGjNLAAdGkmb4Qrpymm5C0x+TdvFhy9pnf9P12NpGjW7cuXky67c573YCwf6W4MBJ+gsgyP40FS/Kx+tq5KKUL9vzJ3Hn+C3DF5znrW6f5uDYhkPecufPd2+++7/cNFkNiGM7Tb6jmz9e/doIFBO/ulluAaX2VrclDubTTV9Iv/e+1eK8SsxySqzVZBu26y07erd3f//mUiTiLXKnFX8jJyXaazJw7/1Mfg0iCk772/ueTzziNPfCVFZEmL2W9JJdyYiOhSK6ElI449BCzFoqsf95+7wNvmZWu7svHHuEFtv79+7pZH0f3RDxYChCAQKsgMPdl54aaftt7dNMMd7U9GuZFxoYZHUDP9VEjR8Rt9WytHMMs3pgoyEpFbikL7F2lNHHX8fZcU0wxe4hWSgMH9Hfn/uAsc9F2d/zuURNZcJzxjZPj1nqm3nP/Q/G6Yr+F56I+LlB8uQULK0R7PcPP/8kP4vbpChKoQiooiMYa1pXrXS8hp7PFr1M65HMHmgXQ2/Zu+cCv64/O68pf/TxFfHri6Wfi7Y1ZkJXoUSbwK+ldd9H557qrrr2x2usxccJ49+HUjxpzCBn1tWhxqkvUAz6zj8Wk+2fKvl8wlskfZKRsZAUCEIAABCDQygk0pqATUARhRwJOuqT6Cy6+wseGTbe9PdQh6rSHq8w5QgACEIAABJqYQBB+GirsaHJMYkCnTh1d306RwKGhy2XXPBML/vFkNPGvupf++5obOXy4uavpbnEAevlF9UXmpu2/r76hok8zZn7sRQhZ2BQU5HpB5o0333VHHPYFt97EhJA0KaYkS5XKSeLQa2+85fYz6xGJFxKSlDZvLrFxvOrL1e0fLHkkUAUBye+Q5k/XRPyfBWksbJ559kUvgkgckeiipDk6WcYkc9GkoVKyizNfUc0fnUO/vh39JGFoon7lzmfDhshCSfVvvPWuO+iz+/nzl+WM9tN10cTl8y/9z08ASsQZm4g/pD5kfbM4EQMoxARSm2RrrhUrV7lHH38yHDrOi4qKY1d7oTJd3cKkic3QjhwCEGg9BD582LmDL6xwtdVYIy8vc+7Dv9att+NNLA4ivPZ86l/PZtTBK6+96Y487BDftoO5ypQg8p/nX6qyr/rWNsVXWb58pZNlaLeukXVp8nGfee5F+7ihJN5f7tzGm8VqSDdce4V77/0p3i2cnqc7mXuy7OQOQsOkfI29x4KbMFmPPnz/HfZRQrGfEJGor2f5H+0DhR9//0y/l951l170Mydha5G9F2WtufNOO3oXc6FbjT9ZfAr1jZH/8a773CGfPzB2z7nfPnu6v/3lLjd9xmwnK1R9lNHPLG/1jh06eJB/Nx1+XIUw1hhjyKQPxZ0LH4+o/Q6jRrhbbvi1F5hkYSxr2hDnLpP+aAMBCEAAAhBoTQSaQtAJ54+wE0ikzxF10nOhFgIQgAAEIACBOhJoDGFHLsS09O/X1wds7mRfDH9qcWGSYwuEYWkC6u77/uJFIAkZEiCWmICwwax0ktNzL7xswaSnO7ld+8Tc2QRXMZWtO27+493Ju1Upa1wfmq98uRGT+xmJP8miR3X7v/fBFKclk3TjzbdX20xfiz/418e8VY2+2NYk0sJFS0xc2Zyyz9/+/kTKem0raq/JOolFsopaZ1+c68vjygLUB1Om+dgSco+nr9K3bNkSdz3NYk5o4m+UxQyQJY5iPWiSsLQ0iv+jeEAdO+b69g898rifxNT840IT8STUJKeH//a4v/7aP6R0dbfeca//eluCGwkCEGi9BNYucO7tu5zb69uNew7qU33XJX3pkMgdpPaRZctziZhitfWheGRB1FHbY486LK2oE6waJfwEy5vKfc+cPcfddsc9KdX3/+URb8UYrGhkrSORIzlpvDUJO88+/7K39gz7dDNxXosEh/C8lRC17957eIvM0E5uNJNdaYZ6fahwxa9/F1YbPdc777c33uotkMRLSec9ebcJfmn0AzagQ72jvn7SCXEPspgNVrOhsrbrE9qRQwACEIAABFoLgaYUdAIDhJ1AomqeXbWKGghAAAIQgAAEIFA/AhJ25IotpMO++PlQrFOuCaZp02e6dy1WTjpBJ7kziSvvmJsuiTSVBZ3QzvdnwkMQdEJ9XXPtP/vjOW6mfTWdLOjUtZ+GtNdXwdNnznYfS6CqJOjUt1/1M2/+Ah+IWnllQSf0K5FGljXJgk7Ytsm+2Fa8IwlYS5YuiwUdbZfLOCWbc/TbdK1mmhu9yoKOb2R/wgRjWFdeuU4xhhB0kglRhkDrJbDoHefeutOeEWZd09CkPt66w7lF79atpy75+a6PidYhKcZMba7XQlu9p1auWh1WzY3o0Lgs8SOkP9x2l3cNuiXh9jPUK99mz9d/PvWMO+/8i305eZueyT/9+a/cqmpEbInq3zzrR+a2tMK6Mjkmjfr697MvuBfNwlXP4ZAkMlV+L1561XXu19ffZO+BinGH9iFXvJ+TT/+uf2eEupBXPu6mTakfHoR2meSvvv6WO/WMc8yF5/wam+scdG61pS3mljTTJBemIdV2H/zl4cfck2bVlcw27Ktrfec9D7ibbjWVMZFCjLywrrymYzQm0+RjUoYABCAAAQg0hEBNcQQb0zVaEHaqG2tN46hun7ZQnzVu8kFJ/6xrmlPauJ4vKJuGLL1CAAIQgAAEqhLYb/JY99p7s6puaMYaiTlyxSaBJ1jwNOPhOVQLI3DgAfu53SftGscfamHDYzgQaFQCjfUM7tq9QmBorAG29P8vKxhmMWm+Wv8YO4qhI5drdbXQaSy+delHFjKTJ010ZSbmzLKPBdK5/kzXn1ygjd9pnMV06+qtRzXRUZ0Qn25/WZqONKvKQotRpzh1NaVBA/u7SRN39VYnEhZk9Tpl2owU13A17d+Y22QFKhd048aO8VZOshhdunS5WYnOc3JP1xJSd4v9M3m3Xb0LNol8b5iF73KzXCVBAAIQgAAE2iqBay6/2Ls6Tz6/xhR0kvtNZxmkf5dccPHlyc1abbmu//ZH1Gm1l5qBQwACEIAABNITaKwJxfS9UwuBuhM44bij4sDedXUPV/ejsQcEti+BxnoG1/V/7DI565Yu6oRzGPVZ50abJ7RuA0JNzfmGZRbD6wXn5v235nZshQAEIAABCEAAAhBoXALJwk5TCTphxMnCTlsSdHR+df23P6JOuCvIIQABCEAAAm2EQGNNKLYRHJwGBCAAgWYl0FjP4Lr+j10mJ9laRJ1wLv12cq7/eOd6jbT/0e3nXMcu0ZYtFjpt43Ln1sw314zTnFsxM+xBDgEIQAACEIAABCDQ3AQk7Pzl4UfNond6kx9awo5crrUVC50ArK7/9u8QdiSHAAQgAAEIQAACEIAABCAAAQi0FAIrZphgYwsJAhCAAAQgAAEIQKDlEmhOgUXCUXOIRy2XdjSy7JY+QMYHAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCDgHKIOdwEEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQaAUEEHVawUViiBCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAUYd7AAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQi0AgKIOq3gIjFECEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIICowz0AAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABFoBAUSdVnCRGCIEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQQNThHoAABCAAAQi0MQKbNm9x3bt2bmNnxelAAAIQaPkE9OwttmcwCQIQgAAEIAABCEAAAhCAQFMRQNRpKrL0CwEIQAACENhOBJasWONGDe2PsLOd+HNYCECgfRKQoKNn71J7BpMgAAEIQAACEIAABCAAAQg0FYEOTdUx/UIAAhCAAAQgsH0ILF+1zh949LABrnNex+0zCI4KAQhAoJ0RkIWOBJ3wDG5np8/pQgACEIAABCAAAQhAAALNRABRp5lAcxgIQAACEIBAcxLQpCITi81JnGNBAAIQgAAEIAABCEAAAhCAAAQgAIGmJ4D7taZnzBEgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAQIMJIOo0GCEdQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAIGmJ4Co0/SMOQIEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQaDABRJ0GI6QDCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIND0BBB1mp4xR4AABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACDSaAqNNghHQAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABJqeAKJO0zPmCBCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCECgwQQQdRqMkA4gAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAQNMTQNRpesYcAQIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQg0mACiToMR0gEEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQaHoCiDpNz5gjQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAIEGE0DUaTBCOoAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACTU8AUafpGXMECEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEINBgAog6DUZIBxCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCECg6Qkg6jQ9Y44AAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABBpMAFGnwQjpAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQg0PQFEnaZnzBEgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAQIMJIOo0GCEdQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAIGmJ4Co0/SMOQIEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQaDABRJ0GI6QDCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIND0BBB1mp4xR4AABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACDSaAqNNghHQAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABJqeAKJO0zPmCBCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCECgwQQQdRqMkA4gAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAQNMTQNRpesYcAQIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQg0mACiToMR0gEEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQaHoCiDpNz5gjQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAIEGE0DUaTBCOoAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAIHmJJCf39l16tSpOQ/JsSDQIgh0aBGjYBAQgAAEIAABCEAAAhCAAAQgAAEIQAACaQlo0rJ3rwK3ZOnytNuphAAEINCeCHz52CPdaSd/xeXmRlPbm0u2uAsuvtzN/nhOu8EwfNhQN3zYEPffV15vN+fMiVYQQNSpYEEJAhCAAAQgAAEIQAACEIAABCCQMYHP7r+v++HZ3/Htf/iTC5lwz5gcDTMhMHrUCPe975zuxo7ZweXkRI5Wysud21hU5KZNn+muue73buvWrZl0RRsINAqBY48+3H39pBNcWVmZO+Wb3+P+axSq7aOTH3//TLf/fvu4tWvXuW+ffW6DTnrUyOHuW984Oe6jzB6MeZ06ury89mOxM6B/P3frjb/2DPbaY5K77oZbYh4U2gcB3K+1j+vMWUIAAhCAAAQgAAEIQAACEIBAIxMoKOjh8jvn+aU1uH+55Bc/df/82/3urG+d1sgk6K6xCYwcMczd8Jsr3M7jdowFHR0jK8u5bl27uH332t09eO8f3fidxjbKof989y3+3pgwfqdG6a8+nbSEMdRn3O1pn149C/zzrmuXfNchJ6c9nTrn2kAC/fv19fdO3769G9iTc4d98fNxH5dc8Rt35PGnuMOPO9lNmTo9rm8LhaOPPNQ/l//w26uqnE5Xew+E1KVLRTnUkbd9AljqtP1rzBlCAAIQgAAEIAABCEAAAhCAAATcsKGD/URsF5uQJbVcApo4/+01l3kxR5Y5/372BffMcy+6efMXeJHny8cd6faYNNFPkF516S/cCad8q8EWExIos00x6tix43YD0xLGsN1OngNDAAIZE5CVitIWs1R8+90PMt6vtTUcOXyYf2d37da1ytA/mTPP3XDzHU4fADz48KNVtlPR9gkg6rT9a8wZQgACEIAABCAAAQhAAAIQgMB2JqBgzjuNHeNWrS50ny5YWOtoutskzpgdRruPP5nj1m/YWGv7TBokf9mbSXvabB8C1119qXclpKP/4tKr3AcfTosH8uHUj5yWr55wrPvGKVE8ie9/91vud3+4LW5T10LH3Fwv6NRlvzE7jHLZ2dlu1uxP6rJbtW3rM4ZqO2NDiyOQZYKh3AnKonHWrI/dttLSGseo+2EXsxpbvGSpW75iZY1tdR9qYrubPTOnTZtRa981dsbGFkGgd+9ebpRdUwnZq1avqTKmnISVWGlpWZVtyRV1ve+S962p3Dkvz+28045uzrxPvTu55LYSp8eMHunm2thXpxm72mpc/cxyacigAVpzH06Zlva+rc2q6T8m9teU6nv++k3p3yvl9t/s2XPSjq2m47KteQgg6jQPZ44CAQhAAAIQgAAEIAABCEAAAu2EwOW/vMDttusu7pXX33QLFy12Jxx3lE3SV/j6L9602Z13/sVuwcLFnogsM+694yY/gXL9729z3//uGWaF0TmmJVHnwl9e4Se4QuUN117hRo8c4eabQPSD8y4M1T6/+YZr3LAhQ9wMmzz92S8udddcfrEbOdImPRMuWr5w8GfdwZ/d37d9570P3KVXXZeyPyvbl4DcFCnNtMm0ZEEneVQP/+1xd/wxR3hXbJ87cP8UUefRB//kNCn+t8efcPf++eHk3dzjf73X5WTnuAcfecw9+a9nvYu3goLucZvLLj7fx0tRxW133uue+vezLtzPb73znp+UnzRxgncDpzayJHruxZft+H/Uqk9jd9zBXXfVr3z5vAt+acLk3GiD/Z04Yby74pLofj3n3PPd2nXrMxpD3AGFVkVA4vSVv/q5GzVyRHzP6ARWrlrtLr7smvgZGE5KwrNEzWFDBoUqs0Lb5u/lrxx/jK876oSvx9vO+vY33FGHfzFFlNTz8tHHn3SPPPbPuB2FlkvggT/d6rp36+Zu/uPdbtJuE9y+e++R4tpv8ZJl7jvnnOdPQGL20Ud8yfXoET2zOlsMnSf+9me/TQLEyad/123cWGT9ZX7fHXvUYRaf5xQvHv72xlvcT350thswoJ9JLVnumBNPdUfa/aXtq1avdo/94yl36slfcXI/GNIKu5e/f+4F7ozTTnafP/gAl9uhYqp91sdz3Lk/uzg09cLmReef6/r17Zvye1BMoOkzZrtfXfkbV1y8yX379FPc5w46wPXoHp1n/7594vNct369+/oZZ/s+5U5V47zvL39Nud/rc/76Td5+9/3+3x/6N0lIGtsNN93unnvh5VBF3kIIEFOnhVwIhgEBCEAAAhCAAAQgAAEIQAACbYOA/NsrsP2B++/rg4pL0NlcUuInwHWGisPz9a+dGJ+svopVe8Wo+Nm553hBR25lNJmppAmaP1x/tdMXwCHJhZr26ZJfMbkUtuVbnbb1TLTfcczoWNAJbbRdS48eFX2GbeTbj4AEEcXNUfrro/+ICtX8feHlV/wWXcfkSbhOFjC8cl3oIrdDbuLeKPD324D+fVMERx274t7o5ncL97MmWyfbpGtZWZnbtDm6n9X+kM8d6C74vx+GQ7gONqkZ+ujWNdVtUMeO0fG1XRP4uuczGUPcOYVWQ0BWOffddbOfyNZ9snXbNnsObvHj79untwV5v9bJwiY53XnL72JBR+11n+XmdnBfO/G4+J4K7ffcfTd3jE3wy22gLH8kECrX81KiOql1EOhsHzDoefDDs7/tDthvby86630Z0mCzZvmsvUuVRo0c7t9ruuYhhWeNniV69tT1vsszqxv1MXBAf3ft1b9ygwb29/eUDpFl7+awXWL7975zuhd0kt/n/exe/uv9d7hDDznYnmcd/D0bxjbW3r16/4Y0fudxTv2ob4klxZs2+U06n112HusuviASr3Ycs4MrMOEq6TTj+1/v95B0zhp7nolbIdX3/PUc/uWF5/l3if7tEaygNLbTTz0pdE/egghUyIctaFAMBQIQgAAEIAABCEAAAhCAAAQg0NoJyIrh2Rdecn+88z6b6NnsJ7Hvvu1GPym0x+SJaU9v/oJF3nJm+fIVfvuJxx/tvmkTKppY+dE5Z7pLr7w27X41VR5/0ul+s75o1gTQcy/+18kiiNTyCOy1+6R4UB/PqbBwiSuTCnJNFNI4c5Xz2htvh9WM8jWFa31w8fE7jXXXXnWJ3+fiy37t3n3/w7T7axLyznsecI//82m/XV+R3/r73/jJxwP228fdVnBvFVdEaTtKqqzrGJJ2pdjCCZz3w+96izEN80/3PxRbEuyz1x7uogvO9c+0C//vR+7M7//En8mXjz3SCzJaeea5l9yNN9/u6yVYXmHWPiOGDfHr4c+ZZ5zmi+vWb3Bf+8ZZodp1794txZIi3kChRRPQhwz3/+URb2WlgSoG3G2/j953ss757yuvu6uvvdFdbdsUS2y3Xcd7AeXLX/tmynld+NMf1em+CztLQNm2tdQ9+Z/n3dSPZrjN9s7eamNKTv95/iV3x91/dkXFxf5evf/uW7xljt71D/3t7xbb5jEvLA4fNtREy1/7XU849ih31bU3+PJ7709xf7j1Lvfeh1NdeMdLhPnTH2/0z1GJOUqysFW64+brnUSt5StXuW+eWSGc+43V/Knr7y65m2nTZ5lVzm1uydLl3r3mb678pY/j1ss+EJEYVZsrxOS+KDc9ASx1mp4xR4AABCAAAQhAAAIQgAAEIACBdkhgY1GRd1siQUdJbmGmWDwUpWR3bL4i8eeJp56JJ3tUJRdCYSJl4oSdk5tSboME9CV6SNXFYwjbFW8kpBHDh4Zik+VFRcWxoKODyA2Q3BUpaUL00C8c7Mv8gYAI7L3nZA9i4eIlsaCjijfeese98tqbftuQwQOd4pMoHfL5g3wuK4gg6KhCwt+LCas03yDxR89XpVxzNdjHYrCEtN5EHk1Kk1oXgakWD0lu80KSe9I1a9f51QH9+4XqWvO63nfJHX7l1O94t5Ovvv6WidtTkjf58o033+EFHa3Izd8C+whDqWRLiRekQqwoxc2Tm1Ul3eMhLbLfwr9MNAqCjupL7H5//c13fBNZWTY0NeT8z7/osvi3I4vMJ57+TzycHS2OGqllEUDUaVnXg9FAAAIQgAAEIAABCEAAAhCAQBsmsCxhgVOXU/xoxizfvFPHChcrddmftq2HQMmWyD2VRiz3ZDUlubAKae68CqudUNccuSY+9ZW60tChqZYUUS1/2yMBuZRUXCel9z6YWgXBy/97La4L7qmCu0jFUMkk/ceseZTkzlIxya6+7CInqzNS2yGw3lzq1SXV575L7n9L0vM3ub668roNG6rb5C19qtsod3Jy5XbuD77rZKFW2U1ldfvVVt/Q86/cv2ICklouAdyvtdxrw8ggAAEIQAACEIAABCAAAQhAAAL+S3VhkDWEXLXoy15S2ySwcuXq+MRktTNl6vR4vXJB7olCeq8al2lhe1Pm20q3eRdEvXoSn6kpObemvhWfJKSV5jqqckoWt4ebldmHZsEYrBTk9iqTJIuH/PzOPkC9no2yZJw44RI3Z+58d9Gl13hLskz6oU3bIVCf+645z173q1ywKu5TU6SWfv5Ncc7tuU8sddrz1efcIQABCEAAAhCAAAQgAAEIQKDFE1BMCSXFNEHQafGXq0EDfO2Nt+L9jzj0kLicrqA4NkoKJq94FNsrKTi40oo0k/fba0wcd/sSSL4XwvMreUR9kqzMFptLKqU1a9b6vGdB9LzzK7X8kbuuE7/+LfdXc1Mpd1hKo0eNcDdcd0Ute7K5LRKoz33XnBx+fM5ZXtCRdeMHUz5yd937F/fgI3938z9d2CjDaOnn3ygnSScxASx1YhQUIAABCEAAAhCAAAQgAAEIQAACLY/A+J3H+UGl+4K9S5f8Og84K4vvO+sMrZl2mDHrYy/SdM7r5D6z714+6LtihFROE8bv5ANoq15xGtKl3r0q4oyk256uLjvbTB7qkHafNNFbkGmXefOruoCrSyyMcNi6jiHsR95yCCjA/NZtkQXXxAnjqwxsv733jOtmzvrEl+fN/9QN6N/Xydqgt8XISY4pJbdS1aXi4k3unvsf8suPzjnTfekLB7n+ffu4Lvn5cfyT6valvm0RqM9915wE9ExXmvXxHPfzS66MD71ly1bXGHHRWvr5xydMoVEIVP9UbJTu6QQCEIAABCAAAQhAAAIQgAAEIACBTAkcc+Shrn+/vnHzE48/2k9QquJ/r0bBxVVesSJyadTN4q4kBwmXr/50X8ZrHwVzVtpp7Bif86dlErjrngf8wLLNp9R9d97kdpu4S8pAD//SF9zVl1/k67Zu3eYuv/q3KduLiov9+rixO7gwGZ5lfZ36tRNjASZ5h8VLK2KY7Js02Z7cRmUJiMcfc0RcXVDQw/3fj8/266WlZRYA/AVfTo7DcOAB+8bt5XLo2KMOj9eTC5mOIXkfytuXgO4t3VfpFo1s2kcz/QBlOXPgAfvFgx1jAdc/f/ABfn3VmsJYeHn6med8nVyp3X7Tb52ehXqe6R479eQT4/1DQe4H5Y4yOW3cWBSv5nbMjcsU2g+But5324NMTk7FdHyP7t3tPo+sLiuPZdXqNb6qjwn0uYkYVZXbVF5vDedfecys148Aljr148ZeEIAABCAAAQhAAAIQgAAEIACBRiegico//fFGV7xps+vQIScONi73Wrfe/qf4eI8/8bSblJjsV5DwNYWFNunexeV16hi3qVx4590P/eTRoIH93ZOPPeDKysrckqXL3Xd/8H+Vm7K+HQlocnv3Sbu6fffew1//q371cyfRpHjzJm99ILFHSe74fnLhJS5M/IUhv/X2++7zB+1vAeQ7u3/89T5XuG6dK+jR3XXIyQlNUvK1a9c5Ta736dXTHXrIwX7CPcfaPv3v59wtSfecjvvt009xp596klkTbXZd87vEItHjT/wrdg0oy6Ll5opN1hKyKHrsoT+5TXY/F/ToEbdPGYCtZDqGyvuxvv0IPPLAnWkPvq201B19wqnu19f/wT1w961OE9jnn/d9Hxi+1LYl3wfXXn9T3Me7709xD/3tcXfSCcc6Waqd9a3T4m3pCjdce6V/3m0u2WKu2wpdXuc818uERqXpM2f7eyrdftS1bQJ1ve+ak8aKlSv9RxtjRo90D913u3+O9jVXhOGZXnksTzz9jI8Vpd/Q4w/f67aZ9Zuzx/8xJ1b/22jJ51/5/FhvGIEKabBh/bA3BCAAAQhAAAIQgAAEIAABCECgXRHYZlYSIcliIqSystJQrDaXT/10SZORcluUbxOUHRNf5q5ctdp9+3vnpsRNefvdD/zEufrQHH9vm5CXoCM//a+/9W66rt39Dz7i5i9Y5HRsTSLlZOe4wsIojkXaHajcbgQuv+Z6H29B4p6SJvW6mWgXJv/WmBBz6ZXXuk/mzKsyxocsRkOIsaP9JNaYPYX751PPuBV2L6VLd/7pz17Y0bbcDtH3v2tNDEpOmjyf9+kCLw5pLLrvJCw98PCjNtbIuii0f/Cvf/f3mdbzzJqip022Swi64eY74vrQNuSZjCG0Jd8+BPykci2H1r2mJHHvuz/8aXxfyVJL94HuG8WBuujSa9zUj2ak9HbfA3911914q/tw6nS3oajILV2+wj39zPPukceeSGmnlWnTZ7jNJSX+uSehOgg6uscvv/r6Ku2p2L4EJEzXN0kMTJfK07xI63rfVdd3OF5t29WurKyaF3roJJFfcsVv7IONTX5NvwcJ3yX2XH308ad8XXmlfl57423/Tte/CfS7yc3t4AXySt160T/UNfb5h363bkt/DcJ28uYnkDVu8kGZ3XkNGNvG9YUN2JtdIQABCEAAAhCAAAQgAAEItC8CXbv3bPQT5v/LGh1po3Uo92lys6X0h1vvMjdWz7shgwc5We1MmTbdJbsUqnxQxauQeyMFWtYEqXzq15YkFg0cOMB9uqBxgjPXdjy2N4zA2B13cDuMGul69Spwy5evdDNnf+wWLFxcY6dyS7XH5IlOX4FPmz7TzZk738SU2qd/5FKtU8eObvmKlXH/1//6cjdux9Fug7m2+uqp3/Eur3axOE/rN2xwH38yN25XuSA3gHtM3s0EqRz33odTbewrKjdJu55uDGkbUtlqCOhe2G3iBLM+7OA+svtx8ZKldRr7Reef6/bbZ08Tcba44086PWVfTY7vY24DJXgqJhXPtRQ87XqlofddU8CTu8Kdx+1oMdEG+vf7sgyfiyNHDLP4aUszeseHcbfE8w9jI69KoK7/9kfUqcqQGghAAAIQgAAEIAABCEAAAtuVQF3/xy6TwSLqZEJp+7RJJ+psn5FwVAhUJVBZ1KnaghoINB2BLvn57i/33uYtyBYsWoK7yKZDTc8QgMB2JFDXf/sTU2c7XiwODQEIQAACEIAABCAAAQhAAAIQgAAEIAABCDgf66uoqNgseZZ5ixtZbU3ebULsEvD6398KJghAAAIQMAKIOtwGEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAwHYn0K1rF+/uTy7/QlI8lnv+/JCb/fGcUEUOAQhAoF0TQNRp15efk4cABCAAAQhAAAIQgAAEIACB7U1AgY3//NCjfhjvW+wREgRaEoE/P/iIGzd2jFtlAehJEGhKAseeeJo7/LBD3M5jd3R9+vRy6+zZOGfuPPf0M8+7tWvXNeWh6RsCEIBAqyJATJ1WdbkYLAQgAAEIQAACEIAABCDQHgjU1a92JkyIqZMJJdpAAAIQgAAEIAABCECgeQnU9d/+2c07PI4GAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCBQHwKIOvWhxj4QgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAoJkJIOo0M3AOBwEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAATqQwBRpz7U2AcCEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEINDMBRJ1mBs7hIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgEB9CCDq1Ica+0AAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIACBZiaAqNPMwDkcBCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEKgPAUSd+lBjHwhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCDQzAQQdZoZOIeDAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAvUhgKhTH2rsAwEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAASamQCiTjMD53AQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAoD4EEHXqQ419IAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgEAzE0DUaWbgHA4CEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEI1IcAok59qLEPBCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEGhmAog6zQycw0EAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIACB+hBA1KkPNfaBAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAs1MoEMzH4/DQQACEIAABCAAAQhAAAIQgAAEIAABCECg3RDodNmT7eZcOdHGJVDyyyMbt0N6gwAE2gQBRJ02cRk5CQhAAAIQgAAEIAABCEAAAhCAAAQgAIGWSICJ+ZZ4VRgTBCAAgdZLAFGn9V47Rg4BCEAAAhCAAAQgAAEIQKBVEthv8thWOW4GDQEIQAACEIAABCDQNgi89t6sVnsiiDqt9tIxcAhAAAIQgAAEIAABCEAAAq2TQGv+n+jWSZxRQwACEIAABCAAAQi0FQLZbeVEOA8IQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQg0JYJIOq05avLuUEAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEINBmCOB+rc1cSk4EAhCAAAQgAAEIQAACEIAABCDQfgjoK9UcV+6ybcmJy86va5vqk3O1zfLtom0qR0vUzpftT3Z51C6sZyX2s8y3j/4kyok+om3WIJF8D9ZBqAm5Knw5sU3t7HC+TvVlWRXrZbbut/vctlnb0kS51Mrarrp0ubarbbRP1MZWSRCAAAQgAAEItAECiDpt4CJyChCAAAQgAAEIQAACEIAABCAAge1NQAJKB5Mhcm2pLtc2vz0raqN2WiTKhHKUR+sSYjRxkWPtVQ7ijfJkMSfUR0JPsqBjZds3EndUX7GfF21sXaKN6sN6cjnL9g311sSX0+XJdSp74UYFS6GcLlddeblGELWLhJzkcsW2INJEIo4EG1ts34r1IOQ4E3SicsiDIBTWfW77brM+VN7mF5fIo3UdL6pPbLf2W61dWLRN5XS5xkSCAAQgAAEIQKBpCCDqNA1XeoUABCAAAQhAAAIQgAAEIAABCLQKAhJLOtrSyeSBKNd6ma1bbqKGyqoPi0SZqFzmBRqVc307CTYuRdTJNUWkQuSJypGoo3JkaaOJCb9Y2yDoRCJPRX0k2qh9VkLYqRBYWgXkGgcZ5J4aGzVoo44gkSZayuPyNtsgYScSd0LZhJqUeok3ts3qgqDj14PAk6gP4o62bdE2E4F8buUtJptVlKN6rft6a1eSKEe51qP2EpxIEIAABCAAAQikEkDUSeXBGgQgAAEIQAACEIAABCAAAQhAoNURkHWJxJc8v5Qn8kiYyctK5AmhRm0k2Pg8IcZImFFdp8S6L9u6RJ1Ovm8JOSbE2Bx7JOgoD+sSaLK8MNPwKfiG92DDIlUiIKpBPEuRw6rDnVJvqk2ckstxZUpBLSQSbbX7Z4tyq1AeLRJ7Ksolqk+IOrGgE4s8kfhTYuubTeTR9ip5eUW9tm1OtJFYhLWQwSVBAAIQgECbJICo0yYvKycFAQhAAAIQgAAEIAABCEAAAq2ZgKxn8mzpbFPT+WZXEeVlrrOJLCpLkIlyWzfRJrT1ubUJ4o4EmTy/HokzUdl5oSbPJu472iS4rGBS5vBrBVe31rV2R4M2RUB3R2SxlWX3rqXKt0u8HgSikKdiUK0si7bY72CzrUgA2qzFiz4qm8iTXDYhZ5MXgLLcpoTAs8lEH7WL1m17ok2xcluK7e4PbbEKMrgkCEAAAhBoFQQQdVrFZWKQEIAABCAAAQhAAAIQgAAEINDaCciaRi7O8m3pYtPVPjdBpktcFwk1+VYXiTgJUcdEGbXNl8jjRR0X53nWZ+esLG81I3Gm9hTPqNfetAW30IR/mPRPdikm6wytl9nGilgz1ZUtXng4HKAAAEAASURBVE2iXehP+4RyyIPFh9aVoig7oZyaR2vR30A65KqNIuRU6Bwp8Xv89ijuj/bR4rdbQfGDVK5xsXa6B9QmcleX6qpO/bWmpPFG1kUmDiUPPi6HKxTOSlc+SpEY5EzkMUHIqjZpMcEn5MVGV8JOsa+TuBMJP8UmAknkKdK2RO7LVl9kVFUn13DhnkgcjgwCEIAABCDQrAQQdZoVNweDAAQgAAEIQAACEIAABCAAgbZIQHFiJLx0NUmhayLvYuJMxbqJMgkBRyJOl4RQo7yL7dvFJqol2siyIT8h0mhyvvoUz2xX36SZt2iiO7jdUlwVudmKYq9EuWKs+HrfzuK2JNpvs/qonJrLciKK7ZJlVLVIsKnIdTy/bsepXK9tZdY2Na9aF2QBtQ1l5ZXXVVchyVjRKmzYPkXbIjEoqqkQbcJVssscV0Z1kTSkclgk3ISy8uT1SMwJwk7Ik8WeijpZeUWijuXWkdYjoSep3vpXvY9hlGgTCSiqq4ht1MEGHq1HdZEFjtzuRfGTtM3XWR/BOkd1Nd+71qCJk86/sxb7LcUpLupiaNHdESWV5B6u2ESgYuV2JYqsSZHPIwGoyC64BJ54MaFno61vNJJFvpwTr0v8UfwhEgQgAAEIQKApCOhdS4IABCAAAQhAAAIQgAAEIAABCECgGgKaXO9siwSablqylJe57vG6xJvE4oUcaxvn2s/ZurlBs0nemv8nvPkngcPUtia0S+wcS6wiKtu6TWL7mCc2bh8Pxdaj4PYSaRJlqwtl5fHi66N225LqK5cl6JTaaUfCjkSdSLRRHpVDHoSbSKiRiKOxb9dU1wHUtX1jnFzimLqzUsWdSOCRqBOLO74cBB0Tc0y4ybH9I2FHIk60aJ+q5SD0VGxT7CW1U1ymuGzj8DGZfF1UlotAxXKKYjepLIu2yPpM4lBz/Cp0HFm95ZkI1Mtyn/yBBVBLhQAkAXKT1Un02WjljXYPb7RRbvTCTsgjwWeD1W0wwutt2WBCz4Zy5RJ/5PZN93JznJ0NkgQBCEAAAm2KQM3/nmxTp8rJQAACEIAABCAAAQhAAAIQgAAEqhLQtKrcokmk8YuJNj2srKW76hMiTjcTaryY4/NyE3fKbbsEmyw/IZx+ejZ9bdVRNKxGU86ygtFks+KPyM1UcrwRH4/EJp8VX8QHnLfJ5lCWUOOD1FtdHKze6hRsXutRXWR5oLayuPF5ok0QdeottGjOnNSkBIRYIlmUQl7LIcN1CXktzb32YW3Uu4SiCiGnLBJ3rE4WPsn1naxOS0f7nUV5Yt1+Y2FdcaJUzkvUaV2xoXy8KL8tihslyxzFiepsI5DVUFNZC2kirZsdo1vA6HNBqnD/pjW5fdtolj/rLd9gv5kNts96L/JI3JHYk23rEnyy3TojpiVaj0QguXnLFL0dggQBCEAAAu2IAKJOO7rYnCoEIAABCEAAAhCAAAQgAIH2SkDzrnk2cSyhpsCmtwtMqCnwZRNvrCwxp4dNGvdQG5+bYGMTxz1sxy6JSeKq7MKsbtUtDa3R9LCEmOAOSl/1F9sMr4/9kRBnVKdA8Mp90HibBI4CxUdxQSTGbLa6OLf9NFEciTnKs704UzHZ39BRsz8EIruWYGkV8ZAztETKVKVItJNVkAQdia5aIoHHyvbb1O/Ziz2JXHGoVKe4U8rVVhZ2ne33rNyv2zbv5tB+uqoL7g4lECWNMoy23rmeDMH9W1/14h8VOqnIiaCqJMIW2RjWWfV6+22us0br7Pe8zn6XymXds85En7WW+8WXO3jxR7/dTFHqWCQIQAACEGhbBBB12tb15GwgAAEIQAACEIAABCAAAQi0awJy9ySBpqcJN34xwaanxJsg4tgEr8Scnja5W6BF7c3SRpO6VSWaqjUNgatJ2CDUFFk5itER4nZEwdoVqD0O2m4jCoHbFaNDAdzDuoQZrWtyV7ksZ2QpQ4JAWyIQxCHFsUlJNSkati1YCsVCTyzyRLGtVK8YWD63Z4KEnqjOhJ7EumJi+XhXduAo9lUU7yr9syJldBmtyJqowH6zBfrZ+p+uTioSfcKzYr1Z+qy1jWtN9Cm0JRJ3smORp9DOtNDEnkJz7KhFQpAs50gQgAAEINC2CSDqtO3ry9lBAAIQgAAEIAABCEAAAhBokwTkwqmHCTe9bOltgo1yv/iyRJsy18uWntZOAk6PJhZuQpyNDTYbG8XXCLlia0RxNkKAdQkzcdkmq33ZJmYl3ATxRtY1xNtok7cuJ9UMBCRweku2ymKQji3FJDnZuuxeZPUTxB4JPl3sWdIlIf4o92Uv9ETlirhZiqelOFrmitFyuWWTC7iGTLhJlgkxfvpprPYMC7F9VJI4vM4EHwk9hXasNfZMKbRljYk6a+xZssaOrmV1orzOyhJ+SRCAAAQg0DYINOQd0zYIcBYQgAAEIAABCEAAAhCAAAQg0GIJaLK1q02o9vHizVaf97Gv2XtnRWJOb18us/VyE3GiWBdV3Sg1bDIzTKJuqCU+RpFNqm606WEFQY/yRNmLOJFog6uzFnurMbB2TEACahCB1gQO+uFXTlYnl3Be9LFnTyTsWG7PqGiJyrLyUfyt6uJwdatWZK58wKrryYJPf/9o00AjC59Ss9PZYONbY1WrTfBZbc+e1fY8krizuryDW2XlVV7syfW5nlOIx1UZUwMBCECgpRNA1GnpV4jxQQACEIAABCAAAQhAAAIQaCcE5DJJMW/62sRkPxNt+pqQ0zeR97Gv5vvaRGkfv0TiTer/0DZMuNlijDfa8RXfYp1Nhob4FgpsrgDmCmoeBTePgphH66qXgJPjJ4SZHG0nNyqn2a4JSJiVmzMt9shIm7JNAFbMnq72POumxZ5fEnnkGjJaj0QfPe+62zMtxPHq4a0KZfGT5Tqm7bnmSgnawaXbKP9ILLOaMnuSRmLPKhvvKnumrbRllQk9K03gWWlij/IVPs/1zztcOdbMma0QgAAEtjeB1H8Db+/RcHwIQAACEIAABCAAAQhAAAIQaBcEFCFDk5v9bLKxf5YtNu2ovJ/qTMjpZxOd/WyCs7d90S43RBWSTUWpLqA09yrhRtY2hZYrRsVam9gsNLFGwcgVnHy9z1WOJmxVr4lbCTe4LqoLbdpCoH0TkMBb5Jdst9xZ9JzK4o+ty4WknoESenqY6ONzlVVn6z1MjFF9T8sL7HmoGGA9DausfCT41OVJqMm/nrZHT9tpjPUjy55yWzbbk221PRNX2PNwhT0PJeyssOfd8vJcG3cHn6+w8esZKHmIBAEIQAACLYMAok7LuA6MAgIQgAAEIAABCEAAAhCAQJsmoAnM3ibcDLRJxIEm3mgZYOsDbNKyvy0DkgScChB1mbaM9tLEY5Edq9DmLdfYRKViTawxwWatCTRBwFFw8bU2YSnRRoHHFW9ik7Vh0rKCPCUIQKBpCUgoXm3PHi32yEpJ2bau+D6KG1YgoceekQVW7pkoR0KPYolFscN6SfCxx2UX61OCeSZJT9fOtgwxkWiIf9TqCbjFhB65bit2y+z5udyekcu02BiXmtDjF5OUNGaEbgNFggAEILCdCCDqbCfwHBYCEIAABCAAAQhAAAIQgEBbJtDJJhvlPm1I1hY32KxxBpmIM8gscKJFVjiRiyE/l+hBVJQy4aLpR7lLW22TnyF2RAgSXmiTkL5sE48qF1ou8aaYr80zQUsbCEBgOxOIxOlsE6g7uiUaS5LoI8En38QdiT09JfSY4NPL8l5aV9mvK+5YFGusd+JZm6nYI8vIwSb0DPaP5MiiZ6MrMSse55aYSL7ErHmiJdctNiueReUdvfu2kozlpO0Ml8NDAAIQaAMEEHXawEXkFCAAAQhAAAIQgAAEIAABCGxvArk266iJxaEm4gyzb7iVDzERZ4hNMA62ycW+NknYKR5k3QScEtuv0FwErbQvx30sCJs8XGUTi1EA8ETuBZxIvGFyMQZNAQIQaGMEIkE7iuW1SI7YkgSfTuXmps2LO9u8ZWRvK/e257DyPpb3kdjun8ey7El+JlcPSU/rbmYB1M0Ko73rtq0m8Wy1Z3GxW2wizyITzhfZ83ihiTsLbDzK19jzeKvtQ4IABCAAgaYhgKjTNFzpFQIQgAAEIAABCEAAAhCAQJsnoCDgQ0zAGWECznC/bHXDbOJwWANEnMj1T7m5/amI8bDSLGxCMO9IzOngJw0382V4m7/HOEEIQCBzAhK0l2sxV2lxMtEnT27aTHSXC0yJO7Ki7Ovzihhm/dO6wIx7SSlIoI/ctpW7va2vEltWlm9yC0zkWWACz6d2/Pkm7mhZbELPRnuGkyAAAQhAoPEIIOo0Hkt6ggAEIAABCEAAAhCAAAQg0KYJ6LtrxXUYkVXiRtk03igTckaZW7UR2WVmjVPu4zlE32Zn9oX2VutvrVngKHbDUpsMVPyGiuDcFrDbJgdXmnuf9TYhuI2vvtv0vcXJQQACTUdAAvgS78qtwrKngz17u5sw39dsavqZwNPPnrL97Xne3+elFveszMc6KzCLniSJKO0gk0Wefa2/IhP7F5slz7yybDfPBJ65Ju7MNVvN+eWdfDyzJOOitP1RCQEIQAACNRNA1KmZD1shAAEIQAACEIAABCAAAQi0awKSZxSse6SJODuamDPGlh2yt5obHsXFSZ7sq13I2WR9LbeJRMVlWGoCjuIyLNNiU4bLbOJvheXrEHDa9f3GyUMAAs1DQEK53KRpmZVQWST09DChp58JMwNM4Bng8ygW2kBzpTnInvv97bnfuYYh6k3Q1foea4WxOWXWg+LxbHZz7Ln/SVmu+9iEndm2zDORZ50dG4GnBphsggAEIFANAUSdasBQDQEIQAACEIAABCAAAQhAoD0T6GzueobbZNxOWZvdOBNyxplVzhizyFFsnMiRTs0ijibqNtl03TIrLLTJvMUm3mhZYuKNvhhfavkqm9DDhVp7vss4dwhAoCURkNCz2p7LWmaUR9JNnsXp6WPC/kATeQaZBc4gywebZY+Wod6ax5nIk2X/pU+y8hls7w1Zc34me4u5aStxH5sFz0yz3plp4s6M8jz3qQk8m3CnmR4gtRCAAATSEEDUSQOFKghAAAIQgAAEIAABCEAAAu2RgCbl+th31TubkLNL1iZbTNTJlhueYJFT3bRdRKvEspX2pfenJuIorsJCE24WaTERZ7HlhTZRiBu19nhncc4QgEBrJSDhXc/wRSbCOHOyKWuenvYkH2zijmKqDbF8qC2KpzbcRB4J/3LHli7pg4ABtn1ATrnbxz4aWGoWPDPKNrppJu5MMxFpugk8q8xiE+uddPSogwAEIFBBAFGnggUlCEAAAhCAAAQgAAEIQAAC7ZKAHOAMtsm53bKKbdnsJtrX1KOzy103/+119UKOJt7W274LyrIsbkKOxUvINTGno1tgE4ALLdfX3og47fKW4qQhAIE2SkDPdMU6W2nP+w9cvhd5etuTfqhZcw6z98gwy0eYyDPS3LUNs/dI92qseGTBM8wEnmHmom0/V+zmlG1yH5Z1dB+YsPNBeb5bbO8R3h9t9CbitCAAgQYTQNRpMEI6gAAEIAABCEAAAhCAAAQg0DoJSMwZapNwe2YVuT2zN7nJFitnSC1WOaV2qmvsS+25Zo0zx6xxFAB7ni0KgK3YOLjQaZ33AqOGAAQgUB8CEl6W27N/uYk875glT2dz16ZYPCPM0nNk2RY3ykSe0WbFM8qseHrZ+yVy35l6JH1AsFu2c+Ptg4LPmnu298qK3dtlnd3b5V3cQsSdVFisQQACEDACiDrcBhCAAAQgAAEIQAACEIAABNoZAdneKDbCXibm7J9d7PYwMWeQn2xLb5VTZu0l5HxiQs5sH+i6o5tjDnbmmpAja5xSm5AjQQACEIAABCTsz7P3wzx7P/zX3huy4hllAs9oc7c2xgSeHe19s0NC4DEdJyXJemekt97Z5vbKXu/eMeudV8ry3Vsm7igWG27ZUnCxAgEItGMCiDrt+OJz6hCAAAQgAAEIQAACEIBA+yPQ3SQYuVn7fNYGt3+OXOXoy+n0osxGm0Kba67VptsX2DPLOrlZNkk3x+W5NSbkSOghQQACEIBA0xDYdZed3clf/bLbdZedajzABRdf4aZMm+7bXHP5xTW2P/y4k2vsq7E3SvBfYVY8K+wd8pbr6nqVb3Ojyza7sSbyjMsusfhtW90oc9HWtdI7SNY8Q+3dNChnm1nwrHOvlG5yz5d3827Z1qe19WnskdMfBCAAgZZNAFGnZV8fRgcBCEAAAhCAAAQgAAEIQKBRCEi2GW5fSn8ha707NKfY7WyfSOdVmkjTgeRebYV9XT2tLMdNNRHnI4tvMNMWTcxhkSNCJAhAAAJNT2CCiTq1CToahdoFUSeT9k0/8vRH0IcAq+yDgFUm4bxjljf9Sre6cRbDbbyJPBNM5Nklu9T18x8ZVOwvcUeWOwM7bLV2a9y/Sze758q7u0/NEgirnQpOlCAAgfZHAFGn/V1zzhgCEIAABCAAAQhAAAIQaGcEcs2uZpK5WjvOvnj+nMUsGGCTZJVtc7Yak4Um5rxn7tXes1gGU8o7u/k2cVZsrnRIEIAABCDQvARO+erx/oBTps1wf3n40WoPHgQdNZAljix8lCT2hD58RQv6ow8Elpo7taUWj+3N8q5uhH1wsKu5WpuciO0mKx25YgspzwqT7FU00N5jw8u2ur+X9bCPDvItck/lN1nYgxwCEIBA2yaAqNO2ry9nBwEIQAACEIAABCAAAQi0cwKdTNCZXLrWnZC9xu2bU1rFzc0247PAxJw3y2xyzWIXvG8TZUuwymnndw2nDwEItBQCUz+aEVviZDImiTwSdlqqoFP5HPThwHTX2dx75rlXS7u6SWXFbm+L9ba3fYAg96Bh4lLyzUBbP8bchvYzJ6D3l5W5t83ip4QPDyojZR0CEGgHBMKzsR2cKqcIAQhAAAIQgAAEIAABCECgfRHoYA5qJpmgc9q2pW6PnDKzu6n4qlmua+Rm7c2yDu6lsi5+ckxfTrfmWDmd+g9wnfr1d+unftiiLvSQU7/peu13gNu04FP38dWXtqix1TSYlsazpXIce9k1rlPffm75k/9wy5/6R01IW822gj32csO+9V0/3ilnf8s5e1aQqhLY8eLLXd6gwW7lc8+4pY8+XLVBI9RMTcTLybQrCTrXXH5R3PyBhx9rFQKPrHcW2jtosVnvvFua794uK3IHZReZuLPNu2ULby/F3/msvc86ZhV6f6Gy9MFiJ77cFCAAgXZCAFGnnVxoThMCEIAABCAAAQhAAAIQaH8ExpZtcCduW+72KN9qgk5FKrHidFNv/m1izotl3bybtW1Jgk9Fy+Yv7XTVdS63Z2+3ZfVKN/Oin1U7gB6T93DDv3OO3z7399e5kmVL3fjrb/brK55+wi164B5fbgl/Og0Y5Dp072GTv0NawnAyGkNuz17NxjO3oKcbcuoZLn/ESJfbu4/LzjWHgZs3u5KVy13RJ7Pdgjtv82NuqRw7Dxnmsjt1ch379PXjHP1/F7ouo3d0pZs3uY/OPbsK72677Grbx/j6lc/+y5UWF6e0KdhrHzfsm2f5uo+v/pUXA1MaNMNKhx4F/p5thkO16kN0HjbC5eTnezG5JZxIZUHngouviIaVcOXWEsZY2xj0YcFiE3eWl+e6aaWdLa7bBneoWe4oDlx4jynfJ7vcFZevc+t8/Lf82rplOwQgAIE2RQBRp01dTk4GAhCAAAQgAAEIQAACEIBARKBP+Rb3hW2r3d7lJfFEmLZsNOud10pz3CMWk+Atc12zwSkUdctJm5cucZ2Hj7QJ5e4ub/AQt3nxorSD63fokb6NLAiK53wcT6ircU4XJvjSQqtDZU7nznHrpuTZ79Aj3OCTv+GyclLvw+y8PNd56HAnwSSIOvGAWnhhy4oVrsekPfz92XnY8CqizLAzznKyglLauma1W/2/l3w5/Om9/4Hxvb15yeJQnXEuq7BhZ57jimbNaFWWYRmfIA3TEkgn6ARXbGl3aOGV+tDgE5fnlluct7nlndyJJuDsl+RCVMLOfjnb3Pzy9W6ZCUArU6LwtPCTY3gQgAAEGkgAUaeBANkdAhCAAAQgAAEIQAACEIBASyQwoXS9+4x94dw9aXBFJui8VNrB3VPa0011LTPItNxX9dznM37UfQ851C28586kM4iKEgC6jouCgW/4aKor27LFafJ77u9+bULQULfyP/+qsg8VdSPQHDx77r2ft9DxIzNxTuLGxunT/LXs0KOHkyDSoVvyHVy3c9herde8+l/X90uH+8MX7LlPiqije1cuAkPqvtvuVUSdcG9vWrjAlW9T1Ku6pbwhQ721kxiS2geB6gSd5LOXG7bWmPThwX/Lu7lC+xhhiyt0B5uQ0yVhWdrD8gOzS9wH5cXu2XLu99Z4fRkzBCBQPwKIOvXjxl4QgAAEIAABCEAAAhCAAARaLIEubpsbb4LOyPKKCDmKiPF+WbZ7oLTAfWiCjuIXtMRUPHeO27Z+vbdU6Lnv/mlFHYkBwbJjxb+eiE9j7TtvOaelumRBtmX9k5XTwW1eZBPmpaXVtdwu9XJD1rFfPy8CyP1YjcnOpfPQYa5s61ZXYtZNtSW5U1Pcl+IF871rs9raa3utPBOdZHfs5AWYbRvWu5LlyzLp2l+/Yd/+nm+rc5hx/o+r7LvuvXcy6iulka7xwEEuu2NH50WRJrjGub16e7Fp88JPXbkFa6+cisxyTOckN3I9Ju2eEmuly47jnLMxhtR1rK0nJd0DOV26+prCN19L2mK7mSCkY3c0F3Vb1611W1YsT3sPd7TrnGnyfdoxt6xeVesuaptnllN20t6Crj6/n0yOpzadh49wpUVFVe6JmgaZyX2YlZ3tOpqVVIf8Lq54/ty0/MIxcjrnu7yhQ8214zJ7Jq0L1XXPm/i5U5ugI2udw487ue7jbkF7KGaO3lsPlJa7gqw17jPmei38ikaZW7YJZZvd6xZbZ2MLszxtQQgZCgQg0MYIIOq0sQvK6UAAAhCAAAQgAAEIQAACEOhbtsUNLUt1u7baLCFethg6LVnQCVdu1UvPuwFHH+c6dO3mXXBtssnz5NTn81/0q2UlJW7dB+/Fm3a75yGbL89yC/50u1ttfYTUoWtXN+q8C1zXShPqa99+w82/5fdm6VPiRv3op05xerZt3OimnmOB4RNJk8AT77jfZXXo4Na997abe+N1YZMXMsZd/hu//sk1l7kNMz6Kt2VcsPEO/uoprt/hR8dClfYtLdro5v3herd+6odVuhpy2hmu3xfNEiSIA3ZtV/z7SVewx94Wj6iXm3P9NW79h+/7/bpPnORGnvPjWCgIfRe++bpbcFcUq6bKARIV6XiO+N4PvSWVAsOXLF3sBn3l5JS+JcjN/OX5bsvKFdV16+v7HPwFH4tEK0sefqBOk/fpOtYE/Kgf/9R1Gz+hgos13Lx4ofvk2qtSxtP/iGP8uGXh9eF3Tk3pbsCxJ7iBx53oJFBN/f53Km37suv3pYTbv8SWtW+94e+NlIa2snHmR677hN3sHhmRskn3mJKur8QbXS8JUBqLUsGee/tcf9a88rIvK8bOkK9/04s58UYVdN2fedotuv9uX93/yGNd3y98KXZFKPd1k+592G/bsmql++gn3/dl/en3pSPcwC9/Jb52EmhKli1xc373myoiYTezihtqLuPyBg2O91f7uTdc638TcWUNhUyOJwuj0T+5MMWSSee49t233Pybb4gZ6TD1uQ8HHPNlO+evpvzOJGYtfeyvKc8L8R5+1vddTl6FC0JZTKndsn88WsNZpm7K5LmTukfd12oTdOreY8vdQx8i6P31sgk447I2uj6J51+eDXl41lbXzz5mQNRpudePkUEAAo1LwPRsEgQgAAEIQAACEIAABCAAAQi0JQLdy7eZ27VUK5Sl5VkWl6Cj2+xa/v8Grnzmqfhy9P3iYXFZBYkrXcfu5OsqWzLIMkLbczppmi9K+np/lxv/GO1jk4CygJEVhZJcY438wXm+vO7D9/y+uQUFZo0w1NfpT9edd3GK76J+u0+cHNer0OszB/p6WRZsnD0zZVumK2N+canrf9RxfqJZE+USRZQ04b/DBb90vQ/6fEpXmszWBLkEndBe1iL9DjvKyUJD4xQH34dNSo869/xo4t4mxzWxX7p5k1+XBUltKR3PHBPadAzFwhn6zTN9XxIoNPmupFhIEhdqS/kjR8dNkq93XFmHgsa5y023u2677Oq5SOwrLS72Pcgd3y6/uyWOYaPKcD2zO4Ww6xUHy+lijp3s/Cq7fet9wEFu0Iknx7FuZLmhiX4JALr+lVPhq//zVdrWZYcx8ebuEp0srXrh2di1mh93okWP3ff0pW0bN/jrpRVZpsk6R0nnpm0+2T2g66AYOko6jrfSSUx2q07n4s+ne4VrqtHnne8kDAaLIJ2HxilWnfr01W4pacxFl3lBR7+d4A5O7Yd8/fSUdtWtZHK8/FGj3c7X/C4WdHRP6Vx1n0usHP+7W50E1pDqeh9K0JUAqXHr9y/RUbm4dh9v900i9T/yGC/wekHH7mn9HvU7E0PtP+pH/xea1phn+typsZNaNtZF0FHbU046oZYeW/5mvb/0HtP7LDn1yCqr8s5L3k4ZAhCAQFsjgKVOW7uinA8EIAABCEAAAhCAAAQgAIFWTmDr2kJzj7bQiys99943xaKkl7lkCxYqir9TWxpy6jf9JL5Eh4+vvtQpBo/2V73EEVlO5JgrpsLXXnHD5Q7MtvX6zGe99Yj61oR6SBIBOg0YaBYNS31VwR57+XyjBaTXxG9dU/cJE123ncb73QrfeNXNu+l3XhyRC65xV17nJDANPe1brtBitIQJ6F77H+jby8XX7Msu8pPsmqgeeMJJZt10fMoQ+h1xdCTw2Ll/eOY3TOgo8ts1Qd0xzeR9ys61rVify574u1v290e8BYUmwXf+7U1+zL32/YxbaNZSNaW8QUP8Zk3cp3NhVtO+lbcNPvkbsVXFp3fcEltddDW2Yy68xE/kjzj7x27WJRdU3jWjdVmMyXJDabNZJ8266Hwvjmldolu4b7QeUqFZgQ3/7g/8qkSJok8+9mXvvsxKci0nMUfiVo9Je/h1Neg6JnLHtv6DyNJKdStM5Fz14nOu2PqQKKfUwUSaXW+5y9+vPffZz6157X/eckbbxl52jesyeoy5n/vUzbggEi1Vr6R4PT12j+7boo9nuTnXXR2JRHbfyxJn65o1UcOkv7JoW3jf3bZttRdWRv34Z9bHnl4o072q32t1KdPjjTznXH8uus9n/+rn3jWa+pRV1eCTT/P31YBjv2zWMo+kHirD+3BwQoCSsDntR9+N+8jJz7fnQ2SRo+fAoK+c4reVmGu7WRf/zFvu6fc1+qe/MMuriSbi7eskQMlNZE0p0+dO+E3W1Fe6bXUVdK65/CLfzQMP/S1dd9RBAAIQgEArI1DxmUMrGzjDhQAEIAABCEAAAhCAAAQgAIH0BNZndXDrK8UWGJhV7kZlbXF5rmoMkPS9bN/aFf952g9A1gT5I0bFg+nzucj1miafJfzUloIVg8QcL+hoB5sIXv7Pv8e75lv8DrlgU4wNJU3Ch9R91918MVjQBJFHX+JL4FFa9fx/fF7XP/1swlpJE9nzb7kxtnbRJPn8W27w2yQkhUn4XgdEgo42yPVVsJqQoLT8icd9++Q/cRwQm7Dv1K9/vEn7BWEqrqxjYWthoRe+gtswiQ3rzE2WUrAAqanLICrFVic1Na5lW6/PHuRbSKRIdru30dzhrTI3cUqyYpGbs/okiUNBSBT3IKyoLx0vMEjuW1YtgXH33SILr87DhkdWPXb/bbSxBtd63cwaTElWNrIiUlrzyks+15+i2bPchmlTUo6raxvuV8XZyTQF4U/3wMdX/arC6sfGtHnxIle6KbJwSu5PLgcl6ChJgFv5fMRU68lWbVqvnDI5nsYffkvLHv9bfF7qS8KthDSl8Nv3K4k/md6HpQnrJv1uJYiFJIuucG6yxpKAozT3d7/2go7K+n3NveE3sXDb/7CjVV1jyvS5U2Mn1Wysi6BTTRettlrvL73H9D5LTuvKs6u885K3U4YABCDQ1ggg6rS1K8r5QAACEIAABCAAAQhAAALtnsDK7I5uYXYnZ86L4tTbJvYPzC52E12xyT2pE2JxoxZUWPPfF2ORo+8XD/Uj06R8lzE7+vKqF2sXUuSuKUySayJ3hFlOhGXwSV+P+w+TvKtffsH3nTdwkLdw0b4SHzQBvuyJx/y2HpMj91hx7BObDK/sBs43zOBPXkIU2jSvasB2L0BZ30oSA5TyBiasW0wwCBPRfkM1fwpffzU+x3FXXuvGXnqNWSZF469mlwZVy7oh07StKHIhplg4DUm6xiH2ybr3363SlaxMQqpNgAjtKudddhzrqyS+ZSIkhv0L33rdFzsPGeYtXEI8nc1LFvnrsu7dt/32Tv0H+O3hnpKIEAuQic7kYk4WZIMs/pLiwgRxMRwr07yT3dtKijWUTozKpJ+S5csyaebbZHI8iaohJV+vULf+ww98MfxOQ311ebr7UHG6lOQeUBZOI3/wE9d5+MiULvKGDvPrus6bFqTG8fIiXeK8OyXFFkrpILFS1+dOuj5qqgtWN2pzwcVXuCnTptfUvM1s03tL7y+9x/Q+C2mzFT4tz3UrHM6IAhNyCECg7RPgidf2rzFnCAEIQAACEIAABCAAAQi0MwJFNrn1UVY3Ny9rgxtXXubPXlNgk7LL3Ck5a93W0iw31QJOb7XA0y01aWJ1/ZQPLI7NJFdgLtA+vf0W11OxQxKTeSv/869ah55swSBhJIgjlXfcUhhZIaz538tu6Onf8cfw4kcihodcLUkgGXLK6eYuK7Ia8mOxjjbMnB5/wV+539rWQ9yW6txXadLdu3yzSX+lDj0iC4OybVFMoNr6lxXMzIt+6sb84jJzMZfvrVVG/+QCp+Dwc669yrvnqq2PptpesnSp6zx0uMvpHLm+qu9xcnv2indNJ3RtSViYqFHegEG1us2KO0sqdA6T/Zsi12dJm2osrjG3ed5Sxe7ZLhYHqvuEyOprvVndKMmFno9FpO1jxrqChFu04rmfpLikkyu1MRdfHsdKqvGgtWzMTVipbDFLq+ZImRxPolZIW1dHv8Wwrnxr4vcpKxqJW3o21DWtfesNe4bc7IZ/52z/+5bLOi2ygJIlktygdTJLKaWyaq6zt3wzQUcu52pKdX3u1NRXum0PPPyYmzB+J/eXhx9tN4JOrgk6E0zQ0ftL77GspHfXXHvFTS3PcxsrWaemY0cdBCAAgbZCAFGnrVxJzgMCEIAABCAAAQhAAAIQgEASgak53d2rZRvcoPJ1FkA6ShYC3h2Us811dGvcI2Wl7q3yLm5DC54IW/GvJ7yoI0sMuc/q87lD/IkUz5sTu0ZKOuUqxa2FFfFBFj94n0tnyaGdgpssudXa9Ok8/wV/z30+Y1siS5m1775pE8trnIK3y7WYJtlDLJz6ul7TcSW6dDRrIFkPpEvBXZiCuisVm0WP4nrUxbqleP48i6dzmpMVSP8jjvUcFRxeljsffPNr9Rak0o23LnWbzFKkwO3jJ9h1TsEVWV36UNtkQaxDj4IquydPwKez4KiyQ5qKkuVmgTTBuew6ClCy6tE9pftX/BU/R0nxdHySyzOz2skbPNR13213l2/3lVLhG6/5PPwZcfaPIjHD4g/Jmkxu5nLtGvb70uEuWdQK7WvK5W5MFmhBbKmpbWNsy+R4JStXxofqYHGkSpelime5PRIiivGqj6ATOhc7ibN9Pv9F1++wI51+B4prtOMvL/exh4IAKCE1XVJsJaXYrWG6RlZX1+dONd1UW93e4uJ0c6Vur6wid2L2OrdfTqnTeyykdfaMfrksz31Q3jCLv9AfOQQgAIHWQgD3a63lSjFOCEAAAhCAAAQgAAEIQAACdSCwKquje65Db/dmVqobtq42IXZgTpk7O6fQnZS9xu3gNptdTyRe1KH7ZmmqiX65PVLygsSoHXx5xb+e9Hltf+TGqswmwpXkRk0xQ9ItahfS6v++6Iua7O2aiHUSJtnXT40sLAYcd4K3oNF+axMutsL+dcmDyNB5xMjYAinsr2DswSpp08IFvnqjWQUpyWKhsvstuXyqNtlkuCwVZl1ygS0X+mbqo8ek3avdpak3BFd3Os6wb3/Pux+rzzGTr7HEocpJll4hbVoUcQzr3vLDYqzUlopmzfBNZCWSNzhygVfbPmH7BruHleQ6zYsFdi0U6yekYLXT56DPx5Y4a177X9hsItyOcbyZxX+51y28906n7cuf+LsX+eKGGRZKVkSu03QeOv+mTpkcb9OC+fEweiTiD8UVVgi/w20b1idX16us2FkSi6f98Cy37B+P+j5kMSaha9PCT/16snVcOEhWhw7xddi8ZHGoTpsn35OZPnfSdtTOK/Ve0vtJ7ym9r/Te0vsrJD3ZXyvt4P5d3t2tdLmhmhwCEIBAuyBQw7/62sX5c5IQgAAEIAABCEAAAhCAAATaLIFZ2d3cIx36u3eyclPi62gaezf7v8Ezcorc93NWuSOz1rrBzlx9tUASa16PJrgL9oqsOjRhWvjGqxmPdMP0qb5t3y8c6rqaC6zkpEltL54kVQZRR+7K9GW+LA22rIosCYKA02PSHn4PiSwaT22pfOsW3yQ7P9XVWLDykSXH4JNPi7vRBPPIc8716+o/WHZIDAgi1Yhzfuz6H3mM6zZ+gut36JFu/O9uifcPBVmp5OR3Cas+32bWRiHVxeIn7NNYuayPVifinGjie8LNd3qRKcTH0XHyzNVVbxM7Ql11HENsGglxySKOhIt+hx/th6yJ+CAQStgLqefe+4aikxuwHmYxUzltmD4trhp17s9SLKVkPRYsquJGSYU1r0b3b7DyKLHzTr5nwrUN1lpb16xJtQTJrpjEdkkijM5N1mvp0paE5UvewMFVhJsgpkm4GPnDnziJFSHpfpBw1Zgpk+PpXgjWLQNP/JoXYMMYZDGXL9HT0prXXgnVdc47mmu1yucml2shZRuHQvVvopvSyB+cl8Jm2Le+G6+veqEinpfibSkF14h+xf7U9bkT9muKfMIuOzdFt03Wp95Deh/pvaT3k95Tel8ly68SdN4oy3IPlvVwM8tTn6tNNjA6hgAEINCCCFS8vVvQoBgKBCAAAQhAAAIQgAAEIAABCDScwDb7qvn9nAL/bXPx5kVuX3NdE7501lRxf4vlcbjV7WJubd4s22RLvnvf3Ngssa+eS5O+iG74SOrfw/Kn/un6HHxI3MG6999JmRSPN1RTWHDnbW7CTXd4q5cdf3mF06S5XHZJSAgT6VPO+kbszk0ijoLIyyWW0sYZFRP6OnZyCqJMcl268tp33jL3W/t4cWLSvQ+7ZWZlsfRvD3lxauDxJ/pj9TfxofdnD7YJ/fWuU7/+8QTykkcedKWbin23sjKYfdlFbtwVv/GT9YO/ViEEhcno5OMPs/ghsnyQy6otZqFRtnWbxbEZ5pvoOIr5sj3TwvvvNpdjO/jYOh0s1svo//u5H45ED295ZPenUtEns13pooWuOo4L/vRHJ9FPAsUOP7vIxwxSHz5GSqKP+Tff4PvSn7Vvv+HvIYl6w7/7AzfwhJNs31yLlVLVfZva635Z/NCf3eCTvu4klEy84z4nqxEJZsmiiNpWTv6ekVCQGMeGjyJrr9DOW+0kba98j20yl3ueh4116KlnuL6f+6LL6tjR3yOhj8q5RAfFi9HYdL95IdCO/8EZJzuJlgOO+bK3OinYY2836Z6H/LlkJwQdxZ0JQkzlfuuznunx5t/2Bzfmwkv8b2SXG251smLLzu3ocnv18oeVILfkofvrMwS/z05X/dbHldLvu2T5Un/tQiyfde++Ff/+V/z7SXPNdpR3lTfxzj87CU6ROBq599poVlsbPoqEYnW8ceZHdu/t6+Ml7WYsP7n6Uqc2dX3u1HRiJ3/1y07CzNRpkaVeTW3TbVP8ndaQcswyZ5BFepuUVez2ztayxQ2z+7ZDpXfRRmv3emmOu7+swL1tLkRbcmy41sCdMUIAAq2TAKJO67xujBoCEIAABCAAAQhAAAIQgEBGBErM/uYdE3ZWlxW7FW6d+5xNlA2wibLw/b/+p3CUrQ/N2WqTaGvde2VFtnR2U+zr5/n2bbRF4MjoOE3VqGTpEm8pIxFGaYWJPLWlcosXFJIm5Kef/2MvGEgs0SRxmChWG2/BsSWypAn7aCI6CCaFb74eql2ZtStZvsxbdGiife07b8bbaiqsffM1V/TFw3wsHk20a7I6pJm/+KkJCz/0k/Cy5ggWHep/4Z9ud6tefC409Xnx/Lluxs9/4uMLyapE7TbOnulWPvOUxcm5zrcJ1jzrp7zv8oeN8OcbRCo1UGygOb+7xpWXWYTxDFIyT1caWSbUuFvC2qHGNrZRE/UzLjjP9T3kMDfoKyf7SXftE9yCyQpCVlKlRZFFRXUcNVH/0XnnuB0u+KW37lGslJAUt2jOtVc6cQtJzFY++y9v4aS6cG8pVtPK555xw00Mq5zk7kyijwRGWblIhJKQtuaVl13ekGHemiSFU6IDnYPci3UeHlmbBMuc0L/GIusd3ZtKq195yefhjwS5uTde50YlrGryhkRio2I/bVrwqet1wEFVrqNEh8LXX3E9Ju/pxyrLry2rV4Uujfm55vLubHMJd4AXm/y52FbFitH9raRxZZIyaZfJ8TZMm+I+vupXbtR553thJwguGoOuyye/uTI1nk4d70MJNz0m7+FjYoXYRupbcY/m33aTij4t+vM9XsQbfNKp3rJH1mIhybJswd1/DKs+lwu3fHMLqXtIlkDBDWJ9njspHdvKlGkz3K677BQv7qvHV25Sp/UHHn6sTu2bq3G+K3MjzJZ016xNbnK2lq1uqL2TjGbKEGRDtcx+cy+UdXR/NwudqfYBAoJOCiJWIACBdkQga9zkg/RcbNK0cX1hk/ZP5xCAAAQgAAEIQAACEIAABNoSga7dE4HBG/Gkiuz/y4bbxNkXsta7Q3OK3c6m1eSl6V9TuSts4mxaWY5NmnVyH5XnmXubPBOEWo71TpphZ1QlwSR/5Cj/5b2CoiuGhixWmivJvVW5TUZLHKqc5MJLk8MKfK9xeRdhGYoj6ktux2SlojT1nG/7iWm/Yn8kknTdcZyJO719v8UmCKSz7Antt1euceYNGuJFqGKzUKkuIH1NHHWNu4zZ8f/Zuwo4OYrs/dY32U02G3clHkIgQRNCIBwQLLjb4XaHHS4BAhz+R49wyOHu7u6BEOLuLrtZ993/91VNzfTMzuzOrEQ27/22p7qrq6uqv66umX1fvffMPbMOL5kRel8kDtL69DWWKgUL5vstokLLhR7T9RmJOZI10RJjoXXEeuzGBwmlgnlzo+4r75HEWCTyheeTMS4KFs4POy5j7Wdt5aNpj67S+J6SnCyYDystj5u02uqv7TzHTovBQ/BSxJvn50is0OtIzqQi1k4zPOuSDeuEFlMk2CIJrf4q8mFDEoYoreu8MxTWObTQObmeZA77TILopVffRFo3a59I913XfFrltAclMyCuWAZj2zGuBBajFdIeZE64SE+MrDYLHPQnFc3lC8TQWYoFB42uzKzrzel1ioAioAjUAYFYf/srqVMHkPUSRUARUAQUAUVAEVAEFAFFQBFQBBoTgVj/sYumL26xHVSPMgzubcbG5cmoBOveJpwSjXXSzc0ixC2YVZUkcypTZC5InoWggrLgECc6G49oeqZlGgKBARPvNvGBqHieesYJDVGl1qEIKAKKQIMhQJvP1lIufaRY+oPEGRBfIoPiyqR3fJXfLWhoY1xksAzk9g8VyfJlVQuZCuscOD4MLabHioAioAhs8wjE+ttf3a9t849cb0ARUAQUAUVAEVAEFAFFQBFQBBSB6BGgQux7KMcWgqCZWlUgoxC7YATc3XQOs0Ka8XeGQhM3BCuqs+C2bQHKz6tMkvlVySB3UmQR6tgIgmdrib8TPQrbbslB9zwkSRkZUgS3UYWLFsDgIEEyRuwmzuXYKsR+UVEEFAFFYGtAgBY5bUDk9AaJ0weWon3jSqUfvm92iKuU1vjOIdEDs6lqXSWZswpkzu/4vvkBsd5+Q+ycVZKs1jnVkNIMRUAR2F4RUFJne33yet+KgCKgCCgCioAioAgoAoqAIrDdIkC3NSuhIHsfFjhT4c5mV8TR2dUXy6CriWUQDA0Vb22R3zauCuVKJauqBIROAYihRKTJshjbEhA8a+CirWgLx+AJ7nnTO6osKUZckC6S3n+g2bx3uOHLz4SB3lUUAUVAEdhSCDSDHWdHLAToCSKnF0ic3tj6xJHYsUSOtbOpTuSwv3RwtwJkzhSQOZMR220yyJzl+K4qD0P8bKn703YVAUVAEdgaEFD3a1vDU9A+KAKKgCKgCCgCioAioAgoAoqAIuBBIFYXDJ5LI+4692vhCiRi/XMXKTVu2YYhvsFOIG76wCVOi1oUaSSHcnHtMrhoW1yVAGInCa5ykmUZlHDLkdKKR5Vx4RCvX176wMGSgRg6ye06iCCGR+HSRZIz5XcpXrWyfhXr1YqAIqAIxIgAvz9ojdMN5E13fI90R9oTbtV6xVVId3yPtMT3SHgKJ9BQHupYiO+RvyqTYUGaatysceGBfn8EMNI9RUARaNoIxPrbX0mdpj0e9O4UAUVAEVAEFAFFQBFQBBQBRWAbRCDWf+yiucWaSB13PRVvbbFWehCInSFxRdhKZCCCV3cKY73jrvGmJThYj1XWS6viQe4kgthJwqprbFDOrUSarSSPFy7dVwQUAUVgm0OAJE4m6JYuIG66gsTpirQbtu6wxukBa5x2+L5IieKuaJWzGt8XsysTZAYsPWdUNUP8tlTZAItPLhhQUQQUAUVge0Ig1t/+6n5texodeq+KgCKgCCgCioAioAgoAoqAIqAI1IAAFWnroVD7FgQMYxj0QAyEgZXFMoBBrbH6um+8VdhFClNNRR7dt3WFm7a9QA4VQeG3BpUuB8mzEiQPt1Wom7ERViPdAJKnWN211fBE9JQioAgoAlsWgVS4U2sLEqcTiJvOmNM7I+0CAodbN5A4HbEaoJnfGqdmmxzGyiHxP78yXubAmnMOyJzZIHKWggZS151b9jlr64qAIrBtIaCkzrb1vLS3ioAioAgoAoqAIqAIKAKKgCKgCGwWBKhgmwNV3Vysns6oKpdeIHj6VTLQdYnsgEDXfaDMa1+DBQ9Ve82h6OuNnd4geRgtoQjb2qpCEDvxIHUSkCaC9MEGImkNSJ51SHMkQV3ubJYnrI0oAoqAIhCMAK1wMqRC2mOu7gjyhrFxOoK86YytE9ypdca83wHzfjP/ZTWTOCxGi5x1IHIWYt5fgFg580HkzMO2GERODoh9tcrxg6k7ioAioAhEjYCSOlFDpQUVAUVAEVAEFAFFQBFQBBQBRUAR2P4QoMJtExRvf2KbCuudViB4elaVSG+QPAyA3RuKv56w4OkC4ibNv1o7PE5UBPaEQrCnIXnKoewrl01Q9q2uigOpEw/CJ0HWop21huBJhCIw0VgO5SrREx5QzVUEFAFFoI4IkE5pCQKnHWbi9iBt2mM+7oD5vINJK0DmVILIqZJWQeR97SQOvzMKUPdKzOtLYJGzCPP5IljlLAKJswRkDr9PlMip40PTyxQBRaAaArG6LfNWEI1rYm/5rWlfSZ2t6WloXxQBRUARUAQUAUVAEVAEFAFFQBHYihGgIo5xcbJBtvwJCie9qsLEVOhZWYpYCtxsXIXuUAZGE1chCfWxXDvoCYfCxY9gK4aCcSOsedZCIbgORI8ldhLgsocET6JsQLoRaRY2dd22FQ8W7ZoioAhsNQjQhVprkDVtsLUFgdMOaTuTwiqHhI6xwKmSNpiPU4N6XTuJw+IuntoyzNmMp7YURM5SEDlLsDGmWj6IeRVFQBFQBBSBhkNASZ2Gw1JrUgQUAUVAEVAEFAFFQBFQBBQBRWC7QoCKOrpomwMXbUmwuKHSsBvIne6Iu8C0K5SFXeGyp0uUJA/Bo0KxCxSLXYwukURPqVEYZqP+9SB61kNpuAGu4Sy5kwACiCQPUh/ZtAn7JRqnh1CqKAKKwHaGQArIm1awvsnE3EsCpw3225j9CkPmtMX5dmY+rkKZONjOeCU6AodXOBJnJebjFbCwXIF5eDkInGUgcJiSdC+D5aaKIqAIKAKKQOMgoKRO4+CqtSoCioAioAgoAoqAIqAIKAKKgCKwXSFABd5axMSh67TfYcWTUgXlIZSKXUHudIF6j8G1GZfBbozHI5Jei7s2ByAVjx2hgGRAblrzcKtE3flw4rMR5kMbQfZshHIxC4ROFhSM2djMviF6cIyUZE8hNl6toggoAorAtopAPDreHGSNIW8wD2aCOCeh3tqQOdg3xyRzKrHR+sbOtbwuINETLrTQ5Fy7DjuMh8ZYaHZLkpWY81eAxKEVpZLpAXR1TxFQBBSBxkZASZ3GRljrVwQUAUVAEVAEFAFFQBFQBBQBRWA7RIAKPrrdocKPkgxLG64c7wTLm04geLh1xHFHKCC5dTDKx1DXP5GBo4KyJUihltBN9goie8pMPIdsKCCzQPZkGbInHrF7SO7ES45JE0xcB+6T7GGw7iKcU8InMt56RhFQBDYfApzfmmFGysAcSfImA3NkK+xn+vYzca6VIW/gVg3kTWvMn5mYBxnXrK7kjbu7YuxsxHxNF5hrMEeaDXPkahD2ZsO8TsvIUrSloggoAoqAIrBlEFBSZ8vgrq0qAoqAIqAIKAKKgCKgCCgCioAisF0hQAXgaqzq5sYo2fHYWkBB2R5WPIHg3AjYzTxY9DDGQ3sP0ROt+pAKzRZoqwUu6G4uspY9VVCIluJcHpSV2Ug3QWG5CYSPI3pyoArNhQIzx5A8CZLLFMdM87CpAnO7Gq56s4pAoyOQjImQc2BLH1FjUu4zD4RNBkkdpJbAIYkD4ga9agGrRVLlwXNi8FE0nccUjLhklsBZh/nQxTBbh/mOFpdrQdwwXYc5m3Ogkt7RoKplFAFFQBHYPAgoqbN5cNZWFAFFQBFQBBQBRUARUAQUAUVAEVAEPAhQQegIlPlVNjR3AggXKjTbgeghsRMI5s3g3sgH0dPWbJa4ieUfWqo86cYtBQrRtuyHh/ARH+FDF0M50HTmQMGZgwI5IH1ysbGfeSB98nhs9t2xJXwYW6gI5StD1KxsRkURUAS2PwTiMZc0w5aO+YzETQvMXy0wQzgSpwXmMR5bAofkDfcx/4G4ycDcRNeU1sbRi13sxI27uhw7eah/A+a3DZjHTGwykNZ0m7Ye7tSYrjNpkpnvKnQuc9BpqggoAtsoAu1bt5CMFqmSk1cs67LyttG7iNztWH4DR65FzygCioAioAgoAoqAIqAIKAKKgCKgCCgC9USAikQG2OY2l8vIIfEgetKh/GwL4qUNXLYxbQslqQn+DUWpDQRuY0e0hs6TVjoJ9tKYPqlAbY1rWYfVZ7IDFWarAsnEwOC08sk1aRwUpHGG8CHRYwgfkD752M9nih4wLXD7yC9AXiGOy23lqEVFEVAEtkUEEkGONMeclIb5IR1kDIkbzlHcTB73faSNIXKw39IcwzIHpE1L3DStbUgyV6dpqufEghFnLJI3WZi+XKyxjZh7NoLA2QjSZgP2N2B+3QgLHKacp5SMjgVhLasIKALbCgIkdChMldTZVp6a9lMRUAQUAUVAEVAEFAFFQBFQBBQBRaBJIECFI61juC2qohrUCmP0MN4EA4S3AbljA4Xj2OzDZRGUqIw1kQkFZ6ZZ/R5JiepqjJxSzUrVQCoUse1YzOhdA6QPs7gSvhBt5SM7HwUCKUkeHluSh0RPoXcfx8wrgNKVpA83xvdhTCJ1d0RkVRSBxkeyZ4caAABAAElEQVSAbhtT8MYxjg0JG0PaYC5Jc/tIzT7mFKbcLKED0hl56Xj30zEvuLQ53vnwq6jrR9oQCc48JJlzMAdmw6owG20xdlg2tizMk1mYSxw5TjKH+4wbpi4kiZ6KIqAIKAJNA4Hw3zFN4970LhQBRUARUAQUAUVAEVAEFAFFQBFQBJooAlRQrkesB25Gy+m7zyQoOhmbIhM0i9mgmGVwcQYVZ8DxVlDAMiXRwxgVrXBxy4ir5qMHj/9ct0SfWjqdbRDxY+txythC9LEAWQVQyJoU1xVAIVvoS+nKzZI7SJFPkseRPe6YZYqRz43niYe6TIr+eWnJpo1AAt5rxqwhSZPq2+gOzZA2mAMceRM4ttY3LJOG8819aRpTQJWGuYJp81rnCjcB1A9fN1fkYq7YhHebMcBI4GwCacNYYDZNAKGDDcRNNkgbbiS/y1BeRRFQBBQBRaBpI6CkTtN+vnp3ioAioAgoAoqAIqAIKAKKgCKgCGxXCFChuRHKTW5GqB2FQCdrlLuMYdEKhI8ldmxQcgYjt8HKGdciEN+iJS5ifIs01AnqqN5CVauz+GnN2vy6V3aSjpMCwqMSKJQLcaoQ+4boMftxUgTlriV1uE9ShwSPJXl4zhE9Jb48nitBOXccSC0ZpC7hArjr3taFAF2dkZyhFQ3JmRTsuzQF5Euq59gROM3w3loix8a1aeYrZ0gdnCNh0xzvHo+b43a5n4J3pGa3jf6XtcEAKkNNBegD43jl4r11cbxyQNKaeF7oUQ4IG0PgYH+T2afVTYJ5xzlrqCgCioAioAhsnwgoqbN9Pne9a0VAEVAEFAFFQBFQBBQBRUARUAS2KwSoAKVFC7c1jqLxaUVJ+FBpTGLHkju+1BwzHoYvyDmUwy1RzgY5p4UPYmSg3sjxMeoOMRXMdOFEhbNfzD477eu4/4SgV4IV+rzHKinG6SLsF0NRTDdNhvDx7vtIHkP2YJ9WPiXmvCV5SPqYPJ8FkD3GOZQpg6VCGVKeJ4FWijI2tZZC1XuGDjQx6dDeOOGT9Rs2SmVlzU7yMjJaSmpKwG0goSgsKpK8vPwmhkrgdjhMnaVMEsZjMkanTaskCe8MremSkTKfJI0jbWwaOE4xZUjokKRB6idxmEd3iL58334zk5KssSQsXarVLN6Xq+aSsZ7le+DicDE8N0kbxuFiDK5cvDMuFlcuiBoek6ihlQ1JHOduki4Yt4f3KVZstXzNCKQ1by7p6bQrC5a169YHZ+iRIqAIbNMIKKmzTT8+7bwioAgoAoqAIqAIKAKKgCKgCCgCikBEBKDckmKoViuDrWBCy1NxalyZpTSXdSxbRnokIPFQQnNVfwuQPNzSjWUPyB3fMUkeFyg9EF/DxdlArA1UxXgbVDbXbA0QaDPWPRsTxFocoBkrLjWq4fDqYeYSnVKzVYHcscpoHluih+csOeRIndJ4uHiqqDT5Jg/nAySPJXvKSQChDhI+tARiGm6f5SrQT6YsZ7akJKlAP8rLK9A3m8c+ct9uJLHi6qfwxj1IElQiJVS9Ry+jR+0p11zxD3PBZVffJHPnLajx4mefeFiScT9emTNvoVx+9Y3erJr3k5PB4wGQkHEZ8aIw98ahQIqA44+Ei90E9my+fRBPiWWlkojxzzyzgTRJQLOJSHlMYoab2Y9HzJiEeEnCNTbflvOTNyhHksYc+8gZew6w8xzySMxYIgfjlmXo2gz3mYLOJuP54q5Nf/3DGMeRJbpSka+P/gzHIslTGzcLMbQwdl3crEDKWFqOvEkAoWPJm3yQNtznRgs7juNaJdbnX2uFURaItV08P2kGag2kpRmvUTZTrVgC3028M8XF1U41agbb5HtWzghpjSRh3s2GbuniC86SfTBPhcrBR54UmrVNHKdgbmrTOlNWrV6zTfS3sTrZOrOVFOGdKCrazO9FY92Qr95xB4yVs844yXzfn3DauY3cWtOqXkmdpvU89W4UAUVAEVAEFAFFQBFQBBQBRUARaFoIUNE2ZCeRfoPgrgxKw8k/iSycH/keu3YTGb6HSDtYUyT4/uUtBUWxYK7I919Xvy4dlMve+4p06AjNMtf6Q6jYW75U5NsvYVZRYBSvBVC+FkBJa6180I/e/USG7QLlI1TPSxdJ0s/fwbKGRA5IH1/K2ByBY8TpwHFzEAnJ3bpIenmZZGStk7TCfLh3YxwPWuY4d1BQaOM4Ht1obEEzUNLbjZZBQTpmnjTiOuJSqLXJDoSIsxYqw31QLVqG4rTsKU1GkHY8x7K4eKnML5SySp4nWQP9KTcSOizXKlNK0lpIMZR4ZXFoAFYwcTm5UrlxHUgeEjqO2LEp2zMkD9ox5E9maynq2Ucq0U7S7OlSCeUXy1BxblP0p0t3Ke7WUwrTENIebZKUSNywXhJmz5QqtMc75OZII3dchTq4PyR/lTSf9jP24mRo4TrEZso3+cjwp9x30KXP/E0SqciFlOGZUxLnzJe9cB1rZDm3kXRx+3HN06S8/yApawVHgVByx5HswPVpixdI0vIlGBtVGI2WqHGpdOosFR07SxkJmiq4JistkdTsLElZthjlK/HIvKQOruXYx3tSlZwkCbj3pKoUew3wSCoqtOMCZIsbH4noXGLzVLxWCaauZBBASbgupaxckkHAsVwS7oAUVu1WMigUJLxzCN/xLSCVaLYEnS4A1gWJiVKYkCRFpeWwqioxsa8Y08rGwIKlTbdekpPZXvKTUrClSn6iL8V+AbbyH76VwuwcKeuMuWj02OjvBnOILF1sy+NdkL1Gi+B5GoKDuST1pvwm8tcUO0eFqznW+TK0jrq026evyB4jwVy3DNSGMSQ/fSuyelUgr6Y9jFnZHXX07A1iiE7xIJyH589BPd/j4URQZHO8hMzF8jPKRyvxeOi77SXSA+3iXTNSACu6ObNF/vglMs7e+tnnPfa2OZ+8K7Jpk/es3R88FJPHsEAbeL9k2RJ8J32DyYuzoE+G7iwyCGVrk6WLJNJ95mDOLPURwIn4DowHARuNJGHsvPzsJFP06Wdflo8+/cLsd+7UUR6673azv2z5qoiE9ITrr5QdBw+QyX9Mlbvue7jatSYjzMe6dRvkwkuvDjrTp1dPueDcM6R/vx0kgc8IwuGQX1AgM2bNkTvvfQivg51Pzckm+sH7v/LSizBNt8EaBM6w5DpLZA2srqZOmyH/feq5bf7Ou3TuKM1JBqvEjIAdETFfphcoAoqAIqAIKAKKgCKgCCgCioAioAgoAo2IQCr+yR82XIRKLq+it0VG5EYH7WgJGm8JaoK44pznQkkdEjmHHRUgf3gdy7O97j0RTAdKb5A6fqFyfuBgkEa7BRSPPNmuvbFCoQslbn5BVV6Jh8I7YdxREtcy3SjPO075WVrMnwVSh4HZK03aLBkK8/4DJBGK8+blJZJWXiotyoqlGRT6PG6Vky3NCwuFcUOoBklNipdU9DkFrAWtKrakUPVG6wvGJzHidIlQjgs3J0ZH5+2sbz93A/xUYQsVD6TuFK/gZokem5aXFErFvJWw/EHDFRVSmVgFVEnQ2K0iJUlKCxdJ5bzFpowhalCWBE5cqwqJzwWBByxd3bzO7TOlZCybIm1fW2n2T9m0Asp/EIYQd94c+D54++3feBzDCYRgQaGsWb7aINMBpS/BPfG82wiJ249LSZaKNPRq+a8Svxy9RJ9IACUgJZmT3LxUEkrKfKSOJVDiMA7i8hZKQu4CU8ZY2KCsSaH5wXAx9fu6ZhNasS3fGJTlPwiDuTlHK4ywwt5vPcJnRxusImBAOoC9trGobFoINOiKMT+jtWzo2kMKQcwUgKRxaYHvOH7lKimZ9hfiWoHUwbvNtHjEEVLV1rrgQ7XVpdmfItm5eBHwNmTUMF+FXsk5jxJuHmM+CRsSHyRPfviaOQGpy3wZuNru1aXdnTBHk9BxQrKCpAzxOfwYkQ/fEVmxzJ0Nn2L+lPEo60h4V4rvcb+BeIk6ibz+PF5iz1tWw1zsLq81Zf3jxotwIQAFY8XM/Zz3h+9qif6P0H/mRxJag+5/UKDvoffA60buYxcmeOsgRiBtzWKCd94IkFYt8GyjGTMk3yLIpCefFW6Uf154jhz0t30jlAzOjgd54pTrnTrimfiEZI/LH9Cvjxx28AHy/kefudP+tEO7tqZct64gIn3ivdblhaZt2pgIc/7sXj27ywP33uYnc9wJPq4WcCu3527DDfk0YeLdMnM2FmtsA3L4oQfJ2WecLEuXrZB/XH5tVD0eM3qk/OuSC6uRcqmpKdKze1fp3KlDNVLnhaf/Iy1btpDrJ9wh02eCmFRp0ghsFlKHJmJkU0tLmz6L2qRHi96cIqAIKAKKgCKgCCgCioAioAgoApsLgdPOtgo2tucUhTW1TcUgLW4o69aKfPWJSE6OPabirXNXu+8+qXxzhE4JVL6ffiSyBqvKqcCj8rQLFH0FjIbhkYMPt/nMcoo+apqilMqdR0hlhy6mdCn0t7nN2qCekNgHBx4t0gllYAXR/NvPpfniedK8rMSSOK1bI02R5hXZJhA8A8PLTruCRKmSVssXSfPcbJA/pZIKxVdcmzaSUlEm6bDWSF+72sYkQcspKbAUSgQRBNKjeVmFJON2qb+P/i6ivNlGLsb+ciMZgqdlhaZBXnol9NmEkkvuOpe6St1xuDQfq/jnY4OAJqzdJGX+QpYCYQfdtCGzzGHNH7Aekk2WLIpYMPTeynHvsJZpSsKnyTsqxTNlnKjiBJBjeJeL8O5WwIKmpLjUxI2iZdcmWMUUJSJKT36BlK1aZQgbxowqQgwbk5KMwWbiasHqi+QM9/P67yM5A4dK3MwZUrFoAciYLDtXjN7Pkrrdh8LKbwXI3cIAtFCqGvnlRxAWSwP5bs/NO6tw3TuvuVybjhxjSGBjzfFNiGI8x2fhwbmMQouchfOq92kwRt6UX4P7FOt8aVsI/oy1Xc6hjtBZNN9aNtIqksTEIUdacmLMASIvPBncTugRy7OuZYtxzyDE1q+z1+4+yhIutKDp1Q9YeBT49ZyLTRdo0ekInW++EJkHJThdv41A/k6wwuS5Xn1EOC4iyd/GBQidcGU6Yi6npSmFz/Kn7zCggVHfAbDiwhhrBUJjxO4iP35ry/z5u8jcCMr4vv3tIgdTF/DeQnL230+RL77+LmY3YLzmm+9+qtbrDRuz/HnUH993562G0OFX7CeffymffvGNLF6yTAYN6CdHH3GojNhlJ0Me3X7LdXLsyWdvExY7vXp0h7vKhLCxjvw3H7Jz/tmnG0KHOLz17gfy+VffSU5uruy8044yZNAAaZaaGnIFhhMIQVpmpWBhgErTRwCzZuPLdVdcYhopwcSVl1cg+fgRlGe2Al8aOHbnymC+q6IIKAKKgCKgCCgCioAioAgoAoqAIrCdIgBSQ5ZBYTpjqsiqlSJnX1iz8oyKOApd37wLRap3VTcVsgugUPMKLW6oSKS89mKwgpRuXZYssue8n3QDQ8KIfVoIRd+Bh1iLHm+ZSPtcub7rnpHO2ny4DzOEDo8+/1gK0QerSoYSGYod2egUy1CCOsnGPSxZKPGAK1kyDNmTmg2XWQOHwf1TJ6HLrk7vvwFLnnJzTgbtLlXNUqVV3ibJmDrZxjJB5SSG4nr1kji44mpRVCBpcD/HvKR0OGWDm64UuAhKz9pog9wjnyqjJCiPkkAQJdMFF9rfFskhB6OmDYcAhyqGA8gYbNBI4m2y+yZFbCUUKIUlWl6bdlLCdzArW0rhKq8EJIyJ3wQrjGwQLUWJSZK4dq2Ug6QhIVPYtY+s79VXynE+6cvPUT4dpE68lBTGSV6HnSW75w4gYouk8p3XUQ9c/aG9WgXupOSv6cEWecuWiPwO0mSfsfbytiBZmOck2adM3QDyYWMYyzJXjvGa1q5xRzZ1bsRKi6ufcyUXQmE/+ZfY+hTrfEmXXLR28caPibXd9h1cj9Ff4EWygpKXC0z/sKRFWnP47MMz9rZD0tzrOotE2luvgszB3OqE7tvo/vLkM2xOB7TlJXXqMxe7Nnr0tHvZsFabO8vus5+//IB5uDNY2I6WcIlE6pCsIWlDrXsoyera2GWE3SPR9+WntixzZs+wxBUtneia7VeQHZirjWWo1zrU1UNXiTsOs0dwwWgIKHduM6d0A3Y14ordfNs9MbW8cNFSmTJ1Wo3X3HvnLZLqIySuv/kO42LMXfDX9Jl4VWfK8cccIaeffJyJVXbx+WfJ/z08yRUxxAlj8BR4SVj/2dp3WrRIl/59+8iSpcvFSzZFupLWTb179UCfU2TO3Png1TnzVZf2cJ9Wk6RjIUZhIeYufJdSMjJaSkv0hTJtxix56tmXzD4/vvnuR7P5M3w7tIqKxdUeSbLcvDxDmIXWFesxcei7Q2+DP+PMRcKB9bLssKFDZNnyFVFhHGtftqfymFk3n6TA5D2lTbK0bRPZTND1hj4CLfFDwidA/uSDFArNr4jw0ri6NFUEFAFFQBFQBBQBRUARUAQUAUVAEdjGEHjqsYACrLau0xKne09b6ncoQ72ETqRrh8G9DoXKvGgVQO+/FX2fbO32k4rrgw6z+3NmQhHYya7Q9pbhPhV8FMZ0CEcq2bPBnz5FJ1VBVHxzM1IF2qVNd9PfZZXAxykedznAnv8NSsQqEE3QRzLwvQ1q30lKho+UJFj4tF/6LILWV0rlLvtLBRROmRvWwOLnB185W57B7suG7YEWYSm0dpU0W4bYQqg9CcdJOJeINpOgO05EHKHyrl1B/lRIi9UrJQnKX8ZnYVm2XQn3O4lVFZKK55BUgnMm35JEibDOiAdxRNdjSWWMCwMXdjivBBJAiEEYM4ZjpALPuwKYU/XIzRj4uH2kZVCoFmMFeDniEsUhLowpA8xJzLBsGfYZD4nH3GecpBy4yCol0bJ+vVRg4zmSM2WwiDEp9kmyBPZhQbPDYNnQdyAImkRJff9tKa1MN2UMsVOB8x2HSUmPHfAuFIi89AxqhLToBQsHEKMcy1/DosEbi4TjPR2kaP5KFPS9A+aiWj4iuQbLzwtc6IgYl0NSggIruEaRuvQppvkyTeSE0yzZQvdoK5fb24i1XS8uznrJAcI5jMJn5dXZHX2idc1G0uwPbBSSOuGk2JHYOIkA8UFS17nYVUIShkQ7Zf5cm3o/Z063pE4mlPEsy/vwCi1s6FaN8tWnImMPsvveT17XrYfNmYfvmdA6ZiOPc74ph7l68ULv1YF9EnAHHW7L0ar0688D5zbz3vyFi6Vvn16y2/CdjeXMrDkhiyXq2Z8OiPFFmTNvYRCh46321TfekaPGH2Jcse03ZpSf1Oneras8+sCdEo856ZFJT8nHn31pLpt40zUybKchhgiZNXuenH7K8bg2XTaBaDv57xeYMuMOGCvnnnWapGBRg5MKECwffPy5PO5zZefymZL8ueOW66V3zx5BfN76DVly4613GsKC5ehybd8xe0urli15KHRR9/6bL5h9xj465cwLzb2wHImQsy64TNav3+CPn8OC5V5C1FwZ/EF3aw/cc5u/DZ695Yar8TOIM77IpCeelQ8/sWOmHwgrxj7KBGnkhMNy1py5ctPEu4Ksr444bJychX6tXbte7nvwMbnikgukY4f2uN84GX/sqaa/tAy649brpQew5zB2wrhHJP0KPe46u3bpLDddd4V06dTJX7YI7zXrV6kbAvyttFUKfQRya9cWE2gtUogVHfkgfujijZY+TEkE2eNAmof84tAvglrq1tOKgCKgCCgCioAioAgoAoqAIqAIKAJbAIFQBVhNXaDizQld+DihliFcPVSScQU5xUueRCpvS4avy52rKR0F5R/jNCCuivzwrcgxJ4Qv7ZSMsfQpfE028DfPOZdyoeU8sYmopOcmOXnoZyaIISy2T2+LDxz3GASfbc0Q8Bz7VQFFkKkOyiBpN9i6pINCOH7JO4akMaQOiAMSNtyS0lpI/vDDQBaVS9svPpbkoiyQOu48mtn5EEPqtIJ1UOrs6b7rLOGTkJomBTvubM//OVkSoUhPQJ0kdfgYKxCXogKKuDQohlPgCqto4BDElKmUFljNngClM9X78bAKKUYw8wTkp61YgXw63oKOlOdMWiUl3bubuDOpm7IkIS/fnDPn0U+mFKatM1pI925dULZK5mHleRH0EU5cOR4TGsqOg/oaJWMenv2CJVCe8wQKuvNMK0F+MDX7vtQds3Wq5nhc1qatFGW0wjHc68B6gPmVsGbJ691XKqHITF22RCpBjDG/AmVIyDDNHTZCyrA6uhnuvWrdGn9+BdqljxSWsWMAnMiI0ZKf3kKS4QosYcMvvnzfeZQPkDp2vxTxaDYNP0AqQOqkgCAoA3vEuqw6EZVHkq5D4Oasr7V8q/S5M/OWpaKvF9xNkbB1QqX3nntb5fbYA2HJ8QVikYBYoSVI73621IL5rnT9UowXI5w/aJ3nhHOE016CgPTvh5tn3DUNlUbqE+uPpf3OXcCo+pTXrNOROpH6GaldWO4Z6xzGLKObtU/eh6Vklq1lAOYNyvIlgb6xnJvjdsCzdaSOKRjmo2v3QCbr8Uos9+u9Lty+Zy70n87a4N+Fzyw7F7ocvEt+kp7WTbTkGetOetI0kGdOYG1WTRAfzTw3jie6oIsktELFu2/kw/cwEZRFKtno+Y8+/rTcddtNhvy4/prLYUh1foO12b/fDv7X6bU3QTbWIF99+4OMP+RA46aNLtuysjfJ3iN398fhGT1qTz+pk4bnkIBnxhg1Y0GwOEnkvAE56/STjFs37nNYFRUXwfIm1VzDNug67ZobJ/K0EVoCPffko34CqAykS0VFpbEwate2tTz24N1y8eXXGAsYkiheAoUVsC+U5r65be+ReL4Qumcbuceu8s77HxsLluxNOZLZKkN2GTZU/n7aifL8i6+FtYDhdR3b+whKU5OdlhLibDsZ+M6iDN1xkPz7lhv8GNOYIgHfjbS8Gjywvzz7xCPges/1WwulgtxnXzt17CD33DEhyAooDvnN8T4/88RDxmKK9ZdiXNIKh/2he7iJE66RK66ZwFOYbpLgVu8WQ8TxuAThWapAmtGFXM8e3ZilUgcE7Aiuw4WxXPL6O+9LOl4iBrRKx2RIRpT7TJvB7Lu+0hx1cKvNnI3tlMPHrHHxZgigYMLHEUN5+AGXi1UZDGSooggoAoqAIqAIKAKKgCKgCCgCioAisJUjACW0Ea7cZyycYcOhfYcSjIpLur1Zucy61HHKsOZQ0jmhYueAQ6BghrsdkheM30Mi5NefcS2UbvWV7j1FBoD4oHwChZzXusDmBj7xf7IRuuHhKvCevWxQdF6zerXI9D+D3UAFrgze6wNluQu2PXdm4Bw1VlxBT4KJZb7/KqBwZSnPqlpJg5KRpA4xoXhXzdsc++msnPD/PRX5tMbgFiTxqKNVJ5O1LpHKSxAh6Ipf2vaxfeo8BG6JVgT3CRYcsuP+tuhi9H3NSv9lcXuOkqohQyUBfUt55TmQNh2kaMR4EC6V0nLVmxJfuNFa9LRuJ3nDx4HsqZLMNW9LQmEuyBxL1lDtRUqlcPCBsFJJkQys2k+aNsXcAe/CbCR20F/uD0eMlROPPcL0/0WsBl8GyxBzt+Z84KZM7ThxxzHnGyXXAhBAkxY+579vlgRHAiVigNBhXiUUvHxM5jzLoHbiWgmdyqYxR0oFFHXNQH4lzM8x+RVtOkjOiMPMdWkr34RCzpeP6yp4HdLyIQfA5R7GVvJU4PcdciMIx8TQfe1JxnsJJfFCL6MyejTIleZ4pnhvyjfyfUFeNNLC985CgRpW3FiklRuUhMbag2696Bprj1FiYp107wFi4DeM5X4YPyB/YAUWMR5J2EYiZLJvg3a0J6m05wNx4sgQHh93EvqG/lH4XlG5Pxnzhptn7JmG+aypT7G2sGyJJaqSU2AS4ZkfwtVTW7uffiBy8OF2vjn+FCz5n26fV68d7Fz6k2e80T0b2+vWy8YECteey+PY2mu0PeKcvGG9O9MwaV3mQtfy7iPt/dJqhsRiJHHfSzwfaf7k+G+Gsest662P7t123tXmcGx5XdR5y22m/TKM7f/8939y2cXnGrLiNLhBew5kQzQyetQe0rWL/R5w5ecvXCSfffGNOaT1j5MFsAiqSRYvWeo/PRBuxH78+Tf58OMv5OCD/mbIljfexndtiJCgIFFCqxVaHNFShm7Ojjz8EFMyG8cXXXK1seAhMXH/3RNlh949ZeiQgYYQmTZ9lil3+T/P9xM6/3v+ZXn9LZCZkD12Gy43XH25IT+uvfISOfeiK+Sq628155549H7p0rmjrF2/Qf5+7j9Nnvt44+335bJ/nAcjhBI/Fjz39nsfyZkgc/gqHHvkYejnwfLXtJnywitvCF2cOSGhdfCRJxlihuQLhVY3v0/5yxUx6VWXXWzqokXQ9RPukOkzZ5v8M087SY458lDo7ZsL4/jw+XqF7ZO4+uTjr8w1NJbgOLgGLviSMR/yVbrjngfMMyCpc+VlF8k+INUG9u9r8CXO1111qeEBWO+kp56T9z74xDRByx9a+oSSUt72dT8yAr5vn8gFGuLMr79PiVgNmVFH8JDkoR9BS/qA+IE5mztHMsj5VYxYWRQnEsFCtgLTya02oS/DXBA8xt0biR6zn2dTX77LI1mkoggoAoqAIqAIKAKKgCKgCCgCioAisAUQcCutqWA98NDgDpDcyICClqvTEWvDrOxPhxLaSajbHCrYqJDs3hNaFSiraoqZ4eqIlDKQ8f7j7Fm6G9qwLlJJ6MKhOXEEinPD5krzvhi0mxuV7XPnuDPVU674dvcE5X+1slN/B2E0Bgp+rJo/6wLremgTlPGIbyI7QDnuFZZxQouIcOJcMDUDcVNXiaVPro1OXaRq6C7mqOLzz6UQrtlMjJAUkBeQjfHse5LZl/Q2WAmP+4PkllMNAmV2qKR3MIra1TlQPEsIvlBaOcnI6ARroOHmcGazd2Vu1SZ3qnqK6wpRliuhN1Q0lz+qbN/8BT311pgHKxg59ETrsg+K5E2T/0Rx37NJaw1SpZW5vJTBagyN5a/N7pSWQ3GM3eYh7XuL1TZuvGXd/t77BiwMPoIS1Ut+uDKRUmeBU9u44vUcW/DKYmTmNJH+A0Voncf3Yre9bD6JT8YsISFaHyHWB4+37yOV9t99Fbk2luU9890lUcqYJ/36i7z4TMMSO7H0KXJvA2dIrrz9auA40l407a5aYYk0R4K5lHXSTRhJda8wTk40Up+xFU39LFPHeUeG+siHTz6yzxmK7LDC7xInxDycwGIh4rsJklnGHWavorXYn5i7twL5/MtvjMuwHrBYPO6o8VDQf2qIkNq6RiU/N68sX7nKT2T0QmwaJ7XFs1m1ao0rKj26dzWEgnGnVoPlUD4W7jt3a+7iU086zm+Bcvud9/vvg8THtTfeJq8+/4Q5zzg+jtTZbYT93mHfHaHD+n757Q/54edfZTQsb7oinh2NGLyWnK7N0JSEFLdQIdnDmDfnnnkqDBmamcUBw3ceKtzWrF0nE9HfxUuWhV4W9rhzp45CiybKx59+6Sd0ePz0cy/JvvuMlDatM2WfvfeqRuqwzPGnnoufTsG/AYbvshNPySIQbK7/1KM/8/wrhtThOcbt+fnX3/3PfcmyFX5Ch+d5Hz/h3ulOTyV2BPhrZosKfQOSKeVWm5ABNJY+HrLHEEFYsWLIIF9Kq6CGsAAiw9gKrC232oR+Ao2Fj5cEois4bC4mEC2BSBCpKAKKgCKgCCgCioAioAgoAoqAIqAINCACCR6l2l9YVDhnhlUmtobidyAIncHYGAOByjgGIKffLid06cSg3utBuKRCedwXSlmuxKbCePTY6JSfrq7QdOw4ay1EYmhKdaVNaHH/MQN3c1X2iuU2iyTTbugTrRHGHIAg2XPDK9BJwow/xiqZqUT80q6G9dfLHRMropO9T96jsyLiOSrGmUcpyA1WTCf7CBJ7NvDpiJ/iCErLQMnIe7H0ibWwTRejaBoIDo/1TthGHPHEk4jvE1bgxszI1uiyfSyeOccvxbhf8mDdEPcWzbixrQc+Bw7GuzXEHv/yY3XCkm7S3NgIXGUtzfjOYVW6gAfxuwHzluG+1yLGKRNJxh6B8U0ChVY5fKcGQ7FI92Actyf/3boAW7bE1sb+tQdZFyq50D9FUpDXhDXr4Xv1zH+Rov+OxOJ9DkE/dt3TErO895rIoND+1HZcW59qu76u52trlxZUtHLs3tNa5TC2TA88i8FD7RxEcmzKZDuXxdKH2sZWLHXVVHZrn3f2O9C+Q5yXaRHlxltN97SZzt18293y9KQHDeFx03X/ksuvvrHWlqn3XbtuQ1C5WbMDBHqph/iijpc61EjS1hOqY9HipZGKBeVXhcGvV8/upgxJnND4QAWwQs3KzkZc+NbSCRYlFOqJXdydKVOnmzzvx7ff/2RIHeb126GP/DV9pvd0zPu0YuK2/76jDfHhXJXRwuWh++4wJFVubl6t9Q7ov4O/zBdff+ffdzu02hkDQifNke3uhC+lZY5XiAN19JQ0/C659carvaf9XHfrzEyTT909Zeo0/DZTaTAEIvyaabD6G7Qi+ufjC8WtNknAl4tx92aIHlgAIQ0cgwTyEUA2L93PzNZWb6TzZE25dQjxYxiufCjJYyyBGANICaBwcGmeIqAIKAKKgCKgCCgCioAioAgoAjUjAPfZRqj8onsmJyRTfvga8Tb6WBc3iP9hSJ3CfFdC5LefEAfC9z9mIZRIJIXaQoFDqxUqhLkKG6tPYxYqlGlZQ+Hq9GSP6/E4KEMpVIrQOofWBew7FSfMmwXFx+JFtgw/582GkrTKEjo87tC5OpFBBethR1uFN5VXH7xlrZJY3is8R+Xr9KnWVV3LDLhZg5KbK8FZx7jDbWkuSGRZ16dUEErhhEQYxYupzYn+M5Y+sdYxf7PKTlpSkJBzFk5eco+upZhfWgwrD49ykP0NtR5gnalQzFPcWLJHW/6TLqgQM8fID99Ud79U33uLdtx4kejRyxKezOM45TMIFRKpjiD0nmMeSR269mN8FVrhhBM3rtwYZJkRu9nxvXa1yPsY3zy3YrlIS6xAH3eoJb5oGfe/SfYcyZ7uPavXTnde4Uid2rB2NXmJNOZRGU3yokMn21637q5k/dNo+1T/loJriKbdnn3s/XLuevUFzDd41/g8SEjvvZ+dQ3fZ1WcNmBVcf6SjaMZWpGtjzd+a551B+P5wY/fzjzG/euawWO+zEcqvXbfeuDE7dNzfZEC/PjJyT7ybtchrb74n736Ae4kg6+GazElvWO04yxiX500Z18zJH39Oc7sxpy6Ou5dQ8lZCYomkDg0MKJ064reBT7z9dXm0PHHSA7Fi6kvquLpIxHDrBRLqxmuvMO7K6E5u/KHj5PmXXnPFIqZdu+A3g09Wr1nrdv3pxo32/YzHghfGDAq1yvEX9O3Q8scJXadFcp+2dt06IQFFY0bKhg1YsKLSYAhsU6ROLHddAZZ1E/z2cYtGGKCqhSN6YAnUEi8s3b+1hF9L4wbOl0cGsr5iXcqlRVUNGdcc/NhhSrM7unvL8e/b/HzEB1JRBBQBRUARUAQUAUVAEVAEFAFFYLtFgApiChXGJEVCVpXKksWwKhgMl00+awevB4VWWEnqSB1bi1U6O1dkdO1Wl9g6XjdERx3vag5Oh++OIC3Y6JqNQcP5v10rKKjZp1DhPThp2yaY1KHlEckYF4SchA4tj2oSxmYIjc+w6x72CiqpqfCkFABbWom4um1u4LONT8nFeCf1lWj71AvKZApJm1PPtvuhn4cfbXPegcJrw/rAWd7H2oDrHnOCq5MdAZHfAPcRaK1+e8NGWJderIWKcroeCxU+Hyex3ltdxg1jfDgrKbre+vxD13pwSusd30ruoBNuXDqcGfsqnJBYpRQW2ZSftFijkPR045PHuZtEPvsIMW5OsW12ggJz1UrEZ0I5xscKlXBxfKLBOrSe0GOOXyrive4dQ8vEctwQfYqlPVc22nb7gPimLAJJ5yW6OH8Yqx2QfxwDdEn3G8ZvbRLt2KqtnljPb23zTgbm/1H72ruYPVNkqWfuj/XeGrH8pCefNe66qONkTJjaXKbV1pWffvldjj4C5CzkEMTGqYnU2Xsv+11V5IvvUlvdkc5vzMqSPiCQksLNVbiouU8H7PSu69YHSAnnzsxbt9eCaCXcszW00N3aeRf/S95+5RljnEBXb9GQOqvXBH4PtG/XtpoVVKbPNRun1doIHd4TST0nb77zgXz3Iyygw8iSpcsRDq3Cf6YNCDKVhkMAv3hViEAhzOq4rV0fGJjhkKGJWUsQPC0QLM4RPzQjC5fH+D31lZYt0Q42CZDQ1aqsqGDsHx/p4yOBQl3BWWugfPzu8f0wr1aLZigCioAioAgoAoqAIqAIKAKKgCKwjSKQFVjha+JtzAhRfnfuam+MbpcocAPOwO4mQDXJniUeqxiep1LYSV3JisJ8rOAHIRROnFsq/n9GAsopRKlcJKnTb6DIT9/CQsjz/1tHT5+83iu4BJZxhLp0sy199K5VaIdrt6Y8uh/baRdbYsbUQEmSScOgiOEKesbYqAwoaAz55BY+hmIYqKHue5H6RAsdZ+0UWrvDlrgSXyqUuJHYIenRd4B1Qee9zlnCMG8liIqtQejOa/e9bE/+mGwtQcL1q673Vpdx0wGrsw8/yvaCLu8+xFiLpGOY8Ve43gbyOF4Yg4bKVNbrJdrYt94+AmeZ791MgiWV/9mWBOpxe15iNgPvEEkdthHNuIwWa9dWpNS5yPO4kYpUtNb8hupTrQ2FFIil3bY+Qq4MJE6ocFxkY/U/rR3DkdSh5WMZW6HXNvTxlpx3aBl6yHjrvo7EJ63ztlJh/JS7739YJt50jfFc1L1rDYrLKO5h9tx5QpKmGWLRjdxjN6MLDedabMfBA6VLZ8wZkBUrV0dRc+QiSxHjZbfhO5u4Zz26d5Oly2Bp5hMSPYwzQ1m71uqK6YasDL8fGCdtp6FDfCUDyV677+o/mDN3gX+/IXfYB1oWpaamYC1CeL1znDON8TU8d16gL6P22l1CXdYx9g2lKBzh7avDm3hx6ABLnPkLQn5DeQtjn/GM0vFbYecwmIUU1cMYEFBSJwawWJSTVrQWQHTHZmL90OoHzLWJBxS0T5dwcA2Hc4k0ea6jJMDEPLNVhtlqq8K4fnNxf7A6jZY/hvDxpTm5ucYqiK7uVBQBRUARUAQUAUVAEVAEFAFFQBHYJhCgy6wVy+DurLuNPbMaK2Tpeo2WCIzxkOEjV1YGFDYy9Q8Rxr7oDrKCVjkL5tlbpeKfpAqFsW28JIbNje7z4/cjlzv+FGv98uuP1t2bK0k3VozpQ0X3qP1Evv/KKs0ZQ2TnEbYUlaUbfKtuqbgZe5C9B5794G2QEp57dPV6U95fSiosGKAMo9skCi1VSAzRWoX1M06Nk+nYHzbcKhkPPNjGdiDZRAX7QbAOopBkiUZ5bktX/4y1T888Ub0O5lAhes7F9ty7r9sx4Eoy/grjf9BFFskEup+jtIFimq6mKIuh+CLZt6WFROPIfWwvnBVXTX2K9d7qOm7GH2vHAePZfPBO3d8N3gutfKi0plXL3zD23ngRcXZI1mFM73uAfQdYbuoUfoL8BHHAcUbrrN3x3tLix1nccdyOsCv3TdnVMSh6Y8W6e0/7jrB9pwBlQHsSg32wUTiO6iOx9inWtqCQNi4M+d5/86W1dGIdsbZLl40t8PzoanL+HDunsB4+w56YV108IxJsTnbZDRZXfazby+VLbS7f/4YcW64tbxqp3Vjmws0x7+yF956YUkia1vX7x9bQ6J90fTZtxmwZOsT3nVnPFp965iW5+PwzjRXKs088Irfcfk9QHJaDD9xfLjz376YV6i0n/vs+f4uMB3PPHRMQ4yVNbr/7/2Te/IX+c5F2PvrkCznmiMPMkL3xmsvlgkuuwjoLqw/lMV2cUT74+DN/FTNmzTHkBC189kEcGsbRofTdobeM3RdzE4RWS4zJ44THJKLoyo1kkWuD50lSXXnZRcYr0+VX32TOUUd85hkny0y0NfWvGf66GPLjvLNPN4QOr/1z6jQmRlauWuN2ZU+QS5P/8H3HIXfZ8hUm3jt100cefoh89c0PIMRWmfKHHXyAcZHGg99+93z3m7ORPxYsXCIDEauHBNyuI3aWyZ5reY8DB/T1W1uRLBs8sL8wJhBdxjk3fMRkryhc90XuxfZ9BjO5SmMhUFhUBEvhIlm3fkOtTUQigDJoqUOrIG7Yb94MP77rIQHXb1gtUYMUFRWbCSUHq9KMCzi6fAPhQ9dv1h0c8kEEqSgCioAioAgoAoqAIqAIKAKKgCKwVSDwKxQrtFYhIXLMSXCptokRfAPKYSqEf/8l0FW6cCJZ0QyKTRIje4yy50igOPkyoMhxWY2akohijJJeIKKoYCXZ5NyfuYZ//p7+UewRlaZOkcycQ490pYJTWqm8+bLNowsrxrmgkLSKh1rAkV7M++aLQP08pmKKpAGVoiTAzr7Yxp1xikeW+e7ryBYbPF+bxNqn2uoLd56kE2OxkNQhiUMSoKwcYwTPn0JF3q8/2/0t/Tl6bKAHI3ZHX7GFk8cfsrmx3ltdxs2oMVZZzxY7dQF5dpFtO/QzGmLRXfPtV7BKOMI+g9PPsSRPWotAOzOhsKRrNSc//wgyYn87Xk8+E+8GdBIcn+3auxKWUNmUFTiubS9WrDln8P4pHDNs3/v+cJ5hP+sjsfYp1rZITLl5Y8hQWAV+Z2uItV26jKRFFUkckjLEY+M6PA/MSyTaKHxGc2fZfRJyzsXjXlB+v+ojdRpjbNkWa253a5x3GIvKyfGnur3qKd/5Tz+onr8Fcm6763556ZlJ9Vqs7rr90adfyPCdh4KUGCEpyUlyxy3XSQUW1xfCDSNJG8Z8oVRiYcG/rr05yOXboSAnSBpQTj/leLl+wh1mv6YPuhH75rsfZd99RkrnTh3wNfk0fjrkmEX5qSnJ5tIlsOb58Wd8B/rkrvselhf/95ghfK6+/GK54JwzTB9btWxpXgUWu+f/HnXFTUpSaKcdB5lr3nn1WXz1lKFsHF6b0+SMU080ZA8Jn4P+tq+8/9FnkoR7P+LQg8zGCkhgVcJLE61znNCq6X/Pv+IOYYCQY/Bw9ZBgSoB1LTH9z3//J4889qRce+UlBtfHH77XlE8EwUs9MYVtPPjoE/76atuhldbTkx4093zL9Vca0oj648zMDGO5xetPOuMC087EO++XZ/77sBDT8846Ffd8glThuXrvp7b29Hx1BHyzbPUTmrN5EYiWAKKJn3HJ5iN5MjyEjyN+mJfim3zqehfNQB5x69gBKyYiCF25GZLHET5wAWfi/fjIH0MGgRQqKSmNUINmKwKKgCKgCCgCioAioAgoAoqAIhAlArQOSaihLK1XXn/RpyQGMUM3Zk4YvPubz63S0eVBqSKvPGctVOiezUvmUBH5xcdQUIIMqVHQp7oIlDNGoNSoJoxRsvtI6wqNBJVz6USrmh++FZkzM3AJlanRCK1vnKxeAUXrYKtIz2zjcpEH5fSXH4nQyilUJoMMo/Ka1iNs0xE6dDP1Gfpbm3WQeNoPrZvHdelTuHq89xmKLc+9+4a1UOg3wFoaOXdeJAA/fCtg+RGu7s2Zx75G+2zZr1jvLdq6vXhGe00o7jXhRus6xjwad5i1wHGxaNguYwj9+Xvw1SQHeG4PvB8kY/nOuveW78eff2CbHHxNbUesL9p7Y12MH9O6te0v308voTMP1iq//mAtimpr1+taMbRsrH0KvZ7HNc2XGzYErqDFnpNY26V7tbfx/EbvZ10bEg/GxXEyf67IL8CDZA+ltNjOMyRSvdY70eJf49jCc4wkkdrdluedGm43Egy15dMzUV0kP7/AxHX5O8iJmqScbk+jEBIAxxx5mJxw7BGGHKC1jCMeeHk2SJcHHnlcFixcHFTbn39Nl9NPPs7keePxVNZi7XTPA49KFlyaHjX+UENMkRRx8svkKXLHXf/nDk1KXef5//iX/PvWGwwZw3AcTki03H7XAzJ95myXZVKSQn9OmyFDBg0wrtuS8a64hfLTZ84yFi98/f6ajjkOUlZaJmtAOHWEZQ6F5QV/TubCCunOex+qFhvnyWdelLNh4WMsgqA/JvlFsofy/U+/SuXdD8i/Lr3IEDv0+ORk1eq1cs2NE4Pi6VQ4S15XKCQlIXbpVTfIzTdcJZkZLc0z8j4nWieVlNoFKMTsqutvkZtB/jAWEQk7CnXh737wqZyIZ837V4kNgbgBu4xpdNjyc7Nj65WWrjcCKfhx2gITC4keuniz+3QDR6sf37HP9Vu9G6ulguLiEp91D0kfn7UP05xc2eRLXdCxWqrS04qAIqAIKAKKgCKgCCgCisB2gUB6S+vHvSFvdrv6vww++YWB1hlvJGsjlKxQJtYkUH6YeCtJWJlLRSXduW1p4Ypkki74Hw7/PMFiAf9X16QIjqW/VKJiVbFxfUVeiG6Uoo0DwiDevJa4kvxqKKlPn2LtA13qtIGyjIoyEnfO8imKevYbM0r+dcmFpuRlcJXjjVUQ7vJ3X3/OKPGorLv2ptvCFWnYvHrcW8N2JMbasApfWre1Y4rxcWrS8HGstIBCEvoOoSt76BWE8bJqUd7G2KOai5OAYvt0vca4W3xHSRRvK0JCk7FjCgsapsd8foxlRHdmWPBrLKwcmeNtwT07rwWW93xj7UdqdxuZd+oKyz8vPMdYf/D6g4+EBes2Jv377SA79O4FHjVT1qxdZ+ZbuhKLJK1AsqalNROvK7JIZUPzGUN9YP++0qdPL1m1ajWImTlBJEdoeR6ToBi20xChxQtdpUXTbq+e3U0sIK8Ltu7dulp9KfSkXqEul0RQp44dMNXFy5o162T23PlBFkre8m6fOHCxP4mXcML26A6NpMpUkE3Uz9ZHiMMQuJEjmcQ2+b1IUiec0IBgGGLrsExWtscSM1zhKPMi/Wbu28OSYqxm/tLwWGxNv40j3UckGJTUiYTMdpJPcz9L8liyJwM/zo31D1y9cZ9WQXQBR/dwjSnl5RVgj0n45NjUQ/go8dOYyGvdioAioAgoAoqAIqAIKAJbIwKx/mMXzT1sTf+4RtNfLaMIbI0I1JXUIUdBbxeUv2bMjMo10NZ4/9onRUAR2LoRIJFzwNgxppPU+ZG3omyLpI7tuX4qAjUjEOk3c1MndbAcSWV7RsDrQq0mHBjkykv2mFg/JHxgdu+IH+YlcJVMHSQxMQGMbqbZIl2uxE8kZDRfEVAEFAFFQBFQBBQBRUARUAQUAUVga0aAilUqWCmptO5QUQQUAUWgERCg5x4Xe6YRqtcqFQFFYCtBQEmdreRBbO3doFnghqwss9XU13QEIvWTPB7Cp6XPAojET5oLRFlTRWHOxUr85OTkWfduPhdv1vVbrt9vZZgmNEsRUAQUAUVAEVAEFAFFQBFQBBQBRUARaDAEjjnpLChY4e7NI163O55s3VUEFAFFoN4I3PfQY/LQY08G1VPXeDlBleiBIqAIbFUIKKmzVT2Obb8zjI3DbdXqNRFvhr4m6dqtFQJpZWBrRcLHl9q8DElvROKHgcIcwWPj+vhi/Rj3b3QBZ+P9VNbkwzfi3ekJRUARUAQUAUVAEVAEFAFFQBFQBBSBporAzFlz5cVX3zS3V1NsB3f/SuA4JDRVBBSBzYEACZySGOKEbY4+aRuKgCLQ8AhoTJ2Gx1RrbAAEGpP4ibZ7uXn5JliYI4BszB9LADGYV/amhgnoFW1/tJwioAgoAoqAIqAIKAKKwPaDQCT/4PVBQGPq1Ac9vVYRUAQUAUVAEVAEFAFFYGtDINJv5vatGTYkVXLyimVdVl7Ybm9Nv40j3UfYjiNTLXUiIaP5WxSB8vJy2Qh3b9wiSU3Ej7P8SU9Pi3R5rfktW6QLt27SOWxZxiMiuePfNmVLtuc4Ny/8hBG2Ms1UBBQBRUARUAQUAUVAEVAEFAFFQBFQBBQBRUARUAQUAUVAEag3AiRyIpE59a58K6hASZ2t4CFoF+qGQDTET0JCgnH1xlg+xrUbXb353L2Z1Jdflx4wyGWb1plmC3c9++cnfAzZky1ZsO4xVj44LigsDHeZ5ikCioAioAgoAoqAIqAIKAKKgCKgCCgCioAioAgoAoqAIqAIKAJhEVBSJywsmtlUEKioqACJAjIFW03iiJ5WIHncviV9XLyfFkKCKBahJVH7dm3NFu664pJSWPawb+GtfYrVB2o42DRPEVAEFAFFQBFQBBQBRUARUAQUAUVAEVAEFAFFQBFQBBSB7RYBJXW220evN+5FwMXNWebNDNlPT0sTunXLzMiQ1pmtzJaZmenfT0lODrmi5sPUlGTp1LGD2cKVLCwsMpY9xqWbx8LHWfuUlpaGu0zzFAFFQBFQBBQBRUARUAQUAUVAEVAEFAFFQBFQBBQBRUARUASaKAJK6jTRB6u31fAI5BcUCLeVq1aHrTwtrTkIHpA8rSzhY4kfS/pkggRKjNHSp3nzZsKta+dOYdvLLyj0u3LLQjwf49bNR/5wn+7fVBQBRUARUAQUAUVAEVAEFAFFQBFQBBQBRUARUAQUAUVAEVAEmg4CSuo0nWepd7KFESgAycJt+YqVYXvCuD7GsicM6UMCKFZJB4nErXvXzmEvzcnNk41Z2diyQtJsjecTFjHNVAQUAUVAEVAEFAFFQBFQBBQBRSB2BOjRITUlRehCOycnN/YKNsMVXGTYpk1r09LGjVlSDlflW0q2Bby2BDbNmzWTFi3SpbKqStav37AluqBtKgKKgCKgCGwjCMQN2GVMVWP3NT+35ngmjd2+1q8IbAsIkNjJ9BA+ma0Crt1a4Z+EhpSiomLZiHg+GzcGSJ8NPvJnU05OQzaldSkCioAioAgoAoqAIqAI1AGB9JaZdbiq5kv0/7LI+Ow6fJhZoDVrzrzIhTbjmScevR8LwjLko0++lKefe2kzttxwTV19xT+EuC5dtkKuuGZCzBUfsP8YOffMU811x51yjlRWVsZcx+a4ID4+Xt5+9RlJQkzV2XMX4F5v2hzNxtzG2WecLEeNP8Rcd+b5l8qatetiriPSBf954C5p376tfPPdT/LIpKciFTP52wpe7Gx938NYxzBx7NmjmxQXl8hRJ/69RhzrczLWftWnLb02OgTGHTBWzjrjJHhcqZATTjs3uos8pdLT0+Spxx5ALOh4+b+HH5cff/7Nc1Z3FYGmjUB9fjNvTb+NY70PtdRp2uNa724bQoAu07gtXFy90/zh6yV86OKNLt2Mizfst4QVUCzSrFmqdG3WKaxrt4qKSr91D4meDRs2ylqsElq7br3k5efH0oyWVQQUAUVAEVAEFAFFQBFQBLZ6BI4/5gg5/eTjTD8n3H6PTP79zy3eZyrISRJ06NBui/elrh3o0a2r0PKgQ/u63QNdW/P6rV3G7jvaPCv287U3391qu8t+UtZvyGpQQod1dkSsWMaMjeZZbyt48b7q+x7GMoZpodOjezc2Kz//+rtJG+sjln41Vh+03mAEunTuWK/5jpaCLUDsUFq2iE0/FNwTPVIEFIFtBQEldbaVJ6X93K4R4Ko060YtKywONKUPkDzWwqdN60xp07o1tkxJTU0Je124TK7saN+urdlCzxcWFYHc2SDr1q83JA/JHu5nb1LrnlCs9FgRUAQUAUVAEVAEFAFFoH4ITLj+Shm+81D54OPP5b9PPVe/ymq42inCWCSjZcNayNfQrJ7awggcfuhBQusVWhL94/Jr69wbZ/1C64pfJ/9R53piubBVRoY899QjUl5WHpVFR7++fTC2raL3o0+/iKWpBi+7JfBq8JtohApPPO4oiYuzFT//8uuN0ELTqDLWsd807lrvQhFQBBSB6ggoqVMdE81RBLY5BOgPeT0sariFkzTE3mnrI3gc0ePSllgRM2/oDwAAQABJREFUFK1wpVwvmINz80ppaZmP5FkPksda9awF2bMBvppVFAFFQBFQBBQBRUARUAQUgbog0KNbF+HiJcaRbEx59sXXpLSsTIqwgOmrb75vzKa07q0IgV49utvx5VvdXpeu0WMCxynlp18n16WKOl3To3tX0/eE+ISorj/lhGNMucrKKnnr3Q+juqYxCm0pvBrjXhq6zv32GWWqXIf/6RvSNV5D93NL1xfr2N/S/dX2FQFFQBFoLASU1GksZLVeRWArQqCgoND4CF+6fEW1XiUnJfktegzR04YWPpl+EqjaBWEykpOTpFvXzmbznqYrN5I76+C6zbhwwz7duHFTUQQUAUVAEVAEFAFFQBFQBGpCIC3NupKpqUxt50gKde7cSTrBPRTdCi9dtrxagPgyEDrPgdiJRtoi0Hyvnt1l8ZJltS5gojudjh3ayxzEWSkoLKyx+s6dOsLdcgZissyvd+wYum4e2L8vXCcXyLKQ3/8pcNEzeGB/0/fQc6EdpCst/safPWd+rf3ntYzp0B8WISxPC//ahP3su0NvQ07Mnbeg2nOp7fpI56N55ry2fbs2kaqIOv8kWFc4ef6l6tYVdKlFTJYsXV7reHH1MCUuxIe4hJOuXTqHyw6bx3qG7TTEnJs7f4FwvEcSjm8qzWfMmislJSVBxdhmB7gFjPb5Bl3sO6gNL3cN3YUPGtBPFi5aKqExX2mp0XeHXlG9g7GO4Vjfw4Yaw/377QCXWXax5YewTKyPZCAeb98+vWQOxk4+5oBohVhxnpw5e26NY4T1JUGHwOeTm5dnnkM0bUT7LIgpCVe+OzNmzq42L8Qy9qPpF8t0h6tI4s+Yat7YXexL/747GAuqefMXVuuLt/5Y33XzXg4dYuboaBfD8tn2w9ywBNaF67GYVkURUAS2bwSU1Nm+n7/evSJgViWuXrtWuIUT54qtfbt20qGtdcvGvBT4TK5N6MqtM34YcgsVunEj4WNIHpeC7PH+iAq9Ro8VAUVAEVAEFAFFQBFQBJo+AndOvNEQJ84t2v6IBbKvbxX75D+myq133CtHHDYOQaVPlrVr18t9Dz4mV1xygSFQ4uC/aPyxp8rfxo6R00853q8odahVVYl8+8NPcvf9j7gs2Xuv3eWqK/5hjs++4DL/AqQXn5lkrn/08adl5512lD13H2EICHfhylVr5JyLLneHJqWi7qrLL5ZRe+4OpbzPlxLOUGk34ba7qylAzz/7dNl/v9FBsRSmz5wj0VpgMMbms08+IlW4MeJy4bl/Nzi4TpXAov7yq2+UHYcMElpreK2eSPpc8M8rTVxPV579v+Sic2XsmL2D+k/XYg88+l/57oefXVF/ut+YUXLmaSeZeJ8ukwu6mqWmusOglEr5O269HhYuXf3uplhgxqw5cvNt90RFCAVV6DtgoPFonjldru2L+2vlc7XXAf/bvP/mC6aWnJxcOeXMC8NVHzbPjUsXg9QVYl/OPes0ScHiNycVcKlNV4KPP/msyzLpxJuuMaTLb5On4H+sFDPWnBsujtfPv/pWHnjkcVN25J67yXmotxUIQArLub5zDJx61kXCe/DKQX/bzz9uX37tLf8p1+6vv/0hObm5GIf7+GMDsdDPyJ+Evl556YVQ4PcPGg9vvvOBPPXsS/66ot0Jh5d7l+l14m1YEZ160nFB45SWKxdfdo0ZY2P33Tuoj3OhaL/sqhuDmq/LGI71PWzoMXzqiceae6Al1Tvvf2z2OQ8OHtRfKuCV4+LLrpUVK1eZfHrPoOs9jpVZs+fJ1TfcavJJBN5+83VB2GVjLEz58y8ZM3qk8epx5nmXmLLej5OPP1qOOeqwIFxJWl953S3V3kW68aNbzEyQC044RmfNmSs3TbwL1o7FLtuksT6L8zAfHjbugKCxlpuXLxxvq1aviXnsB3XGd+Dm9WdfeBXuPXeSoZgbve/bpCefMSTsZf88Xzp64oHRO8oddz8gv+C98Eos7zqvIyl103VXSJdOnfztFhUXm+8yb73efb4jZ5x6gnBBrpPiklJ56D9PyDff/eiyNFUEFIHtDAEldbazB663qwjEigDdqXETmRN0Kf2Nk9zhP0GW+GlnUrfCKKhwmAOu8uKGpYJBZ/lj3v1T5CV8ysvLg8rpgSKgCCgCioAioAgoAopA00SAikMGXfdKAsgGCpWplFQQBszjyvJ77pgQpASMQ/7eo/bwEzpl+B3J35IkGai8G7P3XmYV+3sffGLqSoLi3dWfAMseJyzP/H9ecLbJovKS5IaLV0lLnNGj9gwiOh576G7p5rOioIK2tLTUlKcFxL3/vlmOPfls/yImxhY5/JAD/XUXFRcZRe2Ogwe4LtSaUmnq+n7rjVeb8iRyEhMTTD5JhUf/705/PVQEpiQnGxxImh139HijuHcFHr7vDqElEoX9Ly4pNoQT7/kaEF/8/f76W++74tK7Vw+5/B8X+PGncjIxMdH8j+Av5NmhQvqZJx7yKyfp9o73QOuaIYMGyMQJ18gV10zwXBH9brTPnOPLq5RmCw7D5s2jd/VHSwVHPHqtK846/SQ5+ohDTcc5ZvhcU1PsWBqP500rhGtunOi/MVqksX2ShhSSP6V4TryG4/WAsfuYMXTnvQ8Zwo5jySuu78xjsPTQaKdHHD7OFOez+X3KX/5LXbt77bGryePzZhlHxu2523DhRmGfikvK/O/leCiZYyV1IuHl3mUq0C845wzTHt8zkha8//Zt28hrzz9h8i2egT7SCorPk1YUTmIdw7G+hw09hjn+h+44yHR/9tx5fiuZ/3t4kjz12APmXeF78fdz/2nK3HjtFead5PN6EEQrhRaBD9x9m8GLx3zPSXBznJOgpbDf4eTE444EKWznNjcXcg7476P3BRGc7OO/b7kh0AaeUQLmmSS877T+e/aJR+SE0871z29sK5ZnsevwYcL3g0IChaQz3y/qF2hpxsWfsY59U1nIh5vXzzztRHOGbdHDCOdKjjc3BnmS3x3EkfMTNxIrXlIn1nedFk733XmLf97gXF2F+2Kfeoa4uHfdJlF/6Li/uUPJhxcWkvP8jrzy0ovMQgFaoKooAorA9oeAkjrb3zPXO1YEGgQBrubiNn/hoqD6+IMkYN3jI3zatpN2bYP/+Qi6yHPQDj/auQ3BD0OvbMzKDrLqWQOrnpWrVptVid5yuq8IKAKKgCKgCCgCioAisG0jcNQJZ5gboAUCFdZffP2d3P/QpLA3RSUcFW+ffPyVTIernmIopele6i2s7P74ky/kz2kz/C6IqBCk4pHXjB65hzhSJ2zFnkwq3l5A4HKuFqfQVc8kkDeU8Yce5Cd1aLHiCB3GLXnm+VeMcpKWQNdeeYlR3JHE4Up8WticdfrJpg7+zr3gkqv8/TwWq+bPOOVEv/LUFIriY9GSpXLnvQ+bFf1UFE96+B7pCtdzlJ9//V0eeuxJY8VB5e4LT//HEAWj0DdaY1BIUDlCZ+r0mXLDzf82ilS6TXr4/n8bReIpsCh478PP/K65aBlAiyQ+A67sd8r1EbvsJNdddZmfBDAN4IMWVVxtTiXyHfc8ID/+/Jshda687CLZB+3TbRxdDIVam7jra0qjfeZXXW8tG5549H4hMccFZU5hXlP9oedOPuFok0Xl+rs+gpB9P/LwQ0w+rSQuuuRq4z6MCuH7754oO/TuCcuAgUaJP236rKAqWc+Tz7zgt9RgXf958G6jmN97rz1kUsazZgxyHF58/lly8IFjDY6HHHVSUD3eA5Kgbgx8/+Ov3lP+fbb7zAuvmFg7VJyTXLv79pvMeZIrD8Ia4NvvfzLHp518nJxwzBFGwR1KaPorjLATDq/Qop99+a088fTzxt0f3VlxnJI04Hh55Y135OVX3zTvVI/u3eSxB+8ylx9z5GHGgoIHsY7huryHDT2GaenB8UF5+bW3TcoPLnDkcyH5wIWUtObh/947+QigZ1981VivsCz7xHmNON11/8P+OYlE2q03XR2R0OG1n3/1nTwy6Skzb3Le4DvNNogNLR4///IbFpOrLrvYtEES5PoJd5j5lvm00jvmyEPN/ECLp//893/MjvlZnHvmaea6nNw8OfH088w+PxiHKR3EJy11Yhn7/goi7JA0uvfB/8jk3/80Jdw7xYNVq9fKnfc9JAsWLjbz0wP33Gbe3W5duhiyka4J6/KuX3fVpX5CZ9JTz/m/g0jK0XrRaxnEfvD9PfjA/bkrtEq7+fZ7zNzIOfmJ/9xvxg0xv/am20wZ/VAEFIHtCwG73Gn7ume9W0VAEWhEBLi6a+nyFTJ5ylT58NMv5H/4IXrXAw/LVTdNlHsfekyef+UN+Rz/mE+He4Vofceyu4zzwx+l++49Uk44+gi59IJz5J6JN8nlF50nxx81XkbusZv0xI97rn5RUQQUAUVAEVAEFAFFQBHYfhA4/tRzDTFBguCPP6eZG6dFwvc//eonSpjJ356r16w1553Fjzmo5YNxHRyhw6KMRZO9ydpDULnmxCnzaX3y5DMv+uMvsB9UjlNoVUDZbddd/KTNxDvvD+onLWHKK2K3Uv/w4y/8LpqonP8B7TqhCzlHlDDezbwFdmEWFedOqBynUGl70y13+lfcU7nsLAKoYKflCIWWFBlQuFLoVswROjwm/mt8WPPYyXCQPRQSUHxeFPaVBJgT/uavizTkM6+tfWtdMdgU88YgOfyQgwzJxRO347m6eDDE9Nobb8O9QusOOR7ESKgw7pJzvcVzfF73Q+lMocL+wL/ta/Zj+XBECq95EYRIOGG7b7z9vv950w0exzBl8dJlfkKHx97+MbZHtBIJr9Dr6U7KxZ/Kg9utZctXmiJUoj//0mv+d4qWCS5ukyOtWDDWMVyX97ChxzBda1F4P1Om2vnLZOCDz4XxmCgcM9f8y1rr0D3a62+9Z/L5wZg8FMbR8bpIZIwYd70pEOaD77aLs8R3kW7U3Dil6z4KYw2R5KF8/OmXfkKHx08/95KQmKbsAytIJ7E+i/wCG/8nKSkxyCInFyQPCZ2GFs5XjtBh3W+/96G/Cbo8JKFDISZfffO92ed7SCKYUpd3naQ1hTFxvIsK1qxdJz/55kNTwPcx/rDAfPJ/Dz/un8M5J89fYPvn+uO9TvcVAUVg+0BALXW2j+esd6kIbHEE+GNozbp1Zvtrxkx/f+iegStTOmHr2B4bU2zuH0R/wQg7/IHJbdddhvlL0F0cfZyvXL1aVsCaZyV+BBZFEazVX4HuKAKKgCKgCCgCioAioAhsMwg4hWRoh0k6UClJ6xO6X6MitKGECvdMX1wTV2drLEKiMAaGc4XmzrnU/caluyIKlf1eMsSVa4h040araA1XFy3uQ6WNz60XFajsl1dIwNAKgErN3r16mlO0OHHy9bc/uN2IKRX7LiZEGtwHhWLk6m+daXGMWFENJxrzmXubPeSg/f0u27xxapylE/GjQt0rJCuysrONwpr/+0QjJCkdLrQQi1Wckp1K41gCq5fAfZd7Vt42qWB3whg+0UokvGq7PjfMOHXXkCgNdSkW6xiO9T1s6DFMosQp5SNZUl074XZ5/qlHjVVGcnySsYpjnhMuanSWPtNnBlt/uTKxpJxPN2RlGbd3zt3ZgP6WNGI9tJwMFVpJ0q1lmsd9YazP4rMvvhG60+MzpUXltBmzjIUkSdNo5ZwzT5UB/Sxx4r1mLfQQ3jhq3nNun/HZIslGLAhw4sZ9Xd71FumWRJ8KC9JohBZpFL5qjAXmlTZt7DzJOU9FEVAEtk8ElNTZPp+73rUisNUgwH+wGfTRBX50HeOPOUfwGNLHR/g0a5bqikRMnfu3neF710lW9ibjro0ED922kfBh0EUVRUARUAQUAUVAEVAEFIGmh8CwoUOM2yGn7Nwcd5jWvJlphu6I6X4snLgV7U5Bz5g7W4u4/ufkBBT33r7ReoiWOh06WOskr/KUpEFtwoVYTuhmKNTVkDtHBWxdZHM+c67Sp9C6wqugpRtpSqTnmg+XT1SUp/uUu6ZwLR8O98zMjFpKBp/eEbFLGXuD8t6Hnwaf3MxHkfBq6G7EOoZjfQ8begwHWVLBo0U4IYFMK7Q9YN1HWbhoqXjJNTfmeI7jqyGkEHFbMFDF/e/d1RcnjHU7a0dvO470oCtGkgy0rIr1WXz82ZdCsvfvp1rXk3QBt9OOE2Th4qVywy3/9lupeNsN3d99xC5Y8NkhNFv69O5RLa++GQ73aN916jRIilM2II5wNOLa4HWRvlOcxVo09WkZRUARaFoIKKnTtJ6n3o0i0GQQ4I8TumXg5pWMli2tVQ9+FPFHdRdsXrcX3rLefa6C4sZ/bpxswg9kus+guzi3uZU3roymioAioAgoAoqAIqAIKALbHgL/uvRCs3qdsV4Y2Jqrvfmbcf99R/tjGjT0XdFygNYNjH3g4kqEtkG3OZR16zZI3z69JDk5ObTIFjsuKio2/Wdw8nCSmGDVB+vXW4Ukfz876dCuXa0KZXfvvIbu7L778Rd3eVBam7uooMKeg831zEnKOOuK734IvoeNsHDo06tHRJfQzX0ki3M15el+xF2HO8dMLHLS8TbmTwU8Jnzw0WexXNqgZWvCq0EbQmWxjuFY38OGHsPOkooxXCK5JqdFCMkKJwP69ZE9dx9h4mQxj5Z1dJdGQqVTx+gswFxdkdLQcbp6TYBo5QLKUPIo0+eajRYlJHQosT4LXkN3cx8hFtpxx4w3VpYt4R6S7xNj2kQT9+ojuH/v07sXqwqScJaJQQXqcBDru05vIk6cFZM7jpRmY2Gq4P7p4v4auG8MJ16CL9x5zVMEFIGmi4CSOk332eqdKQJNEgH+IOM2Z/4C//1xBWZnBIElwcOtcye7n5AQ7y8TbqcVgo+2yhiEYKWD/KeXrVjpJ3hI+NDCR0URUAQUAUVAEVAEFAFFYMshEOeWN0fZheE7D/XHf3gVwdVfevUt/5UDEHvCxTXwZzbQThbiSlAJydXV830xayJVPX3WbBm5566GeOqL2CS1lY9UT0PmU0lJ93Cd8HuabqboPtnJTjsO9q8yX+xbdPUHrAec7LvPSKyoX+IOw6Z060SSzVr7tG/Qe96cz9yRJbzJl0Li1CxFrIzdhu9s7pGukxj7xQndZDFOKKUmV0+uPFOuznfDfwni20Qr/P9oyOABpviMmXOqudOLtp6GKFcTXg1Rv7eOWMdwrO9hQ45hvlPOfdz7EUg3voe33XydGQPZWJDI50rS9arL/yEnnXGeIU54/zl5eZKJ/21H7bWHPPzYU15IcK3PPCQoN/IBLW3atm5tCrhxOhexepyM2mt3WQTrGa+4OFhFxUX+7FifhbuQizsZY4vbJRedKwfuP0Y6gEiiazcXb8mVDU3fejcQFyf0XEMfx/qucz7NhwUUred2hiVpNEJ9BOeA1JRUWQX38rXdfzR1ahlFQBFoOgjUrPFsOvepd6IIKAJNGAH6rOYPnp9/+13eePcDeWjSE3L1hIly78OPyctvvG1WAS5YtMT/o7cmKLp37SJ777m7nHLc0XLdFZfIhKuvkDNOOl723Xuk9O7ZQxLwQ1pFEVAEFAFFQBFQBBQBRaDxEXArvgf27xdTY14lZkJ84LcbyZMe3WOPSxJt49/+8LMpSutwxnYIlX6IF5Hus4L5FdZDTm669gppATLICQOiO+sMl7c50k8//9o0k5KcJJf94zx/k1SmXnnZReaYFgFf+oKGU8HIFeSUww45EGRZ4DntOmJnQw6Zk56PBQuXmKORe+wmLOMVkh5D4XLJiTdWEl0X1SR1eebOMoKWJGw7Whk9ag9TlDE8XR3uWloZuFAzN15zeVC9PE6Akp7ywcfVLWeI81HjD3FVYfFZhlxxyYXmmNY2H3/2lf/c8hWrzD719V7MXIFDDz7A39aLr7zhshstray0MZhaghQMlZrwCi1b3+NYx3Bd3sOGGsMnHneUuV0+2/c/Cu8e758XnmPIGha8+ba7ZeK/7zPX8B29+fqrzD4/fvzpV7NPwueJR++X/ffbx1gl3nPHhFpJ7EsvPtc/TkkiMdYVrX4or7/1nkn5v3aez7XbkYcfIl53bIdhrLn387ff/zTl+RHrs6ArvND4MPn5AXfp7h2tbez7O9DIO3V51x3J27NHNxl/6Dh/D2n5t9eeu/mP3c7nX31ndvme33nbjdXwIe7OarAUpLkTzmkqioAi0PQRUEudpv+M9Q4Vge0WAfr25vbH1Gl+DEwwSljyONdt3UDiRHIxwYv4D/aQQQPM5ipZgZg8xjUcVigtxoo5XTHjkNFUEVAEFAFFQBFQBBSBhkNg8h9TZZ9Re5oYCR+8+aJUVlUaV0Pn/+PKGhuZDssE547ohGOPlLH7jTZKS65kb0yhVdCRhx9srHWOPGycjDtgrNB6h3EiWrZoYVbb/w+rz6kopRunt9//WFiO1huvPPtfECRFRmnnFP+N2ddwddNa4JgjD4elUWsZO2Zv2WuPXaUAK8vpWsn16d0PP5E8T1zK2+96QCbedI2xTLnvzpsNyRMPIo1K53By9/0Py9OTHjRY3HL9lUZRTPdBjBfjrBZOOuMC2ZSTI19/+4OcdfpJphoqqunu7cJLAopsb/11eeYkVhi3g/f2zqvPwoqozFg1jD/2NG/VQfuM2+P6+R6wCBU+12+++1FoucTYHm++/LRs2pRjYuikpiSb4ktgzfPjz7+FXmoU6QyGfsapJ5jFaOlpaQYnFnznvY/8bq14TDdT54I4NMreW28QKnSTEpPkqutvMa4GDz/UxvyhZcCMWXN4SaMKiY6hQwYal4LvvfG8TLzzfpkMBX9teDV0p2Idw3V5DxtiDBtLKvyPSZk2Y1aQVZzDhP+DHjB2H3P49bc/+i3bvgeBszesZXaEJdZ+Y0bJV9/8YNw9DujfV3bo3dMo+S/3kLKuvkgpXVKOHTMaViQF0hxxwdy7TjeIUzz/Rz/y2JNy7ZWXmHf78YfvNe9oYmKi/39pjsEHH33C30ysz4Iu1viOFJeUmnmT8XwyW9k4UrPmzDPtsfLaxr6/A428U5d3ne/FM/992NzneWedat71KpB6qakpYXtLEojxlGitQzd0b7z0lCGS+YzcvDx77ny54poJJs4SrZw4P5143JFy1BGHypHHnx62Xs1UBBSBpoGAWuo0jeeod6EIKAJRIkB3ajSz//TLr+XpF16WW+68V26/90F5HivYuLpy8ZJlUl5uV5pFqrIrXL2Nhmn7GScfL7dcd6VcdtF5Mv7gA2Uo4vVkNLKyIFKfNF8RUAQUAUVAEVAEFIGmhsALL78uVIDT8oErx2l1k52dY26zoqI84u3SwufB/zwhXAFPpXd7uEMjobMGCncqyCjeOIoVFQE3YxWwAI9FvNfyunMuulzmzFtoqqCCkop9ujRjP6hwWwjrcSdPPP28fP3dT8YlGc9TGRcfFy9//DnNLExiudp+l7q6aktp2R6NnHvxFfLX9FmmaLPUVMRKb22UvCTJnn3xNWGfvUKl7+NPP2fujfm8hoQOY4T8MnmKKerFiIrQS6+6QehKisLFVVxp7ogSWr6UlNqYHPzd/vOvvxuCjvhEUnyynlifOa8hsfLntBl+/BkPiXFAahKvdcWHsMoJJ/c88KiJGcRxS+U9MXSEDjH55+XXhrvMKLP5v4hzscV7Ju4vwsXbU8++FHQNrZgYl4hjisK+l+OdKCktlXZwVdUR8aMo337/k0nDfTjrmnDnXF607wPf1RyQcxT2nxslGrxqepdNJfggoRutxDqGY30PG2IMk3Rz1jD/z955gDlRfW38JLsLKL33Lr2KKBZABOyKYu+KHcWC3c+Cvf9VsKBiQwULNhQVRYoICKJI77333tnynffO3uwkm2Qn2V02C+95niQzd+7c8ps7N8k9c84Z/MU3Ybv2mFrwQXCN/9evfyDPy6++GbjucE8G6xoIFvY/0/+0i/TBQyjzoJTpP+BjgccKiPtaWuawvsEcBMG9aBU6U2fMkl4h4xTKpGdfel3HGJSfYhQu9uFI3O839uwdpHhEmbFci5mz5gjiktl50yp01m/cZJSEKA8Sbew7OXL/nqpuIr1IrPc6FNhQvFoX75grMa/hGn8+5HtTpbX0s/U/8ezLaqU30nwP4vrAFZ2dlzGvuxVveLAA1wcCN5cUEiCBQ5uAr3GbTvpTI39l5/Yt+VsBSycBEiCBPCYASx64YrMva1LupZpVGqwSFjzLV6yS5StXZnPL4KUM5iEBEiABEiABEji8CZQo5cTeyEsKhfV/Gdzu4LeZdV3jlQme9G7ZvJlZNMPCl9vCxGsZ8eaDkqJpk4bq7q2mbFJrHSzWR2s/XMMV01gWs+bMC/vUfrztiPc8y65SxfIyX+MDIeaPO8ZOuHKhTIC74ukzZ+WoHMH5sKBvrg9FYYESC+WI2xHqzgz5cP3hmgzWOzmJbTcWSmO55ghGv3LVGrNgHKkOtOO7Lz42i/H/TZshjzzxfKSsJh2L7YjfVL9+XY2HsUZgTWRdCrpPfPXFp6Vxw/rGaunSq28y1lqwwoCSxEusJSjFkNcGr7+71y0BCw9r9eSuLz+3cS2xYI2xEiuvvG6XHQuxjOFY78N4x/CH7/Y1ijcoVXDN81MGfdTfKGDmLVgkvR94LGxVGKvoO+K9TJs+K8cYTHCV1qxJI6OMmKqK0W2ZStqwhWtiLNcCnjJOaNfWKKvmzF0Qdd4MHfuR6s/vdK/3ursdYAJLNsx7VsnjPh66DUVpg6PqSyOdK/bqgwsrdK0BVjrh5mXMxRs2bAwtgvskcMgSyM1v5kT6bRxrP6jUOWSHNDtGAiSQlwTw9BsUPDVrQtFTw2zjqUsvsmbteqPcWax/5hcsXmxMo72cxzwkQAIkQAIkQAKHL4FY/9h5IZVIf1y9tJd5SCCRCFzU/Vy5/prLTZPufegJXVCdnyfNC1Xq5LZQuHyDxdSKVavlll735ba4uM/PL15xNyhBTqysVlQfqVIHAhdlsKbJL2nTuqU80+chU/wPP/0q77w/ML+qYrkkQAIkUGAEcvObOZF+G8faD9rjFdiQY8UkQAKFiQB8BC9cstS8bLvxZNZR9epKA33h0x3g1ubBZ9UqlcyrXds2xoXGgkX6tCNei5fImrXr3Fm5TQIkQAIkQAIkQAIkQAIkkIAEzji1s2kVrCvySqGT191sqTGCoNCBDP1xeF4XH1N5hYFXTB3Ko8yXXnR+oKTBX30b2M7NxskdTpT7777dWLMtX7lK1upDhTVrVJOmjRuZYuGSK9SFX27q47kkQAIkQAIFT4CWOgV/DdgCEiCBQ4QA3ILUrV0roOSJ5vfbdnmZmk1DwbMQL1UaUUiABEiABEiABEgABGJ9Ws8LtUR6GtFLe5mHBBKJANyJwc1Smsbb8BqjyEv789JSB+1DOyHhXL15aU9e5ckvXnnVvoIqx14juM1CfJi8ELgre+yhe8IWhXgtTz3/P5meGSsrbCYmkgAJkEAhJpCb38yJ9Ns41n5QqVOIBy2bTgIkkLgE8GO9nip46qpfcSh66tetI0lJThDLSK2GD/EFi5ZkWvEs9uSLPFJZTCcBEiABEiABEijcBGL9Y+elt4n0x9VLe5mHBA4HAnCR1aRxA1m/YZOMGDnmcOgy+5gPBEqXLiXnnXOGieNVRrdhrTNv4SIZpi7ewsVdyYcmsEgSIAESKBACufnNnEi/jWPtB5U6BTLcWCkJkMDhRqBE8eJGuYPgsXDVBpds0WT37j3mR/i8BQsFQS137NwZLTuPkQAJkAAJkAAJHGIEYv1j56X7ifTH1Ut7mYcESIAESIAESIAESIAEohHIzW/mRPptHGs/qNSJNip4jARIgATyiUCtGtWNcgcKHrz8fl/Emvbt228UO/MWQsGzUH0lb4+YlwdIgARIgARIgAQODQKx/rHz0utE+uPqpb3MQwIkQAIkQAIkQAIkQALRCOTmN3Mi/TaOtR9U6kQbFTxGAiRAAgeBQNkypQMKngaq4ClVqmTEWlNT04xiB9Y7UPJs2rwlYl4eIAESIAESIAESKLwEYv1j56WnifTH1Ut7mYcESIAESIAESIAESIAEohHIzW/mRPptHGs/qNSJNip4jARIgAQOMoHk5GSXgqeOVK9WNWILMjIyZO78hfLf9Bn6minYp5AACZAACZAACRwaBGL9Y+el14n0x9VLe5mHBEiABEiABEiABEiABKIRyM1v5kT6bRxrP6jUiTYqeIwESIAECphAnVo1pWnjRtKscUOpXKlixNasXbchoNzZvIXWOxFB8QAJkAAJkAAJFBICsf6x89KtRPrj6qW9zEMCJEACJEACJEACJEAC0Qjk5jdzIv02jrUfVOpEGxU8RgIkQAIJRKBJowaZCp5GUqpkibAt27//QEC5s3DxkrB5mEgCJEACJEACJJD4BGL9Y+elR4n0x9VLe5mHBEiABEiABEiABEiABKIRyM1v5kT6bRxrP6jUiTYqeIwESIAEEpBAsaJFA9Y7sOJJSUkO28r5CxcbBc/kKVPDHmciCZAACZAACZBA4hKI9Y+dl54k0h9XL+1lHhIgARIgARIgARIgARKIRiA3v5kT6bdxrP1IqlC1zhPRwOTFsf379uZFMSyDBEiABEhACaSmpcmadetk+qzZMmXadNm0eYsUK1pMypYpHcSnfLmy0rxJY43RU0/27dsn6zZsDDrOHRIgARIgARIggcQlUKToEXneOP4vy3OkLJAESIAESIAESIAESKAACeTmN3Mi/TaOtR9U6hTgoGPVJEACJJBbAnv37pMVK1fJ5Cn/yao1a8Xv90uVkNg7UPa0atFMqlSuLNt37JCt27bltlqeTwIkQAIkQAIkkM8EYv1j56U5ifTH1Ut7mYcESIAESIAESIAESIAEohHIzW/mRPptHGs/qNSJNip4jARIgAQKEYENGzfJ9JmzZc68BXLgQKpUKF9OiqSkBHpQWZU9xx1ztJQoXkI2bt4su3fvCRzjBgmQAAmQAAmQQGIRiPWPnZfWJ9IfVy/tZR4SIAESIAESIAESIAESiEYgN7+ZE+m3caz9oFIn2qjgMRIgARIohARgjTNvwUKZMnWG7Ni1S0qWKK6vEoGe1KxRTY5p1VJj8aTIoiVLA+ncIAESIAESIAESSBwCsf6x89LyRPrj6qW9zEMCJEACJEACJEACJEAC0Qjk5jdzIv02jrUfVOpEGxU8RgIkQAKFmMC+/ftl6fIV8tff/8h+tdypXq1KwHInJSVZ6tetY15btm6TLVu3FuKesukkQAIkQAIkcOgRiPWPnRcCifTH1Ut7mYcESIAESIAESIAESIAEohHIzW/mRPptHGs/qNSJNip4jARIgAQOEQJQ7syaM98odapXqxroVbmyZeTYNq1130ernQAVbpAACZAACZBAwROI9Y+dlxYn0h9XL+1lHhIgARIgARIgARIgARKIRiA3v5kT6bdxrP2gUifaqOAxEiABEjiECOzevVtmzZ0nq9aslTKlS0vZMqUDvaPVTgAFN0iABEiABEggIQjE+sfOS6MT6Y+rl/YyDwmQAAmQAAmQAAmQAAlEI5Cb38yJ9Ns41n74GrfplBENTF4c27l9S14UwzJIgARIgATykECnDifJWad2Eb/flyel/jbqD/lt1Jg8KYuFkAAJkAAJkMDhTqBEqbJ5joD/y/IcKQskARIgARIgARIgARIoQAK5+c2cSL+NY+2HvwCZs2oSIAESIIECJDDmz/Gybv2GAmwBqyYBEiABEiABEiABEiABEiABEiABEiABEiABEoiFAJU6sdBiXhIgARI4xAh8/9MvedKjRUuW0konT0iyEBIgARIgARIgARIgARIgARIgARIgARIgARKITIBKnchseIQESIAEDnkCUMb0/2BgrvuZF2XkuhEsgARIgARIgARIgARIgARIgARIgARIgARIgAQOcQJU6hziF5jdIwESIIGcCDhWNn/klC3iccTSoZAACZAACZAACZAACZAACZAACZAACZAACZAACeQ/ASp18p8xayABEiCBhCfw26gxAuVOrAKFDs6lkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJ5D8BKnXynzFrIAESIIFCQQAu1GJR7FChUyguKxtJAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRwCBGgUucQupjsCgmQAAnkloBXV2qOy7Yxua2O55MACZAACZAACZAACZAACZAACZAACZAACZAACcRAgEqdGGAxKwmQAAkc6gSgrIHFTk7iJU9OZfA4CZAACZAACZAACZAACZAACZAACZAACZAACZBAbASo1ImNF3OTAAmQwCFPwLHC+SNiP71a80QsgAdIgARIgARIgARIgARIgARIgARIgARIgARIgATiIkClTlzYeBIJkAAJHNoEfhs1RtauW5+tk4yjkw0JE0iABEiABEiABEiABEiABEiABEiABEiABEjgoBGgUuegoWZFJEACJFC4CHzyxZCgBs+Zu0Cg7KGQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAkUDAEqdQqGO2slARIggYQnsH7DxkB8Hbhk++CzwQnfZjaQBEiABEiABEiABEiABEiABEiABEiABEiABA5lAr7GbTpl5HcHd27fkt9VsHwSIAESIAESIAESIAESIAESOGQIlChVNs/7wv9leY6UBZIACZAACZAACZAACRQggdz8Zk6k38ax9oOWOgU46Fg1CZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACXglQKWOV1LMRwIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIFSIBKnQKEz6pJgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIwCsBKnW8kmI+EiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABEihAAlTqFCB8Vk0CJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACXglQqeOVFPORAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQQAESSC7Aulk1CZAACZAACZAACZAACZAACZAACZAACRQqApUrVTTt3bBxk6Snp0dte+nSpaRY0aJBeXbv2SM7duwMSuMOCZAACeQFgeJHHiklShTPVtS69RuypYUmVKxYQfw+X1Dy1m3bZd++fUFpeb2TkpIi5cqWyVbs5i1b5cCBA9nSmUACJCBCpQ5HAQmQAAmQAAmQAAmQAAmQAAmQwKFJQBe3ZK8uRqWneetf0WJO3kRaREpOEUnSv+779njrQ7EjRPbv995nXUyTjAyR1FRv5WPBD5z2emyPt1KDc8XapuCzve35k0RSwDW2xcqO7U+Qh+69w9TR+8HHZd78hVHrGzjgDSmC/rhk7vxFcs+Dj7lSctgsUsS5Rl7HZTx9i3XcJIGf9mvv3hwaH8PhYjqucD0wHr1IrPeGlzJjZe2lzNzmKag2xVov5oYjdP5RpaXnaxiOTX6MrXD1hKYl8LwT2tRo+7163iAn6zwVKmd1vyI0KWi/RvVq8t6brwSlYeezL76WwV9+my091gQobkqXKikbN23OdupxbY+WRx64O1v6K33fllFjxmVLjzcBCnkvyvh4y0+085L1XurU8SSZNnO2bNiwMdGax/bkkgCVOrkEyNNJgARIgARIgARIgARIgARIgATykQAW2pq3EmnYVASLhpMniCxaELnCGjVFjjlepKJaU0AZAoGSY+E8kT9HO/vu9xIlRDqcIlK5iiordEESgkXlFctE/hgpsnuXk+Z+RzvqNRRp3UYXtnXBfdlikb/+dOeIvF2ipMjZ52lfdFH830kiC+Zmz4t2n9BBpHZtkRKlso5vWK/1jBVZszorDVu16ogcq4t45Ss4jJC2c7vIuD+0bUuwFyx+9cR+3Ilafj2RMplPR+9Sy5G5c7RNE8MvyFapLnKitqmCckX/01QJtGaNyNjfRc1Ogst374HvMe1EqtZwUod957TNnQfb8bQptIw62p/jtY2Q4UNFtm51tt3vzVrqeGqd1e89u0WWL9WxMcbpkztvmO3kZL1uMYgPrDJlf6ZSZv2GnJ+YlzJllXdHkSrVHOUJysD5U/4WmTYl/DWKtW+xjhtYHLU7SQScj1CFKQT3CsbwBB3/+/Y6abG8H6kWBSd30fGh48suqm/SxcdJ40VWLs9eEhb8Wx2jL733oHSAoA1LcQ+OjT4WndzZ32Nljfmieq3s5diU34aJbN4kUk3HfEftm1dB++39GmubbB2xzpf2PPsZT731G+h9p+PCPVdt1DE+Qeef0LnK1hP6Ge/Yys1cjDYk2rzT8miRpjpH5SRRvnO2qWWNnWuS9bvE78+ag6IVm4R7K1PS0zMkFXO8yoaNmzNTs3/UrVNLXnzmcVPHDz/9Kp8M+ipbppuuv1o6dThRypQubb46UtPSZOXK1fJ/fZ6Trdu2mfx79uwNtNmv1wTKiEgCa6L+fV+UNWvXyx33PGyyQXHR69brZdLfU+Tl198KOrVWzRry+MP3SuXKFSVJy8Z0sV2/sz4dPER+/lW/vyJIKVVA3XrjtQKFE+Tr74bJF0P0+yuCoN2XX9JdzjnzNJ3KkmXegkXyiPaxIOXFZx+XJo0aqEVphnS/7DpaPRXkxciHuqnUyQeoLJIESIAESIAESIAESIAESIAESCCXBGA50FoXb7HI5VoYl5KlIxfctIWjoHHnwAoOFn9xLFSpA0XOuRdkKX9wHvKjvlp1RIqrQsKt1IEFRJNmqqQ4LmtRG+dUrIT3nAXlnn6OKhTKOXlhSRQqSEObbB4cx+IaFD2oB4vf7oXSrmeKYFE1VLDAesa5Ir/qAjMWvK2gDWeqUgnKL4jtL/p6zLGOcuvn7510J4eW31Ck6xl2z/lEe1DGpVeLfP+1yMb1wcfLah+haKpbPzgd1hhQOLklnja5z8c2uKGNaBfEfjp7zvtJJzsKQncalBONmjr9Rj/iUUy4y4uwPW3GbHn48WciHA1JDjeOkQUL9lCq4NqOGx18Uqx9i3XcYOydd1F2rrh2DZuIVKoqMuRTtRDT+8erYIycf0mWcgbnoTwoDs8+31Gqzp0VXNoZ3YLHLhRduL8xzmrVFhn8sd6zu4PPibYXD+vK2lddoI4omLsgUFREy+fkynq358XTpnjmy6yana146oWCDQodK1CS4p7CNeym4+UnnUvCKedsfnzGM7ZyOxej3kScd0rqve1lzED5FkHeeX+g4AW587ab5IxTT4mQM3LygI8+k6HDfomcQY9AifHck49IieJ6vVUqlNf7OUQeuKeXUei4k6GwqVO7pnz0Xj+5675HZPmKlTJl6nSdCq412Vq2aCovPPWo+5Sg7VbNm8qRahHmdmvZpnULk+ZP8gflRVnP9Hk4SEmEyw6LISiBqlWtLO9/PCjoHFjz3NHzRjm6VQszROzBmmrJFE6K6n1+43VXyqldTg6yyqxRLXx+lPHZh28LlEZQ+syYpQ9T5JMcge9bFTxcULRoESp18olzQRWb+WunoKpnvSRAAiRAAiRAAiRAAiRAAiRAAiQQhsA1NzqLbjhkFwrDZAskYWEQT9BD1q8TGTVcJPMpYLPgj6fm3YJFf6vQgWuzX38WWbvaUWZg8by6Kix2hVignKULykiHQBkCwQqRVzm6rbPYGS0/FtutQgeWQovmOxYaqKeyLhKl7g8+u3Jl5/jfE3TxdJmzoF33KMf6Aee07xys1IEVk1XojNGnlOfrghKeim6r6bB+wDEskC9e6NQDFl1Od7bBc/iP+pjzFm2LLmxj4R0cu6gy5ctPstpVXhd0L7o8ax98rRVUVmrWVqxtyjoza+tU5RZOkWNzwNIIFl8QMJ2gVhGw4GrQWK0plBGYt20nMv4PJ09BvmMsQ2CRg7Zu2Zxp9aHthPKtmSoop0zKUl7E07dYxw0Wm8F3+RJt13/6+P56Z/G5XXtnzMDiq64q/xbNc9ru5b3TqY5CBvcSFImrV6rCSpUlp53lWJ3BgmeJKiSt60Hce3bsjtfrN2uacx9COQvlENqH9oz+zUvtTp5YWeMsXRw1MnG8c885e1nvdt5Bf77/KisdWyd1cpQYsCIbE9LObZoGiadNsc6XTk3B77HWC95WobN4gaOEwz2FsXJ2d2d8dDpNV7DfD64ndC+esZXbuRhtSMR5579/RObpnBxOGjRyHnLAsWjWquHOzYe0+3vfbpQjkYru3Kl9QKEDq5X/6/OsHNh/QC656Dy56rKL9DZKkbtuv1nufejxSEWETW+slieQFatWB47Xqe1Yzi1Y6HqAQY8+eO+dRqGDKeaNd96X30f+IfXr15WnHntQSmrcoQvOO1uGqoWR2zXZh+/0DXytw+Ip1H1moNLMjf79XpIqmXHWvP4sgNUSLKigaMlPefK5V+SSC7vJP1Omyc6du/KzKpZdAAR0BqaQwGFOAH90ih4pvhR9igZPe/j8hzkQdp8ESIAESCA6Af1XgF/saQckY/8+XRDShSoKCZAACZBA3hNAHJzlqqSYOVUXe1eJ3Hhb9EV7KCUgWCwdqgupbosBPLm/UBfH3QKLG6sE+GpQ1gI58uDpf7d1iz1Pn0w2CiO0aZEqPU4/27HoscejfeLJdViuRBNYLsASBxJqYYPvnrXKIVQm/qVtXRTsOmzebJGatR0LHjxFjX5mutKR2nWcErZs0sVDzQdBPJ2J47TuampxoQvkUG5YpU5j5WQVV0OHOAo2nANroV9UwXOOLt5iQb9KVW3fGhzR/1X6Hwv1zZml12+6nq9psOiJJLG2KbQcKGug2AAj29bQPG1UoQbBgvvIX7OUcnNmOovPsDiA+7JJE5SHXv+CFCzYTp6oY9K1CLd8qcg/qsiBogNSoZLjNg7b8fQt1nEDxdK3X6oyZx1qdAQutqB4vPI6Zx+KIq9KnXLldaxpfsiIn1U5ssLZ3q737w9qMXXdLc61bK7XBG4KIVYxi7FlFTpIX7dWlT96D0AZifEbi8TKGmUXKebUAOs0uIqLJIgPhLa5xVqC7d+b/ZjNF0+bYp0vcY9i/cMdSyvWeu31Q7sn6zWCQgeyY7sq/v51lKWYf5J1/nHXA0Ux5lgr8Yyt3MzFtt5EnHdwz7vve9tWuLFs0drZW6LfPVDGF6Acqy7JwsXtcTcJ8ces9HnmJQ21pGNeBfF56tWpLScef6y6BjvKWPiEi7Njz7WfiMkDqavnQpYsWabGi05aJXXJBkGMM6Qd0PHVumVzKVtaFYwqH3/2uQz/bZTZRp5bet0ngz7qb74ubu5xtTz70mvmGN4y9HsECiO4W/t91B8y7JtBUV3YJel9tEVd3qH8r74ZKm+99oKxAAoUGLKB9nl1iRdyatDuEUcUE8RBgoXUCnVnt9Kl5LIZ163fIG/0/8DuRv2EmzqUBT67YrF2jFoqD+YnASp18pMuy04MAvqj3le5jvgq1hJfhWriK1tFfKUriq9kefEV16eAjtAvRwoJkAAJkAAJxEsgPV0y9uxQVzJbJGP7JsnYtkEyNq+RjI0rJWP9crMdb9E8jwRIgAQOawIf9M9aeM8JBFxv1arj5PpHF8PdCp1I57Y+1jkCxYbXBYwfv/XeJne9UKrAFRoE7qSgALHWOE6q8w7FAgTxbcIplZyjwe+RFtFRBgSKDiz4QqDwgHIJsmCe8+l+nzXDWRQvqwvuyItz4ZoMAoUSLKbcAksELM5iYQ0uuKxSB1Yc7usXxVVQXG1ytwEc4XoMMupXx2rI2ct6R1+g5ILM1+uNfrlljqaBvclXy1EQuI8f7O1I7qp27shqiVUOxNu3WMYNasXCezjZ6xoTe51F23DZsqU1bOwk4VrYODI2E5QDK5bqPV1Xx5+OK6vUsX2GMgIvq6jEeZbNXl0Uj0ViYW3LzVxIFjzckx8ST5vc91tObUIco8uucZQtcI+2aoVzRqz12uuBs4vpQ7Jucc8/GkMlIBde7sxBUFDa6xrP2Ip3LrYNwX0T61xoz8XnwZx3oICD20G0GVZro0e4W3LQt+H67OH77jL1zpm3UGrWqBZwweZuTIOj6ptdKGy2b3fNXZr68WdfGqUOMhzXtk3U2DbIAwXGN4M/xGZAEL8GL7e8/Fwfs3tW9yuMFQ52MMV8/+NwdzYTy2f1mrVSvVoVaXO0Wj66JNbYMzfe1tsokVxFhN2Eu7XXX35GypRyFE3I9OSjD+rPlXST/50BA+Wn4SPk6ccfktatmsuYseNl9pz5cu1Vl6pVUQnT5it79JS7e90iUJgVC7HygWJp0OdfB7G8966egnhDu3bt1lv+ZlNPubJlZOD7byqXDOP+7p47exolG4aXlRGjxsprb7xjd/mZoASo1EnQC8NmxU/AV0qVNbWbi79WE/HXaCi+qkfpDz5a38RPlGeSAAmQAAlEJaDfMeYhAX1QAA8RZBNdXEhfvUAyVs6T9GVzJH3pDF0Ay6dFgGyVM4EESIAECjGB0IX3aF2BEsIK3ENZwSpFuHKwSJYZByBIeRIpvy0vXFn2WLTP9ic7brN0YUXG/aGuyS4Ln9suMroVOjm1KXxJIrXrOUesS7nQfOFiE212WR3gyXANJq2rdc6ZYZ4CNmzXqyUC3GLBhZJb4mHltU22HvzPs8oyWBnAuqiLPej6LJ7ZByStVgujUNm2xRknYB3aj9C8BblfJ/Oagi1cDELyum85jRun1qz3GqoEswJFjFdBXCAIFID6gEw2gXUelDruuFOwxjmhg7O4DZeAf/yui9z6mwqWIPUaOkUs1HGQFxKONcrFGMELsk+VT3Y7nvHulOL9PVKbUEIs9Ver7ihicR7KtEod7IeTSPVuVms/KOAQ06jTaY5rxq2bnRIaZyqDMSZs25DPznFHNcpS6oSrE2nRxpYtM9K5saQn+rwDK9TyjiWK/PRDsJVTLP3Mo7xPa4waKBT2qSu1x558Xj4e8EbUkku456jMnLAqwSXE7VO9etWo5+NgamqaWGseG7vH7ifp/Q+LHJS3afNmSc1UItp8W7dvD6t0mT13vlHqFC0SrJCElU8s4jU/YglZN222fPQ/yaffYyqlS5c0n8WVV5J+t0EZ06WTzneZkox5TgUWTlahs3fvPjXETZKUTAa333K9iU+0dp3Oqyq2LMT9sYJYSCgfAqsiCJilpaUbl3jY73pKR3n7vY90euV/VvBIVHFGRKK2ju0iAY8E/HVUidOgrfiPaqNKnMwfuh7PZTYSIAESIAESyFcCxYqLv566S9CXPlNqJH3xdElf+K+kz59srHnytX4WTgIkQAKHA4ESzmKIeXIfCobWx4iU00UwPFEPd1urljvuxexizZEua30slJx2trp2qubEfYE1ChQhk/7Sc3WxP7dSq44IXJhBhuuCnNu6wEnNercKlFRdVIL1SZ26qlTRxW+cs2aNyIz/slxuZZ2Vfat+A8elGI7MU8sgK1j1whP0iMuCPH+OylpwRZ49+hS4leJaL5Q6NhaO2yLD5sGntXKCEigeiadNtp52Jzn9xNPrWOCPJHZ84HikfuzVMhDg3Z03UnkFkV5SxzgC2UOgwAI3iLu9ue1bpHHj1JT9HSuSJ3Z00nHfbNyQPU+kFPQHAu7hxI5FWLkh5hMWauHWC24Cj28vxtVardqqGPhbx3JDHdN67daoIihSPJJwdURKi8Qa+a2VDrYvuSLLhSPuKygVJ+u8YecZ5MkridamWOtYvtRRCmIxG5aD0SSneuEm8qxuzn146VUis2c41wtxvTAmELvKChRAqK+mzmuICRVNcjO2opVrjxWWeQduJY8+1mk1xpbb/aHty0H87HbOGcZlGqp84ZV+stvep2HasG7dBqNsKaZWXM2aNJJZc+YFckGxkKrfa1BGVKlcKZAeaQOKk2tu7KW3+ZEyZND7glg32Iec3OFEefCeXrJm7Tr10to7UETpTNdre3zekZMAAEAASURBVHaHn2M2b3G+3+EKDZZA1j1coIA83ti8ZavAgggsrEXR40+/aOLdhKsKipctW7cZ650Fi5boTxmd/1QGfPSZiQEEnlahdNbpXaXXrdcbJVmHk9rJkG9/DFdktrRNm7fI62++K//+N90cu+zi7nLNFRebcrqdfbqWo79ZKAlLQL8dKSRQOAn4azYWf/MO4m96krpT0z9sFBIgARIgARIoJAT89VqqoqelLiL2kIw1iyRt1nhJnzlWXbXp084UEiABEiCB2AnYJ62xAHz6OcHna0BiKa2L4Xg6/fshzpP91koAObucEZwfC/tYkKxVR+S7r6LHzAg+M/tesWL6yOuZTjrcDW1cnz2PTcEiplWgWDds9hj6hQDxeCG4+ry59kj2T1ia2D6tWJY979R/nGDteGr+hp6OG7aturhVvqLIUbo47hbksRLpiV3rgkld8sQtsbTJVoLYQy2PdvaG/+wspGc+fWyzBD5xTa3YuB92337qU+eCLsA1VaIJXI2ddZ7zWDsUWGNVGWclr/qW07ix9bk/O5ySZdn0sy7+WUWTO0+kbWuBk9O4wvkYWztVaQKZpYuPcMkG6zzcF8ed6KRD8YlYSVCI5kaisQ4tF3nRZ9y7UJQi5knDRiKDPs5bxU4sbQptY7h93APffRnuSHCal3rhghGKNKtwtJ8oCW7CoFR3C2IweZHcjC0v5SNPos87sK4481ynN7DM+0/n7gKUypUqyk09VHGnMv6vyTJp8r9RW/PZ50PkmT4PmTxQYqxes05mzp4rKKdpk4ZGoRO1gDAHcR7E7c6tUUP9vlaBUsctR+D7V2VPBLeQVkmCPFWrVJbFGqMnkWSnWvbC3VqojBg5JjRJho8YJbfd3MPE6vGiJLMFIL6RVegg7Zvvh8nVlztKnaPq17XZ+JmgBPQbkEIChYiABiRManOaJLXuIr5q9QtRw9lUEiABEiABEghPwFe1viTrS7peo5Y7/0ja1N9VwTMufGamkgAJkAAJhCeQ5LgSMQenTdGnwWc6i4kIxt5EFTrN9IUYCFACIAC9PpkbELh0QlBvuIEqdoRIA12UhQUIFow7dvG2+BkoLGSjy5nOk/0Ipj7l75CDUXa3bHKe+LfB46FkOk7bBGsEuDmar088h1tAhxLmvIucRWYs3I4cnr0SEzenqtNP9NFaESEnFsaRBtm1PXhhukiKkx76bhU/e7W+eCWWNqEO1Gndrk3/z4n3E61uq3hCnpTM/oXmT87sX4QFwNDsB3W/i15zjF+Icb/kYp0XffMybpzas96bqPVZk+bO/sTx2RWWcJNmx0bWWY6lGe45dRskqgcJsnxx53NbxFjFD5Sx5+v4hgIFVjm4p5q1clyIYdxe2cNxAbZ8qVMS2lepsrtUZ3u7KhoiLZBHY42zcV99/J5+avvtPYh+Ntd2HHuCo5hF392Kt+wtiC0lpzbFVpr33DnVCwsqWDlCAQ6rHMS0gvu+ZvrgEhRdUEROmezMZd5r1XGVw9iKpaxoeRN93ul8unMPYV6GRZQdb9H6lI/HXnjmMeO2a8fOXfL8K31zrGnK1Oky7JcRcs6Zp5q81apWFrwg6a64d9ZVmDmQw1vjhg1MjjVr9fs6U+rUrmm2Fi9dZpPM5z69V2ElVDQk9ozNhDg1VjZs1O/cBBPEvIkkUIzBPRviGW1Vax70HZZPRfwpkU7xlA7LH2tBZU+oVrWK3HvXbXY36PPTwV/J1On6e4tSIAQi/JopkLawUhKISMBXvpokHXe2JLXFnyL9wZTXcmC3+A7s1W8W/bKMMnHmdbUsjwRIgARIoJASgO/jpBTJSNHFv2R9ii6PxN9QXYnqK6PrGkmb/IukTdI/cKmuhZs8qofFkAAJkMAhR8AGScfiF9wzWYEyZdxodYGpynNYNNSo7Sh1du+0OUT+niACSxXI7l2q4FGlUAV1BwOrFSwIw/ojXMwP54zI71hQhmUNBE+n6wNqAfHpYigEC9ewzoF1AdoOt01Im62LJEsWO3nwPl+fhPfpAg8UOpDK1bIrMrDAeu6FzoI3/tMM+9axSnLOyHrHMSy+zpjqxMIpVVrdWukiN54ERxlndnPywjICeW2biim/cAJFGMTN1Enx/h5Lm1Bqp1OdxU5YrUAhZy2c3Mo9uJZC+n79n6eLkAFBe0OtB3CwWOb/TDuWAicU8Abcm9Vr4DRi3Jjs7pdy2zev48aNoXZdR+GJNIxTXINQgSLVKgjdx5AGpQ5c+yG+SiQLLzuu7BhEGW2Pc8b3ujUiP+r4xrGVK0RKldFxe46j+IJl3EfvOMcQC6ZWHXftzjYUEOGUOjmxtiW5FWlIg6IHyovKVZ36atayOXP/6bVNua8puAQv9dap7/QXc9eXn+l8o/cargfchHXo7MyhbY7NtAbcHFx+pD0vYyvSubGmJ/K801S/P+zYHfGL890Ua//yMP+tN14rlStWMCX2e3uAlC+fqWTWFLgvgxyp93JFzbNL5yTrlg1xWX4bOUbOOLWz1K5VQ6AQWrR4ifw6YrR88v6b5ryVK1ebz2hvqB8WOVAwQOrXqy2vvfS02a5bx7nfEH+mZfOmMvCzL42iYbvOMaVLlTTtMhlD3sqUKWVSMAx27NDvu0IiN99wjZx39hlGb3owmly+XNmAy73Q+lq1bEalTiiUg7hPpc5BhM2qYifgq1hTkk7sLknHZP55ib0Ic4Zvz1bxbV4svq3Lxb9tlfh2rBHfTv2Dt2ez+PbqHxjM4hQSIAESIAESiIcAlDvFSkvGkeUlo0RFyShZVTLK1NBXHUkvr4sJcSh9fOWqSvLp10vyyZdK6l/fS9r475xFqXjax3NIgARI4HAggAViCBaMoRSBIsItS5c4T3+XzVyIsq6ckKdM2Syljj0Hi87WFRlcu8UTW8fthuiCS23JwZ/HtBPBC67Z/tXXLlU+lNEFarQpVNAHKxXUAmntKrvnWB5BGWODkEOhA8ujaILYDKHxGY493jkDi9T2P9IuZQsrEVt2aJnlVQEGQbyT3IrXNtXVxWQIlDZX3+hsh753u9BJ+f6r4Fgv6Me6tcG54QrMKiB25kE/gkuPf691W8elF0rAQjlcj4UKro+VWPuGxdhYxw1ifFgrKbjeGvGTrT34E9Y7bmsbe9SOS8sZsa/CCRSrEHc8DFisQaD0tOMT+9u3ivz2s8a4ucqps6oqPVfr/TFH8yE+VqiEi+PjhXVoOaH7GL+16jhxsEKPxbOfF23Kz3oRywiyWOdLt6IL84ex2lHlH8YAXNL9reM3J/E6tnIqJ9bjiTbvlNb5v/0pTi/mzBJZ5pr7Y+1bHuVH3Borjzxwt90M+jzx+GMFr3F//S3PvfR64NhCjQfz5qIPAvvYgJWJlfkLdPzkIMe1PToo9g4USI0aZH4PZJ5brmwZwatFsyZG0YB4MTWrVxMbWye0inp165gkxOcpLNKyRVM5X+MaQbZt3yHjJkySVavXGIud07t2DijY8rI/y1euktFjJ4QtckpmLJ6wB5mY7wT0Fy+FBBKPgK9kOUnqeIkktdOnbeIQo7xZPU3862aKf/1c8W1fE0cpPIUESIAESIAEPBBIOyC+XRvNSzbMy3ZCevn6klGpiaRXbaGvVqr80YUxr1KsuCSfcqUkH3+epP45RNLGfeP1TOYjARIggcOLwGZ9YMsK4m3MDFn8rlbDOQq3S5BUfbIcT+vDegeufpaqtYFbsChsJV5lxe6dqpBXhVA4sW6psDCNBSW7IIrFRSh1GmofJvyhFkJ63EoVV5syAzybQ3BzhDhC1Ws6OX8e6ixo2/O8fsL9WKs2Tu6ZU7POgjKptX534Ql6xNhIT8s6BuUTXMJBQhk6qbl7j9QmWOhYa6fQGixbcAXfNG0vXhs3OIqpBo1VOTIj+CxrCYPUVaqoSASBO692mQup/6oVCCxBwkm8fYtn3FTWp+S7XeC0AkrFn3SsuZUr7vbNnObey76N8YIYNFj0R7luRRvaVi9TgbM8896Ex47Atd2XvTxrbYcjpfUeglIHdXgZl15ZZ681OMW6yINSI7eSV22KtR2x1FshUyF3IEx/MS62qHUOrB3DKalD2xXL2Ao9N6/3C3LegWXo2ec57uug+Bw3Jq97F1d5O9XCprid60NKSIa1n4qZbvX7YYd9yCIkn3u3x9WXm90t27bLoiVL3YfCbj/65AtSsmQJeVFdwBXROeOp5/8nW9TtGLaRlqbWtPc9/IQ5d/UaR2k/+o/x0rpFM+MyrmP7E2TsuCzFol8525gxsBwqKPFhrotBOnXIUobd93AfVehkPaBwSsf2xt1cDMV5yorYQy+/9qanvMx0cAlQqXNwebM2DwSSOlysC1hX6JPN+uMuBvGv/k+Slk0U/4q/xafWOBQSIAESIAESSAQC/k2LNMj2IkmaM8w0J71SY0mv2U7Sap8gGbDk8SJHlJDk03poXLlTJXXUYI25M9bLWcxDAiRAAocPAbjMWrlc3Z3VcmLPrNGn8+F6DZYIiPFQOlO5smpFFpOp6jIKsS9qqbICVjkL5zvHYO0ApQoEsW3cSgwn1dv7Lz9GznepWhRgAXjSeMfdm80JN1aI6YOF7vadRf4c5ayUIYbI0W2dXFg527je2caCUBd9ahd9gAz7TpUSrj46qcHv6F/RYmrBoA++wW0SBJYqUAzBWgXlI06NlRm63foYZ5Hx9LOc2A5QNmGB/Qy1DoJAyeJl8dzJnf091jZ9PCB7GUjBguhNvZxjQ4c4Y8DmRPwVxP+AiywoE+B+DlJeF6bhagqyZKGj7HP2Cu4disaTTnbqt1Zc0VoTa9/iHTfnXeyMA8SzGfZ9/PcG+gIrHyxaI07OqTr2vh6kcXagrNMxfcppzj2AfFOn4F2Vn6o4wDiDdVY7vW9h8WMt7jBu22ZamSHvmhge6oyVda06zj2C+q3FDwLaQzFYX18QjKPcSKxtirWuZOUFF4a478eMdCydUEas9cJlY0m9fnA1uWCuM6egHFzDOjon2XhGULBZaXOcCCzt4PZyxTInFfd/Xo4tW5f7M1K9scyFB2PeOVHvezCFQGka7/ePU0Kevd90+z0Ry/rqs/elhCp8Ro4ZK6/2eyeQr0rlStLhpONl9B/jZOOmzSY9BQqZxx6UihX0+0/li6++NZ85vUFR41/nN0ocxOOZ+Ld+V6o0b9rYfG5X92nz5gffdyNGjpGe6rYNcXV633GrOb5u/Qb9mvBL31eeNcoenPzeh5+aMg7Wm1sRc0K7Y2Xyv5nfRR4akJycFMjlgzvwTOna+WT9StbvZMphRUBncgoJJAYBf4NjJPnUa8VXRf90eRQslPkXjpKkxWPUndoGj2cxGwmQAAmQAAkUHAFYkOKV/O9ASa/QQNLrd5K0o7p4suDxVaghKZc8IOnN20vqiI8lY9PqgusIayYBEiCBRCMwSRcJYa0ChchF+pDY1q1qRVI8a3EYC8L/TMxqNVw4QVkBax0oRo5v7xyDAsXKyN/s1sH5hCIKMUrq6n8iLLBC2WTdn9kW/PWnLm5nWipg0dQuJOP4Od1truBPWKl887mTBhdWiHMBgdLKr8sCVumFtDG/Z5WP/d1q0QSlARZFoTy6sZcupqsSzS48Is/Y0c5CN7bjkVjbFE8dUDohFguUOlDiQAlwIDXL0giWPZP+iqfkvD+nY5esMtu207bqK5y8289JjbVv8Yyb9p2cxXrUWLW6Ks9ud+oOffeiWLTn/DFKrRLOd67BtTc5Sp7iJbPqgbs5uFaz8td4VUZ0dcbrldfrvbHTGZ8VK9kcjkJlq7OAnJUYZStW1pgz0H8IxgzuD/f9g3kG7cyNxNqmWOuCYsrOG81bikwY65QQa71wGQmLKihxoJQBj02q7Kqo8xIUbRBco3mznW0o5KyLxxNVMfdlplInP8aWU2P0ehNx3kEsKiuXXm23sn/inv91WPb0BEpp2qSh9Lj6MvPau2+/hp7aLyVLlDDDBc2cM2+h/Piz9+/YurVrmd7twj2XKc0ylTrr1ul3XBj5YOBguf2WHlK0SIp89G5f2bt3nxRVJSyGLGTGrLnixf2bkztv3rdqTDcouSqUL6exhk6RLqd0UAVTkvz86++CGETR5M/xE6XrKR1Nlv79XhL0G7GB4I6OcvgR8B9+XWaPE46ATl7JZ98qKVc/6Vmhk7RotBT56QEp8m1PSZ4+hAqdhLuobBAJkAAJkIAXAv6NCyR50gApOugySRn9ovjX6OKFB/E3PVGK3PGOJJ3QzUNuZiEBEiCBQ4SA2xVZuC7BemXIIGcREcfhxgwKHgiCdw9RpQYWHa2k6vYXnzgWA0iDMscqdLAQaSw9wi8U2SJUk5G1GctWWrqTW13GZBPEKJk2xUlG+61LJ1jV/KFP1VsLE+SwK1PZCglJgPWNlTUrlVHmoljZ8lkL0kj74WuR+XNszqzPyaoMGzfGUdygTqvQgZspLOIvXpiVN+yWq/5wx+NpU7hy3P0MZYtjQ9E/tSiA4Klm604ICsCvPo0vdpJTWt6+u/vhpeRY+xbPuPF6Tij3aO2HdR1iHkERAoHVDupBf2DJgTHnFigHRo/IsqbC/WoVOrg/YNX0+3D3GTlvx8raxI/JbC/uT7dCB2PrK52DbH+i1R5tPou1TeHqiVb+xo1ZZ8Biz0qs9cK92nd6/aA0hoAH4uJYhc6CeSLffpk17+7fmzX3uK138mRsRZljItVbmOedKN11LsbBe8/IHDdp9nsts+pFi5cJ3KtBihUtIqXUfRouNVyl/fLbSLn3occzc3r7aJgZQ2eDa/zWrlXDnLxsuX7Hh5Gfho+QF/73hurvdX5QgdWOnWLG/DlBHnz0qTBnBSdlxPg9b3kElxK89/7Hg7Ksl2A5pwJlDyQ9inUWrHpGjBpr8iWpxVG1qpWNQue/6TNl3QbnvoYlk5V0uOdU8dIme479TE11zrX7/Ew8Ar7GbTplXe18at/O7VvyqWQWW9gJ+Ou1MgodX8WanrqSNPsHSZ75vbpX0z8iFBIgARIgARI4BAmkV2stqc27S7q6Z/Mi6XMnSepP/SVjm+sPupcTmYcESCChCZQoVTbP23dY/S8rVkzjp1TS1SNdlNis1ig2Zk0kqlhUgQsgxO3AQiUsUQpa4DoOSpcSarmAxR78r3Yt1uSqeVjVKqUL6GYRXUuCGyWvcUAQxBvngiuUX3kluWlTrG2Am7byuN66CL1JF6St5ZOHcjp3ai/33XWbydn7wcezufwJLWLokE8kRcfXtBmz5eHHnwk9nPf7uehb3jcmhhLhCqxcBWdMIT5ONAUDxkrJ0vrSewPxPLbrwjHiZUVZDI2hJd6y4t5B/XC9hrhbuEehKC4sAqUmXN7v3pU3Lcb1QywjuDNDTBVYWLmV6LYWe+3cFlj2WH5+Rqq3kMw78aK587abjDUIzj+ru1qwRpHatWpK/74vmhzvfvCpDB32S5Tc3g/VqF5NGjaop0qdkjJl6gxZviL6el7LFk3lhaceNRW80vdtGTVmnPfKouREO1ppjJ2Vq1arhc4cVZ6Eeagiyvn5caiMKoWLqsILbuFikXJly0hL7cuuXbtk6vRZeqsVorknlo56zJub38yJ9Ns41n7oL1cKCRQMgaSOGjun67WeKk+a85MkT/tSfDvWesrPTCRAAiRAAiRQWAn4V0+VIvpKr9pSUltdqvF3jo3aFX/jdpJSq4mk/vCmpM/Wp1opJEACJEACGutCnwiHFYBXwVO87ifWvZ6Xn/mgwIE7NrzyWrBgjkXozCeDYyp+my6445XXkps2xdoWLOZtUEXWQZSWzZvKsG/UikNl2sxZ8kif5/Kn9gLoW550BC6Vdnu8ZzFWoBQ42IoBd0cRDwivwipQ4npV5Hrpo7l+eg1zEnvtcsqX18cj1XsIzjtQ5JzWpZMh6IPSKg65+fqr5aYeV5kzP/xksHw79Kc4SnFOgRIFr2hy7DGtpc//3R8tS66PeWlHriuJsQBrnRPjabJ5y1YZM3Z8rKcx/yFGgEqdQ+yCForuJBeRlO53i79Fxxyb618+UWMOfCpwT0MhARIgARIggcOJAFyxFdFXWl2Nn9PmGskoVydi931HlpKUy/5P0kYPllR9UUiABEiABEiABBKLANZW7QJrMVh3UEiABEggHwgUVSssPyw9cyHu+SoFFo35LH4Ny5DbNudzE1k8CSQcAbpfS7hLcmg3yFeplqRceJ/4qtaL2lHfrg2S/PcHkrRwVNR8PEgCJEACJEAChwuB1DZXSuoxOVu4pk8fIwe++V90tyWHCzT2kwQKMYFYXTB46WoiuZjw0l7mIYFEJBCr+zUsiPrhEs0lcJWTCK5/XE3iJgmQwCFCAPNNqCIG840XF11FwyicD9Z8VZB1HyKX/rDtRm5+MyfSb+NY+0FLncN2yB/8jvvrHy3JlzwgviPU72wUSZr7s6RMfFf9r2YGH4ySl4dIgARIgARI4HAhkDxlkCQtmygHjr9FEHcnkvhbdpKU0hUl9asXJWOHxoWgkAAJkAAJkAAJ5BmBWbPnyaAvvzHl5RQbApm8LKTmWeNYEAmQwGFPAAqcfTHECXMDi/c8dxnxbhdk3fG2meeRQEESoKVOQdI/jOr2N+8oKarQiSa+fTskefwbkrRoTLRsPEYCJEACJEAChz2B1DZXqdXONVE5ZGxcKQe+eF4y1i+Lmo8HSYAEEpNArE/reelFIj2N6KW9zEMCJEACJEACJEACJEAC0Qjk5jdzIv02jrUfwTbA0QjxGAnESSCpzak5KnT8q6ZIke9uo0InTsY8jQRIgARI4PAikDzlMynyy/+Jb+f6iB33VaghKdc+Lb7qDSLm4QESIAESIAESIAESIAESIAESIAESIIHCRYBKncJ1vQpda5OOOU2Sz78raruTZ34nRX5+SHw71kXNx4MkQAIkQAIkQAJZBPwr/5EiQ+8UfEYSX8lyknLVE6rYaRgpC9NJgARIgARIgARIgARIgARIgARIgAQKEQEqdQrRxSpsTU1q3VmSz7szarNT/npbkv/qHzUPD5IACZAACZAACYQn4Nu92VjsJM35KXwGTfUVLy0pVz4mvip1I+bhARIgARIgARIgARIgARIgARIgARIggcJBgEqdwnGdCl0r/Y2Pl+QL7ona7pSRz0jSzO+j5uFBEiABEiABEiCBnAmkjOsrcMkWSXwlykrKZequrUylSFmYTgIkQAIkQAIkQAIkQAIkQAIkQAIkUAgIUKlTCC5SYWuiv2ZjSbn0ocjNTt2nTxU/LEmLx0bOwyMkQAIkQAIkQAIxEUj+9xNJnvhuxHN85apK8iUPiiQXiZiHB0iABEiABEiABEiABEiABEiABEiABBKbAJU6iX19Cl3rfKXKS/JF94skJYdv+4E9UmT4I+r//9/wx5lKAiRAAiRAAiQQN4HkGd9IyoS3Ip7vr9FIUi66L+JxHiABEiABEiABEiABEiABEiABEiABEkhsAlTqJPb1KXStg8s1X9nK4dudnipFfntc/Gumhz/OVBIgARIgARIggVwTSJo1VOPVvROxHH/TEyX5tB4Rj/MACZAACZDAwSNQrWoVKVq0aNwV+v1+QRn49CoVypeTkiVLeM0eV77KlSrmql9xVZrDSaVLl5Ijjzgih1yF8/D9vXvJ14M/kDdefT6oA8lJSdL1lI5SsWKFoPRoOwPeetWUdf01V0TLlqtjxx7TWpo2bpirMnhyfAQwJqpXi23OiK+m4LMwJ8Q6T6GtXgVlo45YBHNhLHXEUjbzkgAJkEB+E4hgTpHf1bL8Q5FA8lm3iL9eq4hdKzLiKfGvnhbxOA+QAAmQAAmQAAnkDYHkmd+KpBST1LbXhS0wqf2FkrF+maRNHRX2OBNJgARIgATyj8AZp3WWKy+9UMqWKaOLnD5TUWpamsybv1Du/78nc6wYi5C9et4oHU5qJ0cUKxbIv3nLVnnptTdl+ozZgTS7AcXPw/ffJfXq1BafU6WkpafLf1NnyMuvvyU7duy0WeXRB3tL61bNA/vhNv6vz3Myf8GioEPdzjlDLr+4u5QqWTJQx969++STwV/J9z/+EpQ33E7dOrXkxWf0IUBl8sNPv8ong74Kl00+fLev1lFCLrv6ZgG3I44oJp9+8Jag/zfffm/QOSkpKfLsEw9Lg6PqS9EiKebY7j17ZMLEyfJqv8gPQJQqVVJuvfFaOa7t0eacr78bJl8M+S6o7NCdju1PkOuuvsy0bd++/XJlj56hWfJtv2qVykZhVSlEefPis49Lk0YNJD09Q7pfdp0cOHAgxzZUqlRBUpKTpXLl2BbIcyw4M8OlF50v1155idnr8+zLMvmf/7yeyny5IHDnbTfJSSccJyVLFA+Ugvvz15Fj5N33BwbSwm3EOyfUqllDHn/4XjOWklTpkpEhsn3HDvl08BD5+dffs1WF/Pf3vl1q1qgmRfTehezctVuGfPuDeWU7QRM6d2ovN1x3lZQpVcrMOxjrGzZtkmeef1UWLVma7ZR46shWCBNIgARIIAEIUKmTABfhUGhCUusuknT8uRG7kvLHK+JfPjHicR4gARIgARIgARLIWwLJ/w2WjCPKSFqz88MWnNztDklfvVCVO8vDHmciCZAACZBA3hPoedN1cu5ZpwUVjIVOKGqaNWkUlB5up/iRR8qnH74txYpmj49WrmwZef7JR6XXPQ/JkqVZczueRu/f7yWzUI8yoQhJ8ifpyy9t27SSd954Ra694XaTjuOV9Gn3nCxaKlYoH6TUefv1F6VO7Zo4PUiKFSsqN19/tSRp/775fljQMfcOnrJ/7slHpETxI00y2hxOwKmKtg8KKfQDAm5o7+7de4JOgWXO231fkrL66RbkhfVKndq15O77H1WFR3rgMJ70v0MVZke3ahFQTOFgzerVAnlCN6wyq7QqgqwcUSwxLIKs0s+nmryiOma8KHVsH3LzCSY3XnelLFu+Uu645+GgotxKhdK6EE/JXwJQbL764lNSv27tQEWYc6Dcxf3ZXhU9OSl14pkTWrZoKs/0eTjIEgZ14j7pdev1amFYWd7/eFCgTTX0Huv3v2cDyhzbRswJPVRZivv1ZVVau+WKSy+Qqy67yJ1klMKVVbnZ95Vn5ekXXpVJk7Nc/8dTR1Dh3CEBEiCBBCJApU4CXYzC2hRf+WqSfG7kp5AQuDlp/m+FtXtsNwmQAAmQAAkUWgIpE96WjBKVJL32idn7kJwiyef0lAMfBi+2ZM/IFBIgARIggbwgcPxxxwQUOlu2bTdPks+ZN98UjcXGUzqelGM1sEqBQgfWJkO+/VHGTZgoyTqfYwH9mKNbmoXau3vdInfd90igLDyhD8sLyLMvvS7j//pbsNB7yw3XyFmndzFKj8vVcuhTtaiBvPL62xKqVClXrqz01nKxKIu6p88MtgaqUrmSUbT89vtoGTn6T1m9Zp1ceP7ZcsF555hzrrr84qhKHTyd71aKmIaEeWvU8CiTut1lWdSksaMMW7N2fdAZ6LdV6AwfMVo++vRzo/x54J5exnrlqHp1pNvZpwdZEX34Tl/TXhS0X61arLVAUMGund533Cqndu4YSIGyCcqyRJEnn3tFLrmwm/wzZZrs3LnroDWrri7AQwFXwmUVYisfqBZYYLtHx9GoMX/aZH7mEwHMB1ahM33mHHn9zXdl7TrnXmnVopnUqlk9x5rjmRMevPdOMwagnHnjnffl95F/SP36deWpxx401kIXnHe2DFWLvA0bNpr6YVWG+w2WNs+/0tfMU5hXXnn+CYHC+pSOJ8rnX30rK1etNvnhQvLKSx2FDuakJ555WebOWyDtTzpe7rvrNqPcub/3bXLRFTcE+hdrHYETuUECJEACCUggcX5tJCAcNskbgeQzbzYuXsLlTlo4UpKnfBbuENNIgARIgARIgAQOAoEiY14W39asJ7bdVfrrtJCkU65wJ3GbBEiABAotASgioNgIF6MGShMci2aBggVExPqANUw0gVUJlAuwEMHCdTjBYnZo/Igb1UUQBAuQsIyxCh2kYaHy08+HYDOqwBpl0JffmIXKL7/+XlatXqvWECukzzMvyYHUVHMuXHG5BQupkLXrN5iFUmzDYuPt9z4KWLu445ssX7FSpkydHnjNX7jIWNtAobNX3Yrdesf9Qe7aUN43Q4fJpVfdJG/0/0Bmz50vW7dtkw8GDjZtw3G4PovE/lh1cXayui6LJuAMRVSL5k1NtvXaF+zj1fCoeiYN7cY+BJ/WdRra0+/tAabN6/S8ex/qI9u27zD5Lr6gm/m0bxm6Ar185Sp59Y135fxLrjULzPZYuM+UlGTDZNQf46THLXcJlFp5LVDkNdA+ntCurWAcxyLoL67JX5P+iXgaXPNhLIeO13An4DrAfVW7Y49RRUGdiOO/UsXy4U43aRh7cK0HpaTbSsp9Aq4fFA5wyReLYIzhPq9dK7vVWCzlxJIXPJo3bZyNH3g2adTQxA6KNE8gDzi2btk8IkvbFsxryBeNCdwGugUcO7Y/3iTNnD1XHnrs6YBCB4nTZsySH3/+zX1K2O1Y5wS00ypUP/7scxn+2ygz18DF5C297jNu2FDRzT2uNvVhjNv802bMDMxTUD658/e8+TqTH2+I+YQ5CdKr98OC/sF6b8zY8dKv/wCTjvEAt4iQeOowJ/KNBEiABBKUAC11EvTCFJZmJR3fTfwN24Ztrm/zEkkZ+2rYY0wkARIgARIgARI4SAT279Lv49dkf7fXwlaYrEqdjIVTJH3F3LDHmUgCJEACBUHg6ccfMnFdsEA3e858ufaqS/Xp7hJGWYBYJfb4pL//1QX67dK188kBaxS09y9Nf0fjRNx/9226qIoF68zVPz0GN2BQOFg5ucOJcpdadcAVkRXEmvhv2gzjvsemlSldWp576hGprYu4djERx7CYiKfEoayB4Al0WM5ggfGGnr3Nk+hYjIe7Ici3Q38KKFNMQgxvqGPQF99kOwOL42hzSolk2b9/f9Dxffv2mf2iRYJdttlz4N5oy9atQefYHSwKv9PvZeMWDdYVd6hrt42bNtvDgc9wbcJBlAu3bHha3yqdAifpBhZdH77vLpM0Z95CE0vDumBz5/vi0/eClEKNGtSXoV8NdGeRs8/oal4v/O8Ncy2txcx3P/wclA87sFYyVkplShsrAMTjgXiNO2My69trqvw5cCDYJZQ9Fvr55acD5Mgjj5Cly1Zkc0mGvF8P/sAoJP/WGDNPP/8/gYUFFqRDXe3BymvQ51+HjUkSWue9d/WUTmoBtkvjklx2jT6M6RLEDOqqVkZuZduMWXPV0ii7ovLM07qYexCxjNyC6/rHuAny0qsOA4z7Uzp1MPFNkA9usH78xnnIc5u2+6rrb5MOJ7aTB+69wxRzo94fUDxZaajXtc8j9wcW+JGOOmbPnSePP/2iWvfstVkDc8Cf4ycapehF558bdA/jXrnnwT4ChYRXGfTxOyYm0suvvaVKpaZyyskdDH/MJ7gm9vjAz75U5VEraalKRjsXoJ3vvP+xub6977zVuAm09WIueE6t5CZqOVZuUf7nnnla0NwECzTMT4ghYwUKt8c0Lo2bPaxZhv3ym5njbD7MiVBobdq8Ra5WpTEEVlr2Puj39vs2a64+vcwJmAMhYPL9j8OD6oPCd/WatVK9WhVpc3QLcwwKSyvDfhlhN83nrt27ZfHSZcba6Kh6dQPH7DmYj6zlkT34+6ixxo0i+g5rvLHj/jJKUXvcax02Pz9JgARIIBEJ+BOxUWxT4SDgK1NJkk/rEbGxKeP6iaQdiHicB0iABEiABEiABA4OAf+6WZLyV/+IlSWdel3EYzxAAiRAAgVBoHjx4mYxEgvSd/S8wSxoYvE0OdONmD1+4vHHChacsRC9Z2/Wgu8J6ups4Hv9zBP0GZJhrClsP84790y7adxDYeEbCh0sQMKCY9/+A2a/ebMmgXxY+P54QD+pU8tR6EDBYWO64Cn9p/s8FMjbQd3/QPB0/knaPkjTJg3NJ97cT8Z7sY4InBhlA+6JbKySaTOCXaNNmjzFnFlWFRj3qZLLWg3AOscqUIaFeVofbUOsHJSNvt7z4OPGMihKM7Idsu7SsPAaLp7L0xpzA0oLMH/syeeznW8TFi9ZZpRJWMyG7FBXYijTrWCy+5t1UbtWjSyXUlDghMpfkyYHkhCrw0q4Ntpj4T5jyQ8rBSwywxVWqFsyBLDHGMPxP/6cYKrC2LYKHSjsrFIMFg2333K9wLIsJ7H3Saj1GhbdsdiNOjHuoQCB67gWzWBxojdaiHRQaw+rVEA77L2Ge7KTKkURQwcCpQzaZxUdSEOf8Doy0wIuRa22bBpiLVlBDJbXXnw6oNCxfUZZUGwMHPBmkDWM7RvqR1wV3MM4B/2BoG9Xq9u/WAQxiNA23CeYVyx/q/C1x6+/5nKj9ElLTzNjF3WgnYiZ9eIzjxmFDjjZOQL33HUaF8YKLALPU/5gjTxQ1OETjFu3am6zmZhXLz/XJ8Ae1wn9w3m4fpdelBU3sZnOQ5Dy6ioRrskgjRocZT5xnnVbhoR45x2vc4J137hVFe7h7hFYz0GKFslSpJsEfYPlU6jAGhECaxsrdnvBoiU2KfAJhfWGjZvMPlxHhorXOkLP4z4JkAAJJBIBWuok0tUoZG1J7nqN/lNxzNtDm5789weCBSQKCZAACZAACZBAYhBImvmdpFdpIWl122drkL9Oc0k6/lxJm/hjtmNMIAESIIGCJIAF1i1bt8lPw0cIFu/wtL9bsMj/8WdfGOsXLORBwfKSxmaAYIG3r7rdsovk11x5iVymi6BYYIUFBJ7evu6qy8w+Fkqv6HFroHw8jV6zRparKyh+EO8B+Z572YlLgwVOxIKB67AmjRpIaV3MRvu+/u5H6X3HLab+334fY9pSM9NtFtp7qloVYVEd+dE/LLguWbpcnlJLgB2uWDHmRI9vjz50byDnDz8ND2xj45NBX8rRulAMRUfnk9ubJ9ZhAXVa11NMvklqGWIXWd0nPvfkI+ZpeqRt2bLNWMEgNs2ChYvd2SJuX3Zx94AVyOix47LlgyKgSSNn0fmFV/oFLJ2yZdSEBx55yiTDmgWL9ff/35PGAgPXYJhagoDrNTf2Cpx6fqbiziqBAgcyN9at3xhIqqHXGe7m8ltg3dG2TStTTfduZwdiGCGh29mOUgTu7TAuIQM++sxYec2aMy+wMH7W6V1NkHkoEDqc1M64MDOZY3iDku6Ga680Z8Cqo+ddDwTi7Vx8wbl6T1wepJRBxm/VeuSX4b/Lf9NnBvJi4R6KFrSloyoyfxg2PHCdBrz1qhk76zReSo+b7/TUugd69zJlQbnxSJ/nZMasOeY8uNm6qPs5RgEJ6yK4DXQL7skRo/6Qdz8YaCx5oDBDbCQoLGG5Eo9gjpi3YJG68pqgiszVqhBZE1QMlIqv9H1bJuu9A+l16w3G8gvbiCf1wv/6yUKdrzA+X3/5GUH8pprVqxtLLFjO3Xy9rqWoQIl8+bW3mG28wX1aCVVoW+l9pxO7GPU9/PgzAuUmlBlvqbK1SqWKgusFN4yQX34bae7ROXMXBOYRq1zZtWuP3HX7zQLloXUvCYXLv1Omyqv93rHV5fjpdU7A3AbZo+4iw8nmLVtMMpRT6A8sxKx0PaVjoE82bfNmx5IO1wVMMdenZCr4t0awMoR1GsTyjKcOWz8/SYAESCARCdBSJxGvSiFoE1yu+Vt2CttS/+qpkjzty7DHmEgCJEACJEACJFBwBJIn6h/3A+H/YCd31gWeI5wnOwuuhayZBEiABIIJ7NSFObhbG/zlt2YBdb4utLoFrnmgRMEiHwSu0GBFA1mybHlAoYP973/8BR9GbBwW63YMC9Nu6w48XY4FVCvHZC7Gww2QtfxAnR9/+oXNYmJnYAfHEZwbrqagsIFUq1bVfGIREy6qsLAOhQ4ESgpYInzQ/3WTbhJjeENcmMYN65szflUlEixC3AKXVS+/+lbAggHWBo51k98sKr+c6TrLfQ62j6pfJ5BUsUI5c05fXaB+4J4s5UkgQ8gGYsBYKwm4RnJzQtbKuiB9U4+rzFnj/5oskyZnuaUKKSpo94hijmWJdalVT61eIBgHbimrfCGpaanu5MD2lkx3a0ioVCFy/JfACXmwsWjJUqOgRFGdO2U9YIFF6iaNG5gaJkzMsioaMXKMTFUlitvSYfiIUUaBhcxeLHVMoSFvxx3bJqC0efqFVwNKGmRDnJtwzP6ZMk3+nDApKC8so9asXWdKh2vC3Aji+uCegPzy68iAQgf7H34y2LgUwzZcJYbKzl275PU33w24ZtupCpDpGisGYi1sQs/JaX+qnt/7gcdk6LBfBH0Pde+FecgqdFDWdz/8FCgSCiYodCCYI0aN+dNsY46ByzEI2gxBTCareMH+dlXywDUZBC4bbZwZKLXtfIT7+VdVrkJwP1gZ8OGnJg4UlD9WSqt1HgT37+ldOxlFF9qBF8qGAuW1l572bLnjdU7AHAOxFl1mx/XmVs4jBtgGVf4hPhgEjL77cqA8/n/3ycP33yUfvtvXKPVcp4t7vNn4WO7j2LZKnaJqCQiJtQ5zEt9IgARIIIEJ0FIngS9OIjct6eTLIjYvedKAiMd4gARIgARIgARIoOAI+Haul5S/35cDJ92RvRHFSkiyfr+nDn8/+zGmkAAJkEABEUDg+lhln1o7wKomVLBgasWWiwXkKy650Cxywm3SytVr5KtvfpDfdWHWChbdbXnF9en/px570B4yn2giFknLlc3u5sdmTErKep5yytQZqqT6Ruaq8gUWNHChBAsjWBbArRYW2r0KLD9gbQSBVUTft97LdurZZ5wqt93cw7Tx73//k7kauwaWLHD1VFotAwYPfMdYWIQqzD4d/LVZFE9Ty4na6nau/YnHq7ukFONua6Y+Wf/zr79nqwsJWHB98ZnHTX2wurjv4Sey5XtBWUOpBQuE51/pm+14uAQoMcDZKsqQp6kqwyAbNmZZ3mAfsUkgbvdeJiHzzbqnwi4swQ6WgNmVl15oYs1Yy65OHU8MuMQbOOiroKZA+QUXhLAa26rthFIRSpci/uzjO+jEKDtQIEJwbUKveZTTjJXJGad2lrp1akmquhaDdVleSeNMiy2U9/vosdmKhdUO3KxZK5NsGUISQpUwOHzT9Ver8tNRnrmzr1u/PhATyKanpabZTU+f69ZtiJhvkyq/rNh5BxZ8iAsFhS6snabPnC2ffT5EYJVlpXFDx4oN+4gL5p534HoOgvsBVoVuxZ85kPnmRwYVXGsoCb/5/idZrzGMYKV32cXnG4US2gHLRVjv5SRe54R9GtcLCjWrUAktF/HRrFg3aX2eeckomOBGEvPM8ap8tGLnWPQDirLtO7LmcmuJY/PaT+uezc0mljpsOfwkARIggUQlQKVOol6ZBG5X0tFdxF+zcdgWJk/9XPwbF4Q9xkQSIAESIAESIIGCJ5A0+0d1wdZB0qu1ztaYpBPPl7TJv0jGplXZjjGBBEiABA5FApvVYuPBR5+SpzTIOOJn1FCLmnvUdRqsaZ589hWZM2++wIrAClwe4RVOsDgcSTZscOI7wB3Yo67YMf/+N13w+v6rgUZx1Fjj3HiVo+rXNUHlsW4Lt133h1GeoKwbtC/I89ekfwIKoy+GfKfuorpKz5uvM/Xef/ftctPt9wRVDSsFt3z0yecy8P03jTLmXI3nEU6pg4XU/v1eMiyxEIvg8GDsFrjQqlyxgknqp+7xyqsbLyuwZIJgsbui5tmlSh/Eljn7zFM1ZpCzEFysaDGz+It81lqlWtWqJg2B6OGOCgvXEGsNZXZcbxVd1jmhrrVc2fJ8EwrDyy++QC0jfHJx93Pl/Y8HyTlnnm7qgdsuWBNYufmGazTuyhnm2tm0vPi08UT268K7V2ndsrneIw8GlE9ez/OaD1YpVqz1j93Hp1WMgBviA8GFWazSrm0bvZcrZzutfr3a2dLyOwGu0qAg7nG14+qulcYTatWijyxS60DMD7BkqV7dse5DW1o2bxK2SbjH3EqL0EywWoLidr26G3yj/weBw7h3oUDq3/dFk4Y4YF6UOl7nBChdUC/u43BSpozjng3tty4nYaEEV3Tdu50liA8E5Q7uzXETJsoF558jR+sYhJUSBIodxIDC/V0m0xoptB6rAITrOSux1GHP4ScJkAAJJCoBKnUS9cokcLuSTrowbOvw9G/yP5+EPcZEEiABEiABEiCBxCGQ/O8nsj+MUgctTGp/gaQOfSNxGsuWkAAJkEA+E4DLtouuuF7dE50i3c87yyh2YMWCAOXnX3KNrMtUEKAZ32hskbHjJ4Zt0dJlK8KmI3GVWgBBsCiNuBluqyGkL1q81MTlQb1eBIqmV55/wixqIiD7nff+n8AdVqg00qf9bbD3UCsQLOxCMXTGqacYxRUWQUPdmLnLg3Jm7dr1xj0S3DmFCuJdINYHFnMhr6lLLChZQsXtQuuRB+4OPWz2ocjBa5y6ssPCNKwJrIChex/p6CPSYFUFpc6qTBdWONakUUOjnMO2lZa6iG7FBmG3+/n5iQV4LKa3aNbYWOB8qIoy6wrwux9+DlSN9p2vMYcgcC81Tl2fYQzBYuf0rp3NOApkjnEDC/wN9LoXKeK4pfJy+n1332YUOhhruKboA6yI4L4Li++5lTU6rqxUUmUerrlbrDs9KAHiUeigLIz3+vXquos129s0tkxBCNxG/qxxii656Dy9Bzsby7n66k4QMXgQh8g978DaDexDBRZT0QRzAtyZWSWKOy/GPeKOwaLG7XrSnSen7UhzAmI1IY6Yja0TWk69unVMknWVaY9DWYM5Fi+33HPXbWYX5VqBggfWjXVq17RJQZ9WWbx1W7AlXix1BBXIHRIgARJIMAJU6iTYBUn05iQd3VV8lWqFbWbyf4NEMtLCHmMiCZAACZAACZBA4hDwr50pSfN+lbRGztPB7pYlHXO6pI3/VjI20lrHzYXbJEAChzYBLPTh6Xm8EFz9abXcgfKg8ykdBK6SsKCKwNyV1QXYgoWLY4YxLTPGB0685opL5M13sp6aRxrcm0F2Rwgsbg5mvmEx/c3XXjAWNmgXYn+sXLXanSWw7X7CP9ziNdw+QakDSx4swEZT6qBQu0i7b/+BQB3YgDKlf7+XA1ZM/Qd8HOTCzp0ZC/awUggnUAxBsHiflp6mT/HvELh8+vX30dJbLaiw+Dxc44lgH/LsEw8ba4DX3nxPEGcHLsogf/w5QW65/hrTLyjq5rw036Tbt1M6tjebiPkRzlWXzZcfn3CzBVd/iCFz+SXdzTjDdcTYs9Kpw0l2U93X9QnEG0Ei2h5vrBicP2P2HDnphGONkgaxj3Iaz7gfbLwbKMwQ38oKXIQ1adTA7sb96Y4D1f7EdoH4MbbAppkWbHv2Zlld2GNeP78dmhX3xus5+Z0PrgQRbwqvu26/2cS9gRUbFKxu13gV1LJs7Li/Ym7O3PkL1AKoqblH4OIRCmwruKZ2HOXmHgg3J4z+Y7y0btHMKJ3h2s3ddswVUCZDFi1eYpsT8fOEdm0DsYW++mZoIN/sufPkuGOOlupqpRdqvdWiWRPjwg2ZYemTk0SqI6fzeJwESIAECpJAlmPfgmwF6y40BJKOPzdsW/0bF0rS3GDz/LAZmUgCJEACJEACJJAQBJKnfRGxHUntwn/fRzyBB0iABEigkBKAgsQuStoubN2a9eS+jdewcNFSc/ik44+TY9sebbOaT8S0cFt+YEHxE3VT9sarz5t4F8iE+CNrMy1+TlclSpvWLc25UGJcfEG3gJuihYuyFEYXqsuhzwe+K/fe1dPkxRuCqr+tLpNgmQJFwJ33PpJtATyQWTcQ5N3KM30eDizOIw19h5s5CFwZ2QDtF6lbsFO7dAoKRg6XXQhcjifjIVM1LpAVLNK+qX21QeDh5unHn3+zh7N9ws1bt4uuDvvauWu3yT9yzFhzHGXhSXss+ttrgQVi7OMFV2yQkRqHBfvWugHuq7CgDcE1w8KyleuvuSLghutXVdgdbEF8GMQSgiCeEwQu+KBYtJKc7Ci3sO/zZS3bdO18clgLm9RUR8nmxfpmkst66vGH7xV3fCHEd0pOCn7215cZlwVtSfJntQsKIauMxDG3WKsxjFfcHzkJFHKWSfduZ4vbHdu5Z50WcLP39z//5VRUoTiO+wmKCLfs3OnEgUIamMHyD5Y0kDtvuzHIDSTScN0auizYws0XUMDh3oY8ptfaKudw7r1qfWXFbVH38P13mXkH3K3EOicgfo9te+87bjVzDcrCXNH3lWeNsgf77334KT6MoD7MrchjBcqWh+670+xifLhdxL33gXMuhifmH6sQRkyvPo/cZ87BHIk4QlZircOex08SIAESSEQCwd/WidhCtilhCPgbtxNf1fph25M0Y0jYdCaSAAmQAAmQAAkkJgHftlWSNOsHSWvWLVsDk447S9JGD5KM3TuyHWMCCZAACRxKBB68904Nnl7fBBLfsmWbpGkQ+koVK5ou4in67390Hlx76dU35MN3+hrLjycfud8sQMOFWtmypQMKmSuu62kUENdpnAwsZuMFKxir4Hj3/U8Ei+iIA/FMn4cE5WNh18Z9wQLkC//Lcn955aUXmifpu3TqYBY/EXsCwd6PKOYoMmA5ZGNihF6Tv//9T5545mWjTFq8dJnUq1Nb4Nrpsw/f1iDjOwVKA3e8C8TYsQLrEVsH2pSWlh5w4YY8aPdb735os6vS5NggF0h39LxB8AoVuBFDzIx4BW7rIHPmOcoaLFDDmgrxhNwKEVt+v7ffl37/e9ZYWD107x1yz523mgVju/iLReJPB39lsx/UTyx6X3De2WY8oeKBn30ZVP+f6uIPrs0giFG0bt0G40LLfc3cJ0yYOFlgiYEA8z9+85lal40OiqHizgvF13c6rrufe6aUL1dWvhj4nsACxj0W3flnzJqrfDMM68su7i5dOnc0SoeyatkVSYb98puxEMHY/v7LgaqAPKB99cl5F18T6RR5s//7AoUC+vDuG6+YeylZx7h17wZXXX3fGhDx/MJ0AC7WoJjF2N2sLsUQi6psZmyY2XPnm76jP4i51OvW6829OuCtV036Po2FhLFfRBU/UJxccHkP0/Vw8wXc/f3y60iN24SYVMXl0w/elp27dgWY4kRY6fw0fIQpA3GsOqilFORqtSi0c1escwLO/2DgYLn9lh7men70bl/TVowxqyPEuHJbI112yQXGIgdWelvVJR7GOsYCBIqpl19702zbNyihEScMih8olIcO+VR57g3MXcgHpZZ7boi1DlsXP0mABEggEQlkqcATsXVsU0IRgDuWcOLbtFiSFjrm7+GOM40ESIAESIAESCAxCSTP/DZ8w/SpYH+E7/3wJzCVBEiABPKWQLq63oomOR3HuWlp0ctAnv+mTjcKGiz0I05MFXWvZhQFulj6wiv9AguCWAi/+4FHZYtagECwQIqFRLvIDsuEffudp+pnzJpt8mBxctoMZxsJkyb/K//X51mzkIt9nGsVOus2bDRWNzZoOI7buDBwEWbT/S6rDeSJJFiEt9L7gccFSh60B4K4PbbdcKP2yeAhMuiLb5yD+o4226f7oTiyMXmQYaq6kbv59nuNYsee4HM9WW/Twn2m2waEO5iZlpGZB4okt8ByAdcISiYbUwXxgiBbt251Zw1sI2ZIzzsfMDFpkIhFcKvQWaHu6q67+Y5A4PXASSEbGZLFMeRQ2F0397AZMhM//+q7wPXA2AmN6zP536kyYtRYkxtjpFrVyuaa/Td9pmCsQNx1/fLbqICLNuS31jHpmfeA5WpO1LcBaiExeuwEwxOL7BgPGFuwGLKuuFJTnfsHvPu+PcCMCeStpK7AoNCB5Zm1BAstf7zGQkJbcb1wDtjbIPfua+u+R//UuEHPvvS63kdQAIlRcliFzuo16+TGnr0D197pf873t4chZ5Hk2WfVmi8QAABAAElEQVROcW5Q0Uy11jLxbFSxg2trFTrrN26Sp194NdAWxAF6/a33AhyRr4pa2IEn+jYn0xoNJ4SbL5D+9nsfCeJpYbyAq2WKY7hGvXo/jE0jm8w8dsBsu8dkrHMCCoCiCEpqjAEIXL2hfrR7jLpHfPDRp0y6fZusVli2jRhfVqGDORf3sR1rNj8+weonjUuEMlG2VUan6riHlR/cBbolnjrc53ObBEiABBKJgK9xm06x/UqJo/U7t2cFM4vjdJ6SAAR8FWpIkTvfCduSlD9fV9drWUEdw2ZiIgmQAAmQAAmQQEISONDpfklrcGq2tmVsXCn7+92aLZ0JJEACB4dAiVJl87wi/i+LjBSL4G3btDJWOouXLDWB4N1PeLvPxFPyzdXFGixxoOyB2y/rbsrmg3slxLCBG7BwAtdnxxzdyjxZPl2VKKHn23OguFi0aImxJLJp8X7CGqDhUfWlbp1aGjtnjyxevFSWLFseUFy5y4ULJMRLqaWxforp0/Ww9kHsFbsw785bGLZh5YP4G7D2mDR5SpByIJHbj7HWUmOT7FLriqnTZwksL6IJ3GpBqRDLdYIbNVzjWXPmhR0Ltj6Mn5bNm5nF+SlQhqrFlxfBeFu5ak2ObXeXhfunWZNGRnk4VRUPke4j9zmFcRvXC5YmuN/mzF2QTbnn7pNlkqLWK6tUKYn8sJpzS07zBWLZwKJrzdr1RrEULoYW5kJcM7cVDerIzZwAd3qtdBwj9hfcD0aaW2HJg+teq2Z12aAKLihyrBLX3c9w2zivnlojTp85JyrH3NQRrl6mkQAJFDyB3PxmTqTfxrH2g0qdgh97haIFyV2ulqSTL83WVt+erVL0s0uypTOBBEiABEiABEigcBBIr9xU9nd7PWxjD3zyuKQvnBL2GBNJgATyl0Csf+y8tCaR/rh6aS/zkAAJkAAJkAAJkAAJkEA0Arn5zZxIv41j7Qfdr0UbFTwWIOBvcXJg272RNP9X9y63cyDg1yhWRUv6pFhpnxQprvbBFBIgARIgARIoYAL+dbPFv25W2FZE+v4Pm5mJJEACJEACJEACJEACJEACJEACJEAC+U5Al5gpJBCdgL9uS/GVqxI2U9KC38OmM9EhULyCT8of5ZeytXxSsrJPipYKUeSo88PdmzNk+5oM2bI0XTbMT5cDwRbUREkCJEACJEAC+U4A3+fplZtlqyepeXtJHdoXjvuzHWMCCZAACZAACZAACZAACZAACZAACZDAwSdApc7BZ17oavQ3OT5sm/1rpotvy7Kwxw73xEpN/FKjjSpz6uRgDKc6niPL+8yrSnO/NFFwa2emy8p/0mTbqnwPd3W4Xyb2nwRIgARIIJNA0sLRcuCkOzTKbMj3VkpR8Tc+QdJnjycrEiABEiABEiABEiABEiABEiABEiCBBCAQ8s89AVrEJiQcAX+j48K2KWnJn2HTD+fE0jV80uaqZGlxQXLOCp0IoKDcaXtdijQ9J5ku2iIwYjIJkAAJkEAeEziwWyJ9r0f6HZDHLWBxJEACJEACJEACJEACJEACJEACJEACHghQqeMB0uGcxVe1vvjKhne95l/Kp3bdY6POSUnS9toUKVv7/9m7DsAoqiY8aRBq6J3QexcRQXpRBAUU7KKgYEfFriCoFBUrIEUBG+KPqCiCiCBI70U6ofdeQw3J3f3z7eYlu3d7l7vL5XIhM5rb3XlvX/n27d4x385MYG6rkvXCqfETkVSsemDaM45V9gUBQUAQEAQEAWcEwvctc1Zpx+FVGlrqRSkICAKCgCAgCAgCgoAgIAgIAoKAICAICALBR0DCrwUf8yzVY3il+pbjDT+2mcIunbIsy47KGuxVU4pJmEBLjtxhVKdbJO38x0YHVtoC3XyWaO/hbj2oRmUEpkuVAcP7k8MR+uHpihYuSi/27pc6cN5bvm45zZw7w6STA0FAEBAEQgGBiAMrKdFiIGF5C1B4mWpkPxRnUSoqQUAQEAQEAUFAEBAEBAFBQBAQBAQBQUAQCCYCQuoEE+0s2Fd4hTqWow4/tNZSnx2VtbpEEkKmZaRUaRehpTnYvzz7ETsN6zSk0iVKm+ANCwvLEqROgfwFqEqFKqaxX75yWUgdEyJy4AmBiIgIuq1VB7rlxlto596dNO2vXyn+QrynU6RMEPAfAQ7BFn54HdlL3+DSRhh+Dwip44KLKAQBQUAQEAQEAUFAEBAEBAFBQBAQBASBYCMgpE6wEc9i/YWXq2U54vAj6y312U1ZtX1EhhM6CtPKbSIo4aKDjm2yK1XIbp946EmqVL6SaXxxu+Po6ykTTTo5EAS8QaBHt0eodvXa3lT1uc7+Q/tpzHejfT4vWCeMeG8UFWOPLwgIwg6tb6cezz9EiYlW/hTBGpX0cz0jEH7kP0tSJzy2Ftnol+t56jI3QUAQEAQEAUFAEBAEBAFBQBAQBAQBQSBLICCkTpa4TJkzSOTToRy5XDtPvELhx7e66rOZpmSdcCp7U0RQZ12jUyRdOJJIl06Hbuix3LlyU9tmbQneNEYpV6Y8Tfr1ezFGG0GRfa8QqFezHsWWjvWqrq+VCsYU9PWUoNUvXqR4CqGjOg3n++rOdp01jx2lk60gEEgEwo9utGwurGw1S70oBQFBQBAQBAQBQUAQEAQEAUFAEBAEBAFBILgIZGzMqODORXoLMALhpc1ho1Tz4Se2qd1su42M5rfm2wWfEw1nDqkyh2ILZelyW1cXQgfjhTH6jnZ3hvLQZWyCQEghULhQYcvxIFeTiCCQUQhoL23Yk1yaD8udn8KKmENhulQShSAgCAgCgoAgIAgIAoKAICAICAKCgCAgCGQ4AkLqZDjEWbeDsBIVLQcfflISJVdoFkFRuS3hyXBlkcrhVKxG6N66rZq0cotBu2bt3JZJgSAgCJgR2LpjK11NSDAr+ei32dNcdKIQBAKJgLvveXe/CwLZt7QlCAgCgoAgIAgIAoKAICAICAKCgCAgCAgCnhEIvquB5/FIaQghEFa8nOVowk7tstRnF2WOPGEU2zhzvWXKNoqgE9tCL7dOmZJlyFM4K3gYFClUhE6dOZVdlovMMwAI/L3wby2fjKemKsZWdAnRdvL0SdqyY4un0+jI8SMeyzO78PMJn9IDXR8k3Fvn48/TX//+RSdOncjsYUn/1zkCYad3ExV3zakXXqwc2WnxdT57mZ4gIAgIAoKAICAICAKCgCAgCAgCgoAgENoICKkT2tcnU0cXVqSMZf/hZ/dZ6rOLsmTdzPeSKVA2jGLKhNH5Q6GVW6f7HfekuQxQZ9z3Y9OsJxUEAYXA3EVzCH+e5J477nUhdbbv3k5jvhvt6bSQL1u3aR3hT0QQCCYC4Wf2kc2iw7Ci1r8LLKqKShAQBAQBQUAQEAQEAUFAEBAEBAFBQBAQBDIIgcy3TmfQxKTZdCKQKx+F5YlxbcRhp7BzB1312UhTrHpo3DahMg7jpW9Yp6HxkLbt2kbxFy+YdE1uaGI6lgNBQBAQBASB0ELA3fd8WKFSoTVQGY0gIAgIAoKAICAICAKCgCAgCAgCgoAgkA0REE+dbHjRvZlyWIFiltXC4kM7VJHloAOozJk3jPKXCgtgi/43VagiyCWrd6n9bzM9Zzaq14hy5shpamLWvFlUt2Zdat+8fYo+V3QuqlWtNm2J25yiS+8OQrrVr1WfypetQIeOHKQ1G9ekK8RbVGQUlShWkkoVL0Uli5fUQsY5HA66wATVpm0bCR4gOA62lC9bnmpXr0MleWwHDh+gjTyWoxkUPgwYVK9Sg2pWqUl5cuehfQf30o49O+jQ0UPpmnbpEqW1OcSWjqXjJ4/Tdib+du/fTTZb6KzldE0w+eSY/DGEuWINlShakqKjo7X5HjxygPYylgillllSrEgxqlO9LlWIrUDnzp+jnXt38rWNoytXr2TWkKTfEEPA3Xe9u98GITZ8GY4gIAgIAoKAICAICAKCgCAgCAgCgoAgcF0jIKTOdX15/Z9cWP7ClieHXczeuRzylw4NQgcXJ2/RMIqMJkq6anmpgq7scltXU59JSUm0cv0K2nNgt4nUQaVuHbulm9RB7p6Xn3yFKparRJER5hxHj93/OCUxSbDv4D4aMuI9unzlsmlsVgcgpdq3uJWqVapGIJ7cSfdO3TVCZ9e+XfTpV5/Q6bOn3VUNiP6Rex6lVk1aU55cuSkszHX92ZlcunjpIk2b9QvNmj8rzT5xne66/e6UervYoD9kxGDtGHlbnnn0WSpbqqwLQadOQF+fTfhMI7eULq3trS1vpXvvvJ/y5slL4RZzwPlnz5+lj8YOJ+AabClbKpYGvzYkpdurTG489caT2jEwv/fO+6hF4xaUP19+yhGVg0DnLV+zjEZM/DzlnHx58xGwvbFuIyYES7idpzoBHmzf/fwtLV65SKkst0NeG0pl+HoouXbtGj3xWm91aNqOeX8c5eZ1ouS9T9/h+2+PdojwdFjfmIO7a7Bs7TIa/c0XlJiUqJqQbTZFIOzSSSI7E63h5mcr5cpLlJOfjwlCAGbTpSHTFgQEAUFAEBAEBAFBQBAQBAQBQUAQCAEEhNQJgYsQikMIy1vQclhhlzLWgG3ZaQgp8xR2Napn5vAwnvOHg+8x4jxneHVUrlDFpIY3CwTJ6mGwBwmjpAZ7gISHh5Pdblcqn7YPd+tBHdt0cmucRmMgeiqXr0TjPviK3vvsHY9kwStPvUo31b/J6zHA0F+F5ztm2Fh6+6MBmgeL1yd7WbFo4aI06KV3qRhvPQkM9PmZUOh5by9qdlNzjaDxRGIVLliYchtIq6oVq2mG/r69nqd6Net56korAzHz9gtv07hJY2n+0vke60fnjKb+zw/QiDKPFbkQ62PYG+/T4lWLadTXI9OqHtDy6Jw5TZgAHxB7tdmj7IXe/ZjIiTL1h6cAPHGUgEiZ8NFES9JN1XHe4pr17dVXI1oG8hpyJ/CqMV6vXIypOykYU4AiDEb4+rUaaHmGet73mKkNd+c3bdiUPbNquSWN3J0n+usTgbDLp8mR19VrF78PHELqXJ8XXWYlCAgCgoAgIAgIAoKAICAICAKCgCCQJRAIjeQgWQKqbDbI3PktJxx29ZylPrsoc+YLrZnmzBcaJNPtbTq6ECwz5s5IAWvpmqUp+9iB4bld83YmnS8Hd7S9w6U/d+fDYD/09WEETxx3kj9fqoHeXR0rPcidN559UyOorMr91TVucDONGjI6TULHuf3K5SvT+I8mUMXYis5Fbo+Bz1fDJ3hF6Bgbeez+3gQyz50g9BjGAs8nX6Q5E1PdO93jyykZUrf3g30IZJ8zoWPVWWRkpE+EjrGN6oxPh1YdjKqA7d/X+T7N88pICqXVeAEmqx6866G0qkl5NkAg7Ir1932Ym98H2QASmaIgIAgIAoKAICAICAKCgCAgCAgCgoAgEBIICKkTEpch9AYRFp3HelAJ5qT31pWuX21EjtAgURTCETnUXuZub+WwTkZB+Kb1m9elqKb//XvKvtq5vXVHtZuuLfKAHDp6WAu1dopDodkQMshJQL488bAeTsupyPLwMrd55PhRitsdR6v+W6X9ISwYQro5CzxXAklCwIPp2Z7PWZJWCGl37OQx2rpjq5YzyGquIFoQls4XcQ7HdTUhQZv/Zs57BGyt8geB7OjU7g633fTr85JlCLc9B/bSvCXz6K9//6JlHMLMKo/L3RyezxhGzG0nGVgAcgnrxh/BdTlz7gztO7Rfy3uEeW7avsltnqeHuz3iTzdpnuM8fozrHOfy2cEh97bz2sY6t5IOrW63UosuuyFw7aL1jN39PrCuLVpBQBAQBAQBQUAQEAQEAUFAEBAEBAFBQBAIMAISfi3AgF43zeWwDvETlmhtBLxu5p3GRDh9SUhJKIwH4bwQIsooW+K2GA+1pPAIw4aQYkrgyYE8JBc4t4g/ApJl1Dej6OjxI6bTo5hsQB6SCmUrmPQx7I3TskkrWrh8gUmPg32cuJ6ZC1q1YZVW7m5M7tr21RvFZQAGxaP39CR4zzjLnEVzaMKP401qGO3f7Nuf6juFTQPOrZu2oX+XeQ6PZmwMuXlWrV9J4yd/RRcuma8JPH+GsLeTc+4iZ4xVew1q36CF/VLH2IK8+eCL92nbrm1GNUVwmDyEaEOoMyXo56keT2s5i5QuM7cgQk6cOk77D+/X1nKB/AVoLl8PJYps27ZzGy3iHDlbmAxzJwhx9xZfMyPhAoIM4d6sCC537fiiR/hD5O8BueQsrz3zOucButGktlp/pgpykC0QCLt22XqeUda/D6wri1YQEAQEAUFAEBAEBAFBQBAQBAQBQUAQEAQCjYCQOoFG9HppL9KNC4jt2vUyQ7/mkXQ1tFidpKt+TSOgJ1l5qUyfM92lDxi7u7EHhlG6dexO3079xqjyan/A8P5u89gkJibS60Nfo3c4H03NqjVN7fVgjwgrUufrKRNN9dwdoO0hnw+mCR+b86eUKFrC3Sk+6UFy3WYRims8kzlGEkE1Cg+aYSOHUJ+HnqD2zdsrtbbtdV8vWrD8X0svG1NFPoDXxkdjPnQhc1S9PQf20JyFf3MeI7N3VakSpVQV0/bpR54xHWOcA4YPoINHDpj0OLCx99NQnsN3n08yhTqrWK6SS91gK0DmjPpmJG3attFj18hh9PzbfT3WUYUbtm6g1RtWu+Rwql65hsm7TdVPzxaeOSMnjqDla5e7beaTLz+mSSN+IISQM0rZUrGW18tYR/avcwTcfd97CLt4nSMi0xMEBAFBQBAQBAQBQUAQEAQEAUFAEBAEQgIBCb8WEpchBAfBIaAsxSK0lWW961R5NT60JnY1PvNJpiY3NjWBknAtwdJTYeY/M0z1cNCMQ1z5I/DSSUvGTRrrUgXJ6QsVKOSi90UBL5ZLly+ZTgEZEwi5s31nl7Br8ezJZEXoGPsDKeUcGi46ZzQ1adjEWM3t/mIm3Jy9c5wro46z5MqZy1mleUghL4tR1m5c65EgALGzhokOo8Tks87rZayTkfsgsp54rXeahI4/YwCx4yyxpWOdVek+vnLlikdCBx0A+2Mnj7v0VSCmgItOFNkMAYdruEkgEMY50UQEAUFAEBAEBAFBQBAQBAQBQUAQEAQEAUEg8xBwY7nPvAFJz4JAKCNw6WTmkygp+PBQMns8Ndi7wDkJ+0Y3Xg0gQo6eOJoyfOyAZHEXwstU0Y8D5J5BPhhnqRQADxAQLWbxL/eKuQ2i8mXKO6vIKh+RcyUY5petWeqspkB6uxyw8LJx6ZAVlStUcVH/zV4+acmBwwdMVXLmyGkKUWYqDMKBuxB8gejaea5oE7mUMkvOnDudWV1LvyGNgLvnWgh9D4Y0fjI4QUAQEAQEAUFAEBAEBAFBQBAQBAQBQSBjEMg8K1LGzEdaDRQCbCS2lHBziB7LOtex8vwhe8jM7txBBzkyeTjdOnV3wcMTCfHvsn9d6t9zx70uukApdu7d4dJUOQvixKVSGgp7BnmsFStS3KXnRSsXuuisFCvXrXBRx3IIrUAJQs95I+XLlneptnWHOceSSwVWnDpz0kVdMKagi+56UCQmhlYYS3jXiQgCLgi48chxZNDzz6V/UQgCgoAgIAgIAoKAICAICAKCgCAgCAgCgoAlAtnbQm8JiSg1BJLcGB0jXRO4ZyfEbGzXPr3bToUrZT4finFkpiDRe82qtUxDQKL3HXtciRRV6a/5s+iBLg+YPDDq1aqnigO+Pc7J7Z2lTMkyzirT8V23300NOXF8kYKFKW+efFqukYSEq3T+wnkmHk7R4WOHyYp8MTXi50FBp5BXyEVznvO6eCMHjxx0qZZR43TpyKAoXby04UjfnfzFjy46bxTn4s95Uy2k6pQuUZoeuuthKlGsJOF6RkdHM/nqoIvsqXb2/Bk6duIY2e2Ze++GFGAymJBFwOHu+z7ESMmQBVAGJggIAoKAICAICAKCgCAgCAgCgoAgIAhkEAJC6mQQsFm+2WtXLKfgyJHbUp+dlMe3hgapc3xb5hqG2zZrS5ER5twKCMrzYu9+HpcDyCCjRHHS7VsaNaOlq5cY1QHZP26RK6Ro4aKWbbe5pQ09ck9Pl3ByqJwrOpf2V6JoCapdrbbl+YFQIg+OUZJsScZDj/tWBJYzSeSxgQAVusPX1+btTGhlJfIjP+cAev6xF6hujbquU2UOGHmG8JdR4QZdOxWNIJBOBKLcfN+7+X2Qzt7kdEFAEBAEBAFBQBAQBAQBQUAQEAQEAUFAEPASASF1vAQqu1VzXLloPeWcmZu83HpQwdUe3WinSi0dlDO/mZwI5ihALF05m7l5DTq26eQyZeTXaXpjUxd9Woo723fOEFLHYRGf7prFW+bov0e3HmkNM+jlvpAa8OpxlrCw4HuURTgRfc5j8vb42jU33oLeNhDEesj/8/m7Iylv7jxB7FW6EgQyFgGHm+97t78PMnY40rogIAgIAoKAICAICAKCgCAgCAgCgoAgIAgkIyCkjiwFawQuW4d8cuQqYF0/m2n3r7BT1VvNXirBhODASjc5j4I0iHwclgxhpgIlFTgPC7xUrnKYs0AKQmA5y9HjR02qqhWr0sN3P2zSqYNjJ49pIdeSOMdUoQKF2NOigObJExmZMY/OBCYyonOmhjjMEZVDDSXNbWEOF+cs8RfjnVUZfnz67GkNK2NHX03+0njo1b63Yee8aiyDK737ynuWhM7VhAQ6dPQQh+47x+V5NVzy5c1HIIGcPdYyeIjSvCDgOwLuvu8vB/+54vvg5QxBQBAQBAQBQUAQEAQEAUFAEBAEBAFB4PpFIGMsk9cvXtlmZo4LZy3n6shTxFKf3ZQHV9uoRK1wyl86+N466Dv+iKtXRjCvQdcOdwXUKA0D9x3t76RfZv4c0GkgXJqz7D+836Tqfse9LnM5x3lsBgx/i06cOmGqqw4+HfQZpZWbR9X1ZRvPeXuicxZLOQW45M6Vmy5fuZyic7dTpmRZl6ITFjmFXCoFWHH0xFGqUqGKqdW1G9dyPhnrZ4qpYhY8yMPeORVjK7qMfMofU2jarF9d9FCg/gdvfWhZJkpBICQQCI8gR+5ClkNxXDhjqRelICAICAKCgCAgCAgCgoAgIAgIAoKAICAIBAcBIXWCg3OW68Vx/pTlmB15i1vqs6Nyx9wkurFnVFCnfvGEg3b+k7leOphwi5tbuMwbRuzLnAzeG6lYrhK1atLKVLVds3YBJ3WcyQV0uGf/blO/1StVNx3j4K0P3tA8dFwKMlhx4vQJKlYkldRBdw1q3+BVaLo6Neq4jO7wscMuuoxWHDh8wKWLGlVq0LI1y1z014PitpYdXKaxOW6zW0LHpbIoBIEQRMCRz5UQxzAdF5mc9SHXVwhOTYYkCAgCgoAgIAgIAoKAICAICAKCgCAgCGR5BITUyfKXMGMm4Dh33LJhR142OEdyeKikBMvy7KQ8f9hB22clUfWOwbmN7ElE22YmkUWamKDCXq50OYrJF2Pq8yKTOe68EkwVkw/Cw8OZGGpJ4eyJogThzYoXKU7HA+RdgvBwViHi9h3cp7rUtsZwZ1BcuXolUwgd9A0Spna12thNkXvvvNcrUqd98/Yp56idvQf2qN2gbfce3OvSV/dO9163pE65MuVc5gvPJBFBICsj4MhvHV7Tcdb6t0FWnquMXRAQBAQBQUAQEAQEAUFAEBAEBAFBQBDIaggEP4t2VkMou443MYEc56xDT9kLxGZXVFzmfXi9nXb9GwTPGY62tvGXJIo/mrlh1wDAPUwyOMuaDaudVR6P7XbGbd8ulzpWbbtUYkX1yq7eNc71nn7kGWeV1mdiUqJJ73CYMQ0LS/uxmINzovgrOXK4z5MzZ+Ecl2ZLcl6gyuUru+iNinbN21Gu6FxGFdnsNlqxboVJF4yDHbvjCDmIjFKmZGlqdlMzo+q62XdeT5hYVJRnD758efNfN/OXiVyfCNgLlLWcmON08L3/LAciSkFAEBAEBAFBQBAQBAQBQUAQEAQEAUEgGyOQtvUyG4OT3afuOHnQEgJHoQqW+uyq3L/MRjvmmI3YgcTi2iUHrfsxiU7vtgeyWb/balC7gcu5f8yZ7qJLSzFr3kyXKjfVb+yis1K889K7NPT1YVS0cFGXYiShH97/I6pasapL2ZjvRrvoLjnlq4HnTtMbm7rUg6Jxg5tp4idfUzGLfq1OsMqFU75Meauqmu7gkQP035b/XMoHvzbE7Zi63NaV+jz4hMs5f87706tcPC4nplORcC2BZnHfzvJcr+fpwa4POatdjpGjpvOtXahezXouZaGo2Ll3p8uwOrRyDcmGSsiP9MpTr9Jbfd9yOUcUgkAoIeDue97d74JQGruMRRAQBAQBQUAQEAQEAUFAEBAEBAFBQBC43hEITtyo6x3F63R+juOcUL5KQ5fZ2YtUpogdrh4FLhWzkeLgahtdOu2gardFUO5CqSHF0gvB6V122j7bRlfPm71J0tuuv+c3adiEoiLNXggXLl2gQ0cP+dwkcqw817MvRUamPoZAqNSrWZ82bHUlNpw7QL6cL4aMpouXL9KxE8fo4qWLVKpEKSpcsAhFRkQ4V6ft7EFiNc6TnMcmb24zUfn84y9S3Rr1aHPcJs0QX6NyTarG3kFFChZ2adeT4sjxIy7F8Kj5cfQUir9wnuDl0XfAc6Y6o74eSV99NJ4iOFG5Euy/2Lsf3XPHvbT3wF5tzghXV75sBUuCCSHkJk/7QZ0e9O3k336g9i3am7yHEGqva4eu1I71u5gIOcAE1uGjhzV8ixctTkUKFWGPpCpUIL8e2g+5eTZsfTnoY/e1w43bNrqcgmszeugYmvbXNEpMvEaxHLIQ3lbVKlczXVeXE0UhCIQIAvYiVSxHov0usCwRpSAgCAgCgoAgIAgIAoKAICAICAKCgCAgCAQLgVRrarB6lH6yDAL2o7sp1aycOmxH0bRDX6XWzj57Z/bYacWXdqrUMoLKNbVCznssrsY7aN9SGx1eFxreOWrkd7bvonZTtqt9DL2WciLvbNu1jepUr2NU0d0d7/aK1MFJYUwU5MuTj/JVyGdqw/ngakICjfp6hLNaO/56ytc0+NXBpjIQEG1uaaP9mQp8PEBot/iLFyh/XvP4QDrB8G8lIMl+++s36t6pu0uxuzxBzhXH//gVOYeVc66Tkcfoe+L/JtBzvfq6dJOXPXHq16qv/bkUZkHFUSbukKepfNnyptHDi+zJh5806eRAEMgSCOTITW49dfh3gYggIAgIAoKAICAICAKCgCAgCAgCgoAgIAhkLgISfi1z8Q/p3h1HXMMKYcD2YkzqREaH9Ngza3AO5mCQY2fJyETau8RGV8755mFz/rCD4mYn0dJRiSFH6CCsWaVyFV2gnf6376HXVCN/zPlD7aZsq1WqThEWnjaqwrGTx2iHRcgrVe68Rf2n33ySTp4+6VykHcft3k7zl823LLNSgqTZsHWDVZGlbuL/xlvqPSmnzviJPp/wGSUlJXmq5lJ2mT103v7obVqyaolLWbAVi1YuomGjhhK8hq53GTJysNfXCoTX4lWLva5/vWMn8ws9BOzFa1sOynH2ODkunLEsE6UgIAgIAoKAICAICAKCgCAgCAgCgoAgIAgEDwEhdYKHdZbryXH6CDniT1uO216yrqVelDoCCRcctGehjZaNTqTV3yTSznk2OrbZTucPOejyGYcWTu3SKQed3W+nw+s5xNqsJFo+NpHWfJtIh9aGlneOuqYN6zbUPGPUMbYgOOCp4K8gzBpysBgFXjIIfabEmRTYtH0TDfjwLY30uHj5kqrmsr2WmEhLVy+l59/uS5c81MOJ474fS19PmejR0I5Qab/N/o16v/IYzVlkDj+YmHTNpX+lWL52OY3l9jEeo8C4D68cd4LwdH1e6017ONyaN7Ju0zp6/OVeBJLKk8RfjPdU7FPZlQTPhA3yAz3GY1qxboXXnkO4psAXhFB6BSH5vJELvI7TI/EX4unZ/s/Qrn27PDaDcHwvv/cSe42N5HvHfB3QhpVcTbhqUtvs7p8PvpKApoadDuLjzztp5DC7IODu+91xcFt2gUDmKQgIAoKAICAICAKCgCAgCAgCgoAgIAiENAJh1W9o5ZsrgR/TuRh/1o+z5JRQQCDqnlcpvE5Ll6FEbvqVIld86aIXhSAQbASioqKoVtVaVJvDuBWMKUjbdm4lhIQ774dRGt5IIK+qVqxGZUqW0cigLTs209qNa+n0WWuC05f5xnC+mKoVqtLJMye1kF3engvPpRqVa1BNnidyCUXnjNYIoZ3ssbSZSS6QCZkZbs2beSBUXgXOAVSdcxNVLFeJ8S1LIPDOxZ/T8Dh+8jhtidtMu/dn7fBOCMNWp3pdqlS+Eofdy8+k3B7Nswtzs3sgZLzBUOoIAsFA4Npdo8kqp07SjNFkW/1XMIYgfQgCgkAyAnnzFww4FvLvsoBDKg1mUwSKFyuqzfzkqdNp/saLicnPv99zmpC6fOUKXbjg3QtIphPlQBAQBASBNBDIkzs35c2bx6XW8RPW0UuMFYsWLaL9O92oO3c+nhI4pH1GCuw6hQoWcOnizNlznKfW/IKsSyVRZHsE0vObOZR+G/s6DyF1sv3S9wxARMPbKLKLa16MsHMHKOfPvT2fLKWCgCAgCAgCgoAgkGUQcOQrTgn3T7Ic77WRT5Lj1GHLMlEKAoJAxiDg6z/svBlFKP3D1ZvxBqQOG7foKhuj7DbvmuOXZ7S6oWREiowiiuB0uGl4SadMMDoX0TX2JPd2zmxM4zeEiN3WU5rwuMMv5hBwysgwu76OyeOA3RSGcx7UKODqm7GyRbMm9MbL+r+R+70+kOJ2ePbW/n3qd5QD8zHI9h276aXX3zZo0tjNkUO/Rt6uS3/m5uu6QchszOuq2bM8jZl4Lo7mdYXrgfXojfh6b3jTpq9Ye9Nmeutk1ph87RfPhlz8/GHS0utraIVNRqwtq36cdSH83HEeqqfj1/n51JKfU87S8a4HnVWm4zKlS9FXX3xs0uHghym/0I8/TXPR+6oAcVO0SGE6dvyECxl+S5ObqP9rL7o0+fGIMTR/QeDCy4OQ94aMdxlIFlUgn3OrFrfQhs1b6eTJU1l0FmkPOz2/mUPpt7Gv8+BfMCKCgHsE7LvXWxY6CsSSo3BFCju9x7JclIKAICAICAKCgCCQtRCwx95sOWDHyYNC6FgiI0pBQBAIGgIwtNXm8LxVaxLHAyZavYxot3X+T21MZcoSNeRnWlH2pgAZAgHJsSuOaPG/+rHxM29eouatiYqXYLKCDZIQGJUP7idaOI/IKpQvxlGxKlH9G9iwzQb3/fzvouWL9XPT+sybj6hTF54LG8XXriTaud31DIy7SXOicuWI2Ps3RU6e4H4WER09kqLSdmLLEzViI17hIjpGUCLc65KFPLa9WhXTRzhHYr+pKbfPOTMLJL8djdC12znc5toV1gbZEqWJmvKYijCumL+NSaCjR4kW/UPsdmJq3nQAfBs2JmJPeE1m/qaPzVSJD/wZk3Mb5Xk+N/MYIbM59+e5c/q+8bMWhxKvXT913lcuEx3Yx2tjgT4nY12L/chIvm4+CDzWlaiQzCdOpv3GPBVgb72mLYhKlNLJEzQCUmfdKqIN66yvka9z83XdwOOo8S1EwDkXE6YQ3CtYw8t4/TuFDtYrpPGZmz0KWrbl9cHrSxnVT7PxceVSokMHXE+Gwb9eQ/7jew+kAwRj2Id7cJHntajXdv30FWs8L0rHurajNHNmEp3hSA+leM234Ll5Kxi/ul99HZPqw9fnpTpPbf3pt1IVvu94XRifVad4jS/j54/zs0r147z1d22l51mMMYTac6duA6Ka/IxKSzx855xnzxr1rInk75Lw8NRnkKdmjbmN7XYHJeEZz3Ly1Bm3p1UoH0sfDhmo9fHHn3/T95OnutS9/5676N67O1N0dKrH4vn4C/TlxO9pwSK+z1muXLmaMuZwviYgI9wJvInGjviQjh47QX1felOrBuLiuaceo5Wr1tFHn482nRpbtgwNfPNlKl68KEVw23hcxPN31qQff6ZZf/P3lxvJnz8fPdX7UbrpRr4mLL/8NpOm/MzfX24E437g3rvojttv5UdZJMXt3E39Bw1zUzs46g+HDqQa1aowieagu+7vKV5PwYE9aL0IqRM0qLNmR45zJ8hxeCeFleYvaSexlW9GkULqOKEih4KAICAICAKCQNZEwFae/zFuIfadayy0ohIEBAFBIAgIwHOgPhtvYeQyGMYpX4z7zmvW0QkaYw1YcGD8RZkzqQMi5867U8kfnIf66C+2PFEeJiSMpA48IGrUYpLiplSjNs4pWgyfaQvave0OJhQK6XXhSeQs0GFMqg7KYVwD0YN+YPw2Gkrb3U4Eo6qzwMDa4U6iv9nADIO3EozhdiaVQH5B1Hwx14aNdHJr1u+6Xq/B7VclatdBHelbjAdt3NeD6PdfiE6dMJcX5DmCaKpQyayHN4ZTfkENb1/HZG6VCLhhjBgXRG31I/3zlpY6QWjUgZyoVlOfN+bhDzFhbM/N/oZNW+nNgUPclDqprdYxqsBgD1IF13bJv+aTfJ2br+sGa69Ld1dcsZ6q1iAqVpLo50nsIcb3j7eCNdL13lRyBuehPRCHnbrqpOr2LebWOnQ2r10QXbi/sc5iyxH9+C3fs5fN53g68gfr4jzXGA/PITy7ICAqPNXTa6V+qvP8GZM/z8vUnvU9f/oFwQZCRwlIUtxTuIadeb38yc8SK3JO1cfWn7WV3mcx+vXnWYjzjBLo504+vre9WTMg39zIuAnfEf4gzz/Thzq0b+2mpnv1+G9+oOkzPYdeBokx7N3+lDcPX2+WIoWTv9cMzb78wtPUtlUy0c56kE3wXIxhwuS1fs8SQsX9OXsurftvIz8KHtXOrFunJn3w3gBDK+bderVrUm72CDOGtbyhfh1NFx4RbqqMtoYMetNEEuGyo3+QQKVKFqcJ3042nQNvnr5P96YG9epoS0QVlmVPJivJyfd5754PUfu2LU1emWVKWddHGz98PYZAGoH02bSFX6bIIMmF71sWvFyQM2cOIXUyCOfMajb5105mdS/9ZgUEbHErKdKK1KnYgiLXfp8VpiBjFAQEAUFAEBAEBAEPCDjylyJ7KX5r2ULs21dZaEUlCAgCgkAQEHikt250Q1fKUOipWxgG8QY95MRxovmzic6f149heMNb80aB0V8ROght9vcsomNHdDIDxvPSTFhccvJA6cgGZeghIEMgsBB5Kw1u1I2dnurD2K4IHXgK7d6he2ign+JsJEq6Zj67eHG9fNUyNp7u1w3aFSrr3g84p1kbM6kDLyZF6Czgt5R3sEEJb0XfyHp4P6AMBvI9u/R+gEXb2/R94Dl7Br/mfJbHwoZtGN6BY1smU34y/NuwMBt0uz+QOk7gq7ygUrWpe76OKfXM1L32jJsVkaNqwNMIHl8QYLpske7BVaU6e1MwRsD8xsZESxfqdTLzE2sZAo8cjPXsmWSvDx4nyLdaTFCuW5lKXvgzN1/XDYzNwPfAXh7Xen59/4RufG7cTF8z8Pji/KG0O04fuzefrdrrhAzuJRCJRw4xYcVkya0dda8zePDsZUJShR7EvafW7lK+fls26PchyFmQQxgfxvPvHG961+v4ijXOYuOoJiuW6vecfpT6qZ47mM/vU1P12LullU5iwItsgdM4z7MO4s+YfH1e6j2ZP33tF3grQmfPTp2Eg1ck1kqnu/T10epWtmBPMPfjfOTP2krvsxhjCMXnzvo1RHH8TLaSKtX0lxxQ5slb1ercDNC9yqQMyBF3AuJFETr7DhyitwYOpXN8b1SpXJHef6+/RsL07vWwRuq4a8NKX509TyAHDx9JKS5fTvec27nL8AIDl77+8vMaoYNHzKhxE+ifeQupUqUK9N7br1M+zjt0d5dONJ09jIyhyb4eNyLla12RUCkdWeyMHTmcSiTnWfP2Z0EBJu7gQQWiJSPl3WEf073dOtOadRvo4sVLGdmVtJ0JCIRnQp/SZRZDwL51ueWIEYLNnQHI8gRRCgKCgCAgCAgCgkBIImCrzEYiC4HHrn3fJosSUQkCgoAgEAQEkBNm726iGb8Sfc9GweRQMG57BikBgbF0OhtSlWEVOry5v4uN40aBx40iAaZOZu+Xw6lEDd7+h3eL8xv//GayThj9TTRhjB6izdimp328uQ7PFU8CzwV44kDgYQMvBYwFAmvRMR4jwhoZZQX/e+278USb2cCNucOoGrc1lZTBW9RqnjivXHn97LOn9XpoF/l0VizhuR3Ty0BuKKnOOCniavrP3AcTDPDGgLfQX0zwQGDQL1FS38cnG6u064UxTZmke/Kklrru+Tom5xZA1oDYUBY153Ic38CEGgTrYh5fP1xbzHvbZiYp1uplCF+GPC2ZLTDYTpqoXxOQJxjngX1Ea5jIUVKkmNrzb26+rhsQS9N+0q85yIpEXmdYiyAelYAo8lYKFWbvnuT6c2cxOXJQX1fxvIb/YI8pdS1r8zVRoohZPAsUoYOy47xu8ayAFGOCxxfxFWu0nSNa7wHeaQgV5/yXlHzPIj8Qxmb8U55g166a9aij8hP5MyZfn5e4RyOZlDGKr/2q64c2VvPaxLMHciE+9Z7C88e5HxDFRvFnbaXnWaz6DsXnDjxDsa6c/67y86pOfX3ke3fpZLyaRyZsG3FIMqu8PcahNL25UcrhmC+/1ggdKEC8/DOfSVmWnDmiNG8d7SCND+TkwV+F8uW0mnv37teOoSvGIdkgyHGGY0j9urWpYEx+bf/bH/5Hs+fM55ByNq3Ok8+9kvKIeaJXD62O+nDws+fAocP06agvNe8hhC7zJBHsNXaWQ9797+ff6e4HenFYuOOeqmvj8zYknqeGcuWK1giyJo1vJORDspLjJ07SqLETaflKJgvTEISpu6F+Xa+vRxrNSXEQEHB6ggehR+kiyyHgOLGf7Ae3U3jZ6i5jt1VpT+FH/nPRi0IQEAQEAUFAEBAEsg4CtirtLAdr37zYUi9KQUAQEASCgsDEsamG3bQ6hCdObHm91poV3oWAqp9scAIB4kzeuOtvxjTvx2RsA6QKQqFBQNSAAFHeOLpW/0QoIwjy2xhDpula6093nhFoAwLjOAy+EJAzIJcgO+P0rfFzCxP5MIoXZIM76uJchCaDgFCCx5RRNOM+G7BhREMIrmNH9VIQEcbr5yFUkF9jMo4BOCL0GGQ+kzXwGnIWzKVsOV27g6+3IgxUvW2sA/ZavdhUgkCVB3vrLlzVxQupI1HkgL9z82XdoFcY3q0ExmYlipRQx562VZPtC7gWKo+Mqg9y4OA+vqcr8PrjdYXcUxA1Z4Tewp+R6FXYXL2k1/X20xesVZvJRmO6xqRNRog/YzLeb2mNCXmM7n9EJ1sQHu0wE2oQX/tV1wPnGvKl4FB7hmGL68uG9BTp9oD+DAJBqa6rP2vL32exGgjuG1+fhepcbIP53AEBh7CDGDO81v6daxxJ0PfhgfPmKy9o/W6L20Vly5RKCcFmHMyp06nPjMJOodkOH9G/K7A8VP4f47nO+yAwfv3xa5Ma+WvwZ5SPhg3SDjve9aDmhYMD9PH7jNnGahrBdOToMSpdqgTd0KCOqczX3DO9n+nnVVgzhFv7/KMhVCB//pT+3h3wOr8jYdeOx43/TvNaGjzwDapfr7aWb2jrth306MP3sVdRXm3MD/V6ml587klq0awJ33JmLx8QS5P/94spTxDC3yHf0KVLl/mWf0Lrp1DBAvTdhC8YFwe98Ep/eun5p6kiE2VYXkrmMun22ahx6lC2IYqAeOqE6IUJtWHZNy60HJKtanty5OYf0SKCgCAgCAgCgoAgkCURsFVoTgi/ZiW2Tdbf/1Z1RScICAKCQMARgCXGWwEJoQThoZQYrRRKhy2MZHiDHGIkT9zV12u6kgFKn9a2WUs9bBYbVmiJh2erMjL6MiZ3fZerqJeokHLO9axyE51hrwMlbETShEPUaGIIdaMr+BPXSHn3IISSUXy5fuo8b8ek6uNtfUWWwctAhYxT5WqbJ3kOOD7CHkbOcv5s6rV1nodz3cw8Lp98TTXcj+sjCfTc0lo3zvMvwySYEhAx3gryAkFAANp1o6auSP48wiQiBIStEuWNg/sUIQGRswYCT5CKVfX9XbwOAiFWWKNd9K2eEwlMPhmPA9GvpzbcjQnn+HK/lSqtE7EYu2rTn37PsLef8s5BmDUjUV29pt4i1oQaG3IfqWdc5WqeetTLPK0t1WbaraRdI9SfO/BCLax7otCff6R6b6Y9swypMZhz1IBQSLiWSG+/+77bPv7buFnzjEGFvk8/TrVrpr4o3rFDe+08ECuJyhvVbUtwVrQRSCIjUaSOQWZAsCSgO8beKRCV4+dcfLxlH1u379Dq5cyR/BzRjgBvYvKedxtv60dymFOEaYs2EKC4BSP4ewx/MTH5tA7z8DMdxyBjgFv+fHm1x0xksscbPKAUoXP1agIlwpOTBV5Jzz75GJUoXkw7xodqC3l/lCAXEtrHeEZ/9gFVqlCObPziB66nknatW/DjNfUcpZdtaCHA33wigkDaCNg2zKfI2/vwP35ceUBbjU6cW2dS2o1IDUFAEBAEBAFBQBAIOQTwPW4lCLvmOLrHqkh0goAgIAiEHgJ5dWOI9uY+8m7UZ6+LQmwEwxv1CLd1+IAeykoZa3InExaYCQwlt/KzEEmNkfcF3iggQlYu53PZ2J9eiS1PhBBmkNlskDN6F+ja1E9FoCB8E7xPylfgHCNs/MY5R48SbVqvh+FKPcN6r1KV1GTbcewZpARWL3jwIC8L6iyen2pwRZ0r/Ba4kjzc7wX2DFG5cIweGaoOtsrLSZFAxjJv9v0Zk2q38S36PPH2+sJ/lNZ1q9YHStzN4yq3gQTvxrquLWWeJh+vcSSyh4DAUkZt43jTOzd360bv1fUTFsmmLXQ97ptTujHVtaKFBvOBAHcrUWsRXm7I+QRvD4T1QpjAm5uRlvcpthx7e6zitcyEDkhahFB0l4/Eqg93OndYo77y0sH+vQ+mhjbEfQVScTU/N9RzBnUCJZ7G5GsfB/bpYSRhzIbnoCdJq1+EiezYWb8P73uYaOsm/XohrxfWBHJXKQEBhP7K8nMNOaE8SXrWlqd2VVlWee4grGSDRvqosbZOJpO5ah5B3na+owPVqMbXluWDj0fSZXWfuhkHQn+98EwfLX/O8KEDacu2OP7KjaRyZUtrHjqfjvTOGwTEySO9n9NCg/08eYJ2Lo4hLZs3pddfek4LewavGSUxyaHXrly2fsacOat/vyMUGjyBrly5qk7NkO2Zs+cIHkS1alQj5VE0cPCHWr4bqw5BvJw9d17z3tm5ey//lNHJq/Hf/KDlAAKWilDqeFs7eu6pxzTyp/ktjennaTOsmnTRnT5zlj7/4ktau36jVnb/PXfRIw/eo7XTudNt3A7/ZhEJWQT421FEEPACAXZhtq2bSxE33uZS2VazC0Wu/x+/XZPkUiYKQUAQEAQEAUFAEAhdBOwl63F+vAaphiHDUO38vS8iCAgCgkCWQUC9aQ0D8G13mIfNCYkpho3heDv99585fA2HTFJeAqjpHK4Lhn0YJEHG/DZVz5dhbtH7o+hoona36/URbgh5EtwJjJiKQFFh2FRdzAsJ4vGH5Opx21WJ6xaeJmpOB/e71v1vjZ6sHW/NP/60HobtHBu3ChclqszGcaOgjhLgZiUqBBOH5PFbfBmT6gS5h+rydxhk9izdkG7xEqJWjmuqRHkWqGO1xVvKmAJCU4WaINRYxy6p4ZcWMRmnJFBzS2vdqP6M2+atiZRn0yw2/imiyVjH3b7ywElrXeF8rK2LTJpAtrDxESHZ4J2H++KmproexCdyJYEQTY94wtq5XdTFnHHvgihFzpOq1YgmfxtYYseXMTmP0eoY98BvP1mVmHXe9IsQjCDSFOGotmgJYcKMuc2gM+ZgwrE7Sc/actemsz7UnzvwlLj9Tn3UJ44TredndyZKcfYy6dOLiTuWpctX08rVa9Mczdx5C+jWti01IgOVQWgomfjtZNoWt0MderWtWUP/joqP5xcOkqVaVf6+ZnHOZZML378sV9yEhVQkCeqULFGc9nCOnlCSi+zZi3BrzgJMnWX23Pn0zBO9+D38MJOnjnM95+Mff5qWQuig7NffZ1KPB3RSp3KlCs7V5TjEEOBvQBFBwDsEbGtnW5I6juj8lFTnborcwP/gEREEBAFBQBAILQT435kQB/8nIgg4I5BUqysvDruzmhznT5Htv3kuelEIAoKAtwiwgU8kuAhEGCIKbFjHb4Nv1o2JSMZegwmdWvyH0EAgAVav0MOvqREipNMGNk4hDFR0LqIqbHSCBwgMxi3aemf8VG05b9verr/Zj0Tq61Y5l7o/Pntaf+MfyeMhIJlu4jHBGwFhjnbEWRvQQcJ06a4bmWG4nTdbP9/4qeXNKanPE3NUXkSoA8M4dJBL8WbDNCe0thRF/Fzl/vwVX8aEPtCnCru2cb2e78dT34p4Qp2o5Pk5149Mnp8bA6Bz9aAet+VrrkJbaeGXDFgHYm7erBvnCddg77MatXXtiqWuhGWT5vp1cj7vwD49ZxGHDSLmQUyeL8a6Ro8YRfyAjO3K6xsECrxycE/VqqeHEMO6fagXE3z8hjr6gGB8xYrr+8bP+PPuDeSesEYbuK++/Yq3PH5FYgG/2jyORk10YhZzNxJvxr792U9rTP606c05afULDyp4OYIAh1cOclohfF+tuvozCETkutX6s8yb/lSdtNaWqpfebag/d9rcpt9DeC7DI0qtt/TO28/zPxjytha268LFS/T+xyPSbAXeL6M+eZ9KlSyuedaMm/AdtWzWlOrWrqnxoE/36UkN6tel94Z9nGZbqkL1qlW03aPH+Ps6WcqXK6vt7dlnJmUS+F5FqLOcTrln1HnIU6Pk5Cn+zg0xQc4bdwKCDeHZkM/oHHvzYO5JvE5yhEe5O8UrPTx/0E5Ucqg3nFSqZAl6+YVnLM+f9ONUQpg9kcxBwM2vmcwZjPQa2gg4Du8ke9wqCq92k8tAbXXvocgt0/mtGP5hIyIICAKCgCCQ+Qjwb8AUIsf978HMH6eMINMQsJeqT7ayHM5BJc82jCRp5Z+Z/e9Gw2hkVxDIigg4SKN18Pa2SHAQUEnSYfxCeCYlIFOW/Mv5NirpobXKlNNJncsXVQ2iVcuI4KkCuXyJCR4mhYoU071WYBCG94dVzg/9DPefMCjDswaCt9Nz6G8Na8dhbAyFwHAN7xx4F2DsCNsE3VY2kuzdo9fB5w5+Ez6Mv9BB6ECKl3IlMmBgvbObbvCGMWjmNN0rST8j9RNlML5u+o8Ioeryx3BYKzZy401wtHF7Z70uPCNQV40pmgklKwERBjFiqmu8//RlTGi1VXvd2ImwayDklIeTkdxDaCnor3FIHTZCpgjG6+w9gMJoNsxD1FrSjzL/E+HNKlbRx7FkgWv4pfTOzdt1Y0SiXAWd8IQO6xTXwFlApCqC0FgGHYhUhPZDfhV3Hl5qXak1iDZuZFsECJ3jR4lm8PpG2aGDvIYL8Lq9Qye+4Bn3zTi9DPliYssbe9f3QUBYeT2khbVqyUikQQeiB+RF8ZJ6f2VjVc30b70dU/p7MrfgTb/lK+nzxbPrpx/4ecP3Gq4HwoQ1b6M/Q2/g35o74/gZe8bcvrsjb9aWu3N91Yfyc6cmf3+otTv3L/27ydf5BbD+U70fpeJFi2gtjhwzngoX5pckkgXeIZDcfC8X5TqX+JmEsGx3de6oETp2u4N6P91Py3cze858KsDes0PeeYMqli9HNze6Qcu1s3nr9uTWrDfoHx45IBgglSqWo8+GD9b2K5TX77e2rZprhNF3P/ykEQ3x/IyJyZ9PG5dW0emjQIH8mgbL4MIFw28Cp3qhdvjE449Ql04dNGIsGGMrXKhgSsg95/7q1a0lr52uuAAAQABJREFUpI4zKEE85m9TEUHAewRsK2ZYkjqO6BhKqv8ARa751vvGpKYgIAgIAoJAhiGgETr8AxXi6S0fvYZ8ZkcEEmvfTQ78I9xZOB6/beVM3RjiXCbHgoAgkDYCyTwOaJ0wWAqE2Ekbs0DUgIEYAoMxSBEQEUbZt5ff2mfPgoLJhigVygl1ChRMJXXUOTA6q1BkCO3mT24dYxiiu+9TLZu3DRsT4Q+h2dby3yUmHwqwgRpjchbMQUmRwmZSB0Y1kDEqCTkIHXgeeRLkZnDOz9DoZv0MGKmxfiGXGFt4iai2dW3qZ2EmwCDId5Je8XZMFdiYDAFp06O3vu/82bmbrvl9qjnXC+Zx/Ji5NkKBKQLiYgDmYW7d/6P6N+ohvdACDOUIPeYsuD5KfJ2bP+sGOT6UlxRCb839U/Vu3sJ7x+hto0rVulQ4I/eVlYBYhRjzYcBjDQLSU61PHMefI5ozi3PcPKz3WZJJzyOHibZxPeTHcharPD7eYO3cjvMx1m9seXN4R+c6vhwHYky+9KfqetsvchlB9vDz0kh04fmhee0w+Yc1gJB0q3j9piXerq202vG1PNSeOzH8/G/WWp/Fti1E+w3Pfl/nFqD6yFujpP9rL6pd07bpzY0If0uWr6Jhwz/Xct2gwt79BzRCR1U+x6R6v9cG0rQp32ieP93uupPSInVuurGBKawYCKRqVZK/B5IbLlSwAOGvTq0aGtGAfDFlS5cilVtH9a+2FSuU13avOf9eUBVCcFu3Tk3qynmNIOc5BN2SZSvp8JGjmsfObe3aaOHXAj3sA4cO07+Lllk2uy45F49loSgzHAH+xSsiCHiPgH33erLvXEvhVRq6nJTU4EGK2DGHwuItfjS51BaFICAICAKCQIYhkGyDQfsaoZN8bFBnWNfScNZAwF6xJdmLs3HTgtRJWj6dHIlXssZEZJSCQCgiwA9bpnM0rwqN2AnFMV6PYzpzKnVWyLex2cn4XaqMXo6wS5AkJrXxtj7ykYDs2cfeBkaBUViJv2TF5Yv8Bj8TQlaiQpbBMA2DkjKIwrgIUqcqz2HZQvYQMnx7lzCM6WyyZxHaBnGIPELwuoHMmq4btPUj7z8RfqzeDXr9zf+lngcyqX4hDqtUgb2WIsweniCfEBIO4oyhrk3fp7sxwUNHeTs596CwBa7A12bT/06d1ImpKtWZHNlkPkt5wkB7mImKUBCE82qcbEhdu1r3BLEaF+bnz9z8WTfF+S35znfrozjGpMmfvNaM5IpxfJs3GI9c97FekIMGRn+0ayTaMLaKyQTOgeR7MypHaji3RIsIIcrbDj3F8D0EUgd9eLMuvcXadRZmjQqRB1IjvRKoMfk6Dl/6LZJMyCVazBfr4ix758Db0Yqkdh6XL2vL+dxAH2fmcweeoZ266M91EJ/wzgsBucjeN3nUs95pPJHw9mPRHrccAeBC8ksWRQrz9wbLVYuQlgjzFc/eMQVj8pvIGu0Ei48B735A+fLlpQ85BFwOfma89/4ndJbDjmEfOht7077y5jvamUeO6qT9vwuXUv06tTTiqEWzJrRoSSqxGM44q5wxu/fwd1wmSZiPL/60an5LykhfeXMQEzqpLyi0btFMCzeXUiFAO8g99NFnXwSoNWkmkAgIqRNINLNJW7Ylv1qSOph+0o09KWr+sGyChExTEBAEBIEQRoD/HaUInfDIcHJcy0WOBDa62EP4qz88icJzXqawHMEjFICLPbvhwga5pKZ9KDy3HnLAuIodl86TY9WfHHs6N0VxctZwGO+yoVyMP8s51NlQKSII+ICAnQ0ZiZz3IZGNjdjXiB3tfH4gg+QRyVgEEDLr0AEOdxar5545yi+aIfQaPBGQ44HDvWhy+GDqOP7jkFHIfRHLZAW8cnbt0Mvg7QBSBYLcNnw9/ZK/Zrg/7T72KIABeOVSPdybqokwVsjpA0N3szZEi+frljKEnGpwo14LlrNTJ/R9GITa8lu7mANk5m9MShjmqGvNn5hfzmj2YDiaSu7DUwXEELxV0D7y1CjZxPv1+aU+jQToqOd2ANkE8qQDewdBQLJ4YzzXa7t++jqmb8e7tgENDKJ9ntPLpv+srwFVE/lXkP8DIbJAJiD8HKQwG6YRagqyd5dO9ulHmfcJovGWlnr/yovL02h8nZu/66bLPfo6QD6bmb/7f29gLvDygdEaeXLa89r7ZTJbf3kdYWytb9XvAdT7bx0+mfxk4gDrDN5Zjfm+hceP8rjDur0x2csMdY/y2vZWfMU6trx+j6B/5fGDhPYgBivxHwTrKD3i65h87Qv5MhDCEPf9gnm6pxPa8LVfhGzMx9cPoSZ3btefKWgH17A8P5NUPiMQbEpuuIkInnYIe3lwv67F/R/ItaX6Mm7d9evLszAYz52mfN8DUwhIU3+/f/QWAvbZ59mX3LY19YcJlJcJn3kLFtGnI8el1Dt+/CQh303N6tXISKqAULm7SyeN0EHlrdt47aQhIGrCj4drJA7Cua1Yxd+VLLVrVte2IIjidpjvu7nzFtDTHLYNeXX69X1KKz9+4iR/TYTTiI+HamQPTv7q60laG8H6MBIxTRo3otVrk7+LvBhAZGRESq2wMP6+S5Z2bVryVzJ/J4tkKwT4SS4iCPiGgH3vRrJvWkThdZJ/+BpOt1VqReH7llDEnkUGrewKAoKAICAIBBOBlFw6yZ1qxMWV5H8cBHMgvvbFhJOdx4mfp8EgdrIrLkmNHidHvhKWV8e26CeK4n+I58Sb6yKCgB8IZGdCDCSouncSEtgDhG3ewuX4sYjSc8pKNhLCWwWESPcHOaTaOfYiyZNqHIZBeM2K1B4QwglkBZ55IEZubqaXgUBRMm+O2gvOFkQUcpRUqKgbWEE2qfBnagTLF7NxO9lTAUZTZUhG+R13qVrmLTw5fv2frkMIK+S5gIC0CmezgCK9oFvwT2r7OL7M6xmkAYyiII96P6fnnVGGR9RZ9K9u6Ma+P+LrmPzpA6QTcrGA1AGJAxIgMSnV0wiePSuX+9Ny4M9p0Ta1zRsb81j5z0q+HKlrfZ2bP+umWSvdWI8eS5Zm8uxZvW/nT2+IRXXOwvnsldBVvwaP9tFJnjz5UvvZsjGVcMA5y5cyGdFOX68PPcb3xkV9fRYtplrUCRVv87fgLF+xxjMD84dgzeD+MN4/eM5gnOkRX8fka18gptRzo3Zd9gpMtt/42i9CRsKjCiQOSBngcfoEUVF+LoFog+AaxW3V90HIqRCPTZmY+ymZ1MmItaX36LnfUHzuIBeVkvt6qD3XLe75v2e66kNI8+XE72nYu/215fHGy33p1X7P0tmz5zhEWsGUMGFXE67Rd5OnejXqCuVitXqXcM8lS61kUgcEkpVM/O5HevbJXpQzRxR98+UI9hpK4JfXcmpjQv1NW7bTjp27rU7NMB3Cz506fYbgydShfWtq27o5E0wRNOvvf2jMV9947Hfx0hXUrnULrc7YkcMJ80ZuIISjE8l+CMBuIiII+IxA0oLkfxBYnJnUmH+MRckDxQIaUQkCgoAgEHQEYFPUPFGC3rP/HdqvsXEtCJLlcIFHUTrFXqo+JdXtbtmK4+gesi3/Q/PQsawgSkFAEPAKAXi5iWQQAsZQZFZdwHvl58m6ERHlCGMGggeC5N0/879hYHRUksT7U77XPQagA5mjCB0YIjVPD2tDkWpCZ+9Sj7zes9n1qhwyxkWQo2TDOl2N8auQTgiZuXBeqocJasCY6o3A+0bJ0UOMUbJRrGDhVIM0dH/8QrRjm6qZul3NZNiSBTpxgz4VoYMwUzDi79mVWtdyz9C/Vbk/Y7JqxzhPZ2xRNh3zS34rHG81q3BCIACnTvIvd5LVONKrM87Dm7Z8nZs/68bbc5xx9zR+eNch5xGIEAi8dtAP5gNPDqw5o4Ac+HduqjcV7ldF6OD+gFfTP7ONZ6S97yvWWv6Y5PHi/jQSOlhbU/kZpObjqXdPzzNfx2TVj6f2T51KPQMee0p87Rfh1X7j6wfSGAI8kBdHETo744im/ZT63L12NfXZY/TeCcja8vCMcddvVn7ueJiufjGC96nyt9rU91py1xs2baFhH32u5X6BKoI9ZEBkhMODlmX33v3U96U3KJ5zw3gjVZNz6Jw0rN9ysWW0U/cf4O94C/lz9lz64JNRzN/z84EFXjvqEbNg8TJ6fcB7FmeZVc4vTJpLXY8UHq4lqZoJ305OyTMUBc85FpA9EHh7uxN49cydv0grBp6lShbXCJ31GzfT8ZP6fQ1PJiV2hOdk8WZM6hy1TUpyPw5VR7aZi0BY9RtapV7tDBoL3tgTuf4QiGzbgyJa3mc5sYi42RS16FPLMlEKAoKAICAIZCwCKuwattq/zS7xG6lZTCJijmf4iG3ni2d4H4HuIL24JHT7khyFKlgOK3Hye2SPWyVhxxid7OxtYrk4fFAKdjpY8edO6eHX2HKg26p0A4YPUGbIvZit/l0WHc35U4rpuVTOsDeKylnj7iLAqIIQQMjbAUMlwrlltsDwBdIlL3suwNiDf1cbjDXpGh4WZn42oGtGdG4JYZS8zQOCJN44F7iC/AqUpGdMvo6BjWFUGNebjdCn2SCtPJ+8aKdNq2b0ygvPaDX7vT7QJeSPcxPTf/6eYLTbsGkrvTlwiHNx4I/TMbfAD8aHFhEKrFARfU0hP44nggFrJV8M//G9gXwe8RzGDfmyPBhDfRiJd1Vx76B/EPnIu4V7FERxVhGQmsgdc/lSYEaM64dcRghnhpwq8UyUGkl01Yu6digPprjrN4s8d/yF6vln+mjeIDi/413swepBysWWpbEjPtRqfDlxEk2f+ZeH2r4VlSldSsthU6hgATp46DDFsXeMOzKnbp2a9MF7A7QOPh4xhuYvWOJbZ25qYwz1OMfOocNH2ENnG5MnFi9VuDk3o9QFmBTOmTMHISycLwIc6/JcLl26RP9t3MK3WhZ69vgyUS/rpidsdij9NvZ1Hjod6CVIUk0QMCKQNG8ShddsSmFFyxrV2r6tWgcKP7aFInb87VImCkFAEBAEBAFBQBAIPgKJt/R1S+jY1v+jETrBH5X0KAgIAoJABiGAxMzwAvBW8Bav8Y11b8/LyHogcBCODX+BFhjMYYROfjPYp+bPs8Edf4GW9IzJ17HAmHcy418gMQ6rbu2aNPNX9uJg2bB5C/UfNMxYHLj9TJhbQAaPkEqXvbxnsVZACgSbGDBOFPmA8JdVBSSut0SuN3PUrh9fw7REXbu06gW63F2/1+FzB0TOrW1baQiGgbTyQ554rAf16fWwdubX3/9I06azB2k6BEQK/txJo4b1adBbr7orDog+rTEEpBMfG1HeOT6eRmc4jN2CRUt9PU3qX2cI8OspIoKA/wgk/f2125MTm7/AxqOKbsulQBAQBAQBQUAQEASCg4Ct2m1kq3mnZWeOS+fJNucbyzJRCgKCgCAgCAgCgkBgEIBtFWGH8BctYRoDA6q0IggIAi4I5GQvLPWs8ZPT0byMVRtR8GjMYEFeQtUftiKCgCCQNgLiqZM2RlLDAwL2HavJtuIPiri5s2stTriZ2OIlyvHHC8F1g3YdiWgEAUFAEBAEBIFsi4C9WHXt+9gdALbZEwjEjoggIAgIAoKAICAIBB6B7g8+zsZK8/u02T1UTuBRlhYFAUFAIfDJyLE0cuwEdahtvQk1hrw0d93fy3QeDoLxvFq5em2m9e0yYVEIAlkEASF1ssiFCuVhJs36isLL16awEq5eOfaiVSmx1esUNT+DXMtDGRgZmyAgCAgCgoAgkMkIOKJj+Hv4NR6F9RtvtrV/k23Dv5k8SuleEBAEBAFBQBDIOghs2RpHk3/6VRvwgYOH0hx4MAyiaQ5CKggCgkC2QQAEToIPecKMwPh7nrENf/czs29/xyznCQKZiYCQOpmJ/nXUd9KMMRTV52PLGdkqtaKwSycpcuV4y3JRCgKCgCAgCAgCgkDGIJDY5i1yxJSxbNxxfB/h+1tEEBAEBAFBQBAQBLxHAAmtJ0/RSR3vz5KagoAgIAgIAoKAICAIBA4Bsw9w4NqVlrIZAvaD2ylp5li3s06qew8l1bvPbbkUCAKCgCAgCAgCgkBgEQChYy/dwG2jSdNHSXhUt+hIgSAgCAgCgoAgIAgIAoKAICAICAKCgCAQmggIqROa1yVLjsq26k+yrZzpduxJNz1Otlpd3ZZLgSAgCAgCgoAgIAgEBgHktIOnrDtJ+n0E2Q/FuSsWvSAgCAgCgoAgIAgIAoKAICAICAKCgCAgCIQoAkLqhOiFyarDSvpzHNl3rHE7/MSmzzCx08VtuRQIAoKAICAICAKCQPoQSGz+ItmqdXDbiG3hT2RbN9dtuRQIAoKAICAICAKCgCAgCAgCgoAgIAgIAoJA6CIgpE7oXpssO7LEX4aT4+get+NPbPosIRybiCAgCAgCgoAgIAgEFoHEVq+SrXpHt42CzEmaN8ltuRQEBoEcOaJ8aig8PIwiI31LdYn6OM8bqVSxHN3RoS0VL1bEY/WbGzWge+++gwoXKuixnhQKAoKAICAICAKCgCAgCAgCgoAgIAhkHgK+/esx88YpPWclBK5epsSf3qeonkMprEAxy5EnNe5DlDMvRa7+xrJclIKAICAICAKCgCDgAwIROeha2/5kL9fE7Un27SsIYddEAo9A7ty5aMjA16jlLY0pV65oCgsLo6SkJDp2/CQNeO8jWrlmvWWnzz7xKPV6+F7Kkye3Vh4ff4HWb9xCz786kK5dS3Q5Jy/XQz+tWzQlEEcOh4NOnjpDf/w5hz4Z9ZVLfZA4v/7wFRUunErSbNy8jR7p8yIlXLtmql8gJj9NHPMxE0XhtH3Hbjp95qypPKsd3Hf3ndSn14P01Tc/0tRpM7La8GW8goAgIAgIAoKAICAICAKCgCAgCLhFIKz6Da0cbksDVHAxPmv/ozBAMGS7ZsJKVqKoR96lsDwF3M49Ysccilr4sdtyKRAEBAFBQBDwHQEYegn/8xa7dKmk743wGfeyUbRWjSou5w4a+qlJV6tGVe3tfpOSD6ZOm0lbtu1wVnt1HBFz3Kt66alkO1/cr9OtcNmybaeL4djber4MwgoXR0xpSmz9OtmLVnfblH33f5T4/dtYFG7rqIK8+VMJAKXLblv8dvUWB5Ah82f9RLmio93CNHbC9zRy7Dem8nEj3qeWzW426dTBseMn6M57etHFS5eVinLmyEHzZk4xETQphbwzb8FSeu7lAUYVzfz5G6pUsbz2LDh46AjFli2tlf81dwG99Ma7prpffDKE2ra6hbbF7aK7H+SXb/wUX7DzswuvThv96VBq07IpzV+4lJ59yYyLVw2ks1L8uVMUxv8xw4f/WbzzqjJ26+0aNJ6T1r78uywthKRcEBAEBAFBQBAQBAQBQSArIZCe38yh9NvY13mIp05WWqVZbKyOo7spcfJ7FPXQICZ2YixHb6t6Kznyl2Ri5xMKiz9iWUeUgoAgIAgIAsFHQBE1OjET53EAIG5Qz1ne7f8ydX/4SWd1lj5WuAwa+kma84B3AOobBaGtoPOX7DK2hX17bGNKbPkKOaKtv2e1Ons2aN/H3hA6zu3LcdoIfD32kxRC599Fy2jwhyPozJlzdFfnDvTWK89RVFQUPfV4D/przgLauXuv1iBCoSlC58jR4/RMv7foPHvp9H+1L7Vr3ZxKFC9GI4a/S48/+2rKAD5+/+0UQmf+wmX0zrBPqUjhQjTio3eobOlSGiFzf/cuNOWX6do58NIBoQO5v+ezBA8dlA9680W6tU1zTa8+UBcECOTNQe8rtWwFAUEgCAiUKllC84xLSEgIQm/+dZEvX16y2+x06XIq0ZxWS8WLFaVz5+MplOYVwyR8IntBXr5yJa3hZ7nyV/s9R405hObRYyeo70tvpow/MiKCWrW4hTZs3konT55K0XvaGT/6UypYMIZmzZ5HX3//o6eqfpc1alifLvGLC1u3+/fyj98dy4mENVG8eFFtrdjt9qAikpHPO3ga4/cTXozxdl4FYmLI7rATPKVFBAFBQBDIaggIqZPVrlgWG6/j0A5KnDSIoh4YQGEx1nHc7SXq0LWuoyhy8ecUsXdxFpuhDFcQEAQEgesXgS3b4ly8T9zN1oqkwPmBJDDc9R1sPeZlNV+rcTjXg0dPrRrVvD7fqk2lS6r/ACU16qUOLbf2nWsp8cfBRLYky3JRpg8BhECrUa2y1sh/HDbtmX79Uxqc8ssftG//Qfpm3KdaOLYH7+1K777/mVb+8vNPaFuEaLut68NaqDYo+r4ykH6YOJIa1q9DN990A0VH56SrVxMI4d3acMg1yF5u89mX9H5OnjpNt3ftQSsXzNBCuPXp+UAKqVOvTk2tfmJiokbo4OCX3//USJ0INujAwwgGV8j777yhjXHDpq0Ut9N9XkStsnwIAoJAuhHocGsbeui+blSwQIGU3FhJNhvF7dhFr771rsf27+x4Kz368H0e6/z2xyyaPOXXlDpffzmC8jMx404SmOh4qOdTpmIQ0gPffJlqVK9CuXPl0squJlyj9Rs20dAPP7M0mna+owM9cM9d3Fe+ZA810p5h3/84lX6f8ZepfauDCuVj6cMhAzVM/vjzb/p+8lSraqTmc3+PJwi4IezlpImj6czZc/TEsy+bzsE8hr7zJlWpXIk9HvV8ZyB1lq1YTZ+OHGeqazzInz8fPdX7Ubrpxgaa+pffZtKUn38zVnHZb9GsCfXscb+GdQJj9VCvp13qZJSiZIni2nUqVtT8b+4Phw7k76kqfL0cdNf9PQnfCWlJMSb6ozhvGwz/GSH3de9Kjz50r9b0oKEf0Wo3IUozou/s3Obzz/ShW5rcRPny5kmBAb8x/p63gL6c8F2KzmpnwOv9qH692lZFKbq3Bg2jHTt3pxyrHV+ed1O+/4pzDEaoU01bGxPL9/UwexKDoHru6d7UHOFvDR7TeBYM/+wL2si/a5ylTq0a1K/vk1SU75UIJoIgNia3Fi5eRiNGj/fqHnFuU44FAUFAEMgMBITUyQzUs1mfjiO7KPG7ARR1/5sUVqyc5ewdOfNRYru3ybHpV4pc8aVlHVEKAoKAICAICAKhhgCMZHd3vp1gJHcnS5avCqih3JGvOCU2fZa9dKxDd6lx2DcvpsSpH6pD2WYAAm1bNUtp9fv/pRpQlXLF6vWacQBGxZpsGIWACMKbpJDVazekEDqagj/Gjv+eJoz+SMtt81iP+2gMH7fjfvAGKuSHKdO0rfqAIeLveQu1dViqZHEtxNqBg4eZENLDwcHgqQQkkhJ454DUKVu6pEYgQf+GeOkoeGQrCGQYAk/36UkgZoyCyJgwToL0T0sKMBGkSBZ3dcuw955RQB4pQsOoV/u5onXSRh3jmfXV6E+oeDJBgPEhjGB0zhzU5KaGNHHc5/T4Uy+aiJ0xn39I5cuVVU2kbEFOP/FYD+178tffXb16VUU844a925+QOwwCT0QrAU4l2AsIzz71fANuwOTyZbMHDjxzxowYTgV5axTUbde6BY83ll58dYBpHvAw6stG4gb16qQQUzgXHpHuRJFZMUwEKXHGVOmDvVWGbuR6y8nXzxtSJxBjBCa9ez5E+w8cMnkOoW0jqRCT33xtAtG3tGFGAPfzpx++R5UqpNpiUu5pvj+bMdGTFqlTjO+LtJ47RYsUdiF1fHne4RngiXw2z4ooT+7cNOnrMdpzybmsUMEC9P67A+i5l96gvfsOpBRbjQeFIHfatGxGIEVf6/9eSn3ZEQQEAUEglBEQUieUr851NDbHqUOU+M1bFHnPaxResZ7bmSXV6Ub2UvU1Yif8yH9u60mBICAICAKCgCAQCgggD0mjhu6/1zBGJGtv1eEeuhqA0Dq26h0p8WYOaRdlNsA5Y2FbMYOSZslLEs64BPp4/oKl9NBjfalC+bK0cMkKl+aRByeS33aGnGCvGkhzNp4omfaH65vrS1es0YgenKetrfF8Dr+BquQn9gByFuhALkJact1JTPzsSzZiYAzh4WHaW9qlS5VIOXXX7n3a/oeD+2teOiv5Tel9+w+llMuOICAIBB6Bm5kQUYTOWSZVh7z/Keex2qF1BCKmNYfJSktmsAfLlq3bTdVAuLz56gua0RUeGbNm/2Mq7z9oaArRqwoa3lCP7rpTf27s2GV+u/6NV55PIXTgofLtD1O0t+DRxw3sSQiy5567O9NPv/yumtPIahAtc/75l+b9u5gQWrJb1050d5c7NHLk4QfuIU+kzqv9niUjKZLSsNNOtaqVNU38hYspJTWq62QYQo8ZBZ4JitCZPfdf+mbS/zSMXnvpOc17pXLF8tS5020mL6Kvx41IIXOusVdLDjaIe5J+fZ+i9m1apFQBBurt/xRlJu68O+xjurdbZ1qzbgNdvHgpaCOpwIQZCLi8Bq8Q1fl37IEFbK+wx9T8BYuVWrYZhMCLzz2ZQuggFOvnX3yphShDd/Xq1ErJt+ep+48/H+NCtBYqVJD6cdt4/sD7bSOH+DOKr887eBArGf7Z6DRDosFDD0Qz+v6ZQx4vWbaCf3NFaWRiwwZ1tXFh7i+80l81S5UrVdT21/23iWb+NYc2MR5t+f7t+fD9Wlu1a1YnhIg7cvRYyjmyIwgIAoJAqCIgpE6oXpnrcFyOS+cp8dv+FNn1eYq4wfx2mnG69sKV6Fqn4RSx5XeKWvMt0bXLxmLZFwQEAUFAEBAEQgaBShVT33p0NygYqRDKBN4T6ZXE5i+m2UTS3xPJttRziJg0G5EKXiGQcO0arduwWfuzOuG9t19howJbO1iWLl+jbStUiNW2+Dhw6EjKvnEHYXtA6pTicDoQeOBAbOx1A4OhsxjbqVypgla8NW6XVh9eZJ+8P5CGfDhSy9ODwrPnzmvtVOG6DerV4nRLDs6l84Fzs3IsCGQ5BODdUS62DG3eGueSywWkCTzUtm3f6TanCrzoypYpRVs5V5yn/DF4o7xK5Yqa0Roh05THiBEwGLPhOWLM7dC758NaFRghH338WdN5hw4foUn/+9nYhOX+ufPnad1/G01ln344WCMr8Pb9kA8/pU1btpnKnfOWwHDZpVMHrc6BQ4fp5TcGmerXrF5VO77AJIDKqQI8Brz7Pk3+ZiyHjYthsqaTidT5dfpM+m36LBO2E7/7kRo2qKd58MBTSPOm4bk7SyMOcdaSQ5d5EhAEYYx7ndp6aMkTJ05qOctwTlW+FpADBw9pOnijwDtBhU7D/EeOYYac5QKTQZjv/777UiORQE4ZQ8PheXiQrwXIrH/mL6SZv07WiHHtZIuPqKhIfmnjGodzW0WTfvxZI1Buv7WtRU3/VTBeY/1ifR/k7w2sFW/lOOM0auxEj9VhxMY13Ra307RerU7CdShVqiQh3Nspfllh/4GDpnWszilWtLDaddni+rgLracq6x6uVSn+wgWTp4Uqd7fFGkPIwFOnz2pjc1cvkPrYsmU0DxOsM+P9judEtSqVNXIBYcmsnhOoAwIMeas2831rVUeNNWfOnBoZeT4+3i0mCBtozA8DHFs00z27NzMZ/Mbbg1Vz2nbDpi2Ev7QE9xb+lOD5pghQrP+n+r6q3VuqHFtfn3cgiZQsWbrCIxaoh+fr5J9+NYWahH7QkOH020/faiEEsU6NAhJxxOivTHP5Y+ZsKsZeRnimQUAICaljRE32BQFBIFQREFInVK/MdTyupN9HkuPUYYq8tZfHWdpqdSV7pVYUufYHitjq+laqx5OlUBAQBAQBQSBdCCAXDPK/pEfSe356+s6oc4HL1Gmp4WNGjfuGnnzsIe3NQHd9zpg1N4XQmcpvEvovOjng7ny8PJH0+wiyx61yV0X0QUTg9ltbU+eO7bUeEeZs6jT9twwMZ0oOH7F+ExQx7vNwCCIYcCAqHNE1znthJSo3DspUWBuEWvt28s/0+CP3U4d2rbQ/de47wz7Vdj8crCfTXrxslZYwWZXLVhAIBQQGD3xDy+GwYNFSjWRBHpl8efNy2MDzWq4SVb5y1VqCkbNdm5aaEU+NfTnrx3GeiFdffIbDH1YzGebhMQLCQUnL5k3pBfbqQKgwJbgPkT9m8Af6/QI9kmoPe68/lWMjbjJfq1WHsfSdIR+lEBowDiL0FAy0jz/dT0tQD6O8IminTf8zTYOlGkda24FvvULVq1bSqn0ycgyt4Hl7knKxZbU5wIPvGBv8n3vxDZMhGkZg5TGzdv0Gl6bmzFtA93Xroj1rUFeF8zLm8DGedPbcOY3UAeGUaAgBqerACP/mKy9oh9uYjAappp55qg62UyZ9pZFCSletSiWaPvU7dahtO3VoR/j74JNR2rVUHjPIMeQsSzk0asfb2mpkBkI1IQcHxNu8M6q9z0Z9yRh8oQ49bn+aNF7LkYZ8a31f0p+/xhN++XGi9txfxZ6Tg9//hOBlgDw98EYwCry8Jv/vF5r1t9kjy1hH7b/8wtPUij3ALl26TPc/8oRSa1vkDGrHXgrGsFqbtmxnT6MIUz0cgKjCPegcHgvXdeGSZTT8Ux0DrPvWrZpTgeSwavDqmvHrD1p753ncDz/2DDVv2phee7mvpuvN9weIJyVV+boO6v9qiocV9Ohj6/Y4Gjj4Q/buuaqqknoGLGYCAERX9653mu5hkKcvvT7IZMBPOdnNzuRvx2lz/Ig9RZCbrnXL5hr+eJ7gmqjy7374SSMs6zLJqJ4FGOe4Cd9q+fT6Pf+UFiZQdYNnwbDhn5vuzycZ/ztvv9X0bIIHGp5PPyf/ZsD5CC/4Nue3MmIPjzx4muAZpwR4gJA4feYs9WDSGAIvLXUfjBwzQVVN1xb3/biRH2n3KTyu+nKIs1Onz5ja9Od5V7hQIa0N4OiJ3FId4fpaPXdArOH5HZU3kq7xyzdG+XP2XONhyr7Ry+8cv/giIggIAoJAVkBASJ2scJWuwzHalvxKjqN7KLLzsxRWMDUUiPNUHdEFKPGW5yipRkeK3DCVInbNd64ix4KAICAICAIZhED6CAhiI3Z6CIwMmlQAmgWxo2QKh73CX2aLfedaSpoxmhznzKFnMntc6P/+7p01gwSMZv4KjLgw9jnnk/G3vYw+r3nTm+jjoQO0bvDW95PPw2jKVgoWld8A+zC8WMmVq7rRSiULhgEF4mycMJ4LIwbe+DUapT8e8SWd5reVH3mwGxtp82uGs6EfjaIly1dT3do1tDd+jV46tWpU5VBEt7LhOj8tX7mWfp/5t7GLkNq/nw1VtWpWI3frqm7t6tp4Mc/Rnw7V9p1p0bNZbF2F1AUIwmDy5MmjGSNhkG7LRmIlKqShKm96cyOtCPcY7h11jyH3C/4g8HC7mpCYYhzvwmHHFKmDN85h+IYHAoyJ8AxATircS7U5obYSGL6/HT8yJRwXjJm453AePF8GD3ojxeOl+S36m/Eou4XHB0+Qmnx/KZkxa47a1dowvt2fUuDFTt+nH6ebG92g1Rw7/lsOZbXE41lIDP75R0O0McNj7+nnX3MxnsIjUAnyyziL0esntkxp2r13n3MV07EKlwajryKAjBUGD3pTuy4JTFq/zZ5A344fZSxO2d+zd78W4q1QwYKaERxeRAnJYU1V/h1lWD7Dz1aEflICAsdZlq9crZE60CO3jiJ1rMbofK7x2Jf68Oq6kcPeIbcJ1p0xHBoS2CtyBQnbIVjbitCBkToiMkIjLhFS7tknH9M8to4d9/y9r+4T9ZKAGjuIR4Seg2DdX7l6RSOU6tTSn52qnto2Z28PRSqAnMOLA7jXQGi0YlJ0O88NHg8gZVTIO3WuIhVycw4USBR7bSmdMSdhXSZRkAdFkSTGOYPY+G78Fxoxpe4XNTf0rwTnYK5oA3j24LB/Q4d/porT3GJOGNsrTAbj/lWivltV+WOPPKAVgXyw2exazir0iZwtSoATvHXRDv569rg/hdRp1LA+e8vp+KMNrGe8lAGM69ernULqYL289/brqkmNOEa+JpCyuH64j1UYxFr8HIIUZo8XeP7AKw2eQhAQIEYPLzy7FI5aBS8/cB7yZ+G7F+N+6fWBZPWCij/Pu4IF9ZdebHb9GeTvGDE29YLLhk3mkHDupgnyFIJ7YemK1e6qiV4QEAQEgZBCQEidkLoc2Wsw9t3r6dqY5ymy01MUUb+Nx8k7ClWkxNZvkK1udw7LNp0i4kLXyOBxIlIoCAgCgoAgIAgEGIGkeZPItvCnALcauOb69Hww5e309LQKo0FWIHXu796FBr7xgmbIwXzhFYMY9krOx19Qu9pbrhf57WlnycF5cCBJSbph42oyyQODnjuB8QMCg5ZRvvlhKuHPWYa9oxuJ5sxbpBkzH+txH3s0PJVSDV5GT/fuQZ2699SMdykFIbLTu+cDZMwR5G5YMPa2aZlq8HOud+jw0SyxrpzHnZ2OYWCF4RJvWO/cvZfwtr9RQOYg5wu8X2CkBMEyfOhArQruhxEcdksZyR956F4mmrtqBlYY8RYtWf5/9s4DwInqCePDHb333lFEUJogKkWaBcGCWLChoKIoFuydv2JBRRQUUEEFFbtiQUURBAQFUaT33nvv3MF/vrd5e5tckkuu5rhvNJfN7tu3b39bEubbmTG1FOBwhTPv+u53uv1DTEXUiDUIP6ivgnYvvvqGplT82wgyqAWD1GGnn3aqFFNnO8b31ZgfpM89d5jr8dffJpkuqmikDgzjvUCjiuBUR3vsHxyuKOb9nEYCwBEbiXW5opOJnkBbOI8hnuD+gn0KZrhHDHm9v3E+Y/mmzVvkxq5d5CuNCvAyBUOkU4KYgBRz3mgcrLd163a8GatWrXJYUafr1Z1doeL3KckFp8s6XazcHKdz/wGD3Ugn27/33RYvRzQLnPUPP/GsicDAfo3VSBBw7XZbb3eVK3z1gjA/mG3x7EdlPc6BKe2CrZPWeYjugJMe1vmyjpquLenefJkvHR7Y22M4/IOPTZTXgkVLXEHskovaS+87exjRAvXWUEskWoPT+9abbzCr4eGCXvc94gpMV195qV4T15n+vf1+o+fJz1qr6b+58922uL9CaIGY0UqFTIg69jgNHzLQ3KO3bNsu3Xve6+0q5PQjfXqbviAWPNn3RTeNYI9u18tVnTuZ70xEFw199wO/PnBNjtdUee+8N8pE8tjUYIj4QuRKagz3hCWaMm3SlD9VtEDKu01+3UCEGTBoqMzUqCpY7ztvdUVC1JPq/9pgWa73K5yfEFJRv6lKpUpGcIIY2bNHN7Mertnrbr7DTOMP0qcVVkHbWp97e5lJbO/xZ54XiJtIxzdEhZXyKrrieFlR5+dfJ5hINaSYtPcRK3geOHBI7ru7p0A8LOQT13ZrhOO/s2bLwMFv282l+P7is0+63727du0x20O9qmXLV/qtm5r7HSIxYWBv0x7i2EJw/G3iFHn3/Y8iEqKeeuxBdyzf/zjOnQ41UatGdfO9geUrV6+JaBuh+uJ8EiABEshMAhR1MpM2t5WcwJGDkvDNQDmxYrbEX6Q/TguHf4r3eKlT5HirByWh8U0q7IyT+KW/Sq794Z9OSr5RziEBEiABEiCB7E/g+LpFkjjuPTm+bnFM70y/l9/QSJGrTUHk1A4UTolYiIhKafyPPXCX3HzD1aYZImCefFbzuv/g71Dw1lYqV66M7F+5Jlm39olgFJGGwfkIh3E+n9gTuIJtj/le0Siwnf18TtNG5klxOG+feeE14yh78N6eZjHGh6ioC9q2NMWTn3i4tzqa37Crxsx7v5cHGdaWUeDA6muERenSJbXmw06ZG1BfxLbNLueVHW9OfYfweUN3x7EZjAFqvUBEsYZUaLbA/ao1a11BB8sRMQNRB4Y6LHCeIz0YDI5pRJ7M84lGiMCAA9XaWT5nPJx+NvID19DIjz5z68GgFs1fM/4xy20buz7qkMDwhD1SVHkNIgUiEd4b9obWpnjIjRrxtgmcbtK4oTsrT+7cKl62MC9EJtx1/6OuCGAb5dfoBW9aM4wVL4gLiPLxpiVC3QmkJoNzdcxnI41zG47tGtWrSU19RWIQhDAWGKJJwMlriAK6vbtTY2iaRg/OmBk+bZxdF1EKcPTa+h41NeoFFlgDqYQvOjQhMcGu6ve+y5duDTNRTyMzDFFNECiRhrNt6xauqAPHP+rAwFCbx9p4TXUXaOPGT5S7enY35xFqQKXGztboLpzvMKQX9EYMQSS6UY8bzimv/TMreSo+REZBHERaQaQmTIuhro+NvPz5lwmuoIM+UdepzfnNTQQKUiUGijr7DxyQN956x9089meu1okxkU4acZcam63rP/GME+UZbH3UyLGCDpaP+f5HV9SBwARBB4Z7BK4niDpgjocRcF/BmGGoyYTvIhtlhno4tiYOUpjZqCdcn/Z+hBR0v6iQcrOK1LgerA1X0QMvrxXzpXwto9+HF7Vv7V1k+m7fppWK15VMlCHGmpKdUqu62wR9Ii0fXpM0usym4EODtN7vbD1CMMP9EVFJqM91+90PuNsPNoEaWTYd5S8qqCM6Lpwhquulfk50GMRx1A2jkQAJkEB2IeD/TZ1dRs1xnnQEEudMlMSlf0vuC26W+CYdUty/E4XLSsJZ3cwrbs1fEr9yssSv+kNzKxxLcV02IAESIAESIIH0InBes7PUadXFFLUP1ed4fbrQW4cnVLuI52taioTfNDpn6lcRr5KVDSf9MV3/sT89K4eQKdse9saLmoLmXLMtpKW57e5HZIbvCV7vAFZpLQVrFdQhtyKYqKNOBpitlWNzvdu0U3Z9+17RUwg4pVQ8WKff0w+bVb/Xek9wHqHuDpyKR/QJ8YuucJysiNpB9E6HC9rEpKgzeep0reMQ+rxCyjVE6EDQufuBpywqvmdDAhBIozWcy4iqCTTrLMV82y8cyNdf08U4XF9+/mlZv3GTfPH19/pk+GR3dVwftj/Uu/KmQ3L6ckQhpAYLZfHxTjQdls+aPU8++fxrk7KqkaZaulaFJkQYQXRBWi1vHZ9Q/X039md1fC80xcLh3G+p0UJwAMNp/PRjD5j6I951D2t0AIqKO+mp8qpQXFsaN6xvxAGkjEL9HHv/gNMckT9IEwYRCtE0NqIGkS/W4bpmzXrvJtxpOPhffv4ZwxRRFw89/j93mZ3or6wRpYQIhJcGDLKzw75jP+HkRWSTtboqhsG2bU+KIMJn1CaBedN7mRm+P0hPZQ1CS2YZ6uDccG0XQa0ZG9nVutV5RkDDGEaNTorewWeIX0hBiKgx1PqAqAihKm9c8vMb7SMxCIgwHBuIE5EaHOAXX9BWalSvaiI4EV2WXlbHF7GF/n77fUqybufpvRxp1myUSbIGATPsueydfXuPm9Th74hn3vlbtm71EySwLNEXKettF256y5akukCB7XZ46s3Y+w4i+FAXCoIFop3mzl8oH3/6pdazXOKuXkevQWuoC+a97yD1HAzXQ2A0nV0H73FooIZjDZHw629/1Gi7bXJh+zbS9eorjKCEcSByEfXLUrKPPvnKpKZEmsZqVStLi/POMdF/ODbztR6TrfOUmvvdGI22HD9hsqmbZsdRq0Z1rb/UW6OcKpp7GyKTQkWnIQrulhu7mlURITZoyLu2m5Dvr7/SzxW7IUp5oxZDrsQFJEACJBAjBCjqxMiB4DCUwKH9kvD9EDk+b4rEt7le4qqfGRGW49XOFbyOnf+QxK+ZLnHrNBXChlmM4ImIHhuRAAmQQHACqK2Bf/SnpS7ONfoPr7SsH3xkWTsXXK65spP0fcEpnP2/Jx7wS88TbHQtzm2qzucZpp4JmMBSyyXxv98k8fdPYrJ2TrB9zynzRr4zUJo1aWR2F8Wor7rpDi2UHNzZuWLlahfLRe3PN/Vt3Bk6UVOdZXm13gDsX3X8wlBUGwZHKrYTKBZ16tDOLMefiZP/dKeDTbTTp50rV6pgHHKIdoHVrFHVvOOJa2uoqQNRx+alt/P5TgInGwHUUnn0qefkOS0yjpRjlTWi5gFNnYZommdfGKCRa0s1EqG8u9tIeYRXMINzOJRt27bDLIIo4n0a/N//5qqgMle+/WKUEY7qaPRMJDZdC7fjZQ0F0+EcxtPzqMkRaHgKP7CoOFIxPfnI/Ua4QSq0d9/70KyGtvc88LhJ1XRes6ZSvnxZ8x02T1NJoh7Knbd2M+3Wrt8QuBmTGmrY4FcMS+hxKA5v69XYxkihBVEDNljT45XSSAVrEJFgcHajBtABFX0QcdGxwwV6P3KEmPz58gucsTAbrVKxQgUzD0yQjgqOaxiEo2BWxhOdE5haK1j79JoHwfC6q680zK/ufKmMGDlaOnW4yHSPtF3b1BltradyvlzTsvn88nZ2mt+rVqls+ghXpy1wIw3rn6HXyKOu+BS4PK2fEZVizftdZOdZYQTnB8QlW0/JLo/kvVmTxkHTwdaq6UR7RdJHerVBqjQIxN1vclLdNdB6Qg3O7KvpDNeY+wPEhUr6XW0N9eGCGa6xcHWdELVUTFO6IW3im8Pec7uA+AIBadigl8081AGLRNSBmOy1Dz78VEaNeMtcZ5dqNI0VdVJzv4PwhBqKXkN0W+8+j8vXn75vzr1zzm4aVNQ5pVYN6fvkw+ZaQQrDh4MIyd5+Mf3Cs09IdRWmYD9qasHA6EqzgH9IgARIIIYJUNSJ4YOTU4d2fNU8Ob7qcYmr31pyt7xKcpWrHhmKuNySWKOFeWGFXLvWSNzm+RK3bbG+lkmunSsj64etSIAESIAEDIF6pyd/mjEaNFgfIsiCRUujWS1btUWKn0gsVPqXSNZNanNCEsbEXhqspPHlzKn77rrVFXS2bd8hHbvcbJ48D0UDT6XjCWI4Ii9o01Ke7jfAr+kdtzqRMpj547iJZtk33/8sj2pqN0QL3N79umSizhXqjIXBsYOaI+Gs7xN9zOIv9Wndgwedp93tU/coJm/NPk1sl9n5fCeBk5EAUrZddX0PTU/URjpffokRdlCw/NUX+8oV13Qzgobd76+1tsiUacGjxKwAa9t63zdoBBAMTmnUzfBGDWE+BF+kWcR2U2v/zPrPpEJC2jRENASmJAvsF05MiEwYU41qVQMXB00jhyggGBywgc5kbBe1PuBEhr2uKbG8wpOZqX+QQssaRKVgBiEHr6k6RjimEU1gDeP1fsZ8CHKYh/skRJ0Nmzbb5iYqCeKc1+qrE93amrWOcG4/Z+Q7mMGZfma9OiYC5311iiMVIGzM9z+5m8b47L0daTWn/jlDa7tsMg+SXNS+rTlmbuMoJ+DgP1Wd4LZ+WySrP3T/XcapjhRVOKbYB0QRIX1Xeoj/NiIVYymrYp43JRzm2XR6EDFSI+igDwgOtWrWwKSf7dHaMllhSBv5k4oJ12gtPkRA4dpHdBxq8KAO0RafMImxIdoN7AMNkcHhDGndEL1XvHjRZM1w3iNyDylckXoyNQbBdvPmrWYbEJStpef9DtcMagQhbWG5so4YbLeDd4juA176nxGWwOjeB59w09l523mnH77/bmmkQiUM9/Mh77zvXcxpEiABEsgWBCjqZIvDlDMHeXzuJDmqr/jGF0j8OZdJrvLJf4CFI3OiRDVJxOv0jk4zTVeTa7cKPXs2SK59myXXAX166+AuyXVYf8Qd3S+5Eo5o0lt1zqUixUO4cXAZCZAACWQ2AeOM1XsZ3k8gTYqTtjuzh5Gm7Z0okvRUdJo6CrfyHvUMpNGe+N/Lcv3Vl0t8QO55b7cTJ02THTt2eWdx+iQhgPz/t99ynbs3I0Z+qg6ulu5n7wRSOs38d46ZNXjYB/Li/x41qXeGvv6C3NXnSTO/08Xt5NIO7c30ho2bNUJgmZlGTZEp02aY9G7N1cl5U9cr5aPPvjHLXuj7iFTQp+hhY8dNMO+h/lx2yQX6FH8p44jt/9oQt5lNAed1ljSsX88sD3SsuStxggROMgKITsHT83ihuHo/jdyBeNBWr2mkSoKzEHVGyqkgG1gUPBIUc7RGh7Vu118jb739nv1o3pHKCGbFVvMhyj/lPakYI3noACnIsI+wQ4cPp7g1FKBv1vQs085bTwQzIKYMG/yqG8WEOj3eFHbeznFfQZRCMIMwBMM/yRL132/79u3T2jNfyS+//S59NIIKzmcUZsdn2Av/e9xE9bz+1rumzg5SlMEma42PO3p0M0/uQ6hb9Iq/qNOmVQvTDvsdLFWXWZhBf5BmC6n+8B1y3TWdzTHA+YVzz1rrls3tpDr0+6qgkyRSYezeWmpuwwgn5i1cpFFYTY1Ig9pHKZ3PuB5svRsIZp987nz/YHNIEQYxMq3mrX3S4rxmbv0Y2y/qP8EOHU5KvWeXRfr+jab3ijVDKkHUm8Lrvrt7mro3iGKDKOtNjVdav7tRAyxaW7x0mUYA1TXXCFI8QsC2hmNqz6O0XANIIwg7clR9KT5Lz/sd7i2498BsLSLfZoyw+Nbr/U2UI66hPo9oCs0NG+3ioO/39LrV1GjCwr9UoOw/YHDQdpxJAiRAArFOgKJOrB8hjk8SZ403r7h6zU29nbhaSUVBo8ITFy8nStaURH3RSIAESOCkJQAvyInj+n+inEBxYK01lm9o72y2u7nkSFcnBUzGDfyE5BnQLc3dz9VUNHjRciaBpx65169ew+MPhb7WkFKl01W3GFBjfhgnfXrfZgSWNq3Ok3kzxqvQkmDSFqEBnMsPPPasaWv/vPDqm4JUfqir88TD96hz83bjkEMufRhStTz/SnjHxGMP3m3aQhA66nG+jNUnhR+8t6fmlS8k7w97Tf7WWkA9u99g2k7VAuY0EjiZCSDaAPVmvLUUdu/Wh758husCtnwFImlOkebnnC1NNQ2iV9TAdYhC93PnLTRtz6x3ujzc525BhMUDjz5jhFTUH9msT94jddtFWqvqz+kztbbOXHMdd768o3G6OttZafrAny5XdJKrNEXXP7Nmy2uDhpn5LdXhXVb7mPnvbCNiYCYif5Cmyz55vm37TjeKBg7RHt2uM5EVqOVjoxxQT6e3OjetofC6NewfnoqfqmKyrV+Dp+ERuQQRCD81INpYwzbeGviSeVof85DmyStQ2Hb2PVyx8y8+HmFqXEyYNEUGDn7brmLSMtljAee2FQGQig02Qeuw4N5pDccTDm0IDjhmqBlineI9ul3vpuH6RQW7zDbUh0HUJiJcUM8JhhR83vHnzu2IW1iWK1dSCrn2bc8PGmGTkOA41COJvpmhjmybQu+Zxx+Uu+5/1ERCYFuo75Q73t9N5I3YjNd/U1uDIGTFSDvPvtsIkdKaWi9czRfbfu269S6Tzpd1lImTprrO+UsvudBNs4fvp5PBkAIPkTj2esQ+7d+/3901MEPkn42kufeu2/QetEo2eiLQIMpWUCHXij/B7hcQ4HAPQRrCp/VY97r3YZMOEes+qNFX1rwRdY8/fJ/UP6Ouqfv1w0+/miboA/cz3PdsijTswy03dXVr0sz2pYzFCqm539168/WyXCMWUd/LPlCCtGpI1whBHfaHJ0oS59ZQTR+HKD0nQudJSSnqDikNO1zYzvSFKECkh6SRAAmQQHYl4P9tnV33guPOEQSOL5gmeOWqWEviG7Yz6dlyFUweRpwjYHAnSYAESIAESIAEspxANHn4UVTYa+0vvU4+eHugNNbaFxBq8ILtVkfkQ0/0k7lacNhrqPnQ4cpu8sn7bxoxqEB+x5GJNki51vOeR8M+4d9V07vASYsC8gMHv+vtWh1L203kT7frusi5Zzc2LzTAWJ587hW/tvxAAicbgUcfvFejDWqZdGK7du2RRH0gomyZMmY3IWh8+8PPZvqVgW/K+28PMpEfz2rtBjjlkUKtRIliriBz/S29jMPzFq2TAYcjXhergGMdo++M+FDgRIeD9fm+jxnBBPVBbN0XOCb7v/ami/iGa7uYJ+nbtW4p777/kXG8o/YNat7BAYrUaYePHHa3jxUhuLz8WpLAiyfcr1TRCC8YnMS58+g9xxcRg3mLl66QGTOT6vPA6XlO08Zyv0YOYD9R9Bw1bvV10XEAAEAASURBVKx98c13fumNEEFYvVoVu1jwJDxegQan8HU33xE4O+LPEK9gNooRkQYQmVBDwyuI2A4HDx0hg197wTiEH3vwHnng3jtNRJHdd+zbR598YZtn6juK1uOY2Ho5oz7+3G/7cF4jtRkMNYq2bNlmUmh5j4N3BYiEiMTIlzeP/PD1xxpd9rtfDRVvW4gJY/S87nxpBylVsoR8NupdEwHjPRe97efp95FN09f16s7Srm0rI9SU8EVoeNva6bE//2oiRHBuf/v5KHW6H9N9zSWXX93NNkn2/tawEQJBAfvwzpsDzLWE70ab3g3RZ4OGDE+2XnacgRRrECNw7u7cucs81IHvaNjCxUvNvmMaNZd639nDXH/Dhww0848cPWoip/Kq8IPr+crruqOpBLtfIHXZz79M0LpNqElVSD56b6iJdrFMsR6idH4cNx6Tpo4VhGPYTRpRaO9diCizvztwn0pMPG7GbxrqH9wrA1OYRXu/u/LyTu71gGONexnOBWtbtN7Up1+MsR/l9h43uWOC6GPrA7kNfBN///uf/O/5V80nm9IQH1qce7b8NOYTXyv/t0s6X+8/g59IgARIIAYJUNSJwYPCIYUncGLjCknQl/z0rsTVPU/i9RVX5xyRvEnOjfA9cCkJkAAJkEBKBFAHB04j1MQJtGA1cgLbOeuepquODVw9W3/Gvi9YtEy++vgdv/1Abvm+Lwz0m3fNlZfKNVd28psXrJ1fA37IVgQuvza50zLSHUCkzA097jFiDoQUOF4RCRCuJg6EnVYXXWWeym/WpKFxkCFSbLM6+1KyU7QQ9NLlK+XTL7/TlEZJT7Pb9V4a8JZG6MyWa7tcqk+B59EULUvkjSHvSUr5+u36fCeB9CZwXCNOw1lKy7FuoJgarL//NFoGNSfg5PTWhICzFGl5rFgAR/j9jzwl/3vqEYEzG+29jlFEJhw5qumc1eYtWGiieuCUnOOL3sF8CCdP9H3BFPSGQ9froIfDEo5H1I6whrowqLGBFGF2/j+z5ui86kbsgaDh7WPjpi0q2g4zTmHbB4RczMcT/RAQbLolLMe9ABE6bw8faZub97//mSWNGpxpHKrefUR6pUFD3k1WUD2XOu4jseMAkoLZel5wGnsNkQsQY+BQttENp2nqL9ju3bu9Td1pPLXf695H5LX+z5o6P3CCW1unKZruf/gpOXTosJ0V9F0T2ep8J0Vd0AYBMyF+RGJwTiMiBccE505ghAEiscZPnCIXqIACYaRihXKm2//mzjc1RJCiy7utn3+dKB0vvsCcy2gPXrDjvgcKLFczU/8MV5GweLFi0uK8s43ohfMIhwcRQ7geUPctIcG5BsF70NDhgmgR9F1WU4HBEHm2fv1GadK4ga7rv9+o14SxQmiCwx3s9/rObe+x9V6jf2jdoOMaOfGQ1juBM9+KHNgWzuHHnu7nHnvMi+QeEDAsrJbhFsn35nyN1gIbXI/22GJgW7U2X7/+Sb8nUQcIdRnvvqOH4ehlgn1bpNFo1oLdL7Bs6LsfyA4Vjm667mojgnqvaRyjF/q/brvQdMG4j2mUv/L3npO4jzU9q6E5/jieqgu7NltTSw54fYgb1WcXRHu/W7l6jdSsXs1cE95rFfs5Zdpf8oamWLT3Y2wjzhPBZrcZ7N17naAvK6QGa8t5JEACJJCdCOSq07i1/7dvBox+/95dGdAruyQBfwJxtZtI3CmNJa5mA8lVtpr/Qn4iARIggZxCAP9aSaf0axBqAkUJYPzim7EqbCz1I/rskw/4fcYHiB9ffPNDsvkpz8glxx7KHunXUt6X9G2RGb+pChctkb6Dzoa9gTM5pO7AxQo7iKo9u18v734w2tyzUrc3qV9r7+7t6gZWb616jxwHUuROYbvVjDgHM+MeYsef3d7hBIdzGlE6K1etNunKvA5E7/4gQuQMTbGGSByIPUgFBse815CaCAXYvWndvMuR9u2sRg1MpA3StgWub9tCuFih0XgJAdF+6L+mCj4YA2pILF6y3H26367rfS9QIL8KTbWlcuWKkqDpHpcuX2HqloTaR6RUO+3UU0wEDoSN6TP+Ddu/d1uxOI0on7PPaiSIAJgxc5afOBCL47VjwrlW/8x6cuDAAZk9d4GbVs8uD3xHWi2ICimJVd71kEYtv0aM4cGTUOcD2uMcqn9GPSNEIHWgFRq9fQWbrlG9qp6jm1Icu3ddnN94QAgRILNVeAh1HXnXyY7TOF7nNlM/hl5vixYv8xNSAvfHMsmjgssGvebR3qZHtG1D3S/scqQzg5i0afNWgbB04OBBu8h9x70Qx8ymdbMLMEbUUKqq9b9wvkCEQT2mSM61SO932Da2UblSRZNmcJnep/CwCyKOaCRAAiQQikBafjPH0m/jaPeDok6oM4LzszWBXEVKSq6qp0tcpdqaru0UyVWuuuQq5IQzZ+sd4+BJgARIICUC6SjqpLSpjFtOUScU28z40Rntj8lQY83O82NFmMiODMnOOWoUdbLj2csxkwAJkAAJkAAJkAAJZDcCafn3a2b8+zpSntHuhydoMtJNsB0JxD6BE/t2yglfDR47WiP0lKoouUqUFylWRsznwsX1kaPCkitfQZE8+TSGVy8JxuNaZHwnARLIjgRcUUfTlmi6hhPHE7LjXsiJXVsyeNwZHqicweNn9yRAAiRAAiRAAiRAAiRAAiRAAiRAAjmRAEWdnHjUc+g+G6FHxR5ZPT+HEuBukwAJ5BwCmigFmoX+wX/5i5fOZrt+Qo6+nvpaJZHubF6mGYsUFduRAAmQAAmQAAmQAAmQAAmQAAmQAAnECIHIKhrGyGA5DBIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARIgARLIqQQo6uTUI8/9JgESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAESyFYEKOpkq8PFwZIACZAACZAACZAACZAACZAACZAACZAACZAACZAACZAACeRUAhR1cuqR536TAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAlkKwIUdbLV4eJgSYAESIAESIAESIAESIAESIAESIAESIAESIAESIAESIAEcioBijo59chzv0mABEiABEiABEiABEiABEiABEiABEiABEiABEiABEiABLIVAYo62epwcbAkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAI5lQBFnZx65LnfJEACJEACJEACJEACJEACJEACJEACJEACJEACJEACJEAC2YoARZ1sdbg4WBIgARIgARKInsDx44nRr5SFa2TWeDNrO+mFMrPGm1nbSS8u7IcEYo0Ar6FYOyIcDwmQAAmQAAmQAAmQAAmcXAQo6pxcx5N7QwIkQAIkQALJCBw7eiTZvFiecexI5oyXXIKfBZnFP/jWOZcEsj8BXkPZ/xhyD0iABEiABEiABEiABEgglgnkjuXBcWwkQAIkQAIkQAJpIJBL1z0hYsWLPHnzSVxcfBo6zNhV8XQ7xnrsWOaKOtmCiwpdmcbFxz9Pvtg+XzL2bGTvqSVQuGiJ1K6a7dcz9zDvtYp7MI0ESIAESIAESIAESIAESIAE0pkARZ10BsruSIAESIAESCDrCfjUHB1ILv0PjsYjhw+aV9aPLbZGQC7Bj8eRI3q+6Csn297d23Py7nPf00gA994k804nzeUUCZAACZAACZAACZAACZAACaSGAEWd1FDjOiRAAiSQAwjEBUnQefx4Dtjxk2QXHVlH/+Y6YYQdROzQSIAESIAEMoGAq+H4SzuZsGVuggRIgARIgARIgARIgARIIAcQoKiTAw4yd5EESIAEUkOgZIlS8lif3u6qi5culxEfjnY/cyLGCeRSZ+KJE6rl+LyLrpMxxsfN4ZEACZDASULA3Hb1XkwjARIgARIgARIgARIgARIggfQkQFEnPWmyLxIgARI4iQiULlXSb28OHz7s95kfsgEBCDtmmAzTyQZHi0MkARI4qQhQzDmpDid3hgRIgARIgARIgARIgARiiABFnRg6GBwKCZAACcQSgdIlA0Sdo0djaXgcS1QE6FyMChcbkwAJkAAJkAAJkAAJkAAJkAAJkAAJkECMEghSMSFGR8phkQAJkAAJZCqB8uXK+G3veGKi32d+IAESIAESIAESIAESIAESIAESIAESIAESIAESyFwCFHUylze3RgIkQALZgkC+vHmlSuVKfmONj4/3+8wPJEACJEACJEACJEACJEACJEACJEACJEACJEACmUuAok7m8ubWSIAESCBbEGh7fkupVKG831gp6vjh4AcSIAESIAESIAESIAESIAESIAESIAESIAESyHQCFHUyHTk3SAIkQAKxTaBGtarS7vwWyQaZO55l2JJB4QwSIAESIAESIAESIAESIAESIAESIAESIAESyEQCFHUyETY3RQIkQALZgUA7jdIJZvG5mX4tGBfOIwESIAESIAESIAESIAESIAESIAESIAESIIHMIkBRJ7NIczskQAIkkA0IXNSujdSpfYo70gMHD7nTTL/mouAECZAACZAACZAACZAACZAACZAACZAACZAACWQJAebSyRLs3CgJkAAJxB6BSztcJOc3P8cd2LbtO+TosWNSqGABMy93PCN1XDicIAESIAESIAESIAESIAESIAESIAESIAESIIEsIMBInSyAzk2SAAmQQKwR6HJZJz9BB+P75Mtv5PDhw+5QGanjouAECZAACZAACZAACZAACZAACZAACZAACZAACWQJAUbqZAl2bpQESIAEYodA1y5XSJNGDfwGNHTESFm3YaMkJCS68ynquCg4QQIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAJZQoCiTpZg50ZJgARIIOsJlC9bVi5u30bOqFvHbzAjRo2WlavXmHmJiUmiDtOv+WHiBxIgARIgARIgARIgARIgARIgARIgARIgARLIdAIUdTIdOTdIAiRAAllPoFmTxirotJUihQv5DeaHceNl8bLl7rwEj6gTn5s1dVwwnCABEiABEiABEiABEiABEiABEiABEiABEiCBLCBAUScLoHOTJEACJJBVBIoVLWqic5o2bphsCCM/+VzmL1zsNz8xIcH9zEgdFwUnSIAESIAESIAESIAESIAESIAESIAESIAESCBLCFDUyRLs3CgJkAAJZD6B+mfUlYvbtZWyZUr5bXzDxs3yzsgP5eDBQ37z8cEvUieekTrJAGWTGQUKFs4mI+UwSYAESODkIHDo4P6TY0e4FyRAAiRAAiRAAiRAAiRAAjFHgKJOzB0SDogESIAE0pdA5UoV5RxNt3ZO07OSdTxt+kwZM/anZPPtDG9NnXiKOhYL30mABEiABEiABEiABEiABEiABEiABEiABEggSwhQ1MkS7NwoCZAACWQ8gXBizs5du2XcbxNl1px5YQeSkJDoLmf6NRcFJ0iABEiABEiABEiABEiABEiABEiABEiABEggSwhQ1MkS7NwoCZAACWQcgXBiDkSaP/6aIVP1tWfv3hQHwUidFBGxAQmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAlkGgGKOpmGmhsiARIggYwlEE7MwZZnzpptBJ2NmzZHPBC/mjq5WVMnYnBsSAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIZQICiTgZAZZckQAIkkJkETq99qjRqcKY01lcwW7h4qYnMWbpiZbDFYed5I3WQfi1Xrlxy4sSJsOtwIQmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQQMYQoKiTMVzZKwmQAAlkKIG8efMaEadx/TOlZo1qQbe1Zt16FXP+lv/mhq+bE3Rl30yvqANBJ16FnYSEhHCrcBkJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkEAGEaCok0Fg2S0JkAAJZASBsmVKSyMVchCVU6pkiaCbWLJ8hcyaPVf+1VdazZt+DX1R1EkrUa5PAiRAAiRAAiRAAiSQ3QmUK1vG7MK27Tvk+PHjYXenWLGikj9fPr82Bw8dkn379vvN4wcSIAESSA8ChQoWlMKFCyXrasvWbcnmBc4oo/6GOH2Y02u79+yVI0eOeGel+3SePHmkZIniyfrduWu3HDt2LNl8ziABEhChqMOzgARIgASyAYFTa9V0InMa1FdhJS7oiCHi4LVURZ30Mm+kDvpECraM/TmXXiNnPyRAAhlF4Hhiov7j6mhGdc9+05lAnN638+TJm869sjsSyEYE1Lklh/XXy/HEyAadL7/TNpacSLnz6JM1+k/3I4ci24f8BUSO6n060n1WZ5rm1xUNx46sfzj8wOlwhOOJrFf/VtGOyX/tyD7Fab3IPOAa3a/bVi3OlccevMdso8+jz8iSpcvDbm/U8DclL/bHY4uXrpAHHn3aMyeFSY3SN8co0vMyNfsW7Xmj3y/6BaPnweEUBh/F4vx6XuF4RJruOdprI5KhRMs6kj7T2iarxhTtdnFvKKD3HxUtIz6GwdhkxLkVbDuB82L4vhM41HCfe/e6Vc7X+1SgXdL5+sBZfp9Ro/fdtwb4zcOHjz/7Sj75/Jtk89NzxtlNGsmTj9yfrMsBg4bKxElTk81P7QwI8pGI8antP9bWg/+mdavmMmf+Qtm2bXusDY/jSSMBijppBMjVSYAESCCjCFQsX05OP6221K1TW6pVqRx0M/v2H1AhZ44RczZt3hK0TVpmJib4O0AQqUMjgVghULhgvmRPvsbK2LJ6HNt37c3QIRyP1FGYoaNg55EQgKhDI4FsTwCOtjMaiNSuK1rgT2TmnyIrloXercpVRM46R6SMRlNADIFB5Fi+ROSP353P3r+FC4u0bCNSrryKFeqQhMGpvG6NyOQJIgcPOPO8fzGOmrVFGjZWx7Y63Ndo7cK//vC2CD1duIhIx8t1X/T6/HeGyLLFydti3Oe2FKmmaXYLF01avm2rbmeKyKaNSfMwVbW6SFN14pUq7TDCvP36XTB1so5tFT75W5w+JHT2edp/TZHivqejD2jkyOJFOqbpwR2y5SuJnKdjKq1csf+JKgJt2iQy5TfRsBP//r2fwPesZiIVfL9nx45xxuZtg+nUjCmwj+q6P+foGGHjvhPZvduZ9v6tV1/Pp4ZJ+33ooMja1XpuTHL2yds2yHTu3NHdV5HC2NpRnyizdVvKT8xLcY3KP6+VSPmKjniCTrD+rL9F5swKfoyi3bdozxtEHDVrLgLOBVQwheFawTn8p57/Rw4786L5W1AjCs5vp+eHnl/Wqb5DnY8zpomsX5u8J3yvNThLX3rtQXSAYQyrcQ1OCX8uOq2T/42WNe4Xlaom78fO+XWsyM4dIhX1nG+l+xapYfz2eo12THYb0d4v7Xr2PTXbrXWqXnd6XnjvVdv1HP9T7z+B9yq7ncD31J5babkXYwyxdt+p30ikrt6jUrIw3zl7NLLG3mty63dJXFzSPShct95/6x8/fkIScI9X27Z9Z8jValSvKi8//4zZxvc//iIfjv7CbTts8KtSpnRJ93OoibeHjxJE5Ngxx+kxgRgRyhBNNGzQy7Jp81a554HHTTMIF73v7CEz/p4lr74xxG/VqupLeebxB6VcuTISr33jdrFXv7M++uRL+ekX/f4KYUWLFpE7b7tZIDjBvhozVj77Ur+/QhjGfd01naVThwv1VpZblixbIU/2fTFE68yZ/fILz6hP6VSNKD0hnbvewqinzMGeaVuhqJNpqLkhEiABEkiZQOlSJaWuCjkQc06tVSPkChtVwEFUziwVdCDsZJQFS7+WUdtivyQQLQGkMmnQUJ0ytGQE5syeLRkt7CTbKGeQAAmQQHoTQORAQ3XewsnlcYxLkWKht1T3TEeg8baABwfOXywLFHUg5Fx6ZZL4g/XQHturWl2kkAoSXlEHERCn11OR4uwkpzbWKVMWf1M29HtRJxUUfI4uRBIFGuZhTLYNlsO5BqEH24Hz2+sobd9BBE7VQIOD9eJLRX5RBzMc3tYwhg4qKkH8gtn9xb6e1dQRt3761pnvtND+a4u0v9h+ct4xHvRx7U0i334lsn2r//ISuo8QmmrU8p+PaAwITl5LzZi862Ma3DBGjAtm351Pzt/m5zsCoXcexInT6jr7jf1IjTDh7S/E9Jx5C+XxZ54PsTRgdrDzGE3gsIeogmM79Xf/laLdt2jPG5x7l1+VnCuOXe3TRcpWEPnyI40Q0+snUsM5csU1SeIM1kN/EA47XuGIqosX+Pd28WX+5y6ELlzfOM+qVhP5ZKReswf91wn3KTWsy+m+FgtzH8K9CwahIlw7p1XSX7teasaUmvtl0padqdRsFwIbBB1rEElxTeEYXqbny496Lwkmztn2eE/NuZXWezG2G4v3nSJ6bUdyzkB8C2FvjxgleMHuvet2ufiCNiFahp49/IOP5buxP4duoEsgYrz47JNSuJAebzX4MrwGQacgIrdSsAr6MOtvv0/RW8HNpmX9M+tK/+eeCrlWgzPqmn69aS0bNzzTzIsLyGqCvp7v+7ifSITDXkwFG4hAFSuUkxEjR/ttC9E89/S6TRppynu0tVZFI5mCWT69zm+75Qa5oN35flGZlSsGb48+Pn5/qEA0gugzb4E+TJFBVgDft2p4uCBfvrwUdTKIc1Z16/u1k1Wb53ZJgARIgATwg8JE5KiQg6icUHbgwEFZsHiJLFikL33PDAuWfi0ztsttkAAJkAAJkAAJkIB0u81xugGFdRSGwwLHIJ6gh23dIjJxnMiePc5nOPzx1LzX4PS3gg5Sm/3yk8jmjY6YAed5JRUsDgREoFyiDmXMh0EMgXm9Ps6c0H8bNXGcnaFbqDDRIUnQQaTQiqVOhAa2U06dRAlH/dcuV85Z/vef6jxd4zi0a5ziRD9gnRZt/UUdRDFZQWeSPqW8VB1KeCq6ic5H9AOWwUG+crmzHbBod5EzDZ7jftDHnHfpWNSxDcc7OLZTMeXzD5PGVUodulddl/QZfG0UVNLcpKlox5S0ZtLUBcotmJBjWyDSCBFfMDD9U6MiEMF1ah2NplBGENGaNBOZNtlpk5V/cS7DEJGDse7a6Yv60HFCfKunAuWsGUniRWr2LdrzBs5m8F27Ssf1nz6+v9VxPjdr4ZwziPiqof+WWRHFv1NaX+AIMriWICRuXK+ClYolF17iRJ0hgmeVCpI29SCuPXvuTtPjt2COcx1CnIU4hPFhPL//6vCL5G+0rNGnOkeNTZ/mXHPOp6S/9r6D/fn2i6T5mGre2hExEEU2KWCce3QeLDVjivZ+6WzJ/2+02wVvK+isXOaIcLimcK507OycH60vVA/2CP/tBH5KzbmV1nsxxhCL953//hFZovfkYHbqac5DDlgWLlo12LoZMO/hPncbcSRU10+oYFG4kEbiBbEL27eWVs31O0dt7vyFQVqEnlVHI09g6zZsdBtVr+ZEzi1b7nmAQZc++uC9RtDBLebNt0fIbxMmSy19ePa5px+VIlp36MrLO8p3GmHkTU32/tuD3K91RA8Fps90N+qbGDb4FSnvq7MW6c+C4ircIYIKQktG2rMvDpBrulwm/8yaI/sz8GHgjNwH9h2agN6BaSRAAiRAAplNoGiRwhqJU9MVc/Lm1X8sBzP98WGEHJ+YcyCap86C9RflvERfyLVdLT7KdBN2Pb6TAAmQAAmQAAmQQNQEkOpxrYoU82ers3eDyG13hXfaQ5SAwVn6nTpSvRED+A21XJ3jXkPEjRUBvhid5CBHGzz9741usevpk8lGMMKYVqjocVFHJ6LHLg/3jifXEbkSzhC5gEgcWGCEDbxFm5VDoE3/S8e6wj912BJ1klWp5kTw4Clq7Kf9XVetutPDrh3qPPQ501BPZ/pU3XZFjbhQBznEDSvq1FFOVrj67ktHYEMPiBb6WQWeTuq8hUO/fAUd3yanb6T7wfYWaZTF/Lm6vs5GRE8oi3ZMgf1ArIGwAUZ2rIFtGqugBoPDfcIvSaLcovmO8xkRB0hfNuNPFc70+GelwWE7c7qekweSRrF2tcg/KuRA6ICVLuukjcN0avYt2vMGwtI3n6uYswVbdAwptiA83nCL8xlCUaSiTslSeq5pe9j4n1QcWedM79Xr93uNmLrlDudYnqHHBGkKYVaYxbllBR3M37JZxR+9BiBG4vyNxqJljb7z5ne2gOg0pIoLZagPhLF5zUaCHT2cfJltl5oxRXu/xDWKaBdvLa1ot2uPH8Y9U48RBB3Yvr0q/P3riKW4/+TW+493OxCKcY+1lppzKy33YrvdWLzv4Jr3Xvd2rEhjeWZD59Mq/e6BGJ+F1lRTkgWr2+Md0lJNPxbKHnmgt1m0a/cemTNPvycisDw4b9RqVNfvNrVVq9Zo8KIzr6ymZIOhxhnmHdPzq2H9M6REMRUY1UZ+/KmM+3WimUabO3o/JKM/GGa+Lnp2v0leeOV1swx/Tuj3CAQjpFv7beJkGfv16LAp7OL1OtqlKe/Q/xdffydDXu9vIoDcDgMmML5IU+IFrOr3sUCB/II6SIiQWrd+o6z3iFy24Zat2+TNYe/Zj2HfkaYOfYFPZvudwg6MC0MSoKgTEg0XkAAJkED6EqihT4/UqlFdauqPkFo1qukDkfojOoStWrPOjcrZtj3MPxRCrJ9es5l+Lb1Ish8SIAESIAESIIGoCbw3LMnxntLKiMSpWt1p9Y86w72CTqh1GzZ1lkDYiPTBmR++iXxM3u1CVEEqNBjSSUEA8aZXc5Y4tUIwjfo2wUQl2877HsqJjj5gEDrg8IVB8IC4BFu2xHn3/l0wz3GKl1CHO9piXaQmg0FQQsSU1xCJAOcsHGtIwWVFHURxeI9fmFRBqRqTdwzgiNRjsIkq1iBqKNCwLxC5YEv1eGO/vLZI50HUMe2qOgKBd3lmT4dKV7V/X9JIrDiQ2n2L5rzBVuF4D2aHPefEYRUqIrXadZyWOBa2joxdF+LAutV6TdfQ80/PKyvq2H2GGIGXFSqxnmVz+IDtJbL3aFjbHn2OZDmqok1GWGrG5L3eUhoT6hh17eaILUiPtmGds0a027XHA2vnz+e/Ve/9J9F3/0GLLtc59yAIlPa4pubcSu292I4S102090K7Lt4z874DAQ5pBzFmRK39Pt47kkyfRkq1xx+6z2x30ZLlUqVyRTcFWySDaX7u2YKHXGGfqwgSiUHA+PqT9/2aon4NXl579cW+5uMlna83UTj4gFvMtz+M8zaT3Srub9y0WSpVLC+NG2nko8eirT1z2119jIjk6SLoJNKtvfHq81K8qCM0odGzTz2qP1eOm/aoLfTjuPHS75nHpGGDM2TSlGmycNFSufnGazWqqLAZ8w3de8n9ve+QVi3O1UvOP8oHwtLoT7/yqxP04H29BPWGkPWla7eeZjslSxSXUSPeUi4n5L6HnpQH7u1lfFQ4vayNnzhFXn/zbfuR7zFKgKJOjB4YDosESCD7EyhVsoQj4FRXIUdFHHx5hrOt27a7Qs7qtb4f1uFWyIRlTL+WCZC5CRIgARIgARIggeAEAh3vwVs5cyFCWEN6KGvwUgTrB04yXx0AP/EkVHvbX7C+7LJw7y3Od9JmqWNFpk7W1GRdg7e2TkavoJPSmIL3JFKtprPEppQLbBesNtFOz8NEeDJci0mrt85ZM8hTwIbtVo1EQFospFDyWmpYRTomux08rW/FMkQZILqonV3oefemANqoEUaBtmeXc56AdeB+BLbNys/VfccUbJFiEJbe+5bSeeNsNelvZRXBrEGIidRQFwgGAfC449R0Zvj+IjoPog4EW2uIxjm3pePcRkrAyb+pk1uFFUSC1KzttFqu50F6WDDW6BfnCF6wIyo+2enUnO9OL5H/DTUm9BDN9itWcoRYrIc+raiDz8Es1HZ3arQfBDjUNGp9oZOacfdOp4c6PjEY54QdG9rZe9wppyWJOsG2iXnhzi3bZ6h1o5kf6/cdRKGWciJR5Mfv/aOcotnPdGrbT2vUQFA4cvSYPP3sSzJy+JtR9dz9JhX21LD+92P9xZZQHSUkJMr2Hc65ZWv32M/xev0jIgenxI6dO8U+mGrb7d67N6josnDxUiPq5MvrL0giyicai7R9bn2o16Zps/3j9hGfS7/H1IoVK2LeC+k9PV6/2yDGtGut9zuf5cZ9Tu28c5q6gs7hw0c0EDde8vgY3H1HD627PFc2b9H7qprtC3V/rKEWEvqHIaoIBmaJicc1s6QT+dS+TSsZ+u4HenvNIOHabJV/0krAOSPS2gvXJwESIAES0H9L5PaJONWMiIPInJQMYb3LVqzS10rzSql9Zi9P1B9PXgsXXeRtx2kSIAESIAESIAESyFQChR1niHlyHwJDw7NESqoTDE/UI93WhrVOejHrrCnoPCVsxghHyYUdNbVTRafuC6JRIITM+EvXVWd/Wq1qdRGkMIONU4ecN7rAmZv01wooCepUQvRJdXVqw/mNdTZtEpn3X1LKraS1kk/VOtVJKYYlSzQyyBq8XniCHnVZ0OaPiUkOV7Q5pE+BWyuk24WoY2vheCMybBu82ygniECpsdSMyW6nWXNnP/H0Ohz8ocyeH1geaj8Oax8o8O5tG6q/rJhfRM9xFLKHQcCyTm3veNO6b6HOG2eryf/CI3leK2c+rhukY4vUsD8wcA9m9lxElBsyHCDaA2m9kCbwnBZiUq1VrabCwN96LtfWc1qP3aYNoeuRBNtGqHmhWKO9jdLB9DXXJ6VwxHUFUXGm3jfsfQZt0svCjSnabaxd7YiCcGYjcjCcpbRdpIm85DLnOrz2RpGF85zjhbpeOCdQu8oaBCBsr4re11ATKpyl5dwK169dll3uO0gr2aipM2qcW970h3ZfMvH9sk4Xawp5PbZq/QcMloP2Oo1wDKfUquGmJkNqs0gNwkm323rrZV5Qvhw9QlDrBp9h57c8Tx7VdG6bNm/RLK193C6L+VKvHToY/B6zc5fz/Y5UaIgEOnTosLtuRkzs3LVbEEFU7/TTxEYUPdPvZVPvJtj2ILwgPR2id+Az2qOROLDhH3xsagCh1rIVlC65qL30vrOH0ZhbNm8mX37zQ7Auk83bsXOXvPHWO/Lvf3PNsq5Xd5Zu119t+rms40Xaj/5mocUsAf12pJEACZAACaSWQKUK5QXijXlpWjUbRhyyP/139FKfgAMhZ/1G/cd5DNsRmxfZN8YC+fPH8Gg5NBIgARIgARIggRxLwD5pDQfwRZ38MWhBYimmznA8nf7tl86T/TZKAC0D03XBsQ+HZNXqImO+CF8zw39LyT/ht1P7Ds58pBvavjV5GzsHTkwroCAVmNewXygQjxeKqy9Z7F3qP41IE7tP69Ykbzv7H6dYO56av7WXk4Zttzq3SpUROUWd415DG2uhnti1KZg0JU+qLZox2Y2g9lD9Rs6ncT85jnTf08e2ifuOY2ot4Petna1eQhV19BNSU8WaIdXYJZc7USEQsKaoGGctvfYtpfPGbs/73rJNUmTTT+r8s0KTt02oaRuBk9J5hfVxbu1X0QS2QJ2PSMmG6DxcF2ef58yH8IlaSRBE02LhWAf2i7bYZ1y7EEpR86T2aSKjR6avsBPNmALHGOwzroExnwdb4j8vku0iBeOSRUmCoxUe0RPShEFU9xpqMEViaTm3IukfbWL9voPoig6XOnuDyLz/9N6dhVaubBm5vbsKd2rT/popM2b+G/Voet1+i1nnuKZI/eDDT6Nev+7pznfU3r373HVPq63f12oQdbxmfReHQqSFtCIJ1qlQvpys1Bo9sWT7NbIX6dYCbfyESYGzZNz4iXJXz+6mVk/5cmWTLQ8145PPv3EFHbT5+tuxctN1jqgDAY4W2wT0G5BGAiRAAiQQKQGkULMiTs3q1aVsGf3HRAp2UJ/4WLZ8pYo5K8wTFvaJkBRWi4nFgQXyChYsEBPj4iBIgARIgARIgARIwI9AvJNKxMybM0ufBp/vOBNRjP10FXTq6Qs1ECACoAC9PpnrGlI6oag30kDl1986p6pTFhEgcBi30lxekTg/3c4CJtp1cJ7sRzH1WX8HLAzzcdcO54l/WzweItPZOiZEIyDN0dIlwR3oEGEuv8pxMsNxOyFIahtTN6eCs5/YRxtFhOHAMY55sAN7/R3TvrQszkLPXyv8HNbtpdaiGRO2gW3atGtz/3Pq/YTbthWe0CaPb/8C2+fO48wJ4QAMbJ6pn9vpMcf5CzPplzys02PfIjlvnK0n/T1do89OP8P5PH1acsESadLsuZG0lhNphmtO0waJ6iB+kS/edt6IGCv8QIy9Qs9vCCiIysE1Va+Bk0IM5+0N3Z0UYGtXOz1hfGXLeXt1pveq0BDKQR6ONdbGdTXyXX3X8VsRC/t5ho6j6bmOMIt99wpvyUcQ3ZyUxhRdb5G3Tmm7iKBClCMEcETloKYV0vfVq+/cgyBEzprp3Msi36qeVymcW9H0Fa5trN932l7kXEO4LyMiyp5v4fYpA5f1f/5pk7Zr3/4D8tKAQVFvCenQ6tQ+1aw3Z978qKN8sKJdf9Nm/b72WfVqVczUytVr7CzzjgdU82utp3wBtWdsI9SpsbZtu37nxpih5k0og8CG9GyoZ7Rbo3mw7wl6nuSNyxNqlYjmI/IH/SCdm7WK+iDzg/fdZT/6vX/0yRcye67+3qJlCYGko5Qlm+dGSYAESCC2CSAMFyJOrRrV5ZSaNQSROZEYwlgRiWOiclTQCfV0SCR9ZWWbgwGhygh3ppEACZAACZAACZBAzBGwRdLh/EJ6JmsQU6b+rvU2ajmptSpXc0Sdg/ttC5G//xRBpArs4AEVeFQUKq1PuiJqBQ5hRH8Eq/nhrBH6LxzKiKyB4en0vJ6I51zqDIXBcY3oHEQXYOxI24R5C9VJsmql0wZ/l+qT8LnUwQNBB1auYnIhAw7WS7s4Dm84g8Z+40QlOWsk/cUyOF/nzXZq4RQtpmmt1MmNJ8HRR4fLnLaIjEBbO6b8IX4HQgiDeZk6cyL/G82Y0GvrCxxnJ6JWIMjZCCevuIfUUph/VFPqqBPSNYw3MHoAC/OrYx5mzyXnU9b/RXqzmqc645g6KXn6pbTuW6TnjZdEtRqO4Il5OE9xDAINQqoVCL3LMA+iDlL7ob5KqAgve17ZcxB9NDnbOb+3bBL5Qc9vLFu/TqSo1i7t0MkRvhAZ98HbzjLUgqla3bt1ZxoCRDBRJyXWtievkIZ5EHogXpSr4GyvSlXbMu3vkY4p7Vvy7yGS7Vav5ewv7l2ff6z3G73WcDyQJqxlW+ce2ripLxpwp3//oT5Fcm6FWjfa+bF836mr3x/23B3/s/PdFO3+pWP7O2+7WcqVKW16HDx0uJRSgcYa0pfBCuq1XEbbHNB7UrC0bHfcerMJakPbYcNH4S1iw/YRkQOBAVarZjV5/ZV+ZrpGded6Q/2Z+mfUlVEff26Ehr16jylWtIgZl2kY8Kd48aJmDk6Dffs8vwkC2sXax563dpPLO17ssszo8aFWtE25F7itBvXrUdQJhJKJnynqZCJsbooESCD2CSBy/pQaNYyAAxGnWtXKEQ362LEEWb5ylaxas9Z9RbRijDeCGIXQaPeHGkWdGD9iHB4JkAAJkAAJ5FACcBDD4DCGKAIhwmurVzlPf5fwOaJsKie0KV4iSdSx68DpbFORIbVbamrreNMQXXmt7dn//axmInghNdu/+jqg4kNxdVBjTIGGfbBWWiOQNm+wn5zII4gxtgg5BB1EHoUz1GYIrM/Q9BxnDTip4emCHVC2iBKxfTtzk/6WUgEMhnonabVIx1RDnckwiDY33eZMB/69rIsz59sv/Gu9YD+2bPZvjd+4VoDYnw774d976j81bOKk9EIPcJQj9Vig4fhYi3bf4IyN9rxBjQ8bJYXUW+N/tFv3f0f0jjfaxi6156XljNpXwQzCKsz7kBki1mAQPe35ic97d4v8+pPWuLnR2WYFFT036vWxSNuhPlagBavjEwnrwH4CP+P8rVrdqYMVuCw1n9NjTBm5XdQygq3U+6VX6ML9w0Tt1HCOB1LS/a3nb0oW6bmVUj/RLo+1+04xvf+3aOPsxaIFIms89/5o9y2d2qNujbUnH7nfTvq9n3dOU8Fr6l9/y4uvvOG3LJ+mkjun2Vlm3tr1G2T9hiDXpd8a/h/ObtJIvGnFICCddqrve8DXFFlV8Dqz3ulGaMCDtlUqVRRbW8e/R9XKa1Q3s1CfJ7tY/TPryhVa1wi2R1PQTf1zhmzQlP6I2LmofVvXb5Oe+4Pj9fuUP4N2OctXiyfoQs7McAL6i5dGAiRAAjmXQHEtnletShV9VXZeEYo4ILZu/UZZtnKlI+KsXqsZBI6clCCRgq2Ir2hvIaZfOymPMXeKBEiABEiABLI9gZ0akWMN9TbmBzi/K/oe1EHaJViCPlmOp/VRjwSpflZrtIHX4BS2llqx4uB+fYJfBaFgZtNSwTENh5J1iMK5CFGntu7Dn5M1QsgnrKCP8p4x+Qo8m67xVBLqCFWq4mzpp+8ch7bzKfK/SD/WoLHTfv7spPUgJjUsqWmV1EGLGhvHE5OWQXxCSjhYIENnbtr+hhoTInRstFPgFixbcAXfRB0vXtu3OcLUqXVUHJnnv5aNhMHcDSpUxIIhnVcznyP1X40CQSRIMEvtvqXmvCmnT8lfdqUzCoiKP+q55hVXvOObP8f7Kfk0zhfUoIHwg369QhvGVtMn4Kz1XZt58ialczsW5N9dNtoOWyqm1xBEHWwjkvMyUtbJ98J/jk2RB1EjrZZeY4p2HNFst7RPkDsWZH9xXuzS6BxEOwYTqQPHFc25Fbhuen/OyvsOIkM7Xu6kr4Pwiei8GLD9Gn1TyN7rA8aTG9F+auZ2q98P++xDFp52N994rUndhlkffPiZZ0lkk08921+KFCksL2sKuLx6z3jupddkl6YdwzTmJWo07UOP/890tnGTI9r/PnmaNDyzntluqxbnypSpScJinHK2NWNWrNTvuCyyXLjXRWGtWzZ3Wz/0eF8VdJx9xcw2rVqYdHNug3SaQO2hV19/K516YzfpSYCiTnrSZF8kQAIxT8CIN1WTRByIOpEanoRYunyFvhwhZ3ewtA2RdpaN2iEFmxV1CjJSJxsdOQ6VBEiABEiABHIQAaTMWr9W051VdWrPbNKngJF6DZEIqPFQzCeubFiXBGW2poxC7YuqKlYgKmf5UmcZoh0gqsBQ28YrYjhzI/v78w+h212rEQVwAM+Y5qR7sy2Rxgo1feDobtFW5I+JjqcMNUQaNXFawXO2faszDYdQu4udfcCcsWNUlPDso9PK/y/2L19+jWDY5KR8w1L8xoMwhGgV9I86Ndbm6XRDfcLaiACXOLUdIDZBPLlYo4NgEFkicZ47rZP/jXZMI4cn7wNz4BC9vbez7LsvnXPAtkT9FdT/QIosiAlIPwcrpY5ppJqCrVruiH3Op6z7C6Gx+fnO9m0UV7jRRLtvqT1vLr/aOQ9Qz2bst6m/NrAviPKB0xp1ci7Qc++r0VpnB2KdntNtLnSuAbSbPQt/VfxU4QDnGaKzmul1i4gfG3GH87aJL8oMbTfpuR2pRcu6anXnGsH2bcQPCtpDGKylLxjOo7RYtGOKdluol4EUhrjuJ01wIp3QR7TbRcrGInr8kGpy2WLnnoJ+cAyr633V1jOCwGat8dkiiLRD2st1a5y5uP7T89yy2/K+h9puNPfCzLjvnKfXPZjCIJqm9vvH6SHd/t5+9wMh+/ri4xFSWAWfCZOmyMDBbwdtd/EF+n2mtlfTnM2Yqd9zURqEmrgtcUbEQSaR6X87fZxRt47pCf0uWep/3Y2fMEl6ado21NXpc8+dZvmWrdv0ayJOBg14wRWZ3n3/oyhHk7bmXiHm3GZNZea/vu+iCLrNnTvebZUrl37f+ax92/P1K1m/k2k5ioDeyWkkQAIkcHIScKJwEIHjE3GiiMKxRJYsg4jjvDZt0R/uOdAOaqSONUbqWBJ8JwESIAESIAESiDkCM9RJiGgVCCJXXa8p1XZrFEmhJOcwHML/TE8aNlI4QaxAtA6EkXNaOMsgoFib8Kudypx3CFGoUVKjpuNghdhk05/ZEfz1hzq3fZEKcJpaRzKWd+psW/m/I0rl60+deUhhhToXMIhWceoWsKIX5k36Lal/fMZvQYgGcIpCALutt1N3xjoe0WbK746jG9OpsWjHlJptQHRCLRaIOhBxIAJoCmU30giRPTP+Sk3P6b9Oq3ZJfTZppmPVVzB7Z7AzN9p9S81506K146zHFitUUvHsbmfbgX8jERbtOpMnalTCFc4xuPl2R+QpVCRpO0g3h9Rq1v6apmJEe+d8vaGHXhv7nfOzTFnbwhFUdu9M+pzSVLSscc/A/sNwzuD68F4/uM9gnGmxaMcU7bYgTNn7xhn1NSpwitNDtNtFykhEVEHEgSgDHjv038xl9L4EoQ2GY7RkoTMNQc6meDxPhbnPfaJORpxbzhbDbzcW7zuoRWXt2pvsVPJ3XPO/jE0+PwbnXN6pg+TP5wgOX3+b+jGj1jEMmUSs1fOJOlu26HdcEHtv1Cdy9x3dJV/ePPLBO4Pk8OEjglRwOGVh8xYslqXq88lMw8PB23fslNKlSsrFF7SRdm1aqsAULz/98psMffeDsEP5Y9p0ad+mlWkzbPArgv1GbSCko6PlPAK+u2zO23HuMQmQwMlFIJ/+SKhcsaK+KkhVXyq1aKJwQAPhu2vWrZe1+sI7XjT90YTUJD5jpI4lwXcSIAESIAESIIFMJ4DokKSHVJNvHtErX472OYlVmEEaM2so3j1pvON0tPMS1AH52YdOhArSs3nFHDgif/tZHZTBHUW2C1UykiajmUo87rTWlDHJDDVKmjV3UqFBoLIpnVCMfOpkkcULklaxnqmkOcGnEH1jbZP+xj1Qz3Gklyhl58JTJjLhJ41y2Jg0z07NVDEMjjREj2CbVtBBmqlfdbwpRQelxCk1Y7Jj87579zOQLZZ995UToVBbn+7GU832yWYIgD9+kxT54e0zK6Yx1kiPLcYX7b5F2reXZ6TrBHIPxw/Rdah51OFSJwIHUTswbBc1hP77x/ls/0IcwLJz9PqAGItr1l63uD7++1dfM23ryN7RX6T7hh5RP6ZkSWe8uD69gs5SjVaZMdWJKEpp697UioFtox1T4Pr4HO5+uX170hqI2LMW7XaRXm2MHr9WbZ3UhuCBujjWli0Rma48IPbAjh527jNI4+WN3omUf9hzS49jKAu13ex83wmzu6EwZNT8Ezhv1BLt91rAhrpc0dHMOaZpT9Mi6tT21dDZ5jl/bQ3kNWv1Oz6I/ThuvOzTiL4H7+sleXLndtOTYciTp/4prwx8K8ha/rNOmO8vnwrkvyjoJ8sj6ELfzBEjR8ttt9xghB2MC9FHNhPM8TDRWYjqGT9xilzQtpWJNKpYQQVUtf/mzpeKFcpLuTKlTV++zWigV6KZjGRMdh37npDgrGs/8z32COSq07h1ht8K9u/dFXt7zhGRAAlkWwJeAaeyFr6DkFMGxWKjsOP6g9AKN/Z9r6ZXoyUncE3ny+TssxqZBfs0l+6z/Qckb8Q5MU2gQEH9B+9JaIUL5pNmzZqdhHuW9l2aMWOG7D/oe4o77d359YB/HBzBU6i0bEEgt9YgyIM6BLRMJXAItVTSaIWLlkhjD8lXz1H/LsufX52MZZ1aKjt3qJNVnYnhTJ0qghRAuF7gqEQ6t6w2pI6D6FK4iFZE3qMRC/rv6nCO4GjGCydqUXWgw4kOXxXSKEVaBwRFvLEuuEL8Si9Ly5iiHQPStJXC8VYnNIQ7G/kUQT9tW7eQh+67y7Ts8+gzyVL+BHbx3ZcfGmfinHkL5fFnng9cnP6f07Bv6T+YKHpEKrCSpZ1zCvVxfI7ioD3gXClSTF96baCex969+tJrJIwzNGg/aZmJawfbR+o11N3CNQqhOLsYBE3Ujjl4IH1GjOOHWkZIZ4aaKoiwsmKOdwv22HkjsLzLM2o61HazyX0ntVjuvet2Ew2C9S/prBGsYayapqkfNuhl0+Kd9z6S78bqgw1ZYPXPrCv9n3vKbHnAoKEycZIKg+lg8B010Bo76zds1AidRSp+BHmoIh22E00XxVUUhn8LaeGisZIlikt93ZcDBw7I7LkL9FLLRveeaHY0wrZp+c0cS7+No90P/eVKIwESIIHYJZAeAg72bsfOXX5ROOv0i5wWGQFveDPDeiNjxlaZQwCiBcQLWnICGSXoJN8S55AACZBAjBI4rCIOogAiNX2C2NSYibR9ZrSDgIN0bHilt8FhDic0XtHaHnW445XelpYxRTsWOPO2qZCViVb/jLoy9uvRZotz5i+QJ/u+mDFbz4J9S5cdQSTYwQivWZwrEAUyWxjw7ijqAeGVXQ0ibqRCbiT7aI6fHsOUzB67lNql9/JQ2z0J7zsQci5s19oQzAXRKhXWs8dNcnv3G82a73/4iXzznUZkZqA1Pauh9H3i4Qzcgv4kUB8QXrFkNjon2jHt3LVbJk2ZFu1qbH+SEaCoc5IdUO4OCWRnAukl4Bw5clTWb9zoF4mzXyNMaKkjcPBg0hP58fFxUkCffD0ERwmNBGKAAMWLGDgIHAIJkAAJkAAJkECKBOBbtQ7W/IjuoJEACZBABhDIp1FYcYj0TIN571d5ENGYwRanNWXSOuYMHiK7J4GYI0BRJ+YOCQdEAjmDQPlyZaWCvvBevlw5KV+2rJQqqeHaUZoVcNZv2KRCDl4bZdt2TQVBSzcC3kgddFqwYAGKOulGlx2RAAmQAAmQAAmQAAmczASuuv5WdVZqujeP5fRUOR4UnCQBEkhnAq8NHiaDh43w6zWSVGOoS9O5a3e/9fAhM+5XM2b+m2XbTrbDnEEC2YQARZ1scqA4TBLIrgRKlSjhE26sgOOIOanZHwo4qaGW9nW8kTrorZDmS0Y6OxoJkAAJkAAJkAAJkAAJ5DQCCxYukdGff212e+269SnufmY4RFMcBBuQAAnkGAIQcI5EUSfMCya163n7SO10Vm47tWPmeiSQlQQo6mQlfW6bBE4iAkUKF3bEG424caJvHPEmb97UhepSwImdkyN5pI4WwaSRAAmQAAmQAAmQAAmQQA4kgILWoz9zRJ0cuPvcZRIgARIgARIggRggQFEnBg4Ch0AC2YkAUm8hVVq5smWS3suVkcKFCqV6N3bv2Subt2yVzVu3yoaNm5lCLdUkM2bF5JE6BTJmQ+yVBEiABEiABEiABEiABEiABEiABEiABEiABEggLAGKOmHxcCEJ5FwC+fPnV9GmjCvelFfhppyKOUWLFE41lEOHDssmiDe+16YtW4yQg/m02CVw4NBBv8EV1PRrNBIgARIgARIgARIgARIgARIgARIgARIgARIggcwnQFEn85lziyQQUwTy5s1rxBsTfaPCjY3CKV6saKrHmZiYGCDeOELOnr17U90nV8w6AozUyTr23DIJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJeAlQ1PHS4DQJnMQESpcqKeZVspTzrp/LlSktJUoUT/VeJyQkytZt253X9u0q5GjkjUbhbNu+I9V9csXYI4BCi4imKlAgvxkcI3Vi7xhxRCRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAjmDAEWdnHGcuZc5hEAw4cbOSwuCAwcOyhaPeGOFnJ27dqWlW66bjQgcPHTIFXUKMf1aNjpyHCoJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkMDJRICizsl0NLkvOYKAFWlKeyJu7Ly0Atixc5cbdWOFG7wfOOhfUyWt2+H62Y8AzoFSJUuYgRcqWCD77QBHTAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAInAQGKOifBQeQunHwErEiTEcLNsWMJrnCzTVOmuRE4Kt6gFg6NBIIR8NbVKVigYLAmnEcCJEACJEACJEACJBAlgYoVygserDpy5EiUa6a+Of6tsXuTwur5AABAAElEQVT3HkmIod/+xYsVk+MnjsvevftSv2MprFlMa4YeO3pMEIF+stnDfXpLs6aNZNPmrXLPA4+7u5c7Pl5at2ouc+YvlG36771IbPiQgZqiu5j8NG6CvP/hJ5GsEnWbpmc1FGSDWLh4adTrcoW0EcA5UU5r6eJcQZrtzLJyZcvI7j17M+xeFxcXJ+XLlTXp4DNzvzKLH7dDAiRAAoEEKOoEEuFnEsgkAhkp3GAX9ug/iEy0jQo3+AG/Vevc4PPuPXsyaQ+5mZOJgDdai5E6J9OR5b6QAAmQAAmQAAlkNoGLL2wrN1zbRUoULy5xcbnM5iGwLFm6XB5+4tmww7n0kgvl5huvDdtmzPc/yejPvvZrU7VKZXm4z91SpXJFyZsnj1m2X53qX37zvXn5NdYPr77YV2pUrxo42/38fP/XZfbc+eZzn3vulObnNnWXhZqY/ve/MuCNoe7iM+udLn3uuUPKaJ3PeHXIwhLVyTz5jz9l0JDhcuzYMbetd+L9dwZJ0SKFpetNPY0whbqPH703RHbu2i09737Q21Ty6L6+8L/H5dRTakm+vM5+Q9T5c/pMGTj4bb+23g9FixaRO2+7Wc5u0sjM/mrMWPnsyzHeJsmmW7U4V265qasZ25EjR+WG7r2StcmoGRXKl5OCBQpIWWXptZdfeEZOP+1Udd6fkM5dbwnJ1LtO2bKlJU/u3Mbx752fXtPXXnWF3HzDNaa7vi+8KjP/+S+9umY/YQjce9ftep2eLUUKF3JbHT58RH6ZMEneGTHKnRfpBMShd4cONOf7ipVr5NGnnku26mWdLpbrru6sbYpILudWJ9jmh598Id/+8HOy9nZGzRrV5O47bpVqVSuZWY89/bwsX7HKLnbfMYbevW6Tls2bSYH8Tv1XLMS94JXX35K58xa6bTFh7x1+Mz0fjqjoe8Mtd3rmcJIESIAEYpcARZ3YPTYc2UlAIKOFm4SEREG0DQQbR7jR6W0q3ui8o0ePngQEuQuxQsAvUoc1dWLlsHAcJEACJEACJEAC2YxAr9tvEQgzXjtxQgTOyXqnn+adHXS6uApBcN6Hs8qVKvotxufBr73gijnYHhyshQsVlO4qQlSvVlVeVQeo16pUrhR2OxBirEFISGlMaIs+rQXjgGUQd9qe38KIE488mdxJDE7l9Yl/iD820gjcsH3v71X0hcicoYNekRL67jW0bd+mldnv+x9+yi9aAdEE96iTuFGDM10nNNatEsDU2591XBdTIchagfzhj5Ftl9Hv1tGdSw94vnx5IxJ10mNMYHLbLTfImrXr/SKH0LdXVChW1P/YpMe22Yc/AQibA19+TmqpUGLN3gPy588nLVToSY2o88wTD5lrEX1CCAy0oW+8rNdYlcDZgm327HGTxOu1/PW3Y/2WNziznvTqeYtU9dwr0KBUyZLJRB3Uef3o/aGSX8/rQCtZori89OxT0vuBx2TV6rXuYgjpVtx1Z3omYuW69QyJkyRAAiQQkgBFnZBouIAEUiaAH0il9AdDSa01UqqEvvTHhivkaFqD9DKEKe/YudMINo6I44g3O3ftSq9NsB8SCEvAG6mTJ09uyatPOh7VJ5loJEACJEACJEACJEACkRE45+yzXEFnl/6+f/6lgbJoiZN+CsJLG02TlZL98OMvsmDhYr9mEGgef/g+I2wgIuOncb/5LUe0BqJzsOylAYNk2l9/mzRFA176n8D52abVefLpF9/I+g0b3fVsNM+ESX/I75OnufPtxNLlK+ykvDlshOnPneGZKF26pNx/d08zZ8XK1e6SU2rVNNOzZs+TsT//KvPmL5J2bVvJLTd2NU7aM+rWEaSm27hps7sOJk6rfYr5vHfffnf+6XUcMQzppLyGyAQr6Iwb/7t88NGnhtEjD/Q20Sun1Kwul3W8yC9i4P23B7liztFjx1whzNuvdxpRShfouK1BbLJRR3ZeVr4/++IAuabLZfLPrDmyf/+BTBtKDRUKIcAV9kSF2I2PGv2FgO0hjZiaqOcXLWMJ3N/7DlfQmavX2RtvvWNSlGGrEFGqVkkSWyMdCaLSmjRuELY5UqHhevj1t99lwu9/6LW8Rbpc0VGuvLyTucZuvO5qP1EHKfmefeoRt89jCQkmYsydETCBCD0IOoi8+/KbH2Tqn9Mld+48Rkw8q1F9sw3s+30PPemu+WTfF1RUSorowYKzdD86X9rBtPHe19yVOEECJEACMUqAok6MHhgOK3YI4OkhFIjHP3gg3FgBB+8I+08v26V5rbfv2CnbVbzZgXfPNH7Q0EggKwkcPHjQb/N4MuroUaby84PCDyRAAiRAAiRAAllKAA9XVataWeYvXJKsbgNEk3L6NPmixctC1lSBExLpyRYuWireB1oCdwq1G049paZxWiNlmo0Y8baDMxuRI97aDrfdcqNpAifkzbfe7bceBJWPPv3S20XQaaRSnjV7rt+ygS/3M2IFnr5//uWBMm/BInc5HJ9W2Jgzb74RdLBw85atckfvh+SLj0cY5yeejn+y74vuern1IR4YWARuz23km4DwEii+2DbPPf2omcTYvPVZ4MwfNORdWbtuvW0q348dJ2VLl1Knb0czD45Z2y8EglzK/cwz6pplW7duM6nV8KG2HgsY+sJDd0jbhnebOg11WwYPHW7a7FMx6MHH+sqno94RRNZcfeVlfqLOCR3oOj0WSLf228TJMvbr0W6KPNNBwB887HRYU639Of1v+eiTL42A0uHCdgGt0vYRxxDnL87vdes3+olvKfW8RTm9Oey9sM0gnpUoXkwFxmV+52uwlXAcKlasIEj3tl2zRaxZu87vPLbrlC1Tyk4me8fx+VCFnXCG41e3Tm3Zu2+fX6RFuHWwDFFYp9c5Vf8tvcuMLaX26bEcqQ3hF8B55r3ecZ847dRTzPW1dNmKoJzQBgJYEV1/vl63we4ldoz58uUzYuSevXtDMkHaQG9dKnBs1eIc08V8FYMfe7qf7c68z5m3QPCKxrCNB+9LObXg19+NlTHf/eR3v31v1CdyVqMGJoIHETMmwk7vh7DcmvIPwjPuXx/rvRBsXn7+6ZBDw/119OdfJ0s12ff5V2TM5yONIITz1GuBNZwgHl/e8WLTZO36Debe4G3PaRIgARKIZQIUdWL56HBsmUIgb968PtHGRtto5I2JuoGQU0J/XMSn2zh2aG7XQMHGijeJMVSoNN12mB2dNAQO6I9mrxVUUQdCJI0ESIAESIAESIAEMoJAv2cek4YNzpBJU6YZYQF1ZIoULmzqQ6JWiV0+Q+u0wMnZvu35fk91/6Xz39Y6EQ/ff5c6h0/zc8wj5Q+ci9bOb3me3KdRHUgLZA11H/6bM0/69R9oZ0nxYsXkxeeelGrqxLX1IbAQztL/Pf+q67yEKIHUU3DQ3tqrj0mTDKd8xQqOg/Gb734M67x1NxjBBFIg1aldy7R8bfBQQd0ar53brIn7cezP491pTEC4Wrl6jXmK/5SaNfyWxcc5/wbatXu33/xoPsBh27hhfbPKnPkL/CJFfhznPxbbrzfaZrfnt+ZnH71rHMC23Wmn1pLvvhhlP5r3jhe3F7z6v/amOZY2YgY1hgIN0UqXXNTOiBl4eA81OGCR1p2x/b3+5jsqIr1lP4Z9//yj4VKwYAFZvWZdspRkWPGrT97TFGn55G+tMdPvpdcEUQaIiAhML4Uor9GffiU//eIfkRVs43C+t9YIsANaP6lrNydiyrZDzaD2GmWE42Rt3oLFGmmU/N+/EKpwDQY+1AixbvLUP+WVgQ4DnPdtWreU4r60auU0Nd8PX39sut+j476xx13S8rxm8siD95h5t+n1AeHJWm09rn2ffNgVIjEf21i4eIk80+9lje45bJu694A/pk03QtdVV1zqdw1DPH3g0b5+wqG7coiJ0SPfNvv46utDNIqlrrQ5v6Xhj/sJjoldPurjz404UV9FRnsvwDjfHjHSHN8+997ppibDpnAvePGVN/yuzzuU/6UdLvS7NyECDfcn1LqyhvSCTz/+oB97iB+IcMM9zhruiRBCd+zcJTepaAxDlJa9DgYPHWGbpum9/3NPmXst+C5fsVrqn3F60P4C63rZRrinIC0beHkfXv1rxj9+1x9qboUzbD/YNiCs4f6dp3DusCnpq1WtYu7nqGu2Wc/B3vc/5ifKhds2l5EACZBALBCgqBMLR4FjyHAC+KGO/KlOxA3SpDmCDd6RSzo9zYg0GmWDdGlWsME7xJzj+OVCI4FsSODgoYBIHc8//rLh7nDIJEACJEACJEACMU6gUKFCxhkJh3Q7dRJbw9PcMLv8vHOams9wch46fNgtln2upjrDC4YUQIePHHOd45drqh0r6iCiBo5vRCDgpzoiA5CeBwLPGR6nIhzfI4cPdtNxIX0UniTHenjau1/fx9ynvFs2d56Mx7LmOj4UBK97em0zFvz54adf3Wn04X26310QwcQ9vW6Vc5o2Ni2HDR+pqaymhl0LEQVwnHoNkRaotYFoEGsYk3VUQ+zAZ1i047z1lutdh/U7Iz603Yd9h4gBw7GYNn2m23blqjUmxRseuoMTdp+mEjty5IhZjggWGP7NBdupTm2kurMGASfQ/pox04g6mI+aQlbUQRRJNBZNe0R1IWUVeOO886ZDQwF7K65M/uNPMwSc21bQgZM6Xh82zKPnPyKv7r6jh4mgQsRVOLPXCcQir0F4ROo5GFgfOnzICEpn1qvjbeZOt9RoDyvowBGfoC/U68F50lpF0cW6b4i0gihjI8PsylZUwENhsDwaoWHnoa6KtfoqoqAOij33vPsMYWPU8LeMMGXPQ7tv2L41rIN9RR/geZOm+Hrhldft4hTfsU8Y20MqBuP6tWYFX7u8R7frzCKINYmJx02dFmwTtaKsgRNqGaEfvG7R+lVWdEWqsct9/NEHzmfUGgJjiNlW1MH5YqPd0C+EDNR9wTWA44eH7D7/6luzyXp6H4LBx4HIH0SlIVIIhvW86RVTe9+59qor3Do5z74wQLpe09n0H80fm0IR12vg9RP4OZp+bVv4fmzdpjnzFtrZfu+oCfbGq8+b4wKGve59JN2Edr8N8QMJkAAJZCABijoZCJddZx4BK9oEfy+WrgPZr085oZbNzp0adYN332uHfmaNm3RFzc5iiECwSJ0YGh6HQgIkQAIkQAIkcJISgIMVTjdEdixbsUrwtL/XIOaM/PgzQfQLnL0QWF7RGjIwOHgHadot6yTvdsM10lWdknCwQjyYMvUvU8MFn+HYvr77nW7/SFuEVGzWIPygzgzavfjqGyaNGRyjD/e5W87Xvk4/7VQpps52jO+rMT9In3vuMNv/9bdJposqGqkDw3gv0KgiONXRHvsHhyuKeT+nkQBwxEZiXa7oJDbNF5zHcJTu2bvP7JN3fUReWGvfppXrALbz8G8aGBhYR2/RIkXsYhnY/zlXmEGaMdQAelkjYbwpntzGngn01U63B9uwcXNEqbBq1ahujh/WQQSRdd7j8yNPPoc3E80CZ/3DTzxrIjCwnbEaCQKu3W7rbdrgzxW+GhmYH8y2bN3uzq6sxzmlFHNu4zRMILrD1iHpfFlHTdf2hdvbZb4UUGCM8xI2/IOPTZTXgkVLXOf3JRe1l9539jCiRcvmzUwtEbeTCCfwb+Zbb77BtEZUR6/7HnEFpquvvFSvietcYcV2+Y1Gj/ystZr+mzvfbQsxDUILxIxWKmRC1LHHafiQgVKpYnnZsm27dO95r+0m7PsjfXqbviByIBWgTSPYo9v1clXnTuZhTEQXDX33A79+cE2O11R577w3ykTyQDBDbSQ8vInIldQYroclmjJt0pQ/9fxFyrtNft1AhBkwaKjM1KgqWO87b3VFQtSO6f/aYI1iWWWuKYgHqN9UpVIlIzhBjOzZo5tZD9fsdTffYabxB6nNCqugba3PvU6aM2zv8WeeF4ibEGCHvPGyiQTC8bKizs+/TjCRakgxae8jVvA8cOCQ3Ke1rSAeIo02bLdGOP47a7YMHPy23VzYd4jCEMlgv06YbI5PV4lO1Ol6dWdXvPx9SngROuxgwix86rEH3aXf/zjOnbYTuGcMeb2/EeIwb9PmLXJj1y7ylZ7jgd8vdh2+kwAJkEAsEqCoE4tHhWNKRgD/sChZ0lfTRn+E4gktr4CTbIU0zDh2LMEn1FjRRsUa/bFrBZwj+kObRgI5jUCwmjo5jQH3lwRIgARIgARIIPMJ4IEqpFsLZUghBhHFGlKh2QL3q9asdQUdLEfEDEQdGOqwwHlu04vBMV21ciWZ5xON8MQ4HKjWUEwbBrHBRn5AdBj50WdG1MEy1ABBJAyW2zaYD0MdEhiesEeKKq9BpEAkwnvD3pA773lI/y3iCC3eNoHTTRo3dGcheqPt+S3MC07Xu+5/1BUBtqlTHaIKHOx4jfl8lEkrh/1DXaDyZcu4/QSbQKQBnObgg6iRRvXPkI/eG2LSO4UTdlCrBiIYDKJbSoboipf6OVEaEKmeevaloKsgSgHjsbV4amrUCyywBlIJ/TcjLCExeG3SXR7GqOOTGbZi1WojUKJ+TdvWLVxRB05m1IGBoTaPtfETJtlJ933c+IlyV8/u5jxCDajU2Nka3YXjCUN6QW/EEArOo4A9zimv/TNrjvejmUakBRziSCuI1IRpMdT1wb/vYT//MsEVdPAZtZjanN/cRKAgVWKgqLP/wAF546130NQY9meu1okxkU4acZcam63rP/HMCyFXRY0cK+ig0Zjvf3RFHQhMEHRguEegfhREHTDHNYj7CsYMQ00mCC82ygzXlL2ukLLRRj1B1Lb3I6Sg+2X873KzitS4HqwNf/8jwctrxfRcg5UpXVIuat/aTNs/6BtCbxW976HOlFdEtW3sO87RlzT1JO5fuD95eds2Kb3jfmNFIUSY4d6Z3ob7jk1H+YsK6oiOC7T8eq/xZmvBfRsvCK2IeAyVGjKwH34mARIggawm4P9NndWj4fZzLAE8bWKFGryX8gg3+JyedW0Aebf+Yy1JqHEibFDvBvOQcoFGAiTgTyB5pE7SPyD8W/ITCZAACZAACZAACaQfARSuj9bwEJYVFLzrWmcp5tl+4UC+/pouxuGKotzrN26SL77+Xn5Tx6w1ODRtf4X06X9vOiSnL0f0wL9bQll8fJy7aNbsefLJ51+blFWNNNUSUhohwgiORqTV8tbxcVcKmPhu7M/q+F4oKBYO535LjRaCkxZO46cfe8DUH7GroHD466/0MymJUJzcpmzDcivYIDrCOnV379ljol7gvLXz4LS/47ZuRsCCw/8ZrfHx0OP/s5tI9o5IJBgiDAIFrmSNdQbGZx2tqM8S7Il57Ccc44hsslZXxTDYtu1JkTf4jNokMG96LzPD9wfpqaxlZp1I1MG54douglozNrKrdavzTKQUxjNqdFL0Dj6XU9ENKQgRNYYaQxAVIVTljXMEM7SJ1iAgwv7P3nnASVE0UbzIUXLOQclJBVEJAqJiRMygIGJEEUVExADmLAooIKCCiDngZ86IiCBKzjkfOUmU9NXr2Z6d3dvd291Le8crfrc7oaen+z89c0e/qSpcc4gT0RqEtw7ntZPq1aqY8GvwLksrq1P7ZLeqn36d7C7bBXjtIMya9TKx28N9hwpLd2uPrjrh74hn3uM2b9ni5gSy248eOWoXo/revHlr2HIIw27NPnfgwYe8UBB04e00d/5Ceff9jwVeWdbqqPedNeQF8z53EHoOhvsBXoXhwpbl9Kl3uNYQCT+d+LVs0fwx57dvK9ddfbkRlNAOeC4if1k4u7fXbSb/FJ4XDz76VLhiYbfj+fH8UwNNe9GWSM+OsJWksANecN1vuM6UgofYkNdHhTzioHpKTdDnrxOqL696WdYyub8gWCF83j+z5kio8ROyMm4kARIggUwkQFEnE+GfKKfGf4DwByt+kRfzfTvruq2Is83GqE0rJm6INP2PCN7C2qHJ+PCfEidU2i6NexvbH2lp1S7WQwJZlQD+o4C3wmy89UiTFlm1j2w3CZAACZAACZDAiUcA/0fo/8gT8oQmGYcnSiX1qLlPQ6fBmwY5IxBuDF4E1uDZEs67BZPD4Wzr1u1mF8KBeb1Q/pk1VycR58rEj8YZ4aiOvjEejSE3h83PgfJImI7JYbyRj5wcXtuYtMmEeOp02UWCvBvIN4FwUlOmTpMrNIwbvG+8CehxrPUcsPVA6EHYtVNq1jCeGTWqOR4ydr/3GxPENqcFxKeU7OnHH5JqVSqZYl9riK9gEQjeBBdfeJ7W6Qgx+fPlNyIQDrDeKhXKlzfbwAThqDBxDbO5W8yK56O0xzsnOLSWp1iaL0Iw7Hz1Fcbj4epOl8qYsRPkkgsvMOdB2C54Vlm77eZumnelg5kIt9vS4hthtGD//Rd9BIomOkaeGNjfFZ/Soh3eOuCVYg3eP8FmhRFMvENcsvmUgstFWm/e9DQzdoPL1KwRfiwHl02rdYRKg0B8U1cn1F1jzSfUuOEgWaFePHg+QNSsWNHx7sM5GzWoG/LUEFnCCTo4AF5LRTWk2xYNNzhsxJtuHRAXISCNGPK82YY8YOFEHVx73IMwHHdIxw1y0sDg9QLDfYZtGFPBgiz+/zhi6Avm+Yr2PvPCq1F5I5qKo/w4uWZ1GfRwP3OvIIRhvwGPhT0SQvWEDz4N2I+wdA8/cK+5Ly+7pIOMevOdgP1cIQESIIFEJEBRJxGvShZqE95aKw6xRt16wwk2BQum/Rv9EG3wHwu8reQVa6x4E8sfqFkIN5tKAplKAPdaxQLOpAa86WgkQAIkECuBejWdCYry5crGemi2KI+JqoUrkrJFX9gJEshOBBCy7aouPTQ8UVvp1PEiI+wgYfmLzwySy6/pJpt9AgH6/KnmXZj8x7SQ3V+9Zl3I7di4QT2AYJiURt4Mr9cQtq9Yudrk5bHJ6LEtVvt75iyTZwf5QODR4A1JholMtB0/XrvvnjvNKnKrRGOrNaQdwm1FeikPCeFheCP/g48nRqy23713GVEJhcD19TfeSlYe3jjwJrAGht51bIcgh234/ylEnQ0qZFnDm/gQ57zWSCfRra1ZG/662TJp9Y0JeEymN6xfx3jgvPXO+yYUIOr//H/fuKdB+y7XyWUY8q5MmTrdjCF47FzQvp0ZR27hGBcwwX+KToLnzZs36iPvv/dOI+ggNB6EM/QBXkSY7LcCXtSVhSiYtMkviJZRccAbEg7FbTg9iALxCDqoA4JEzRrVsRhguzW3TGYYwkZ+oyLmNVd1NB5QuPdrajhB5OBBHiLvcweeLWAfbEdCbPOWgTgL771ixYp4N5tljHt4q+BeRujJcNbx0gvdXRd3aG/y9rgbfAsmv9KooWZu5oYezjMFu/AsQv4fCEuwVzRMnleMNhtT+QHR/aVnHzPCEhj17vtQMlE6pVNASIbgjmdL9apVUirO/SRAAiSQEAQo6iTEZUjcRkCQsaKN813Mv65CTmr+0xGu1xBsdkOw0bdTkLwPb3pgGX9sGSFHl+lpE44et5NA+hGAaIr/FMBKlAgfXiT9WsCaSYAEsjIBCDoQc8aOHZuVu5Hqtl/Q4UIKO6mmyApIIO0JQPTA2/P4QXL1J9VzBxN87dq2EoRKwmQhwo6V1RBgy5avjLkBczRHh7VuXa6R10b635rH9qo+TxWEU4vXynkEc+QVSsnOat7UhGxDuY8+/SKl4ma/9VqCYBPKamvIKOvJ9PuUaW74tlBl7+55s8mVgn1/qlDw3EtDQxWTdzQk2fc//Sp91IMKk8/faT4RrMOefmyACWH1ymujTJ4dvPQH++33qXJ7j27mzX0IdYteCBR12rZuacodOHgww0MtIcwWQv0hh0znazqZcYbxhbFnrU2rFnZRQ1UNMnmR7Aa0PZKoZsuF+563cJG0OKuZmXBHnpOUxjPuB5vvBoLZex9+5laNEGF1aycPaeYWiHLBm/uk5dnN3fwx9nDkPIEdOBj//fHZF1/b6hLmG6EEkVsGP/fcdZvJe4PQfBBlvaHxSqlnGXKAxWqLly5TD6B65h5BiEcI2NZwTe04ihRu7F8NTx/ufodoYw1lEG7RGgTWEUNfdJ8HyFfjDWtpy6XmG8Lia688Z7wccQ/1eUBDaG7YGHOVCMeI5z0MzwQaCZAACWQFAhR1ssJVSqc24pcsQqIhUWO477waczmtDW8a2TBoXi8bR7jZrfF5Q/8HIa3bwfpIgARiI4D71ho88/AMsXHW7XZ+kwAJkEAkAie6oAM2ELborRNplHAfCWQsAUwKIseCN2TQrl3+N/cL+3JXLF8BT5qTpcWZZ0izpqcGJElHTgskup87b6FpfMP6daVfn7uMh8V9/Qea8EjIP7JJPX4geFxwXluZOm2GzJw910ysd+p4sZl0xcHLV/gFoys1NNpVGqLr75mz5eUhI0zdrXTCu4zWMeOf2UbEwEZ4/iBMF8KowbZu2xEQkunSi8435/5Hk93bv90g6Dx4f29THhOx3tBLHS+5UMsdlb/+nuV6C+BNfLSlWtXK5hh4FoWyO2650WyGR8Wot8eHKmK2IbTYheefa5an6FvyCMkUzpwX+3Zrzh0njwgmt60IgFBssJ81D4vtG9ZxPTGhDcEB1wwh4eykeI9uXdwwXN+rYJfRhvwwYA4PF+RzgiEEn7f93pyyOXL48zG1b3dOSA+bI0ccES8a75vpKqDdofxhyI1057395V9fDiLkd8qdK3CaKIcvLwvK58rpn8SHIGTFSOzzmvUQwbiJlPPFHrN23XqXCRLW/zJpijs5j/Frw+xhTGYHQwg8eOJ4vY727nXyQKF/YAbPP+tJ0/vOW/TZsEoQStEahAj8TWHFn1DPCwhwuG8RHu1RvdY9e/cz3jQ4tq96X1nzes8M6HePhnurZ/J+ffnNDzJ46EjzY8t6vxE6Ec8d5LCBd5E1/D/xtcHPui8EIvSbV7S05VLzjbE1XMPHwUvP8dB5WCJ53aFNPbp1Nl5myGtm2Z/WpJH0UoHZ2vcqGtNIgARIICsQCPxtnRVazDZGTQCxS0N511jPG+sCG3WFURZEQvVQoo3dRi+bKEGyGAkkGAGIsF7D213B8da9+7lMAiRAAiRAAiRAAolOoH/f3po8vaZ5E33nzt0aEeCIlCld2jQbb9FP/PJbs/zC4GHy1sghxvPjcc3dgEl5hFArXryoK8h06d7TRBbornkyMOGInw4q4GBiFPbGmHfMJDomWJ8a9KCgfuQHsXlfMDH5nOatsXb9tVeaN+nPbdNKRr013ky8I98DEt3ffGMXFQGOqyB10D0/joOY8vzLgR4v111zhfHIwT5EQkBy9ny+l/eOqofSi6+8Zk9pvq9QkQm5eZA0HPsx+YljrKGdz7w4xK6638ipYcOiQVDxCmVuId+CDS2G1Zaaz+Kbz98LLmLWL+rUxd0O8Qq2aMky842/RfF2PXJoeAURs1M/hg4fI0Nfftp4WD3Y9265r/cd5qUk612Aazj+vY9s8Qz9RtJ6cLZ6ybh3Pww4/+8ais7mMUE+ks2bt5oQWt7r4D0AIiE8MXBdv/z0XfUu+zUgh4q3LMSEz3Vcd9KwWiXV+/6DcaOMB4x3LHrLz1uw2Iw1sL7u6k5ybrvWRnQori95hbOvvv3BeIhgbE/8cJxOuh/WvuaQjlc7YlKo414bMUYgKKAPbwx7ydxLudU7zoZ3g/fZkNdHhzo0y21DiDWIERi7OzT0oZm70ZdtYQsXLzV9xzJyLvW6o4e5/0a/PthsR04bjH3kLoboc0Xnm1BUQj0vEO7v2+9/1rxNyElVSMa/OVz27nMERXOQfsBL5+vvfjSruIchHMO6qkehfXaZDTF8tDizmSsA4zB45eEn2PDCb+cbbw/eHNX6rT26SoH8jqgLL0qbHyj44L/+mSWPPfWiFNb+457DDwzscufJbYR1e8zipStk+ox/7Cq/SYAESCChCVDUSejLE7lxIT1s9A8BK+Tgj4T0MLi079Q3n+wbUyZMGrbpD4QbvOlGIwESyH4Ekos6xSnqZL/LzB6RAAmQAAmQQEIQgKdIJEtpP46N5mWyWeotg/CymPCEkGENE34IB2bFAkyE3/vAI/LYIw8YgQTl7WQzjsGLLof+c/4fNG/BQuPVAxFljs97B2UwWfjQoKdNQm/8X807QY833THxaD0mUB55YZBjA+GA7Pa/1dumZvVqRuzBJLu3jo1Jm/WN+hFmUhjHW5uh3g0QCFDeOxGP/9P1f/gJ1yPClv9n1hw5V8POYaIUk/LecyxXDx1w2artDTa8BW9t5JhxdjHkN9hYQSNkgaCN8FyAGANByb5hj1BvsF0aIjiU4a39nr0fkJefe9zk9MAkuLV1GqLp3n6PyIEDkUMtHRdtqDhhmeyxkb4htEVj73/0ucAjBQwwdoI9DOCJ9eMvk+U8FVBwDZDHCDZr7nxdLicI0eU917c//KK5Ts4zYxnlwQt2zBcm7ziAe2y0ioSI1tHy7DPMdcY1RhF4DOF+gGeMjaAB3kOGjxZ4i6DuMhoKDAbPs/XrN0rT0xrrsYH1I0cJ2gqhCeMI7Pf4vIGOHj3mtsR7j/6ueYOOqcfW/ZpnCcIO5jusYWw/+OiT7rXH9mieAUHNstWl63dKeW5w8vnqrQU2CH9mry22b9m2XZ58bjAWjSEP0BEVmu+6vYfh6GWCvi1S8dRaqOcF9g0f9bYgZ1bXzlebZ4D3uYVr9PRzr9gqZLt5jh02/IPHpFvIs3DMdy2Pq/jrtRw6TqKxYxEukHdshKorp8eDLdR+u83eJ4dUQMM4gncT7jsbeg7lIF7DQ2fk6LH2MH6TAAmQQMITyFHntDaBv33Tocl79+xMh1qzb5W59I/VYkWKSFF98wXeNEXdZawX0X26LcJbMakhgxjORqTRnDY7kddGhRq7bkQcXU/3AZOaDvBYEiCBdCNQrkwZub93T7f+T774SqbxTSaXRyIvFChYOJGbx7ZlMQKYIDp06EDMrUZOne+/c954j/ngbHRA9+7d5ec/ZmdYj3LnyauTe+nzok+GdSILnujAfn8YnXibX7hI2uev4//Lwl8NTIJjchpeOitXrTYheqygE3wU3pJvoCHW4IkDsQehwIK9lxFeCTlBw3mrIOzb6ac2Np42CNsWfLw9J4SLFRp2KTinBeqvoYIP2oAcEouXLDcv3dnjgr/hhQEPnyqVK2p4tu0a0m1OwAR5cHmsn1yzuilfvFgxIzos1n4GJ7APdVwiboOXzxmnnyrw9pg+Y2aKfU+UPmCsNWpYX/apd8XsuQsCwuqFaiPCakFUSEms8h6LMGr5dXwsWLTEFTG9++0yvEkaNahvJsMROtAKjXZ/uO/q1aroGE1Kse3e4zG+MV7hzTZbhYdw95H3mKy4jOuFUIgIDbZo8bJk4p63T5ZJHhW8Nug9j/Lg47VwzwtbBvc0xKSkTVuMsLRv/367y/3GsxDXzIZ1c3dkkwWM47q1a0mlShXkyOEjsnT5CpPDKdzzPpt0m90ggWxNIDV/MyfS38ax9oOiTgYP63x58/rEGkegMaKNCjRGxDHizUlurOC0bhreUIAwA48a91uX/eLNbn277L+0Pi3rIwESyCYE8IbdM4Mecnvzy+Q/5JsffnLXuZC4BCjqJO61yYoto6iTuqtGUSd1/LLK0RR1ssqVYjtJgARIgARIgARIgASyMoFYxRBvX7OyqMPwa94rmcplvN0CDxokEDceNerO7Hz7BBzdXkDda9PL8DaOFWjcb4+AgzfFaCRAAiQQLwG82WiTuqKOkvrmII0ESIAESIAESIAESIAESIAESIAESIAESIAESCDjCFDUiZJ1RMHG52njjckZZbVRFzusbqEQZRAKzbrxG+HGet6oeMNcNlHjZEESIIE4CSCvjo3DXKJ42oeFibNZPIwESIAESIAESIAESIAESIAESIAESIAESIAETggCFHX0Mme2YIOQZ4gR64g2/7qizS5fDGZsDxXr9IQYoewkCZBAQhHYuXOnVNVY7DDE+KaRAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAlkHIFsL+pktmBz8OAhv3dNkKeNFW0OHDyYcVecZyIBEiCBVBCAp461QoUKSr58eTXRLHNxWSb8JgESIAESIAESIAESIAESIAESIAESIAESIIH0JJClRZ3MFmz2HzhgPGx27VHvGtfTxh8eDaLNoUOH0vP6sW4SIAESyFACO3b5RR2cuESx4pK0eXOGtoEnIwESIAESIAESIAESIAESIAESIAESIAESIIETlUBCizrFihaV4sXwU8z37SwXzYAcNvv27RfrSePNZePNaYOk4TQSIAESOJEIeD110O8SJYpR1DmRBgD7SgIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkkKkEMk3UyZM7txQLEmscAaeoOGJOMcmRI33Y7FXBZvfu3SYsmhVu7Dc8brB85MiR9Dk5ayUBEiCBLExgh+bU8Ro8dWgkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIZQyBDRJ2WZzV3PG2M500xFXOKykmFC6VLDyMJNtbLhoJNuqBnpSRAAicAgVCeOidAt9lFEiABEiABEiABEiABEiABEiABEiABEiABEkgIAhki6lx+cYc06ez+Awdlp+Zz2LVLvWzUm8Z61ZhvX04bCjZpgpqVkAAJkEBIAseOHTNejsU0DCasRPFiIctxIwmQAAmEItC9e3cZO3ZsqF0nzLakTcxDdsJcbHaUBEiABEiABEiABEiABEiABNKBQIaIOtG2+9+9+4xos1NFm507d+myfquI4/zsloOHDkVbFcuRAAmQAAmkEwGEYPOLOgy/lk6YWS0JZDsCC1ckmT5d0OHCbNe3WDpkOcRyDMuSAAmQAAmQAAmQAAmQAAmQAAmQgCWQoaIOPGx2uEKNFWz83/SysZeF3yRAAiSQuAQQgq1GtaqmgfTUSdzrxJaRQCISoKCRiFeFbSIBEiABEiABEiABEiABEiABEshKBDJE1Hnm5SHG6+b48eNZiQ3bSgIkQAIkEIKAN69Ovrx5pXChQrJ3374QJbmJBEiABEiABEiABEiABEiABEiABEiABEiABEggLQnkTMvKwtWFCUAKOuHocDsJkAAJZC0CCL/mNXrreGlwmQRIgARIgARIgARIgARIgARIgARIgARIgATSj0CGiDrp13zWTAIkQAIkkNEEvJ46ODdFnYy+AjwfCZAACZAACZAACZAACZAACZAACZAACZDAiUqAos6JeuXZbxIgARKIk0ByUad4nDXxMBIgARIgARIgARIgARIgARIgARIgARIgARIggVgIUNSJhRbLkgAJkAAJyK7du+XYMX+ONHrqcFCQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQQMYQoKiTMZx5FhIgARLIVgS8eXVKFKenTra6uOwMCZAACZAACZAACZAACZAACZAACZAACZBAwhKgqJOwl4YNIwESIIHEJeANwUZPncS9TmwZCZAACZAACZAACZAACZAACZAACZAACZBA9iJAUSd7XU/2hgRIgAQyhECgqENPnQyBzpOQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmc8AQo6pzwQ4AASIAESCB2At7wazlz5pBiRYvGXgmPIAESIAESIAESIAESIAESIAESIAESIAESIAESiIkARZ2YcLEwCZAACZAACOzYtSsABEOwBeDgCgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmkCwGKOumClZWSAAmQQPYmsH37joAOli9XNmCdKyRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAmlPgKJO2jNljSRAAiSQ7Qkkbd4S0MfyZcsErHOFBEiABEiABEiABEiABEiABEiABEiABEiABEgg7QlQ1El7pqyRBEiABLI9gSNHjsjWbdvdfpYvS08dFwYXSIAESIAESIAESIAESIAESIAESIAESIAESCCdCFDUSSewrJYESIAEsjsBr7dOuXL01Mnu15v9IwESIAESIAESIAESIAESIAESIAESIAESyHwCFHUy/xqwBSRAAiSQJQls2rzZbXe+vHmlRPHi7joXSIAESIAESIAESIAESIAESIAESIAESIAESIAE0p4ARZ20Z8oaSYAESOCEIJC0KSivDr11Tojrzk6SAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAlkHgGKOpnHnmcmARIggSxNIMnjqYOOMK9Olr6cbDwJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkEAWIEBRJwtcJDaRBEiABBKRwLbtO+Tw4SNu08qXZV4dFwYXSIAESIAESIAESIAESIAESIAESIAESIAESCAdCFDUSQeorJIESIAEThQCXm+dcuXKnijdZj9JgARIgARIgARIgARIgARIgARIgARIgARIIFMIUNTJFOw8KQmQAAlkDwKbNvvz6pQtXUpy5uSvlexxZdkLEiABEiABEiABEiABEiABEiABEiABEiCBRCSQOxEbxTaRAAmQAAlkDQJJmzYHNLS8euts2JgUsI0rJJBdCBQumC+7dCXqfuzdfyjqsixIAiRAAiRAAiRAAiRAAiRAAiRAAiSQ/gQo6qQ/Y56BBEiABLItgSSPpw46ibw6FHWy7eU+oTtWqngRadykyQnHYM7s2XLw0CGhuHPCXXp2mARIgARIgARIgARIgARIgARIIEEJME5Ogl4YNosESIAEsgIBb/g1tLd8WebVyQrXjW2MjQA8dE5EQQeU0O/8+U48D6XYRghLkwAJkAAJkAAJkAAJkAAJkAAJkEDGEaCok3GseSYSIAESyHYE9u7bJ3v+3ev2q1y5Mu4yF0iABEiABEiABEiABEiABEiABEiABEiABEiABNKWAEWdtOXJ2kiABEjghCPg9dahp84Jd/nZYRIgARIgARIgARIgARIgARIgARIgARIggQwkQFEnA2HzVCRAAiSQHQkkbdrsdqvISYWlUKGC7joXSIAESIAESIAESIAESIAESIAESIAESIAESIAE0o5A7rSrijWRAAmQAAmciASSNm8J6Da8dZavXBWwjSskQAIkQAIkQAIkQAIkkF0IlC1T2nRl67btcuzYsYjdKlq0SLL8dPsPHJB/PSGMI1bAnSRAAiQQA4FCBQtK4cKFkh2xecvWZNuCN5QuXUpy5sgRsHnX7j1y6NChgG1pvZInTx4pUbxYsmp37Nwlhw8fTradG0iABEQo6nAUkAAJkAAJpIrAps1+Tx1UVL5sGYo6qSLKg0mABEiABEiABNKMgE5uyUGdjDp2NLoq8+V3yibSJFLuPCK59L/uhw5E14f8BUT++y/6Putkmhw/LnLkSHT1Y8IPnA5G2Z7oag0sFWubAo+Obi1nLpE84BrbZGXrlmfJg33vNufo03+gLFm6POL5xo0eJnnRH48tXrpC7uv/qGdLCot58zrXKNpxGU/fYh03ucBP+3XwYAqNj2F3fh1XuB4Yj9FYrPdGNHXGyjqaOlNbJrPaFOt58WwooM8fFS2jvoah2KTH2Ap1nuBtCfzcCW5qpPVePW+Wc/Q5FWwXdeoSvClgvVLFCjLqtZcCtmHl3Q8+kfc+/CzZ9ng25NZrW7ZsaUnatCVAED+j6any8AP3JqvypSHD5ZdJU5Jtj3cDBPloxPh460+048C7TesWMmf+Qtm6dVuiNY/tSSUBijqpBMjDSYAESOBEJxDsqVOuXNkTHQn7TwIkQAIkQAIkkJYEMNHWoLFIrXoimDScMVVkxbLwZ6hUWeT0M0VKqzcFxBAYRI7lS0R+/9VZ934WLizSqq1I2XIqVuiEJAyTyuvWiPz2s8j+fc427yfaUaOWSJPTdGJbJ9zXrBT583dvifDLhU8Subij9kUnxf+ZLrJscfKyaPdZrUSqVhUpXMS/f6t6SP85WSRpo38blqpUE2mmk3glSzmMsG3vHpEpv2nbVmEt0HJqJPYzztb6a4gU870dvW+vyOJF2qZpoSdky1UUOVvbVEq5ov9HVQRKShKZ/JOo20lg/d418D29ub75U8nZ+tXnTtu8ZbAcT5uC66im/TlT2wj77guRXbucZe9n/UY6npr4+31gv8ja1To2Jjl98pYNsZw7t163GCwHWPnsP58os2Vrym/MS7Hiyru1SLkKjniCOnD8zL9E5swMfY1i7Vus4yZfPpHmLUTAuYAKpjDcKxjDU3X8HzrobIvls6B6FJxzro4PHV92Un27Tj5O/0Nk/drkNWHCv/Hp+qP3HkQHGNqwGvfg5Mhj0Smd/DNW1nheVKySvB675YevRHZsF6mgY7619i1aQ/vt/Rprm+w5Yn1e2uPsdzznrXmK3nc6LrzPqm06xqfq8yf4WWXPE/wd79hKzbMYbUi0506jU0Xq6TMqJYvwO2e3etbYZ01u/V2SM6f/GRSp2ly4t3x27NhxOYJnvNrWbTt8W5N/Va9WRZ5/aqA5x/++/l7emfBR8kK6pfedt0qLs86QkzweRAf1hYvvf54kb4wZpzrgQbfNOfWaQIwIZ/AmGjHkeSMM3X3fAFMMwkWvO3rI9L9myouvvh5waJXKlWTggL5GTMqldeNxsUd/Z41/72P55nv9/RXGihQ5Se645UaB4AT75POv5IOP9fdXGEO7O1/TSS658Hx9lOWWJctWyMODnglTOmM2P//0QKlb+xQV0I5Lp+u60+spY7Bn2Fko6mQYap6IBEiABLIngaNHj8oWfeujjP5xBYOnDo0ESIAESIAESIAEUk0AngNNdPIWk1yeiXE5qWj4qus1dAQabwnM4GDyF/uCRR0IOZde4Rd/cBzK43xVqokUUkHCK+rAA6JufRUpzvBPauOY0lH+/YN6L7hEBYUSOEoEnkTBhm1oky2D/Zhcg9CD82Dy2ztR2v5CEUyqBhsmWDtcKvK9TjBjwtsa2nChikoQv2C2v+jr6c0cceubic52p4TWX0ukfQe75nyjPajj2q4iEz8R2bYlcH9x7SOEpuo1A7fDGwOCk9fiaZP3eCyDG9qIdsHst7PmfLY4xxEIvdsgTtSu5/Qb/YhHmPDWF2Z5zryFMmDgU2H2Bm0ONY5RBBP2EFVwbaf8GnhQrH2Lddxg7HW8KjlXXLtadUXKlBf5eLx6iOn9E61hjFx+jV+cwXGoD8LhxZc7ouriBYG1dbgscOxC6ML9jXFWparIe2P1nt0feEyktXhYl9W+Fo3wHMKzCwahIlI5p5T/0x4XT5vieV76z+wsxXNeCGwQdKxBJMU9hWt4mY6Xr/VZEkqcs+XxHc/YSu2zGOdNxOfOSXpvRzNmIL6FsZEqkuAHBjGlw3ltw5QMv3n02+/KF199G76A7oGI8czjD0thX07dUiX1fg4yhFUb/PwTUrO63ps+s79y8ufPJy1V6IGoM3P2XH0U3GhKNGpYT5574hFbPNl34wb1pKB6hOXH/eWz05o0NNty5sppN5lv1PXUoAEBIhEue1EVbCACVShfVsaMnRBwDLx57u55i5zauKEZInZnZfVkCmX5tB23dL9ezjv3nACvzEoVQpdHHe++NVwgGkH0mbdAX6ZIJyuA37dqeLkgX768FHXSiXNmVev7ayezTs/zkgAJkAAJZAcC8Nbxizpls0OX2AcSIAESIAESIIHMJtDtFmfSDe2wE4WR2oSJQbxBD9uyWeSX70R273bWMeGPt+a9hkl/K+ggtNn334hs2uiIGZg8r6iCxb4gD5SLdEIZ22GYmYJhhihaO7WpM9kZqTwm262gA0+hFUsdDw2cp6xOEh35L/BozWdoPDj+mqqTp2ucCe3qJzveDzimZbtAUQdeTFbQmaRvKS/VCSW8Fd1Ut8P7AfswQb5yuXMesDj3AmcZPL/7Ul9z3qlt0YltTLyD47kqpnz4jr9dJXVC96rO/nXwtV5Q/q3+pVjb5D/Sv3Secgsl5NgS8DSCxxcMTKeqVwQ8uE6po94UygjMmzYX+eM3p0xmfmIsw+CRg7bu3OHz+tB2QnyrrwLlzOl+8SKevsU6bjDZDL5rV2m7Zunr+1ucyefmLZ0xA4+v6ir+rVjitD2azzbnOYIM7iUIiRvXq2ClYsn5FzleZ/DgWaWCpA09iHvPjt0/9PotmOPchxBnIQ6hfWjPrz9Ec3anTKyscZROjhqb9odzzzlr/k/73EF/Jn7k346lFm0cEQNeZJOC2rlbt8HiaVOsz0vnTIGfsZ4XvK2gs3KZI8LhnsJYubiTMz7anK8z2GMCzxO8Fs/YSu2zGG1IxOfOrL9FlugzOZSdUtt5yQH7Inmrhjo2Hbb163OXEUciVX1vr9tdQWfu/EXy6mtvyCZfTt7GDetLlcr6XI7R6qjnCWzdho3ukdWqOp5zy5Z7XmDQvf379jaCDh4xw0aOkZ9+/k1q1qwuTzza33gNXdHxYvlCPYy8ocneGjnE/bUOj6fg8JnuSX0LI4a+IOV8edai/bOgmAp38KCC0JKe9vgzL8k1V14mf8+cI3v37kvPU7HuTCCQMxPOyVOSAAmQAAlkMwJJm3TixGdwNQ71lo7dz28SIAESIAESIAESiIoA8uCsWiHy5aci7+ikoC8UTNhjIUrAMFn6hU6k2olVbMOb+8t1ctxr8LixIsBHE9T7ZYNfqMHb//BuCX7jX99MdgSj70XGDHdCtHnrjLSMN9fhuRLJ4LkATxwYPGzgpYC2wDBbtEnbiLBGXpv2p8i40SLzdYIbfcek6pKFflEGb1HbfuK4qtWco3dud8qhXuTTmTZF+7bJ2Qdxw1od5WSFqy8+1nOowABvDHgLfasCDwwT+uXKO8v4RLgfXC+06YPxjiePf2/ypVjbFFwDxBoIG3ZGLXg/1k9TQQ2GcfGzXj9cW/R70XwVKf5x9iF8GfK0ZLZhwnb8m841gXiCdq5dLfK3CjnWSpWxS/H1LdZxA2Hpsw+daw6x4rCOM4xFCI/WIBRFayVKqnePr/yP36g4ss4ZV3t0DP9PPabstWyg18SaFWYxtqygg32bddziWQErowJPLBYra9SdN79zBninIVRc8M8R3z2L/EBom/fHeoL9dzBwO8rY/ETxtCnW5yXu0dwqyngt1vPa64c6ZujYxLMH9u8e/z2F50/weSAUey2esZWaZ7E9dyI+d+AZinEV/HNQn1cNmzgtX7XcEeNtPzLhu5mGJAuVt8fbFHjptG7p/F6ev3CxPPjok66gg3Jz5i2QL7/5wXtIxGXUh5/q1aqacqtWrTHr2GZfMEWOM6zDmjRqIMWLFjHLY999X7774RcNKXfU5EG7vdf97iPmtpu6mjL247g+e9au3yCDh71hvIcQuiyS5VKvsZ0a8u79jyfKFZ1v0rBw/rmRUMehfdGGxAt1vN1WoEB+OeXkGnJW86aCfEihbPOWrTJsxJvy53QVC1MwhKk7rUkjKRTKgziFY7k7cwgEPcEzpxE8KwmQAAmQQNYmYN+2sb0or/+h27Zd/+NHIwESIAESIAESIIF4Cbw5wj+xm1IdmISoUs0p9fe06EJANWnmlIcAEizehDvfl59F3yZvHRBVEAoNBqEGAoj1xnG2Op8IZQRDfhtvyDRna+jPcJ4RqAOGyXFM+MIgzkBcgi1b4nx7PxfMcybFi+uEO8riWIQmg0FQgseU18zkvk5gYxINIbg2JTl7IUR4r1+EUEFxtcnbBnBE6DHYLyrWwGso2NCXylWdrUv1elvBwJZbpNvA3pSr4hcI7P6M/g4Xrmrvv/6WWHEg3r7FMm5wVky8hzJMNluzooRdj/Rdq46zF9fC5pGx5SEOrFut93R1HX86rpB7Cmb7jNBb+PEKvZbNQZ0Uj8ViYW3r9U0ay38q2qSHxdMm7/2WUpuQx+i6bo7YgvBoG1RQg8V6Xns9cKyG0gow7/NHJ9Jdu7Kz8wyCQGmvazxjK95nsW0I7ptYn4X2WHxn5HMHAhzCDqLN8Fr79UdvSzJ8GaHPBtx/jznv1BvkZwAAQABJREFUoiXLpXKlCm4INm9j4CGCHDawocPHeHfFvAwB49P33go4Dvlr8OO1F58ZZFYv6tRF4IUDwyNm4pffmWX7sUvF/Y1Jm6RihXJy2qkN7WbzHWvumVvu7BNVWDOEW3v1xaekWJEi7vkef6S/viNxzKyPHD1Ovv7uR3ly4IPSpHEDmTT5D1m4aKnceMO16lVUWNDm62/qKfB+at3yLL3lAr18ICxNeP+TgDxBfe/pKcg3tG/ffr3lbzPnKVG8mIwb85pyOS733P+w3Ne7p9RQoQzDy9qPv0yWV4aNtKv8TlAC9NRJ0AvDZpEACZBAViKA8GteK8e8Ol4cXCYBEiABEiABEoiHQPDEe6Q6IEJYQ3goa95ZCrsN35gkwxvkMK94Eq68UzK5GGC3p/Td8hwnbJZOrMiU38KXtpOMsbQpXG1Vazh7bEi54HKhchPtUK8DazqJZMwmtvaEurFFzGyZ9e5BCCWvxXL97HHRtsmWx4ShFcvgZWBDxtn99ruQTmJb26geRsG2e6f/2gb3I7hsZq5X811TsEWIQVha9y2lceOc1f9ZSUUwaxBiojXkBYJBADzmTGo6G3yfG1VEhHnfGrfeOLhPERLQ5tSAJ0iNWk755ToO0sJCsUa9OLd9ThxS8cm7nhbnjVRHuDbhmFjutwoVHSEWbbd1xnPeHertZ71zEGbNK1TXqefUiDFh24bcR/YZd3LtSGd09kUaW7bOlGtJuUSiP3fghVqylNOPr//n995MuWfpUuJJzVEDQeHQf4fl0cefDXuO2qecbPbtP3BA1nt+fyAXT6x25MhR89Ko98VRLOMHYgYMQwLrm9Q7BWajh+zasyek6LJw8VJTLl/eQEHysPWONXtT/oi2fG4Nc4owbcglZA23IIQv/BQtepLZXEif6ViHGHN3z5ulyEmFzWMmt8/j7ewzm7mCzsGDh+QwPDnV4JV01+09xDsXY+tC3h9r4I/60Z7XX3nOhMc7qi9+4Hpaa9+2tT5e/cfY7fxOLAL6m49GAiRAAiRAAqkjsH3HDv17XuPN5nVcncuXiyH0QupOzaNJgARIgARIgARIQPNwOJMh5s195N1ool4XJXQSDG/UI9zWhrVOKCs7WVPQJ1iAHSZKztc3epHUGHlf4I0CIWT6n3qsTvan1qpUE0EIM9h3OiHn9S5wtvo/rYCC8E3wPqlWXfumk984JilJZN4sJwyX/4jQSzVP8SfbXqKeQdYw64U36JGXBWV+/8U/4YoyOvnmWiE977/qGWJz4Xg9MtxCumC9nKwI5N0XzXI8bbL1Nm/h9BNvr//2k92a/NuOD+wJ14+DWgcSvHvLJq8p87acpGMciexhELDspLa3vantW7hx45w1+SdmJM9u7WzHfYNwbNEa+gMD91BmxyK83JDzCd4eCOuFMIFnthST96lKVfX2+EvHsgo6EGkRQjFcPpJQ5wi3LRxrlLdeOli+pos/tCHuK4iKM/S5YZ8zKJNWFqlNsZ5j7WpHFMRkNjwHI1lK50WYyIsuc+7Da28QWTjPuV7I64UxgdxV1iAA4XyV9bmGnFCRLDVjK1K9dl9Wee4grOSpzZxWY2xt9Ym5th8Z/H3ZJR2kbm29tmrPvTRUINiEMyuq7Nt3QO656zZpcdYZbmgvCC3/zJwtg4dG5w0C4aTbLb3M8R9PGCPIdYN12Dmtzpb+9/UyYc/gNWOtqC/02oH9odu4Y6fz+x2h0OAJdODAQXtounzv2LlL4EFUv25tsR5FA5983uS7CXVCCC87d+023jvLVqzSP2Uc8Wr02++aHEALFi1xxaqLLmgvve7oYcSfVi2ay8effRmqymTbtu/YafIc/TNrrtl33dWdpFuXq009l118gdajf7PQEpaA/nakkQAJkAAJkEDqCSRt3ixVNQ4rrDw9dVIPlDWQQIIQOIh48DQSIAESSHQC9k1rTABfcElgazUhsRTVyXC8nT7xYw1fo8816yWAksHhujCxjwlJiDGff+TkywisMfq1/PlF2l/olEe4IeRJCGeYxLQCig3DZsuiX0gQjx8kV1+y2O5J/g1PE9undWuSl539t5OsHW/N39zTCcO2Sye3SpYWOVknx72GMtbC/T6wIZg0JE/cFkub7EmQe6jRqc7ad984E+k6CRbScE2tWc8Cu26/8ZYyuoDQVIlmCDV2UUd/+KXJKsZZS6u+pTRu7Pm8363ailjPpm908s8KTd4y4ZatB05K4wrHY2ztVdEEtkAnHxGSDd55uC/OONvZDuETuZIgiKbGIrEOrhdl0WfcuxBKkfOkVm2RCWPTVtiJpU3BbQy1jnvg8w9D7QncFs15EYIRQpoVHO03akKYMIjqXvPmYPJuD15OzdgKrivceqI/d+ApceGlTuvhmTdLn92ZaGXVy+TWm1S4U/vjzxkyfcY/EVtTtJj+7lUrXaqEXNC+jVm2H/AqgTdI5UoVpe+Dg9RZL4S3ni3s+a5X1/kdtWePvnDgs9q19Pe1WnAumwL4/at2IExYSCuSoAxeSl2pOXoSyfaqZy/CrQXbjz9PCt4k3/34i9x5200mV4/XUydZwaAN7334mVhBB7s+nfiVdO3siDon16weVJqriUZAfwPSSIAESIAESCD1BJBXx4o6pUuV1Jdec2tOV/3PFY0EsjiBvfsPyfTp09XNXf9jdYIZBB30n0YCJEACCU8gl2cyf85MfRt8vjOZiGTsdVXQqa8/CA0EEWDGNCf8mu0UQjrN0ckphIHKX0DkFJ2UhQcIJoxbnxvd5KetK/j73AudN/uRSH3mX8F7w6/v3O688Y/k8TCITGdom+CNgDBHS5eEnkCHCNPxKmeSGRO3P3/nHO/9NHlzyjv9RB+tFxHKYGIc22D79gROTPs8sp2dnk8r/BzU88VrsbQJ58A5bdi1ubOcfD+Rzm2FJ5TJ4+tfcPnceZwtYSYAg4tn6Pq5es1taCsTfsnDOi36Fs24Ce5wXfU+q9vA2Trtj+SC5VmtnOsUfBy8RHDPadggUR0kwPPFW9brEWOFH4ixl+v4hoACrxzcU/UbOyHEMG6vv0m94fQNdZwDhvaVKessez/3qNAQboI8EmvUgftq7Cj91vZbEQv8Gmg7mp3lCLPou1d48547nuWU2hRPndEck9J54UEFL0cI4PDKQU4rhO+r38h5BkGInDnDeZZFcz5bJqWxZcul9jvRnzvtLnDuITyX4RFlx1tq+x3n8c899agJ2/Xv3n3y7EtDUqwlJ8ROtSPqZQch4tOJX8sWDY12fvu2ct3Vl5vwaLVPqWnywyB/TDRWp9YppljSJv197bNqVSubpZWrA0WZQ3qvItRZvqDcM/Y45KmxtnWb/s5NMEPOm3AGgQ3h2ZDPaJd686DvR3Sc5M2ZJ9whUW2HRxTqyeML9YaDKpQvJ33vuTPk8ePf+0hmz9W/t2iZQiDMXzOZ0haelARIgARIIAsT8P5hhW7AW2edJ3ZuFu4am04CRtiguOEfCDnx5iaNBEiABBKJgE2SjskvhGeyBjFlyq+ab6OmE1qrUlVH1Nm/15YQ+WuqCDxVYPv3qcCjolCpMo7XCiaE4f0R5VvETiW+T0wow7MGhrfT8zpvDZv1HL7nKCau4Z0D7wK0XSdUzCT3Qp0kWbXSFDUfS/VN+Bw6wQNBB1a2QnIhAxOsl17pTHhjMuirzxyvJOcI/yf2YfJ13mwRhKorUlTDWukkN94ERx0XXuaUhWcEyto25VdBKZRBCIN5mTpbov+MpU2otc15zmQnwq5BkLMeTl5xD6GlsP0/Damjk5Cuob3B3gPYmV8n5mF2LDlrmf+J8GY1nIlMmTIpefil1PYt2nHjJVFV3+CG4AnDOMU1CDYIqVYg9O7DNog6CO2H/CrhPLzsuLJjEHU0PcMZ35uTRL7U8Y1969fpGC6m4/YSR/iCZ9zbI519yBdTpZr37M4yBIhQok5KrG1NXiEN2yD0QLwoW945X+UqtmTqv6NtU+rPFFhDNOetVtPpL55dH76rzxu913A9ECasVTvnGXpaM5834I7A+sOtRTO2wh0b6/ZEfu7U098fduz++K3zuynW/qVh+TtuuVHKli5lahw6fLSULKkvSfgM4ctgBfVeLq1l9ukzCWHZ9up30SInqZCzTYaNeNNXWuSb738ShA4bMeR5s61VizMlJVEH54dHDgQGWM0aVeWVF540y9WrOffbuW1aSaMG9WTcux8aoWGPPmNwfrQrlBUrVsRsxjD491/P3wShCifQtttu7iYdL+5gHAQzolklSxR3Q+4Fn69xo/oUdYKhZOA6RZ0MhM1TkQAJkEB2JoDwa16D1w5FHS8RLpNA9iCQUyd/8uUK/Z+j7NFD9oIESCBLEsAEMQwTxhBKIER4bfUqfWtfPQuK+yaibCgnlClW3C/q2GMw6WxDkSG0Wzy5dbxhiK641tYc+H16cxH8IDTbP/qzT8WHYjpBjTYFG/pgTb2iZdMGu+Z4HkGMsUnIIejA8yiSITdDcH6GZmc6R2CSGjNdsH3KFl4itm5nq/+zpApgMOQ7Sa1F26bqOpkMg2jT9RZnOfjzsiudLRM/Csz1gn5s3hRYGqHArACxNw36EVh7/GtNmjohvVADJsoReizYcH2sxdo3TMbGOm6Q48N6SSH01o9f27MHfsN7x+ttY/facWk5I/dVKIOwCvPmw4DHGgyipx2fWN+zS+SHbzTHzQ3OOcur6LlR749FWg75sYItVB6faFgH1xO8jvFbpVpgeMfgMrGsp0WbYjmfLRvteZHLCLZSn5deoQvPD+O1o+IfxgBC0v2l4zcli3ZspVRPrPsT7blTVJ//Lds6vVi0QGSN59kfa9/SqDzy1lh7+IF77WLA99lnNhP8TPnzL3nmhVdl2/YdUrFCOf2V5ogn3sJr1q6Tg+qtB0+aKpX0mZKCndH0VPGGFYNQAy8fr5UoXkzw07B+XSM0IF9M5YoVxObW8ZbFco3q1fBl8vOYhSzw0ahhPblc8xrBdmsIuilTp8uGjUnGY+eC9u1M+LW07sba9Rvk18lTQ1Y705eLJ+RObkx3AvoXL40ESIAESIAEUk8g2N25etUqMmXaX6mvmDWQAAmQAAmQAAmQQEoEdqhHjjXk25gfNPldoZKzF2GXYAgRi7f1kY8EYs9q9TbwGiaFrcUrVuzfq7NFKgiFMhuyDBPTEKDshCgmFyHq1NI+TP1NPYR8wgrqKOdpky/Bs6kaIW6QRwheN7BvvnAmtJ216D8RfqzxaU75+bP9x0FMalJCwyrpBC08NY8d9e+D+ISQcLBghs7W1H2GaxM8dKy3U/AZLFtwBV8N/WN+tm11hKlT6qg4Mi/wKOsJg60bVKhIBEM4r+a+idR/1AsEniChDP2Lp2/xjJuy+pb8ZVc4rYCo+LWONa+44m3f/DneteTLGC/IQYNJf9TrFdrQtho+AWet797Mo55U7rU9lLw+622HPUX1HoKog3NEMy6jZZ38rIFbbIg8iBqptbRqU6ztiOW8pXyC3OEQ/cW42LnDCX8XSqQOblcsYyv42LRez8znDjxDL+7ohK+D8AnvvAQweN0Uss/6oPbkhrefmnnc6u+Hf30vWSxeukwaqwgBAaZBvToyf+Fi90iILxB0YAjjnpI98vhzctJJheV5DQGXV58ZTzz7suzUsGNYxraj6k17/4DHTDUbkxzR/tff/pAmDeubkHGtW54lk6f4hcWcytnmjFmxUn/HZZLl8IWoi/b0bVq1cIveP2CQCjr+FxTatm7pMnULpcECcg+9+MpraVATq0hrAhR10poo6yMBEiCBE5QAYr4iuWCN6lUNAesGfYLiYLdJgARIgARIgAQykgBCZq1fq+HOqji5Z5L07XyEXoMnAnI8FPWJKxvW+Vs1W0NGIfdFFRUr4JWzfKmzD94OEFVgyG3jFTGcrdF9fvtl+HLXqkcBJoCn/+GEe7MlEcYKOX0w0d2yncjvvzgzZcghcmpTpxRmzrb5JsEwIXSuvrWLPsC++lxFCU8fna2Bn+hfvvw6k5bkhHzDXniqQBiCtwrqR54aa/N0ucnpziTjBRc5uR0gNmGCvYN6B8EgskQzee6UTv4Za5vGjk5eB7ZgQvTWXs6+Lz52xoAtifwryP+BEFkQExB+DlZSJ6YRagq2arkj9jlrmfcJobHFOc75rRdXpNbE2rd4x03Hq51xgHw2X02M/95AX+Dlg0lr5Mk5T8feJxM0zw7EOh3Tbc937gGUmz0Tnyp+qnCAcQbvrOZ638Ljx3rcYdw29XmZoWySju1oLVbWVao59wjObz1+kHcRwmBN/YFhHKXGYm1TrOdCvgyEMMR9P+lnx9MJdcR6XoRsPEmvH0JNLtMJezxTYLiG1fSZZPMZQWCzdtoZmiOsphP2ct0aZyvu/7QcW/Zc3u9w543lWZgRz52z9b4HUxhE03h//zg1pNnnrXfdF7auj94dI4VV8Pl50mQZPHSkW+69Dz+TqzpdakSVRwf0lZ69+8mOnbuMONP33jvdctP+0t97KRiEmpybcxoR55j+/rHHQCyC7dHwaUuWBt53yOPTU8O2QTzqc/cdZv9mzekDQWfIS0+bduHYUW+Nx1eGmVeIOat5M5nxj+93URQtyJ07l1sqRw79feez9u3O0V/J+juZdkIR0Cc5jQRIgARIgATShsAKTdBnRZ0iJ50kiL8Kt2caCZAACZAACZAACaQ7gelTHW8VCCJXddGQarvUi6SQf3IYE8J/T/M3AyGcIFbAWwfCyJktnX0QUKz9/INdyphvCFHIUVJdhShMsEJssuHPbAv+/F0nt32eCpg0tRPJ2H9JJ1sq8BueHJ++72xDCCvkuYBBtMqp0wJW9MK2ST/568f6fvVogmiASVGIR7f0cvLO2IlHlJn8qzPRjeV4LNY2xXMOiE7IxQJRByIORIDDR/yeRvDsmf5nPDWn/TE2Xw1qbtrc+Ql1ljeGOltj7Vs846ZlG2eyHmcsX1HFs7uccwd/RiMs2mN++0W9Ei53rsGNtzoiT6GT/OdBuDmEVrP25x8qRrR3xuv1PfTe2OuMz9JlbAlHUNm1w7+e0lKsrPHMQP9hGDO4P7z3D54zaGdqLNY2xXouCFP2udGgkXoFTnZqiPW8CBkJjyqIOBBlwGP7FpHS+lyC0AbDNVqy0FmGIGdDPJ6twtyHPlEnPcaWc8bI503E5w5yUVm7tqtdSv6Ne/77r5JvT6Ath3U8fPv9z3LJhefJSYULyfg3h2t6s31m2TYTXjpff/ejXY34jUggsH2453xW3yfqbN6sv+NC2Jvj3pO7br9J8uXNI2+/McSEfMunIiyGLGzegsWydNkKZyWDPndpTjeEpitVsoR0OK+tnNu2lQpMuUyuoeGj3o7Yit//mCbt27Y2ZUYMfUHQb4S3C5c3KGJl3JnlCfhlvSzfFXaABEiABEggswmsXLU6oAn2D6+AjVwhARIgARIgARIggXgIeEORhToe3isfT3AmEbEfYcwg8MCQvPtjFTUw6WjtiC5/8I7jMYBtEHOsoIOJSOPpEXqiyFahSoZ/MZalo8ec0hoyJpkhR8mcmc5mtN+GdEIy8t/0rXrrYYISdmYqWSVBG+B9Yy1pPWbFnLXiJf0T0tj2v09Eli6yJf3fM1QMmzLJEW5wTivoIMwUJvFXLveXDbnkOX+o/fG0KVQ93n4Gs8W+L9A/9SiA4a1mG04IAuBH+rZ2PLmTnNrS9tPbj2hqjrVv8YybaI8J5h6p/fCuQ84jCCEweO3gPOjPXyrSYsx5DeLArzoBjNCJMNyvVtDB/QGvpp++c/ZF+xkra5M/xtde3J9eQQdj6yN9Btn+RGpDpOdZrG0KdZ5I9W/b5j/CetdgS6znRXi1z/X6QTSGgQfy4lhBZ9kSkc8+9D93/zvof/Z4vXfSZGxFeMaEO29Wfu5E6K5zMTLuExE7YEft7zXPqSFSjJvwkcC7BpcZ4o61WXPnS68+A+xqit+1fDl0tnrGb9UqlcxxyNETyiAYPffyMNXv9fmgBq8d+4iZ9PtU6f/IE6EOC9h2PMbf85ZHQCVBK2PGTjDCDjbngeecGsQe2LEI3lnw6vnxl8mmXC71OKpQvqwRdMBy81bnvgZra8cQnlMtmjbZY+z3kSPOsXad34lHIEed09r4r3Y6tW/vHr6lnU5oWS0JkAAJJBQB/EHy7GMPu22a/vdM+Xjil+46FzKHQIGC+h9eGgmQAAmQQIYROIBcKqm0wkWKp7KG5IefUP8vy59f86eUcXKp7FBvFJuzJjkWZwsmVRACCHk7MFGJcG6ZbQgdB9GlsHouYLIH/6/2TNakqnmY1SqiE+hmEl1rQhilaPOAIIk3jgVXiF9pZalpU6xtQJi2krjeOgm9XSekredTFPW0a9NS7r/nTlOyT/+ByUL+BFfxxcfvmEm7OfMWyoCBTwXvTvv1VPQt7RsTQ40IBVailDOmkB8nksBgZoeL6gyx3hvI57FHw7ghX1aEydAYWhJdUdw7OD9CryHvFu5RCMVZxSBqInfM/n1p02JcP+QyQjgz5FSBh5VXRLdnsdfO64Fl96Xnd7jzZpHnTrxoet95q/EGwfEXdVIP1ghWtUplGTHkeVPijTfHyxdffRuhdOy7kMMG4dKSNm2R+QsWBXjceGtrpHl4nnviEbPppSHD5ZdJU7y7416uVLGC5vipL+s3bFQPnUUqnoR4qSLu2uM7sJiKwvny5RWEhYvFkJOokfZln3o+zZ67QG+1LPTsiaWjUZZNzd/MifS3caz9cOTAKCGxGAmQAAmQAAlEIoA3YFatWSvWQ6d61aqRinMfCZAACZAACZAACaQPgYP6Rji8AKI1vMXrfWM92uPSsxwEHIRjw09aGybMMQntezM4purhzZIeHi2paVNMHdDCmMzbqkJWBlqjBvXkq0/Vi0NtzvwF8vCgZ9Ln7JnQtzTpCEIq7Y/ynsVYgSiQ0cKAt6PIB4SfrGoQcaMVcqPpo7l+Pg+qSOXttYtUJj32hTtvNnzuQMg5/9w2hmIOiFZx2G09usqtN91gjnzrnffksy/UgzSVtnzFKsFPKGt2ehMZ9FC/ULvSbBvEHPwkklnvnFjbhNxEkyb/EethLJ/NCOjrKTQSIAESIAESSDsCK1atcSsrU7qkvsipb7DRSIAESIAESIAESIAESOAEJoC51ZzqfYWf/PDuoJEACZBAOhDIp15Y9lkTp6ZjQpTZOvLAozGdLafmlLHnwzeNBEggZQL01EmZEUuQAAmQAAnEQGDlaog6rdwjqlSqKPMXanxpGgmQAAmQAAmQAAmQAAmcYASu6nKzTlYGvk97oofKOcGGALtLAhlK4OWhI2ToiDEB54wm1Bjy0nS67qaA47CSEc+r6TP+ybRzJ+swN5BAFiFAUSeLXCg2kwRIgASyCoG169YHNBUh2CjqBCDhCgmQAAmQAAmQAAmQQBYlsGDhEpnw4aem9cF/94bqUkZMiIY6L7eRAAmcmAQg4ByKIU+Yl1K8x3nriHc5M88db5t5HAlkJgGKOplJn+cmARIggWxI4KD+Abla3/KppokWYTa/TjbsKrtEAiRAAiRAAiRAAiRwghFAQusJHziizgnWdXaXBEiABEiABEggQQgE+gAnSKPYDBIgARIggaxNYKUnr06VShWkYIECWbtDbD0JkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJJAABijoJcBHYBBIgARLIbgTgqeO1KpUrele5TAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkEAcBijpxQOMhJEACJEACkQmsWR+YV6dKpUqRD+BeEiABEiABEiABEiABEiABEiABEiABEiABEiCBFAlQ1EkREQuQAAmQAAnESmDfvv2yZt0G9zDm1XFRcIEESIAESIAESIAESIAESIAESIAESIAESIAE4iZAUSdudDyQBEiABEggEoGVq1e7uyHq5MmTx13nAgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQQOwEKOrEzoxHkAAJkAAJREFgrcdTJ3fuXFKlEvPqRIGNRUiABEiABEiABEiABEiABEiABEiABEiABEggLAGKOmHRcAcJkAAJkEBqCCTPq0NRJzU8eSwJkAAJkAAJkED2I1ChfDnJly9f9utYHD0qVbKEFClyUhxHRndI0aJFpGCBAtEVzmKl+vXpJZ+896YMG/xsQMtz58ol7du2ltKlSwVsj7Qy+vXBpq4e3bpEKpaqfc1ObyL16tRKVR08OD4CGBMVK5STnDkTezoQz4OTTiocXydjOCq9nzsxNIVFSYAESCAmArljKs3CJEACJEACJBAlgT17/pV16zdK5UoVzBFVKleK8kgWIwESIAESIAESIIHsS6DD+e3k+muvlOLFiunEag7T0SNHj8qSpcul30OPx9XxQQ/3k4b168jhw0ek8423J6vjg3dGCTynQ9nRo8fk2q63urse6d9HmjRu4K6HWnho0DOydNmKULukdcuzpHvX66SITsgeOvSfXH9Tz5DlsLFG9aoy4P57pEyZUpIntzM9cfy4yMakTfLY0y/Iho2bkh371htDTN3Xdb1NwK1Agfwy/s3XZcfOXXLbXX0DyiP879OPDZBTTq4p+fI6oYD3HzggU6fNkMFDRwaU9a5AXLrjlhvljKanms2ffP6VfPDx594iyZZj6Xeyg1O5oXy5skawKhMk3jz/9ECpW/sUOXbsuHS6rruOj8Mpnslei7JlS6dYNp4C1151udx4/TXm0EFPvygz/p4VTzU8JkYCve+8VVqcdYacVLiQe+TBg4fk+58nyRtjxrnbol2AODRq+GBzL65YuUb6P/KEe2ifu+/QczVz18MtTPvrH3np1eHubojcA/rdIzWqVZUczqNRjh47JrNmz5MXX31d/v13r1t2xNAXpXSpEu56uIWRo8fJT79ODtgdz3MnoAKukAAJkEACEKCokwAXgU0gARIggexKYNXata6og7w6OfSv8+P4nzqNBEiABEiABEiABE5AAj1v7S6XXnR+QM/xpxEmSOvXrR2wPdqVC88/V5r7xIfj+ZMfhTfyIbBEa2XKlE7Ro6V0qZLJRJ3LLukgna/uJEU93jYF8of3jGl1dnPp37e3K2yhfWCByVx4ElTTvx2DRR1wKqftw0QvBB0YuMEDZ//+A2bdfsAzZ/iQF6S4fnsNZeG9gvrv7feICh7H3N1lte67e94ipzZu6E4qY2flis5LSm5Bz0Ks/fYcmu6LBfI7AwJ/g+fLlzcqUSctGgUmt3S/XtasXS933zcgoEqvqFC0SOC1CSjIlTQhAGFz8PNPSE0VUK3Z+yx//nzSUoWeeESdgQ/db+5F1Akh0GsQF6PxiqvsCc8Nj5kRQ19wxV3c37ly5tKfnNL0tMYycthLcuPNd7n3PQSdaM4BwdNr8Tx3vMdzmQRIgAQShQBFnUS5EmwHCZAACWRDAt68OoULFTR5ddasW58Ne8ouZScChQsyDE5K13Pv/kMpFeF+EiABEiCBIAJnnnG6K+js3L1Hnnp2sCxastSUqqSiQdvWLYKOSHkVE6F33n5TxILFPKLGC6+8LvCmjmR4cx71eq1EieLSp9ftRuiAp8vc+Qu9uwVv5p/XrrW7DaILJmPDGSZjraBz+MgRGfL6KPn9j+lGdICXzHntzpFVq9cmO7x2rZPNtj2eN/br1nHEsKRNWwLKwzPBCjrf/firvD3+fTMJ/MB9vYz3ysk1qsllF18gE7/81j3urZFDXDHnP/VqyasT4pEs1n5Hqis99j3+zEtyzZWXyd8z58jevfvS4xQh68TLXBDgCnu8QmzBcRM+ErA9oOPol0m/2838TicC9+p9awWdufMXyauvvSGbNjv3SuOG9aVK5YoxnxleaRBawtmwEWOkXNkyIXeXUjHm3rtuM/tWrFztlsH9ar31nn7hVfnjz78EgtTtN3eTiy4419zLndXDcfx7H5lj4C1YuJDf68itSBfOb99GWrc402zyPqvife546+YyCZAACSQKAYo6iXIl2A4SIAESyIYE1q4PFHCq6NtYFHWy4YXOJl2CmJNf8xo0btIkm/Qo/boxZ/ZsOXjokFDcST/GrJkESCB2AhAiqlapJPMXLtGwX4HiM0STsvo2+aLFywSiRCjDJCTCxi5ctFT27d8fqojZBs+XU06uYSatETLNeox4D8BkNjxHvF4gt3S/wRTB+b1vnGPj+g0bZfz7H3uriGr5uScfjSieoBIIMtam/DEtZHvtfnyv1Rdw8GMNfbFix0ENp3bH3f0CwiChXJ48ufX3wn8a1uwvnXT92AgJ8CAKZzd37+J66PTs/YAJt2bLQnT6dOJXdtV8QyDIodwbNqhn1rds2WomfLFSS68FDG3GJDBCjOHbhk5buHipDB0+2pRB+Ka+Dw6S98e9YTyKrr7isgBRBx7l6/RaINzaT7/8Jl99OsFtp6kg6CPWfgcdHtUqwsth/GJ8I7Qxxkq0tlk5DRvxZsTiCHlVvFhRFRiXBYzXUAfhOlSoUF7g/bBt23b1xFkXcjyVKV0y1OFmG67POyrsRDJcP+Tc2fPvvyHFvXDHYtK+bp1TZNv2naZt4cql5XaEmIYnHMaZ937Hc6L2KScbkRChCkM9J1AGAhhyx8xfsChkGdtW5N5CKL3de/aEZQJB1CvagmPrlo64MX/hYnnw0SdtdeZ7zrwFgp9YDOfoe0/4kIqoC+ET8RPKnni0v9kMb6G33nnPLVKzZnWzvEnHLAQdGMbK8FFvG5EGY8+bhylc+EccB+EWtnPX7oD+xfrcMZXwgwRIgAQSlABFnQS9MGwWCZAACWQHAohtvn5Dkv5HtLzpTpVKyKszPTt0jX3IhgQo6ER/Ua3wNX36dAo70WNjSRIggRgIPDnwQZPXZdLkP4zIcuMN12ouiMKya/duk6PF7p+uORkwydlePTvsW944zZ+6faTmieh37506EVg7YGIegsGb4/yTiee0Olvu0bfEEYrIGnJNzJozT558brDdJMWKFpVnnnhYquokrs33gJ2YLH3sqRddseiKjheb0FOYxL25Zx/ZunWbmZSvUN4JA/TZF19HnLx1T5jCws03dhHUicnRb77/SS7u0D7kESVLOF43KBdqYjnkQb6NmBQeqbkr4HEN74q773tQJ8x3JDvklWFv6ATsa8m2h9vQro3j1bN46Yqwk7/eYz8YPyog1FLtU2rKFx+N8xYx/QeD514eZq6l9RT6/H/fBJTDCiaNzdv/KmaUKF7M5OPB9mjzzqAsLJZ+fzh+tBQsWEBWr1mXLCQZ6vrkvTc1RFo++UtzzDz57MsCDwt4ROTXsGleg5fXhPc/Mdfcuz3UMibf26gH2L59++W6brcFFEHOoPbqXeUNYTVvwWIT8iqgoK5AoMM9GBzGD2PqtylT5YXBzrVHyLW2bVpJMV9YtbIahuvLT9811e3Wdt/Q405B+KsH+t5ttt2i9weEJ2u19LoiP5T1sMJ2nGPh4iUy8Mnn1bvnoC0q9hnwuwqVELquuvzSgHsY4ul9/QcFCJTuwWEWJowdafr4onq0NW5YT9qe08rwx/ME18TuH/fuh3L6qY2lkYqM9lmAdo4cM9Zc3z6973BDk+FUuO+eUe8T5JCxdrvyv/TC8wOeTfBAw/Pp48/+Z4uZ8IKPDugbwB45kr769gfzjLMFweP0UxvJ9h07pauGKYPBS8veB0OHj7FFU/X93BOPmGct+C5fsVoZ1I26Poy105o0MuXnzF8Q4D1mhfh8eQPHO4QyPI/xDNq5a1eK50LeIDtOP/z0i4DysT53Ag7mCgmQAAkkGIHw/tAJ1lA2hwRIgARIIGsS8HrrxOPenzV7zVZnNQLw0rFCRVZrO9tLAiRAAtmRQCENq4PJSExI393zZjNJh8nT3Lmd9xLt/rPPbGYmnJF74cBB/4TvWRrqbNyoodKgXh05rv/gRWKt46UX2kUTHgoT3xB0MCm7W71EDv132Kw3qO+frMRk5NjRQ6WaegKhHRA4rECCczw56EG3zla+sD94s7yFtg9Wr24td/+X3/zgLuNN/XjsZH2r/YqOl5hDP//f1xEnrosXL2rKHT3m5KCJ9pwoN/zV543ogb7e139gshw3tu14oz5aA5d8eZ2wZlOmTnMPi9SulavWGDEJk9mwfzWUGMQlr8Bk13fopDa8w63Zt/7tOr7/nD7DXUVuHWux9APHxFIeXl0Y0wiFFRyWDBPRGGPY/9vvU01zMLatoINJbYSpg0HwuOv2HmHDW5lCvg97n0As8hqER4Sewzkx7jFBj5B5DevXCRAZ7DGt1NvDTpSjHfZew73QRkVR5NCBQZRB+6zQgW3oE34KFiyIVcmj195uy6VjwVojFVFeef5JV9CxfUZdyJs0bvRr2jb//WL7hvPfcN1V5p7FMegPDH3r2vlqZyXKT+QgQtvuVzEYQpblbwVfu79Ht85G9ME9hecFDO1Ezqznn3rUCDrgZJ8RGPPdu17ntqLZ6U2ko/LPmTOHKQOhDmXBuEnjBm45hDh78ZlBLntcJ/QPx+H6XXvV5W7Z+vocgpVUzzx4/sDgKQTDcV4PLy9HUyDKD5yvWtXKpvTjT7+kY8Z5pkR5uHg9Zd4Y807AYdNnzDTr8BoDfzCDwTsHgg7sK8+z02wI8XFT185mK67L/776zi0Rz3PHPZgLJEACJJCABJy/iBOwYWwSCZAACZBA9iCwZt0GObu5M6GB/2RUKFdONm4K7Y6fPXrMXpAACZAACZAACaQVAUywIoTO19/9KMtWrBK87e81TPKPffcDgfcL3uiGwPLC0wNNEUzwDtGwW3aSvNv118h1OimJyT14QEye8qd0v+E6s46J0i433eHWDw8VhGKzBuEH+VVQ7pkXnXwPmBjt1+cuOUfrQlikojqZjfZ98vmXmmPmdvN2+Q8/TTJVVNbwWTC0F/liMKmO8ugfJlyRP+YJ9QRAeLCUDOd9+rGHzCQyQhWNGTvBzdUT6lh4OMHQbxtODP04cPCAhhibLKPeGh8QNsrW8czjD0vFCuXM6s6du40nDHLTLFu+0haJ67tyZb/gUkgn+kcOe9GE84KnFcQy5MZ5TXNyLFi0xK3/gYefMMvwZsFkfb+HHjdCFlh8pZ4g4Nrtll5u+ct9wp0VgdwdvoXNW7a5myrpdZ45e667nl4L8O6weUg6XXaxmxsE57vsYkcUgfiIcQkb/fa7xssLHKx4dNEF7aXXHT3MtW/Vorl6dHxpysbyAc+km2+83hwCr46e9zzgekxcfcWlek90DhBlUPAz9R759rufZNbc+W5ZhIOD0AIxA/lLMIFur9Po1websbNZvdRuuq13VM17oE8vUxfEjYc1X8o8DUcG69Gti1zV6RIzsQ/vIoTj8hrG8o8aKu+NN8cZTx4bLhBCADxX4jHcK0s0ZNqkyVNVyETIu6SAaiAqvjRkuMxQrypYrztuNp5fWN6YtFm9xYaqF8sqI0K9+uJTgvxNlStWNJ5Y8Ei5rUc3FDUicucbbzfL+EBoM2+emD69e5p9ON+AgU8JxE2E43tdxdZyZUoLrteHn0w0Zb794WdzjyLEpH2O2NxY+/YdkHs0jw3EQ9xzsF3q4fjPzNkyeOhIs57SB8LMWZHsh59/M9fnOumU0mHuftyr57ZtbdY3bNyULDzeOxM+lFNV0IIg2+6clnJW86bK/w8NvdbWHDNdWSPEXSSD2G09IhE+0WvxPHe8x3OZBEiABBKNgP81h0RrGdtDAiRAAiSQLQh4PXXQIXrrZIvLyk6QAAmQAAmQQIYQ2Ktho66/qae89+FnZgI1OI8Cct9ARLG5LBAKDcIAbNWata6gg/WJX36LL2M2D4sN54OJaa93BybRMYFq7XRfUvCVq9e4+R5wzrHjP7BF3HwP8Ay5qsvNJtQUBBsY8pDA8IY9QlRhYh2CDgwiBTwR3hzxqtluNkb4ePiBezUUXSHjWdH/kScjlEy+Kwc6qoYvnBdv+78x7KXkBXXLyTWrudtLa3JzeC4M0Qlqm6/C3RnjQvVqVdwjOl/TyXC3ofMgnFVV0ef5pwaGnJAvkN/xLEH+HFgN9XqBBedAKq58YUeOOt4tZsXzsVNDBFsrUyp8/hdbJi2+V6xabQRK1NWuTUu3Skx2Iw8MDDmJrP348ySZrSKKFXSw/bsffzECFpbDJaLHvkh2RrPTXNEG4QX3qmBgDSJRKGZ/z5wjv0/VkKuesvCMStq02RyK0ISpMeT1wT0B+/b7n11BB+vIuwLxCYZQicG2d98+efW1N9zQbGjjXF+eGOthE3xMSuuz9fg+DzwqX3z1raDvmzZvCTgEzyEr6GAHvOWsQWCCoAPDM+KXSb+bZdxzViRFm2HIyWSFF6wjH47NRYM8SjYMHURt+zxCCLrvVVyF4X6wNlrF2cuvudGIP3ZbUfV6geH+vaB9GyOMoR34Qd3tVWR55YUnjfhkjwn1jTH6rIaexPML4bXBO1ZD/irc3zAI8cGGfr04+HXX0wpeUXjm4DkJD8oXfSH+go/zrsNTCgYx9+133jfL9iM1zx1bB79JgARIIJEI0FMnka4G20ICJEAC2ZDAVk2imqT/ESqvyYdhmDCZNsMfTzobdpldIgESIAESIAESSCMCSFwfqx1Sbwc7eeg91ptA3NaLCeQu11xpJjkRNmn9xiT56NP/qQfLb+6hmNC09RXSt/9tom9bAE3EJGmJ4sXtpmTfuXL536ecOXueilSfymINx4U30xHSCB5G8CxAWC1vHp/givCmPd5gh1lPjuAyweufqxfTj/pmPfIRWatZvZrmNeml3gMVzEQz3vgP9voY/94nJqTVUfWcqKph51qefaYJm4ZwV/M19wry+MRjuXL5pyHWrt9gBLs/p/9tvHXgkXGu5tvB5PF999wp13e/wz0FRAxwtkIZdtRTMQy2dZvf8wbryE0C84b3Mht8HzY8FVbhCZZRBmbXX3ulINeM9exq0/psN9TUuAkfBTSlrHpjIAQhvMZ2aTshKkJ0yZvTmRwPKBzlCgREGDxigkXSSFUghFuH89oJJsePaGgxeJelldWpfbJb1U+/TnaX7QK8djDurJeJ3R7uO1iEQblbe3SVOrUc8cx73OYtW9ycQHb70SOxhRXbvNmfF8jWYb+3e3JQ2ecOPPiQFwrCKryd5s5fKO++/3GAd1qdWn4myAvmfe4g9BwM9wO8Cr3Cnz0vvnOigBquNUTCTyd+LVvUuw/eL9ddfbkRlNAOeC7CKyac3dvrNkFYNDzrHnz0qXDFIm6HdyIMXkehQiJe3OE8ufO2m0yf/vpnlixeslzgcYeQdEXVg+m9cSONJ1i4MQtxzF7fOfPmBzwncN54nzs4lkYCJEACiUjA/9dUIraObSIBEiABEsgWBPA2pRV1qnvilmeLzrETJEACJEACJEACWZYA3jrv/8gT8oQmGUf+jErqUXOfhk6DNw1yRixaslTD+TghyNBJhDzCTyjD5HA427p1u9mFN8gfefxZt9g/s+YKfiZ+NM4IR3U0f0Q4Q1gphHuDIXwRhJDSKg7AvCKF3bZVQ1/BMKHrFXSwDV4jvfoMkE/ff8sICmee0SyZqAMvBa/hzfdxY14zb85fqh4+8Yo6GzSpvbXXR77lemXg70WEgqperarJOwNPAkxYI7zdxReep95Jhc1h+fPlN94FWLHeKhXKlzfbkIge4agwcQ2z3lBmxfNR2uOdExxay1MszRchGHa++gojWl3d6VITOu+SCy8w50HYLnvNsOG2m7tp3pUOZpI7LRuCMFqw//77L+pqmzRqoPdIf1d8ivrAKAvCK8Wa9f6x6/i2wgjEPohLCGEWqzVvepobmst7bM0aVb2rGbKMUGkQiJH/BbpLY80n1LjhIL0v15jnA8I4VqzoePehQY0a+PN7eRsIkSWcoINy8FqCILJFww0OG/GmeyjuXYT1GzHkebMNecDCiTq49vDogeG4Qzpu7DMmv14LGO4zbMOYCg6Rif0QjeBdCAt+rpiN+nGzPnPBAs81K2x/8PHnGtauvfS8rbt5Pva79y659a777CEB37fffKN7r4wYPS5gH1Zife5E4pqscm4gARIggUwgQFEnE6DzlCRAAiRwohFA+BP8RwqG/0TX0P+s401DGgmQAAmQAAmQAAlkNgGEbLuqSw8NT9RWOnW8yAg7eDscCcovv6abbPYJBGjnpxO/ksl/TAvZ5NVr1oXcjo0b1AMIhklp5M3weg1h+4qVq01eHpw3nJ15RlPXYwhhnMaNGpqsKCZF7faLOnVJtt+7AZOWyL2BN/DLlnHEIe/+4GUIYJs03w3OjXBO8dqatevdQ085uYYr6tiNUzTMV01fWDWEyYM3DrwJrIGhdx3bIchhG7yqIOpsSNpkiyvXWkacczfoQiOdRLe2Zm3462bLpNU3mGMyvWH9OsYD5y0VymwowM//9417GrTv8ks6mHWEngITjCF47FzQvp0ZR27hGBcwwX9KzeqSN2/eqI+0iesPq4cOhDP0AV5EmOy3k/VRVxaiIPIoWSuj4oA3zBu223B6EDHiEXRQBwSJmjWqYzHAdmtumcwwhI38RvMUXXNVR+MBhXsf4x45eJCHyPvcuX/AYwL2wQaPqUiGEHm4X4sVK5KsGMY98o4hRJ039GRwwY6+/FTYfnGH9uYnuIzJr6TPIzwjbuhxZ/Bu6d71OrMNAvMHHzs5gLyFaqtXEu5hWLC3Gq4bcuV0OK+tEdjhrRUcbhFC35nNTzfHw/tvvUc4Nhv1I9bnDsY4jQRIgAQSmQBFnUS+OmwbCZAACWQTAkhsfETDGOTOncv06GT9DxVFnWxycdkNEiABEiABEsgGBJD7Am/P4wfJ1Z9Uzx2IB+3athKESsKEKvK+lNUQYMuWr4y5x3N8OT5wYLcu18hrI/1vzWMbwpvB9u93cvCYlaCPQwcPGq+boM1mNVfOXO5b6pg4PXr0WKhiAdsggMD7B2ZzfAQUCLGCkGGwQ/85eYtCFElxEyZkkfcIIe2QM+MzDQ/ntVMbN3RXl6vYBa+k73/6VfqoBxUmn7/TfCJYhz392AATwuqV10YJPH0Qogz22+9T5fYe3QwTCHWLXlhqttuPtq1bmsUDyjRUqC5bLj2+EWYLof6QQwY5hTDOML4w9qy1adXCLsr9AwYZBnYD2h5vrhjUMW/hImlxVjPjdQNRLaXxjPvB5ruBYIb8VtYQIqxu7eQhzez+aL+XLF3uFm15dnM3f4zdWM/nwXbgYPj7w5YN9x08zsKVy8jtCCWIvFz4ueeu20zeG4Tmg3DhDTNWSl+Kmzzlz5ibtnjpMuMFhDBvCPEIAdsarqkdR5HugX///Tfscyd3Luf/dqgTzx2EVgs2CDbWu/H3KdPc/Gfecl5PpFAiG8LTQdSBaI1nULCoc+MN17peeW+/84G3anc51ueOeyAXSIAESCBBCfgD+yZoA9ksEiABEiCBrE8Af5wvX7nK7QjeDqSRAAmQAAmQAAmQQGYTgLeBFSpsW3bt8r+5X9iXu2L5itVmd4szz5BmTU+1Rc03QoR5PT8a1q8r72iYsmGDnzXhw1AI+Uc2+Tx+LtDJydOaNDLHYlIUCcQx6QpbvsIvGF15+SXy/rg3pO89Pc0+JKq/7KquIX9GjhlrysCTAWU6XXujWcfHzTd2MQnmrYCDbXjz/YWnBxqhCuu/e7yPrtKwYOed20aKFXWSrGM/QnYNfOh+k/cH67M1L1BqbNLkqeZweBF07Xy1WxXY1qtby6xjghheGQgdh0l/ey0wuY11/CAUG+xnzcOCdevdgBBQmNCG4Zoh/JO1Ht26uGG4vlfBLqMN+WHs5DfyOcEQgg/CojX7IhTWc+TwT9u0b3dOSA+bI0cckS0a75vp6mljbeCAvgGh+5DfKbcn55Fzfp1J9xnEQ2sQhKwYabfZb3iIwODBgfsjJYMgZ5l0uuxi8YZju/Si890we3/9PSulqrLEftxP8C7x2t69Th4obAMzeP7BkwbW+85bAsJAYhtCLtbyeLAFPy9QBgLcUd+4elSvtRXncGzfe/0eNfC+sjag3z3muQPuMIREDPfcmTV3vimzWUM9okzP3v3MuvfjjlucZxGeTaPeHu/d5S7/PXOOu/zUoAFuO7ERz2iEw4ShLxs9Xnhmo34g1xMMubSmR8jdGstzx1TIDxIgARJIYAL01Engi8OmkQAJkEB2IrBMJylswk/k1UFohS2+WO/ZqZ/sCwmQAAmQAAmQQNYh0L9vb/37pKZ5y3znzt3q4XJE/0YpbTqAt+gnfunklXlh8DB5a+QQ86b44w/3MxPQCKFWvHhRV5Dp0r2nESC6a54MTGbjB2+Xf/nND6a+N8a8I5hER/6JpwY9aBJ5Y2LX5n2Bt8ZzLw9z4V1/7ZXmTfpz27SSUW+NN6HS3J0xLFzR8RLTbhwCDxlMrubL659ox4Ts+x997tYI75EC+R2xBG2C148NjYRC4PL6G2+55eNZGDP2XQ0/drbx1sH5OvmSqHvP89KrrwdUjbB1sEVLHLEGE9Twcjl46L8AQcQeNHT4GBn68tNGuHqw791yX+87tHxONy8MRITx731ki2foN5LWI3E8PA9g49790FnwfUJks3lMRgx9QTZv3mpCaFnxL6CwrkydNsN4YuC6fvnpu+pd9mtADhVveQhfn+u47qRhtUqWKC4fjBsl8IDxjkVv+XkLFivf44b1dVd3knPbtTaiA3IehbOvvv3BeIhgbE/8cJx6Ih3WvuaQjld3C3eIvDZijEBQQB/eGPaSuZdyq3ecDe+GsTvk9dFhj89KOxBiDWMdY3fHjp1SoEB+EwYRfVi4eKmbA2vM2AnS644e5hkz+vXBZjty2mDsw9MNos8VnW8yXQ/1vEC4v2+//1kuMTmpCsn4N4cbrzzLFAfCS+fr7340dSAvTiv1lIJ1VY9C++wyG+L4QH02VCJE1lD5dlAtRG9EcUCIboSge/et4UaggbjpHfPIsRNsHS+50H0+ITxmJIvnuROpPu4jARIggcwk4H/lIzNbwXOTAAmQAAlkewLLPJ466OwpNWtk+z6zgyRAAiRAAiRAAvEROHbsaMQDU9qPg49qOKCUbNbsuUaggccM8sSU0/BqRijQydLnXhrqigWYCL/3gUdkp3qAwDApCi8TO+EIz4RD/zlv1c9bsNCUgXgyZ56zjA14g/yhQU+biVys41gr6EBY6d334QDhxuaFQYgw5L6JZAhzCzuOkwYZJkvtZkwEW0EH235Tr5c77u7n9hOHos327X6EnPMKLbM1jNxtd/U1wk7QaQJWIQJEMuRM6X5rb9mybbsphnPY82CiG+LWjH9mu1XAcwHXCCKTzamCsE6wXbt2ueW8C8gZ0rP3A4KcNDD03YaLWqc5N7rfdrccOHDQe0iy5eMSuR/BB6TUb1seIpq9Jhg7wXl90Pcff5lsimOMVChf1owXeEZgrMC85/r2h1/cEG0ob71jjvnugeBxMVpFwl/VWwo8ISxhLOZUjyB4DNlQXHZMgfeQ4aPNmEDZMhoKDIIOPM+sh0Vw/X/8+ZegrbZ+sLesvaEBvfcoPNGefuFVE9oP50GuJys+bEzaLLf07ONee6f/Kd/flrEBlkEfKeW5QTPmq7eWyWej4x7XFn2F4X548rnBZhkfyCfz6uujXI4oh1Bm4Im+LVKhxFq458XwUW+bPDUYL+BqmeI4XKNefQbYKmS7eY4dNuvBY9It5Fk45gvzeNzjZebZLT26dXZXR44Z5y6HWujzwED5659Z7n2BHEP2+Ypwj++897FM+ODTZIdeefnFZhvGWkqiTqzPnWQn4wYSIAESSCACOeqc1ia2v1LiaPzePTvjOIqHkAAJkAAJZDcC9/S8VSpXrGC6hZjOY98LfCsxu/U3UfpToGD4pMuJ0sbMbkfhgvmkeXPnzcTMbktWOv/06dNl735nEjMrtZttJYH0JrgkkdUAAEAASURBVHBgf+QJ+GjOX7hI8WiKxVSG/y8LjwuT4E1Pa2y8dFauWm0SwXvDYXmPxFvyDTTEGjxxIPYg7JcNN2XLIbwSws+GezMdIYVOP7WxCjwHZa6KKMHH23ogXKxAbsIoBCp7TKhv9A8e0whrhRBdy5avEOQ8xJv8oQweLShfRXP95FdvIghDyL1iJ+ZDHRPvNuQPaXp6E8EkLv4+xFv7aW3w8jnj9FONp9L0GTMDxIG0Plda1oex1qhhfdm3b5/Mnrsg7PWy50RYLYgKsVwnhFHDNUZi+HBjHvXDm6RRg/rGe2wmxNAUhEbbpurVqmji+qQU227L4xv3T/26tY14OFuFh3D3kfeYrLiM63VW86bGg2zR4mXJxD1vnyyTPOrJtEFFSZSH15zXUnpeIOwicuskbdpihKXg3DSoC88KXDNvTh/vOdJ7GeOs1sk1TRv2aY6xlStXy6o1ayOOzXjalBHPnXjaxWNIgARiJ5Cav5kT6W/jWPtBUSf2scIjSIAESIAE4iRw0fntpV3rFubog/rW37MvD02W6DLOqnlYBAIUdSLA8e2iqJMyo1AlKOqEosJtJCBCUYejgARIgARIgARIgARIgATSn0CsYoi3RVlZ1GH4Ne+V5DIJkAAJkEC6Eli+0p/8F28EnqJvi9FIIBEIwNtkzmx/mJdEaBPbQAIkQAIkQAIkQAIkQAIkQAIkQAIkQALBBCjqBBPhOgmQAAmQQLoRQJiPHZqE2NrJNWr8n72zgJOq7P74obu7l+4GkZRS7O5AxUSxk78Kr/lar4qJor4iYmG+oqIo3Uh3d3cv/T+/585z587szO7MbM3u/g6fnbnx3Ce+N3Z5fvecYxf5TQKZTgDeYxR2IjsN4EQvnchYsRQJkAAJkAAJkAAJkAAJkAAJkAAJpCWBvGlZGesiARIgARIggeQIIIkqvHUQ0xxGT53kaHFfRhOAtw5+IFbQkicATjQSIAESIAESIAESIAESIAESIAESIIGMJ0BRJ+OZs0USIAESyNEE4K1jRZ0ypUtJnVo1VehZk6OZcPDxRYCCRXydD/aGBEiABEiABEiABEiABEiABEiABEjAT4Dh1/wsuEQCJEACJJABBOCpc+zYcbeluirq0EiABEiABEiABEiABEiABEiABEiABEiABEiABFImQFEnZUYsQQIkQAIkkIYEDhw8JCs8njl1ajOvThriZVUkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQALZmABFnWx8cjk0EiABEohXAitXrXa7VqNaFalUoby7zgUSIAESIAESIAESIAESIAESIAESIAESIAESIIHQBCjqhObCrSRAAiRAAulIwOupg2borZOOsFk1CZAACZAACZAACZAACZAACZAACZAACZBAtiFAUSfbnEoOhARIgASyDoGt27bLuvUb3Q4zr46LggskQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkEJYARZ2waLiDBEiABEggPQl4vXXq1q4pxYoWTc/mWDcJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJZHkCFHWy/CnkAEiABEggaxLw5tXJly+fQNihkQAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJhCdAUSc8G+4hARIgARJIRwIr16yVnbt2uy3UqVXLXeYCCZAACZAACZAACZAACZAACZAACZAACZAACZBAUgIUdZIy4RYSIAESIIEMIrBy9Rq3JXjq5M7NX0suEC6QAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQQBABzp4FAeEqCZAACZBAxhFYscov6pQqWULq1GIItoyjz5ZIgARIgARIgARIgARIgARIgARIgARIgASyGgGKOlntjLG/JEACJJCNCKxYvVoSE4+6I2JeHRcFF0iABEiABEiABEiABEiABEiABEiABEiABEggCQGKOkmQcAMJkAAJkEBGETh8+Iis8IZgY16djELPdkiABEiABEiABEiABEiABEiABEiABEiABLIgAYo6WfCkscskQAIkkJ0IrFi12h1O1SqVpEqlSu46F0iABEiABEiABEiABEiABEiABEiABEiABEiABPwEKOr4WXCJBEiABEggEwis9OTVQfMMwZYJJ4FNkgAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJZAkCFHWyxGliJ0mABEgg+xLYvnOnrFm73h1gnVo13WUukAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJ+AlQ1PGz4BIJkAAJkEAmEfDm1WlQr45Uqlghk3rCZkmABEiABEiABEiABEiABEiABEiABEiABEggfglQ1Infc8OekQAJkECOIbBwydKAsTZp2CBgnSskQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIiFHV4FZAACZAACWQ6gc1btsqyFavcfjRpRFHHhcEFEiABEiABEiABEiABEiABEiABEiABEiABEvARoKjDS4EESIAESCAuCHi9dapUqij169aJi36xEyRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiQQLwQo6sTLmWA/SIAESCCHE1i0ZJkkJh51KTAEm4uCCyRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRgCOQlBxIgARIgARKIBwL7DxwQeOu0adncdAch2H4b/ZccOZIYD91jH3IYgbKliuewEUc+3MSjR+XgYb8AG/mRkZUk+8g4ZYdS6X0tZQdGHAMJkAAJkAAJkAAJkAAJkAAJBBOgqBNMhOskQAIkQAKZRgDeOlbUKVa0iMBbZ+bsuZnWHzacMwlAVGjeokXOHHwEo543d266iTpkH8EJyEZF0vNaykaYOBQSIAESIAESIAESIAESIAESCCDA8GsBOLhCAiRAAiSQmQTgqbNj5y63CwzB5qLgAgnEDQEIXkULF4ib/rAjJEACJEACJEACJEACJEACJEACJJCTCFDUyUlnm2MlARIggTgncPr0aYG3jrXGDetLhfLl7Cq/SYAESIAESIAESIAESIAESIAESIAESIAESCBHE6Cok6NPPwdPAiRAAvFHAN46XqO3jpcGl0mABEiABEiABEiABEiABEiABEiABEiABHIyAYo6Ofnsc+wkQAIkEIcE1q7fIGvWrnd7RlHHRcEFEiABEiABEiABEiABEiABEiABEiABEiCBHE6Aok4OvwA4fBIgARKIRwJeb51qVStL3dq14rGb7BMJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJZCgBijoZipuNkQAJkAAJREIAeXVOnTrlFqW3jouCCyRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAjmYAEWdHHzyOXQSIAESiFcCO3fvloUq7FiDqFOgQH67ym8SIAESIAESIAESIAESIAESIAESIAESIAESyJEEKOrkyNPOQZMACZBA/BPwhmArUaKY0Fsn/s8Ze0gCJEACJEACJEACJEACJEACJEACJEACJJC+BCjqpC9f1k4CJEACJBAjgUWLl8q+/QfcoynquCi4QAIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkkEMJUNTJoSeewyYBEiCBeCdw9NgxDcG21O1mk0YNpFzZMu46F0iABEiABEiABEiABEiABEiABEiABEiABEggpxGgqJPTzjjHSwIkQAJZiMAij6iTK1cuhmDLQueOXSUBEiABEiABEiABEiABEiABEiABEiABEkh7AnnTvkrWSAIkQAIkQAJpQ2D5ytWycfMWqVq5kqmwccMGMnbi5LSpnLWQAAmQAAmQAAmQAAmQQAwEKpQvZ47asXOXnDp1KtkaSpQoLgULFAgoc/jIETlw4GDANq6QAAmQQFoQKFK4sBQtWiRJVdu270iyLXhDuXJlJbe+TOm1vfv2y9GjR72b0nw5X758UrpUyST17t6zV44fP55kOzeQAAmIUNThVUACJEACJBDXBBYtWeaKOgnVq0rtmgmyas3auO4zO0cC2ZnAvLlz5eDh9P2PXXbmx7GRAAlkMAGd3JJEfWadOhlZwwUKOmXjaRIpbz6RPPpf96NHIhtDwUIiGsY24jHrZJqcPi1y4kRk9WPCD5wSI+xPZLUGloq2T4FHR7aWO49IPnCN7ndal07t5clH7jNtPPTEAFm2fGWy7Q0d8o7kx3g8tnT5Knn4iWc8W1JYzJ/fOUeRXpexjC3a6yYP+Om4EhNT6HwUuwvqdYXzgesxEov23oikzmhZR1JnastkVp+ibRfPhkL6/FHRMuJzGIpNelxbodoJ3hbHz53gria33q/vbXKWPqeC7fzLrg/eFLBetUpl+ejd1wO2YeWLr7+TL7/5Icn2tNxwRpuW8tTjDyap8vVB78uYcZOSbI91AwT5SMT4WOuPt+Py6r3UtUtHmbdwsezYsTPeusf+pJIARZ1UAuThJEACJEAC6UsAeXV69ejqNtJEvXUo6rg4uJAOBBJ1MgHCRfMWLdKh9qxfJfikl5F9epGNz3rT81qKzxGzVzETwERbk+Yi9RqJYNJw5hSRVSvCV1e1mkjrM0XKqTcFxBAYRI6Vy0QmjnXWvZ9Fi4p07iZSoaKKFTohCcOk8oZ1IuP/Fjl8yNnm/UQ/atUTadFKJ7Z1wn3dapGpE70lwi8XLSZywSU6Fp0UnzVdZMXSpGXR7/adRWrUECla3L9/x3ZtZ4LIls3+bViqniDSVifxypR1GGHbwf0ik8Zr39ZgLdByayT2Mzpo/bVESvrejj6kniNLl2ifpoWekK1YRaSD9qmscsX4T6oItGWLyIS/RN1OAuv3roFv63Yilao6W0f+6PTNWwbLsfQpuI4EHc+Z2kfYqJ9F9u51lr2fjZvp9aS/4+24jxwWWb9Wr41xzpi8ZUMs582r5y0KQwhha8d8osz2HSm/MS8lSynvLiIVKzviCSrB8bNniMybHfocRTu2aK8beBy16ygCzoVUMIXhXsE1PEWv/6OJzrZoPgurR8FZPfT60OvLTqrv0snH6ZNFNq5PWhMm/Ju31h+99yA6wNCHtbgHJyR/LTqlk35GyxrPiyrVk9Zjt/w5UmT3LpHKes130bFFaui/vV+j7ZNtI9rnpT3OfsfSbu26et/pdeF9Vu3Ua3yKPn+Cn1W2neDvWK+t1DyL0Yd4e+40aynSSJ9RKVkyv3P2qWeNfdbk1d8luXP7n0HJVZsH95bPTp06LSfwjFfbsXO3b2vSr5oJ1eWVFwaYNv736x/y+fBvAwrdctO1cuF5Zwds866M+nOMfPzZcNUBE90+59ZzAjEinMGb6INBr8iWrdvlvof7m2IQLvrd3Uemz5gtr731XsCh1atVlQH9H5EKFcpJHq0bj4v9+jtr2Jcj5Lc/9PdXGCtevJjcffvNAsEJ9t2PI+XrEfr7K4yh39ddfZmO9xx9lOWVZStWyVMDXwpTOmM2v/LiAGlYv656lJ6Wy669hV5PGYM9w1qhqJNhqNkQCZAACZBALAS2bN0mS/UNyAb16pjDGzesL7/9+Zccj/Rt0lga5TE5moD1Qpk+XSfaaEkIWD5JdqTBBls32acBzCxQhT3fWaCr7GJmEYDnQAudvMUkl2diXIqVCN+jRk0dgcZbAjM4mPzFvmBRB0LORZf7xR8ch/Jor3qCSBEVJLyiDjwgGjZWkeIM/6Q2jilXHp8pG+rtdaEKCqWdsvAkCjZsQ59sGezH5BqEHrSDyW/vRGnP80QwqRpsmGA99yKRP3SCGRPe1tCH81RUgvgFs+PFWFu3dcSt335ytjsltP56Ij3PtWvON/qDOq65SeSn70R2bg/cX0rHCKGpZu3A7fDGgODktVj65D0ey+CGPqJfMPvtrDmfHc9yBELvNogT9Rs548Y4YhEmvPWFWZ63YLH0H/BCmL1Bm0NdxyiCCXuIKji3k8YGHhTt2KK9bnDtXXJlUq44d/UaipSvJDJimHqI6f0TqeEaufRqvziD41AfhMMLLnVE1aWLAms79+LAaxdCF+5vXGfVa4h8+Znes4cDj0luLRbWFXSsJZJ5DuHZBYNQkVw5p5T/0x4XS59ieV76W3aWYmkXAhsEHWsQSXFP4RxerNfLr/osCSXO2fL4juXaSu2zGO3G43OnmN7bkVwzEN/C2OCPhwp+YPffc4ece3a3MCXDbx7y3y/k55G/hy+geyBivPTsU1K0iJ5vtbJl9H4OsloJNaQwvLfCWNWqKlqrzZ47Xx8FN5vlZk0bycvPPW2WQ300b9LI1OkNa9mqRVOzLXee3AGHoK4XBvYPEIlw2kuoYAMRqHKlCkZU8h4Eb577+t4uLZs3NZeI3VdNPZlCWQG9z2+/5QY5u8dZAV6ZVSuHLo86vvj0fYFoBNFnwSJ9mSKdrBB+36rh5YICBfJT1EknzplVre+vncxqnu2SAAmQAAmQQMoE4K1jRR3E2m3SqIHMmb8w5QNZggRiJMDJ5hjBpcFhZJ8GEFkFCWQXAr1vdybdMB47UZjc2DAxiDfoYdu3iYwZJbJvn7OOCX+8Ne81TPpbQQehzf74TWTrZkfMwOR5FRUsDgV5oJyvE8rYDoMYAsMMUaTWso0z2ZlceUy2W0EHnkKrljseGmingk4SnTgWeHSFCs7+GVN08nSdM6Fds47j/YBjOnUPFHXgxWQFnXH6lvJynVDCW9FtdDu8H7APE+SrVzrtgEWPXs4yeI76RV9z3qN90YltTLyDYw8VU7753N+vMjqhe+V1/nXwtV5Q/q3+pWj75D/Sv3S2cgsl5NgS8DSCxxcMTKeoVwQ8uOo2UG8KZQTmbdqJTB7vlMnMT1zLMHjkoK97dvu8PrSfEN8aq0A5e7pfvIhlbNFeN5hsBt/1a7Rfc/T1/e3O5HO7Ts41A8+nmir+rVrm9D2Sz65nO4IM7iUIiZs3qmClYsk55zteZ/DgWaOCpA09iHvPXruT9fwtmufchxBnIQ6hf+jP2D8jad0pEy1rHKWTo8amTXbuOWfN/2mfOxjPT9/6t2OpY1dHxIAX2bigfu7TbbBY+hTt89JpKfAz2nbB2wo6q1c4IhzuKVwrF1zmXB9dz9EZ7I8D2wlei+XaSu2zGH2Ix+fOnH9ElukzOZTVre+85IB9yXmrhjo2HbY99tC9RhxJruoiPsEHeXIHDxmapOjmLVuTbEtpQwP1PIFt2LTZLZpQw/GcW6E5eb32xCP3G0EHj5h3Bn8sf/09XmrXrinPPfOEFNO8Q5dfcoH8rB5G3tBknw4e5P5ah8dTcPhMb/1Y/uDtV6WiL89apH8WlFThDh5UEFrS05596XW5+oqL5Z/Z8+TgwUPp2RTrzgQCuTOhTTZJAiRAAiRAAlERWKR5deCSba2xhmCjkQAJkAAJkAAJZHMCyIOzZpXIL9+LfK6Tgr5QMGFHDVEChsnSn3Ui1U6sYhve3F+pk+Neg8eNFQG+Ha7eL5v8Qg3e/od3S/Ab//pmsiMY/SHy8ftOiDZvnckt4811eK4kZ/BcgCcODB428FJAX2CYLdqqfURYI69NmyoydIjIQp3gxtgxqbpssV+UwaSaHSeOq5HgHL1nl1MO9cIDetokHZtvgg3ihrUGyskKVz+P0DZUYIA3BryFfleBB4YJ/YqVnGV8ItwPzhf69PUwx5PHvzfpUrR9Cq4BYg2EDTujFrwf661UUIPhuvhbzx/OLca9RF8UmjfL2YfwZcjTktmGCdthnzjnBOIJ+rl+rcg/KuRYK1veLsU2tmivGwhLP3zjnHOIFcf1OsO1COHRGoSiSK10GfXu8ZUf/ZuKIxuc62q/XsP/U48pey6b6DmxZoVZXFtW0MG+bXrd4lkBK68CTzQWLWvUnb+g0wK80xAqLvjnhO+eRcha9M37Yz3Bjun/bbzbsWzzE8XSp2ifl7hH86oo47Vo27XnD3XM1GsTzx7Ygf3+ewrPn+B2IBR7LZZrKzXPYtt2PD534BmK6yr4J1GfV01bOD1fs9IR4+04MuG7rYYkC5W3J7grhX3eqNu27TDeOPDI8f5s3aZjjdDy6XWDn5rq/QNbs2adWce28hqSDYYcZ1iHtWjWREqVKG6WP/viK0GotxMnT5oyd/V71H3E3HnrTaaM/Titz571GzfJG+98aLyHELosOcujXmN7NOTdVyN+ksuvu1XDwm1LrrjpX6Qh8ZKrqFChglK3Ti1p366NIB9SKNu2fYe888EnMnW6ioUpGMLUtWrRTIr4zlkKxbk7DggEPcHjoEfsAgmQAAmQAAkEEThw8KDAW6dtK+cPWeTVKVO6lOzavSeoJFdJgARIgARIgASyDYFPPvBP7KY0KExCVE9wSv0zLbIQUC3aOuUhgASLN+Ha++WHyPvkrQOiCkKhwSDUQACx3jjOVucToYxgyG/jDZnmbA39Gc4zAnXAMDmOCV8YxBmIS7AVy5xv7+eiBc6keCmdcEdZHIvQZDAISvCY8pqZ3NcJbEyiIQTX1i3OXggR3vOXTKigmPrk7QM4IvQYbIyKNfAaCjaMpVoNZ+tyPd9WMLDllug2sDflqvsFArs/o7/Dhas6eMDfEysOxDq2aK4btIqJ91CGyWZrVpSw68l912vg7MW5sHlkbHmIAxvW6j1dU68/va6Qewpmx4zQW/jxCr2WTaJOikdj0bC29fomjeWYijbpYbH0yXu/pdQn5DG6trcjtiA82iYV1GDRtmvPB44tWACffvM+f3Qi3bUrrnOeQRAo7XmN5dqK9VlsO4L7JtpnoT0W3xn53IEAh7CD6DO81saO9vYkw5cRTq3/ow+YdpcsWynVNISaDcEW3Bkbem3/Ad/vo+ACEa5DwPj+y08DSiN/DX689tpLA83q+Zddb7xwsIJHzE+/jPIWk70q7sNLqErlitKqZdOAfdHmnrn9nociCmuGcGtvvfaClCzuCE1o9Nmnn9B3JE6Z9uHJ9Ouo0fL8gCelRfMmMm7CZFm8ZLncfOM16lVU1PT5hlv7yoP97pIundrrLRfo5QNhafhX3wXkCXrkgb6CfEOHDh3WW/5O0w4inwz9+F3lcloeePQpefj+voIwebi8rI0eM0HefGewXeV3nBKgp06cnhh2iwRIgARIIJAAvHWsIUFti6ZN7Cq/SYAESIAESIAEsiOB4In35MYIEcIawkNZ885S2G34xiQZ3iCHecWTcOWdkknFALs9pe9OZzlhs3RiRSaND1/aTjJG06dwtdWo5eyxIeWCy4XKTbRbvQ6s6SSSMQ1RY8wT6sbZoJ84R9a7ByGUvBbN+bPHRdonWx5v61uxDF4GNmSc3W+/i/jGgPXN6mEUbPv0RSHb3+BxBJfNzPUE3zk13Lc5PUnrsaV03QSPv6qKYNYgxERqyAsEgwB4ypnUdDb4PjeriAjzvjVuvXFwnyIkIHLWwOAJUques7xSr4O0sFCsUS/ats+Joyo+edfTot3k6gjXJxxjr9/kjrf7KldxhFj03dZp94X6tmW81x3K7VZvP+udgzBrXqG6gU8MxjVh+4bcR/YZV6d+qJYCtyV3bdk6A4+IbS3enzvwQi3jeKLIr//ze2/GNtpUH/W85qiBoHD02HF55tl/J1ufDS924IAjSCMPTyx24sRJ2blrt/mxx9t1iBkwXBLYtlW9U2A2x8/e/ftDii6Lly435Qrk9z1HzBrwHvctRfYVafm8GuYUYdoKegRQ3IJ5lAl+SpQoZhosos90rEOMua/vbVK8WFHzmMnr83jrcGZbV9BJTDzq5hqGV9K9d/WRihXKux23dSHvjzWcA9SP/rz35stSu2YNOakvfuB8WuvZrYs+Xv3H2O38ji8C+puPRgIkQAIkQALxTwCeOnBlrlTRCdPQWl2Dx02aIie9b37F/zDYQxIgARIgARIggfQgUNSZDDFv7iPvRgv1uiitk2B4ox7htjatd0JZ2cmawj7BAn3BRMk5F2jOEg1fgrwv8EaBEDJ9qh6bBl7B1RNEEMIMNkon5LzeBc5W/6cVUBC+Cd4nCTU1x4hOfuOYLVtEFsxxwnD5jwi9VLuuP9n2MvUMsoZZL7xBj7wsKDNxjH/CFWWO6Fvg1opou5iIs7lwvB4Ztgy+rZeTFYG8+yJZjqVPtt52HZ1x4u318X/ZrUm/7fWBPeHGkah1IMG7t2zSmjJvSzG9xpHIHgYBy05qe/ub2rGFu26cVpN+YkayQxdnO+4bhGOL1DAeGLiHMnstwssNOZ/wNz/CeiFM4JmdxOR9ql5DvT1m6LWsgg5EWoRQDJePJFQb4baFY43y1ksHy1df7w9tiPsKouJMfW7Y5wzKpJUl16do21i/1gkjiclseA4mZym1izCR51/s3IfX3CiyeIFzvpDXC9cEcldZgwCE9qrpcw05oZKz1FxbydVr92WV5w7CSrZs6/Qa19YOn5hrx5HB3xdfeK40rK/nVu3l19+Ww/Y+DdMPm4/movN7ycUX9DKljms4SYQFe/WNd2XlqjVhjgzcDOGk9+39TGiwEcM/FuS6wTrsrM4d5ImH+5m5AnjNWCvhC7125HDoZ8zuPc7vd4RCgyeQN9y7rSMtv3fv2SvwIGrcsL5Yj6IBz79i8t2EagfCy569+4z3zgrltM8nXg357xcmBxBeerWC0vm9ekq/u/sY8adzx3Yy4odfQlWZZBsin7z17ocya858s+/aqy6T3tdfZerB+Rrxg/7NQotbAvrbkUYCJEACJEACWYPArHnz5cKKZ5vOIm4uhJ0Zs3Ryg0YCJEACJEACJJCzCdg3rTEB3OvCQBaakFhK6GQ43k7/aYSGr9GQSdZLACWDw3VhYh8TktUTRH781smXEVhj5GsFC4r0PM8pj3BDyJMQzjCJaQUUG4bNlsW4kCAeP0iuvmyp3ZP0G54mdkwb1iUtO/cfJ1k73pq/ra8Thm2vTm6VKSdSRyfHvYYy1sAtlNkQTBqSJ2aLpk+2EeQeatbSWRv1mzORrpNgIQ3n1Jr1LLDr9htvKWMICE0Vb4ZQY+df4g+/NEHFOGtpNbaUrhvbnve7czcR69n0m07+WaHJWybcsvXASem6wvG4tjQcs7FFOvmIkGzwzsN9cUYHZzuET+RKgiCaGkuOdXC9KIsx496FUIqcJ/Xqiwz/LG2FnWj6FNzHUOu4B378JtSewG2RtIsQjBDSrOBov1ETwoR5c5thmzcHE9bDWWqurXB1Bm+P9+cOPCXOu8jp9fZtInP02Z2JVkG9TO64VYU7tclTZ8r0mbMi7g1uEXur5NMXKapWriRvv/6iPPvSf6Kqp1FD53fU/v36woHP6tfT39dqwblsCuH3r9qRMGEhrUiCMnhxdLXm6IknO6ievQi3Fmyj/x4XvElGjR4j99x5q0Cg8nrqJCkYtOHLb35wBR3s+v6nkXLTdY6oU6d2zaDSXI03AvobkEYCJEACJEACWYPA7LkLpGunjm7MXoo6WeO8sZckQAIkQAIkkO4E8ngm8+fN1rfBFzqTiUjG3lAFncb6g9BAEAFmTnPCr9lOIaTTPJ2cQhiogoVE6uqkLDxAMGHcpUdkk5+2ruDvHuc5b/YjkfrsGcF7w6/v2eW88Y/k8TCITGdon+CNgDBHyzUsbagJdIgwl1zpTDJj4vbvwDwCpi6TN6eSM06M0XoRYScmxrENdmh/4MR0/nzO9uBPK/wkanuxWjR9Qhto04Zdm68v+CDfT3JmhSeUyecbX3D5vL7xhZkADC6eoes99Jzb0FYm/JKHdVqMLZLrJnjADdX7rGETZ+u0yUkFy/adnfMUfNz6tU7OIg0bJKqDBHi+eMt6PWKs8AMx9lK9viGgwCsH91Tj5k4IMVy3N9yq3nD6hjragKF/5Ss4y97P/eq9F26CPDnWqAP31Wcf6bf2396D4NdE+9G2vSPMYuxe4c3bdizLKfUpljojOSalduFBBS9HCODwykFOK4Tva9zMeQZBiJw903mWRdKeLZPStWXLpfY73p873dWzBdcWnsvwiLLXW2rHHePxL7/wjAnbdeDgIfn364MiqqXP3Q/KIfXmtB4lCOl16UXnGeEAAsTjD98rV1zXJ6K6UKhBvbqm7Jat+vvaZwk19GUHtdVrA0WZo3qvItSZDQHnK+5+IU+NtR079XdunBly3oQzCGwIz4Z8RnvVmwdjP6HXSf7c+cIdEtF2nCfUA+HNWuVKFeWRB+6xqwHfw778VubO17+3aJlCwH+WMqV5NkoCJEACJEACkRPYryFAZs+dL106alxhtdo1E9T9u54sWbbcrPODBEiABEiABEgghxKwSdIx+YXwTNYgpkwaq/k2ajuhtarWcESdwwdtCZEZU0TgqQI7fEgFHhWFypZ3vFYwIQzvj1A5P5wjwn9iQhmeNTC8nZ7feWvYrOfSyVAYJq7hnQPvAvQdYZuwbbFOkqxZ7ZTB53J9Ez6XTvBA0IFVqJxUyMAE60VXOBPemAwa+YPjleQc4f/EPky+LpgrglB1xUtoWCud5Mab4KjjvIudsvCMQFnbp4IqKIUyCGEwL1NnS+Sf0fQJtXY925nsRNg1CHLWw8kr7iG0FLYfS1QvDz2v1tDfYO8B7Cuok6cwey05a5n/ifBmteo6/Zg0Lmn4pdSOLdLrxkuihr7BDcEThusU5yDYIKRagdC7D9sgpCK0H/KrhPPwsteVvQZRR5sznOt72xaRX/T6xr6NG/QaLqnX7YWO8AXPuP8OdvYhF0z1BG/rzjIEiFCiTkqsbU1eIQ3bIPRAvKhQyWmvWnVbMvXfkfYp9S0F1hBJuwm1nfHi2fXNF/q80XsN5wNhwjp3d56hrdr6vAF3B9Yfbi2SayvcsdFuj+fnTiP9/WGv3dG/O7+boh1fGpa/+/abpYJGyoC9/f4QKVNGX5LwGcQZWGG9l8tpmUP6TLJh2fYGPWuPqkD7zXc/SflyZeS8c3oIvGmQ+wa5cJIztA+PHAgMsNq1asibrz5vlmsmOPdbj66dpVmTRjL0i2+M0IC5gxLFi5l+mYJBHyVLFjdbcBkcOOD5myCoXLyt3nlbb7nkgnONg2BG9K1M6VJuyL3g9po3a0xRJxhKBq5T1MlA2GyKBEiABEgg9QQQgs2KOqgN3joUdVLPlTWQAAmQAAmQQJYmgAliGCaMIYpAiPDa2jX61r56FpTyTUTZUE4oU7KUX9Sxx2DS2YYiQ2i3WHLreMMQXX6NrTnwu3U7/WNGfxCabZb+HFLxoaROUKNPwYYxWCtbJlDUwaQaxBibhByCDjyPkjPkZgjOz9D2TOcITFJjpgt2SNnCS8TW7Wz1f5ZRAQyGfCeptUj7VFMnk2EQbW663VkO/rz4CmfLT98G5nrBOLZtDSyNUGBWgDiYBuMIrD32tRZtnJBeqAET5Qg9Fmw4P9aiHVss1w1yfFgvKYTeGv2rbT3wG947Xm8bu9del5Yzcl+FMgirMG8+DHiswSB62usT6/v3ivz5m+a4udFps5KKnps3iSzRcsiPFWyh8vhEwjq4nuB1XL/VEwLDOwaXiWY9LfoUTXu2bKTtIpcRbLU+L71CF54fxmtHxT9cAwhJN0Ov35Qs0msrpXqi3R9vz50S+vzv1M0ZxZJFIus8z/5ox5ZG5ZG3xtpTjz9oFwO+O5zZVvAzaeoMeenVtwL2Ba/MX7jEiDrY3rB+XZk4RX//JWNntGkZEFYMAlL9ur7fA77jSpcqKfhp2rihERqQL6Zalcpic+sEV1+rZoLZhPw8WcWaNW0kl2peI9g+DUE3Sblt2rzFeOz06tndhF9L67Gs37hJxk6YErLa2b5cPCF3cmO6E9C/eGkkQAIkQAIkkHUI4I8WuPi2aKZvL6m1aNpYxk+aIhs2hfgPW9YZFntKAiRAAiRAAiSQGgK7d/qPRr6NhUGT35WrOvsRdgmmiZpNuCDkI4HYs1a9DbyGSWFrsYoVhw/qG/wqCIUyhNSBYWIaE0p2QhSTixB16ukYpoxXDyGfsIKyFT198iV4xmbzui7yCMHrBvbbz86EtrMW+SfCjzVv5ZRfONd/HMSkFqU1rJJO0CLHxqmT/n0QnxASDhbM0Nmaus9wfYKHjvV2Cm7BsgVX8D2p/cXPzh2OMFW3gYojCwKPsp4w2LpJhYp4MITzauebSJ2lXiDwBAllsY4NSS6ivW4q6FvyF1/u9AIh737Va80rrnj7t3Cedy3pMq4X5KDBpD/q9Qpt6Fstn4Cz3ndv5tN7xj23R5PWZ73tsKeE3kMQddBGJNdlpKyTthq4xYbIg6iRWkurPkXbj2jaLesT5I6HGC+uiz3qfQFvx1AidXC/orm2go9N6/XMfO7AM/SCS5znOoRPeOfFgR1U75si9lkf1J+88PZTM49b/f1wwL5kEVTOu1qnVoK7unW7PptTsKeffVmKFSsqr2gIuPz6zHju3/+RPRp2DMvYdlK9aR/t/y9Ty+Ytjmg/dvxkM1eQR5l26dReJkzyC4u5dZvNGbNqtf6OyyTLhWddFNa1c0e39KP9B6qg439BoVuXTibcnFsgjRaQe+i1N99No9pYTVoSoKiTljRZFwmQAAmQQIYQmDVvgSvqoMFW6q1DUSdD0LMREiABEiABEohPAgiZtXG9hjur7uSe2aIveyD0GjwRkOOhhE9c2bTB3/+5GjIKuS+q13S8clYud/bB2wGiCgy5bbwihrM1ss/ffwlf7hr1KMAE8PTJTrg3WxJhrJDTBxPdnbqLTBzjzJQhh0jLNk4pzJzt3O4sY0Koh761izHARv6oooRnjM7WwE+Mr0BB9WDY4oR8w154qmCCH94qqB95aqwt0OUWrX3i0flObgeITZhgP1e9g2AQWSKZPHdKJ/2Mtk+fDUlaB7ZgQvSOfs6+n0c414AtifwryP+BEFkQExB+DlZGJ6YRagq2ZqUj9jlrmfcJobHjWU771osrud5EO7ZYr5tLrnKuA+SzGflT7PcGxgIvH0xaI0/O2XrtfTdcBB406Fu3c5x7AOXmzsanip8qHOA6g3dWO71v4fFjPe5w3bbxeZmh7Ba9tiO1aFlXT3DuEbRvPX6Q0B7CYG39geE6So1F26do20K+DIQwxH0/7m/H0wl1RNsuQjYW0/OHUJMrljrPFNSDc5igzySbzwgCm7VWZ4jA0w5hLzesc7bi/k/La8u25f0O1240z8KMeO500PseTGEQTWP9/ePUkGafd9z7cNi6vv3iY5Pz9u9xE+SNtwe75dq2biG1aiaY8OkrVqrAqoacOuf0OEsu0bw6sMSjx8TuMxvCfECoyb0ttxFxTunvn2kz9HelWpNGDcz3fg2ftmx54H03+u9x0lfDtiGvzkP33W32b1MBCYLOoNdfNPmBcPBHnw4zdWTUh1eIad+urcyc5ftdFEEH8ubN45bKlUt/3/msZ/ez9Fey/k6m5SgC+iSnkQAJkAAJkEDWIoBwa6vWrNOcOjVMxxGCbby+eRMcszdrjYq9JQESIAESIAESSBWB6TpJCG8VCCJXXq8h1faqF0kR/+QwJoT/meZvAiGcIFbAWwfCyJmdnH0QUKz9/addyphvCFHIUVJThShMsCIEnA1/ZnswdaJObvs8FTBpaieSsf/Cy2ypwG94qXz/lbMNIayQ5wIG0Sq3TgtY0Qvbxv3lrx/rmuTaJKTHpCjEo9v7OXln7MQjykwY60x0YzkWi7ZPsbQB0Qm5WCDqQMSBCHD8hN/TCJ4906fGUnPaH2Pz1aDmNu2cn1CtfPi2szXascVy3XTq6kzWo8VKVVQ8u9dpO/gzEmHRHjN+jHolXOqcg5vvcESeIsX87SDcHEKrWZs6WcWIns71ekMfvTcOOtdnufK2hCOo7N3tX09pKVrWeGZg/DBcM7g/vPcPnjPoZ2os2j5F2xaEKfvcaNJMvQInODVE2y5CRsKjCiIORBnw2KViVzl9LkFog+EcLVvsLEOQsyEeO6gw941P1EmPa8tpMfl24/G5g1xU1q65yS4l/cY9/8fIpNvjaEtX9Rzp1qWD3HzD1eZdgSMqghbSXFm4XKy99e6HdjHF75o1qpsyh3DP+ayxT9TZtk1/x4WwT4Z+KffedasUyJ9P/vvhIElMPGqEJduHBYuWyvIVq0IcmX6bMGeBHELIJXTu2d2kR7fOKjDlkd/++Eve/+i/yTY8cfI06dmtiynzwduvCsaN3EAIR0fLeQT8sl7OGztHTAIkQAIkkIUJzNbcOtbwRwyEHRoJkAAJkAAJkEA2JuANRRZqmPBeGTHcmUTEfoQxg8ADQ/LuESpqYNLR2gld/vpzx2MA2yDmWEEHE5HG0yP0RJGtQpUM/2I0SydPOaU1ZEwSQ46SebOdzei/DemEZOTj//Z7mKCEnZlKUknQBnjfWNuyURn5JsVKlfFPSGPb/74TWb7ElvR/z1QxbNI4R7hBm1bQQZgpTOKvXukvG3LJ036o/bH0KVQ93nEGs8W+nzE+9SiA4a1mG04IAuC3+rZ2LLmTnNrS9tM7jkhqjnZssVw3kR4TzD25/sO7DjmPIITA4LWDdjAeeHLgmvMaxIGxo/3eVLhfraCD+wNeTX+N8h6R8nK0rE3+GF9/cX96BR1cW9/qM8iOJ7nWk3ueRdunUO0kV//Onf4j4LFnLdp2EV7tRz1/EI1h4IG8OFbQWbFM5Idv/M/dY4n+Z4/XeydNrq1knjHh2s3Kz51khuucjIz7PO27bk7a32u+phEy/fAR517BKcb/2e2p3qMhvQa++FpASLSUelzPl0Nnh+f6rVG9qjls3Xr9HR/Cfh01Wl7+zzuq3+vzQQ1eO/YRM27iFHni6edCHBW46XSUv+ctj8BaAtc+/my4EXawNR8859TsC6qnkvHOglfP6DETTHmElatcqYLhOkdZb9vh3NfwZLJ2CuE51SLpkz3Gfp844Rxr1/kdfwRyNWjV1X+206l/B/fvSaeaWS0JkAAJkEBOJYA/fh7ud7eUQ6JgtW3bd8ob7w3WkOn84yP4mihUWP/DSyMBEiABEsgwAkeQSyWVVrR4qVTWkPTwHPX/soIFNX9KeSeXym71RrE5a5JicbZgUgUhgJC3AxOVCOeW2YbQcRBdiqrngr7ZK/h/tWeyJlXdw6xWcZ1AN5PoWhPCKEWaBwRJvHEsuEL8SitLTZ+i7QPCtJXB+dZJ6F06IW09nyKop3vXTvLoA/eYkg89MSBJyJ/gKn4e8bmZtJu3YLH0H/BC8O60X0/F2NK+M1HUiFBgpcs61xTy4yQnMOBaKVZCf/TeQD6P/fv1R++RZCZDo+hJZEVx76B9hF5D3i3coxCKs4pB1ETumMOH0qbHOH/IZYRwZsipAg8rr4huW7HnzuuBZfel53e4drPIcydWNPffc4fxBsHx51+mHqzJWI3q1eSDQa+YEh9+Mkx+Hvl7MqUj31WxQnmpXStB8L1j5y5ZsmyF7PAJEKFqada0kbz83NNm1+uD3pcx4yaFKhb1tqpVKktzzce7UXPxLli0RE5FI0BH3VpkB5RUUbhAgfw6l5HSCySB9ZUuVVKa6VgOHTqk+YYX6a2WhZ49gUNJk7XU/M0cT38bRzsORw5ME4SshARIgARIgAQyjgDetpmtuXV69ehqGq1Qvqzx1pkxa07GdYItkQAJkAAJkAAJxCeBRH0jHF4AkRre4vW+sR7pcelZDgIOwrHhJ60NE+aYhMZPtAZvlvTwaElNn6IdAybzdqiQlYHWrEkjGfm9enGozVu4SJ4a+FL6tJ4JY0uTgSCk0uEI71lcKxAFMloY8A4U+YDwk1UNIm6kQm4kYzTnT89hSmbPXUrl0np/uHaz4XMHQs45vv8j54JoFYPd2ecmuePWG82Rn37+pfzws3qQxmhbt20X/CRnyL8z8P8eS65IqvdBzMFPPJn1zom2T7v37JVxEyZHexjLZzMC+noKjQRIgARIgASyJoFZc+fri5X6HxKfMQSbJcFvEiABEiABEiABEiCBeCKAudXc6n2Fn4Lw7qCRAAmQQDoQKKBeWPZZE6OmY0KU2TrywaMxnS235pSx7eGbRgIkkDIBhl9LmRFLkAAJkAAJxDGByy++QDqc0cbt4SfDvlKX7uXuOheQ/5nh13gdkAAJkEBGEmD4tYykzbZIIGMJRBt+DROiuRESzWMIlRMPoX88XeIiCZBANiGA502wEIPnTSQhugqEEJwz6nmVmW1nk1OfY4cRbdgyLyiGX/PS4DIJkAAJkAAJZCCB2eqt4xV14K1DUScDTwCbIgESIAESIAESIIEcRGDR4mUy/JvvzYjXb9iY4sgjmUhNsRIWIAESIIEICUDAORpFnjBvtbEe560j1uXMbDvWPvM4EshMAoGvi2RmT9g2CZAACZAACcRAYO36DbJwyTL3yBaaMLCaJkGkkQAJkAAJkAAJkAAJkEBaE0BC6+Fff29+jhzR3E00EiABEiABEiABEshgAhR1Mhg4myMBEiABEkh7AvDW8Vor9dahkQAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkEB2I0BRJ7udUY6HBEiABHIggfmLFsv6DZvckbdu3kxKlijurnOBBEiABEiABEiABEiABEiABEiABEiABEiABLIDgbzZYRAcAwmQAAmQAAnMmjdfqlerYkAULlxIWrVoLmPGTyQYEiCBLEygaOECWbj37HooAgcPHw21mdtIgARIgARIgARIgARIgARIgAQiJEBRJ0JQLEYCJEACJBDfBGarqNOtUwcpWbKE6Si8dcZPmiInT56M746zdyRAAiEJQNApWKCANG/RIuR+bsyaBKZPny4UdrLmuWOvSYAESIAESIAESIAESIAE4oMAw6/Fx3lgL0iABEiABFJJAIlqZ81b4NZSoXxZac3cOi4PLpBAViRAQScrnrXk+wyhjkYCJEACJEACJEACJEACJEACJBA7AYo6sbPjkSRAAiRAAnFGAN46J0+ecnsFbx0aCZAACZAACZAACZAACZAACZAACZAACZAACWQXAhR1ssuZ5DhIgARIgARk2/YdAmHHWu1aCdKwfl27ym8SIAESIAESIAESiCsClStVlAJx7sGG/pUuVTJduZUtU1qKFSuarm2UKFFcChcqlK5tZFbljz3UT7778hN5541/B3Qhb5480rNbFylXrmzA9uRWhrz3hqmrT+/rkyuWqn1tW7eQRg3qpaoOHhwbAVwTVSpXlNy54386EM+F4sWLRTTQfPnyCZ6nWWFcEQ2IhUiABEggBQLMqZMCIO4mARIgARLIWgRmz50vbVv5c3C0bt5clixbkbUGwd6SAAmQAAmQAAlkWwLnntNdbrjmCilVsqROQOYy4zyhOQCXLV8pj/3fsymO+5abrpULzzs7bLlRf46Rjz8bHnJ/rZo15N67bpMa1auY/U8+84KsXLUmZNl77+oj3c7q6Aohx44fl9Vr1svT/3pJDh85EvIY78bLL7lArr/mcrNpwHOvyOKly727zQRs/8cekFoJNSSXg0FOnjolc+YukNfeek8OHDgYUN6ufPrhICmuAtC1N90p4FaoUEEZ9sl7snvPXrnz3kdsMfONid4X/9Vf6tapLQXy5zPb0Pcp02bKG28PDijrXcFE8t233yxntGlpNn/340j5esSP3iJJlrt0ai84N+jb0aPH5IZb+yYpk14bKlWsYM5T+SDx5pUXB5gXnE6dOi2XXXuLHNdzmJKV1xDG+fLmlQoVyqVUNKb911x5qdx8w9Xm2IEvviYz/5kTUz08KDoC999zh3Rsf4YUK1rEPTAx8aj88fc4+fDjoe62cAuvvTRQaiZUD7dbXnj5TZk7f6HZDxHxg0GvhC2LHStWrpH+A15IUgbPqP6PPiD2OkSB06dFNm/ZKv968VXZtHlrwDHXXnWZXH35xVKwoD+86779B+TDTz6XcRMmB5TFyteffyR58+ZJsh0bEPHhmpvuCLmPG0mABEgg3ghQ1Im3M8L+kAAJkAAJpIrAitVrZNmKVVK/bm1TT4tmjWXc5CmycdPmVNXLg0mABEiABEiABEggtQT63nGLXHT+OQHVYMISb883blg/YHu4FYggyXmcVK1aOcmhzZs2lr533iLVqzpiji1QpnTpkKLOy88/I82aNLTFzHd+FUga1Kstnw15R+554HHZuWt3wH7vCt6Yh6eHFa0KFw70kMEb+B+8/aoRD3AcxJk8ufPoT25p06q5DH7ndbn5tnvNdm+94FSxfDkj/uAYGLiBx+HDgUITPHPeH/SqlNJvr6EsvFcSalSXBx97Wk6pkGStgtZ9X9/bpWXzpq7QhH3VqiRlao+5+MJz5TqdWC7h8SgoVDBwvLZsRn8XKljQNJlLVbMCBfJHJOqkRR/B5PZbbpB16zfKfQ/3D6jSKyqUKB54bgIKciVNCEDYfOOV56S2iiXW8MyBkAohpJMKPZGIOtX02ZHcc8frDQZRMLmy6Ae8hYKtc4d28sQj97vPDey3fUV53LNeUeeRB/pKj66d3WogPOM5hXvx8YfulSKFC8uvo0a7++HFA9GVRgIkQALZgQBFnexwFjkGEiABEiCBAAKz1FvHijrY0b5taxlBUSeAEVdIgARIgARIgAQylsCZZ7R2BZ09+/bLC/9+Q72JHe+VqioadOvSMaIOFSlS2JTbuHmLDB6S9A17vNHuNYS6evbpx91Nx0+ccMUUd6NnoXvXTq6gs2btennx1bdkn/b3+muvkMsuOk+KavuPPHBPyLfsbTUvP/90wMSs3W6/4TWAiV8Y6p88dYZg8vmu23rL+b16GCHmOvVmGvblt/YQ812/Xh3zvd/jxdOwgSOGbdm6PaAs2rCCzqjRY+W/w74yE82PP9zPeK/U0TC9F1/QS3765Xf3uE8HD3LFHDtB7O4MsfDQfXfL2d27uHvgaQRhKl7s2Zdel6uvuFj+mT1PDh48lGHdqqmT7xDginq8QmzjQ4d/K2B7RD2mxoybaDfzO50IPNjvLlfQmb9wibz17oeydZtzr0DsrV4tUOgN1w2IJbC/9ZyNHZ/UA2b5ylXuods1JPbTz77srtuFG6+7ygjDWB/993i72XxDBLKCDp5Rg977SCZOnm6ESHjOnd39LMHzyBrKW0FnrYqH/zfgRdm7b5965dWSfz/3lLnXb7/1xgBRp6RH4H31zfdkv3r00EiABEggqxKgqJNVzxz7TQIkQAIkEJYA8urgTS/7VmW7Nq1kxqw5sm7DxrDHcAcJkAAJkAAJkEDWJgDvjxrVq8rCxcs0/NXRgMFANKmgYaWWLF0RNnRYxQrlpZp6uSxeslwOHT4ccLx3BW97Y+IQk9YImWY9RrxlMJkNzxGvF8jtt9xoiiD8V7AXCjyKh301wltF2OXC+vY5bNs2zSWoL7KkZHlVPEH4rQWLlsgX2gb6/8oLz4Q97MZrrzT70M/7Hvk/dwxDPh1m/raCJ02zJo3MhH0ooeC+vrcJzoV9wz5UQ7Vr1zSbt+rkLwQdGEKDvf/Rf+Wcnl0NW2/OFbDOpf1uqu3CMGkMEQhWT88FbL3+nYdtqAffNnQawr69/f4QUwYh3R55cqB8NfRD8zb/VRq2ySvqnNZOb9BzgXBrf40ZLyO/H56sOJUvX15J1FBrU6bNUAFqhBFQzjunh2krrT4QXg7XL5hu2Lg5Ku9z5Jt854NPku0KvKpKlSxhwhV7r9dQB+E8VK5cSRDubefOXeqJsyHk9V++XJlQh5ttOD+fq7CTnOH84fzvP3AgYCI/uWOwDxP9DRvUVS+yPaZvKZVPi/3Vq1U13h+4zrz8cJ/Vr1vHiITLNYpAqOcEykAAQz6phXp/hipj+4jcVsgVum///rBMIH54hQpw7NLpTFPFwsVL5clnnrfVme95CxYJfiKxvHqtw/B8TOm5g3EEl7lBRWF4+sFG/j46yfPutlv8nn1973/chFszhfUDY/r+p5F21Xx3OLOtu/7+h58aQQcbVqxcrffuBCPYIuQivHXs87x06VLuMZMmT0uWt1uQCyRAAiQQpwQo6sTpiWG3SIAESIAEUkdg2sxZrqiDms5s24aiTuqQ8mgSIAESIAESyDACzw94Ulo0b2Je5DTzAAAuYElEQVRyImAS8eYbr9FcEEXNxB1yldj902fMMpOcPfUtbuv5gU5O1e2DNU/EYw/eo5PD9QMm5jE5+MnQL92xnNW5gzygXh3enAzINTFn3gJ5/uU33HIlS5SQl/QN8Bo6iWtzwGAnJkv/9cJrrliEXDIIPYWJzdv6PiQ7duw0k/KVK1Uwdf3w86+pmky0YY283ipuJ0MsTJ3+T0A+laaNA8OqeQ/BJDDELdjMWXMDJqmx7ZvvfjLh0TD+8zQ30IgffsFm15o0aqDbHVEDOWiuu/oyd593wYpuBfLn92427YE9vIH27N3r7vt62EcB4Zzgkf3zt4FeShec21Pw8/J/3jHn0nrM/Pi/39x67AKEJOMRpGJG6VIlTT4e7Is074yt5813PlQR6V27muz3N8OGCMLQrV23IUlIMhz43ZefaIi0AjJDc8w8/+//CDwskKenoIZN8xq8vIZ/9Z389sdf3s0hlxGeqqt6gB06dFiu7X1nQBnkDOqpXkb2esLOBYuWmjB4AQV1BecU92Bw6CoId+MnTZFX33AY4LrvpuGwSvrCqlXQ3Cq/fP+FqQ7eXjf2uce8ePX4I/eZbbfr/QHhyVo9Pa8Dn3rM9bDCdrSxeOkyGfD8K+rdk2iLus+AiTo5D1H0yksvCriHIUo+/MRAI/a5B6WwMPyzwWaMr6kXR/OmjTSnVGfDH88TnBO7f+gX30jrlo64aZ8F6Ofgjz8z5/eh++82YQJtc3gWvKQeadO0Hmt3Kf+Lzjsn4NmEexrPpxE//M8WM+EFn+n/SAB7iLQjf//TPONsQTwTW7dsJrt275GbNHQhDF5a9j54+/2PbdGYvhEeEea9LyOt6PxePU0eMZSHlw/E22Dr3tXxeFu6fFWAoBNczq57wz+WUcHTa5vUixGGcwKvMGsINwnD9uQENFue3yRAAiQQzwTixy84nimxbyRAAiRAAlmOwPR/Zsv6jZvcfrfVt0qTS+7pFuQCCZAACZAACZBAphMoUqSImYzEhDQ8PzCZjMlTeJ3A7H68rY0JZ0w4Hkn0T/i211BnQz96WyAynNZ/8KawdomGELMGjxpMfEPQwUQfEmwfPXbcrDfxiB+Y+P5syNuSoJ5A6AcmCu2kINp4fuCTtkrp3NF5Mx5eDR19b5M3aljP3f/Lb3+6y3hTP1pDbhTYAfVigEVSB7wjIjF4KlmbqV7OwbZoyTLDCdurVPaXxToEoX89/RgWzcT21yoAhbPpM2ebXfAQeVSFN7CCwTsDgg5spIfT6jXrTA4fTGbDDmgoMUzqeid27fpundT25g6ynkDmQN/H1Okz3VXk6bAWKadYysOrCxPsyG0SHJYMCexxjWH/+IlTTPW4tq2gA6ELIalgCCl37119XPHNbAzzYe8TiEVeg/CI0HNoE9c9BBCEjmvauEGAyGCP6azeHlbQQT/svYZ7oauKosihA4Mog/5ZoQPbMCb8WA+zfOo9Ybfl8Z13lGumIsqbrzzvCjp2zKgLeZOGDnk34Fq3Y0P78C7DPYxjMB4YxnaThvuKxpCDCH3DNYnniuVvBV+7v0/v64zoc/LUSfO8QBvoJ3JmwQsOeZ/AyT4jcH3fctO1blcQEvES5Y+cUygDoQ7fYAwx2xq84l57aaDLHucJ48NxOH/XXHmpLSqN9TkEK6PeKPD8gcFTCIbjvPlFI3lmmAN9Hyhvz+nuPXvNeYi0DlzbuF5hiJzw2lvv+Wr1f4EPvGpgk6ZMc3ck18bc+QtdvvgdgeewtfPPPdssIhSl954uVaqE2Y7zBkuuflOAHyRAAiQQxwToqRPHJ4ddIwESIAESSB0BvA3n/U99e/XW8cZiTl3tPJoESIAESIAESCC9CWCCdc/efSYvwopVa0xuF2+bmOT/7IuvBd4vCH2Eib1XXxxgimCCd5CG3bKT5L1vuFqu1UlQTCDCA2LCpKlyy43XmnVMlF5/691u/RAovAIHhB/klEC5l15zcsBgQvAxTcZ9ltaFsEgldDIb3gjf/fiLPHTfXWaC+c+/xpm+2JCw6C9yQ2BSHeUxPky44u+T59QTAOHBUjKb2+Ki83uZiV2UxwQyPB7gMbFSOcVq1ar682vs2LErZDXHTzjJyMtrODuvQdDBRDr68uSAF7y7kix/PvwbaamT1/g7rftZnaR9uzbGK+ucnt1M2enqrYJwVtYef+o5swhvFrTx2P89azwwcA5GqicIuPa+vZ8tLpf6hDsrArk7fAvbtu90N1VVISs4VJS7Mw0X4N2BSXrYZRdfEJAv6OILHFEE4iOuS9iQ/35hvLwgpNmJaXg89Lu7j5lg79yxXRJPKXNgCh/wTLrt5htMKXh19H3gcTffzlWXX6T3xHXuBL6t6gf1Hvl91F8yRyfSbcg9hIOD0ILJ/i4qZP5v5Cix52nIe2+o6FdRtqmX2q133m+rSfb78Yf6mbogbjw18CUTLhAH9Ol9vVx52YVG7IN3UbCXB+7J0Roq78NPhhpPHghmyI0EcRCeK7EYnhHLNGTauAlTZNNmhLxzPD9sXRAVXx/0vszU6xTW7+7bjOcXljdv2abeYm+b+xDX51uvvSDI31StShXjiQUvtTv79EZRIyJfd/NdZhkfCJ9WVAVtaw/d39csor3+ek9B3EQ4vvfeesUIRzhf8J6D/f7n38ZTDSEm7XME5wh26NAReeDeOwUCC8KRwfZqGLdZs+fKG28PNuvJfRQvVszd/cbLzxlRCRtwvSIv2CvqHecN+2YL166ZIP0ffcC9nk6eOKnn8iL3eW3LVfPk9UH/Br/zmgnxB+9LCOjIl/XuBx8L7gWvIbQgvCzxTMBzH/sh/NfQ+nBc8Njg7QnD+bWhFXH9HEk8YkK2faQhJr0h9LxtcZkESIAE4o1A9K8FxdsI2B8SIAESIAESCENgxuw5gsSZ1lo1b6r/qappV/lNAiRAAiRAAiQQ5wQOatgohFv78psfzAQqclN4DbkSIKLYiTiEQsNkHmzNuvWuoIN1b+4Um4fFhhLCxLT3RRBMomMC1Vpr32T86rXr3BwwaPOzYV/bIsbLBCvwDLny+ttMqCkINjDkIYHhDXuEqMLEOgQdGCYk4YnwyQdvme1mYwQf6DMmJGGY/Kyqbbz9+ovSrm1rZ2MMn+U1XJa13Xv22MWA7xMq2sBsiC0s9+zWRVo2a4JFk+A81ASv2en7QBit1954z+0/PCAcj6vcZqL7NV84L+8xWC5U0PEsQf4cWC31eoHZnBlmRT9KKV/YiZNOX82K52OPehtYK182fP4XWyYtvletWWsEStTVvWsnt0pM/CMPDAy5eayN/nucwBvBCjrYPmr0GCNgYdmGycNyNHZG21buJDvCC1qRBnUgnF4oZv/MnicTp0wPKAvPqC1bt5mmEZowNYa8PrgnYL//8bcr6GD908+/NCHFsIxQicF28NAheevdD93QbBjPfF+eGOthE3xMSutz9fiHHn9Gfh75u2DsW7dtDzgEzyEr6GDHj//71d0PgckKq3hGjBk30ezD/QqhC4Y+w5CTyQovWMd9A+8SGPIowesJ9uuo0e7zCPfOH6PHmu24H6wh59WlV99sxB+7rYR6wsHKlS0tvXp2NUIX+oEf1I379s1Xn4/KYyWXHmyfO/Bkwn0/7JP3jCBl27XfTZs0dAUgbIN4Cy+nEcM/Vo+1BGwy5o2mgJCNeBbbcJoQsSHSvPLCgCQiHe4RiErW8BxFaEbYJ58ND9hny9hvjAOGLzyD4fn04Tuv2938JgESIIG4J0BPnbg/RewgCZAACZBAagggt06Chkqx1v6MNrJy9Rq7ym8SIAESIAESIIE4JoDE9dHaUX173HqzeI/1Cg22XkwgX3/1FWZiD2GTNmouhm+//5++tT3ePRST7ra+Ivr2/3PPPOHuwwK6iInB0qX8SbgDCuhKnjz+9ylnz12gItX3slTDccFbBSGU4GEEzwKEKfLm8QmuB+t97n7QiBh2sh+hteCZglBTEI0ef/heueK6PqEOTXEbvKKsFS9WXDaJM8Fst+E7Tx5nGsFOTGNC//577zBF4MkxZtwkb/GQyxdoeKR77rzVcENIpqXLVpoxIPxUCfVW+HLoYOP14RXxIGKAsxXKUHEjncSF7djp97zBus035A3vhe3WbHgqrHvHbPen1zfy4NxwzRWCXDPWs6trlw5u+Lmhw78NaLqChvFCCEJ4je3VcwNREaJL/txOqKqAwhGuYOIbBo8YL9+UDsd1du7Z3QUT8BD20tL7vUH9Om7zf42d4C7bhQWLlpgwb9bLxG4P9x0swqDcHX1ukgb1HPHMe9y27dvdnEB2OzxKorFt2/x5gYKP26XilzX73IEHH8QHiAnwdpq/cLF88dWIAE+UBvX8TJAXzPvcQeg5GO4HeBXaZ4Ftx37nRgE1nGsIIN//9KtsV48+eMRde9WlRlBCP+C5OG7CZHtYku+9+/YZTziEXrMCOu77u27vbTwVIcAM0Lw/j/b/V8CxeNZ99e2PGi7xoJRUgQl5iBB+ECLui8/+n5vnyT5TcDDCZ0PERy6wShUrGC+tHppvB8+2hx+4R2645W7TBjyW3vnPvwW5yiDkI4faWZ06SLMmjQwXhMJr2aKZPPeSX6j5UT06R/893uRmsx2FuPT4I/1MLlaIbvB+Cs4VZsvymwRIgATiiYDz11g89Yh9IQESIAESIIE0JPDPnLnSrk0rqVmjmqm1uf6hP6NuHQ2psDINW2FVJEACJEACJEACWZEAJimfePo5eU6TjOOtc3i7PKyh0+BN8+yLr5s3veFFYA25MvATyjA5HM5sKDOEA3v62X+7xWbNmS/4+enboUY4aqA5ZVIyTLB6DeGcEIKpfLkyxtsFE6Z4+9+bb8ZbPrnlDZ58hOW0vqBoR+ZQ+wa9HdPLLzxtRAmEXfvgo8+knM/bJ796IVjDBDC2Q5jABPRtyhfzzZi4tSLW1yN+1BBWPaXvnbcYFo89eK/cce/DxpvggvPOFhs6qWCBgsa7AHVbb5XKlSqZbQi9CxaYuIZZbyiz4vko5/HOCQ6t5SmW5osQDK+76nIzQX2VhqH6WL0JLjyvl2kHYbt2aLgya3fe1lvzrpxrONltafFdvZrzstOxY8cirq6FemM8N+AJV3yK+MAIC8IrxZr1/rHr+LbCCCb2IS7hmo/W8P8BCADBVrtWjeBN6b6OUGkQiG+9yQl111zzCTVvOlBWqXcgng8I41iliuPdh840U4+XUAZBOZygg/LwWoJIul3DDSJUmTWIiwhV9sGgV8wm5AFLTtRBoeDnCZ5DCLtWt3Ytw7VWQlKO8KgbpmKVNXg2Pv5wPyPQQcCtU7um8WratGmzLSLvDf7U9dTC8QihVlPrhhgE7yIrYl128fmmXTxTb+/7kOnfqD/HCJ41L/zrSUF/zlSvNAjm8OCEQdwKfn7Cg67fQ/3l+68+Ndf3mWe0pajjng0ukAAJxDMB/19Z8dxL9o0ESIAESIAEUkFg2sx/XFEH1cBbh6JOKoDyUBLIIALz5s6V5i1aZFBrbCYjCCTGMBGXEf1iGzmbACb8rry+j4Yn6iaXXXK+EXYw4YgE5Zde3dvkqrGEvv9ppEyYPM2uBnyvXbchYN27skk9gGCYlEbeDK/XELavWr3W5OVBu7Ha/IVLjKiD45HjB+GyojVMolrDZKjN72K3QeDCGGDWG8KGrYPY89G7/rfi7TH4fvh+5+36l3USGLl/bAL6YM8UTDZjovfcs7vphG1Fk/8D3jg2pBLqQvvedWxDfdgGryqIOpt8Iaywr2H9eknCMDXTSXRr69aHP2+2TFp9YwIek+lNGzcwHjiffv6V2FCAP/7vN7cZ9O/SC8816/s0JNckPZe4huCx06tnd/ccuAdEsYAJ/rrKOH/+/BEf9eiD97jCHYQzjAFeRAjfVUxz2KTWkDPFGkIAekPCYbsNpwcRIxZBB3Xg2qodIgzzPs0tkxmGsJG/aZ6iq6+8xHhA4d6HcIEcPMhDhPvEGjxgIJoGmw2FGLzdrkOIgfdJyZJOGDe7Hd+47pF3DCHq7D3s3R/p8loNcwmxLNJQdxhzV18YPTxjEKpunSdcdt06tVxRx/YB1z/YwHC/4PqzofgQZtMrOEG0eejxAfLD1/81ou4VKp5aUcfWF/yN+xJ5iEqpN1GFoFxhwWW5TgIkQALxQoCiTrycCfaDBEiABEgg3QjMmjvfeOvYN8ia6ORAI30T1puAN90aZ8UkQAIxETh42HkLF8IOLXsQgKBjz2v2GBFHkZ0IIKQQ3p7HD5KrP6+eOxAPunfrLAiVhAlViBYVNATYipWrox76PF+ODxzY+/qr5d3B/rfmsa2GL1Ts4cNODh5si9aQjN3aVs+EsN0WyTfydUBEwNv9Hc5smyQp/RWXXuBWg9whMLCx+Sncnb4FJCSHnVS+CD11JDExwOsg1IQ6QlFB1IEnD0KUfa4hyf74a6w8pB5UmHwepflEsA578V/9TQirN9/9SCBIwRMINn7iFLmrT29TB4S6Ja8uN9vtR7cuncwi+mPFKbsvvb8RZguh/pBDBvlDcJ2BIa49a107d7SLGtJqoAo6W9119D3SCXT3IM/CgsVLpGP7tkakwQR6Stcz7geb7waCGUJjWUOIMAiIqbVly/0e9J06tHPzx9h68Xc7DAntY7UfNPRWvBlCCcJ7BT8P3HunyXuD0HwIM+cNjVdWPcuCBdZIxrJ0+Qr1AGpk7hGvxwqOxTm111Fq7gHryQgvmEjMmz9n/4ED5hDkxEIINYS5RG6t4HPVsnlTt+qVKoDDbC6iRL2Hgw0iDUIwwrPHevMFl/GuQwwu6hMnbVhJ734ukwAJkEA8EqCoE49nhX0iARIgARJIcwJTNbeOFXVQefu2bSjqpDllVkgCaUsAAgBFgLRlytpIgAQCCcDbAIIjwh1Z27vXv1zUl7ti5Sp40tSRjmeeIW3btAxIko5wQEh0P3/BYlNF08YN5bGH7jXiyMNPDDDhkZB/BEILQrf1UsFiyrSZMltfOoHocdklF5hJVxy8cpVfMLri0gs1n8RFmqh9rvxn0Aem7ratW0itmgnmWDsZj3BU5/Q4Sy7RvDqwRM0pZPeZDVF+YEL11puuNZO+mGge9N5HpgZM7p97dg+zvG6DZtvxJY+/5KreIVsAl581rBzs2RdfMwnnsbxz5y4TdgrLLwzsLwOff0UQBg+G84HQdzAIQTZpPN6+t+cCk9tWBEAoNtjfmofF5vrAOs4nJrQhOOCcIWeInRTv0/t641mAcn+oYJfRhvwwBzQsFjxckM8JhhB83v7nzeuIYdiXK5c/H1PP7meF9LA5ceI4iobcZ3Z4Pqarp83dGtoNhjwo9zz4hPFSwDryO+X15UzCOswr2OXJ7e8XBCErRjol/Z/WQwQT7zZcln9v0iUIcpbJZRdfYPIybfSF5Lro/HPcifkZ/8xJenAW3IIQePDE8XodHdS8M9bADJ5/1pPm/ntuNx4t9n5AOeSFQs4ZK/6Eel5AgMMzBGEIn9Fz3ff+x8y9hmMfUe8ra/C+stb/sQdMXhrk/frltz/N5ksuPE+vz5MC/taDCOcWdSfUqGbKwNvQGsaHXFGzZs9TL7kV5trGmJAfqs/Nzv0Nr6uJk/3ehOMmTDHPMXgWIT+YDduG522jho6oh2vEMkMeI7SNfEPe+xsCzeX6TIWgA1u8xAm9huXbbr5ec6uu1fttnusNBs/Au2+/2Yj2KDMxjCcm9tFIgARIIJ4IUNSJp7PBvpAACZAACaQbgTnzFki71q3EvsWK/+Q3bdRQ8LYijQRIgARIgARIIGcSeOKR+zV5em2Ta2HPnn1yUpPQly/n5MzBW/Q//fK7AfPqG+/Ip4MHGc+PZ596zExAI4RaqVIlXEHm+lv6mnwNt2ieDEx44gceJ3Zi9MOPPzeT6JhgfWHgk4L6IcjYvC/w1kB4Mms3XHOFeZO+R9fO8tGnw8zEe1f10uimk6U333C1er44nguFChYy/bLHvfXuh3Yxpm+EmLtEQ3/hTf5ePbtKD/VWOnnylBTIn8/UB7Hl3Q8+jqluHASBa/XadeZlG4RU+uLT981b9RAykDjeGnLseA1h62CYJIahf/BygYjlFUTMTv14+/2P5e3/vGgma5985D4TAg4TvtZ7CBPEw7781hbP0G8krcfEsy+PvQz94puA9jGxjNBmsA/eflUwgY0QWl4+3gMgEsITA+fol++/UO+ysQE5VLxlMSn/o17Xl6kIWKZ0Kfl66EfGA8Z7LXrLL1i0VPmeNqyvveoy6dG9ixFq7KS5t6xdHvn7n8ZDBNf2T98MVU+k40YcCicA4jhcUxAUMIYP33nd3Et51TvOhneDJ8eg94bYJrL0N0KsIWQgrt3du/dIoUIFTegvDAqRBGzeF+Rc6nd3H3Peh7z3htl+VHMh4dqHVwtEn8uvu9WwCPW8gMfK73/8rXmbkJOqiAz75H2BJ4pligMhzv46arSpA3mvOqunFOwm9Si0zy5cq+XKlpa+d9xixFYIK95rEc+ul14bZI7DxxkqxFyrAiF+wj2nvvz2+4B8QB9/9oURgjAueLBB7IbZUI1Yfv2t9/Bl7MNPPpeXnn3K3EO4vyGk71FxuHSpUuZaRSHw9YZ4vPySC917DtcT+mafayi/TXNaffVt4HMH22kkQAIkEI8E/K98xGPv2CcSIAESIAESSEMC09Rbx2vIrUMjARIgARIgARKIPwJ4Kzw5S2k/jj0ZQTigOeotg8l9TPRj0hKheoxQoJOlL7/+tisWYCL8wceflj0+jx5MiuKNcjuxCc+Eo8eOmi4vWOR47GDCcJ7Pewc7puvfIf838EUz0Yh1HGsFHUwm3v/IU67HBPbbvDAIEYZ8D7C58xcaMQjLEARQhxUG0LeB6hFjPVJQJpSlxAUCya13PWDeaMfxYGMnPiFEPfDoUyanRai6w207fjwwHwhyXsyYNcdMquIY5BOxLI8eOy6ffzlChn/9vVsd3vJHPzB5bN/Ur6+hv2B79zpePm5h3wJyhvS9/3HjMYVNmCy2gs4G9QK55c77BOHmkrPToicxCoP4EYlh4hjXBwzXTnBen5mz5sroMRPMflwjyFkCPnP0/ONagXnb+l0TxNsQbSgPXqaM7x5A6DuvDVGRcKx6RoCnvY5yq0cQPIasB9aJE849CN6D3h9iJvNRtryGAoOgA8+zf9QTAxZc/+SpM0xfbf1gb1lDILTmvRaRA+rFV9/S+wgCkObR0fwmVnzYvGWb3N73Iffc4/hIngFBw7bNput3Snlu0PhC9dYyXjgq7ODcYqyw7erF9vzLb5hlfCAP0FvqKWc5ohy8/cATY1ui3mjWQj0vsO/9j/5rhA1cL+BqmWIfrqd+D/XHorFd5jl23Cx7r0l4tqAPMFxf9l7FOjxfcJ/t8F2X2LZ85Sr3WWmvL3zD8Ax554NPAu5vbEcepVvuuN8wwDrEHCvoQJyB4I37whpCWr702lvu/Y1+QUjH8xu2as06ue/hJwPyl0FMttcEGNrnGraNVw/Au+97zH3m23b4TQIkQALxSiBXg1ZdA3+7p0NPD+7fkw61skoSIAESIAESiJ7AXbf2Nslh7ZFffPOdzPXEubfbs9N3ocKxJ13OThw4FhIgARLIKAJHDvvD6MTaZtHipWI9NOxx/H9ZWDRmErxNq+bGS2f1mrVGtAjl/YEa8JZ8Ew2xhglEiD0IBeZN1I0yCD+EfDHesG7Ybg1hxlq3bK4CT6IJ2xZ8vC0H4WKVJhIPzlcB8al2rQQjQu3QiWB4r3gnVe3xqf2GOIC37gtrjo9ZGgbOhklLbb32eHgo1KtTW5Bn45DmE1qtE8RIfB6OvT0u2m94+ZzRuqXJ2zF95uwAcSDaujKyPK61Zk0byyH1rpg7f1GAZ0OofiCsFkQFK6CEKhO8DWHUCqrH2CJNPp8cd5yrZk0aG+8xhA60QmNwfcHrOLcbN21Jse/e43D/NNYcmBAAIGSGu4+8x2TFZZyv9u3aqBCRW5YsXZFE3POOyTLJp55Mm1SURHnw8Vq454Utg1Bj8OjasnW7EZaQyybYcM/jnNmwbt79OL56tSoqQpU0fV2qzz6IMeEM1y/6hLw7u9QjCXV6Q8iFOw45hdpoqEmIvQsXLzXefeHKYnvVKpUFfUN7GzZukmXaDjwpQxnGh1xQOCZ//vwaqnKVrNBnLLyaaCRAAlmTQGr+Zo6nv42jHQdFnax5vbLXJEACJEACMRJo3qSR3HTtVe7Rq3Ti5oNPhrrr2XGBok52PKscEwmQQDwToKgTz2eHfSMBEiABEiABEiABEsguBKIVQ7zjzsqiDsOvec8kl0mABEiABLI9gXkLF2tIgNXuOGvXTJDWLZq561wgARIgARIgARIgARIgARIgARIgARIgARIggXglQFEnXs8M+0UCJEACJJBuBKbO/Ceg7vZtmVsnAAhXSIAESIAESIAESIAESIAESIAESIAESIAE4pIARZ24PC3sFAmQAAmQQHoSWKDJSRED2lpCjWrStlVLu8pvEiABEiABEiABEiABEiABEiABEiABEiABEohLAhR14vK0sFMkQAIkQALpTWDazFkBTbQ/g946AUC4QgIkQAIkQAIkQAIkQAIkQAIkQAIkQAIkEHcE8sZdj9ghEiABEiABEsgAAguXLJUly1ZIw/p1TWvVq1aWDu3aypTpMzOgdTZBAiRAAiRAAjmbQGqS2uZschw9CZAACZAACZAACZBATidAT52cfgVw/CRAAiSQgwlMDfLW6dz+TClcuFAOJsKhkwAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJxDMBijrxfHbYNxIgARIggXQlsHjpMpm3cLHbRrmypaWLCjs0EiABEiABEiABEiABEiABEiABEiABEiABEohHAhR14vGssE8kQAIkQAIZRmDC5Kly+rS/uc4dzpSK5cv7N3CJBEiABEiABEiABEiABEiABEiABEiABEiABOKEAEWdODkR7AYJkAAJkEDmEFi3YaNA2LFWoEB+gbBDIwESIAESIAESIAESIAESIAESIAESIAESIIF4I0BRJ97OCPtDAiRAAiSQ4QTGq6izZ+8+t912bVpK3dq13HUukAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkEA8EMgbD51gH0iABEiABEggMwnsP3BAIOxcesG5bje6qLfOilWr3XUukAAJkAAJkAAJpD+B8qWLSYliBdO/IbZAAiRAAiRAAiRAAiSQYQT2HUiU7bsPZFh72b0heupk9zPM8ZEACZAACUREYNLU6bJm7Xq3bMP6daV1i+buOhdIgARIgARIgATSnwAFnfRnzBZIgARIgARIgARIIKMJ8G+8tCVOUSdtebI2EiABEiCBLExg/BR/bh0Mo3OHdpInT54sPCJ2nQRIgARIgASyFgG8xUkjARIgARIgARIgARLIXgT4N17ank+GX0tbnqyNBEiABEggCxNYuHipzJ63QFo1b2pGUbVyJUEYtrETJ2fhUbHrJEACJEACJJB1CCAsB0NzZJ3zxZ6SAAmQAAmQAAmQAAlkPAF66mQ8c7ZIAiRAAiQQxwQmaG6dEydPuj3srKJOqZIl3XUukAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkEBmEaCok1nk2S4JkAAJkEBcEti4eYtMmDzN7VvxYkVNGDZ3AxdIgARIgARIgARIgARIgARIgARIgARIgARIIJMIMPxaJoFnsyRAAiRAAvFLAN46zRo3lLJlSptOIgTbvAWLZN2GjfHbafYs2xAoWrhAthlLRgzk4OGjGdEM2yABEiABEiABEiABEiABEiABEiCBuCBAUScuTgM7QQIkQAIkEE8EDh46JBOmTJPLLzrf7RbCsK375jt3nQskkB4EypYqLs1btEiPqrNtnfPmzpXEo0eF4k62PcUcGAmQAAmQAAmQAAmQAAmQAAmQgIcAw695YHCRBEiABEiABCyBKdNnysrVa+2qtGjaWJo0auCuc4EE0poAPHQo6ERPlcyiZ8YjSIAESIAESIAESIAESIAESIAEsi4Beupk3XPHnpMACZAACaQzAYRhq1MrwW0FYdgWLl7qrnOBBEggPggULFCAnjrxcSrYCxKImMDB/XsiLsuCJEACJEACJEACJEACJEACfgL01PGz4BIJkAAJkAAJBBBYvGy5/DNnnrutVkIN6dCurbvOBRIgARIgARIgARIgARIgARIgARIgARIgARLISAIUdTKSNtsiARIgARLIcgTGq7fOsWPH3X53bn+mFC5cyF3nAgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAlkFAGKOhlFmu2QAAmQAAlkSQJbtm4ThGGzVq5saemiwg6NBEiABEiABEiABEiABEiABEiABEiABEiABDKaAEWdjCbO9kiABEiABLIcgfFTpsr2HTvdfnfW3DoVy5d317lAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAhlBgKJORlBmGyRAAiRAAlmawJEjiYIwbNYKFMgvXTq2t6v8JgESIAESIAESIAESIAESIAESIAESIAESIIEMIUBRJ0MwsxESIAESIIGsTmD6P7Nl+cpV7jDOaN1CmjZu6K5zgQRIgARIgARIgARIgARIgARIgARIgARIgATSmwBFnfQmzPpJgARIgASyDYGxE6cEjKV7l06SL2/egG1cIQESIAESIAESIAESIAESIAESIAESIAESIIH0IkBRJ73Isl4SIAESIIFsR2DFqtUyYco0d1zVqlSWbirs0EiABEiABEiABEiABEiABEiABEiABEiABEggIwhQ1MkIymyDBEiABEgg2xAYM2GS7Ni5yx0PvHWqV63irnOBBEiABEiABEiABEiABEiABEiABEiABEiABNKLAEWd9CLLekmABEiABLIlgYMHDwmEHWt58+aht46FwW8SIAESIAESIAESIAESIAESIAESIAESIIF0JUBRJ13xsnISIAESIIHsSGDm7Lkyf+ESd2hNGzWQdq1buetcIAESIAESIAESIAESIAESIAESIAESIAESIIH0IEBRJz2osk4SIAESIIFsT2DMhIly/PgJd5zdzuokxYsVc9e5QAIkkHEEEo8ezbjG2BIJkAAJkAAJkAAJkAAJkAAJkAAJZCKBvJnYNpsmARIgARIggSxLYOPmLSYMW68eXc0YypYuJd27dJSffh2VZcfEjmcugYOHj8r06dOlYIECmduRLNY6BB2wo5EACZAACZAACZAACZAACZAACZBATiBAUScnnGWOkQRIgARIIF0IILdOw3p1pXq1Kqb+Tu3byZJlK2TZylXp0h4rzf4EIE5QoMj+55kjJAESIAESIAESIAESIAESIAESIIFYCTD8WqzkeBwJkAAJkECOJ3Dy5EnjreMF0V3DsNFIgARIgARIgARIgARIgARIgARIgARIgARIID0IUNRJD6qskwRIgARIIMcQWLhkqUz/Z7Y73to1E6Rrpw7uOhdIgARIgARIgARIgARIgARIgARIgARIgARIIK0IUNRJK5KshwRIgARIIMcSQBi2ffsPuOPv3qWTVChfzl3nAgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmkBQGKOmlBkXWQAAmQAAnkaAK7du8JCMNWuHAhgbBDIwESIAESIAESIAESIAESIAESIAESIAESIIG0JEBRJy1psi4SIAESIIEcS2DytBmydMVKd/ytWzSTFk2buOtcIAESIAESIAESIAESIAESIAESIAESIAESIIHUEqCok1qCPJ4ESIAESIAEfATGjJ8UwKJ7l45SoED+gG1cIQESIAESIAESIAESIAESIAESIAESIAESIIFYCVDUiZUcjyMBEiABEiCBIAKr166TsROnuFsrV6rIMGwuDS6QAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAmklkDe1FbA40mABEiABEiABPwExkyYJA3r1ZWKFcqZjcits2TZClm7foO/UAYvHTl8MINbZHMkQAIkQAIkQAIkQAIkQAIkQAIkQAIkQALpQYCeOulBlXWSAAmQAAnkWAJHjhwRCDvWcuXKRW8dC4PfJEACJEACJEACJEACJEACJEACJEACJEACqSJAUSdV+HgwCZAACZAACSQlMHvefJkzf6G7o1GDetL+jDbuOhdIgARIgARIgARIgARIgARIgARIgARIgARIIBYCFHViocZjSIAESIAESCAFAvDWSUw86pZCGLbSpUq661wgARIgARIgARIgARIgARIgARIgARIgARIggWgJUNSJlhjLkwAJkAAJkEAEBLZs3RYQhq1UyRLSq3u3CI5kERIgARIgARIgARIgARIgARIgARIgARIgARIITYCiTmgu3EoCJEACJEACqSYAb52Vq9e69bRu2UzObNPaXecCCZAACZAACZAACZAACZAACZAACZAACZAACURDgKJONLRYlgRIgARIgASiJPDnmHFy6tRp96hePbpKhXLl3HUukAAJkAAJkAAJkAAJkAAJkAAJkAAJkAAJkECkBCjqREqK5UiABEiABEggBgKr164TCDvWihUrKhB2aCRAAiRAAiRAAiRAAiRAAiRAAiRAAiRAAiQQLQGKOtESY3kSIAESIAESiJLAX+MmyPKVq92jmjVpJJ3bt3PXuUACJEACJEACJEACJEACJEACJEACJEACJEACkRCgqBMJJZYhARIgARIggVQS+EO9dU6cOOnW0qtHN6lauZK7zgUSIAESIAESIAESIAESIAESIAESIAESIAESSIkARZ2UCHE/CZAACZAACaQBgXXrNwiEHWsFCxbQMGzd7Cq/SYAESIAESIAESIAESIAESIAESIAESIAESCBFAhR1UkTEAiRAAiRAAiSQNgTGTpgkS5avcCtrWL+udO/SyV3nAgmQAAmQAAmQAAmQAAmQAAmQAAmQAAmQAAkkR4CiTnJ0uI8ESIAESIAE0pjAn3+Pk/9v545xEoiCMAA/Ra6hoMRYcwCDIcQz2xgtiCcwxpoA3gClgWzBC0OyVGCm+LZh5m2yTL4p/+z+rdf1qc3bOv2b69orCBAgQIAAAQIECBAgQIAAAQIECLQJCHXaZJwTIECAAIEzCMzmi9IEO7ur07n0GbYdhl8CBAgQIECAAAECBAgQIECAAIGjAkKdozxuEiBAgACB0wu8Tz/K59d3ffDgtifYqRoKAgQIECBAgAABAgQIECBAgACBNgGhTpuMcwIECBAgcEaBl9e3slr91n+YPD2W+8Fd7RUECBAgQIAAAQIECBAgQIAAAQIEDgWEOociegIECBAg8A8Ci+VPaYKd/et5PCrd7tX+kZoAAQIECBAgQIAAAQIECBAgQIBAFbh4GI42tVMQIECAAAECBAgQIECAAAECBAgQIECAAAECBAikFPCmTsq1GIoAAQIECBAgQIAAAQIECBAgQIAAAQIECBAgEAWEOtFDR4AAAQIECBAgQIAAAQIECBAgQIAAAQIECBBIKSDUSbkWQxEgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIEooBQJ3roCBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQIpBYQ6KddiKAIECBAgQIAAAQIECBAgQIAAAQIECBAgQIBAFBDqRA8dAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQCClgFAn5VoMRYAAAQIECBAgQIAAAQIECBAgQIAAAQIECBCIAkKd6KEjQIAAAQIECBAgQIAAAQIECBAgQIAAAQIECKQUEOqkXIuhCBAgQIAAAQIECBAgQIAAAQIECBAgQIAAAQJRQKgTPXQECBAgQIAAAQIECBAgQIAAAQIECBAgQIAAgZQCQp2UazEUAQIECBAgQIAAAQIECBAgQIAAAQIECBAgQCAKbAHd+m1a55tyMgAAAABJRU5ErkJggg=="
+ }
+ },
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "instance_type": "ml.t3.medium",
+ "kernelspec": {
+ "display_name": "conda_python3",
+ "language": "python",
+ "name": "conda_python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/sagemaker-pipelines/tabular/train-register-deploy-pipeline-model/train register and deploy a pipeline model.ipynb b/sagemaker-pipelines/tabular/train-register-deploy-pipeline-model/train register and deploy a pipeline model.ipynb
index 2fbf96183d..2b7f52f0ef 100644
--- a/sagemaker-pipelines/tabular/train-register-deploy-pipeline-model/train register and deploy a pipeline model.ipynb
+++ b/sagemaker-pipelines/tabular/train-register-deploy-pipeline-model/train register and deploy a pipeline model.ipynb
@@ -1130,7 +1130,7 @@
"data = data.drop(\"medianHouseValue\", axis=1)\n",
"\n",
"pred_count = 10\n",
- "payload = data.iloc[:pred_count].to_string(header=False, index=False).replace(\" \", \",\")\n",
+ "payload = data.iloc[:pred_count].to_csv(header=False, index=False)\n",
"p = predictor.predict(payload, initial_args={\"ContentType\": \"text/csv\"})\n",
"print(p.decode(\"utf-8\"))"
]
@@ -1231,4 +1231,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
-}
\ No newline at end of file
+}
diff --git a/sagemaker-python-sdk/paddlepaddle_sentiment_analysis_byo_mms/Bring Your Own DL Framework to Amazon Sagemaker with Model Server for Apache MXNet's (MMS) BYO container.ipynb b/sagemaker-python-sdk/paddlepaddle_sentiment_analysis_byo_mms/Bring Your Own DL Framework to Amazon Sagemaker with Model Server for Apache MXNet's (MMS) BYO container.ipynb
deleted file mode 100644
index ab9f3d330b..0000000000
--- a/sagemaker-python-sdk/paddlepaddle_sentiment_analysis_byo_mms/Bring Your Own DL Framework to Amazon Sagemaker with Model Server for Apache MXNet's (MMS) BYO container.ipynb
+++ /dev/null
@@ -1,539 +0,0 @@
-{
- "cells": [
- {
- "attachments": {
- "image.png": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAscAAAFwCAYAAABU0dZ+AAAgAElEQVR4AexdCXgURfb/Vc9MLpJAwhVABQEREMgKKioSDjkEV2CV6Crogq6g/gXEAwRN0klADg9cUBRcBFfBA1DAAwXl9ACVIFEBkUNEbgiQhFwz0/X/Xk9X6ExmkslBMslUfd9MVVe9evXqV9XVr1/XAUgnEZAISAQkAhIBiYBEQCIgEZAI6Agw+k9MTExSmKIKTDSuqSkpKcniWqZLfGT/kPeHGA/k+CDHR/l8kM9HMR5I/UDqB7VZPxD9XPoSAYmAREAiIBGQCEgEJAISgcBGgN78AhsBWXuJgERAIiARkAhIBCQCEgGJAKAQCGaTuARFIiARkAhIBCQCEgGJgERAIhCICOhThQKx4rLOEgGJgERAIiARkAhIBCQCEgF3BMhgrFuO3RPktURAIiARkAhIBCQCEgGJgEQgEBHQlWNafR6IlZd1lghIBGotAvpOPLW2drJiVYmANCJVJdrlL4vaqbrve9lXyt9+MqdEoFIQoEGA05RxN259APzkFleWSyuANqYMBwC0MF3X5OBGH+pC2DX0UMkoABEe4kuLegTAU6URlSG9Iu3bDMAZoyxzO4u+ROnV5YQMSzwIQHJRXx9hpFEb3emBzhxF9NFGfSksfh8DoLYUzhdeREt5BA/3e07wqgk+YUn3dFU60X43VWWhlVAW3beizcl/oRJ4XiwW1dWuFR1P6H40Y0zheYaSS+MtXV/jBpq4F0V/onQzD5KJ8PDkyjLOCNlorHR3dA/R80S4mtRXhMzS94KA3KTCCzA1JFrc5O4ParpJxYBVnqqIB5nIS/zKoxSK/P7kEy6lKfreaOjrSnkejjRwi0G8MrAgZbu87RtsUtTN7Sz6krcHSmXIXRoPIQM95EhOs6OHJcVTXyRHDyYRNqKKeIQ5YSR40jW1e0cApHwTLwqTK42XQaaXRw9DTw9KQVMTfMKB6l+VTmBe1Up5RepIL2KEExkKqM2FAiZe0CrC+2LkrY52rYzxhPoE9Q8aewjr2wzcxVgr7n3zc47ubWEAImMBtRONiyQPPatEHk9GDjEmUJ7Sxhkqg+hIJrMjOSle9Oea1lfMdZFhiUCtQ0Dc5OSbHd2oZuWJBg26kelHg5Cgp7dvurnFTU7KAg0mgpZ40IBEAw3RUhopB6SUEA2lmy3MpDyKeCqHfp4cWfxEGVS+J8WbaCg/lUe0JAMpmCKfWTEiGUQ9aDAzD4hCSRI8yBfKMflUB4ojC4FQeihO0Aj5xYAtaAlDKoswEzypXCEH8aBBmxzlpfqIPN7q4U0eGsAFDuRTGfRzd4SXGMSpLiSDwIIePJSH2pHoKJ7kph/RWUw40DXFmzE2l+WtnhTvqX9Q+xJPgS/xor7irmQQPkImc5qIJx5CJqqLCJtlE2FKJ5xFXvLNjvJSWeRK40U0os+SDAI/6jMinu4TESa+xJPqTeVSPxEPcEqjNjLfK+a+Zr5XKUzOl3YlOsJXWLpITpKHHLWLWdZLTHWndFJCSF5PziwPySFwJN50T4p+SXU0Ky9mXhRP9RbKJfVBcmXBhmipfOJDPyqbnBjrSH6SgXzxgldR2aluhJvZUR1Ev/N2HxA9jQvUBiQrtTXx8SSXaOOyYOGtXOJP5QlnbldvYwu1I2FI2BJ+ZudLv6uM8YTaTOAgyifMhDyEDdVLtIW4z2hcFGli3BP5yac+Qm3g7kQe4lnaOCP6t/v9QbJQfhFfWl9xl0Fe1xQEpAm5prRUETnFTU4PXhq06UeDMt2w4qalm59uYho8hCJHNzI58aCnAY4GUqIjJUkoDsLaSXSkgIrBlwYtCpsHH/EgEnKYBw6jON0TPEhmMcjTIOzuBD8aCM3KJMlhHhxJAaGyiAc9AMRgRooC1YvSqG5UFslN10QXZoSpruLBIQZjqi/FmR3xogGR+JMMAnviRzjQQC3CRCswJMWAwjTYm/NQW1CbUB6SrSR5iIZkIuxEewpZzTJSOSJetLUY/MUDmupF/EgWIaNZiaQ0d9nMZVCYaKjO7vUUbUsyuPcPyiNeFgQO7hiLeIGLKJfyUX+mPkH1IEfXJL8nJ7CkNMHTXXETslJ8SbwEf6ortb1of5Gf6kUYi2v3fi3KJzqqr7jPiA+1O/nUtuREv6Z7SPR5wsKXdqX8xIfKof4p7h/iI2QzyyrKpHakeE8vqKKveRo7BH/yRZ+isCdnbk/qG4QBubJgI+SksggfUU+SmzCnPku8RV0qQ3bRBsST6kY4mh3Fe7oPxJhEfZXaWLQLye6tjcuChbdyST5P7SruB+pHJAPRiHFCtCNdU78xO1/6nWh7kp/oSbayjidUNt2D1Ib0E21HYeFEOUJe6lPkRN82v3gbSYVp7ve+wNrcLymPt3FG9D0aA8hRWVRPyi/wLq2vGFmlV+MQUJNUamzpahYC4iantnP/icGP4s0PLTHI0IBBNzY9nMWDUQyONKCb+wPRUT4xEInBRtAJXjTwC0dhMXCIOPJJiaRBiBzJTwOPJzoaaM1KM9GIAZHyknwkkxgsXRxd/5RG5XtLM+cj2ennjgtduzuSiX7kBPZmbM3yicGSeFMes3JsHvTd6+EuT12jrvSAE44wo4eJuxPtQfFUHvE29wMqV7Qh0ZjpRX08yeZejrd6Ct5UB3KCP10TTkJm6mckm7sTMogXDcpPjtqe+gw9NH1RjglvdwVMyGSwLMSB4n1Rjikf1UG0v6ireGB669eiTgJXKo/qLuQRfIg/xVMZlId+VFeSTeBINN7aVdCIe5loRT8RZQhZ6ZowFYo6ye7JCXlEmvkeIRyEJY/SRR8XtGaf+FDbkRP3hagjpfmCjei7xEMoe+Z7VCgsYmypLNkJKxqHiJ/4CXm93QfUhuYxzSyvkEvUX7QxXVOa4F1SP/FWrrd2JXmIN/Gkn7kd3WV1tZLrX/QpuvLW70TfIhozvXt9KJ1kMLeZq5QLLzWUbv6JthR0or+KMc1cprifBC35Qjb3NCGbr+MM9V0qW4wphL/7fSnK89ZXzHLJcA1AQO5zXAMayQcRydorBlvy/w9AcyOOsn9p4vGHEW4CoAOATgAyjUFpsJEWZKJ3D54FoLnRUZn1AHxjIl5vxJmi9GA+gFuM8ojPKAAH3YmM6z1u8T8b11SecHUAvCguDJ/qSPWntAluaSS/2TkB0E8obvXNiR7CxNPsVpou6IEoBvfNpnj3YJYpojR5Whm0eaY8e71ge9KgoYfCv4y2jTWUCUr61cSDgp7auSTZRPaS6umpf1C+RcaDkR5U9KB1bxfBm3zqI/MBTDGsNNS3vjIevmY6b+GZAJK9JRrx7u1YCrmeTHnc8wklqLR+LXCl/uHJiQf4q8b9RfcGnVp6DoAv7SraMtfE/A23fiJkJRLC9AmDtoEpjwiKe8zb2EF02wRxCT4pTOQWGveGuC+6GPHklYYN0dA9/anB47wprwjajT5D8laW7PSicRjAPQbPSACbAKQb197uA+ojzwnBAIg2KamNBbkvWHgrl3iU1K6exjqS9T+icDffl35nziL6oDlO1Ifi3Mc6QUcGABoPqN3Ej55ha41rQSfGjCEiwuTbTGERdL9XRbzwfR1n6OVmqul+WQXgXsHE8EvrK27k8tLfEZD7HPt7C1VMPvEgbmRiIyyQxwEUGJ/GaWAZBICUCvFm720go8HX3Yly2pkSenkZDMnCRA9lGrhoIBSKvClrmYMPuOWgOogHKFlGzE7IT+XTg45koJcL+nU3KSLmPOaw4CvijhoBUkjpwX+twdPTYC3yePK9yXPIIBYPVrosadAnpfJ5ox3pZYLacRaAHaaXGnP53trZTGMOl1ZPga85D4VzDAXnIQB3A3jdncB0TW3yrPHiRMoitZPoYyYyj0FheTM/lInQPf8HhjziRc8jszJEVrRfCzn6Gv2H2puUMfE1pqztSqLTPViSoz7/LoC/PBAJvLyNHR6yeIyiFxxSbGnXEJrrTP4nAP7rkdp7JL3ArzDuU+of5n5LFmy6Hm1kryzZqUyydAtHfUooZqXdB8NEJgChRri0NjZl8RosrVzK6N6u3sYWofx6Lcx44biY44ko231cpX5JjtpaONGu4pp8UYex5kgjTPe4t3GPSHwdZwg/8WJJL/bkyBBkfhkoqa+Y62Bkl15NQEB/6Mp9jmtCU5VLRrKqktJGFhz6jCSshw4AJwwrHoU3GNzphifFjpQcYfXxpWB6eNObPn26pQH8fbcHmOBBAw0NWGRNIf6k/JgfdIKuJN882Cw1ZKVPXRQvlOG3AVAaKcriM6RII970+SvOmL9ID62hBk5m3p5koDyePkMLZZis4HRPfWRkFvF0WRJvb/KcMvgQP+JL9SGLojdr+zRD+SSFkhxZhcg6/6RxbZbH3M4lyWZk1T2Rv6R6munN4XijvUlZcldezXQkCz30qF+Q7A+aE03h1saUIOpH4kftTi857o6+JFDfJPxoKhFZ1PubiDzxMiWXGvTWr33FlQqge5XuIXqBpXxkoRRfRUprV7JwkqN+QlMMqJ70svuMEW/2qA0JW+r39xkJpNy7u5LGDndaT9dUB2o/andSLkhG8scY+BNmvjiBIVmqSWa6n2l8Ios3vTz8HUBXAzey4FH9Kyo7yUVjGo2d9MJF9x61Cyno5KgMcp7uAxp3qH/RtACi+9agJc9bG4s6mkg9Bku6/7y1q7exxZcyS+t3Qh4SlsLiueELb3MFB5juZeq79OWRnHihMC49esKwQ32B2okwpykQ1Aa3eszhiiQZfRlnBAsy5NDYS4o7KermupfUV3ypgyhD+hIBiUAlIEA3N92kNCCYHSmBYpoAxYuVtURLDydhPRYLWyiefjRHkRzxJTrBm8I0YJHSS2Hh6FrQUJx5vpV5oZGgJ58UFFGeKJN8Md9Z0LrPYaT6kAzCUR5xTYOimafZ2iMWd5jTRT73NGF1FPUVZQmf0okPpbtjT9dUZ1GOwJwsDaIuIo/5pcNcljd5iF7wFb553p2Qj3xRhrA4ivm9QqEnXqINiZbCxJOsTeR7k02UUVI9S+sfQjb3uYRm3iSD6M+iXUW6wJGuzVgLTIRP5QgnyhRp5BN2og8QnTdeQg7By1y+e1299esr3XAV8gjeZj6UZpaF+rxQwkQ+b+1KMnqSgeLNZdC1uQ/QNbUH4WKer0zx5EQ/pnTqK2LsMGNBdDTmCKuaK6dLqaR8JLu7o3hSNsgXfU7U0RM2oi8QPeFCbSjC5Jt/Qo6KyC7kNY9pojzCyb2tRFmibIEp5RFtSvV0zyfaWNS9NCzc85vLLaldvY0t7u0o6i18IZe3fmcuk2jLOp5QOQIfcxtS+4q+5i4L9Wd3Ry9M5vwUdn+miDyiTqKfib4l0s2YUPvQNTmSh/iKscP9vvLWV4zs0qtJCMhNKmpSa1VMVhoQxGDgzslbvDtdSdc0OJGyLRwNKDSweHPmMs1hb/S+xJfEp7xpvpRrpiGc6UdO+Malz543Wb3F+8y4EgnLU09SKujhcjHrIZTJSqxqmViZ62YOl4UJYVvevFQO5S1v3/MkZ0Xl8cSzPHHucviCkXueipTrCVOKE/HCp35u/kpFCqR7vydaX+T3Jq+ncr3RmuMrUqaZj7+G6f6vzjqKdhV9wV9xknJJBCQCVYSAsJSQJUNYAry9uVeRSLIYP0JA9Amx4tuPRJOiSAQqFQFhYSQrKn2NI8VYfJWr1IIkM4mAROAiIyBNyBcZ4MBgTxYTUpLJiuz+SSwwEJC19IYAvSiZvyx4o5PxEoHagABZL6m/0yd78Rm+NtRL1kEiEFgIyH2OA6u9ZW0lAhIBiYBEQCIgEZAISASKIyD3OS6OiYyRCEgEJAISAYmAREAiIBEIUATkPscB2vCy2hIBiYBEQCIgEZAISAQkAp4R0Fd2yn2OPYMjYyUCEgGJgERAIiARkAhIBAILAdrfFBs3bqTV5NJJBCQCEgGJgERAIiARkAhIBAIWgbgedN6XdBIBiYBEQCIgEZAISAQkAhIBiYBEQCIgEZAISAQkAhIBiYBEQCIgESiGgNznuBgkMkIiIBGQCEgEJAISAYmARCAAEdAX5NG2FQFYd1lliYBEQCIgEZAISAQkAhIBiUAhAnKf40IoZEAiIBGQCEgEJAISAYmARCDQEZD7HAd6D5D1lwhIBCQCEgGJgERAIiARKIKA3Oe4CBzyQiIgEZAISAQkAhIBiYBEIJARkPscB3Lry7pLBCQCEgGJgERAIiARkAgUIkD7HLPCKxmQCAQgAqNGjbI1bdC0o2bVrrQAbWyMtWGMXco4b8DBIpxAGAeCOGAFh4UrUDgvvG8UcPDWCjbu1dADrDC+KpHUANAXoEKfMXASgGnQwOBUGOyMI18BcgBkMuCExnAw38n3MM72wImd6jT1N8CVryqFl2VJBCQCEgGJgETA3xCQyrG/tYiU56IiEB8fb7mqzVXdFCsG2IBBdqB1BENONKA0VJQ64Yxb6oAhjAE2AMEMoM8rpH2STzcMhRm7cOtscWi43qrPULqosvvKnHOua8qkITuNnwNAAQcKAORw0pI5znLYT3HkZGic5QJBQUB6Ptc+dHC+JjU1dbuv5Uk6iYBEQCIgEZAI1CYE9Cc8bVuRkpKSXJsqJusiETAjoI5Sw9AYj9gsmFCH86DWFktYCwW2RqQEmxRdc55ACudyjmMacIDzvAMaL7ADR3M1baqiKItVVSWrtHQSAYmAREAiIBEICAR0c5fc5zgg2jpgK6kmqPdZm+BQKytTb7cqDf8VbK3bzcpszRQmFWOjV4QyhsstDL2tSsgDQZbIfhblyiaK8ooN2K0+q/YM2M4jKy4RkAhIBCQCAYUAGYytAVVjWdmAQmDy5MmNw222d8IZ69rHyiKaKBemQgQUEOWobEsLQ0uLJXKPU4tcz7VPpiaqb57KOjdx1qxZueVgJ7NIBCQCEgGJgESgRiAg9zmuEc0khSwPAgkJCVeH2YJ2drBaug+3ScW4PBhSnjYWBSODLHWaKnigQUTkNvUJtUF5ecl8EgGJgERAIiARqAkI6NMqNK7J46NrQmtJGX1CICEh4apgxfL1rTYluruFBZsXz/nEQBIVQSCIMQwJsoS1tihXIBwnJ02a1LAIgbyQCEgEJAISAYlALUJA7nNcixpTVgVQn1VvUSzKt0Nsiq25nEZRqV2ilYUpoYB2zGJ58MabblqxadOm05VagGQmEZAISAQkAhKBakZA7nNczQ0gi69cBBISErqHKJbPBtuUcFpsJ93FQeB3p6atcfCMXIe989SpUw9dnFIkV4mAREAiIBGQCFQPArrluHqKlqVKBCoPAXWC2jTIpmweaFPqXmaRinHlIVucU32FMc5gOwWlX1yjnm9u2LmBtlOWTiIgEZAISAQkArUCAX23CrnPca1oy0CuBLOFYvl1VhZ+bdsn6YSOQMbi4tWdc5z6/UWdf1eLYjuhaa0OtcM4AM9fvEIlZ4mARKCyEKATQRs0aBAdxIOinFZnFGOsnqIpdTRFsylcsWnM5dOJmxrT8unsIEVT8jVFy+ecZwLIcDqdGVlZWRlz5syhdOkkArUSAV2LUJNUriarUqOolU1c+yulJqj3R1vwn+E2JbzhlU/h1J4Xan+lq6GGDdo8WQRbOjhkUYGWk6s5b5In6lVDg8giJQJuCKjxahCuQBvNqjVPSUn51JysTlY7cBunky8zGGdnOONnGGe0biCPM25nnNnJp2vGmY0zHgwgiHFGvpUzHgUg2vRLS05OvqFIGU+qjXgwb8Ny2E51lpphTpNhiUBNQUDuc1xTWkrK6RWBMWPGBFsVPN/PqoTLXSm8wnRREujgkO5WFrLZaZkD4KaLUohkKhGQCJSKgDpGjUQUvgfQgjN+QOHKVwCKKMcIws5kNdlWKjMfCdQRakgxUivCYcXzPJK3T0pKIiX7LTVFnVCMTkZIBPwYAdrnWB4C4scNJEUrHYH69eo/eKnCbI3lArzSwboIFO0Vpmxx8k7qM+qN6lT124tQhGQpEZAIAIiPj7e0bdv2Rguz3Prrrl+fWbp0aeFc/9M4nV+P17vj+PHje+bPn0/W32Kuso+BVxepee6FqNPV/QB0azKtA3EGOxu70yQkJPSwwNLFCecXqampv7qny2uJgD8goCvHcp9jf2gKKUN5EAhR+AMdFSWiPHllnoojoDCGDgoL3WbV7gYgleOKQyo5SAQKEVDHq9EIx9/BMJAz3h/AfnB8alaMidiY/+tXiqY6Uz0CgH5FnKZpexWr8k+FK48mJSUFM86+AMcanMVn6hyV5jVLJxGodgTkPONqbwIpQHkRmDRpUv06QcF/PRSkhFiMRXju82LLy1vmK46AN2xPaxzvO7Tjz6hqTPFcMkYiIBEoCwJkIW7Xrt0tCpSRnPGbGWdfgeMz5OEz9Xn1WFl4+Tut+rTaWgvS+itQbgbQhzO+lnO+RPlT+dSTZdrf6yPlqx0I0JxjqRzXjrYMyFokJibecZmivHlHkCVSAOBNgRPp0i8/AiVh+1q+M6cgH1ep09U/yl+CzCkRCFwE1KfVljyYPwxgOOPsgAZtoXJKeVedq2YHAioTJ06sGxoUegcY7uGMdwGwQtO0+ampqd8FQv1lHf0LATnn2L/aQ0pTBgSCFeXmFgoLL0MWSXqRELhUYY59Nh4HQCrHFwljybZ2IpCQkNBFUZSnOee9AMxzOp09p0yZ8lvtrK33Ws2YMeMcgDfppz6lxmih2l2KovwvKSkpk3M+W9mkLFY3qA7vHGSKRKDyEFCIFZmQK4+l5CQRqBoEFI5ujRSm9+GSSyzAshWvImryMvyY60apZWDmfxMQ9cF2lNk84ziC0ZOXYUeJu30WYMk7r2LJyQK3gi9cfvftMl2279xlu0DiPeQ4jtGTE7DwWHH+2ce+Q9TsH8peL++leU1pwhBpVXhXrwQyQSIgESiCQEJCQkc1UV2hKMoKprHN53PON09OTn4mEBXjIsAAoOkjKSkp/0lOTm7DnGySAuVe3oMfUBPUR2ivZnd6eS0RqGwEdMWCtq2obMaSn0TgYiPgAFrX93FiUEE2TdXbgcX7im69efbwb5hG66szy2GQUCw4ikx4XBpuqnxWxjGcshcuLDelAHAcx6JPduiyfbDnZNE0n66cOKrTFX9HCIm4FNOvqY+q+DxUX2FUzjU+iSyJJAIBjID6hNpATVRfVxRljQZtPdvJWqmp6uwXXnjhfADD4q3qXJ2irlFT1D7Mwf7BFT6gSUyT3xITE2kBsHQSgYuCABmMiz9RL0pRkqlEoHIRoPlpDFDCfD0NL8hV/qItv+FsoSgaNm37rPDKFSjApu/XYtDkBERNTsCwT3/AgUK9ORerN63CTXram5j94+/IgNjq04Hvvl9rpL2Kyd/vR7F9jtxKostj+9PwQczNWHnPjVj0/q84ZdDknUzHsBU/YPWOtYiavAQ/5hfgq29XF/IfvSYdYmUObdWxYMtaDJvtknn8t66yHXnnsPtMLhznD2L07GX47rxmcHdg9Zo3MX4HKePlk9u9KvUYwMEuc4+X1xIBicAFBNQEdTjq4Bc6aCM3N7cNWUfVpWrxzz4XssiQgYA6Vf0xOTn5NuZkIxhjj6uJ6iba514CJBGobATIYCyV48pGVfKrEgRCLaGXhgI+P1TyM4HuPftj4P7P8OVpQ9vNPYSZ38fg1XtuBnJccTu3rcLgFZtw0z0jsWFUPPDNKnT+MF2fmrDj249wz+c/4N4RD+ObUdfgyxVfgPZOom98e3aswMAVe/DIAw9j7YjOeG3FQkz6paiVujgwBVi96VsM7NYJcW074ip8hY+N6REOZz4++34V7nn/ECbdcT0cO5Zh6Cff6mVvfbgnsjcsRbt1B3WWNOn61++P4h+DHsbae27Gok8W4pOTDjjyM7Hom0ygTjQuObYDi/bSYVgA8k/gtQ0HcG3TqHLK7WJj/icZCoAG5jgZlghIBC4gkJCQEMsV/pSTO/uqKepjM2fOzLqQKkO+IqBOVTclJydfB44ldevWbeFrPkknESgLArpyLPc5LgtkktYfENCsWuM6CvMyV8GzhBH1W2B0H+DBHYd1ggP7fsCvLbuje+M6cJlhz+PL5TswcMijmNChJWJbdMK8RwcBP/2IffkF2PLjLgy8YyxGt2mK9i064X8P07ajZB/Oxfr3d6D7kMG4vXkDtG3ZBStvicGiLftKnO/rOLsfj+8Hhl9RD7A2wmPtgce3uRReUYO3nhiBCV0uwa4tF8puc+lVePHheEwKd+ql01zpSaPuwtAWTXFNh65Ips1F7Xag8NU3AncMaYcPNv6hy3P26B5sRhx6N7SXS24hm9m3MgYL4FAfU+uZ42VYIiARcCFgsVvOHT169JrU1NSfJSYVRoCrqerrv/32294Kc5IMJAIeENCnI6akpNDzVDqJQE1CoF6wSf3zRfAsZzCuix0EvLgDh3o3wpolOzBixAA0wU5XdkeObgnu2qRwZziEx1yGgViFtJNHsekY0LXJhc0x6jW5At2xB/aCM9gEYPOKeWiywiRJe1PYQ/DXnWl67D3Tzethd2BHnyvQCmTJ7o+4+grgOIMvjwF9mtUt5BJzaSdMuJRmRRwBmZ/qBok1KhZExECPKyQG0L5tZ2DFV9id3wU5v36F7n9/GDH55ZPbzNcctgIORwhIOb4wc8VMIMMSgQBGQG5zWPmN734YijpZ7aw+p7oG1sovTnIMEATIYFwVa3UCBE5ZzapEQOFK3VCQsbIMzqEhpH5bPIxVeGlTKBahHTa0rAOcMqZZWENwCc0DzjWuiXX2GXyGGDxav5GucB475wCausp0nDuOzRQMigLtYdbo3vGY1S4SeQ4FednHkJ4VhguqtJucWgYWfbILI+74N8ZeFoZcJ2DVTiP5lcVYtL8PUqOI3uJaTKcEozmAv87TthiuydNnD23HzAOhmHyjUOQvGNE9zjWJbI7pMcfw4e5dyP0G+PeEGCA4v+xyu1XDfBnM4DhvcwljQBIAACAASURBVF7Q4M2JMiwRCDAExo8fHxocHBwyffp0elnkAVb96qmuFS+pk9XHpYJcPfDXllLJYFz44bW2VErWIzAQ0JgWatW/5JelvqRARmD4kHZY9PkmoNu1iLUCeWKdGiLQo08MXlv0Mb46eR7Z549j9geLAcTi8tBQ/K1nDF57+2NsOp2LvNyTeOmjpQBCYEMo2vWJwaK3t+LHTA1WZybeX/YaBm87ptt/PUl49s+dunI+IrY5Lm/YEO1jGqJN07YY3RNYtHYXaMPPQqdE4uZuwEsL1uO7zALknT+OOa99iNccoQi/oBMXkhcGCutFMaEY2PdGvPb+e1gUMxBx9ejWL7vchbw9BOhNxaJZQj0kySiJQEAhkJSQNCEyMvInRVFoxa5UjKuq9TW8BBu2JSYmDqiqImU5tRMB3XJM21bIqRW1s4Fra60UrgRbymo5NsBo0/ZaXIVdeOBqshPDNTnDOPg4rucwvJS5GENnTXelxVyLz56+HpQc0/tuTD87C4Nf3KWndW9/uYsGQFzcMEw/tRh9p3/rimsZhw3928AKh27rLboVsgObvv8C6HYfrnL7dtO50yBgwyp8c+4WIEa8uyq4uc9YJJ+ZjYHTf3Dxb98fW7s1Bywn0YRUfpuYVuGyLVN5VpsViLlgBb+0RXt0x7do1ftKfe4DMfIud2HVfA6QcuxUnHL1uM+IScLaiICaoD4EhvuRjW7Pvfic2ICmNlbV7+qkTlFXJSQk3KgoCvlDU1NTN1aXkGq8GoTWaKHZtGZwfW+kBcv1FCj0da1w5OeM6xuSMs7OaNBOK1w5DY5Tdm7/eerUqa4FMtVViQAuV28UNUnlarLq446xAYyWrLrfIJCYmDjxOqtlSjerUjjIkHAlHXFcFuEdjgLkORWEBxdhr7PIyy+Aw2JFuFUorxc4l5TvAlX5Q3mOAji8yFV+rjRZ2Ht9Bd/SsP3Arp057HTelZKSslbkkb5EIJAQSEhI6KYoynTkYVjyjOQ/A6nugV5X4/jvBxlnsRy8LQOjCXh0YijtunmEfM54Fuc8Q+FK4dcEjWlc4Qp9A2zAFV6fcVafwhq0j1NSUl4145qQkNCdMWbNycn5Xu6LbUamcsNkMC7+5K/cMiQ3icBFQYBxZgF83eS47CJYrUEI93J3hAQbmyZ7YFtSPg/kZY4KsQaZbA5lzu41Q2XIzThniqaUbR64V4lkgkSg5iGQmpr6DYDuNU9yKXFZENAPcnlRLfJVQAvSLgdHDtPY6044f7Z+bT1U2cddM8bCGWPPhoeFX52UlPS7pmmPGH2uUPwfR3WxxbRs0I8prKcCXMk5rweG80zDEQ72vdNR8Mmlz66TFulCxIoHaJ9jL4//4sQyRiLgTwhwxi0KuALIDx7+0i7UEpqiSeXYXxpEyiERkAhUGgLqJLWNZtPuYIzdDo4DAO40M09JSfkKAP0umktJSVkNYDUdod24ceO/WRyWwlMVSSlu0rL+Q0yxJDLwwj3nmbAhuZ6W91uDgpxHZvT/GHBObjrxS9ccwYsmcc1lrCvHcp/jmtuAgSq5whW65aVm7EcdQDaGHzWGFEUiIBGoNASefvrpKB7E1zOwDznnE5jCqm0uM1Vq/vz5dgDGAhTgz9TerWxBtpVg7Cof1n9aGGNDwC1/Pzqj/9NNJn7xYqUBVYsY6cqxXIxXi1o0QKpC87TKuR4vQBCq+mpK5bjqMZclSgQkAiUjoCaq0zVoO1JSUt4tmfJCKh1mpL6sFu7XHrIlJEvdoNIK7sK5wheoqzd0ZEbfa8CULwAWXSZJGCP974WjM/q3aDLxizFlylvLiclgXHxFUS2vtKxerUHgwoqGWlOlml0R15rrC+fy1ezaSOklAr4hoCaoj6qJ6nu+UUuqqkbAyZ0rFSjmk5a8icDUZ9W/q4nqt84I51VmImPusN8pxn9M7d+EQVnJyqoYmyvH2KOHZ9zylDkq0MNyn+NA7wE1u/6atFT6VwMyDiiafGfxr1aR0lxMBNSeqpUrfKKTO6dezHIk7/IjkJqa+h1nPPTZZ59t541LYmLiYDVR3cEtPFWDNst9kZu3fNUdH2Rl/wXTd8WokCgK41MPTe/TkZj8PqZ1MIXpd0TtElYhxjU4sz6tQu5zXINbMHBFl1qYn7U9WY41Rb6z+FmzSHEuIgJanDZA4cqh1NTUny9iMZJ1xRFYZWVWOhikyAI09Vm1J7fwGXSak8a1p1KSU76oeFFVw+HozL49AAysnNKYzaJYXjoyo/9egN3HGHSlmIc1zD8yo/+H3M4nNnt2zaHKKatmcNGVY9q2AkByzRBZSikRKAEBzvW9jkugkEnlRYD73VfF8tZE5gswBNRn1T7G2ev0zLPCihWqqhY5Q1IdpYap89WcskCjQLlXg/ZGWfJI2qpHgGlsB1d4N1Fy0sSkyxCCFzl4Z6axZ9RU9X1/nE8s5PXoc+XJylySzsD6gIHuk0LHgGAwdjeCWN8/p/brddkza34pTKzFAbnPcS1u3ECt2qnf5cLbQG17We/AQkBVVQX5oGMoO4GhA2f8CnC0Sk5Jvq4YEgrGGqeS0ZGRjiNHjnxMOw+a6XgTvk9NVOvQgQ2c8UOMs33g2KsxjfaT/WXKlCn7zQrUmDFjgjnj/ZVzyigzHxn2PwRo32EFygP6NJjufAIUjOecz1Z2KcPUpWqB/0lcskSHxt8QysH6VdXUQgY0sFqV5T+O6tLhmvnbaKeMWu3kPse1unll5SQCEgGJQO1BID4+3nJVm6u6aBatB2OsJ9d4N2ZjdBBDugZtOzg+UZzK755qrKaogzzFm+OSk5ObTJw4sa7NZmtqhbUZGNqAoTVjrLvFYvlbUmJSfQaWDuBHcGzRmHaGcbbLvKuBmZ8M+w8CFqflPFd4B8ThRwZ2BLm4Onlm8l/+I2HZJGExdWIZg/fTqMrGzidqxtCmSev6twMgK3utd/q0CrnPca1vZ1lBiYDfImBZlksWvKoygvgtDpUgGHcODa1VOxA9+eSTdcJDw2/jCo9nnPXljB9QuLIRTizIc+QNnz59+plKwK2QxYwZM84BoB/NTf2yMAEAKc4h1pCOTGHXg+F2hSl3aFzrZ6aRYf9EIE/LOxyCkHBwvKCmqu/4p5S+S8WY0tx36sqjZGA0zzlwlGO5z3HldR7JSSIgESgzAsxxR0iZM8kMRRGwLs+rFS8YarwahLYYQgoxgH7g+IZz/i7LYg8mz0rOKFrrqrsyFOevAdBPuhqEAL1ETZgwIXLmzJlZNUhsj6J+EA8L44itHnMCiz8y45YwBm2941jWB5fO+i7Xo5A1PJIMxrrluIbXQ4ovESg3Almc4z07Ry7n/re7e7lr5XvGvjYF7ZVaoVP5XmlJ6ZcITJo0qaHNZnsEwMMAfuKcL1WylNHqLLXaFGK/BEoKVS4EaoNifHha/z6KBfMBdnm5QPAxE7OFAlYbeG5mkRw09xgM/wKUf1ma1H3h6PT+TzR5+ov/FSGqBRdkMJbKcS1oSFmF8iOw2K6hdePG6Nv2SgRZLOVnVMNyOjUNZ3Jy8ebW77GbAR0sDG0steqLfA1rkcAVlyzFvD2fB45/MLD3HJqj15QpU4psuRW46MiaSwRcCBye0X+kwhjtjOLxQWVr1h71bp2gG3lO/+9R8LxshMYORN1+Y3Bq0SNwnDyAoOZXI/rO55D7y1qcW/2Szrj+iNcAruH0W/8Ha6NWqNt/HGwxV+hpjoy/cPbj6XCc2FesGXRFWWFvHZnZ78b5OWseUdWiC1yLZahhEfrTkLatqGFyS3ElApWCQLTViqsvaRZQijEBZ1EUNAivA4Ux1GnYCKsdHMc0Dofcrq1S+pVk4jsC9jb2JtDwG8tiLdUU9SGpGPuOnaQMDAQOTbulZ0mKMaEQ3KorWHAdWOs2RnDzq3VgHMf36r61/mWGf6nLb2BMWWYKrNHNYD/uWsdab0iCzuPU/8Yg44PJUMIbIKLXg3oeb38MyuhRof2f95ZeU+N15djY57im1kHKLREoNwK0cy+rnslb5Za5MjPWrxOGGy5vjjCbDe/bNbxaoCFPKsiVCbHk5YaA+ogabo6aOnXqweTU5Oly+oQZFRmWCLgQoBPrrAoWeLMYC5yCL78Geb9tgvP8GQQ1/5sebT95AFxzwtqwhX59QUl2KcuWek3ALDbYj/0OJbKRrljn79sCUqoLDm5H7i9rwPPPiyK8+oyxxw/P6NfXK0ENSyCDsfyOWsMaTYpbuQiQchyYs41dOAo9uGnduri9U0fEhNfBb5o87KNye5nkJhBITEx8gNfnNK9YOomARMAHBMIubTkMDC1LIlXC68PWqCUK/kyH/fBOBDXv7CJ32uE4dRDW+i5LMSnHBYfSwWwhsNSNgbW+y5JMyjFNw+BcQ9AlHQCra5e4rK9ew9kVqSUVXZjGwKYXXtTwABmMpXJcwxtRii8RqBwE6BWBo1nduhAKc+Xw9R8uWWfO4MyZLNApENJVLQLqBPUSNVFdxxh7iOWyRVVbuixNIlBzEWBgd5cmPVmNyRX89QsK/voV1noxuvJLcfbje2E1plGQMpz76zpwzvU4Upa506HPR+YFOchJWwVbTBs0GDkPdbreqU+rKK1skc4Y63xwev+rxHVN93XlWO5zXNObUcpfdQjkYMG4HgiLK/67ZdmeUsXI3vMuwuLexdlSKUsjyMHckT0wd0+ZTrstjak+yaS22Y3z9n2OexlDZHQ0oqMjYWNDsGqfa0enI1tXYdnWIz7gUjqJ48hWTB7CwBhD7JBUbD0p1XCBGg/ly+mQDsZYV/UF9YSIl75EQCJQCgKcdSmFAsEtr4XjzBFo2adRcPhXnTyoxYV5x5Z6MVAiGkIJq6cr0M6zRwuVY8epPwCn69C7rHXzcOajFDjPHEb4Tfeh4aiFCOtc6vk5heIFAaXKWkjs5wF9twq5z7Gft5IUz48QCMOwKZ+Ahgtbzu/oF/8Kpi79D24Ic8JhiShVzpAmcdj4XyvqlUpZGkEwIiKBU3CWRljG9No2ySQLb90+AIdnfIXMx3sjAmfw+bSeGNB6ITL5WJzeOBjx+dvAuzYtI07u5Ecwrdn1mDZpKf5441r8MGsQrm8UjBN8Ahq6kwbgNctld6gz1Rp7IlkANpmssp8gwBiiShRFsehzjLXsDF2hheLahCy4eWfk7litL7ZjTEFIm27QCvLgPHMEjhP7Xcpx9KWwHytq1Mnf+x3op9SJRtTQVET0fBB5uzdByyndpHMiK/8GADV+azcyGMtpFSX2OpkoESiOQEhEBBpGRKBeVDiiEYl6EREIjwjCFwtew/sbVuG6uB6YtTMLv33zrh7WrcyT38IvOYAjYxdeWPUzspGD9+e8grmr3jCs0PdjbtpxV2FZezBtsmGZHvkiNhwv0OPzTqVjgmG1vmXyNIzeDgR73tWnuNABG+NAfjoQ27YpIuiZYY3CLY+9j4WvN8XPKyaj00QAiV0QO34Z8uBA2rJU3fJL1t/Rs9chD0De7iWIZb0xZEisK633eGw+Qikml3UIy9AL2xKHonnD5hg6nubpfY79Wa78jN2LtBp//ICpvqUE3RfdScW4FMBkskTACwL5djpA1LsLatYeSnAdQFEQdNnf9DnDWs45BDWP1b8E2k+4FuWFXNkdjlMH9FU29pOkHLeARVeOXTtV0BSLhg8vRmgH17o67XwGcn/5EsxihSWqmXcBTCnncp0lK/ImWn8OksFYKsf+3EJSthqFQPbepRiZuBVPvLIQwxv/jqsnvY6kd1cjZ/1KzMGbePKz/XDkn8PHq+h0WoDon9wSjV8+XY1fXu6BJx9bhT9xFrNufRC/d38NGevX4tthwRgY/y6O4hRm3T4GG656Dge/XI3/3NrYwKayLcc1CnIfhI1Cv5UpeHlwOzAWi3vHz8Syn4Dho4fiupsfxdJRACatxKrJN+PM5mnoEv8XthzORO6JbQgbdzNS1h2B3Z6FdKxH++HvIDPzMFb2Woe4Zs+jyGSMiK7Yal+DzvpBf3n4fNZgoNd9iKWPCaExSJkxGJeW/mHBh/r4P0liYmI86sG1PN7/xZUSSgT8GoFzOSVrx0GXX6vLf/ajFGQseVz/nd+2AkpIBKy0X7EjH7RfMSnRjhOkHEO3HNMCPiUopNBy7Mw6BSWsLoLb3AQwUg2ZnocW6TnPHfMJo98dDWPVnrXjcDldOZb7HPvU7pJIIlAyApnAU688jbs6tUTD+rHY9+5C1PtrOxZ88A7mfQ3UpaVgpmN38jOBqSMGoGVEGFrGdkccvsPx0yfwBYDzh3bgzQ8+xreHjwJ4E78dPYGP0AqzhndDw6AwXNltGKYCyC9ZIpkKoO2gBPDME0jfMgs3NzmK+Lh2sMVOxkFbU1x5VS90qncJmjeMwtHt6wEcx8ZlCzH/3a9xCsC0L2nze0J5BiYN7YiIiKYY9NR8Mjdj98mi8IZYrcijecexoRgwbRS2LR8O0pVDmvdGwoShATG9IjExcQxj7GUcgWuD1aIQySuJgESgjAj8cTK3RM2U5huTYus4/Wch54JDP+vhwv2Oj7msw2QxJmc/4fK5o0DfzYLiaEFe9nfvIqTVdWj40Nto+PDbCLnyJpzf8r4+l1nPWMLfqawCnEDdLN6DL46Pj/d4UEkJ2f0uSX9UG/scJ/uddFIgiUANQiAfrXBd0zBdYsfxLWh192Q88PDT+GfXgZiUtRTTXGse3GpkWH6dtHgrkt7p9fQebVrhCpsdBRiIhZN6oZ6DMhc/pciNmbx0RyDvZ4zu+goe3TYPHbv21n8jxt6Pe0M7YcmOp3CrO/3w3ujUpg0KCgrQYuliDAivY7yA6CbhQupOtDLcTm114W3n5NbZaHT9ODy2cBMyR3RHgBiKCzFJSkqaBuB25OEGdZHqNu+kkEwGJAISgTIg8GdG7juXnQ2ZEFOv6BgkWGR++aq+DZu4Jt9+dDdOvzsBWs4ZPfrc6hdBP+G0rJM49vwAcVnon/92sT7fmHas4PY82I/vgzPjUGG6t4BT4/jpj3OZ1uPbeuGGy5a1a9duAtkWvNH7e7zc59jfW0jKV4MQcAKRTdAozPXCnHeO3uLvR+LdA9CtoQPfvQNE23yoTmh9kOL1Bxqgb7du6HGJAyOnbUFo02b4B4DZn6Xr82BP7tyIZ+hUJB9YBjSJ1Qakz8e4aatwJI+UWQcO/vAt3gHQLiYUyDuN9LMnkOVwIKZNLPDOATTrdgsGDeqFgrXD8MGxMATrKI/Dws0H9fxp772JdDyGKxpdUIyR9zMevH4cJq3ei+dH3ABkZSEry6UfOk6mYckiKr/2tkRSUtJbjLNeLJPdkDwj+YIJq/ZWWdZMIlAlCDi0gpe/33v27Jls19oT90Ltf/0CfccJc4LmhP2vn+HMKPsaWFqsl5v+OfJ2bfBJMdY0ji17z+Bsrj31qTXp53Nzc+9UoAxTE9THzCLVpLDc57gmtZaUtUYgIIav8CZd8ADeRHPa8u3W6Th4NbBp9lxsywXQyqUlF1ds6yDU2hiJbz+H9MSR+kK9RvcmImHaSFxpaYAxbz+Hc7PHIDquB5o/5NpvXS7IK6VbWNvihb2r0SxxMJqF2sCYDS3i5mLGyl0Y2jwEl/cZD0wbgMgucxF6SwpWphxAp0jaji0S8b/PwPR/ti0sYFxcCz1/l5FbsHBbIlqZdOOs37/FSjKVDGgNG7MhMjISkYPng+w2uYe+xrCRb+J0IafaFVAT1dng6I5j6C1PuatdbStrU/0IjP9w11G7w3H/xt0Zzv0ncvQ9iqtfKpcEmbkOrN95GkfP5H2yaVn6LIqdOXNmlt1pHwCGCeqzqu/7wPlLpQw5GPlkQpbbuflZy0hxSkSA+uz1Vot6g7Via0rf0xj6xnYCnRBX+c6JvJx8WMPC9I/v2TkFCA9znTxUellOZFPe4DCEFJm95YoPMXiWzqdkije3fI/bOrTH5n370bFpE/x5OgNhx46iczlwXW53nv3Tqd2XkpLyccmlFk21LMvljjs8fzIsSlmxq7ysLOQ6gIioCNNkiOI8HXlZyLXbEBHhkikrbTYiRwYjd8do3SJsjSg5f3GOVRNjXZ4H59BQfUyvmhKBpKSkVwFcw46yXup8tbI33a6qashyJAJ+j8Ds2zsNVRjejAyzRbRuXAcx9YIRGlTk4VAldaApFDS/+OCpXBw6nUvK+hKek3P/2NV7iyyBUZ9Rr+EW/rnGtb6pqanbq0S4SipETVK5bvuQinElISrZSASKIGABKbHC+a4YUw4Lwk15BQ/v8RcoqiOU7+R+ZdHwhAFtweeLCm4NiYChFxts8oD0PJDRPyoi0GYSe0LSFacmqtM5+HW5ubm9Z86f6ZNiLBZ/m585Ms5loCJUzbh4R16mBCICYz9MXzbr9o5p53LsqWl/nLuLHhJBVoZgmwWWKnglpsOhChwa8uyafooqB3YCmjpm+c9LPbWHOlX9kY6LV5iyfOLEiVfPmDHDtU2TJ2I/i6N9jqsAUj+rtRSnViBQMyzH/g91ZVmOZ+3Oxfx9edxiwS/n8/kaxpHmcNrTsra9QjvMl7hRZ1VZjsvdGo4snMkCoqL8WzGuSstxYmLiRAXK3TiDOHWOmllubGVGiYBEoMwIzLq9XRMLrP0VhcVyDtrXs8gsPc3uZOc1zqxQeGiwUuqhp/Z8J8sDZwwKD/dOzwF2mjN+UNOcax/78Ncf9U2TS5E+KSlpDoCY5OTk+FJI/SrZNGvOr+SSwkgEJAI1CIFfzzmR7wSDEx0ZYx3BAKsShKjrx2eDs584eBo40pgTaWe2bd8FbHBty1ET6miNQFSt2Nq+csBWn1VvATDO7rRfO3XOVKkYVw6shVzoxV9akAvhCKQAfWY8D4C2lXjSqPhtAFYBoPmDQsn9afyHu2ge4OUmcCjt/wDMBfAUTf01pe0A0LWEnT+XALjbRL8JwM3G1knEqw4A1ZRepmBGRsaT0dHRW9QE9SE1VT94rlFF+JWp8AoQy32OKwCezCoR2L/hFX3h3Nw9RY8/cxzeqMfftazo0ZylIfbTssdwSwl5svcvR9jI5cgujVEVpx/J9WYcZuFguIkxNpYpbBFsLD3q+s5ZUV3Hb4m67vG5da977N9VLKosrgIIPPvss1dAwf/gxNCpU6ceLisrMYWirPkkvUQgABAYYdTxCf0EDtfF10ZcA8MngyYdfUeH7AhrsfikRWuC2xiKMZ0MQvodKaJEv9vI7+7RqSCkGF+pz+Vz0ccBcNvF3T2b79dz5szJZ/ksnjM+pUf7+r4dtec7+4tGqSvHxj7HF60QyVgiUFsRyMl0nTf/5Kp0Y4diV023bnrLFfB1/Z0BUGgQkOFxP2QTgn643fFnPSMxuWf4uauaWadyzlPA+afgnE4w8eRCwFhXKHhYUZQ3PBH4Epf182yw2Nko8lqSl4bejGHR7pL2TcvCvFiGmWmuPUB9KcsbDS3WY7HzispAJpcjW7Fk2VZ92z3oMvXG1iKCeuPov/ETJkyIsCrWj8ExWZ2qfuu/ktZsyaTVuGa3XwWkp8Wt4ryJLgYfMUhdb1y3M/F3HY3nUn4pmg7tHAyAHkpiygMpuZcAWGPKJ4KkYNMvGoCY/kb0dE1O1w+NMHmkjG80LNgkFyni5BoCMC/Cdr9W1enq73MXvXcyL+v0k9bQCJ/WJxi8q8Wjl3g5raJaoJeF1joEVn2GraO7oRu9wzv3Y8Fr+3DbTbSCwVXTk3s24tF/J7pGkKvjsXbyKHRrHAQ4j+PtudMweul24OqbELd9O6LHFs/TYdATWDZ2UOGo5Y/4RYUqvPdVIVu/Xj6zcKCsc+0jMRYlpLMCrQugdGaMdwbYZZUmf3pxTnSUSmYOLZ/zvvyuyLLq4ix8jgkNCwbSi3NznN6IYfFALqevmS4XFipCNdMPCwl7G8Bnaqr63/LWoCKKn6qqCuzoDAU3gqE9Pdg54zTfkpo8nHFGeyTqzzTOuAUcHIwm+FTQGXwYZ+KsdgdnnF5h6QNOJuPsuL41OcdOaPgWNqSpqurtU0oFhZHZaykCwqJKyjEpxjS1oodR108A0EH3NK5OMhToPwD9kFSiuRcATZ2gqRV0zCdNqSCL8ETjmr7wjPaAG80BJkVaKOCChK49TSQjiwOVQ8ozffH7zbBM07SLv4vMxjQMcT3PkL37yT9/63QSeLVFXL+uf2xabiL3v6Dc59j/2kRK5CMCCld4xZ96PhZWIlkucNNDmDPoayzY6jr74OSOr/DeTU/j4etvwgF6hJ5Nw63/TkTzxNdw5PPlWHn9XvSNnw+i3rDgToxOuxrff7oaBx/vhwxR1tl0Pc/fX1+OzC9X4gm8iLaL0kSqX/rMNSOuiFJw/oe5xzK3vvTZ2a0vp57d+tI/zmyZ1dyh5TfQOO8Pzmmg97jSuSIVFJNg83YvQSzrjSFDYsEYA+s9HpuLncThQNqSVMRSOmPoPX4RDhpG592rZhbGsyGp+Fm3/OZh87zxLn6sN0Y+8hDQS3zddEntOLgKXTvRc2kiQmMng4zYkViPcSOHuPLFjsY6o5CTacswxCg7dvS8wrIrUv+LkVdNUEdwxi/HLjx9MfiXxDNpYtJl+qIeDce4lS8AQxtw/KRB+w+zs3+xfHaz3W7vlJufeznO4JLsnOwm58+fb8SOsUbkV/Qn+BBf4k/lUHlULpVPcpA8JJcun4ZjJC/JXVK9ZJpEwITA8wBori+NopMB0NQGYbz8j0n5pCkQNEf4U4OGWJDi/JzBiyzGZFGmnSHeB3DC4Elzl90dzW/21QnlvbOhTJO8pFjTPGf3b53ma5KN5KHpIXPbNsbHuaFRt6hPq2Sx9mung0/bVvi1lFI4iYAbAhrTGEfV7/HoJoZr/cT5huj77ycwZsQ3mN2nGT5/6x089cBKNDn8hU6efeI3/IL7saZPe9QD0PfOsbjttZHYdnwEjr4DJLx+OzpEhAERPTD/gVYYZQeyT/yOXwDs3bkR83cC2caSmQAAIABJREFU50hhW/UzTvUJLy6Cn8QYbyvun+KKSZf1/at0HgZ95tM/9Vkw1aVWF6OsWITdnoV0rMetw9Px9tv1sf7lAYhr9jwO8wsHN53cPA1dhiVi8bY/MODSTLxxVye0GBcN+/RwtBs8ESv3ZmJQ8zzMG9oI4xYOwTsd3kXcQ+uwctcJ9Gt8DCnRnYBeReW0NuuFd5Y+hk7x0Uhf/RBahLiOX428diwy3liI7c/3xM0vbwCfXAd9usRj/JbDWBZrwwfjGqFFyhXgz/UuyrCar+hBxhX+PLOzXupSVZxzc9GlIkuxpmnjDWvZPH0B4NSpdEyhPzjztlRFvl8888wzza1W6ygEIy0xMXGaoiizymJJlgvy/KF5q1QGsvOIBXHmsZDOdyZrsZjCdI0h1V5D4aVLEbfWSCOdjhTkvxnXNMXhTWNRHz0wzcYLmt5AjyR3R/LQgkD9UA+3RHN+6vclKdjFngW7j+Pz6LOnQhF0yesAaGGv3zpd+Ip86vLbmknBajUC/mM51r/ho1HLODyK1/G/b9ZgzvabcE+neiUMG1Z9+W+Bw/WVVj/VuLC1zBOKB+Fvl16K5k1i0KlXChYmtoORpZDanwJkOVa00rcNqmyZ3be9oG/sLuMvTXeYgUlDOyIioikGPTWfjjzCbtNSk0PfLQMmbcI9nZsjqmFHjJ37OjB/PQ5HxOHw3nRE/b4e8156DnNXkvXXjmM/fgqkzMegtg0REtURk7alAOvdplVYI3DFlbSQvC5aNG2oT+7IRC8kjuqNqJAo3Bj/APByOg4d2g56uuzZugxz57+LffSomvZN5a2EqQSg9akMQfgf53y6+pxK72sVcr4uyFPj1SDO+VIFyj8cTkeX5OTkZ6b6j2JcIgYkJ8lLcpP8nPMkHccSc8nEAEagm1F3WhRHAwfNEX4XwP+MeDFH9wdjWgMpqKREk6WZ4siJqRFktTUbO2m0+4dBQ9MfzI6sz+SEVdi4BMlj3u2C4l3HugoKl+/ty0gTw6osqM1rTxplbF3xDRx4Wn1CFYsMBZ3f+GQwLqbZ+410UhCJQAkIuCzHJRBUeVI9PDDpJjw5aTrwwF360l/xbSkkmsaQN7Ek7TgcKMBPa5biPfTF32LC0X448Px7a/BnAeA4uxMLFgDRNiAkOsb1sh/TDrd264bW9h8xcl0m6vmDsdwLtmQ51hStCme70HSGFThgTIMQYtG0irb1xQLuovOOO9E0cLuYOiqmJV9YNWm1Ec8cZP/5GZq17oR30jPQfsD9SJiEC+XQdOZSnGh7M5ndiLTrj7oQ45vpKHRu0wYtWrRAbPxSLF58Lezu2r6ZSRWHNU2jBT6bUlJSaA5kVTmGdpjLOKuLTehZU5Rid3B0uTehJwNL5Jx/p/ZUxWdyd9Ii19JYVQSOQLigufz05k6L4mguMc0RpqkVZNUVAxmlk3vZ8Ml7wQjTnGThaM5yEoCORgT1uelG2H2TI1Kcqby/jIV5NHZTvs0AiKd5JBIKrpieQXQ0NYKmeAjFma6pPFLYacoFKfHEn5Rw4k11IdnOq8+pP6kvqqeE0P7m0z0olWN/axUpj68IaAzGLFdfc1wUujr6ciAaRa684Q50APBEz7aFJZGia23QDb88/xCefOxORMb1xY3TDmHhfx/FlRYLeo54GwnnZ6Ntnx6IHPQwFpC9MciVZ/u0+zHs3r/rW8JdnZiBpf93o2uJWSsxFhUW4x8B1wdB82e3iypXxBU3YjDWY957m5FFDeDIwua3ZmE9hqNNMxqjSdEdh4Wb6Uu8A2nvvYl0PIYrGl14w6jfpjcwbRrWHcwC8o7gnZkjgeF90ews7XyUgikTRqD7JXZ8Mw2oHwLEXHOrTq/PXXYcwfJ5icXmHOuVtpPGfg4nsvKKPGH0NKP5rDH0LJkPtOiKQYMGoU3BWgz7IAPRPqlQFxXaQuYpKSkfqSnqs4URFQz4ovipiepiMDwABf3UDar5AV3B0qs+uy4/g4Vx1oT34FX5glH1lZUllgcBGqRoIEhxy0xKJTlabEdO7OojrL0Ut8GVBJqTLBxZjcnqTB+laESmV3LaGo6s0uYpG4K+lWGBpgV8NHZTPlLEhRIs6OjzWF9jegbxITqab0yKPMlKVmziQeUJ2SnvVcZWcsRbLAcpaSqGKK/afX0YlnOcqr0dpABlR0Bz6odYsiq0VBYXssOgqcgZZMTX64zvN9FONy7XYejL+NwIt+x6N3I23YHsHId+pHSh/hN0GSb9ZyPG5+QAwWEIuaC34cpu/0LO+ruRnW/OcwdyFooS/Ms3tOIqU44R0hFvbFuMu7rEIXKkwKIXFm97H22tKNxebVxcC4zTkzth4bYNaGU1Ngi1WdH8lhSsnnEvbm5BkzEAdJqEbRuGoh7SMApd0IglUiQG9wLWj3sShzJXYmXKYMQ1M2094TbnmNhEXN4Do3A9Wkeux5YTZOBphjDxTkOPj16RCGk6CLtWpqBdO7Fz0mCs3HtbCXtsuESszf9qgkqr4NtiJ4LVpbVjxweab6zGq61ZO0YHIfy7Irt91Oa2D9C6kdLp7Rlmjqd5xOZrgouWCLvHUfw9AO4DQIMUrRFwm/dVBGkar2nHC3ok0Sc0oje/kNLCO+G+NPZCps9x7nTEgxR9Gt3MzwD6TkYy0gEn9M3Nk4Iu+PuVr1uO5T7HftUmUhjfENB4DbrRXFUKQnhYWOESZHM1Q8KKKsaFaRbveQpp/CRgjHrmgfGiS9aw8z1Yx+3IyDiBExkZsPN1uKczrUGhYTof6PQ6cjlHbmYm7HwHRnSmHYoiMHoHx9iO9JUvArdMWAGem4vMzFzwHc9BJ4nqjHncXphvxTqOzMzP0DUiAoMS1sGem4nMXDs45+DrRhd++yyscFRXzKM0vg5dG3bGOv42OhozPCI6jwVfN0LP03ZQAridyib5VmBQK/EVtZBTwARoDiJX+DQnd95blQv/qgJgvT52DOeMTy9trqWv87KrQm5ZRo1FgBRcUp5LUozNlSN6UmTNirE5XYRpfPdGR2V5G/8pTzHFOCEhoeuzzz5rPulPlFOtPt2DhQasapVEFi4RKDsCdBPTjWiytZadicxReQhwxriiKWQ5qGJnRVSUoRAXKTkPSM/TzRVREaUonSEhiCg6PVk3poSY8kWYCKwhEcUV4iJll+HCGgIz7zLkrHGkJX6lrAPaRuS91NTUXytSsS3DBkQGO5UOXNFagystFAXNOdAAnNcHQxTjLASMB4EzG2dcAycrGCtgDFmc89MMyOCMHeWcH6AfHNquLku/oJWyxR7uZZFTnabuTEpKeheuelbaVJWyyCBpJQJ+hkCQxWKhD6w0F7FC91dl1osMxlI5rkxEJa+qRKCgAKCVVeJjdVWWXbGynAXIc1oQElS79Ho754xzXmVbfZXWCBGdHkZGhufd7EvLG8jpiYmJ9ypQHlRTVNprtUoc7ebAOR/J8ln3sha4vmdPa72YOv25hd+mcNYDnF8JCxijA76Mj866J2ZguS70NP2MEIOGyqW9rnVfhOk6SMH2YQNpvuQWDr4GnC3rvOSzcm0px/LZLATha1VVE71t7+bLvOyyYiTpJQL+iEBqaupmNVE9n5iYeEtKSspqf5JRn1Yh9zn2pyaRsviCgMKVXCfn3j7h+MKi3DS/ffkiwuLux9pTpl0P3Lgd3fkNPtrpWoybveddhMW9qy/fzTv8Da7r1RfRfVbo127ZSrw08yyRsJoSqTE0RfNhL4cqEtAagaioUizGVSRKTSmGdlRgjKlw4qJYNr0qfgX6WtZMdbq6vyxY/XT3wJH1mtY5yCz4RAEbDYa2hRpuWRiVQsvAIhlYPwXKCwpw4Kd7Bi7/Kb63+xZYpXABqH6c8XNOp1PsJlBqHkkgEajNCGjQZilQaD9zv3K6cux1wPIrUaUwEoELCGhMO58PVtr8qAsZKi10HEtSVgHYh/9uMO9JXLSA0zsmY9gPrvPuQprEYeN/e+v78uz9bil+GT4LmZvu8Lj7elEuRa/MPIum+MeVnUOxOCz+uxI5Lw29WW+k6Sfdecbs59m9ETuzMk8izMLsWIZ5ruP1ihSaNpOBpW4tElftF90xAsB+dapKq8+rzllwvemwA5/K3Ty0z31Q2JuMoalPGSqLSD9ukd2u2UL0Q2zKypZx9g1jjOornUQg4BFQFGWxBu1VdYxqrIqufkjkPsfV3wZSgvIjcDaPe538X36upeTM3rMOz+MhbF/0ND6e/TH2G8bj7P2rMebdL7FgTg+E3fkErnsNwIIHcd2cjcjO2IUXVv2M7V++iOtmbwfeGY8bp6xGds6fmDXlfn2rtrC4Hpi2wWU0c5zaiaRxPYz4Kfhofxb2f/NGUZ5Z+zGtMO9jWJBe/VtGFtCKZ0eZDeKlIF6JyVbXDJySJkWXlFYRSfI9zP65/I4t2HZf+4qwrdS88fHxNM8nBnl4oFIZ+8CMKzyWaWyHD6SFJHsyzlzj1Krl45Euw4mcHNoeq8xOg7ZDgRLrLaNckOcNGRlfGxGg6UUpKSkr1Tmq2Oqt2qtJBmO5z3G1N4MUoCQEIq8aHx3VdVynyK6PD6zb9bFR9bqOT67X9fEFn2zL+1t+lU/gd2LtB6/jtsQ4XNnyJiRgFd76lfY6J2fHgtdSsTQyAd89PxaLaXu34c9h2b2dgfxz+HjVOVzRZxRWDm+FDsOfw8pxPbDjnXvxTN0HcGLTRhx8/QmkJs7GTwWn8PztD+P5q1Jw8Mu12D6tMYaNmI682H8U4bn7kylIxT04smkjdr/cFWMefQ9l+h5tSF2ZnoPDdjrvtDipqVJY/7xoMibPno0hjOlfyyfPW4LZo2P1MOs9GS6DrANpS1IRa9D0Hr8IB42DQY5snofeRvyQkeOwHpGGmupA2rJUFx/GMHr2OuNUPc9i7141s5A/G5LqKjcvDZNHz8SyeePBWG9sPXMESyYPMXjGInXZzzqz4PrAuHH/NPLHYvY613TV41vfw7z1O7Hk3t6YuVkc25eFRUN6uyzNZ9KQOsRVbxY7GutEpTyLWOHYpUuXOtVUdUryjOQ/K8zMCwNvih/jrBU4aHNpn93hrJzdGw8dxpHsbGi86tby5Doc+PXUafxw5Jjr05DPErsIFU35DUDLMmaT5BIBiUAVIqAvyCtxBXEVCiOLCigEWJ1rH2lsZcGXMIZmDPwSgF3CXUdnXgKGS5h+rCULpQ0pXEvXXO9ynPPl/a4IW5zNtUmgRTdV5bJ+xbQ1QIe2e7Ehzaqf1/nKez9iUqc+xvaO9+O9f/XRp0tYWlyNDvaGuKxeBLILH6ERaBIdCdgboklEGBoOX47v/ziCjV+uwsYtKwA0gT3rBD5CK2wc3gMNg4CG3R7EkU+zEB4eAc3EE5fdALyWin+GHMDI7l3xy6q21fq0zeGcVkDlz5kzx9etg3xrtfydmDbuEDb9kYH/7HsDLW4ehkkr05H7QiTmD26BhOV3441WK9BlWCIWb/sDAy7NxBt3dUKLcdGwTwlCs7iHMGnlLnzWrz7WpPTBSri+3J3cPA1d4v/ClsOZiLX9jkmNuiClw2EM9STVmXVoN3giVu7NxKDmeZg3tBHGLRyCdaOAnfMnYlqvFGxKX4jsVwZg2LRbkX7iPVxyeg2i23XCpbtO6ofEAL3xYcbbwPaX0frmeRjMn0POqXXYkncvHr05EsNeXIux3e9ByMlvMHIlsO3tPMyM7II9C7cgd1ksdn4wCV1aPI/DPKGK5xB4AqTy4zjjTTSmiVO4fCpA084tyi6o2+PHYyfutCkKGoaFokFoKOoFByMiOAgWsQDPJ27eifIdTpwryEdGbh5O5ubiTF4+LavPAC+fhd3JnEcVptARux6dnOboERYZKRGoUgR05djY55iOHZROInBxEWg9JjiqgfUHztGWMWbaaeLCKvGSBODAV2dPO4aF/ARnQU8liPaZFSvMS8pXGWm/fb0Kv6Av/nF+D9b/CEQNHwS8k4qvTvVCD0cBOgxqjXCjIG+ToS/EO/HFgjsQnzYI8/7dF/fd+S+8suYjI3ckbIX7yBTg+NEzQFhEkQ0oL+t2P458OBA//ZKOz1aOwcivb8K3n0/F32ir9Wpw2RywMRyv7KLt+Zno9J9UdG8eBTQbgF74HEP7dURICNDrrl5YcCIHh04sAyZtwj2dm+vFj537Oia2W489/yb9YwaeGtRWP1hjUOJ8dJo2Sd+l/tD29SCtdeOyhaBZvzQp5Z0v92Eondrt7qLicHhvOvb9vh7zlq/H3JXA5b1dEzDoO+BXyxPQPSoLszulI2XL1+jYMARoOAi5tFVGhBXzDwAzFo5EK1ocGDcEvTAcx7Kegw319ZI63joWGDkeO/LuQczaxcBjE9AZh/AkSbhnI+a/tBXIp8OnXsbukwlo6mnXOkPmqK6PHwX4Ng6WBq6lMaeWdmbb7ItmCXaHqrTrEhS/aE3TxGeY0tjo6VfWv2x02wbR01b+dmBGvsU58kj2+duOZJ93dQI6AcFqRZjVilCbFUEWC4IUBTaLBRbm2pVCAdP3jqIxhCzPDs5R4HTC7tSQ73Qgx+FAjt0BuzF1g9MesBzfMfAPLq8X9MEldeoPww9lMnbrclvyLWcQYjS+TzWVRBIBiUBVIkAG48JHcFUWLMsKYAT2zsnn9R9fwRgSyooCB344m5k3BHvn5qt7ged6qCczOGta37QVU1l5+k5/CkumrcVTL6/EpM505D25AjT6ddX/s3clcFFV+/977sywqIPglqmlZRlWYkmGLVZg9fT1Av8FLYqlvVJfi4i5hAlcB5QkC9CeT3mvsFIrsRLfK63cUitxQcFcSs0lxZ1tEAZm5p7/53dnBoZVlhkY9B4/13vuuef8zu98LzPzu7/zWzB781Fs9AM69fSpTPBhLMKv+gIUm+uKaFGC39OBacl/x5hB3vh1/QcA2gNuHRGAPViw4XekjuiHCzlf4u7XM7FjfTJgR/OXpCCEdE1C0Ysj8MhdPbF52xv4+WwJ7rq5daTjS3LCC96s+LRWUGueDFZLYJPlbKSThyXHhxzJT45PTMmdLEWtoURNJXJ84+ox8CnnaUUJD4Jfv34oLy9Hn/TlGNGhPcp+r7hbUTGd+BY9bwnB+HlpCB/xEqILkhFHZhsyI5PRl/KKUPED3O1e9/LOnYIRN1hvWl+L5DVYhGLrDaDr/VgekoPPN2fC9z/LkJKyCMAB+XbQID/0cytHOV7C8rRn4C2npa4RkLmCFBi6A+wJBjwBJgCCAO+AKRcZOHkZZkmchGa+u3BnkkOtcLyHTB7LONrnZyaTtX1TDIE7aIo0hZULuXLtnKHUvT9ju0J8b/5aAv+YF52b8fWpvK5uGrd7wTCgxGi8pcRougkGfiM468KYnAHsyoTB9ZzjDKM0uAzHOWeHuMT3oOTSrifu8O+p0rBnGBOyTuuLDgJIagDBql1KUcQ9efuqjcqVgsC1jYD4tngPVJgr6sTHWxsJJc5xaz+Ba3X+Mul9uAuTwNCxERAcMhuL/4oDqcW2MQz4NY+jRzVRw3bboWfDiZ14F49hz502wZjIu+Gvo8LxxrTvcWxJZ3S0i/Dbxz8MeHkauv0yCYfpNaCvRWqqfBvVIjAqGPdPDgHl57zzwQcBbEP0dy/i86XReG7sK/BKsCxhfvJK3OkGFNjRPPluEp57KhJeH1r6PPn3uXihlQRj4uAiR7lBkpWwFoZa8P/O/YKAiARsnPApgq7TY1niOCA8HQN7kh5/BD7d+gImDe2JzM8/krXF9CS69xsIRBxDz0WTMECrx6oJXlh530FQsujqpfQSaQd1iJ8+Fl3zsxCZAHROsfTyCrwJluTPnugzBAhJ+hLj0sZCe34jRvQfhlfJrOKKxQOPvTkPox+iAAY6nJYz93UHeWwdQ09MCh4A/aFV8Oq/FgeNlBm2cYUxdAEY/eA8LsgvkgzeAZGkpd1DAnM7DTswcnC7V0yXzw9LTU2lTFaNLxyBYMIL3kOmvMiN0sTC3cmNDfnhUagutL4FNWz6AoNhfanRNNtTow5VgYVyr+uMof2v2wFgFzh+hcC/M5rNJ4+bv899Jh1m8fauHYztvXyYkXswjeCmAtPALEhgvBwCL0eZVJyrOZqfuhtGSiSiMplvVDHhZiawOxnYRM763seAG23cnS0ubVKoEVqnF7zqfMNRzBxtCCvnawqBn7CXP8wDoqKiOickJFxq7bXLv9VKnOPWfgzX1vwFe5MLvIdMSWZAbINWzvGnQTI8Xro7tUpIhjIJ352U+EO3qlidPzQNot+ATh69R6Bky4gaPa8PeAUl1qBXX/SvvO3dr2r/kjTLvTtDk0G/3lTuGvEmSoa9gWKo0IESgpSXwKCiNNL9sG5LIIpLyqB2r0wrXZ3mR1t+xKLyEkAeYyXaSqcTZqmUcfazo6e3U8TWIG2713u4DmvnjcGwPtZIQH5R2L05FGof4GCGDv0f6oOIitEhcq3HcB0ydGPg52Xddgich4MpvjCmAp2r/TVpb3oU4+GPboxEZz+EBAKbIqYiM8zy9mLRCasR/M5uRD3ij26acfIc4Skb8KKvOz4Gqjn7eaGdlXnbXF3vexqTMQMX00ZabYp7Q3cwAyH9/Wx5LKDLOALfyrerihXZV0wmsy8T2CCBMX8AgwDczRizf6OTu1vbAgEElpqAz34hmbj9OZ+AKXs5+G5wlmWSTFnFu84cBNLr2v6omJoxgeYiXgdDw3Z4B0T+swCmaGQurOKBXo/gJ3Tc07FRwjF2/Lb3UvfrWC+NJZY1I2GX4QEAD1hAY3BTCbhVeMK8ZxTPY0A+GPI5WCk4NwLMSEkdweEGxjRctg739Xn1Nvl9pwPUVf0Z7DeozJzj3OUSmx1UBQ4NqXTs2LGMc36FJ9kQSkofBYGrBwFxs2iKfTh2o0ajCQKQ3tors/+8tzYvyvzXEAIdBr/xsFpQ/1DV7rgmAJzjotlsflC/K4U8vKuU6OjowT6C+odx7kJjNNBVaHwuMTw20A89OjaZRBV6be3io+078OSdt2Pr0T8woMf1OHkpD+3OnsGgaoJBfesiW83FZVJZnr7QJykpqdFJQFSrSrnp6WoSaX0T1nXPYIDeiJqpmE0G6EtraQdgMuhRatTUHFNjDhMM+lKotVrZdEavN9Q5xqDXw6jxhNbDEfKPCXp9KTSeWlyJnPpLA8yhnjW+0zsOjrwZjPkLjJMAO4iDDbJok2ssspYGXgqObAnIAkcWk0xZBbzsV+xOrRRke0V6eveCnoFVT/mYK0mYXLjj/YofurqEYzFGlCBAXVfWuFoYk5uWPT7kwJ1dO9u9ltbV07Htl0oN2Hr6xA1i5pFTTaDMxFhREmeLNZ5VE2gpQxQErhoEYmJiogQInUWdSC4XrVYUm+NGQq9aVUq2dMoXWiNxs3bn5lBPwfueN+6GShPDGEY2gIyecz6iNsGYxsbFxe2KixVZIefo6CDP9AbwpHSphsAfEoeaYWdTBONqpJp36eEBbW0yttoD2joS5ak9tLWPqcGJGh52RLS1TmQZRP1qY6MGyQY1qKG1m7dBQ6p1stoXk41xhZDq4z/pRq4SBmk9hGGdtKpXTl40k1a1FndE5gmGIQIwRP7mEzTw5ppyBEzZR3bMEgnM3FzGoK4uGBMXPQQBK32GRK6TzHiN+KjTIY+BN1YwpgmKysrWA2hx4bjAYChsomBMbHP5X7XnpFwqCCgIyBurDdtRdiJY9D3lCNWGE1l0OdLMIRoul1uW8xlSf2lg3gFTvq4uFHPOOasl3AQHL+MSH1m4I3lXPdzRnuiaoxKeH2SL9lZP59pu0ZsOU953aoOmwW2/S7y4TMLyBg9QOrY6AtYIFicnR4t+XOD/mj17dmQ7/8nXu6swiDMyx2D+DGwQWIVHYQXPVsc2f+pjsWG+0s8IGy6osN97SOScgqJTiTiQbmedX0GWbBjo49iogMUXSi5nlJnNb7irapPNK2g7vKIvNzbWprqCBzFGFGXj8ooWpaIgoCBACAhFwm7uxf1cAQ3ZqIpUyK7AjMLD1Y2AvWBMQrFVk7Ww+qo5uBngowp3JG+sfq/6tUGSEjONUjlt7TelWH6Nmza2KfO54pjmvBzkSxwnOHhJWYkiHDfj4erz85Gfr68Srq8Z5Bo8lAv8/yRJWkUDSnYnn8nfkfxNQWZyXEFm0sj8zPdvNBmLu0qc/wUcMznnqzjnTY1y4UHJsn28bsj2HjL5kVoYpF25Rn8Q9WWF2/MM8o5eLSSd00RfXRdLL/+3qdRFnSiinmg9yu9xU5FVxrV1BMRksYCBlc+cObPOOOAttUZZOLbGOW6pOZV5rmEESCimJB4mCQPyt7//TEFB4SwOXpEmg6DhEiYUbE/+qiEwxcXF7ZMY37pXIgebxheBc/x65izKTCY51qkt5inFPb0aD9v6jGYzLhQX4+Lly1CrqjoeNQbFn8y82CxJ7yUmJuobM66hfXMz12BVZm5Du1f2M+zDBMqMN3NdiwuclUxcuWY4ug5jGINXp07o1MkLGjYSa45aoDTlZmJmkCVD3sAxcci6UBkl+8qUr9yDfoAYZ70PHTq0va7e+t2pFwszk77Pz3w/oSAzKawgM6kvN0g+kiQNk7g0DRyf1fA3rIuYpd2XQdjkHRD5SQe/id3sujbpj3B+zrnLhQbjaTs6Tq/qy42UFOT7Zk7UpPU2c05luIJAW0DggJvg1qTU7I5anGJz7CgkFToNQoCDf81MiM3fnWTJq0ujfvtIzwKmzAfDXLqUJGlG4Y5ka4CyBpGFqZT9PdNTOnQTY5rOln3ehg2kQF8Cx39O52LvaYsARhYeFqV2g0m0qY60PtKWC9Z1Du/vC29PT5glDpOc7KDheuTfzRJOmvklflCwBp1zPBSXfgxBWNlu8IAejSKeu/kzpNKIhMXYHTUcAXXYHDeKqMM76/HxUyNwet4GFE0Jghb5WJfwCEbckoYiPgZpI4bgm9C1OP1Ff/yGvN6oAAAgAElEQVScFAz/Z29A0caxcNRS3FRuTwBYR2mjG7M0ijYDgHZ1NuL2MDdvba/QWiyj6iXJGBujbtfubx0DJr9VmJn873o7X+GmvqxsK4DGx7i7At26bucbDGVxO49YAlHX1akZ7XXaZTeDpjJUQaCtIMAZP8MYa9wXvoMXp8Q5djCgCrn6ESjYnvRUbT3y9YaF3lr3NxnwUeGO5MTa+tTXJiaKp2bHxLz5P5M0f7RG6KBuhHNee8YQ4a7CZjNHtknCwOu7Q6VSycLj1Swo09okieNicTG+O/gbjuXl4cG+N+F0Xj6lgb5iKeIcG0y4bODm5+PS42qzIb0ijSt1OLpmJvxmUC9/DMxLR2bSSBxYkYBxo2NAyTwCJ6chLWEsetfwfsvHqhkJ0G3JQd/3/PDBf48iYFRfAHqsiIzBRV9PREwkeT4Ei9NHIituHFJzgJCodKTNDYWP/hASX30WM5ZZUobo0nMQHdoHK8Y8iNE5FMytMzp3voRLt0Zi25JwHK6NJ+zDzIjV6N1rNybGZAB+47FhTQqCqjBrQlkOMDCuByXSA+CD4ZO/QFq3X6EvPoX9GI9l04ajhwfwl5BQIOG8Y7XgDH8FR4N2aOp6Vh211w+4UsSZusYC0AqMTfYeMtmWWLKernXfOltS/LVJkkaphZZRxurLyynZTaNNQOpegXJHQUBBwA4B0lS1qnBMvMjfJkqcY7vHolRbHoEDi4olsL/lZyZNb+rksTrdksscX20x80Zl2bLN97AAPKRi2Jt7Brv/PCWbVNA9D7VaFpTpfLUd7dw0UKtU6NqhPf7P7045xW5W7pnKLH82cKqdyznHl0ap2MT41Li4uF+q3XbYZe/A15E+HkBUBtbMHIb8rQnwHx2DGbuPI+98DoZnj0OfiDU1BEbTiQ2IyAnHqKED8OSbOiwbvQK2VBz67GREbOuN40V52DAPmBiWhMe+OI+i4xtQlBCGlYf02JrQHzO6xIFeAM5vX4yYsAhkGbR48v21OL52PdZ8MBKbNuVgdPjjMNTJkxEHUmOwtuMk5JXmYUPodgxL3lwNGx88nqFDckh/MDYQYyITsWovED4hFD06DMCS7CUYoD6KBRNGwmtIDManPw1bMr5qhBp9GRYWpuLgQQaj4ZtGD7YfwNRyfGP7pup1LscU5gdlm2VgtmTmzxrN/M4CY3G7/O1JtxdsT36/+pjGXJdeLt9aYChrzJBm9b1UYmiuSUWz5lcGKwhczQhwzi9ygTvqq67JUMn6CmUbp8n4OXagQY98vUmOpeqYGKmOZc+Z1Ioy36/T7rGh857NuzQenTrfXMol/7+qmWdjtnqp791qBl8VR2q5hF9Pn4YKTM6D22gX+oYy7AL9bGtTMYBx4F61gDvpoo5i4BxLyiWowT+KmT17cR3dHNKs1vbAbXcEws/QC727+iArbRUQtQWjBvWW6U9atBgz+m/C6X8Go7ddwITdX8SBIo/tWbcRbifInD0Z3xyajLG+QNklYF7KKPTWanHdiCBguTv+5tsVHrgfz/oBhYUm3Bd1GjkHjmLTiiXYtJZSOd8kZ4rWdu0BrekoIntGIHz5QUwf2gNZiXXw9N4YFCEQ88cHwccDuD/s70D/HOQnDa8i4PoGR4MXTcS+A/uw+8f/Iuyh/oBfFI5kzkVfWSPeCQ+GT0RKu2OICPsQ041z0ddurU0F+o5+d9wN4OQ777yT31QaNE7gkr+cpppUqZwbGdhhMOznwAEuYb+Z8wPF0uXfq8RFbs6EtYyd++uxc/69e1zq0s7T6ckyS40m5JeVNdkZrxb2azSRvaPym1wDFqXhGkFA4MJlAK3qkEcKYwd8zV4jT8ypy9Rj44JXMSxiWcUsIbq1+DR6OLSmXKxZtgsDnwuuZfu4onvjKoZ9GOMZjdeLVruoLWbjlmPrvXDhwjJxrPjYyd7SlmVGNnC0RnAj29rGFE+rmUVjxlwrfYs5x/JyqdydS7OidDrKeu30UsXLUhYW3SrmVGvcKcYCqmQdMezDBzNyEB4VioNb1qPUsxPGBwLjlvyEsUmUPA2A0eLYJtPOKQOdPWCErHvUmPBtTE+EbByPtLhwvBQZjeRlJCBTyceSx29BctQG8FG+lqYr8GS0EIdRTsrsUVUrT06DAR/g9d1LMCAgSD7GTnoJYzz9sGzzg/DMMeEf04MxaOhwDBraE6uTw3GxdC76OsDoWFJJ9zHGmq/1l7BVYthgZsb9xaayw1cSgp0l+BWWlVPIx79YHorz/s8zGMy4fKi+8JLOm1yhrCBwbSBQzBn3bM2l0stpyxhpteYq28Dc+n1pGBZxGhuOF8nOYHlH1uJYzAj8K4uUOmcRPS4EZx2xDhK0lyRiJP34ArCl33UEaVehIS4VDTNnz763mOPbL4xSMQl0Smk+Auckjs/KzZdNEua0lGAsc224hJyC89CbTOjcLwhISMDGE3rAkItlieOA8Mdwi90rPjniLUMU3p8bjei5czE3OhopKYuB5ERkNiieRjEOJQNRSfEYGzwUOPQT5KzCMGDdzEcwcZMOR6LuB2XBoyx5DeFJXof1w1Yl3oRaA+SkIiJhDXINdMeEEzt/lj+bd/QQMGNGCFbIkTpMOLpuNTahMzQO+tAKEO5jUvOF4/ydycspA15x5sIDVxKMm/9XWDeFi8WlGS3hSFtUVn5MPACn2NjbVqdojW1IKOdrFAE147SP2bpFFo6VuIqt+xBgNAAYiB6dLCohn77D8UVGGm42HMRMf3/Z+WiIZxBWHTUA+VmIG2kJ78QGTsDGEzSWHI1GYuSEMRhI0QgYQ+SSraA7NYp7NzyTppObK7VyeqwYwzBySZPj2teYprUbZs4W/++ChKRPyqTSP8yt/jlrbTiaPD+Fs8sySaZVlheNcbPiRMsfT5MpNm7gTY9GAgkj4OW/CF7DdVg7DxjWxwvMsyfGbY/C7gWhdtpYPb5fnIDw5S+gq900HgOGIwqb8NF3x0C65irFLty85Z43Hk0bj4Rh3eTPUfjKYwAyELX4KyxOIAe9GNzi5QlPLy94haRegaeeaGcTZunDFuhlxysAtS/mH1mLnjEh6OmpAWMa9HloEeZlHESo33AczNBh4pCecvstI1Zh8ZY0DKrhfFhlNQ2+4IzfZ+Km5muOGzyjpaOzBL8iQ8n6wjKnyqzyAgrKDNUNxxuJgNJdQUBBoD4EJCZRRh/nf5jrY4K+num+Nc7x7Cv0VW47CQHtrcHQBfZHf69kIDAc854NQfBT4QjtCty/bDkS/OYhIycNQ3vmI9HTH7+nbUfpqoE4sDIK/n3exWk+GfrsDGRcikLO+SJ0P7sJj/o9hHdvPY3oIDunT3UPBI8dC5j24T/jNtmtRoPuD+rwzOAb7NrafjVWJ8ZER0f/d50Jq7qa0elRtdDBp5Gh3to+Ck1fwWmJ4weTVFwK7C81GcPmzJnzZ9OpNW2kz6Cx4HxsxeDh01eDTzJAbwRqpnDWYuxqjsretmG9Mde2gxBa+aKkHTAJPNvWR4sJ2dZ7g5aAP5cCPdSQbf8NehjUWnhMqz1aWO08DcJG/qmNOLSDJoHXktJG23c4PuUc/9brUWoCtD7aCgFatkfmUTDIfgjVTDIqKDe+Ik4Vu3HwDvHx8b81frRrjojf+8fhe3r1KPH2cG/nLA4pNvh5ffFqZ9FX6CoIKAjICHTgjFexlmtpXEhhrJhVtDTqtc2n9UX0Ro7zR3Kw4dVhOLM2DP27aTBzzQn0uNUXgeiMXrf2ho/xDNaRnvj3H5H6fiq2HSWtVgwOkSv+JUCXOg0DumrRdUAwSDkcs/5QbbMBpkqdsaWDB4ImRGPUIHt9W+1D21prXFzcTuMB3JprRvRyo1T0pVHSHzNzGG3CUltbkJP5LeEch8wSPimX9Bkm6XShCWNmiuJ9rSEY17lUD49aBOM6ezftBs3hYbXX8NDCVq2TWDN58tBq4WMnGFfOo4aH1nGCsUzXA7cDqOPLoXLmtlbTlxsr46c7gfk8g4Fzs5HsbJxalJ1cp8KrEHdxBAQInSGRRNN6hRTGinDcevhXzHxo6QSMWXoIXfsOQFDoWCSt5jiYFoKEkC9hcyWXHXtk1yEgaJAf+vn2QZ+BL2F52nJ4Gy1hjNw1dsaX7n6o6qlUMd01VxHTxXIxTkzOKyrsfsrEJ39nlHYvMfKyL8rNBdtM3HjUzJEnWTLiXUvg0AvCeckiDG80SYZPys0FHxml0k1mviHfxEcZgRvFeJE0ZZXq1msJoKt1rQzkTdgqwrEzBb9LpYZvnfnIisrKz4p7j1MCFKUoCCgIOA+BTgzsovPIN4yyLE0pcY4bBpazemncSrBs9KsI81+Gvw7oAeiP4sfVGcD4cGiNRlzCJeTn5cPUszsGAjiGnpgUPAD6Q6vg1X8tDhqfpGhTmKFLw7OfT0LP0iwsmZGD8RmU+KAhxYCsNV/h1HWPILiRmcgaQt1V+iQlJdFWzUd0TJ06tb3Zo8PQc4IpwJ0Jj0gct5kEdFFzGD3Bje0FQXLjXOXBmKBmUKkAQc1BQc4Egcy6wcDB5VgYJDnaQqK19Fpt81aeGefgkMAkDkhGcMnM6QyzATAbAemyxFWlgNrModEwnKbQW2VmbOQS38F+YtujN4tV/MZaek3KfE5GgMGXc94qwrEzV1ZkKF172Wic3d5RXovVmC0oL2t2uMlqJGu9dJZddq2TKY0KAq6HwC3gWNnabMnCsfJhbN3H0HfUIqw99SpG+PWsZCR8Hg7OD4VaewGR4cCIPp2QsjsPuoMZCOnvJwtj1FmXcQS+auDHIgCbPkQfzwiZht/k5dgcbIkHW0m0rpoR26JHY3VkzlUtHNuvfv78+RRLkaxU6JDt7SkxQr9+/bqXCkL3fM67UJgCgQtaiUkUVoZiiNFhU8/TrgsdEp0FLnCJSY2LG2fPUNPr8vw2Pqxnibx9GRg5NZCgWy4xidZbLEhCISScL+NlZxISEujtXNEKNx37tjqSNMc/tAbzzvytkXb+vif/+u7G9hrHS8fkmHqppESxN26NPxplzmsNAS8jNx5uzUUrcY5bE/0qc2sxfPqn4FP+Db2+FPDUVto7oivGfpqNsRW+PcHYyI1yP42nzRZSLyc3SMnZhkkDNNAbrI5EVeawu/AgZyF77yAtJmVzTLLrci1W09PTzQBOW49rEQJlzdcAApzzzUKx0OKRKpwNrQiYVpSVH4EW/R09V76hDHpTub0Xs6OnUOgpCCgIABB14j2tDQS9xCs2x639FOznV3tA6+NjJxjb37Svq6HV2gRjS7shBygsISWhnSOR/RClriCgIKAgQNskcbPfEZNESh141ZX80rINzlhUgcFQOGfn0RaJ1uJMu2xnYKPQVBC4GhGQt4jpw+jM7a6rETjXWpMW/8jLozhQrsWWwo2CQMMQ4OovDa1hktIw7tpOL5c3kXH2b82F0uKMvecvvN7Pxwft7B2Um/gMzZzjtF6PI/mF3zeRhDJMQUBBoA0iIAvHSpxj13pyJn0+8g2Ah5a0yA3jjTTOSlEQaIsImEM9lR2stvjgXJFn6fLPJ4vUB08W6ft3dHdDZw9PeLm7gZz0PNVquKkEqIWaf25kU2yUJBhMJpQYTdCXlyO/rAwXS0phkng55xI58rZIURRVLQKzMomCQJ0I0Eu8zbmozk7KjRZEwJSLpdPCMS650rQtfN5aLJo+HJbceU3nJTdzDX7GPQi9iqNRNB0dZaSCgIJASyBQj+BHjqXNLuLu3BLx9ty7WPvbwgrKyv6voKxsKAPrZk+YtigExuSDVO0kGNNhXzi4ARx7wNlaLhjTxB1HTtnfd0DdIet1AB8KidoRkHeyRFFkoihyOtfezTmtNKcdZfu6XfPVVRVFUZAkadjZs2c3p6amVk/G0KKLJYWxIhy3KOT1TWbCmtd6YlxqFHaf/xaDunog/+g6jLtlBMZ0O4jVY8nBvOnl0o8hCCvbDa4Ix00HURmpINDGEZg1a9ZtaqgHiPHiKhdbCqlzSQBptiAgHqDUs78tB0AHxMF9ugvcvS8X2I0AunDOvM1M8mBgGs64GRLKBQF6LvFLYCzXKPHj6l2/HSUHPydiJKuvtfe+1s/EeanG7O4uCWYPlQruklnwEAS4cyZ4MMlUVrhzgVPsqJ24NpcgTeE6PT0972KM+QoQKK5pLwDd6W8AQAc6OOO2SERqBkbyED0XSwQgDvrbEMQYUaJzSy5KnpMmZKBUyjZ+6LNBf5MmzjgJjwaKQASAYlVR/O3zjLPT4PhDYtJhoUTYI84Xz7ck382ZS5KkJwUIEampqa0SSac677JwrMQ5rg5Ly1+bTnyLkFQg/bgOg7pa3ll8+g7Hoi1p+PDABRgMJdBFrMegQWcQNjEb24u+wJ8JryAsIQOAH1I2rMGkoN7QH1qDV58NwbIcWkMI0g+m4e7f34XfDLr2x8C8dGQmjcSBVQnwD4uRFzo+ZQNSJgWhgRYcLQ+OMqOCgIKAQxBQM7U/GP4GwNWEY4esrzYi4s7jZwHQ4fTsdrXNX1+bWnD/TcW5kamYQeBqWZvs7gZ3k5kbKJI6U6m9vQZH3Fu0M2VnfXSUe4D4iKjGgwiGgCAAjwC4GcBezvhvkHAUwPcSky4IJuEcOErKWXmhJEnlJpOpvMOuDmX7u+7n6enp9Aya/YLmhOdBGmy2f/9+dofPHe4GH4PG0+SpMXmaOqoldXsI6AIG2iHpDobbGWNPoh0Gx8bGXmac/ShB+lEoEtLFZNFlk9gwxp6SuPSlE7BrEklZCqtnq6tJRJVBjUegtOg4RS3Gw70tgrGNQo+hYxE9lN4Rs3AgdQYSAnXYkpOG4uRHEfZnJE6XroLm8Ep08wvGnefX46f+Ieiy9gj48N7IXBKKIc+uRN6215E+PgFhnTOwJnIo8reSYHwK208XYaDmMKK6+UN352nMDephm1Y5KwgoCFyFCEhMcmeMWVJqtsL66nTIIy3dtVQoUw8Ao8k0QKNW/8zB2zHGSEsIswQwxiy6Co7Cop0pu68laBq7VvFVsQO6YCqAiQBywPG9GeYxhw4dyrGG52wsSVfsT6Yd8t9MOtJL7BisN5Oc+JbYBxo8KjDhEe7F342Njf3abDbr4uPjj9nRaPWq+IboxTkPMRqNU1qdGdnUSlLMKlzhQVh4cAfle65haGPS44Jeg66elr2TDV9GY6iPHkvCc4Cbfseq1EVWpUgO1u8Dok4fwYGjh7FiyZdYuygD6DwC0PbAbXcEws/QC727+iDrM7Jp9sKPq9KQCYA+XcvWH1WEY9f5Y1A4URBwFgIkdNF2rKsVKSwsTKgmzHQFIO95AUgF8Lsd0+SGQZqEfLu2K1Wp/xC7ToTDLrvrhlYHANjX0M7V+5FtJedcfhko3rXg146DI+9iKrYRHNeBVW7gcU5b6HwNEMaA9Opkrvlrq43qBACxnPH/snL2iJggXnWZH5vzoMV3RNK6/YcO8U2xi9ReekWlUu2KjY1dxi6wt8VFIplltHoxe5v7qiTVSwkJCZdanRlSUypxjl3hMVh40LSj7/oE7MqtytO+RQ+i29Ofw2AksXky+lqDUpR1BsKDBqFfvz7o02cI0pcvx0Paw0joeQvGL8tBh9sfRWTcZOBSmfwLUkPoDg+CX79+6NOnD8LSl2P5Q+1d8hezKhrKlYKAgkBzEOCc72Vm9t/m0GjO2Lp2Kcn2t1evXpSB0lb6kA0lgF8B7ADwG4Bp1pvzrXaWf1i3wNvZBl3hTDanWwGMtB5k2/jjFcZUv022p7LRWvUbtVy/asdz5e3jcGOcVWjKC3cm/VFQZLiTg/8CDspkaSkMpRLHDu8hvd61NSlnCwKxM2JvhITNAoQxJskUOHv27FcUwbj+vw7xPfGiTqdLKC8v70cJEXgXniPOEsn8pNVLXFzcHjFe/KrVGbFjQDYyp60uuzal2goIePR9EstDgJARM5F1QS/b3edmrYBfRA4mT39cVid4Bd6ETjJvnug3EFh2DAgYHozgx/vhh9GjcVYw4RsASfHTETz0BhzalAx0tnqVGC4hp+A89CYTulsH93xgOIKDA1H+w2isPNuuUmXRCutXplQQUBBwPgJxcXG/iPHit86fqdEzGDqaOmrsRr0O4D0ASwGslLMbWW7Sb9abVuc9UhVsAUDOVg0tZHNJW/B09ATwkHXgCqszFl2OBUDCOZVHrQI4bWk/aG2z2W1SJi/b9jQJHNSHDhpDvP0TQKJ1HutQoKRbiUaOhlHRAuDAouKCzKRhEvhicG7ZNue8vGhH0j9h4Dr7rj4BU8K1/uPJqeyaLDExMX+BB3aCYxUEPBgfH3/wmgSiiYsm7ezs2bMnMImNh4DPY6Nj32oiqat6mGzg2ppxjvfs2XOrIAh+jLHuEmO03ZQncH7wt99+y37mmWcq3q6v6qcgL06LUZ8fwcVXnoJ/t4SK5U5evAW64T0AvSU5k8V9Wo3huoPQhfRHJ2uAmRDdWjzt3x8Y74dh3SyNgSGBwKYIfJo1BmMejQT8R8DrmxTkZeuQoRsDPy/r4MB5OJjSvGgYFQwrFQUBBQEFgUYiwDi7DE94WTXCNDoJAIVPI0F4OgASXm0aVPriIjs0ynhPwi1tHTek0AaaN4Al1s7jrQI4XY6wI0CCMWmxaTuPtMukkyATD9Jek13wCavwTE5ydI94oXvkEEWmGhQ9gNo/s9KklPQVRaPReIHZaYgr7oAXZiZNffTZGX12nzA+LUlIJmG7YG+yTRiXe5olflEQ2t8FYH3l0GujJkaLz3HGk5iJhYlzRHoxckjhnPN+d9nekxxC0qlEDmdvtf54N30aMV5c//bbbw9Wq9VfxcbGXj979uyIplO7uka2Wpzjbdu2aTt4eUUyYBxjzPaGbomVwhh5I8DX1zc/Jydnpdlsfufuu+9u6Jdf235CHn0x6dNsvLpID73BBLV9EhBtAFZvDKhcn9YX0Rs5pun1MGo8K1JOj12Sjefm6wFPS3ppk14PaLVQYyw4J4WIpQRHr4Zxmh6lRg20Dc00YhusnBUEFAQUBJqAQJ0OeRSKSiULxzaqJFCSJpmEVtLykgaWNMlUp0KmFDZpZiCAbdZ2OolyaB4gpQ4B0mbAS1t0JHyT5qyKAGqlRQ5eNCfZNdNBZhmkGab5SGMcbG0nG2Qqt1nPRIuE6T0A2lvbKk4aSdMRvG5b6ftvc9+3+7hBV7QzpVbzjaKdSesqiFkrPkOm/JR/ij+KU0ml1e9dLdfR0dEPg+ETZmT3iHPFWrG5WtbaUuuYM2fOn+KrYiC6Qi/GiEZRJ9o+Xy3FgkvOQwrjFo3dRyhkZ2e/rPXyOiowNtteMK6BEGM+YGyCSq3+LTs7e/7KlSvt7dFqdL+aGtRaLXy6Niw7nodWWyEY2zCgNg9r0AuiZa3ablec1R5aRTCuQEOpKAgoCLQWApzxM1DBPlwO2RVTDFqyj35Y9iC2CLIkbJLZAgmrTwJ41mo/bM/6Aau5xTn7RquwTYIraVzpIEGArinkV13FnoZ9nRwE11gH9bfS8ZPjagJvAzhTF0Fap7zeOjqQXXZ+Zu2CcR1DYDKZX4I6nwR3uWjvfa1zx3tfv8l23dbP4iwxSCWoNsOEIYpg7NinKTvl5aEjfa7EaHGWY6nXTU0cK3rExsT+Eft2rM1cqe7OrXBHFo5bIs7xrl27NNn79v2bCcK/GWP0Vt3Q4sYE4U1fX9/NO3bsoADeSlEQUBBQEFAQaAICs2bNukmMFkc1YahDhtTlkAfgDzCQ3a6tkABMArKtdLZWSElCpg42xc4wayIEWz86k40ymWBUjyhhM6ugLWk6yOaYzCwuWM8kIFAhH5xyWEJEkMaaCmmqaTzNSzuZFCWBCjndUfgfokOhg+ggx0EyEamhNZZHWNZJzoQOK/pdKb/h+NLKKCRmIxOYZqHDJmhFQuI0sTsELJO49Kg4V8xqRVau2qnFhWIRLuNRLvAJMTEx9JlyfrkRMxnY9tlzZtvv+jh/3gbOIH/B1POF1UAyV+6mcXP7mAEvX7lnHT0Yu8/dw2P99u3b6UtHKQoCCgIKAgoCjURAbVZ7coFHN3KY07szie0DA2lebYVsyMgm2ObkRmYMg60aY7LlJX8Uukd9bAKsbeyVzpTogQ6yabaZRpC2N89K0zaehOD/WdsoigRpqSu0s1YbY3K6Ixtj0iTbeP0FAJmFkJ00CdoVpoNEmAt8AOeconA4reh3p17Mz0yiZC8VpeO9U8J87o2kqPltq3jiP5zzD3Q6nZIp0IlPjqJZwITnBQgfz5gxw/ai6JQZo6Oj7+aM/wOlcIm4xtUXSQpj29t39XsOvc7Ozp7JGHu+uUQZY3e0a9eOvnCabYzeXF6cNT5rQRCCFtTzcmzIQhALQiZZyzmo6PctABu4ADVI6jMpGD221rjhoIkVMgoCCgIti0CxLBCS7axrFQnbOeP32TFFiUroe57iMtNBdVtMYtJ8kz2yrd0+KYIdiRpVEliJjv1hC2tHGl+iSb+JdN/m50KmG+SER+2kkSah2mauQBpn6ku8kiaZxlNfm2ae4jLTOBstG0P3C2aBBOhai7OiR5WXcUo2YtPAy3N73zWZNN4uW8jOmDN+BzvE7HcRXJbfts4YaXE54z+6u7tPdtZaKJOhIAhLmcQixXdFylzpcoUUxk4XjrOysm4HY+Qg4ZjC2BPZ2dljHEPM9ai0cwcuVW6OtQyDchBkd/mbvcqEnv2wfUsO7iKfbaUoCCgItHkE5O1T8lh7Q2yVHbg6BT8Nshhn14vTxeqCOwmetWX0o8A9tbU35xkRTXvNsI1WQ1MK03jqa1+q0KP10TqhQT0aEPvhjquXZiedLtiRvLoKRQ/hA+973ri7SpsLXaiYahrnfJaYLpKZi1JaAAFWymYwxl4nm2BnTMcf5gsZZwfFOHGZM+g7iqYsHNf5heWAWVQaTQxjjN6oHVYYY7GUHcdhBJtDyLAPM8fMxIK4kbKWlY2ciRUrErqYBoMAACAASURBVDGQMfl65iqL2ZvpQhbiRg609GFBWLKVogFROOMTWBoZZGkPGolXJ25CZ+uf5IWsVRhppTNwwhKcqE9o1h9C4hgbfYY467z7ls5E3IoViGQMAxMzkZu5opLmyDjsI7cWjTuQMxGvWPkbOGYBTtDXvOkcMt77CL9lL8XAoAWyYR6xrN+3BANHLoUeJmStirOuiWHCgo1KIhHLU1X+VxBwXQQ4/jR7m29wJQZFUZQ448vhIYdncyXWHMuLBybROmm9dRFuCTNH29wF27PGmnmZJU6otdHn3kib5tvWrVXO4mSRtNoPGQwGm+Njq/DRmEmv69YFKpWcBbzGsN439qrzXo3OrdggJoqnGGe/STdKDrc9jomJCWOcPYZ82RyqFVd55allAdMa5/jKvRvZY9euXV3A+VONHHbl7ozdHPL00w5/cFeeuLYeRhxYloCd3d5EXtERpBQlYPS8AizLK8LxLfOQEBaNfYZcvN/NHzE3zcDxvDwcXBuIiQ/1wZoTJmyM6YNxGwORk1eE84vCUZE78cJWPOofhpHbT8NYeh4zMBF9dBtrY0Bu25rQHzO6xKGIc5zfvhgxYRHIImG67ABiRo9Gp+VbsOb/8jFiyGjcnp6D0tI8xN0eA7+nl1aoXwZP+gqlRUcQuiwCyespVV8Jtmdkw9zDHzdtisA3hyzS+c6PJqJzkB8MWxPgH3YK208XofT8brSLGAbdxmop/urkWLmhIKAg0BoIMLBTjDH7yBAtxkZ9gp/JZJrPGR8nviVWsdFtMeacPBGti9ZH63TyVI0gv9lE9sn2AzjDcPtrq2lItSbnX0pe0mMANiYmJrq8YZ+7uxumvPEKNn6zEhlffIjB/hTtz1J8fDoiQXwL361ehq8/+w/87qTgJi5eOFYxxv7qSC6jo6MHM7B/mbk5xLaD5Uj6jqRFCmOnal/d3Nwed7TW2AaAwLlDH5yNblPORQhE5ItD4aPti8CRgfD7+/MY4KNF7/tGIBBFKMn/E8sptpBuFHr7+MB3+DSkBQLfZP+OXxMAXerrcv+uvqFI0/nJZhX6P/fIOUp/z1yFRamf4ShZ1SX8VKG9rc7nfVGnkfOcDzatWIK5H5D5nBfkjNNlRUDUFkSPGopO+hzkYB6iQgfAw8MHwXNLkfflc3CXxeMUjA/qCw9tX4xcHIjknXbKBI8BeDMKGLeGIiQdRVoyMD3UD3/uISftc/hxVRpSP9sG+oZNWH+0OmvKtYKAgoBrIfAr48zlttLnzJlzmnOeyN15msvsDDroudF6aF20Plqng8g6hUxBZtIL9oS9Bkf4+9wbOdO+rSXqjLFHwV0/0cn9Q+7BN6s+xqC7BiDkuZfwxao1+FfSXLw3NxovPP80vlu9HB6e7gj86zNY8833+Dg1CW9Pn4R2nhSp0EWLGesZZw5TQIpR4u2CIHzFwf8eFxe330VXXcFWS8Q59q+YzfGVQY4n2XSKsiBqjfUDg2zEC5is5xrWvGq4eQElxZZ47aVVrKnsY5uPx6B+/dCnTx8MDEvH8uWDYSRzhxrFhG9jesJv/DLkdbgdL0WSMzo5UANGAzB+yI12I+zMiAx5OHXqnOz2Lfe19jKWAX7uVS1hBr+QBsz4BlmZ32CZ32I80MMaPTk8CH5WHsPSl2P5Q+0V0wo7tJWqgoCrISBB2gMBLiccE06CIFDSDQMkOS20q0HXdH64/DVbYl1fvXScaeZY78R13CzamZtlhrTb/rZXwKRbna1RZpwNhoQd9vO6Ur2Tjzfmz5mFpYvfx6eff4Xwv0/CkaPH5fpfRobDzd0Nr45/EZOnxyJyxmycOXse//n4MwSHjUPfPjdi7def4JGh9j6orrM6ca64nzN+gyN8E2Q/Ajfs55zH63S6DNdZZf2cyJpjZ8U5liTJadtjnPPe9S/Nhe526IwgiumTtlGOCJGbtQyjM4ARAbfhzigg4b1PZXti04VMLImBbHPs2Z2gSwX6BCA4OBj9yn/A6JV56FRrRg89DiUDUUnxGBs8FDj0kzXMJmHghb7XWXxvPDtTrPsIfJlFTtYGbF0wAn7h62CTt21nC3I2wd5y5eH7BOb5xcB/SASikp6U86p27zcQWHYMPR8YjuDgQJT/MBorz7aTXchdCH2FFQUBBQE7BASTsKe1NMdXEvzIFreoqOgpDu4rxojnrdEg7Lhvc1Umxoj/4eAXDxw4EFyfrbHrrizdXLQj5Tt7/gSuut8nIJIyCDqlWHcOfC8VXcp2ygTNJBo68gmsW23xJyu+XIIt27ZXoXgpLx9frv4WJ0/l4ufMKu8V+PP0Gbz06lS8tzAV7+iikJI4G106U7ZxlyqccbbH7G2utA9pAntRUVFd4YFtkPB3nU5nS9veBEotP0QWjuuzA2sOS5fy8uzyHTeHUs2xZWVlFMDdRUpPtLMqWqvqW23s9YbuyFp4RQyDF2Po6T8Ok5fvxqi+HRAUcxC6ogj08WTQdBsiB8v0cgfUPYJxMEOHsP6dZIe3/mHkHPekVfD0gqbKRD54NG08EoZ1k/uGr6SQoBmISqvqEK3uHYqc5VEY50/9PPHQDD9sWPMi3EkODvSS01HZOO7ckWLeU7HN1RVPR1N0l3A8f7/FXLHHcB0ydMfg50XOh14IOzwP7zznax2nnBQEFARcEYH9h/cfBHDj1KlTa09S0cpMJyUllZ45e+Z6zvjB2NjYr8RI0eUkh4ZARHwT/5zxW9kFdlN6ejrFZr5icdbv8RUnbkSHgh1JH+fryz6xH+IdEPkm/MY45G+Kl/JenPFLCxcudHREEnuWm1S/y+8ORL35GibPEDH17XgYbdvGjaS25psfMHxkONRqNeJjKG+MaxXO+DEVVLawhY1mbubMmde7ubltlCB9KMaJHzWaQCsOIIVxrXpIR/FkNBrr9Mht7hxGo7FKiJzm0mvyeI9B2Mg/rRg+YNJGVLzqyvesTnTa4VjNOQx6PeBZmd4ZHr6I3sgxrXo7AN/gaHDjNOhLjfCsSANN81WNxkOTDxq7BPy5FOihtqSTNuhhUNM8q2FvfzJg1FzwZ2Kq0uw9CdzO12/QpI2wXdrP1Tc0CbwK6loER6+GcZoepUaNkoq64q9AqSgIuC4CJKTdHnv7vg7uHUgr9HNLctpQwS81NdUohomPoT/iuRffHxMT846QL6S5uiMPYUlb0ZKPNI4z/hbj7FMcxNUZiuzAomL7vx3GccbL06dHEXDY2k4xoKv8Ytj6ewdM2crAP8vPTCIHmRpFcpNuUHEVaXlcrmg0apw9dwE/b7eF3a5k8d577sLlyyXYf5BCXNdengoegakRE7D15x2YET0Xa3/YhKeerO4DWfvYlmxlnB2TmNSkqDaUiVOlUm3knKfqdLqEluTbEXPR95RTheNivb7Kh8cRTNto6IuLq1oB2G447+wQ50UPbe1Bg+tqh9qj4UKnh4ds7iBD4KGt27yhMTQbgKfaQwutnSlzA4YoXRQEFARaEQHG2QaoQE7NLSocN2bJ1ti206Ojoz8VBGEWfBAXGxv7C5PYT+D4FRKOw4jcQlNhYVJSEoXSqVUQa8ycjejLIiMjPTw9Pb3c4NYTAvqA4U4u8AfAcR9jbK0kSY/HxcVVT2HdiCnaVtf8HUmUoKuy3B6m8dbe8HNB5vv3VDYC2nsmP8AYvwscgzoGRN5cmJlEatMqz44x1oUzToFGXb5s2ZaJklKL/9Dfhj+K3DNn6xWOX33lBaQs+hAzp76Bhx8cAg8Pd2Ttcb0/EwlSPmPM3mGpQc8iOjr6AUEQVlJ8ap1Ol9agQS7YSRaOyQ6soW/0jVnDhQsXcvvddptT4pbknT/vNMG72hpZx8GRYYLAYi0ubtXuKpcKAgoCCgJtDAEzN68VBCEZwCxXZ90qYD5LZiAdPDo8DAH3c4GPgQp9mJpd5wUvLzFWbA9uiUrJGSdDMQEcHEzOYNe8JVbSkRi3xuxncOecX2ZgRZzxc5QFj3G2n5lZSrGh+Kn58+dTuukmFWf9HjeJmeYMOpBebvZ/PdyehPaeiNtUKjaRc3gyxlQCx0TvIZEDC4rK/g9VNdHejLOKyKb2NFytPm1WfKNY2vJTJmZNn4Tcs+cxebqIB4bcg/sDnBm7oFHsVXQWuHCJM26f0r3iXl0VcZY4EgypEpfG6HS6KnbqdY1x1XZZOLbGOZ7taCZPnz6dU3L58rB27R1ihlTBHpcknDx50tkxw1jHgMlPC0yIBXBnxeRKRUFAQUBBoI0joNqq+oU/xPuKU8Vu4nzZ8a1FVtQcwc8qcH4LgI4aZfz48Zoe5T1UzI0JxV7FrENRB07nGh0b2WCjQ2eUQ8p1yzWT2UcjyVyT3fW7Pzhkv3DOTGow9zBmSbNNybfbM84e9PZy38cHRw4r3Jn0h7V/B864RR1rT6CV64wxDBk8CJ07+8C3X18c+r1SDPHu6IVb+vbB6dzKjMhuVZ2DZO517yRj+Rdf41TuGZSVlePmPjegV8/r0eP665B7ht6zXKRwlDDOat/qrotFAfPBkaiLa9uCMX1POdWsgklS1u+//4677nZs1KCTJ0+ipKSkqntoXQ+r8e3Me8jk/2OglNdsQOOHKyMUBBQEFARcGwFxs2gSHxLXw1NO+FDFscq1Oa+bO6vA2uaFVmfs4taNWsveUTHNdYzzMjDmXjEzky0AbxBUbK/3kMnBBduTNwNyW305YSuGt1SFkneIM6fA09MDGf/7Hp9/vAhfr1mHhYvT8MRfgjDp1b9j+84sfP3ftTJLFNbNS9sBS5ckIVqXKEepsPF69Jg1Qy6AzVt/Adkqf/Plx/jw48+RmrYC5eVV4rvahrX0mfBvnMHkSdxZdn1ZbdrQOu3PW3pRDZnP6XGOS8vLvz965IipqMhxBgkmoxH7cnJg5rxW7UFDFl5XH+97J4/0DojcwyB8WU0w1ksS5tQ1TmlXEFAQUBBoawhIkNZygT/RknxfzYJfS+LYVudighDJwTpU55+BUc5lLYOwqePdr99jNV9xGeGYQreRkEvOc0+GjUXCex/giadfRLeuXfDzhtUIe+pJvP7mLLzxZjTOX7BYg5BmmPocPvIHMlam4eUXn8e48GcQ+ND9VZZ/uaQUsXPex6hxb+CBIYPx7VefQK2uPQV1lYHOvyjnjDeKEXGpaEhISKhhDiPGiPvEWWJoWwrN6NQ4x/Pnzz9vlqQ1P2/bBqMD3oQ458jcvh2lJSV/zJ07d4Oj/ja8h0QEew+ZksUE4WvGmF1cP14scSnBrOd9Cne87/K2eY7CQ6GjIKAgcPUjYDAYVgJ4nEwrrv7VKit0CQTMPAsMuzn4j+D4jnOsloAVHPxDzvkCznkK16g6N1Yoc/ba+t1yEz797Ev8O20Feva4Xp6OzCdem/I2Cov0iJgWg8yde+T2nj26V7BTajBgzrsLMW5CJMaNeQZ//UsQ5sXNhP9dNTelD/52GOEvT8KNvXrAw6NxCtuKCR1bqbSxbyZdCdKbXMVnxMbGZlG2vGaSa5HhslmFM9/mJUmaXVRUFPLLzz+r7n/wQTmmX1NWRoJx9p49yM3NBTgn++hmh4nzuTfib5ypRAZUtYbnuAzOPzCh/F39jn/avwVx9ZeGZtuwNWX9V8GYKt7IV8F6lCUoCLRpBBITE/WxsbGrpXbSGACUmU4pLoBAc+yyXYD9elko2JlEPjx01F+Gx9zLOHMpExk3Nw0mv/YyxoaHYczLEdi3v4o5tbwerbYDVn/+IX7YuAWJSf9CQaFl1zz714PI3ncAly+XwsfHG97eHetfv2vcNVEmEEewYnXO+y4mJiaEqRlFyTngCLrOouH0OMfE+Ny5c3NmzZo199y5c9Eb16/HA0OHon0jHfQoyPaO7dttgvE3cXPnVgYWbgI6XkMmjxAgkIA9uMqT57wEwD+N5aZ3i/cspDRyVYo51NMh4dyqEFUuFAQUBBQEWgsBEz5kKpbaUsLx1Sz4tdYjvErnFRwlmDkKn5fGPIusvb8iNHyCnCaa6LZv5wmNWo0OHSxmthS99m+hL+LtaZPwXcZyxM55D+t+IBNqIGH+B4h8Yzz+++0P2PgjZbEFRj0zEnfefpuskT524k+5zZX+Y1w2d3EYS3Wkj3Y5e2RSGLeIsBcfHx8Lzr8uLCzE9+vWYf++fQ3KKiNJEv44ehTrvv1WFow5sL+gqGhU9ZiIDX1yXvdG/MUnIHK7CsK3DBhcOY6Xcs7fM5aW3pSfmTS9NsG4sq9SUxBQEFAQuDoQmD1n9jYGdlGcKdrnCro6FtdGV+HMndw2ColLsL36f9/h+XGvyYKxIAh4LjQE6//3Oc6cPY+0f72Hp0NIIQqcO38Rk6bF4J9LloIEaluhtNFT3potxzimnfC7B96JNyaOA9FKmP2Wrds1d46Ojh4UFRXV2dUW7tQ4x3aL5X+ePj2qV69e/zaZTOEHDhzA4cOH0bNXL1x33XXo6O0t29jQ6wN5aZID3/nz53Hqzz9Rag2uDeCXsrKypxYuXNho776OARGPyZpixu6z44lk7FIOLDZyad7lzBQXiqFSlUvlSkFAQUBBwFkIiDrxIWfRrk5XEfyqI6Jc14aAwAUuQWq26WRttJvaRkKwraQunIce3a/DWzEJ+HHbdtw/5B7MiZmOvw0fhhkxc2WnvJOncgFWZW/aNlw+m81mqFQqtPP0xO2+/UCZ80ir7CrFzMxcYILTzSE55wVuGred0dHRIa6UNMepcY7tH/LSpUvJ83TMrFmztjNAZzQaOx0/dgx0XKGUc2Dhb7/9NjM9Pb1R8U06Dp40TFCpZgPsgWpzGCitYbmJv1OyO/lMtXvKpYKAgoCCgIKAgoCCQCshIDGJMcZaZGe7KUuMm5eCP0/lgna3qVAqaYpMQULz/z05Aks+WgY3N7d6Sef8ehBzEhfgzjt8Ic59H5Q5L/y5p+od0wo3nf4M4uPjj8bExEQLgrBenCU+L8aLG1thnVWmJPMvpy+8yowA4uPj/6kvLu7LgTgAJ6vft11zzvO5JC0xmky3xcfHT22MYKwdPOl+74DILYJKvd5eMObgZRz8g7Ky8r4FmUkRimBsQ1s5KwgoCCgIKAi4AgL0w+wKfCg8VCJw+sw5PBcaLB+UCOTEyVNyGLekebF49ukn5Y6UQvqP4ydBjnsvj30eifEzKwnUUcv45ntZQP5qzVqMeSUCXTp3QvHlEpSXu5QvYh3cO65Zp9MtlyQpGAI+F2eKrZ50jeIcOzUJSF3QJScnFwCIoWPW9Om3QaUaAJXqOkr5yTnP45wfPHz4cHZ6erq5Lhr1tQtM1Z8xNtTWh3OUc/D/lBswtzQ76bStXTm3PAKqVaX0ql33XlPLs6TMqCDgSAS44rhbN5yKQ17d2Ch3XBeBj5en449jJzBz2ht4Pmwktv28A6OeHYlv1m3A9xu3VDDu7u6Gf7w8Br8dPoq0T7/AA/fdW3GvrgqZVrw4KhSvjX8Rm7dtl6NcuEgSkLpYdkp7XFxcphgtToUG+2JiYoJ0Ot0mp0zUQKKycExhKxrYv1Hddu3a1U6j0TzEGLuHA7cA6AbGPBjnDdJYR0dHW+ZjjOxeyLwin3F+XJKkPYyxTQMHDqw0ArLjrHDHno+9A+6eCbBejPGPyqSyuaU7F7meK6gdz9dQlZmedokYjtcQ5MpSWwqBthzqMTo6eihj7BGdThffVKfnlsL5ap5Hsct2zae79ecdcgKQSf94Cc889STC/z4J+w/+LjN7XbcuiHrzdTxw32DEJy7AZ+kZePjBIQ0Sjh8NfBBjw5/B+DdmYPfefa65+BbiSowTP4mJiTnNGFsZHR39aFxcXHYLTV1jGlk4dvSHcfe+fX4aziPAGEWWkCWhKqrCeozUa3Bo1yDTYAyCSk7awnNyctabGUu5e8AAypZnZzi+2WQ23/WCmpv/zN+9oE7TDTvSSlVBQEFAQeCaRiAuLm5rbGysKEaLpWKcON8ZYDj6t8YZPCo0FQTqQsBkMoOE5GEPP1ghGP/9hefw2oSxWPfDJjweMgr5+YW1Dr+uW1dEvv4yTp0+g3+mfgyKWEFFo9HgyNFj17xgbANNp9NtEGeJ4wSV8D9xmjhYfFc8a7vXUmdSGDdIg9tQhvbs2eOdk5PzqQbIBmMv2QTjho5vZD8Gxh5TAf/LycnZkZWVVSXrin5X8k+KYNxIRJXuCgIKAtc0Amaz+WUu8KhZs2b1vaaBUBavINBABLy8tHjpH29ipjivhmB8U+8bsPC9OLz/Tqwc7k3boQNGPjkcQQ9XjxHQwMmukW5ivPg/xtmnvB3/WBQdK6c2BEJ6iXeYcLx37957BZXqVzAW3pDJHdqHsXvUGk1WTk7OeIfSVYgpCCgIKAhcQwjEx8cfY2b2ilpQ/+MaWrZLLbWtO+R1DIh83PveKS/g9lc7uBSwTmIm6YN/Y2/O/lqpF+n12L4jC/t+PYhTuWdw4w094O2lRX4BuV0ppT4ERJ34NgA1N/Op9fVz1j1ZOG7uhzEnJ8dfUKk2MMZ6OovRBtB1B2NL9ubkvN6AvkoXBQEFAQUBBYFaEBDjxa9EneiUH6Tm/tbUwq7S5GIIMMZ8mYCPfbQeZ70DJn9MIVXJ2d7F2GwyO5JZQpcuneTYxPZEHnogABNeqqobvJRXgOVffI20ZSvx2pRZcj1ihihn2qOxZFIRcM/dkKwmFvb0lDo47WRBwAzxLfHmlsZD/oOlsBVNnTgnJ8eHAxkAXOItUWAsJScnp8WC2jcVN2WcgoCCgIKAgoCCQHUE2rpdNpO4JfgvQ3vGhBcopKpPQOQJn4DIhA5DJvevvt62dr0nZ78ciWL5Rwswc+rr6HNjL6QkzkZi/Ns4d/5CncsxGo34fNUa/PTLTrnPYP+B+G96Gnz79cW89xfVOe5aviHvZElsNtywuCVxoJf4Zr/NccbmtLLGuDpmlNLl35s2bWqVMHXVmVGuG4GAQY/8C/nQG0yNGOS8riZ9Pi7I/DhvjgZTdjFsGsy30rHNIzBz5kwKs+mQ0tYFP4eAcLUTYaiZ2Y6xXmDsLQ2EA94BkTt97o18Xes/vktbhIISfyz+cBmCn3kJt9zcB9+vWYHeN/bEX0JGg1JMU6EEIBSt4oae1+MuvzuqLNO7oxcSxLewODkBn372JZ554R84fPSKydCq0LiWLnLP5f4LwM3iLDGopdZNCuNmCcc7duzoDs5fbimGGzoPY6yfT5cuzzW0v9KvtRHQY+OCMWCeXujUrRO8PDUYGbcOepktA7JWrUBmLiVYbH7JzVyKIEa+nAxj4lah1vd8Uy6WRgZB49UJ3WR+GMYk2vhpHg+5mWuwKjO3EUTqw6YRZJzWVY8lAxkSs/KbPcO+JWPk50LPpvIIQqblD6FB9LMWBCFoQVaD+lbvpM9KBGOJqG8ljX9+1Wdpe9dqtXq8KwTmb3vIXaMcs/ptBCi0KwS2UKVun+sdELnae8jkp3B7WP3p5FwQSsqQ99KrUzHlrdlyQpB3dFHofl03PDL0Pnz71Sfw7XcLPv3sK3y0aD5mv/0mtNoOCHnicaxbvQwdtO0x4qkxspmFLWqFCy7RJVhKTU01SpCmQUBiSzLUrDjHHh4ez4MxTUsy3NC5GOdjASxraH+lX+shoN+XhmERp7HheBGCemuRf3QdHrllBP71RB6mD/LEtrDRKNxdhIAezYyPrM/EiCHjELo2B1/4GZHU0x/PdsvBxgkD7BZvwprXemJcahR2n/8Wg7p6yPyMu2UExnQ7iNVjfe36Nr566ccQhJXtBg/o0aDB9WPj0yAazu5U5rAJ9IDfPBzZ/A90Mhlg2T9Qw0fb8AnauQOXao+kdEUinjc9je271agP1cY+vytO2gY6CCYhA2qsj4oSH0lIEH8HHhHg34+h0F2ASc/Q2U1AqVqA2cB8OqgFbm7HJCMTePtyxo2CwN0EAZKRcZMgcEnD4FYmeEhqxjV0bWIwM4GrmcC52VLXqOUzuJlpVEzgEhOozlV0VjEuQYBaYpZ2iakFWPpAYBCIjsTAmQBBIjFNoDZwgam4WT5zAQI4Z3SHc4FB7mOW6yqqc84sfYieJNB9zimGKNGx0BYEbhnHGeNyu5Umg8BhvScQMxDAJIGDWftzevMTAGbljVv7EG1OAf2FPl019x2/UL6D+sh9aX5aEHFMfIMJnN4eiTbNJbfLNr3UKoB4JRtfxi1zyHVY+xNNCEymQ30s7TIaMo9yDgJ5DqJtoWehXdnHQtsyv/weK2Nk4Zfmh3dD0jwxi+wQArAQb69eeTwg8nPOpE+KtqdktoGPRQWL/1u3AVt+ysTUiIlY/98V0BdfxrvJi0FZ76jQOTZqMn5e/zXIBjkq9h1s2vJzxXilcmUEdDrd6tjY2LkxMTGBLZUcpLlxjh+98rJapwdlyNu0aZNHYGCgY1SOrbOMa2NWIz2igejRySIF+fQdji8y0vCr8SJWTAhFBKHg74W89CNICvXCqpmvICyBzNz9kLJhDSYFdcKKyDFYWaLFsdRlyAEwefEWJEwYagmybUVRf/wgOoenYdrwAXJ7iM4PCedLqmBsOvEtQlKB9OM6DOpqscwhfhZtScOHBy7AYCiBLmI9Bg06g7CJ2dhe9AX+TKjOT2/oD63Bq8+GYBkxgxCkH0zD3b+/C78Z8mIwMC8dmUkjcWBVAvzDKFkkMD5lA1ImBVXhGXViUyqPWZM4CSEzLO+AIVHpSJsbCh+YkLl0GoaMSwYQiMmTewIDpyNp7ABcyFqFV/zDZCcBv/GLsSZlAnpjH2a+8hm69zuAiJgMICQKy5/xxrzRM2Qso9JzMDd0QK1rCvW1WS/R2YCNia9g2M8DcHzVdLTLqTlXp8NLMXWtGwadGY2JG1OQlz3JThgtkvME9fbRQg17iViPFZExuOjriYiJCXbPvTdgOoGl08ZhXPImIDAEgZs2oXMKsbIPkQHhOObXGRnLKNFS4P+zdy3wUVRX/39ndpNsyOYFAQwoKIggkigRgq2gICrRV5cFTgAAIABJREFUmqAFbRFaqF/BJxBUFDTJZHkZagXUWqBVaAVUwEpoFa28UQtiUIICylsEeSYhm7Cb3Z17v9+Z3U12QxLy2M0D5v5YZubOveec+5/Nzpkz54H5mxdhXP9OAE5j5bRJGJ7pxs37XcHJbZi1AHh7QSz+XQW/e0sWVLp+w/yvlXZFgOjk9JVuhcWrpFShjGiKj1fZIeWDFCaPMuKjpAhSpDQFy6NwkQZCCpA2pkJJIaXKq+h4lRTSEN1zKxQvbXa5UuXuZ+65bqWK1EKv4uZRvOauOycx4grsiek3ybNKANpL8Vj3sWYicT+8krlE1nZDgVDPcMpNr6Wnp+MwbZ8IMimEuLsbM1Tsa2Pd3y2m7dPyPLU1tfGyZ6x7snuMhwxBqTW5ohanxswtgJuO9j+h6BnjGes+WcFLg1pD1EOc4KFd0h8rNu597YR2GdxjvLR9ZNcYekj50tFIkqYKHDnjIp34F+5RXh5u2m6BK0Tw7JVvtDmeodTpw67SmPJBWn/5UVUTvKy9AmnHnvX79XkOyol5T158y8BiGcPjgPx4THL6/06c41uviJbr8M7o4jyCOaLYWoLM6S9jybv/ws8nT8FqLSlnd+LkKTw2cSp633gD9n5/AFRmWm91RkAwznIggQKFg145j/Ice+9sdZaUJqic30ylD5tpC4ls06YbgF3AcAk4zXB9nARHe1Zu7XDaGMqMEqJbsegyWRJqGRM8jImwUknwUFatxYOrTBjpxmJkQoUEo8qESsfUX8nqIWQG7pQEbckKolkepIp9Dskge6wgZPmQhdsyQuPIIkH3VbKUyILJtNVueJLHslHJ+kHWAA8tmX4WGRklvFYU941Ss4CUWxskJmmWEbcVQrN4SB5LCtGqygpC/ZLXIuG1bJA1w2td8VhChHaPqLjReiwtVf3ama9NhWVgD/SInAsMHImch9KQ+sBIDIszwHX9EmxfmABTbj7S+3fAlmnJGH40HcdsK2HctxxtE1Jxw6nPYN2Zi9yzU5B/qhjtT2zA4IQB+NO1x5AxqMJCa+41GuvfBuxHPsa4MSlYuIGU4ES/r6+t+DAAC27r5P+nEd9/NDKoILl9B3YvfA6zBlqwOX8RSuYOrkKetfi8RxrarNkPMaQTti0Yhn4PLUfBZ09ixdhZGN46F6vT+6NwCynGP2HrsWIkGvdhStskWG44hpm+MteATeGWqUh77krsLxboZN+GYW374e3fFmPU+b+i35idyN1TgIGR3+LxDgNwbF46cHoLBicNR/rWY1iZaMTyCW3R2XItRGY0di+ZhaPzN6Og+M94O60rHs6ZgvyCYkR++1d0HpCB39r+hlVVrKl4J9X5IXXnFD6e+gBSPrwXez6bjE6FW5BYBa/i35Zh4XNjMNCyFPmrU3wUY6ISCeT/E89O/QEmjaoNtui7YZn8S1h3zsWEQ/Owv6AY+Houut6xAGliJg5kdsaY9RbkF+Si/clPMLhHLlprc504lJ+PQ/euwKniXJzYMBcJA1Jx7ak8GOcPxvDMQdh6rADXFG/BQz0GYMIVh/FyxzPIXQhgAarkd2+x//Xz/4ZoTLX/GGO/du9U9PlpKb56RVXKCE3z9Hv1EH8NxkPXe9KPng9PUrW8iopmMKw0z4ePdsZLr5y9Z7KXhi9pfV9HIIAICLpRMfFfrmLRuUJXbvso6fmWmOHih/0Hq0VlxzffVntOP3FxBM4WnX0nNjZ2zgsvvNBhxowZxy4+o/4jKDaiut/3WlFljHnMBrUa3miD9h4+g3Ez/k38dlZYOq704e+1dkQA4Z7uMLqPuG/Jfok3qrR4GMrvOW5riAHQkPQ8KGgbN7TaPU4yVoyvMJ+4jUWakUNCheWDaTakChMKPBYXzb5Tvgb3fcx919J4uDvcg72GE83o4G/pcN9zPXc7beM5r91HffrLb86+N1jvXbOyRcI9r5w2HXpIkcB+/eUr8Nkxd0fGeoFHD+zCrq/z8O8lw9HjUWBK7n7MTO2M7gnAuY6dERfjxL9W5gNX/4CVCym6lwrn5GPtrjPodBawLHwWveLMQFwqFlmApLV7/ZRjL0dD5LUYOXkFws8Ox7S5/8Wv5qT6WADJ1GWD0zvYu3VZcdpqRJwJIPvmuvcz0D/GigUjq5IHmHJsP3Yf2IdlC97HmjdygdYpgDke1/UciAR7R3SKi8GOd+gBOBKbVi4CvUc8Q35Aaw/4KceoEZsp2J+/G/s2LMP7G9Zo1uBBThcObVqKhJxFSO1ODgL9MX3FSKQedMJ69GvNEvzDtpV4YxtwjozmCz/H6cx7UYyByPx9f8SEAQOHDkQCfote5M9wSwoG4mOcd8ZhYhVrIteHyKuBMUlUHR5YcWwnupsB676qeZ35bZT28PF+xohKirEX6GvRp98ARMBBFeOBkA4go2TZWSDn5THoQjINGIqBGIkT1in4dhZg2fqkW9aYYVhkScAYz7uiYiRg4ZRhcH8lJiIHmVi76yjaZ+ZjyuaNSI6PAeJT8cb8NPT4cCdyfJJAVsXvDGb6XT+vxJfLVgjtiZkcCehHQXAIJwMTEIJDC8KifXDq0I7piPq1J3Tm3qdzgnEwokaBW55+okF+ABodGgOuGbC1fve+1uehrXFCRb/Q9j3zqFqqxlcTlANa9gSyC9C+mzbx9fQz6tdkccstvGuisZAouEyTl2ljPLTJY4N4kOOCe64mL+1rmNA5TqYQgkkixY/KoXnGkDyamYKDu9dPQ8vXr9EkvkRbcDcmTHCNDyO7PwfnHqzcmNPlcNsyBFeJtiQ4VHLq8Mz38AHjhL7GSyU6KiMHFOrhgCqYZOA0BirNk4SLMk9IpLhKWp82n2i7JPc88vBQBXcSDcklmEtwSTY+IElsRm3+LgRwgIEvKoP6D9vW136qmON+m1ZxrO9d7gi89tprZVlZWcsNkmEUgJeCjYemwVHaivpEEUuS1CDlOliL8+h1wSKv0w0wAnsXj8MMpOPt0b0wqEsvDBo2GuMWD0WPtPfxrHgMvj6tZa2BkYN6o1u3EDgcnbFi6VJERLtwBECo0efrGJoAFPgLenzLYiw52Q+Th3VH/yFdcEvneTD22Aubj3JsDKfX+bPw1fGZSK0wOmPXG7ciYVU6bLmUiWgiungcU6uUx7wPszr8Eh+OzcG0kYORPm0ilmSUac9PFyjdIwchoVs3OBwOdF6xFCkRrUC6nde7uiZsfpP7MxLT1iNn0TQMHvckpsxdos31XzXgIj2zvI1F727dEOJwAJ1XYGlKBJye5CBOEi4M7gcDu0dSl3d7ALM6dK1yTWXFQFrOUgz9+WEMf3wxileRuz+1C3m57PlIGJvo5zThGQztsSMhBQ+kDilfv/uc932DR1BNJrd9mM7b/Nan+bF4SLaGpll7jghTm8NDwyf8x2g2A+cdHh9nz2BtcyG/C66f73DPPhcYxjRlhzxeBa9SGSEdkhQPUi40JcVHGSEFRlNSVAEVXFN0KisjTBWQab6sKSSM9p30eokUMKdgTvLIlYXdIDhzkIJjEIz2bWWCGTmHbBSSjXMmh4qiUJXDqnIYrAIFbTgMhQLhpRymGIG8Qg6sIAXRXesWgDJWCUd7kI29ryrUMdOmTfu8Chj0rssUgejk9J9rXLoQ5wXDCqhYVLR9zmbf71aN8/STlz0C7GeWjtbo2BhAaNqEJ89xdmMwbAwebvc4NyfNPqFZIzQLh2ZRKLd2VFgt6LHe/VSvPa27LRt+VgC3xUGb77Ys0JN2hYWE+c130xJuqwfFYnisABXWEz8rhOalQHM0K4SPhUHzifBYQbSbU83WD7fsnvley0yFFaVcXnekhkcmiWtGEF+MyKLC3BYJXysIhW5wzSLia+HxjGUeWb2WEJqv3fx9LSRkrcGcyt8BY8h5LHn4cQxPWoJ7esUD1gPYtCoXGDsSZo/KUnTqFFyuTuiWCEw4BLw6PhUx9r0YZ+qBW/acBq4GnrMswkPvjkcH2w4seC4fY3P9K+Aa8QOeG74Kt516F8kxNqxd8SaQ8Ljb6O8RKqzLfViaBqSlTEXe2inoHWfC8R3LkTAhHxPX3IUwHEXkwKvhfvdgqlqerzbhQ1ro9MkYFHcay9LnghxhNVXLfhb5RadgdbnQ3rOYDm+MRy+zFSvHRWL5LXvgdlRwC1QTNmV7lwJT5mDy6EEo3LUM5I1L7rbtE+5FfooFW0a+iz7YCeXhJWg9Lx2m9p0B8uDu/JJmVd67chx6LL8NtgfaVr4kFx7bz1S7JrKyDkp5EKN79MIqYwKeWdkff/lF1bxOZwKtu7Tzw9yf2TkcKyxEpMujmJL3scf3238cHZlwwxRgwp/fxrg+49HBug0LiD6BoNnkN2DWwi14d3x/WHe8r/mur+hxJSImAhOmvI1xuY+jnT0Psx9egpFLFYSiFllEfK6f2eDzMOZmqf1/btsr7/scXnK7ykKF3jmMzMzMTJOZvELJVFY5XI7smTNnnrzkFtsEC6qvsaoJRK2apWYl93l16B0lxP+4EG+dO1f8Hr5/y/vE6z2rb3UELoqA57fnh4sObOAA+hus+te9gYSbenq3q1pj48LROHjggHL//fdfMkp/U+MaCP4yZlygHHcZ8QbW/PQ4UhJ8CiyOzMGel4dpStStj49FUkpXbJ2Xh/WWPbCk9UCs57c3zbIGv+4eimXk67DhTXQ2aeF7SJi4FBtTKfiqosX1n4Jcyyj0a+txn0kYi3Wrf1/JimnGiHf348wfH0BSW1I33Y2CtixDSHE/qnW4VTcDhlQlT1IPYGwC7mjrFnJg2kBgwwS8vWMURg1OB5JSEPkhBaNZNHkSIj2LGZiDPfP8s2HUhE27fSVISLoDTItRG4iBpPo+s1zzwd087wkM6EDrTMDINC1WDob4VOzJtaBHD683VBpy99+HMOwD0AHhnrwzVaafibgG6VWu6X4t5qoMNsDQC3/bnIO2Ax7AQ6fyquQVeu6viPR9FeAFmKpFmROB/OfQNVaLWvScGYitxbkaD//I2kiEGw1IztwDyz09yq87TUrzBoEByJ0wAJ6vBCYuysOwTmHArD3I+U0PdI50f1cGTlyK90Z00aITkOCeTP9fyA/o6Hf9fIMJfRZymexaLJZcZaKySUSJF4xG424lQ5mDEryhzFEqvbOpAKTFK34VS9H3qkOAS1p0jHZa4AQY/ulyOhZZ817fW90UvV9HoDkhQAZj7a6sZClCyXbv10XA/F27yl+11WVeY43dv2+f5YEHHshqLH46n4sjIK+0CdevvU4Dlca77LBabYDJDHNYzc9tdqsVTqPJM86KVxMjgSXFGN/LCKvdUPN8lx12lwFhF+Hhslq1giQGcwzM1YjsXYG/PO5e6qO1EBuiBTNlYbiwuexW2JxGmGtiUh021G8j0iSgXVu7dfubePPA9Xh2NGXrcOHjdCOeuyIPOyf3djPX5jhhqkaeCyWs6KntmspnNIBXOY1a7PjKpQ2378AgkwXzbKvQC1bYDe7r4EvKbrfCCe93yPdM/fcN79uhDjN5nnbqT6elzVSeVzqLUJEB4AEAbzEHe1OZpeyuvA5dOa6MyKV3HJmc/hsJ7CHGxVuF279eA2z0vgaq9WLpe8IYk7Kzsxvl/k3vWbvd2HIK6+7buSXovzEZGRm3SZJkyc7Ovq3WF+4SGUg6sXavprQVwVjT93v3okPHjoiICHxl6WM//QSDwYB27dsHQ3SdZlMgYAiDmSLCatHCzGY/v1R7PlB2nn6DSbG+CAFDmKawXmQUDGZzrXPsVpaHaFOftxGt6pohrHYyV4kNYVZOOkxbu/mGG5E3oB9MY8iavAEbMBKbTyVUsNfmXAykiuG+e7VdU/mcBvAqp1GLHV+5tOFOJzYgF0XkJFzpu+IlFxbm/x3y9uvbuiOgvKRQmpdHpk6d+mKIIeQpESI+zcrK+pFx9h6cWF3iKjn58ssvl9YntqXu0ugzmhKB4m1z3gVAH73pCLRYBDTlOFg/WPk7d8IcGRkU5fjQwYMwmUy6ctxiv3qBFNyMxwoKAHNN5RsCya+Z04pJxiphw/Ejx1BsM6BD104wV2WybubLaJB45iQUFBT4PDg0iJo+uZYIzJw5k4KxpiqK8iLn/E5IGIZQzGgV2opeqSfVkow+TEeg0RH44RuKDWwZzTeuqmVIHFgplUzl1DnruU5z5swJStJoMhj7JP0KrPA6NR2BxkTAHBNz+SmANQIchvhOXdC9+2WoGGu4GBATE1OlG0uNsOknA4KAoijcYrF8IoRYD+A853xiQAhfBkTIpeAyWKa+RB2BhiBwKsoUdW1DCNQ0lwzGQVWOzxYUmI1G46mahKjvubbt2r0UHRv7Xn3n6/N0BHQEdAR0BIKHQFZWVjZjbDpzsoHTpk3bQpx0xS94eOuUdQQuFwQEE/u4gQdNOSYcNeU4WD9Yb7zxRgmjlF5BaJIkOWTGapN2NAjcdZI6AjoCOgI6AlUhoAxXQrKysqYxwe52OBx9lZmKXhqsKqCq6QuWm2M17PRuHYGWiAClc6MKyEFrmifipZbnOGho6YR1BHQEdAR0BKpFgAqEiCvEB1pFlx9x+6zFs/wy4umKX7XQ6Sf8EeBaNn3/Pv2oERGQuSwEVUNshi07O5vyfQYtYwcZjC+3MJ1meJl1kXQEdAR0BFo+AspEJRqR+IgJtlexKI/olc9a/jVtwhVIQqv+3YQSXO6sGaRm/oASFK8EuuxkML5klePdB0/j4I9F3moHl/vXvDmtXxjetwftia85LVSX5bJEIGg/2M0ZzfT0dJOIFORX/N/s7Oynm7OszV02slrpFvYar1IvACcAnK5xVCOfdBXuxfK3V+DbE0C3fnch7Z5kxARCw3Kdxra1eUDnZCR398vIFEcFUeEuX1TdagmrXdWd1PurR0C7dMHKc1w92+Cecbk4Zr61GT+dKn4suu8kB1NLLIV5C88Fl6tOvTYIqMNMQQ0CrY0M+hgdAR2BwCJgNptTwfFO9rTsmTVR1hW/mtC59M9F95vYWXCpj5B4iaSKUsFZCZedJaqTlZwPYSXY9lpJVK/HogBU51ZB9498D1LNx8hi3YZhsf1gtizC8H4h2JDRD2Pmzkfx+nGVKrDW4xrbjmJKSgo2DFwE2/rRvvn9vckOqsPBi1V152sURmWqkJh0WT7sEzCaonKpPaV+sGEPjp4shhCQmYRJwhCxPyp54lhguFzjt0E/qSOgI6AjoCNQZwQ45/kXU4zrTPQynXCp3Y/9LyPbwCTxjiTYMsjyKmaQ1hmk0O2hIcZ90TCejuk3SWWhob+iOYKJqu7XgwDs9ND0mlFJ+dsE4N8eV54HPcek2A32jO0MoNBzfpmnb7Tn+NDQQYOQvpgMrFZ8PHsUKI8wSxyF1XtpCrB32VSMSp+KoYnu/vVH/FzpYd39X+QiB69mjEZq6gjM2XkMiwYCp+yA6/QOTB2aqNFMHJqOLcfdc/euno1BjGHQqHSMGzUOqw/Q4ONYlj5UGzto1GzssgIwGhFJQmyYg+1ucejIu3YqvkONKt5/41kPbcM9/UWe7c0ADnn2KZCNsPHFhzB72dOn6YWMMTKeNkufY8866qX0e+bWuCGDcYOI79ixo8anit69e7O1a9cei42Nja9RknqcPHjwYJYsy9d06tTp95Wn7z9agL+s/Ar5+7wPVt4RYhdXMenc9jlrvT36VkdAR0BHQEegZSGgTFbi1VC1kyzkdmCIJmWAM26UhGTkjLNg+EoKRvFJkuCMOyUhUaak8xAoUpl6Ui6TjyizleMtC8XGlzYmeeITgJQDhlYXchclgiOv6MuvB2dl3DaZMRahWJSplcaRengNgP8DNBpU3Zf0GFLi+gOgm/73nv2fPAohnSddhcqEkgZM+1d7xpIcnQBst6w7ht8VZKPz8NbYU5wJ4+cWdE05ij3Ot+F8cxASHk1E3qmpKJo/GHfYXoeYSew8zboDoyKTsATA2CnzcG/KQNzVv5dm5d02jWFs6GbkTb4Fm6cZkR6ah7yHDsLYeThW7DmGmw4vQdeU5zAvrwA3rIzFHchFgWUgvn7lVtzxxXOwvdsdvzE9g3YjzwJp/8KCYV1Iec72cB4K4EaPYlsCgPAgJZf2LQB2AKAxpBiTm+l5AKSdt/Vsiz39vwXwFw8umsKdmZl5J2NscnZ29p3eZTaXbUZGRn9JkmZmZ2f7XITAStcgj5gP/vWvi0qzcePGHQwIuHIsVHUHk6QTO/LyLlCOSagBnYCyYiln7wn1IcYYPTXS31AvScan0cmTVnGneLZ4x5z97n79fx0BHQEdAR2B5orACy+80EGW5QcYY0OYYLcIJspkIR8RkjgGoIgJViZBsoOBS5BcYO63ooFcDwPjYDBIkCQwhAkmyFoXLQu5gzCJTkqmEiqY+J8Q4mNVVf81Y8YMkk1vPgg4nY71BmOY8QKrnBDnucBfzn05h5RhLtgAJwPTLJg+08kaSg9C5HZB/sazPcqgl9xnPmN994kOKcHXA3jcZwwpivT5MsGyGRmDzFiQuBCWrcXobg4DhkzBvIRIbN3/NySci8SUzZnoHRcD+/BHgB7/Q+HM/uXmW5h7423nKTy5dhM2fZKLtAETgIQcHNs5GckTj2HhN3uxatmb+HQl0PoRYM/HbwCWrRjWPR7oPh7z8BzsOIOvZgEj5zmwbe3ncLQdBOT+BBu6oxgd8OjYh5Dy5Fr8ZZiZlkCFYkjB9eo/kwH0BEAWYAqGJfcmeghI9CjGqR7LOfkgU7vOsyXLMvkuEy1SNL2WaEhckoUsHJ5xzWrDGAsBEFTZNOX4UvUDG3y98bOtO88oMe2jnwbY8wCLoCvMGIZKRnZPTHL6PLhKZ+j+yM3qe68LoyOgI9CMEVAURQLHSxz8iMViIWtT0JqSrsSKSDENAg8xsNVc8IUOp+N3s2bNOhs0pvUkPGXKlNZGo/FWCVIak1lWVlbWe6yYZShzlIK6kLxU78eEgcEQ2h1MnADYVXQshOAMzCYYH3Fu27zVXpwkIZWBlbsGeLufoAchAKsAeGOIyPD1o49SR4qwV8HzKs00vxQA1Yee5tmSckWNrKzRazNILyxEWT4Q5XVIgEszNkeY3DZEk2eGU6uuEOZXffPA6mlYht8hI3UYkocMw+Q/KUg3dsXKXWPQ+a0OSFs/EvPSh+POR0Zin90Jp/0sEsoZeco1qAU4QK9A7MdxhLL4hnbH/BU3wqSJaUWHG4ZgSv4YrKKak24czngeFuiYjNYpANI9irGvZX4hAMKW8OjhmUtWdGovAKCS79T8XrVziZsYWFAVUA/fOm8kLoUKWdCDTdCadtXrm+dYcP7MRSVT1TcFYxsvOq6OA1xC7JaE+FESoloZaAwOL7YXHsaM8KSJb4UYMQsCv2NaQwjAnhXGVqOj+k164dzWo28BK9Q6iqEP1xHQEdARuLwQ4JgPIH7Pnj1T6rrwuih+GRkZtwhJvEs39jJH2bUvvfRShcdlXRk3wniPwp4LIPf555+PCQ0NtYhI8XVGRsZvpk2b9r9GEKFZsYjsmR4rRWBF0bY5d3gFK/pyzgdRfdO7MgnZEGSnYqddqnqn9at55Arh2+yCCaNvh8dSTK4B3u/BswBeA5DmoyTSFLIu+zavBfo2T+enHqsj+eFSZpVWcS57qR0xuGEKkL7qG4zv1R+uA+swIb818tpd/AW7syAPmWOAEbYMdAkDXIVnQJrrwNbA3rnAov1vY3QXYPW4NJztmY4rb7gX+XeswfHxvRB7YAMmAJgnX4VbEoCdN/8W4wbFoXDbq4idtQcjfkV6bDGc5k4YNg9IuuNRWsbvPJZh7zrJLcKLDfle04MAKcP0oDCOvD08VvMVHnze8Ewkdwv6zlbVQphgtqpONHUfl3hrBub1pw64OPQ7dfGrXgPb6TNn/rmG09qp6bNm0VNeMJs3crVGHufz5v58HhgdlTz+NSYMc8FwK01gYHEMWBid3PFJISamn/tyrvu5rEZq+kkdAR0BHYHLDwElQ3lGMJHMTrNfrlgRPGNCRkbGbTKTV3DBR1kslk9aGtIeRf4p5UXlE1mWv8jIyLhh2rRp39VmHS01IC+638Tbi5znv0TeQs2iJ4eVqsJh/EPlNTvKsCzMRC4R4r+FReeG4fu3KOyscitmgnmDzuhcB88Ar2JMh/SQRq4VpAT6Kkq++zSO/JEpiM8bI0X7X/vMKWVGEwbOy8P6Z/PQLzYJLNPNbcqKfPQOczvukg9NeUvwO0L30X/Dov8NRleTZyLIPWId7oqPQcy8NAzoyjCGtPiRacif8AyOFuci1zIKHRhze0IDaBMVg/v+lYs5XdtqC6ITK/Z8BjP2ARSS5wIS7iJXa3ppkv/fSusmtwrvGwpSjAmXOeXyut0myDL8JgCyJHuxeAdAde4/sYIJr4Xeh1Sz2I1mggXt7REZjLXXDkqWIpTshgXnNQu46iBEVL8JDzHIOcztjF8+U0B8IJyuZ8/lvUZvOPSmI6AjoCOgIwBg6tSp7QwGww3Sj9LnymLFP1w/gAhlZGT0lCX5W6hIU6Yr5a/aA8iiUUkpLyi/gAGfw4XblBkKKS6XSiOLbHk2g+jk9PeLzrpGYP9rZRdbYEzy+FsLt736uY+S5jclMzPzbgnSBMWi3ON3omEHpNF6ZSPdx6sgkntH+T7Fq50+bYUhLAYx5rrZD+2Fp1FgA8yxcSC3ZW+zFhbCZTIjJswAu9UOV8EXeGuDEWNG94fZdQDjjH/EH4rXI5nciV12nC60wRwX45u2zUtK29LLb78O9wEJS2+/aS1+16aasXTtyq9f5TGZmZnPSZCiqgiKrDy00Y+zsrJywFGYPS37pWAwJ51Ye91wqeU5rg1Y57bOe6/oREF3AZEBCIrs1BoDu58ZjLtj+k3KQfJTWgYV7zl9qyOgI6AjcLkiMHPmzJMWi2VdMBXjF154oZMkSWvB8ciloBjTd0WZoXzBBf+VMIj3lOcVT3B4y/4WRfebND4gBTkyAAAgAElEQVQyedIQ31UUbZvz69ooxjSncNurFDDno5D6UtJ8kSnzh+aX7H+mQUdexZiIVMsbCENcXFydFWMiGhYTh/h4f8WY+s0xMZpirI0xhyEiNhrbxwxAZOIgMGNXnJyXjSQtzo4cs4l/9Yox0aimkZO0d13VKr2euTS2xjESpCs5uNcfuRqWTdbdkYFRNpKgNe2xqKW+xmkwKocX24sOY3p40sQ3QwziJYCNqvBHxuRoZhwt+k584dyXc9+62BepLrLIK230pazqya8uZJrzWKEX+2jOl0eXTUegWSLADAbD34UQL2dPy6bf3EumWSyWD5UM5c8I0V5p3+2jxFywxrr4ZV8wORgdXZ8KjYyREoq3z9teTl5VvynmtqD5UctF8hERK64s53ep7VB2C2HDK8dPwmmMRXycVzNuVgvtCID8s5tju0qFeiRYgpHBuG7vDC4uCSXiHgCAkndXfiohnyHS9Mn1ZvHFSTXeCI8/8u+j+qa/xpiYC7BfEncGtGWS9Lfo5PQnAZZetO2VDQGSirl+7fPOJUBEmwsZvTx0c7kSuhw6As0LgZoUP+VF5V4B0VaSJF9fyea1gIZII+MVIcQoWqcyXflPQ0g15tzoNvJvwSW6n5crx0Xb5wXVPUR5TSnOysoqoTcJM2bMCJoS1Jg4XsgrDHHxlGK5eTbBRCJX+e5mKZ1AD9kmVw7iDJioZDD2RnEGiijlH7xYo5x6zbKd+3LOV4Vb59yqCvFbAZT/QTLGEhnDevKpikp6qkuzFF4XSkdAR0BHIMAIKM8rXZVM5fVnnnnGNzVUgLm4yQlZPC2EsCiKUtmwEhR+jU2U1kXrg4RJNfFuyje5Ub0ei4nuN+lvvvIVbT32dtGXr/zTt68x9plgO43M6M3LG1SW7jfGLef/oIIBYPLkyWTKbjN9+vRmVwuCMsGAQVZeVvxSzwUaE005pqf5QBNuyfSKt815V/NHFjwTQsuPqC2HMfaA5o/cN/0l3R+5JV9hXXYdAR2BiyGgjFbCRKj4kIOve/nllylPbINbdYrfc889R4aVm6UfpQ8bzKQZE6D1CSb6eNbb5JJG9X3yauD28jfIrjAWqnJeyaUleFlJagKAg28Ssmh21dlqkvlSORcWFkZlt2v0C2+qtYaEhPRlglG2kaA27Y+ivnmOAyAZXQBv3j0iR0m+vfn3vOSpHCIp75SehS4W5ffzpnOhVC/k6kD5TWgM1WSvyqXDS6v2W7c/8rTwm558MyTESBGRI8v9kRl7LhrGMaJv+tRzX85ZVIULSe356CN1BHQEdASaIwJXIZMJlpdtyf4g2OKFhobeygQLahaMo9MHdTCEhPwCQnQXkOIBQdZwh2AoZEIcUFXx9X6HI2+gspGClYLSKJgxKyvrM5PRRFUnmty1QpJCViOpW2/kuUsRlG5/4wQA+jR5k1RpnTCIfzS5IJehABKkO8GxrjkuXRJSHyGJchefYMhIBuNAu1XURU5KwO119qY64aT8UrWlBT5ESAkmpZcSWZNiTMov5fLz5jykWjZUHpGAonMBb+e/fv140bY5v+OcJwP4wsuA/JElif09Ojk9j/I7evv1rY6AjoCOQEtHgNwpBBPjYKvZBSBQ62SM9RVMbAsUPS+dn2emxB3PGTLl59kpuwwhoT8BbDmYZGEMjzKmBWA/IoE9w5j0V4NB3trdZCo4njPknz/PTvEWjPCSCthWCLEVEvpWRzBYb3JjkieNpBRqvnwLt77SC3kLPSXafM80/b4yQ/kKAqaMjIybml6ay0eCsWPHGgUTDzm5871muWqGvkKIL4MpGxmMm1I59hYQae2pj05WYFKQqTY4NaqBTpVuHgJwNYD7fOqBL/OM8W5oHqVdowwQQfFXo0jdwq2v/FLlfAQgqFyl1hhjNzJIG6KT01dG9Um/xtuvb3UEdAR0BFoqAiJUzGOcTVP+pATUilid4scEu45xtidQeO1TUiKP56S8LAw4whibCeCGctqSDEO7rgi9rj9Crk6CFEG3IE9jMJPSDGDjz7NTth576a5feE8FaisJiQKJugWKXnV0ovtN9Esbp0K4CotKqABGi2kMbJnMZKrwprdGQiC+XfyvAeyeMWPG0UZiWVc2fVVVDapyTAJpynET5Tne60FkB6BVqyMrMCnI3qo4VBqSGiUKJ+dw+niD5Cgjhm8qNPJLqqrKjodE4DbFX859p/AndBdCZFXyR/41k7E7Jjl9Fq77Q8DzslAS8cJCq1btPXCr0SnpCOgI6Aj4I5CZmTmQlFVswev+Z4J6RKmZAnIzPpZz150R4fieMTzNAFNlqUOvvhmtR/wZMalTEDtsGiLvHg/JXGWceLIkSZ8dn333G4eU2wOWXsizzmpz+Fbnl115HTUdR/Wd+H9MML8CGhRLU001uppINek5W5ntTwCGZmRkNEpgXpMuthkwpzgDMLzEXOy5ZiDOBSIok5WOggnRGIq7phwH4o+x0iq8iagrdfsdkn8xPcWSW8QWQAt8IzcK72sfb9lCSv9W7Pn4VmXyVY6pjHnjtZ/m2Iq2zbHY7eI6IcQSb3UdBhYKxp6Pjo7aF508kcpmNtgybz/wMUYxhsjYWMTGRsLIhmL1AfdzgOv4NixbuQ2+oDQUBPuuBWCDFjTOk0ZDhdXn6wjoCAQcAQaWw8FfUDYqAfe9re5eI5iIE0I02Ep9LCflOYlJVG66fVXASFHtEXXvMxBcRfHGv6Psx50Iu6YPjFdcV9VwssEwBumxUFPYF4eU26ukWc3Eartlp0yFFdpWO0A7UREkV/M4gILDY/qmT/Udd46pywu3zakcv+M7pEXs5+TknIPAJColnp6efsGDTotYREsS8irMF0x8SoVrmqPYPIynAFgfbNnIYNxg5a2SkF6FNqJSPx1quYM9AXR0TD+8N1JRGQB9AFCNb3KjIEW5PHoWACWijvX50DH9sATFfaIKuavtsu2cc6xo25xRXEU/CFGeEJ0x1o4x6c3ofpO+iu6T3gDfNSv+8UAKjuWsQ7FTQDgLsMZyCGldF2nKq+vsJjw8nFJLN7y5Tu/AgtnpMCU82nBiOgUdAR2BFolAZmZmCgMzWCyWRvU3ZIKZVVUlI0i928+zU3IkBirm5Gs48aMnhZgghbaC4+hOnN/+L5R+QbcdwBDn54HgN4cOGGM3hYWHffFTTgrdfxrWnCgWTFT7drFVn8fbx/Tr7YzoM/GBKhkljPJLqxflQpwA/F1Str3WICyr5NtEnco0hdwod0RGRs5vIhEuC7ZKpkLuR/1YAXu6uS5YgpTCOFsTbPnoIT7QyjEF1FF72bP1bkjZ9QbaHfB0kpX4kKfW+VcARngUZDrdDoBX0aY/chrr/ZAlOej+Jl7Ba7Mt/mrOl4Xb5vwCXDwMgfJXgwy4iclsY3Ry+gp3ypzaUPMd40JZPpDYPR5aiXdDDIZMfA+L5sej8OBqJCfQm4/nYEqcir124PSOlRjKmGbrSBy3AEfsgH3vMiSyQRg6NFHrZ4PSseV41bbm0Kg+WJST5iuANp9c8HY0itOKH+t6HyiKkBRFkRQhJCFEtTfKejPQJ+oIXKIIMMae5+D0KrtRm4CICD0Xaqsv0+MvDaHcwZMvNp/brFBLChDauTfMA/+ofWiO81htah2wq2WIjw+8NLg2+fyrF8UGOwT8FFzv4MjkSf3CjWEHBeAySKxKy29MeBu/4gfn8l47UPTlnKBnFPHK2BTbs4VnqXjYNVlZWZTdSm8BRiAzM/MRMExxqs47qABLgMkHhJxyu1a0bjDO4+OAELwIEU05ri5I4iJzqzq90tM5FsA3HhcJCrCjaFiyClNVHVJyqZH2T4/rpEiTwxe5U5DJnNoxj7JM+3Sh6Bz5HHnnjncPa17/F345Z1nhMXGd4FyBEOe90jHGhjHJuCcqeeLMuvkjx+CuXAvmpvUAY4kYlT4bK78BRo4bhquuGoglKyYCsCB/TTo6W7dgcNJwDN16DE7bKTyHR9HZsh5OpxX52IDrRy5BcfEx5A5cjwEd/gQqXO/bDHG9MXrcCIxMTQE2+JSgN7WHJScNV1Zr5/Cl0vT7v3t9besf23+cfKhNn1sPLfjvrY8uWNMjaewCY9NLpkugI9C8EcjIyKCMPPEWi+XdYEla3b2GrNXHXcd9fnhqL8Hx2Xf3ZxKbXZsZvOQMzv1nNoSzDK1uvh/Gdl1w/tt1cJ6sZa0DxnqGSwa/Ihm14es3ZjfsYLjgNymqb/qjMsQ6F4eJAQbG0Cqqz6THYpLS/fxtC4Xrej96l8HBa6+9Vvbzzz8PAjBIyVT+S8aPy2DZjbJEJUP5jQRphsPpiJ8xYwbpXs2yqf3VXwomvlf+rDSKG632BfPkOQ4EIOTqQC4QlD3C60vsfT1HOR19U55RbmIaRyZ8qnRC7hSkQJOLBTW6SN59OpfvOU/WgX97xng3AUlQ7yXWoC35I385N9tuRzcBLPX1R5aYNCUmOvqHutDvnpoBUXwK+Vvn4I4rfsbwAT1gTJyKAy4zrr2OknhEoXN8HJxHv9YA+mHbSryx8B0cINV81uc4A7rf5GDKsF4wm+OR+uxCAJnYe7pqKWxO//tTWKdByJg8THt6qXpGM+uVXDeVOthfrS6+9LyTLyuyu569pnunFqLaNzMsdXEuKwS0rAAcpPjVJmYksNgwGAsLC+vsKqcFyglG1dsov32tGlmPi9cvQNFHf0bRf2bDeew7CFtdjGVs+LGXhvymVsyqGnQ7OBOsQjnu+lRodHL62xJjL4MxCkzXmhAshDExzIVS8lGuaJeQy0TFoi6+t3DhQic7wjoIRiFZ4hPlKYUyVOmt/giwrKysPwkmZrq4q//MmTP9v2f1pxuUmTLkK5jKcoJCvBJReogPxtMXWXcp4wT5Ent9hemHi1Kx+f7okvpG4+gLTj7E9KFX4ORi4W20TzISHRpHdHxf+ZECTXOqUfW8ZBp/q/kjb31lJGf8FlBeS29jVQeKeE/7be27MC5xHHaZ4tAreRBGT54DYcvHyPxZWLazUDPH+43HWPTu1g2dO3dG4vAVWLq0D1wqjfAPtE6gzPfOgMfa+IvSREdOu9Pk5LwDV1lHzqUOKlibEntxrW+cTSS2zlZHoEkRIEVDMHE/K2F/D6Yg1QXkEc8VK+peiS3UZHqaMVazw3ClBfEyK6Lvedr9+dVkGFpfWWnExQ8lhj/XN4OFb3lsc98nWse0NuQx4AEwf1cLxhDCJHS15i1sFEvZxVfd9COoiArbxFKoOA1ikK+8qPhl5Gh6CVuGBJ6y8JuYYL0cTkfy9OnT9zV3yZVpyrvKdMXrnRBUcYOd55hMkF4/4ZosAuTNSsptdQouKdREh8bVRCeoYNWXePHWeds0f2SBURCC/KVr3wxGIH8hJsxajeN2UmZdOLL9CywB0KO9CXCS7/A5nLLaYWxP94eFQOdkpKamopvjUzy8vAAxciiACVi0hbLgubDj3beQj4m4tq1vzGP1IlGg3rLFxL/6Mc3pDDfKLiZYKSReAsFLIZjNEOLyfShrTuLqsugINAsEeAyn3KbrlTkKFVlqikZGkDrFBxxXksIB4U35WWuZeUkBhKi4lajWs7WeWz6QsfhQU+jo8uO67jD3Wg1S6BmAdRYMHAI28jX2JSWADua+TwQ9J7Ivz+a+T1lUFIvyPDj+T8jiNSVTeVd5XtFrDNTiwk2ePNmclZWVgxB8AYF3IeGeWbNmVad71YLipTtE05CaKM/xpYvqhSsThdteWYKksf+KNkaQWwhV/bt4M3THy/vX4PGuKeiQ6R2egJzcPRjWKQyIvA1j0Q9dIzdga/F67Mm1oEcPMrJTS0Pu/vsQet5dL2XCgM6YoPUnYFHeRnSpSTcm07Kn2Y5+hofHrEf+b1IR7+1sou3w5ctNYSfD2ttVY0hoiANGEX3eoZYKIVSTUxgkI3NxxpnDBb5bcvEoYQRkxg/KssnvhlOT+BTA9/yKvMicB2+m96y6Ul0TWPq5SwYBCdJvodLTdZM10lbr9PcmTG0ekhire3Cc4OClhZA9xT+4tX6GWZVrLoH1zaCgaeeCi9GCsRLGEQEmWgkuIrq2C7ln/ynXtxCIBmNhUJ1N9cDSZF+G2jBWpitrlbFKL9FOPMlC2NasrKxNzMVew+f4IhhpCGsjU3Mdo0xRuokQQelzRzDB3ocdvZXZSt2Mdc11cUGSS1ORanrVFSS+lyfZvIXniwBFxozaKcdU+aTLELwtBP5mtcLmAswx5oo8dzHJWCBEeRoQkH+y81lYbU6YzO5x1h1lQMJ82HaOA6xWGDz91V0Ac6/xED41lMy9x0OIZhL/eDCkqysq/FEZzk6qCIfE7LvBhAo5pIvMRbjTJVzmcPlTlxMOSIbzLERmsku4RIi91jfd8bOXdihpfcXQ0a/9e+3is3k/QFEqTEzVgab36wi0YAQ0lwqIvuwnlhrsZZAvXzX3mzq7+KkuMVaq8Nytk+jcerZcOVZL6qccG2Sp65GX7u7Z6flPvqsTc/dgbb1FX875R+W592RmmqrBqPLQy/5YWaiQe+Zs5SllvogRD8OAeWwA65J1W9YmKjEscSkPLuz97sB3R+rjttMSAZ46deoVRmbsImTRhzHWG8BgAeEA8E9WxvooLymHW+K6GlNmMhjXZD9sTFl0XhdBIMxsruQ5XM0EQxjMZl8fYzuQbwflSIoxt+y4NFeIFMsd/HbOcb0kCwio0S6Vc8HEjRzCLJzCVVLmOKGq0kBVsEjZCbiEsJeeRUg1aF3Qfc4ce7/tvCsnRDb++/dXJE79B8RBgNVaub6AoN6hI9DMEeAx/E7G2P/In7OZi1ou3r6nUkINBnFzeUcdd1TrGRivcHsr1NdyTCyFKsjntT7KcbUS64pxtdBUe8KTfuyvAP6qpCuxwixuY4z1FbJ4ikmsR88ePeOvz7qeYpTOMMFOCiYoiL+YCVYmmHB6tvQ7T8GSWqSOx42THmLIQFLnh7dqha3FCQo5pLhD4uvZp7gZTQbBBPlKhniCOikWK5yK6DDB2gkmKA0uPTD8wMC+EUKs55xnTp8+nRTiFnkfy8jIuEmGfK8yTZleC+gCMoT+BnXlOCBQNl8i5oTHUFBQUZO7+Up6cckcLs6NcKjgKleFBIOBOZmAy8UdLkFOhEzYJA5nmcupApyrDDCqTtUQYqrVj8Ly5cvlFT+rdzs5D+EqezCchx8c/pcVM1Y8gZKLS6eP0BFomQhIkO4Ex4eNIX2gFL+wjjyRMane9y/n6cPgtnOQIttCLamHz7EHLIeLk7XdN0i8MWDUedSAgMdvnvI+l+d+Hjt2rDEuLi7eCGM7MLQRkjBLQqJc0+ECwiggDBDaS1mJzC6SkGTOuJCExL3bGlgG/BQTjPgy4k/7lA5XQJAhSIXQ/NIdAsIuCamUM25lLlboYq6Tdrv9+Msvv9x8sncFABlJkl7iglNGmkZt2o9LDa+6GlUYndnFEbAWnoYdBphjYmppSTYjhnKCtMCmLNoQdtyBiFCjzMqc4aLQ+qNQgWMcMNFjtMzEcchClQQ7zF3CDAYbZHbGoLJDXMhmBiEJ2fDzVWZTqzFvfcQiEKFpuUbnnqKF48ZR7m2/VhoXZ8TxkvbgnKmCiDkeiYuIfed2ZcPum64PNRaWlonORzY5fKPN/QjoBzoCLROBX4AjqFkqAg1LaanrlqiIWr8Q8mMvx3QALy1AWNe+cJ75Ecb218F58gdArXVoQjk9iTE9WK4cjea7Q2ngAFBUOn301kIQePHFF7swwW4qKCwIustXZUg0M30A8xxXpq8fBwgB1/FtSE9kiIxti7axsTCxRLz6sbfYYAOYuI5j9eLVWjW9BlAJytRDttKeTib+WFrmfEJlpx+NCo/obAqT97UKNeaZQo07wkJCfggPNf4QERL6DfWFm4xfRxnDfgwNNew0hRp2RISF5IUZpUNlDsfd4KZxpWW2J2B3PFZY2r4TUHXlPMGF+w4pVHCwOJuT/yo+tvT2syeL73ecK73v+7jkX4x67ZOr9cp7QbnkOtFGRkAZrYQJJj6EETsamXWD2JWpvGddCEjmOLTqOwytf/86QjregMg7n4Dj6LcIie8B0/W3I2bYdEQOmYiQTjcCrPZv0A0yC3iu3eoKpdRlvfpYHYFLAQFZlh8VTCyiIjCNuR76G6z3a6nGFPSy5+U6gGc79MPcKStwbNtQxIcBB9a/gq53dEXkHhtGd/f1Ma4rWieQMSYNC38j0KmuU4M7nhWVum4OCXVM5VxEgLtcBtn1nlNl/QXnV0kGAzjHJpfqUrngSSoXUUzwUtVoK3GpeIBzNUpSGSRJmMpcvDWc9pu5QIgQZRAufAkl+yAUfx+sI5s2OUKi++xRBXoLlUuqy4USIcarQkjgZVFCgssoSWdk2fXBmjVrqHZ3o/7BBhdunfrliIDHz3hKY609UG8pXapoXxuZTYkpMF0/EMYOPcGYJ1OcJGv79j0bIcfEQ6KsFaoT4b3u0j5UXtr+/Wac3/FvqEU110WQJZ9iHrURSB+jI6AjUGsEbDabZfbs2Y3u1kgGY105rvVlarqBRz56HXMxFvtnDitPp9Zl0CRsnl+G3cdPYtfWBVgVcgMKHn4Y63O2Yu3go/hj0nDkAkgYOx+r540DZX7bu3o2Hkp7TqukhzQL8heNwDu3J2nH/UyDsGL/RxgWuxvTxiQh0z0Z61bPwyCa3ARNgHOXi9yJBSA4Z2Dl2UnJCUulSDkuGCU2hhBQuUs2IgSchoMShwLMBcahCjoW5JQhOFfhpFMXNHKXeGzhR4udpfyuMuaG2ilEO+2WKkuUmDTECB4aU3qy05fbjmZmZyp0v5WJONOqNjFB2wsIN0KHFr5RLgMIMlUAmo+aEMIhCYkUeQq4KuGMFzKVnWUqO4GtOKynPWqEC6SzCCgCZ0oc98XHmmqkGd47DZF3jLtgjCG2I8oO70DrUXPBHedhXbcAkYMfLx8nR8SiVdJQhF59M84sfkJTnMtPVtpxqiLgBYYC5ZddSVT9UEegxSEwe/Zsqm/RJE1TjvU8x02Cfa2Znj28E5jyIrr4zTCg/7gM9Aewa8FuZI6ZBcvSzVjdpwSDuw1H+tZjWJloxPIJbdHZcq2WKr9H2nPI3V+M1E52LBjWFhPevg9LlizFrIQc5OYvQv8OhZhtSsIPi7bCtjIRu5dPQVLnP+GYyChXyv1ECOABuSlkL14caoq9xWiyFmkKZl7B6TKXbNwnOMJliTsMknS6jPND4DgvG2SEGNhP3ChzB0ek4GglBCsLNxoLyoxiP1fVVgajAZKk/qw65VIuSZFClYwCXBilCD4r4o6oVkuGqAUogP3Q930hiSFCNgy0FR67DobW4YxJVI5RyNwpQmQGE1TEijJx9fmCuKucRUMNMhvq1YJpS8q3dxtAWGpNysvbvXVP00Kv3Q8JnBJ3uOg2L6CWgqk2iQm7EVLpbYiYcbtyVga+Oc/5BopulmV5u+5XXWvo9YF1QCBQil+YUX4fABUuqbLJ0fEwD/i9ds556hDUwmMgizD5GlMAnm3vJlg/exvgKuRWMbBuXgSpVSxIMaZtSMfrQUq0uf/vYN34ZpU8qNOp8ro7KldLTT+hI6Aj0FwQ0JTjQP1gNZdFXS5yuKyFsMIMlBUDUzYjY0R/WHe8qlmCf9i2Em9sA85RUpeFn+P0zCk4tj8fB/ZtwIL3N+CNXODqQRzx13bHQLRGx2s7Ica5Ax9Tne4fNmHhK9uAskMA5mLv6QzExwUX1XEL/2Mqsbe5nZXuu4WBSWTalSVDvJEJO5eFi4GihsXVRlCGG1bMuIupTqk1mZWNjJUJSahCyJwL3l0CK5NkSWWqk6kqOkgMBkmIUiFBAoX8queH/+AqGtnq59IkqI5rYYyAyRBmKGOhMmV9c3ABJkkYxAtYHD/PTFwgkqnowO1ow8soErCuxbyCC97FqZMTpU/0UoVPJReCSk/GneDizpOSfNsBLp4/z8FmZiqrHSpe/+6H77ZfLvlBLw6jPqK5IGC1u45WLwtDVEo6mDEM53d9iuKP59aYxaoq7dbY8QbE/FpB+M33w/7DF3Ae31Mlu1K7qrtWVYmM3qkj0HIR0PMct5BrFxV/LTDh3zgyc5CPX7AVb0TGYtWiPXjZDoztd5XPasaid7duCHE4gM4rsDQlAraDH6FT1zSMzVmEkSl/QEbRXEzzyWrq1HI3uBM4DOqdgG4hDjjwByxd9CCitTLVwXWtkHhIhCqcd7oc/I+CkY+vZvek/z05JwVjYDdoPhbMnXeYslGQuVa4U92Q4wT96yUkULobrTGIHuRyQCepgwkhM/A+NmZgJRIV1xKM/A9URqRkzZIEScaVKMMdrlNoK8oaN8Glz1VsjF2JMRAKUTLDdUDIACCkWAjs4/jNN4yn3diz59me1/UcrUxXNjaGPDqPxkVAeVEZBhe+ammFAU4U2L7gnaMm0ve3cgu/eShCOvbE+W8+QvGnr1c+Xatj50/fonD5C4gZPh1R90xyu1e4qI6Cfyu2OU/69zT8KFB+2Q2XRKegI9AkCLDMzMwHJJv0ufIn5URTSEAG4woTUlNIoPOsFQJdfvUkRmIuUqeuxHG7C3BZsWNZplYOevJdXcnWiy7t3EHTpvadyVQMdE5Gamoqujk+xcPLCxBetBeABdMnj0b/jk58PgtoTfqu04mzOIvCgkK4TO2RCOAQOmBIaioGdnPg4TFrEN42uIoxgSBxl6RyhKhchKsublK5MKmct+KcR7o/wqxyHsGFMHv7VC4iVFVE+BybVcFbcdU7h0fSGC54xRwhWrkEk1wczEEfMKh0g+UCQnWC0iWTYfgWtRCtheOSVoyr+/JFMoYkmcmPhMoR9xqkTmEyPpydlTXjmWeeobygeruUEJAwUciiY2MtKVCZGM6XuXYUlFyQjRGQDWjV59cozVtVb8XYi4Xz5+9R8N4USGGRCLuOHNgubGdLHXkX9uo9OgI6AvVFICsU0wEAACAASURBVDMz81HG2NimUoy9cmvKcaB+sLxE9W2AEQjrhb8dXod+s4ajg8kIZoxE0sPrMX/zYQyJ94+pNMSnYk+uBcN7xGoR2T2Gn0Tun+9Dm6sHYywy0Zbsr7FjcGggsGHCM9iGa5A+EkjpHIs38iNh2ZOLncMTtLmRPYbDkquguz+LAC/OTU4wUcaZWgKmlbl01/IhW29jfshv2CDjGn4eN4tCUFb4y711khj+ECKFXyNJE2NaReS/+OKL113umFxi6w/nEm+yoJf6Yjl+1bcHjhfYzl0wX3Xh7D+fgnX9Qu1U+2c/QqvkoajPlgi4Tu7HmUWPwv7dugtYlZa5cPxsGfk+B7Tpbo4BhVMn1oIQUMYq4YyxTM75800ttqb2ePIcZze1MDr/6hEI6zQIC4TAy4WFWnkck08RkN6TV4EKqHtb99QMCOezsNqcMJnNWtkfoDcWCCfmWW0wePqsVrtWajr57Z0Y/bZ3dirWCyesVhuMJjPCGkExJs49Okvnz/6AU06IUjAeKrRcE16Zath6X6tSRouGNklCNBO4i59BtMuuPSA0lOSlMN/IGO4wsvDWquj0OQxblalKf2Wm8u2lsDZ9DYCsygHPuFAdroFU/I6etX9wfUfzaIPs/wKUlxZWx75e/dXRO3T6vMvFz31SL6L6JB0BHYELEODteToD2zht2rSvLzjZiB1kMPb/VWlE5jqr+iFAlfFifBTjaqkYwmAuV4y9owwI8+kzm6tzlzBocxtLMSbpnkpJcRgkHDIYpGNVJ+H3DYJz+xkaJLlUZpKVkV+ypiT7+h96971bLwbebaV+xhAmSbiVF+AG9RxIIdSbPwI3ykwebJCiYMSuKVOmtPY/qx+1RAQ4+D+cLPB+s42BRUmZ+ubBUxRx3PjN4eI4eLJkTfqqw0WNz13nqCNw6SGgpCuxjLFJqqq+2NSr0/McN/UV0PmXI0CJgsfMX7tXLXN8x8B7CsqlplmDSUn1tQq7lVZJkmyhoYbTEthZW5mji0sIiitj2r9yqt4dLQ0yPHF83k5PxgkBUqxlyYC+vAi3O08jjPIhU0Lg4tPYcugwfjxXBoRFokfHq/CL+GiPJd6HTKPschw6fhTFoW2R2Lrm/K7BFOc6mTEHJLGJhR564YUXesyYMeNYIPnJK21aeupA0ryEaQl1mKlBBg6LxTKvpeIz8YOdny148Ka8Tm1MSaHGRjN+a3B995MVdrv6UjCw0wPygoGqTrO5IyAixQwAS6ZPnx6A0r8NX6320lzPc9xwIHUKDUfAyGyHJcG2yJJ0i1BFpwrXCrr/u/Veb5UrWYIqM1ZmkJnL7gDVevbXoSuJo6nUvjq297zEYJRk9BVFuMt1ErGq251iZ94a3P7+F+5R7dsDJzxBszemYs+DfVCr8lxeHtVs7Wf34pGlh/DCuBRcH1rNoPLuMvz79b8j6/b/Q+FdTVvLsJfMmE0w01dG4zplrNJbWagE0nzHXL+u7o1GORj6DgDD+/YW93oj0Ipfmcv1/I7D5z695drYRvtOnC4uw8ETJR9NyP3W8wPRaKx1RjoClyQCGRkZSUyw+21ltmYT06JZHQLpB3ZJXrlLelF2FJ4+jdOn3b7MTbnUhePuO28KkdfJsuFjJjErY8wGsCImsSJZFmcNknxUlti3IUZ5ExfkUu1sU2JzduNCRFJKNq0KB6kLtf0YZEQyCbeiECmuE+jgKtX8jM8c2Kgpxj1/ORz5WdkoHP8ECmdmYONDA4BvVmPs5iMBgcnArfjoxM9wGmtDzoQHx/8fNia1rc3goI/pa5AMXRm7MuwK8ULQmekMmi0CMUnpvUx9Hr8SCaOaJJPJ+H/tWvvTWdt7+040ToVZm0PFtv1FNodLnRCsi6Lfj4OFrE63mSLAJEmaD4HJOTk5FwbZNoHQep7jJgA9oCztOzDKlIQlfkQHYv7mRXikyx4kdUjBvWsOY+YQr6XRjtXpJqSdWQrb2yNQsG0xUvqNcZeT1mikIXfPIqR2j/Gj2JgHV/UIO3Bgl/UdZpCu4MJ4lcvhXCOBF1AqYlm4SmTBCuQw+Sw4HnIJ+UGuumLcsXhVGdEq93mOyZ+YMXSWnEhyFeAWVyFiXTZPAF4R/vrmOqD93Vh1bwLalC8+BImJA/GPn45itrUEdCuOgAP/y/sCsz5fhy0ngJ7X98HEWwdgWOdobdahPRvx4sFW+EPHUgx7j6Ld2+PB2/sje3ACbN9vxKi33RHwT7y+DBMfHIZh7SV8tXMr5mzfiY8OnkDP9ldjQL87kNm3E8LgwNf52/FV/G1IbC1j9YercfiaRJgPbsGkzw8B7a9G9qB7Mf6Gdm6Jy05j0dpNmPT5To3v6CF3IGtAd5BkP3z7KbKPt0N65zJYFq/G/U9OxZj4urtq3G5g4T86MEF5UdmuTFdWlUOl71w+CBiwMIyF9QsLD4Pol17GBAoA0N+rtqV9+lA/BwrmrbHuieqTPhhcFDAjCoqEWoBtrxU3BDCVFz+68zDrZzLKnToG0eWI/Iw/+/4sbGXqk5NWf7e/ITLrc3UEdATcCGRmZv4fE6xMmaaUpwVoamzoAbVB/mpNvQCdP6i6GSzr9qO4uACnTh3Dunkd8OiAF7Endgj+tXQsZqVMwA5PoqbCbQuRNhdYk/MgcGAZOvQbg3tX5MEmBISzAJvnm5HW43asP910yCoDB7ratG/zZXhoyF9CZbYlwhRS2Cbc9PFVHR3zlz+d9vd3nk59f8ljKRujwkMXhIUY1kkGucTralF13jdaC/lTCECSIEkS2sgctzAr7nP+jDtcp9FGtaO8mEDZeVBG6P43X+OjGHvxMCD13j/gs3t7IgLA/zYvwT3vr0NB+zvwl1/fg567t+OPC/+MBUdt2gSb9Tg++nw1hr33HbIfehh/GdwOyzeuQNbXpwG5Ffrc2EMb911sHGKMEo5++ynufO8T4Ire+Meo4bi7vQ1/XfV3/OO4u/jAiR934pNid27XksKdyHr7n5hUeCX+Nmo4JsUeQtay17GmmPxLijA7+1VM+vwkpgx9AC/dHoPFHy/Fr9Yf1DKduM6fw0cbV+DOxauxpX0i2ofWz18zhDGkGuVWBkks1gP0vN+RlrVVpio3ZmZmPlZvqZn4j3cuAwsFY1eAsZ6Msf6MsTTG2BjG2NOQ2AxJYn+VZPaeJLNPJaOUxyAdioaxMLpfOgXg0Be3Xo2C4rgQv9q6r7Do0KnSC2iYB4zV+uq69SVEFuONu8+gqMTxyvgPdr7le66e+/Vebz356dN0BJodAsrTShsJ0jRVqE/U7BjZ+KJrPseB9gNr/GVcvhzJ5NI2vhPMZsowEYNB96YAE/4OUqG6jHgJ8/8ei6TMjyHm9ICl3wSMXLoHQ+JdWDnqYWBiLmYO8ySBM8Sg/7g3MP+9SEx/ZwcGjfdNDte4+M558Bc2ZdGGzUeYKOFO9oCNYYStIHrX7xeuz7/6ishDyn032zoc/yL/xyuSXxXcUFLGMYRzVzvBmURqsDvwzm0dJgsxKb4GVUUsc6GTsKG7akV3VwnaqecrlGLvEj164q3xFN9XQys7jtc/PoSev3wYa+/tDvKSHZHYFa0zX8Xzm/bi9yNvKq/Y/Lf0cRgWR39q16Ds2514s8yJq5P64E8xwOJvgM9H3InrJWD34VLg+gfw5r03afRSr4nCJ9/8He4igW7BvJ6VWs3aa+7GnpG3uv2fr4rAJ7sX4ceSMhw99j/MAvCPpx9Damt6/r0J/Vuvxi/f34rvfnkNvF4ck0aMR8YNDasL3lZiuE6WQr9nITMBjKsBMf1UM0TAJbnOyUyeCuCv9RFPcOk/TML0es0FTgkuHj63fc5a3K1Mq4iQrTu1CR/kfzs3LWHIV4eKPiosdcYmdoqCLDGc+NM9dSdWacaJIju2HyiC3cX/Mv6DXc9UOl3fw2oNU/r9uL6Q6vNaHAKt8JZgYsE0yzR6xdmsmvYH6slz3KwE04WpHQJUF++9Gc9i6tSpmJo+DoldHwZyZqG3FtMUg3FL1iBhbgoGDeqMuQPn49UR3aksHo7nA5b/Z+9M4KMosj/+rZ7JSRIIyK1yBBAEgxAFVA4JgiCa8Ffw4NgFD0BXgbgcBkjSSTjkUA5dFNTFlcMDPIKuuB6ggEpQQAICCiiooHIkkMFkksxM/T/VM5NMLiAkgQSmPgzdXV316tWrmfTrV+/93v1Fsz4Fc/N9PVn/yjeGRfrcOKicVvqIntYmR/ps9hH214Rm+kWD1prN/n/7Dx39+9/+9eE9P9W+IdJhI8esyQ/8fMRHmsl0TJjN0mQyIYTJQJQIFg6uEnlcL07Ti3SibL8TnfcH3fOO09CRXVwx9pjK76cN9dOjxnlqy7bwa2Y2pzP+5EPg3nDl8uAq5roMHdAGdv/B74ZdSFl8b+dWQzF2tQkEt4JrNUAxrCpJoVGujRjIvn71+Xjr10x+byWhiS/zvatbsYMVul3bsiAw0M8/P7V4usWJ8/ry2jcYtXwlQ95azVNvfwPsYfMJZdVWfHXnkXIqxm6eupmFv0AM1WP1a9113mP1kMC0adN+FlLUUDBK58PxyS3P7kDya1n7SsmG3DzH9YZi7OysfjElhcyeM+lxKWmpAnnLT3/+tefjtKMcybCec9+SGipr8ZYDGWzcm26z2uwTx7y94/Hy8pg/jizfXPPpeE+8EqimEoiPjx8mhWzy+++/n9fLdWVOW72gXqAUD5U5DS/txtfdSPfwIHJzc2nAPsZOmsnGEavpppSyRn15c8VI2gzZzNrDD+HpTeznNiFWURHqulAPzDR98fv7D/r5h0mrvZUmRH0EV0uT1lyqpNO2nBxN0w7UtltswZqZK4SNQGwEOGzUdtio5cgx6q6w5RAkc885scerh44yp13tIrBtDj7532wGb7mZb2KceBVOBddDgIaB9y8MQDKs0MCvQHn2aFbS6Y4tq7n1vR3QoD2xN7Rh5eBmDF6pVPDSihNyzn3XnebMvaxhjRrR0H2z7x30x0yXmn7wm+KrluEa4r5dnqOfEHQya37fSoeyHg8oD63K6mu1ZGCxgsIJv5D43ZU1n4qkK4X8WgbLnkDZsr21eMKvZh1zDyk4KeCqc+NJSoeUT59KPRwHqwp9gQcNGmRatapw3bnRLGj1xNs79+p3NbpBUnfWVz+mP1oz0Gxq2SCIxqH++JhLNdYWEABOWHL5+VgWvxzPwu6QKtnNI2PeTttcqFE5LnRd187kROINyCuHcL1dq4UE9Al6AynksyJP9F6yZEkJeeAv7jS8OMcXV/4VMrpyq+jX7276XueyX0bdjnV+CJ//ZKFbXacq3PS6thDellvyU0370CgcBi1LZWLHvh58WPjqzfX0fGguwR61F/tUH3WXggrbCex67F/ra6TnnLxC03zqEECQsDt8zKdP1a9rP1o32iRFoJAEIg2sYn9po5A37bkk9jDXY+gt8OGXK3ivUxwD6/oWTD/7V5ZvATpdzZWBAbQFDhzNgKbuYLZsNm7fA82juNoMvxT0POOZj6HNZrNt8w64bQQZkc2d7bN/Msbw1QrN4oy01M0AP+UR3YB7b7qVm9ysZf5E8sYT1A7QKmVXIFzDlCq4XX9Kb64/rf90ViYvVAPrARY+0oKxHlGrSWv3E9c3DGxHWLP8W9rfH0WTfPP/+TJmI/XVeLqMUA4t4SStWErs4I5FXq7Ol3bl9pNSrhNCRJ6LchwYMa6hr5l+CHGnkKIPgnNGqZBSnnAIOSwzdf7aojOSSFvbY23FKlYVvVXma/39I1lw5Ilno69bcjLLNvXbn07es01gqhPkS51gX4L9zfj7mjBrAoeUqEC7v3LsZPyVx3FLDtm5xrbPfulwzE7P2LlU/9xw1S8zH6V1+P7770XbNm1tpd331nslcMlLIICXhRQv6DP076rqXL04x1V1Zc6FL9f71tFjh7FkhGC1wYm97zAJWHGFWyvCuW2f5knQnzunrIA2/Zh8y1biB3TEnww2vjKG0evD+ezNi+dv7MllCedy0T96KqAI9Tnovj89QV/c0SS4RtPKuTGrKJrpfcvfaPvlazwy7yWO3NePvlfWIvvkbyx6ZZXhSvFSt2vwrwHjrodH3nuBOvyNoc0CSN3yMU/9BMMHh52bVdahno8/8+nOfQS0ugoClfr/AxuurUWDvKM888IKw61i35/HsTUowM1wz7u0Y7PWHbmDb7jjpTWsHngTzTnJS6tf44U/ujO8f2m9ylevgvMiTELb5ifV16/K+B7vXPIIY9NmsT/zScKCYe9HM2nT7276ZO6gc8AfxI2IZsn9Mt8l5XylcGzjTLqM+C+rth7kRr4kKiICrjpKXLfy+XSfLz9l6afZtE8ws7qUPiLkxrE3aJp2J0L0Q8obhDsCtigYTCkEXNVf5Thy7s/+ZlHJLhiSPK7Fn8+N3/aZKZ3j3SdTdqoX6vtmRV3TKNDke99RS27fo5k5Nwkhir/7SykRYreEz+12++pT7+3aoJcjSPBMLDZp0sRfClnlrGVn4tl7zyuBipJAQlzCRKAWG0iqKJqVQcdQjr3bOJUh2gtA0yeQ9kB8rxbEeww3K2UPg8MKTGE+Pn4QXtiH1r/1YA5vzmVolwgK1OihpOzZRGTVf57nz1a/VTdLuL+1SXiYePNvn9eJuVZLPvrnCGa8u5SEN5eS4KbSoD3/ubcfUXWc3kgD7/4nFt93ePK91/KjmR6952Hi27ndN/2dCq+7f5FjUM1GDG8ACW++Bo9O5t7bonhlyRqiFzpzC9x7Wy/u3fUZL7z5Lne1fgS/giUtQkldmgxrv2Fl9ruSf425n38sfIOBzytfY2VIvpHV/+xh7H3vVo4eShGv4KLWYKtd3q/fqv9D/1yvEpaxPOsJiGxNvWDnmrXuO461L9aDjN1M7hphwBh2CYhk1f4PGVh7N8kjIohPUcbfkXy2ZgGR9fcR03koP4fXIWX5esAJlTiqmxse0SnEX79eDUlLGNhR1Tdh1sghTPr6J+Ju/INRAeF0TMtk1HXFdbIKXoLzIqfP0NMSEhJMCrnCsORc+1hQrWC/Xk7rMHciClzbnWnaC4aRUu5D8IGQdEGImwruFJxJKZ85mbr9Kfi81O+EQGTl1MxRqXAqHLB40pofjgDzXB+UsuxnNjeWDlFDk+Q6NDJOZZz+Wf/8YPmclAumfMazIHuQn0SWmjjHG5B3RvF5b1ZjCehT9C5Sk//Ms+XdMP3z6aX+PbjYU1Q4x2V797/YHF8i45tWZ8uqkwXMhiXDgg0zoaEV8/A2v23FPjDggny39Kn6HXVNYuVgP+0s8BLn9+WxZv/FyTwHmMxcUSOgxG1ymy2X03l2/P0C8D83t8bSmXE4sDrUtq6Gv/KRdORyMk+jlp9TuSu9Y0l3HJzOcf79CfKrsHeHkgbKr1uZ4zj1p7Q/kJSUVGzrPL/RGU4q+rdh2buG6DbRGGrt0Fjui+7H3QO6Udds48jOt2gcPouUtDV0a+nLSwGN2Ll0My8Nbc/ut2KJGFKbw9n9eSwggp9jV/Fp7O38sX4+4dGr+ezoViI9Ay1tVqz4G/7MtkMf4dO0Hy/uyWRU06MsnvNfbhz9GB09259BBud6qyJ/Z6PGxC3ecyT3qp2/5QopRaQQlPiFkdKweG5CyA8cuWJN5rZ5Bt5vzU5PjtI0XvTkXUIG2Ief3Lxgjbu+NMVPj9f3k0ufCnLJUb/Cm91jehw3uc7V3yYFX6f8l5QLkOLb8KUAunq0V6cqit7tzl/k1vldTp06tZnJZPo0MTExrCQKpcmopLbeOq8EqosE9HF6LVlT7pBSPp6UlPR+Vef7fJ64VX1OXv7KJAGzEaRUpi5VqLHZJPu3MpWwTXqePC74YiP6/z4+z97Vs5t+ex/G9iiKXHJ+c2llIjjDLu4Ezks5Pr9RS+8V3DqKdTKTQzt3883WL1g+qDujCSdlfypRLVvTkzpc2bIJoXnb+AgI+fELljybCjk/A/PZe6w/mYSzJHYgdYOhbtQ4ZhHPpzsziPTcYjH744+VbSuTiBgyk5FLtzKqtXrZDGZU3JjSGbxodwaZQjs3vAlpUpbhO9/a8pdyoTfQ1Iq65ks4ipT/kw4+OJV5ai0//LuYspgr8v7rnw8SaHg3fQOOe09uXpDv/nSWqR63+9hVBpuK8Fe/GtgIvO4xplqML40tFgXXAwqfeQHwIPAvMOJmVb3q94yxcGrx4AEFew64FWsPkud3KoRoIKQoFU3eu5N7fnL19qriEgimj7CLRxKnJVaLB6yhHHvfVKv4l8rLXqkSMAnRq6EmymuvzaevlMSKUhTziV5GJ2otTA7Rq2pM2cLKYdHkTvmQ4dd1psl1nRk4fAwrBwQQvXIHcoIT18MJped0AY3sGE4r31xyeZAVS++lVp5yR6qDh95naFHZuUVdRo+xeEA9RqeMY8PBTLo1qZhdmIqUY0jbmNoE0UcTxsuLAgAONVKtlzCIlDJNCj5wOOQHli3zUz0sqyW0huzU537z6xyzQwjRXkq58KTltwnsXuXMXuPRozTFTwr5q8lhauzRtLynJxX0eAlEVFrKREB33fsUUNkdbwSUP5NS5j2xjJVSrBBYKkw5NklTYwS/lcCbt8orgUtWAnqy/lZ1mpyhHLtwjtUfDG/xSqDaSEAfrvvnSZrVvyAOHNVGLBeVUbUWak3U2uiv6hfEh7P0Cfvgy3qG3DeHiE/HcV3dYCyHvmFtCowcegXkHecEJ8hIz8DWuIHhv/8zjRkTdR2WvasJabOWPZYYQljPzCUbeWNMNyzb3mYssKpNvULD7n31EUanJLE/M5YmPtmGq5JPcDD+HOGj5Z8T2uduOjc6o9N4IXoVdRHUZVwbk0PcKTRUKGZXgSgF+kRmhwSYfmjbyMd//f7TvZWyW1YepGSFhORTqfPKBgnnHOhHBAqEvSKKenNRmdKVAqygW1T5E5hjQInA31117oP7oa1espu6PsqtRMHIK6tyxWyruEcTtJZC/uC+9B69EvBKoGpJQBmMK8ziVrWmdnlzcyR1DatTVQxK6cWybTZCzFZOgZVSQrs8+WXIjWOVNabySmPCg5HZ5qL7wJU3opfyWSSg1kKtCY0JP0vTC3Dbn4Ev7WdB+GrC64UYGNchTbvDrBSeHhgGwc2JGQr9mtZmUVoISXtS2DEo3NmuzSCSUnRauxzPUsZ2J0AI6kWMYNzSrQxs4umRZmHzeyqKL54WIT74BIQQUjuEJWkZkP0Hk0YM4TtX2u9Kn7TCHu48tnetzjELanWO+dkHbbemidkC0aOYYqySd0jHYhz2uzL+yKh9e8vMTp3C/IIn9apz7tAoHhM6tWXenFOpz56PYoyUcovUZIkBfR5DlPVUWYEVTrH6HDqHzu4kJMqSrBRmN65cxWbvEnQRduGKlC3OlXowF6/11ngl4JXAhZKAF+f4Qkn6Ao9z4otoBuVsRXZuVOrIAc3uYfNWc6GkIKU2Pr8bN2uallqr87hlubm22Kztz59ZW1djRIz0YWsZAME12tXTyggCfH5z8fYqgwTUmpzSZDtAoUJf3OIfxphlO3jsJQuWbBsBwZ5JQOoyfNkOhi9zs6j8k/OwWLLxCQh2JguxbiOTaNKy3+M6LFjNrnp3F+MYzPD3JMML1bkvQtkh3TqXu65ij07sYaEAy+8UiNtLwx6WEodAfqPcJbDLD05+M78QxuiSJaDH6c9iRqWTvrdiuXRSK82FLy8vb5Ofj9+ykSNH+lRQUgDlVvFqKXO4zhMKEpjrQslQ0FJKob7eo58KHBrtsjp7VJ/fqZqfSk+ZY88par0+P4LeXl4JVFEJ6P/Ur6AGSdh5VZ+uX/xnQRnlZFiOFWxFGft5m1eCBI6krmSAEIblqv2AZHa6zLrHtq3OrxeRMWw8pHarLayMiWHh4slGeyHas3DdIQ6smUy4QpqNj6B9zGqsZLBm9jBXG8GAyasNa7Htz1RmLv4SSyl0jOllbCN5gJMf0X4U6zzGXbl6Me2FYHZq6bZnhYcqhPY3P1/fH2t1iZnKlTEFqHFF5BfS+ck7Qs1B/ylSfcZLsyY7NBD526ZnbOu9eeEkUF8Q5CMKKRgXbvBSRjL7BxN6TtnxzAQrdwi3YTgvj/WkcFJt1Pt71JcyzoWurtXlyS2+ZnFYCPFvIcTdxRRjySkp5WqJY4Tddrp+Ruq8Lic3z5tWVDHO5/tXFimkB32KfkN+3QU4mTlz5gkp5PcNGjToUwHDKQVUuVWo55taSfdHke4NKPQMd7JQBVz5T5f7hLqv+nmWDXDuiU48O5Z03qheo95qnmq+Jd1XdaX5ZZfW3lvvlUBVk0BsbGwdavCVA8f31VExVvI0lGPvj7EKfLWOraNflyFcuyqN7Ox0kq+NJ/yeV7EeW8dtEYNotmIz6ZmHWdt3B92bxhp7hJYd8xm7tgH70zPZ/9lAxvZaDD0fZ9VIIDaFNZN7Ydk4h+hJV7E/U5J3dDPMHMSynRbyso6TsuS4MfGS6BziGLNrR/DjgM1k52WzdVIgvZrOQZl/Vfshg9YyaUMaI9q7nzGFZSiR7+bXCGoIRHLoleyt2WXsffn1+Se3mk3IZxE8YCjR+fVnPjEjwmt5XSrOLKSLcFetiaZVBbeKCph8cATp6encVPXi64zJCbgxPymHa7oSuV9Kx3yH3XZbhu103ZOp8wad3Dz/VctW1w/+DGIx/MQl06RZqlR/FV7O9KyRUs4XQkytgEHd0ZIqPbU6d3+UP/G3gEpCkI4zZdBRIApwo0coi7Nn+csAt/asKc+5RrxwCIW57C1eCVySEtBH6oG+vr4fSiFXJSUlKZ/9aleUwdhtH6l2zF9qDFt+/ZY0ZrFp4HVGNHzUjGzSMyDv54WkkcTngzsbpo6+Ty4gelI43xxKIOcEzJo7gjCFT9x9AD0ZynFmcE3bnoRbr6SJSh9dN5b9abvZt34lb69fi/KMjMyzFYq+L4nOH78PLAXaahycgNgNSxncrWTFdgBppgAAIABJREFUWK3Nyc3z7q7VZdytSKEeeCpXiYKJulrD9EZol5gnHA7GndoyTz2oqNW5wz8Q4hqjiSSpVqeY709umVegXBs3iv/nEDQJLkcwnkode1I6QUwtUnJawmlNw4pAYRTkSrAhVYAZdrUnrUL2pcS9Sa6O6kogjDpnki2h/CeLM1uJNSphmXtsjERfih8nX0o86qOpHQmFYyVA+QWrH76PEKisCypMLFA6lK8wwQiUTGsK8DvPF48QNVD5k85VosTKQlrhf5f+PS8Lpcpq68IeVjBlHziwr8lMXbivXGOZWILkUT1Ov/dCRphrmrYKB2Pj4+MfSUpKeqkcczjs+tqXRkIF5ilXChV0pxA13D9Y9RNvVqSTsqSrT7mLHqePlELahSZKy0ZY7jG8BLwSuMgSENTlepEp+unzdPUCWi2Leon3KsdVauk8otmt6fz2W56R0awwiz4G+GauzZ1cxnW0KeNIHaOp22yiLg6tiadF9DpmLU3mtlGPEzt/OSVDCBSl46RSErRVDuF0Ubl4z1JObp7/OdCxZpcnHxIwTYArxF/cIoTcUqtLzGu27Nw5CJEfgOKygi2rdeO4rqVu/brGtTuoZyhiZ+HDfVsptocl/CLhNzSO2x2E+PoQ5OtHSIA/wYGBXBUQgL/ZTICPDz4mk/ExaxomTWDSNEPBVMqoUjbdRV079UGnIup+0rrvV/ax0NhKMXcpy2pcQ4FXCr2UhoJvdziwOyR5djs2h53svDysNhunrTlkZmfzW1Y2mTlWTuXkEiigkYCrcNBME+esLKs1seWvdWXN3sLi9iGcWprOxI4FyuvOxZGEr32Y7PcGG0p/SaPvXBjJUOtcdkwsb5p0Cwvbh8DSTMZ09Pg92I6wZvm3tL8/iib+zjZ+yysnQ57DLu87pdk+IvW5zJLmej51uq474uLiHtc0bfn48eP/O3fuXGU9rfSixp06deoIs2beGBcXtzs5OVkp/JVV1E+jcNrQyhpJebnFx/eXQibb7fbu06ZNcyccKXHE0vyyS2zsrfRKoGpJQOrTdWeK16rFV5m5MdwqvNGxZZZbhXcIqKOSNY3l7W1qd8/KxoX9CB/6EeYGCt0onmXrDmFTSQaWz2M5Q+nYRNn8SinWE6SdPIrFZuPE3nUQO4+Jw6NolncAtVfqoYKXQkABIBVAW/WNiqJnq1yGjFhLYD0T1GlGfVc63tIJ5N9xnNr87EsnZV5LpJwjpWGpUT7QSqX8u9nfd7so8P9zdlJuGCaRUqPzWJUUoMQSExMTIAU+52LdPOaQfG6XvGyDr3388Wl8JZHtr+PxHt14uOst3N/pBu64rh3dwpoT3qghrerV5arQWjQICaZOjUBqBvgT5OdnKMz+Pj74mc35irNSoJ3Ks1boqOou1Ecp7fk8uPhxK/a+JpPBr+K7hq8vIf7+hAYGUC84iEY1axJ2xRW0bdCAzk2b0LtNa+6J6MCIm29i7K3dufuGCBqHhfFTSC1eyZOk2GG/XRrW8xIXxVWp1kSCr1qjM7WrtHs/Hyf7DMQ9Xx7P0OwcbgUQUkf9Wt0vqu4ufxA3Ipo/3JeGFubEVPaoqpDTU9/Me6siFWM3U8nJySoZxkc1atR41l1XEcezPWumTZv2gwPHQyZh+iIuLs6VmKQiRr54NBKmJHTVhPaBw+G4W83v4nHiHdkrgUqTQIG1qNKGuLCEDeXYhXN8YUf2jlZIAuYmA0lbEcuIiHoIEUD3SeF8tubvBDWKYv/aWYzt1RQfEUDEiH2s2PqsATFVXD0OIdAHmt0WAzP7ERKxiNDbYgif2csIyKs99GV6KhV8/FucVqOHOymUTKfJGaGtiqH7F5pNCRepz2VmpM6bKG1510ppgO4bjYQQpWgN4mpfYXqHFk8UZ0+5AQQGXuEj5RktP785JO/Y4X1MBF91NX/r0olhN3WmW4swrqxVC6U4ekvJElCvLlcE1aDDlY35vw7X848e3Wh3zTV85x/IUjt8Z3MYluiSe4NaG7VGpd2viPrSF98Ps3UnMe3bM2BYpCsYNZLFG4ujeZUc7Aoq7fSw9q5gVDGA1XudgafWIxuJiXTWRw4YwYj16mXTYwPOdoDJERGkAV0CIll9IAe/OjB27P1GAKs7cNY5/2OsnjzAxZ8zoLYi5FJRNDIzM8cprN+4uLgOFUXzXOio1LLSLm81aaZdSrE8lz5VtU1cXFxnYRJvYSf6XC3hZ/LLrqrz9PJ1+UpAYdrHx8c/fClJQL3Ee/xVv5SmVj3nct3gGch747Fk5xEQHJz/yA3rOxEpx2CxFK4ftcNjA9+/I+ukgudUjgzDkbIAWGpH3v1YsiE4WNmMrVisZoL9zUgXemepdIzUu0WgrYAx61zjnIeYT2197gDwf7U6P6n09BQDFrd0OjfXqmNacnJ/MdB+NE27wk8UM9kZlJT/8OcOQbpm5qYWzWnXsIGhgJQ+jPfO2SSgXiSubVDf+PxpsfDlvgPsyMwkUpNcpRU3Gqi1ydI0pRz/ejbalXM/j5/T0vi5/yqOZqbwx/r5hHePouXRrQVwBK5g10gV7HrXVaS+MNQIdj0oJ/Nam2iuWLsf2bcJqYsH0uW+t8jccRfzG3dnXWwKRz/syYmPZ9JGOfF7fg3NTXh8+Qpmhs8iJW0p3RqbeUulkiCSd9KXwfb5tOi1mGg5g1+Sb2PQrzEczl6Nz763qBceRbujO/DMSl05sjk3qvPmzVMGeJU+ucLKuSp+idMTN+lT9f6YWRUfH/+i9os26+InlTl3MSiFwXG1Y5IQYrRDOh5MmpZUJdKpn/sMvC29Eji7BPRxei1CDPSXV87euvq08OIcV8W1Mvu7lNiizJVWX7RdCdcGTXe9P4aO7L4869EJbXXWZmVsIKQjEE3zcNQsmYCCgqvZKWaXSjDg2UIIUStA04r57v3hkHxgh+uvvpL/a9bU8BP27Oc9L78E6gcHc3fH6zl4Ip0Pd31PhMPB9UUUZLU2J6UsCotV/sE9KKgtBWuJPhI5hrqaSThLYgdSNxjqRo1jFvF8ujODgS4aziDYkoJd5zDu8H52H9jHysVvs3ZRCtTph83yK6sJ5/nYKOr6K5qxzGJmER9+M41atqYndbiyZRNC/S3k/AyzlhYOnP3DEsvu1WnQ7EdWL1HxXsoJI41Pdx4jsqpoxx6yvhin+jT9wylTptxgNpuflU3kzwkJCa85HI41f/7555YKwkKu0GkpDOP69et30jQtSiL/JhAbbDbbDdOnT1cBgt7ilcAlJQF9ot4Ifz6WQn6SlJhUJhjW6iAIw3LsxTmuDkt1CfEYMdJHCu3Z4vbGkucohHg6tNPYPRlbFnzg0SLIB1mIhFVK3sxzMOC6dobfsEfbczy189OP3/O71UaNOi25vrFTd7ce/4WtRxTCkz9t27ai1jl4Y9gsh9m07xTXt7+25PZ2C9/t2Ede/Zbc6BrHzeSxQ7vZtPsQp30Cad2mPTc2rlQd0z1smY9N69RmxM1deG7DJhxmjY4KCsNVXGvjTt3rrq7YYx34KPU3JnYuCMgzQqy6NDWCVo0AVQ+nHbVvkp1bojadz7Va8Vzrj8xs2pb/jpxF8tDbiEkex/K4HNdOjnKYOLeSp4bKd/AvGvAKOXVgaGRHWrXyJTe3KatWrCCo1pn4O7dxL6VWLsXyvqlTp7Yxa+YHNE17tmHDhu30eP1nV8KO36SQp4QUOQ4cbldzJWxNSOGxtVY+qUhh/K1RL+PGM1NDC5BC+gkpakohGwspmkkhFdLFLiHF/2wOW+S0adP2nM+o3oC885Gat8+FlMDUqVPDpEmuk1I+n5SYVMhwdSH5qMyxjB/6uW51VSYjXtqXjwRqmYP+IQStznXGChwCYVoZdGPMTae/mfe96qdJrYaPdOJ0u+n81yHoHtb8PBVjRSWHj2Y+wXjl+NH1KdJn9DN0m42vDyPaSCTbgS8+ms+Nge4RSz9af9/AHeNe5IuPvii5fc6vPDUuhvRHX2DLA9fmE/rh0+fpkOTOWuusfjxpGbNvvTq/TVU6Uagej9zUmWWp3xCmSWq6UDzU2qg1qjxeg+nxt2hGj3iejdFz6dYkGMuhjcweu56hK17CzClCWM/MJRt5Y0w3LNveZqzKB9ymHvzo5CrACHaNZtm6v/FYZH3SXMGuU+pbmAXMmzaRyLrHWBkzH+oswBZ8lWF1fmbpRm5UNFPfRuXbWVB0knl5nOAEGekZ2AJK81wLoFV7GPszLBwTRah1L6MC2nDTHrd+V5Toxb1+4okn/OqE1OlMLj/qc3TPWMMyMXa+ip9L0YxX0cn6IN2XlrRymB1KGW2gObSaKgxBE1oNKaRJSKH0YiOdUpmYO1NjgSIpnf+EHUmukCLTIRz7Nbv2IQ5+EvvEj/oqvczhGGca1nvPK4GqJoH4+PjbNbT/4GBiYnLia1WNv4rgx4tzXBFS9NIomwSufSwI5FNO0LMydQ02a+L94IiRnVzJDAL9hDACShUV5U5xWjPTuUn5lMjghoBSjjdtYqelHzcGH+aTfF21BgWGSDunLVn4e/iGG7OxZ3GaQMx+ytrbwaM9WC0W5fjtNCaazagnek0fT+XJwqb31WCD2L7+ca7J2EKnuyfwfPyXTN5wtdNX1p7FyRwztQIVRGvhYs3Kxb+EencrW1YWVnMgQcW62rHmgr/vOZjE3cQ8jqGBgVx/1ZV8d/g3erhIONdGnsNrhAehMp62HrqIFT8qP2HDqdfo3TN2FW8ODgPrNuM6ZWx3ApRWDIxbupWBTczsVKCH/mB2Bbu26NXUUJyhJyu2vknrWhAzMpxe9ZyW8J7RPWH9WJZtG8a4PSlEtymgqej65UcHOMchuDkxQ6Ff09os2PqLgSVdGD5RBc6a6Zy0h6ToNtR2Gdyjk9ZyT+t8U7OLWNU4PPfcczlTp049Zgo07dSn6g/o0/RPLxZnLgV0l7LSXiweKnNcr7GqMqXrpV0eCcTHxys/+nF2h32QC9GmPOSqbF/1G8xXLqosl17GLi0J7F50+mTq9ittNntr6ZB3g2OqhBUStiNl1pkmKwTNTOagt4kY6eMQDj8N8rW5Aw7JtY0alj/wLl+d28Sn+yzYDqfxfBGmrH9uY+LYSOr1v5OQ7tN49ydnUq1ju9fSqWc/6vXswWOrNhVknc36hUVzH6R2/zup3X0cb/5YkITrVCHa7unksP/AYU6HtGPJnCRWzb/FyJH9U+rbBv1GfXvTd8mnHLODsjR3GjuNiYp+3/8wY3IP7vvPFsPn9vdtrxPY/Rm+s2TxyepnCOnbj3q39SDh0x+xkcu7S6YYfR8cEUnt18uHMNWuUUMOeGxim53ZNytX0zM3YvCMdeRlpnP06FHSM/NYN2MgKh+wKplEk5YtkdmZZOdJ5g134hpfN2Yd68Y4z53BrtlkZmaSJ9cxuKPqXZfhi3eQnenst+49NUYmj3UMJdgVpOps78SPHuWJcWyMXJfhy3YY2NJjOl6FCnjNx2J2Bc5epyQT3Jq4ddIYJzM7j/fi+rrcQZz8V7X/lfXW4XAMkCb5uj5ZPy+QaK/iV9VW1cuPVwJnl4DaOUpISHhTIO4R2SLiUlaM3dIwlOOzYU+6G3uPXglUjAQ+t1m+XfCDyoKXsXn+9JObnx16cvOzHTNS5wU5HLnN7ZL+DumYICX/RsrNSPJ1SCHoXssnaJEmNR9fZ+4Ng6U/NZMBz1Zu/gz1PIx2wLtfbmPT1q1Ab+KGuhCl7IeZMCiG5+uOYfuat1gx9CeGDH+Sz4/uY9Lop9kV9RS73lnGLValHKuSy5vThjF+TUe+eH0lK4ZmMuLh59lV4uZrIPc9OYOHwtYw6OHB1LutHyNTdkKdUGyH1tJuwkJujX+BXS8nkb48mcc//oW8rJPs2v4Jz1t78+GLd9CnYwfefyWFXblZbFj9InRoiUx9luiF37P0xWV8lTSUOUmP8PohC+kHNxl96RPHF7dd5eL3/A61AwONHL1ZruyAZuUK48xAdn4Ey9DLHBxK3bp1CfXE3s7LYz0pnDT8foPx9zTQF6Otgl0L0GHct9WugLufudB9Z5DqGUm6iZzDUY2j0GOqQ1FwZMIm+ksf+YQ+QW9QHXj28uiVgFcC5y+BuLi49nVC66gHYVZ6Rno3fbZ+5PypVZ+exl9kF85xYvVhu9pzKs1vWwuil6r9dIpNwMOGWOzemSrkqS3Pq0Ab9fnQs2Fgh8cb+Zi1NkJo1wq0Njt/zWl1fXN/t6kVBd+mstxVSOkzgITWaQxaGM8dQPdH59Gv7nqSSYfcv1C5eefe249ragUS1n8QLH+ab3bs4g3gubt70fwKX64e/E+e+Hgd5B7kHUNPXkXc7P1QoyHdw7LZ90dJgVcWDv4VyqhpKcwJtbF373e88lwyg4btZ9kYJ2Tw8yuWkBYCzbp2gF9/JbeB8lHtylcTH+B6X7DVGQQLJ/PK66vYtwkmzOnAtpRnDLE8s/hZalODuzqEsf/nE7RQtR3GsPCB2wzLdHllV9PXD4stx8isJ0BzCEeBF0p5iZe1f3AE6enpyovFWypYAvp0fQugPt5SCRI4X7/sSmDFS/IyloCu65rD4finQExCMi4xKXH55SIO9RusHuaKS2xF7AMDvO4sZVzTrO3Pq7dV9flMdW3bN36SSRpZmw1KuQ5ppH0uI9mSm2cF0unWPrRb+Inh1DikeztC0py6ep6vPy2B8W99St/Hb2bPp/8Dwri+bTvuB5545zN6P9yRL95RWNANwfdK7uwA79cYw6LYPuTt/pj4lNO0b+BjKNPK7zi/2DNIHv0o79OVFS+Ookv9RjQyfKBb0Kq98qX+hOlPTOCR5jm89vIrnGrbFN8Trt4qPt8XzPU78FxXeOKVfxsW77kdroI/u8ImmD3xKdra9jBn8cd0almf9I9V34rTX1XmwFybMzWHSRpoAfkvL/lzvGAnZkJDPVAsLti43oHOJAGv4ncm6XjveSVQNSRgwLQ5WKmpuOoc2VGfpf9SNTi7MFx4cY4vjJy9o1SCBIQUJiMDtYu2Ch9XaZQrpARC6BXX80RXGPX7g/Rt7Ev6VmWhrU2I39UkvxzHvoeTafex0yI7ff5b9G5Un7A5D/LGhKdpvcbQlwGVEjyQB+Lmsf7uGNr1X2iw93j8SzT3NVMPOOoZHGe6mlden8OYqRMYMtrtltGbpS8P5bqWgXwyJo3e4wYzRVEJG8pXoxrDpqIZmgO5Y+CDPLHp33R/6A7aKfr9xzA37V7ueOBOY/x2g+KY3NiH99VVjYpTjk2awG6MAMr5WyEHuC8r+mizZJBhBf/g0DLidlc0J156SgIq1XNycrKBJOOVSPkk4PXLLp/8vL3PXwL6rbqZboyRmoyVUs7VNG1O4qzEYvkEzn+E6tPT2Nr3vs1XnwXzcuqUQEJcwuROPlryzWanRrwo18Fj3buirJcXptg5nZWD2S+QAucO5WKcy2mbiaDAonqhHWtWDhRtXwqzVoUsYYeg4MDCWAgGfQg6AypFKSRRNG1mP4LOE5WiNLru+tVbt9H2tIXmJsHXNodjS55M0JP1ae7753I0rc6WtnvO4B5jO8KrE4YyYv76fHJDZ61l0cTzC2Y7krqGr7iBgZ0b5dMremLZNpuQCEiXE6kwW7TtCGuWf0v7+6NocobpFuXF89r8thX7wIAq4Z6lx+sfIVmuJ+uXzdar51p4z70SqO4SiIuL62YSphekkIdFrnhCn6m7QC+r+8zKzr+eoEvD1OZ9Uy278Lw9Lq4EBMIsPPDglIZwYbUEpQAXUYyVSHx9S1CM1Q0T/iW1L0WMqm2toopxPn1Pc3MpBEqoVjQrSzFWwxVZA3VZQaZ892RsrPlHY0bM78LWo9kGGkT6/rVYJvVj2Kt73Y3KdDzxRTSDPj4zbG9As3vYvPW+ilOMDQ7/IG5EtJEXr0wMV9XGuYyWmkxMSEiYWVVZ9PLllYBXAiVLQJ+id9c0baUDR0JiYuLtl7NirCSkcI4r+OFVsuC9tV4JVLQEHMIhPcAqUBGA5xsFWNG8Xa70PNdArY1ao4qUhe3Qh0QvgVUHk+io8jcDoWF9WbRhKRE5x7BiY9vKZNob6R8EkTGvcsgAGLawMiaGhYsnG1B/QrRn4bpDHFgzmXCVxSM+gvYxq7GSwZrZw1xtBAMmryYDsP2ZyszFX2Kx7mTyqGQWJw9wtmk/inXOASBjG8kDROF66zYmj5rN6sUxCBFJqsUlDdsBJkdEoHLtdQmIZPUBK7Zjqn9719iRLN54qCJFV+m09Kf1g7m5uZ2EFDclJCS8O378+EpMAFPp07moA6id3IvKgHfwy1ECWX/99VfrpKSkty/HyRedsxfnuKhEvNfVSQKaLJI+ujoxf+nzKoWGVtS3pFzTzs48CCTRo0lh15lG3YYTN6oblo0ziRgSz6StB0k/mkbfHSNoOnaNgfls2TGfsWsbsD89k/2fDWRsr8XQ83FWjQRiU1gzuReWjXOInnQV+zMleUc3w8xBLNtpIS/rOClLjgN57F4Sz9qaY0jPTuezgZvpNf9z4Biza0fw44DNZOdls3VSIL2azjGiR3cvmcSgN2uzIW0p7d3IGeYmPL58BRBOStpSejU+yrP1IohvNomD6ensWduT0d2bsuaQK+V0uaR24TrPnDnzxJE/jvQWUvwZFBi0PS4urkPR0b2KX1GJeK+9EriwElCpnydPnqxCvfOLPl3/du7cuX/lV3hPnO6MXp9j7zehuklAk5qsfDeKXL5c+z6/Nb6JG/2P87vVRo06Lbm+sVPLsR7/ha1HVEIPf9q2DSP9wPdnadOKWqZcftj9HVsOncA3oA43Xn89zYMzeffdLwi9qR+3Ns7PQmIsycnDP/LdCWs+poSvfy1ahl1NrQpVOytn9R1UrOVY5aKDbANPuRDHNgvHLD78+vVqiN3A4I5NjNtjFr3IpDbrOfyvnuScgFlzRxAWGgzdB9CToRxnBte07Um49Uqa1A2FurHsT9vNvvUreXv9WlKAyDxbIUCPTHoyd2Qkof5w86CHoE0aGUn1+AgI+fELljybCjkKiXA+e4/1J1PBq7wdR7dCzspmGrVsTU/qcGXLJoTmpaJU5Q1Jg2mivlp9J7C0Zzz/3XGYqCbOuRSabxW+WLJkicIoHK3H6fdqmvaxHqfP1JP1eaVt7LiVZU/Xvsu9zlMWVXipvaxVLwmI+Pj4KCHEY0BHc475uurF/oXn1tAvlPOxnqhXvq5x4efnHfESlYB6gHYxm/SbzE7PoMoIyDv90xrqDX+GxcvWYNGjGK/SSnd9ivQZ/YwU0J8814NoI7V0B774aAbf/KPfWdok89srYxmyShEqKEuXvUPus3czqm4SmVN7FArA+271OG5euL2gsXE2lL3rH+HqKqYgewbkbbY52Gx3JCUmJpZpi/hMAXnWAysJaDGElMOSKI/4uZ0L2xP+XgxfDZjHzX8sQc7obEjJduBVfFp8zZ68uXwcEYJ1abozU511G5EB45mZuQ6fpZGMsM5lx8SOHFoTQ9PodcxamsxtXeqzuk0Xam1N51GfZYSEQ2Z2V6Jd/ToHg2XbQiNQLzOzM9EhXRiwai2tfHNR+V1Op5+m9S1XMr7VuyyV8yim4nrw0JlttA+J4PlMSTfjvcvGygE+rL13D8sGty6y9gWXVSkgr4CrgrMpU6Y08TH5vK6SFebk5QybOXPmsYK73rOKloD+T/0K/RldbXF4i1cChSQQHx8/QUP7PySLTpw8sUqlgy/UwHtRSAJKv/D6HBcSiffCK4ECCRzclQoM4pYrfQh2b0Jt2sROw3f0MJ8YirFqX8Ow7J61TfavLFaKcVQSmRu+4MCcocZgz3z+J7fcFQUfb+RAkcx5Zl/lutmBD1etJf2/y5huJOo7yJ8ef9rcyBYFnAP2XE5n5QOr5d+yZWVxusgY+Tcv9knEyDPiyvmH3cWKaIjuN5ltx9Qi2DiybSXhY9MYN7EPjVpFwsyZrDtkAesRls8eAUN706KwF0bhWVpPkHbyKBabjRN710HsPCYOj6JZ3gFUZNkZgSRc3NqCG9DeyFzTmL5RUfRslcuQEWsJrOtHSM9m1C48ovMqL48TnCAjPQNbQB0igYSl61CzOrJtOUNSoF/npiX1rDZ106dPP/T9nu+7qYQhvr6+u/U4/eELHjdbbaRVPkbj4+OjqcGRmJiYotiO5SPs7V3pEjDg0ypoFD1Gr61P0TsVJadla8v0JP1mhSbjVYyLSqf4tRfnuLhMvDXVRAKV71aRy/ebN0HUDMNC+2W+t8MmPt1noUP9NJ4vKqtzaaP6WI+z6/BJWl8XRUpSK3yaNaIR1wJPs+X3J7mmST4h1wg1qKGQLoIb07WjM6GH0stOHtpCnD6BV1yG6AnxLxF3W1NS1yyh91yX5t5nNNuffIBrArP4ZPULRC9UIMzgbNuqkJXaNViFHNQ2lJTyrPiYQeGj65kD/e9Aiv5CiNuVG0LpJZjBb+zn+CN3E1GvABRh3IsbSOrbiGCSWDtrGL2ahjhJhMey9fOBmLEYDhlGbF4+8RACfeDK22Igoh8h/13AT0tjCI/ohVCkw3vSExg7/i0GLVDXyqVDlcZGP+NUORD0DMFME5L2pBDdJjwfMSUpZT+tTU4jXomew8HNiRkK/ZrWZsHWdJL2r2VYi16EjHWOMm7FVgaHnVE1dzas4v+vWrXKvopV8XFxcW9qmvbvhISEv9vt9pHTpk3bU8VZrzbs6fH6DCnkMIncV7NGzd6A80debWZweTOad0tew4QeCd8Bm4VDfA3st2P/wW63/3H8+PHjLlelfCENGjTI1KJFi1p+wq+Jw+xoLKRoi4ZykwhHGptU84tmsNTn6GeG5Mmn7j1xS8CwqSjYCneF9+iVQHWQgEM4hKQS/Qpyf2PTJrhrfEOnApmlpBJGOw7w7pfbuKmJSjXfm7ih2SS7kV3P1ibgWhbMHM2w2IXc/LEzIcj9g8bwZEQw/n5h3AW8wezCAAAgAElEQVTs/MMChZRjZQj6hB79exQsS9ho6gdm8eXnG/nrpqfY+68b2frKRIYkLeLODiOJUYpx1FNsubs+77/zFd/9dhJ+eZ7ohT+x9MVlXHP0f9wc/wgtWq5lWKGxCoYo75mCqRBClLgzVevGcddjEncKyZ0ScaNKNZ2vVZ5tYP8wxizbwWOLLFisNsyFkoAE03fie8gxVix5EBzsVi6DGbXDAzjDvyPrpMpgqLzvhiPl8PxRd+TdjyXb3deKxWom2N+M3OFssk4uy28b3HEMcp3rsnUU62QeFks2PgHB+Bt/WcN4b53TxSO/U/5JXYYv28HwfHJ9eU9KrBYL5PfPb1ztT1wJQrrocfqjJs20UY/Xn+c4c/VF+ulqP7mLOAH9Kb2FFLJzbm7u9T4+Po9hQlnqvcrxRVyTsw0dFxfXXtO0MYmJiQ+pttOnT/918uTJ7cxmcxchRARwtwlTC81Ha9iwYcMr9Hg9Wwqp/oDlCilqIPCTyHSJ/E2T2q9Skz8Ih/ifHftsk8m0U9f1sxolzsaj9z7O5743AMD7VfBKoIgEbDaOqqpcD7tfnwEktE5j0MJ47gC6PzqPfnXXk0x6QecztLGfPkxmcHveWLOWerYMvv1uAxOTFtJpvw9H5zh9S9N+OgGd6xfQQ2Xm68ona5K4pZaJX7a9TutxLzLvy0hGt24KK//HwH/8Dw4YDtH4BLTgX/GjeeH9p+k0HNp1iCLWnMumdZ8YNJ9Z/Cy1qcFdHcLY//NxaKLSUldyiRgZGGoKiESIO40PorExoidQdRlZMAcHo2LrSiz+/uefNc/sT3A+3bLSMRNc0LlE1s5W6V/O/mejf5HvSz1ZXzR58uR3fXx8ZosrxH49Tk9iI0v0z3WPH9pF5rI6De8gSwjRe+bMmQ59sp4ifaSKI51QnaZwufGqadpwKaWC3skvM2bM+B141/XJr1cn+mN6kDXYagrIDTDxF1Z9iW6YYQo18l5UqAS8OMcVKk4vsUtKAn41jP2pU575NrIC6XRrH9q5JjqkeztC8pTyqkC+XOUMbayZafR4/FHazU5hT46Zq69uRDPV7aqa+NvgFNCjrUekmZsmv7P3h+/5ZncaX3y7y6gN0H5h2oSFvNHi/1i3NJnYQWFGvf3od7zw/i5ueegtjrw+A7avYcjG43TuopyVuzJ7YjLLnoymSY3mdGrZIH+Eij45ke1g/W5rRGjnmP+G+gSdQDO9j9BG4VaMXQNKsEnkFw6HnJiHQ/mWeMslLgGlCCQmJg6zS/vtUpPRsofco5IQXOLTrpDp6U/oLp8hJzl9tn7EbSnUZ+hpQgq7Plm/vkIG8xKpcAnog3RfIcV9Wp725rkSV7srs2bNOqXP09O9ivG5Sq187ZTB+EyhKuWj7u3tlUB1loCpATdFwfOb93I66krnTAIh9IrreaIrjPr9Qfo29iV9q1KOaxPi/iWdoc0VV/dj15yT3D/hRXpsetFJs+uDfPVwV04ffI8NwKj6Rf2NlVvFAZ6Y8IRLmmE8Pn4eE25qz45Hu/LGC/HUyw8MbI6pXjvu6ZjCoMfvxegRFsWHfVvQ7ooxzE27lzseuNOg025QHJMbe2r+LvIVcBi79TTvHzZeF/ojioPgqC1BEGsdUn6g5ciPTn43X+HhGcVkhMG5ry7Vo42MYxlg9ie0VPP3pTr3gnklJycrZ5Xb9an6bZj4m0KzK7jrPfOUgD5c95dXySelJhXwySjPe57nUsqXhVk8DqjgR2+pahJozVAp5JeJMxMv29TMVW1JSuPHeHIp2Aqva0VpIvLWV0UJXAgot2Np/6HJ41/w4Zp/c2utipSCndNZWdjwo1agU0H9fEkP7vh6NEeWPkBZhrLlZmEzlZDGWqFV5ECQi76be4VsYTP7VUoaaTeUW8o+K8/9WDj8DSm/R/IBwv5BRurvX8Oq4lAaKsn26mxpu8ftK+zm2nncu3IUbYZsZu3hrfRt5H4bKdzmYl9Z964koM2QYmzM2pzOxM6hHEl9lX5dRhjZ8ZyNhrJ2/yL6hgWzc/Ewwke7HdhdJHrGsmF5PN0aFZdJVYdyKyaEMlaoKH6vuwXoU/W+aCxRQVY5eTmPngkSTx+v15M15B5xSoTp8/X8l84yit7b3CuBy14CRsCMgq247CXhFYBXAkUkUDe8P3P7hPD1D38WuVPeSxNBgcH5ijH2w6z/vgPPTehXJsVYcWH2LUExVjdMvsUUY1XtHxhYKYqxp0Qi6/ug4KdrB4n9SMfjEkezjNR57TK2zHsqI3XhptIUY08axc8P8doQpR+k8eJqlXi5apa8PAXG1pO1B9PJTE/n6NGjxufRiFBsh1bTuMsI+q/aSraUyLx0PltgoV+LYex0pbkmfJaRxS89PZ30o3t4MWQm3eeoLHyXX5E95Ct6vP60/pRevTHtyrt0Gg9KuxysJ+kDz6QYq2H0uboKlVhJCFPLO6y3v1cCl6sEvDjHl+vKX4LzVlsgxTfwyzvRK3hs6nxiCwXIlZdmCf1NjUlcMJ+Hri2LzbgEOhe5yr0G4bVMLLkthKG3BK3MSJ3/r5Ob5xcKPjkfNi3b3mQms9iTtpSUsYs54Arfsux8lVGTF5I8QCh0DAZMXszKhaOMcyEGsHpnhjHc3jWzaS+cbcSAZAOrWvVVfdq3b09kZCTtRXte3WnBdmwbyQPau2hEsnjjIYPGzlcnM3nxQoa56AxbuI4i9nHX1EJo3DiU4NBQ6tata3yCzTY+nDEIhq4iaWBHJ36yOZTIxxawNCmCY38qSgrIriaNQ4MJDQ0ltG5rekT2hHWHDPzj85Fbde6Tl5f3lHr/k77y24SEhLXVeS7nyruu68UQXvQk/d7E6YmbzpWGzWZ7Wgr5d32i7vIHO9ee3nZeCXgloCTgxTn2fg8uKQl4gHVdUvOqbpNRCqe/WYC9olbExv/mTSJ6xX5aX1ebJEbwytfTmNGtLpDDkpljeXHDQdIXHGBE017Mik3haPZc9i6MpvvYFLLfvpo20ZNI2Z9JVBMriwfWY+zSAax77B4OHuxJYKCNd+5rwejwBfRpY+FZnwjix63gYHo/slOfp033pjQ8mEeznN3MHL2bz/ans9Ccyq1Ne/HxwDyiirl4pDD2H5PpUse5ctnZDRg1cwQHN6scI4UzIGJuwvC4OKPhTkIgbS8fb9xIUF4ep//4lrix60lau5R8AI3q9mUoB7+uCP7x+nB9quNqx6BykKryXfXJejvpI0dIKVXQwaPlYXj69OmHE+ISnpEB8gUwECLLQ87b1yuBy1IChuOeF+f4slz7S2rSKu7LYUBBXlLTqlaTsUtZUvxd+eeQ8bWBJR1+43bWrfM1gPPmP/MJ8d0GG8ox4Qv4e7cm+FOffuGQ80Af6io4t6gBMCmT7NDuHN6fxoF961n89noWpUCzyDwwB9OkSbCRNnr0+pHsyR5Do+xUVqjIsKTBNFEaad8JLO0Zz393HGZUTibhs+YSGRYK3Mpj4bDjVwtRjdS1Zwmnb48utA5y1uXmBhUEbBbgmuR3sBw7BqFK0VcDzuffy6E+WWT9ucTwTU5u40S+y+9wmZ3or+rKrJ6PCO2efnx8/O1CiHullB9qGdon+nP6mXPIuDtWkWNsbGwdP7PfAwiGSyHrCyn+k2vLnV0R7Im94lnaMDg+Pn5YUlJSMdlVxBheGmeXQGxsbF1fX9/PpZRzk5KSlp69h7dFVZGAoRx7g/GqynJ4+ThfCZiFwO64UNjnFr7bto2dv2URVKcxXTqE07AoyMT5TqRov9zjfLPrF+q0bE/z4EpMelJ03PO4VvKvjDC5vSlKSRzKwFPb+PRTqB07EmYO4eMj9xpZ7JQGqbAx/MlDZdW2GvB6/uS58fUOfUjjFtGMnLWUof0eJO7kfJJd/hAZ2xbSNHqdEeTXWsW85RXNYG3GNwSyTjsh+wrE4hzLr2hzo0Ez+t0bxXWFhGGlUTiMfT+NGZEecH3WbXStF0HMnmwiOAw9X+SNxaNcaav/xaBRPsS9mUbUxI4FQ3vPDAlof2gbHQ0crYQQD8va8lU9Xt/qwPG+lPIr017TVn2VXuUSpetT9BvQuAPBHVLIayTyQyllrCa0z/TEikveoOYeFxc3RNO0dfpT+tf60/p+79fmwkpAn6A3kL5yn3CI2MTkRK9ifGHFX67RlMG40J/vclHzdvZK4CJKwF8IsvPyCPJzp/mtJGZyfyFh1DDmuFI2O0fpQMqq2fSuX0ZotNwfue+2R+jx8gc81qrkjfPTB7+gx7iFTH/xA2KuLblNJc20zGSteQp/o6LLEV4bsZzYz44SF6msq6pYqbd5CXGr09ikcjwDZ8ogYT2xF0hi2sTh1M3YRsxMqLMArIc+4taIscSu3c9t9WxYMrLxCa5DJJCwdB0pYyKxbFvOkBRY8UxT+K9zLPf/aq4l+xxncvjQMa4MKeDLP7gud05ZCm36MfmWrcQO6EhA9hHeio8gjVj6tPbnxBfAiRwj7YsTm8JMs7Y9STv1l3tI79FDAi7M1+eA52JiYgJq1qjZU9O0vlKTg+W1snVCQsJ3QoovHTi+dDgcu81m8wE3JrAHmUo7VckbqM0NDs1xoxCik5DiFok8KRAf4mCy2CQ2VCYaR3Jy8s74+Pgn8eVjPUa/QeHkVtpkvYQLSWDKlCmNMbEeeFZP1p8vdNN7UeUl4MU5rvJL5GXwXCUQIgQZWdnUDXLtZZ9rxzK2++mrVU7FuM9THJrai+9fT+SOFzYxZd1+ej/gymGRm8VJm4JpK27ptWVZsPkF469umVzvpmZPlVLBvOUYqBLqblCre8jacE8xLk9bsvAPDixkqTVSDwcHu6yOHl3suVjxdY7pUV2Rp1JKTubkUNOnYsMirXs/ZiZD2XOzWzFWXPtz18RYRvdbxs9bmkHPEBQatCrFFNZwCGp2GyOJoJ6IB8KJ7gnrx47nLWuI4baQ1q9FPrrygq3pJO1fy7AWvQgZ66Q5bsVWBof5s9N5Wej/mj4l2RfW069FvULtnFBuwzn4WSZRvSLyxyN8HJ8djEXZkjODG0OzkEJrGtKqC/Rby6HYbjQpaahCo1y+F/PmzVOm/Q9dH8aPH18jMDDQUEg1oT0kNNEOBw30eH2vFPJ7HHwvhVRZyX5zOBy/Wa3W3+bOnXtebyFq69zP5HedQ3OEaWhhUsgwIUVLKWRL4DtNat/g4N08R9746dOnO6M7L9BSKZeKhISEcIJZExMT09slpws0+uU5TFxc3C2apr2Fg5mJSYlexbiafg2MJ5kX57iart5lzHZRnONvbA6s9RtyW5trKlEqubw5tzcj1nTgwzXzndjH9j9ZNG8h34b0YeHIG9mx5t/0nuvMynH/QzOY9fdbCD70KQP0lah4qw3bnWmeU14bwpq/PcorLm7vH7+USfW3M2zCQpw58Lqy9OUJRNf4ngEPvMKIV5/n+p/+zbD3T9Kuxie8oWLXu45ml/4AzW2/sGiRzvg1inYHlr6sc1/TTBIm6KTWbUv6x2v4vxc/ILYSLc9HTp1i7Xc7GOp6H9hsc7DZ7khKTExMKMuCnAnnuCx0ire1YbVko9JOKx3TYrESHFwcO9izn/GyERCMf4UrpcpKbcGGuVxJQC51nGPPtaiIc6UwBwQEXGvC1FZq8hohjTTmCtHB+Dhw9E9KSlLWvvzihHTSbpFCqm+2Jmxiij5d/yq/gdplSEh4RUihlOIDODigjppN+4Gv2FWZlmFPHs52nhCX8JRAHDlx8sSbzz33nPI+8pZKkEB8fPw/NbQJDhzK1/uTShjCS/ICScD4s+/COU68QGN6h/FKoMIl0MokePPPP+nRqgU+puIW24oZ0MZptTHZ9Xa6uFHXTPV5bPx0g/xPnz9D77lrmDt/GXcG/cjAhyfTn+dY1y2LDQcOcP+YeeydCPoDMUz5YhAfvz6DfQ9MpsecZTx6jZ2lz2+jX+wLfHyTmRlRjzBiZkd+ivVhAweIstnJy/qdXds3MXz+Mqbe/w3tHl/Im3t70fyNYYzfNIgvXk/mt//GMeTh52n7/r3s3X7ASDk2PX4O0VdVllO0U7LfHvyFVih0ioq1HFfMuikqZvyDC9xSzqYYqx6e7SuODycvCubNWy6sBFyW4W8A9SlWVNKRopWaTXsfja+lJlXSGofIEspHp1BJTEx8qFBFFbxITE58ugqydcmwpGDzZIBUfsX+eba8zhd6h+CSEWQVmYh6KS72x6CK8OZlwyuBMkmgphA0E7Bq+3cMviGiTH3L1FhtvG4/wBGgudExi3eXPMv7dOPh2r9C2Bge7Hg1/jTmn12TGfH5Pk51c0Zt3dKxHVc3hp5d4Y1tB+CBDtQE6tVtQK1aNtq1bsgzb82lz0chTutxQx+l0xUpXene7mqam2zcD3yxO43vDATUVcTN3g81GtI9LJt9x5zRaN3HPEXMba2K0KjYy18zTrL3+HFG+xaDaK3YgbzUvBKoRAmUZOXVZ+jbKnFIL+nqLwERHx8/Tgo5VUo5W9O0OdOnT79QkeHVX3pVdAbKYOx9mlXRxfGyVXYJ3KpJfjuVyfs7nY4JZadwth6B9L53KLCK++e+zee703jz9RcYsvwT/mpwFQ1qXQUHFrIi7TC//7SJpZug3U3NqOEim2NTYWM2nCH0PphtNk4pp8Rdu9i36xOiF64i/O86X80cw+OqTykekAYZu824XdNcjzs7KGv2GBZNS2ZBdEdqNmxF+8YlwiicbYJlvv/7qUze2v4d9/to+Ck8PW/xSsArgWohAX2qPqAKb/VUeRkqKEEV9Kmh9Rc5onNSUtKsCxnwWeUFVM0ZNOxSXpzjar6KXvYNCfgIwaO+Gu+cOMGqrdvofW0bagW4Q7UqRkhX3/IgX8X7cnPSQu5Y46R5/6NzWBjVnCAeYumeXxnx+GCeULf6jGHvQx0xH3Kmn/ZzWYENTAulMfvV5PauMGVuDKmPPs3crjA+fhj5ERxurdoVaIaPay4uOur2X77+PBA3j/V3x9Cu/0KDocfjX6K5CadS7lvM9FwhglCwbZt/Psj2X36lv0lQX/MqxhUiWC8RrwQukASkSfbT4/XxNoftoWnTpv1wgYat9sPoU/SbpVmqzD3NpZQTE5MSU6r9pLwTKCYB7xOtmEi8FdVBAkUD8jx5VskotjtgmwNa1a9Hl2bNqBlw5uArz/7ndG7P5WRWDma/QIJ8C/s423KzsNrMBAWWEdpNQYNlZWEOLIxCcU78YMealQN+gZWKSpFnt7PzyO+k/vwzDaSkuyYJKsFiXPUC8s5NitWplTcgrzqtVtXkNT4+/h9CiCQp5dCkpKTLIkX3+a5EfHx8Lw1tkkIhkVLO0DZoS0tyxTlf+t5+VUcCXp/jqrMWXk4qUAImIbjBBG01yfZjR/n/9u4FPqrqzgP479w7M0kmmfCQAAER5SH42FZUrK0saBUr+LG6Vqz6cauorLaKSvupq1KSS0LQ+qitWj+72rqt2lalrasV0bIqoLBSEHyACAiCvAKBEPKYTGbmnrN77swkk2SSEDKZzCS/+Xwuc+feex73exJycnLu/zy3/wAK8304bfhwjCkYlJwH9kwP+vsSd35dHt1hPrYbyvYe64NzphP+7dhK7TjVnqoj2LSvHJ+Vl2O4ITBdKAw19e/W/P26Yz1eQYH0FCgpKfm1dY/1qsgS303PGvZ8raz7rFPhxstOTRQeESvEC/OXzW8vtHrPV5o16JIA4xx3iY+J010gRwh8ywTOMRS+qK3Gp5vr8NamzzE0LxcnFRSgsF8+BuflIdudmvm56e4Vq1/YtlFVH4AOz7arshI7Kiud2MljoXCdC/A5/eFu6xQr118C3ZZ57B57ybsOD8IXBbokYD1k7QbwVMtMrBmWJx1XGWxZz27/bGKUhLyrpKTk7W4viwWkjYAzIVEPIXMJ6bRpE1YkyQJ6aenxpsB4KDQYwB5/HXbtrMMWYaDStmErIN/jgS8rC/28Ochyu5Hj8cBjmvC4XNAj0aZhOJueQWBAQMRNJYjbTXLNO5+dUoCuj35XULqmkEo6Qdb0PGFbKthKIqRDw0nprCrYEAyipj6AmkAA1cEggraNfi4TgwUwTElMNAR0NJBUjBLbV+XwIeHONztTUCD5Aqfgs+Li4iUiJH5rLbQ+Sn4BaZWjsOZaE2FgklVq/SK+ZtYC6/X4z9zvGwJO55hxjvtGY/Mu4URU0A+rRcKwKcAwEFQKNTKEmvoQav01CCjAD6BKCISEgA29ATo+T2zTli2H7WJRflse72533W2Nld2yrFiXVvc49aZnR+vNpRQ8UPAohX4CKIRwRoR9JuA1DaeDHcmLfdWWpvxMgb4gELJDU0zTvAVuvFpcXFyplHratu3Xy8rKdvWm+7eKrF8qoWYopY4IJZ7tTffGezk2Ac45PjY3puplAh4hcJwAjnPuK9Ff9FPd3U0VcKzrnKryWA4FKJApAmVlZXsA6MXBSoqKir4thLjR5XKVWEXWPiXUYinlG6WlpTrKekb8B3nPPff4vLVeZT1l1ca3gYTcaQSNC6wHrC3xx7nfdwX0gHH3xHnqu6a8cwpQgAIUoEBvElDR+bZvW5ZlIIxz4MKlhmH8sri4WP8Rbp1QYjUk/hGUwdULFy7c19M3f++99w5wu93jTZjfgMDZAM5WQo2ACb1CVLNVDktKSh7r6fqy/PQTcDrHjHOcfg3DGlGAAhSgAAXSSSC6yMUHAPQ2z5pjDUQuzpGG/IYwxSy34f6NVWTpWWnblVBbIbFVvweDwWUPPvjg4WTdy5w5c3L6ZfUbbrvt4UKIYQCGCSFOFUqMU0KNE0pkAfhMCfWhUuptpdTPP9/0+WeLFi3SM+T4okCHAk7nmA/jdejECyhAAQpQgAIUiBOwHrMqAbwZ3Zwzc+fOHeEW7rHKUGMNYYyFwHkej7Ma0aK4pCgqKppjwJiqhKoH4JdSPl1aWvpei2tmGTAuB5CvhOqn3wHkCSV8Sqg9pjL11I+9SqhdQorVNuznzHpzs/WwVR6fD/cp0BkBPWDMaRWdEeO1FKAABShAAQq0KRB9YE8/tPdOmxfph5ulfN2AsVkZSi/96VVKHWx5vWEbH0NgHxSqpSGPKKWqXXWuI9FOecvL+ZkCSRFgnOOkMDITCnRVIIwVqz5A3dhzMK3gGFcP6WoVmJ4CFKBACgUWLFiwFYDe2nxZZdY/2jzJExToRgEnTpMOW9GNZTBrClCgXQGJbWvfwnWbDrR7FU9SgAIUoAAFKND9Ak7nOBrnuPtLYwkUoEACAQNZXuCfs7lSXwIcHqIABShAAQqkTIBxjlNGzYIo0FIgjI+3bMbOoAm3GcQaP/Def6/Da/1PAoJB5A48CRcO87VMxM8UoAAFKEABCnSjAOMcdyMus6ZARwJ7d6zGDcu+xGlDgY3Os9WrcMOb24Hyclx9xQ/ZOe4IkOcpQAEKUIAC3SDAOMfdgMosKdCxgAvTLr4Jhy/WV4bxx9/Mx4un3oHXvjWk46S8ggIUoAAFKECBbhNw5hwzznG3+TJjChyFgESDcxXj0x8FFi+hAAUoQAEKdJuAjnPsdI67rQRmTAEKUIACFKAABShAgQwRYJzjDGkoVrO3C3gw85ZSzOztt8n7owAFKEABCmSAAOMcZ0AjsYoUoAAFKEABClCAAqkRYJzj1DizFApQgAIUoAAFKECBNBfQcY455zjNG4nVowAFKEABClCAAhRIjYCOc8zOcWqsWQoFKEABClCAAhSgQAYIOJ1jHbYiA+rKKlKgUcBQhhKNn7iTbgK6bZRSMt3qxfpQgAIUoAAFOhJwOseMc9wRE8+nm4AUUqh0qxTr0yig28aAwd9fGkW4QwEKUIACmSCgB4ydFfIyobKsIwXiBbo6cqyUwhEA1QrwK4U6/S4EGvQGgaCzbh1gK71+nYIeAtUdPqWi785nBQEBhdbv8XVN1b6ui37F6hPrmep3IXRnVW8CLgGYANwAPHpTCl4l4RWAF8J5HyiAHJ2oCy8Jyd9fuuDHpBSgAAUokHoBxjlOvTlLTJJAZORYd/GO7hVQCrsksAfAQWHgQFjC63IhPzsLuVl6y4YvOwtDPB54TBNZbhdMYcBtmjCEgGkI5x2IvUc6nLHSdTdS9wRj77HjqXhv6qBHSot14nXfViqlpzc4dZNSwVYS+j0kbYRtiUA4jKBto7ahAVWBAHYHAqgPhlBRX48sARQYAoOljRGGwFDdwe5Eh5kjx6lofZZBAQpQgALJFnBGjnXYCk6tSDYt8+tmAdnRuGaDUtgqFbbAwH4JDPflYWRBAcb58jDU50O2W4+d8pVIQHeoq+oDOFBTg/LqaiyrqEBNQxAnGcApQuF4oX85aL8FJCTXw06Ey2MUoAAFKJDWAk7nWIetADA/rWvKylEgTsBQhp5zHBusjTsD1CmFtRLYJIETBwzAWcOGYdSg4+AyGJylGVQ7H3THd4A3x9nGDRmMKWPHOKPLmw8cwHu7diMUDGGCkDjV0NM0WneSddvoNmqnCJ6iAAUoQAEKpJ2AHjDmnOO0axZW6GgEpNDzWVtPq1hvK/xDAl8bPgy3nDgSuR49q5avZAjkZWXhrBEjnG3PkSNY9cU2rKmuwdUuBV+CDnKkjZJRMvOgAAUoQAEKpEZADxizc5waa5aSZAGhhB2dSts4OvmerbBZmPjBuWehf05OkktkdvECw/v1w4yzzsSm8v14duNnuNptoDBuoDg2zTk+DfcpQAEKUIACmSDAOMeZ0EqsYyIBKXWgiOhrh1RYF5aYdd432TGOoaTg/ZShQ3DFP52ONyUQ1KE8oi8pnAAfjHMcA+E7BShAAQpkjADjHGdMU7Gi8QICIizhdMCcw2uUcDppOtIEX6kVOHlwAQr798fnzSO36Z5yOLU1YWkUoAAFKECBrgnoOMd8QqlrhkzdQwJSSDvWOdZRKSpsiTEFg3qoNiz2lGHDsEM0/XcSBqShDEar4JcGBShAAQpklICO3n4teXUAABDISURBVNb00yyjqs7KUgBBCeH82b5KAQOzszsVg5d+yRUYlJeLw02zKvR8F902ei0VvihAAQpQgAIZJeB0jnXYioyqNStLASBoQzmd4wCAbDefLe3JL4octxuBuGkVeuSYneOebBGWTQEKUIACxyrgdI6jcY6PNQ+mo0BPCASCkVWdnUWKGcO4J5qgqUy9cp5ejS/20m0jhayPfeY7BShAAQpQIBME9IAxp1VkQkuxjq0EDGnUBeMeyGt1AQ+kVKAxnl60VN02uo1SWgkWRgEKUIACFOiigB4wZue4i4hM3mMCtSG9xjEiy+Q1jVn2WH36fMHOHJeogm4baUh2jvv8VwUBKEABCmSegDNRU4etyLyqs8Z9WkCh+mif9grX7MHqrYcwYvzXcIK3Sa12/xas3QOc8fWTgfIt2BouxMSRvqYLGvf8WLZkFY47fRSqD9ciFNYRylwoPGEMxg2KZFi1p730QNi/H6u/2A+EwwhF880dMhYThycqr7HgHt0J13yF9ZX5mDiyf6frERJCKaWqO52QCShAAQpQgAI9LOCMHOuwFT1cDxZPgU4JhEX4UH2i9aMT5BLY9wGm3j0bP13+VdxZP1751SxMv/tBbG0AdqyYhSnLdsWdb9oN71+F6S/7UbN6Aabe8Sze/fRTvLvyNUy4chq+/+dPnGC+7aXXOQV2v4+pd8zGi2s/xbtrP8S7K1dhzY707jsGdr2LKW/FmzWZdLRXr5Rphs1DHV3H8xSgAAUoQIF0EtADxnzEP51ahHU5agFXnetQMB/uo0rgiozu/u2BlaiYdgIKdKKqDbj1fb1TGMnEPQGnI/G3w+pXX8dPZ1vwbn8Hp995J+ZfNcop9qeXfg2Db5yNt/75HQxvJ71zsc569E/w8L99F9nOgRb/2EGETU+kBnYQtbaJPE/zBU0Cfj9cXq9zTcAfhMsbvT6aVTjoR9j0Irt5shYFNf/YZhpdH5cHk7MSmzTPpfWnBgUPAjjY+gyPUIACFKAABdJXgHGO07dtWLMOBKzHrMM24IqPkNBmkrAfmDADd4xeijd3RiZjbFjxKiZffxMuAxqnOSRMb2/Hb18YgRu+Hp1aEGpa9C1v1AUoA/DJIX/CpK0Obgu1WDLOj5ceuRvff+JReC+4Dav9fiz90wJ4L5iKwRd9Gzf9eR0CiFxzzl03YeAl05C/4Bk89fQCDLxkKvInP4o1NboUPxb/fi7yL5qGgRfchOc/q0Jg5xKcc/8S1AKo3f4avDN/D2cMuGodvn/XX1CBmrg0U/DASn22eX3W6hh5Ht05trHsT3PhXbAEFa1uqvUB3Sa6bXQbtT7LIxSgAAUoQIH0FnCmVTDOcXo3EmuXUEC5BA47fcOEp+MOOoOfo3HdLWfi1tc+18PGeO6RffjJdyYiF+0/M1ax9g1suHkaRiUcjfVh6p0T8MrHiadjxNUAcManH8e377obl9x1E7yTH8VHfqB213r8bd+J2LD4P3Be8Au8f/Bc7F2xHJWLFuLFx5/Ep37g0Mb1uPbmh+Bf8TrK/v4CNp08E/4Vy7H0+tcw793t2P4/v8CMiouddHufvw633vYkdhaehtPfX4SPg8CO1e8A257FhweBfR+9gS/PPAWVSx7EjM2TnDT+xc9g633/iud3hprV59xsD3BwG55/YhbuqZmOAz+bFhl1b35jrT7pNtFtE31WstV5HqAABShAAQqks4DTbYjGOea843RuKdatlYCpsKdaHVV/DagOYcy50zH5vlew8sKxeHLCDJQcn41HW+Uaf8CP1/64DvfN+2H8wbh9GxvXrce/XDMC2BJ3OOGufgzvNiz+1bUYYEdWVXaZDVhVDZTdPg2jfB7AHoJT85fi1vs/wGCvnpGgp3zYQP4kTBqvl8b2I380MGrYAKeEfgMnoLIugG3l24FK4Mf3/x113hxcMykH+6qH4+brgVfXrEPhtjH4Q9FpeGfVOtR+sh2zZ47BwcV1KLvxQjjj4b6T8eM7J+A/d1fhlLj61Fa4sWLRg1gB4LI7C5CX8L5aH6xWgG6b1md4hAIUoAAFKJDeAoxznN7tw9p1IKAUttbELTzR5uV6JsS2EFzmKNx7/UZMve1RzLthErLjJjk4vyUGAwjDRiAYdLbwwTWYjSswdVBk2LjZNf4qrFzyS8x8fxKmjfFF5gq3TN+qQh7k6FFV03S2ptORzvK+D17AzM1fxx8W/gyPzZoB6FFtp1AgGJ3N0eAkilwfm+BxwohC4MTpeHphGV665wcY681HYb6Js75zHZ68bw7mZk/AtG+dh62PzMGtFVNxyXAPBo3IxdzfvY0qnV/NFvzi8fU45/hYVIpI/gjXYfLNz8C/4nnkPj4Lj33iXN1U7Tb2dJvotmnjNA9TgAIUoAAF0lZADxhHf/SmbR1ZMQq0KRBQau0BJS4/QcDT5kW6M+pyA6PhdIXPvegK4IVPcOXpPsDe15TMMwgbHp+N/N82Hbp9+om444ZfNY2Ytrxm0vVY+tfZOMMLbGh5DsAjv3kdPzo5LlTb6Ka8I3suHFcIHEKk811w2lRcdt9s5E8uxekTpuIyrMcDb+7AlMZkLvgKnd5145GTcrMxbtocPHL/95A/OXL45vuewTgtMvIMzAOw95tjkO314bujgWGXnecMtRdMm4MnPvwehk1+0El0zZ1P4NqROfhLXH30iX6O2gl4/Hc/weAbH8akxWWYGHdLjRWJ2zmgENRtE3eIuxSgAAUoQIGMEXAWttJDyAznljFtxopGBYqKiqYNN4w/nm0a/Tfm+XDVWWcm1aa2xo9sXyQ6RFIzbjczPXINZOtIFXYQAXiOOvpEm5En2inPSQMv9PTirryC4TCeXPE+bvcYeDloV+2R8rqSkpIlXcmTaSlAAQpQgAKpFrCKLeWMHLNjnGp6lpcMAdu2NxwShqfl0sXJyFvnkeeLWzEkWZl2mI/Z1FE1PYnDvrWRh8vT+Y78saRJWLwQiLXDIQWPbpuE1/EgBShAAQpQII0FdJxjLh+dxg3EqrUvUFZWtksqNPihImtIt385z3ajgF7J2xDAEaWg20S3TTcWx6wpQAEKUIAC3SLAOMfdwspMUylgCvzvYakQltGHyFJZOMtqFLClhAGBvVJBt0njCe5QgAIUoAAFMkyAcY4zrMFY3eYCDRJLDwN+f0iHSuOrpwTqgkF4DYHdCnW6TXqqHiyXAhSgAAUo0FUBp3McjXPc1byYngKpFwjgxa9smEcagghF4wenvhIssby6BgMBbLGVGbSDL1GEAhSgAAUokIkCjHOcia3GOjcTsB62yk2B9ccZAp/sjQvN1uwqfuhugY937UKutPWUivULFy5kQ3Q3OPOnAAUoQIFuEdADxnwgr1tomWkqBRpsPBqw7bpV27Zh35HqVBbNsgCs3P4lZCCA3UrU1EvZ/qKDFKMABShAAQqkuYDTOdZhK9K8nqweBdoWcOGvfogjZ0Dh+bUfYsO+8rav5ZmkCYSlxJKNn+GDL3fgXEhUK1VjGMYrSSuAGVGAAhSgAAV6QMBZmmv58uXLe6BsFkmBpAgsW7ZMnTdlco0S4vxLXUbWioOV2FxRgbzsbPTPyYYQsQi8SSmuz2fSEA5j/e7d+NvHnyLH78eVLoF3bVVTKeX98+fP/7DPAxGAAhSgAAUyVmDylMmNcfsz9iZYcQpogRkzZphnnHralqkuY9QYA9gsFdbBQIMwcErhUIwbMgRDfHnsKB/jl4t+2HFH5WFs2VeOrYcO4URD4CyhMNgQ2GorLLXlto82bhy3aNEixtQ7RmMmowAFKECB9BDgkFp6tANrkQSBefPmTfEZxuKZHjPXFR0tPigVNitgGwTqlcLI/v0xuF8/DM33YWh+PnLc7iSU3PuyOFTnx/6aGuyvrsb+qirsra1DocvEGGVjrCGQE/UNK4X/Ctp1NVJeWlpayr9A9b4vBd4RBShAgT4n4HSOddgKLiHd59q+V97wwiLrxdEmLr/YbWa3vMEapbBHKlRA4IAwUGFL2Eoh1+1CrtuN3KwsZ3O7XMh2u2EaBtymCUOIyGboJZIj+7G8nT6iApy/wSg4I9N6tbiUvvR3cawOuuDogoH6sFTKOaX/taWClNI5FrRt6DnDDcEgguEwagMB6FjFdaEwAnYYeYaJAgMokNIZHR4uAHeC6Sl/D9mBbTZevb/Euial98zCKEABClCAAt0k4NL5RuMcz++mMpgtBVImUBXwz/oi23v+GENljzKb/2HEJwTGmwLjndoo/YWPoAL8sFEXtOFvCCAAhQYFNAAICQEbArYApNPnFNGOZqQvGuuPxr+n7EbjCkrQN3bO6uNNm4J++tZQgAkF/Y3vVgpeAAMEcBIEvALwGnozdEi2yMtoO6DNdlvhC4UjRwL+WXHV4S4FKEABClAgYwX0gLHTOc7YO2DFKdBC4KGHHqqx5lpXvink0muF4R1gxHp5LS6MfvQIAQ+A/o2XNe4kSJDiEeEENUjOodg9xt47n6uervJWWNUFpP09bd75HJiCAhSgAAUokH4CjHOcfm3CGiVBwCqzVoUkbv9zSPr9qZ7ikIT6p3sW2vQPIYmgVHeUlpauTPf6sn4UoAAFKECBzggwznFntHhtxghYpdbvQkoueCYo0cAOctLaTVu+FJL1HqnmauOkZcyMKEABClCAAmkiwDjHadIQrEbyBd5Zvvz9i84/v/8XCqeNNZCV6IGy5Jfae3PUI8ZrbezcLdULc0vm/6z33invjAIUoAAF+qoA4xz31ZbvY/e9oLj45y6IO65yG97jOpiD3Mdojvp2D0kFPU0lpPDreSXWPUedkBdSgAIUoAAFMkzg2J/IybAbZXX7tkBRUdHlHmG8MNElss42hFuHZ+OrYwEdCm6tVKE1YdUQVPL6kpKSVztOxSsoQAEKUIACmSvgzDnWYSsy9xZYcwp0LKA7deEGTPgwrJY/F5LVn9hKBTkXuU04baONtJU203bsGLfJxRMUoAAFKNCLBJzhM6vYUtZ8i0NpvahheSttC1g/s6a7DfxIClx4vCFCYw34RhoCeX18NLlWKeyUClslanZL5TYU3g5JPGUtsN5oW5NnKEABClCAAr1HgHGOe09b8k46IRDt7L1hzbEGfumzLy2XxtXLoCYbgGuogeAAIXIGCGR5oZdJBvRSey4BZ+EMHRhc/xap/+SS7r9N6qjMkcVLgDCimwICAOqdxU8UDis0HFaqvlzCI4GwCayoV/Jlo8ZYPO8xq7ITrLyUAhSgAAUokPECOs4xFwHJ+GbkDRyrgBXp/D0PQG8o/vfiE75yizN3GPZojxDjXcCJgBikgP5KIEcqZCnApbdomQZUpI+sBIRQUPr9WOvTlXSxsvW7k09kMT/dN4ahlA0hQoZAg1CoF0AVoA6GgR1hKTZBYbsKqXXzfz7/q67UgWkpQAEKUIACvUHA+SEvlbR6w83wHijQFYFo55AdxK4gMi0FKEABClAgwwUY5zjDG5DVpwAFKEABClCAAhRIjoCOc8wXBShAAQpQgAIUoAAFKEABClCAAhSgAAUoQAEKUCCBAOMcJ0DhIQpQgAIUoAAFKECBPifgLAKiw1b0uTvnDVOAAhSgAAUoQAEKUCBOQA8YO53juGPcpQAFKEABClCAAhSgQJ8U0APG7Bz3yabnTVOAAhSgAAUoQAEKJBJIGOfYGVKOm2qh4yCXlJTMj2XA80XF8VNR6MOvD35/8P8H/v8YEeDPB/584M/Hpqmq7B9kXv8A/7+Y1v8BHvJ6LTV95FwAAAAASUVORK5CYII="
- }
- },
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Introduction\n",
- "\n",
- "Deep Learning frameworks enable Machine Learning (ML) practitioners to build and train ML models. However, the process of deploying ML models in production to serve predictions (also known as inferences) in real time is more complex. It requires that ML practitioners build a scalable and performant model server, which can host these models and handle inference requests at scale. Model Server for Apache MXNet (MMS), was developed to address this hurdle. MMS is a highly scalable, production ready inference server. MMS was designed in a ML/DL framework agnostic way to host models trained in any ML/DL framework.\n",
- "\n",
- "In this blog post, we will showcase how anyone could use Model Server for Apache MXNet (MMS) to host their model trained using any Machine Learning/Deep Learning (ML/DL) framework or tool kit in production. We chose Amazon Sagemaker service for production hosting - this PaaS solution does a lot of heavy lifting to provide infrastructure and allows users to focus on their use cases. We will be using 'Bring your own Inference code with Amazon Sagemaker hosting' approach, where users could bring their models together with all necessary dependencies, libraries, frameworks and other components compiled inside of a single custom-built docker container and host it on Sagemaker. \n",
- "\n",
- "To showcase the true 'ML/DL framework agnostic architecture' of MMS, we chose to launch a model trained with 'PaddlePaddle' framework into production.\n",
- "\n",
- "The overall picture of steps involved to take a model trained on any ML/DL framework to Amazon Sagemaker using MMS BYO container looks as follows:\n",
- "\n",
- "\n",
- "\n",
- "As shown in the picture above, in order to bring your own ML/DL framework to Amazon Sagemaker using MMS Bring Your Own (BYO) container, we need two main components\n",
- "\n",
- "1. **Model artifacts/Model Archive**: These are all the artifacts required to run your model on a given host. This contains the following:\n",
- " 1. **Model files**, which are usually symbols and weights. These are the artifacts of training a model.\n",
- " 2. **Custom Service File**: This file contains the entry-point which gets called every time when inference request is received and served by MMS. This file generally contains the logic to initialize the model in a particular ML/DL framework, preprocess the incoming request, run inference in a particular ML/DL framework and post-process logic which takes the data coming out of framework's inference method and converts it to end-user consumable data.\n",
- " 3. **MANIFEST File**: This is the interface between custom service file and the MMS. This file is generated by running a tool that comes as part of MMS distribution, called 'model-archiver'.\n",
- "2. **Container artifact**: To load and run a model written in a custom DL framework on Sagemaker, you need to bring a container that will be run on Sagemaker service. In this document we will show how to use MMS base container and extend it to support custom DL frameworks and other model dependencies. The MMS base container is a docker container that comes with a highly scalable and performant model-server which is readily launchable onto Sagemaker service.\n",
- "In the following sections, we will see each of the above components in detail.\n",
- "\n",
- "## Preparing a Model\n",
- "MMS container is completely ML/DL framework agnostic. Users can write models in a ML/DL framework of their choice and bring it to Sagemaker with MMS BYO container to get the features of scalability and performance. In this blogpost, we chose to showcase this by bringing in a model written for PaddlePaddle framework. Lets look at how to prepare a PaddlePaddle model in the following sections. The model artifact is readily available at <*TODO: Update this with the S3 link with model.tar.gz*>.\n",
- "\n",
- "### Preparing Model Artifacts\n",
- "We are going to use [Understand Sentiment](https://github.com/PaddlePaddle/book/tree/develop/06.understand_sentiment) example that is available and published in examples section of PaddlePaddle repository. First of all we need to create a model. In order to do that we followed instructions provided in [PaddlePaddle/book](https://github.com/PaddlePaddle/book) repository: downloaded container and ran training by the notebook that is provided as part of the example. We used 'Stacked Bidirectional LSTM' network for our training and trained the model for 100 epochs. At the end of this training exercise, we get the following list of trained model artifacts.\n",
- "\n",
- "```bash\n",
- "!ls\n",
- "embedding_0.w_0 fc_2.w_0 fc_5.w_0 learning_rate_0 lstm_3.b_0 moment_10 moment_18 moment_25 moment_32 moment_8\n",
- "embedding_1.w_0 fc_2.w_1 fc_5.w_1 learning_rate_1 lstm_3.w_0 moment_11 moment_19 moment_26 moment_33 moment_9\n",
- "fc_0.b_0 fc_3.b_0 fc_6.b_0 lstm_0.b_0 lstm_4.b_0 moment_12 moment_2 moment_27 moment_34\n",
- "fc_0.w_0 fc_3.w_0 fc_6.w_0 lstm_0.w_0 lstm_4.w_0 moment_13 moment_20 moment_28 moment_35\n",
- "fc_1.b_0 fc_3.w_1 fc_6.w_1 lstm_1.b_0 lstm_5.b_0 moment_14 moment_21 moment_29 moment_4\n",
- "fc_1.w_0 fc_4.b_0 fc_7.b_0 lstm_1.w_0 lstm_5.w_0 moment_15 moment_22 moment_3 moment_5\n",
- "fc_1.w_1 fc_4.w_0 fc_7.w_0 lstm_2.b_0 moment_0 moment_16 moment_23 moment_30 moment_6\n",
- "fc_2.b_0 fc_5.b_0 fc_7.w_1 lstm_2.w_0 moment_1 moment_17 moment_24 moment_31 moment_7\n",
- "```\n",
- "\n",
- "These artifacts constitute a PaddlePaddle model. We copy these artifacts from within training container to localhost so that it will be easier to begin preparation of the model for production hosting. To learn more on how to copy files from inside a docker container to location outside of it please refer to [Docker CLI](https://docs.docker.com/engine/reference/commandline/cp/).\n",
- "\n",
- "### Writing Custom Service Code\n",
- "We now have model files required to host the model in production. We can now define a custom service file which knows how to use these files and also knows how to 'preprocess' the raw request coming into the server and how to 'postprocess' the responses coming out of the PaddlePaddle framework's 'infer' method. For this, we modified the notebook example written to test the trained model **. Let's look at some code. \n",
- "\n",
- "We created a custom service file called 'paddle_sentiment_analysis.py'. Here, we first define a class called 'PaddleSentimentAnalysis' which contains methods to initialize the model and also defines pre-processing, post-processing and inference methods. Refer [Custom Service Code](https://github.com/awslabs/mxnet-model-server/blob/master/docs/custom_service.md) document to learn how to write your custom-service code. The skeleton of this file is as follows:\n",
- "\n",
- "```bash\n",
- "$ cat paddle_sentiment_analysis.py\n",
- "```\n",
- "```python\n",
- "\n",
- "from __future__ import print_function\n",
- "import paddle\n",
- "import paddle.fluid as fluid\n",
- "import paddle.dataset as dataset\n",
- "from functools import partial\n",
- "\n",
- " \n",
- "class PaddleSentimentAnalysis(object):\n",
- " def __init__(self):\n",
- " ...\n",
- "\n",
- " def initialize(self, context):\n",
- " \"\"\"\n",
- " This method is used to initialize the network and read other artifacts.\n",
- " \"\"\"\n",
- " ...\n",
- " \n",
- " def preprocess(self, data):\n",
- " \"\"\"\n",
- " This method is used to convert the string requests coming from client \n",
- " into tensors. \n",
- " \"\"\"\n",
- " ...\n",
- "\n",
- " def inference(self, input):\n",
- " \"\"\"\n",
- " This method runs the tensors created in preprocess method through the \n",
- " DL framework's infer method.\n",
- " \"\"\"\n",
- " ...\n",
- "\n",
- " def postprocess(self, output, data):\n",
- " \"\"\"\n",
- " Here the values returned from the inference method is converted to a \n",
- " human understandable response.\n",
- " \"\"\"\n",
- " ...\n",
- " \n",
- "\n",
- "_service = PaddleSentimentAnalysis()\n",
- "\n",
- "\n",
- "def handle(data, context):\n",
- "\"\"\"\n",
- "This method is the entrypoint \\\"handler\\\" method that is used by MMS.\n",
- "Any request coming in for this model will be sent to this method.\n",
- "\"\"\"\n",
- " if not _service.initialized:\n",
- " _service.initialize(context)\n",
- "\n",
- " if data is None:\n",
- " return None\n",
- "\n",
- " pre = _service.preprocess(data)\n",
- " inf = _service.inference(pre)\n",
- " ret = _service.postprocess(inf, data)\n",
- " return ret\n",
- "```"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Note about Permissions\n",
- "Running this notebook requires permissions in addition to the normal **SageMakerFullAccess** permissions. This is because we'll creating new repositories in Amazon ECR. The easiest way to add these permissions is simply to add the managed policy **AmazonEC2ContainerRegistryFullAccess** to the role that you used to start your notebook instance. There's no need to restart your notebook instance when you do this, the new permissions will be available immediately."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Creating Model artifact file to be hosted on sagemaker\n",
- "In order to load this model onto Sagemaker platform with MMS BYO container, we need to do the following:\n",
- "\n",
- "1. Create a MANIFEST file, which is used by MMS as a model's metadata to load and run the model.\n",
- "2. Add the above custom-service file and the trained model-artifacts, along with the MANIFEST file, to a .tar.gz file.\n",
- "\n",
- "Let's use 'model-archiver' tool, to accomplish the above points. Before we use the tool to create a ''.tar.gz' artifact, we need to collect all the model artifacts, including the custom-service-file mentioned above, into a separate folder. For ease of getting started, we have uploaded all the model artifacts onto an [S3 bucket](https://s3.amazonaws.com/model-server/blog_artifacts/PaddlePaddle_blog/sentiment.tar.gz). Lets run the following commands to get this artifact onto your host:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!(curl https://s3.amazonaws.com/model-server/blog_artifacts/PaddlePaddle_blog/artifacts.tgz | tar zxvf -) 2>/dev/null"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!ls -R artifacts/sentiment"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now that we have the model artifacts, let's convert this to a model artifact that can be hosted on Sagemaker. \n",
- "\n",
- "### Prerequisites\n",
- "Before we proceed with preparing a Sagemaker model-artifact and endpoint, we need the following:\n",
- "#### Software packages and tools\n",
- "1. pip\n",
- "1. Docker\n",
- "1. Model-archiver tool\n",
- "1. Sagemaker SDK\n",
- "1. Boto3 \n",
- "\n",
- "#### AWS user account with following permissions\n",
- "We will need AWS account user with permissions to \n",
- "1. Create roles (or access to an already existing Sagemaker role)\n",
- "2. Create Sagemaker Endpoint\n",
- "3. Create an ECR repository and upload a container to the repository\n",
- "4. Create an S3 bucket and upload an artifact to S3 bucket"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We are now ready to create a sagemaker model artifact. For this, we use the \"model-archiver\" tool to create a Sagemaker model artifact. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!pip install -U mxnet-model-server"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!model-archiver -f --model-name paddle_sentiment \\\n",
- "--handler paddle_sentiment_analysis:handle \\\n",
- "--model-path artifacts/sentiment --export-path . --archive-format tgz"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The above command would create an model artifact called `paddle_sentiment.tar.gz`, which we will use to host our endpoint. Let's verify if this model artifact is created."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!ls"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Next let's take a look at how to build a container with it and bring it into Sagemaker.\n",
- "\n",
- "### Building your own BYO container with MMS\n",
- "\n",
- "In this section, we build our own MMS based container which can be brought onto Sagemaker (also known as BYO Container).\n",
- "\n",
- "To help with this process, every released version of MMS comes with a corresponding MMS base container, hosted on [DockerHub](https://hub.docker.com/r/awsdeeplearningteam/mxnet-model-server/tags) which can be hosted on the Sagemaker platform.\n",
- "\n",
- "For this example, we will use container tagged *awsdeeplearningteam/mxnet-model-server:base_cpu_py3.6*. To host the model created in the above section, we need to install 'PaddlePaddle' and 'numpy' packages in the container. This can be done by creating a Dockerfile which extends from the base MMS image and installs the above python packages. Here is how its content should look like:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!cat artifacts/Dockerfile.paddle.mms"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now that we have Dockerfile that describes our BYO container let's build it:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "!cd artifacts && docker build -t paddle-mms -f Dockerfile.paddle.mms ."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Creating Sagemaker endpoint with PaddlePaddle model\n",
- "Before we go on and create a Sagemaker endpoint for our model, we need to do some preparations:\n",
- "\n",
- "### Upload the Sagemaker model artifact to a S3 bucket\n",
- "Upload the model archive **sentiment.tar.gz** created above to a S3 bucket. Here we uploaded it to the S3 bucket called paddle_paddle. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import boto3, os, uuid\n",
- "\n",
- "s3 = boto3.resource(\"s3\")\n",
- "s3_bucket_name = \"paddle-sentiment-model-\" + str(uuid.uuid1())\n",
- "local_model_artifact = s3_model_artifact = \"paddle_sentiment.tar.gz\""
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now lets create a bucket called **paddle-sentiment-model**. Here is where we will copy the model, **paddle_sentiment.tar.gz**, that we had created above."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import sagemaker\n",
- "from sagemaker import get_execution_role\n",
- "import boto3\n",
- "from botocore.exceptions import ClientError\n",
- "import json\n",
- "\n",
- "sess = sagemaker.Session()\n",
- "account = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
- "region = sess.boto_session.region_name\n",
- "\n",
- "s3.create_bucket(Bucket=s3_bucket_name, CreateBucketConfiguration={\"LocationConstraint\": region})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "s3.meta.client.upload_file(local_model_artifact, s3_bucket_name, s3_model_artifact)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We now have **paddle_sentiment.tar.gz** on S3 in our account. Now let's look at having the container that we built on ECR, so that we can go ahead and set up our Sagemaker Endpoint."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Upload the container image to ECR\n",
- "We had built an image called **paddle-mms** above. We need to upload this to a Amazon ECR in our account."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "%%sh\n",
- "\n",
- "# The name of our algorithm\n",
- "algorithm_name=paddle-mms\n",
- "\n",
- "account=$(aws sts get-caller-identity --query Account --output text)\n",
- "\n",
- "# Get the region defined in the current configuration (default to us-west-2 if none defined)\n",
- "region=$(aws configure get region)\n",
- "# specifically setting to us-east-1 since during the pre-release period, we support only that region.\n",
- "region=${region:-us-east-1}\n",
- "\n",
- "echo \"region is \" $region\n",
- "\n",
- "fullname=\"${account}.dkr.ecr.${region}.amazonaws.com/${algorithm_name}:latest\"\n",
- "\n",
- "echo $fullname\n",
- "# If the repository doesn't exist in ECR, create it.\n",
- "\n",
- "aws ecr describe-repositories --repository-names \"${algorithm_name}\" > /dev/null 2>&1\n",
- "\n",
- "if [ $? -ne 0 ]\n",
- "then\n",
- " aws ecr create-repository --repository-name \"${algorithm_name}\" > /dev/null\n",
- "fi\n",
- "\n",
- "# Get the login command from ECR and execute it directly\n",
- "$(aws ecr get-login --region ${region} --no-include-email)\n",
- "\n",
- "# Build the docker image locally with the image name and then push it to ECR\n",
- "# with the full name.\n",
- "\n",
- "docker tag ${algorithm_name}:latest ${fullname}\n",
- "\n",
- "docker push ${fullname}"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "This pushes the \"paddle-mms\" container to Amazon ECR in your account."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Creating Sagemaker Endpoint\n",
- "Now that the model and container artifacts are uploaded onto S3 and ECR respectively, we can go ahead and create Sagemaker endpoint. To do that we need to complete following steps\n",
- "\n",
- "\n",
- "#### Sagemaker role\n",
- "\n",
- "Before we go onto create an Sagemaker endpoint, we need to setup an IAM role which has **AmazonSageMakerFullAccess** and **AmazonS3FullAccess** and **AmazonEC2ContainerRegistryFullAccess** policy attached to it. "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import sagemaker\n",
- "from sagemaker import get_execution_role\n",
- "import boto3\n",
- "from botocore.exceptions import ClientError\n",
- "import json\n",
- "\n",
- "sess = sagemaker.Session()\n",
- "account = sess.boto_session.client(\"sts\").get_caller_identity()[\"Account\"]\n",
- "region = sess.boto_session.region_name\n",
- "# NOTE: If you already have a sagemaker execution role created with above attached policies, use it instead of calling get_execution_role()\n",
- "sm_role = get_execution_role()\n",
- "inference_image = \"{}.dkr.ecr.{}.amazonaws.com/paddle-mms:latest\".format(account, region)\n",
- "s3_url = \"s3://{}/{}\".format(s3_bucket_name, s3_model_artifact)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We created the role required to launch our Sagemaker endpoint above. Now let's use the Sagemaker SDK to launch an endpoint."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "inf_handler = None"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from sagemaker.model import Model\n",
- "\n",
- "endpoint = \"PaddleSentiment\"\n",
- "paddle_model = Model(model_data=s3_url, image=inference_image, role=sm_role)\n",
- "try:\n",
- " inf_handler = paddle_model.deploy(1, \"ml.m4.xlarge\", endpoint_name=endpoint)\n",
- "except ClientError as e:\n",
- " if \"ValidationException\" == e.response[\"Error\"][\"Code\"]:\n",
- " print('The endpoint \"{}\"already exists'.format(endpoint))\n",
- " pass\n",
- " else:\n",
- " raise"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "This creats an sagemaker endpoint using the model artifact \"paddle_sentiment.tar.gz\".\n",
- "\n",
- "### Testing the endpoint\n",
- "Let's test the endpoint. To do this, we will send a movie review to the endpoint \"paddle-sentiment\"."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from sagemaker.predictor import (\n",
- " json_serializer,\n",
- " csv_serializer,\n",
- " json_deserializer,\n",
- " RealTimePredictor,\n",
- ")\n",
- "\n",
- "predictor = RealTimePredictor(endpoint=endpoint, sagemaker_session=sess)\n",
- "\n",
- "message = \"This is an amazing movie.\"\n",
- "print(predictor.predict(message).decode(\"utf-8\"))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "You would get a response showing that the review was positive.\n",
- "### Delete Endpoint\n",
- "After testing your endpoint, you could delete the endpoint you created as follows."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "sess.delete_endpoint(endpoint)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Conclusion\n",
- "We have just shown how to build and host PaddlePaddle model on Sagemaker using MMS BYO container. This flow can be reused with minor modifications in order to build BYO containers serving inference traffic on Sagemaker endpoints with MMS for models built using many ML/DL frameworks, not just PaddlePaddle."
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "conda_python3",
- "language": "python",
- "name": "conda_python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/sagemaker-python-sdk/pytorch_mnist/pytorch_mnist.ipynb b/sagemaker-python-sdk/pytorch_mnist/pytorch_mnist.ipynb
index 4c78865757..7c91e84339 100644
--- a/sagemaker-python-sdk/pytorch_mnist/pytorch_mnist.ipynb
+++ b/sagemaker-python-sdk/pytorch_mnist/pytorch_mnist.ipynb
@@ -69,7 +69,7 @@
"sagemaker_session = sagemaker.Session()\n",
"\n",
"bucket = sagemaker_session.default_bucket()\n",
- "prefix = 'sagemaker/DEMO-pytorch-mnist'\n",
+ "prefix = \"sagemaker/DEMO-pytorch-mnist\"\n",
"\n",
"role = sagemaker.get_execution_role()"
]
@@ -114,11 +114,11 @@
"MNIST.mirrors = [\"https://sagemaker-sample-files.s3.amazonaws.com/datasets/image/MNIST/\"]\n",
"\n",
"MNIST(\n",
- " 'data',\n",
+ " \"data\",\n",
" download=True,\n",
" transform=transforms.Compose(\n",
" [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))]\n",
- " )\n",
+ " ),\n",
")"
]
},
@@ -144,8 +144,8 @@
}
],
"source": [
- "inputs = sagemaker_session.upload_data(path='data', bucket=bucket, key_prefix=prefix)\n",
- "print('input spec (in this case, just an S3 path): {}'.format(inputs))"
+ "inputs = sagemaker_session.upload_data(path=\"data\", bucket=bucket, key_prefix=prefix)\n",
+ "print(\"input spec (in this case, just an S3 path): {}\".format(inputs))"
]
},
{
@@ -202,16 +202,15 @@
"source": [
"from sagemaker.pytorch import PyTorch\n",
"\n",
- "estimator = PyTorch(entry_point='mnist.py',\n",
- " role=role,\n",
- " py_version='py3',\n",
- " framework_version='1.8.0',\n",
- " instance_count=2,\n",
- " instance_type='ml.c5.2xlarge',\n",
- " hyperparameters={\n",
- " 'epochs': 1,\n",
- " 'backend': 'gloo'\n",
- " })"
+ "estimator = PyTorch(\n",
+ " entry_point=\"mnist.py\",\n",
+ " role=role,\n",
+ " py_version=\"py38\",\n",
+ " framework_version=\"1.11.0\",\n",
+ " instance_count=2,\n",
+ " instance_type=\"ml.c5.2xlarge\",\n",
+ " hyperparameters={\"epochs\": 1, \"backend\": \"gloo\"},\n",
+ ")"
]
},
{
@@ -532,7 +531,7 @@
}
],
"source": [
- "estimator.fit({'training': inputs})"
+ "estimator.fit({\"training\": inputs})"
]
},
{
@@ -562,7 +561,7 @@
}
],
"source": [
- "predictor = estimator.deploy(initial_instance_count=1, instance_type='ml.m4.xlarge')"
+ "predictor = estimator.deploy(initial_instance_count=1, instance_type=\"ml.m4.xlarge\")"
]
},
{
@@ -600,16 +599,16 @@
"metadata": {},
"outputs": [],
"source": [
- "import gzip \n",
+ "import gzip\n",
"import numpy as np\n",
"import random\n",
"import os\n",
"\n",
- "data_dir = 'data/MNIST/raw'\n",
+ "data_dir = \"data/MNIST/raw\"\n",
"with gzip.open(os.path.join(data_dir, \"t10k-images-idx3-ubyte.gz\"), \"rb\") as f:\n",
" images = np.frombuffer(f.read(), np.uint8, offset=16).reshape(-1, 28, 28).astype(np.float32)\n",
"\n",
- "mask = random.sample(range(len(images)), 16) # randomly select some of the test images\n",
+ "mask = random.sample(range(len(images)), 16) # randomly select some of the test images\n",
"mask = np.array(mask, dtype=np.int)\n",
"data = images[mask]"
]
@@ -710,9 +709,7 @@
"metadata": {},
"outputs": [],
"source": [
- "sagemaker_session.delete_endpoint(\n",
- " endpoint_name = predictor.endpoint_name\n",
- ")"
+ "sagemaker_session.delete_endpoint(endpoint_name=predictor.endpoint_name)"
]
}
],
diff --git a/sagemaker-script-mode/index.rst b/sagemaker-script-mode/index.rst
index d289bf1480..5d2f1a1e1a 100644
--- a/sagemaker-script-mode/index.rst
+++ b/sagemaker-script-mode/index.rst
@@ -16,3 +16,5 @@ SageMaker Script Mode at Increasing Levels of Customization
:maxdepth: 1
sagemaker-script-mode
+ pytorch_bert/deploy_bert
+ sklearn/sklearn_byom
diff --git a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling-multi-gpu-single-node.ipynb b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/gpt-2.ipynb
similarity index 71%
rename from sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling-multi-gpu-single-node.ipynb
rename to sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/gpt-2.ipynb
index 6c4ff1b8aa..c68f7f0089 100644
--- a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling-multi-gpu-single-node.ipynb
+++ b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/gpt-2.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "markdown",
- "id": "c5608edd",
+ "id": "aa619cfc",
"metadata": {},
"source": [
"# Compile and Train the GPT2 Model using the Transformers Trainer API with the SST2 Dataset for Single-Node Multi-GPU Training"
@@ -10,7 +10,7 @@
},
{
"cell_type": "markdown",
- "id": "ec894c6c",
+ "id": "2f479baf",
"metadata": {},
"source": [
"1. [Introduction](#Introduction) \n",
@@ -25,7 +25,7 @@
},
{
"cell_type": "markdown",
- "id": "9e9d46c4",
+ "id": "83e922ff",
"metadata": {},
"source": [
"## SageMaker Training Compiler Overview\n",
@@ -40,14 +40,14 @@
"\n",
"In this demo, you'll use Hugging Face's `transformers` and `datasets` libraries with Amazon SageMaker Training Compiler to train the `gpt-2` model on the `Stanford Sentiment Treebank v2 (SST2)` dataset. To get started, we need to set up the environment with a few prerequisite steps, for permissions, configurations, and so on. \n",
"\n",
- "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, `Python 3 (PyTorch x.y Python 3.x CPU Optimized)` or `conda_pytorch_p36` respectively.\n",
+ "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, `Python 3 (PyTorch x.y Python 3.x CPU Optimized)` or `conda_pytorch_p38` respectively.\n",
"\n",
- "**NOTE:** This notebook uses two `ml.p3.8xlarge` instances that have multiple GPUs. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
+ "**NOTE:** This notebook uses 2 `ml.g4dn.12xlarge` instances that have multiple GPUs. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
]
},
{
"cell_type": "markdown",
- "id": "3977fe0f",
+ "id": "7bb2751c",
"metadata": {},
"source": [
"## Development Environment "
@@ -55,54 +55,44 @@
},
{
"cell_type": "markdown",
- "id": "dbc4930a",
+ "id": "b945c6f4",
"metadata": {},
"source": [
"### Installation\n",
"\n",
- "This example notebook requires the **SageMaker Python SDK v2.70.0** and **transformers v4.11.0**."
+ "This example notebook requires the **SageMaker Python SDK v2.108.0**."
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "7045eb46",
+ "id": "37613be5",
"metadata": {},
"outputs": [],
"source": [
- "!pip install --force-reinstall sagemaker==2.70.0"
+ "!pip install \"sagemaker>=2.108.0\" botocore boto3 awscli pandas numpy --upgrade"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "25f110f1",
- "metadata": {},
- "outputs": [],
- "source": [
- "!pip install transformers==4.11.0"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "f7a8ceb9",
+ "id": "5bed8ad5",
"metadata": {},
"outputs": [],
"source": [
"import botocore\n",
"import boto3\n",
"import sagemaker\n",
- "import transformers\n",
"import pandas as pd\n",
"\n",
"print(f\"sagemaker: {sagemaker.__version__}\")\n",
- "print(f\"transformers: {transformers.__version__}\")"
+ "print(f\"boto3: {boto3.__version__}\")\n",
+ "print(f\"botocore: {botocore.__version__}\")"
]
},
{
"cell_type": "markdown",
- "id": "6bcc3a46",
+ "id": "51a693fa",
"metadata": {},
"source": [
"Copy and run the following code if you need to upgrade IPython widgets for `datasets` library and restart kernel. This is only needed when preprocessing is done in the notebook.\n",
@@ -118,7 +108,7 @@
},
{
"cell_type": "markdown",
- "id": "5e5c0cdb",
+ "id": "5a4f105f",
"metadata": {},
"source": [
"### SageMaker environment "
@@ -127,7 +117,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "655beb77",
+ "id": "8a56b484",
"metadata": {},
"outputs": [],
"source": [
@@ -152,129 +142,148 @@
},
{
"cell_type": "markdown",
- "id": "12032413",
+ "id": "97e3b0d2",
"metadata": {},
"source": [
"## SageMaker Training Job\n",
"\n",
- "To create a SageMaker training job, we use a `HuggingFace` estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through `entry_point`, which `instance_type` to use for training, which `hyperparameters` to pass, and so on.\n",
+ "To create a SageMaker training job, we use an estimator. We use a `HuggingFace` estimator for SageMaker Training Compiler. Using the estimator, you can define which training script should SageMaker use through `entry_point`, which `instance_type` to use for training, which `hyperparameters` to pass, and so on.\n",
"\n",
- "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the `HuggingFace` Deep Learning Container, uploads your training script, and downloads the data from `sagemaker_session_bucket` into the container at `/opt/ml/input/data`.\n",
+ "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the appropriate `HuggingFace` Deep Learning Container, uploads your training script, and downloads the data from `sagemaker_session_bucket` into the container at `/opt/ml/input/data`.\n",
"\n",
- "In the following section, you learn how to set up two versions of the SageMaker `HuggingFace` estimator, a native one without the compiler and an optimized one with the compiler."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5f608b6c",
- "metadata": {},
- "source": [
- "### Training Setup"
+ "First, we define some basic parameters common to all estimators.\n",
+ "\n",
+ "**Note**: We recommend you to turn the SageMaker Debugger's profiling and debugging tools off to avoid additional overheads."
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "182822a2",
+ "id": "f8f50795",
"metadata": {},
"outputs": [],
"source": [
- "# Here we configure the training job. Please configure the appropriate options below:\n",
- "EPOCHS = 100\n",
- "\n",
- "# Choose between Causal Language Model and Masked Language Model\n",
- "LANGUAGE_MODELING_LOSS = \"clm\" # or \"mlm\"\n",
- "\n",
- "MODEL_NAME = \"gpt2\"\n",
- "TOKENIZER_NAME = \"gpt2\"\n",
- "MODEL_CONFIG = \"model_type\"\n",
- "\n",
- "# For more information about the options, please look into the training scripts\n",
+ "estimator_args = dict(\n",
+ " source_dir=\"scripts\",\n",
+ " entry_point=\"run_clm.py\",\n",
+ " instance_type=\"ml.g4dn.12xlarge\",\n",
+ " instance_count=1,\n",
+ " role=role,\n",
+ " py_version=\"py38\",\n",
+ " volume_size=100,\n",
+ " disable_profiler=True, # Disabling SageMaker Profiler to avoid overheads during benchmarking\n",
+ " debugger_hook_config=False, # Disabling SageMaker Debugger to avoid overheads during benchmarking\n",
+ " base_job_name=\"trcomp-pt-example\",\n",
+ " metric_definitions=[\n",
+ " {\"Name\": \"summary_train_runtime\", \"Regex\": \"'train_runtime': ([0-9.]*)\"},\n",
+ " {\n",
+ " \"Name\": \"summary_train_samples_per_second\",\n",
+ " \"Regex\": \"'train_samples_per_second': ([0-9.]*)\",\n",
+ " },\n",
+ " {\"Name\": \"summary_train_steps_per_second\", \"Regex\": \"'train_steps_per_second': ([0-9.]*)\"},\n",
+ " {\"Name\": \"summary_train_loss\", \"Regex\": \"'train_loss': ([0-9.]*)\"},\n",
+ " {\"Name\": \"epoch\", \"Regex\": \"'epoch': ([0-9.]*)\"},\n",
+ " {\"Name\": \"train_loss\", \"Regex\": \"'loss': ([0-9.]*)\"},\n",
+ " {\"Name\": \"learning_rate\", \"Regex\": \"'learning_rate': ([0-9.]*)\"},\n",
+ " ],\n",
+ ")\n",
"\n",
- "# SageMaker Training Compiler currently only supports training on GPU\n",
- "# Select Instance type for training\n",
- "INSTANCE_TYPE = \"ml.p3.8xlarge\" # ml.p3.8xlarge is easily available. However, p3.16xlarge provides better performance.\n",
+ "# Since ml.g4dn.12xlarge instance has 4 GPUs, we set num_gpus_per_instance to 4\n",
"num_gpus_per_instance = 4"
]
},
{
"cell_type": "markdown",
- "id": "03b85427",
+ "id": "6c2b1bb3",
"metadata": {},
"source": [
- "### Training with Native PyTorch"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "2b6e9683",
- "metadata": {},
- "source": [
- "The batch size below is the maximum batch we could fit into the memory of a `ml.p3.8xlarge` instance. If you change the model, instance type, sequence length, and other parameters, you need to do some experiments to find the largest batch size that will fit into GPU memory.\n",
- "\n",
- "This example uses HuggingFace training script `run_clm.py`, which you can find it inside the `scripts` folder. \n",
- "\n",
- "To get the most performance out of the multi GPU configuration, we use a wrapper script to launch a single training process per GPU using `pytorch.distributed`. This allows us to get around the Python GIL bottleneck."
+ "Next, we define some basic arguments to be passed to the training script."
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "2d1efd5b",
+ "id": "db0f6871",
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.huggingface import HuggingFace\n",
+ "# Hyperparameters are passed to the training script as arguments.\n",
"\n",
- "# The original LR was set for a batch of 32. Here we scale learning_rate with an adjusted batch size and the number of GPUs per instance.\n",
- "batch_size_native = 8\n",
- "learning_rate_native = float(\"5e-5\") / 32 * batch_size_native * num_gpus_per_instance\n",
- "\n",
- "# hyperparameters are passed to the training entrypoint as arguments\n",
"hyperparameters = {\n",
- " \"training_script\": f\"run_{LANGUAGE_MODELING_LOSS}.py\",\n",
- " MODEL_CONFIG: MODEL_NAME,\n",
- " \"tokenizer_name\": TOKENIZER_NAME,\n",
+ " \"model_type\": \"gpt2\",\n",
+ " \"tokenizer_name\": \"gpt2\",\n",
" \"dataset_name\": \"glue\",\n",
" \"dataset_config_name\": \"sst2\",\n",
" \"do_train\": True,\n",
- " \"do_eval\": True,\n",
+ " \"do_eval\": False,\n",
" \"fp16\": True,\n",
- " \"per_device_train_batch_size\": batch_size_native,\n",
- " \"learning_rate\": learning_rate_native,\n",
- " \"per_device_eval_batch_size\": 16,\n",
- " \"num_train_epochs\": EPOCHS,\n",
+ " \"per_device_eval_batch_size\": 8,\n",
+ " \"num_train_epochs\": 100,\n",
" \"block_size\": 512,\n",
" \"overwrite_output_dir\": True,\n",
" \"save_strategy\": \"no\",\n",
+ " \"evaluation_strategy\": \"no\",\n",
" \"logging_strategy\": \"epoch\",\n",
" \"output_dir\": \"/opt/ml/model\",\n",
- "}\n",
+ " \"dataloader_drop_last\": True,\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6b5204bc",
+ "metadata": {},
+ "source": [
+ "In the following sections, we will create estimators and start training.\n",
"\n",
- "# configure the training job\n",
- "native_estimator = HuggingFace(\n",
- " entry_point=\"launch_pt_dt_sm_native.py\",\n",
- " source_dir=\"./scripts\",\n",
- " instance_type=INSTANCE_TYPE,\n",
- " instance_count=1,\n",
- " role=role,\n",
- " py_version=\"py38\",\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
- " volume_size=100,\n",
- " hyperparameters=hyperparameters,\n",
- " disable_profiler=True, # Disabling SageMaker Profiler to avoid overheads during benchmarking\n",
- " debugger_hook_config=False, # Disabling SageMaker Debugger to avoid overheads during benchmarking\n",
+ "### Training with Native PyTorch\n",
+ "\n",
+ "In the following sections, we will create estimators and start training.\n",
+ "\n",
+ "The `per_device_train_batch_size` below is the largest batch we could fit into the memory of a `ml.g4dn.12xlarge` instance. If you change the model, instance type, sequence length, or other parameters that affect memory consumption, you need to find the corresponding largest batch size.\n",
+ "\n",
+ "This example uses HuggingFace training script `run_clm.py`, which you can find it inside the `scripts` folder. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8dbc83ab",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.pytorch import PyTorch\n",
+ "\n",
+ "# The original learning rate was set for a batch of 32. Here we scale learning rate linearly with an adjusted batch size\n",
+ "per_device_train_batch_size = 10\n",
+ "global_batch_size = (\n",
+ " per_device_train_batch_size * num_gpus_per_instance * estimator_args[\"instance_count\"]\n",
+ ")\n",
+ "learning_rate = float(\"5e-5\") / 32 * global_batch_size\n",
+ "\n",
+ "# Configure the training job\n",
+ "native_estimator = PyTorch(\n",
+ " framework_version=\"1.11\",\n",
+ " hyperparameters=dict(\n",
+ " **hyperparameters,\n",
+ " **{\n",
+ " \"per_device_train_batch_size\": per_device_train_batch_size,\n",
+ " \"learning_rate\": learning_rate,\n",
+ " },\n",
+ " ),\n",
+ " distribution={\"pytorchddp\": {\"enabled\": True}},\n",
+ " **estimator_args,\n",
")\n",
"\n",
- "# start the training job\n",
+ "# Start the training job\n",
"native_estimator.fit(wait=False)\n",
+ "\n",
"native_estimator.latest_training_job.name"
]
},
{
"cell_type": "markdown",
- "id": "85e624f7",
+ "id": "2ef182d4",
"metadata": {},
"source": [
"### Training with Optimized PyTorch"
@@ -282,68 +291,55 @@
},
{
"cell_type": "markdown",
- "id": "d63763c1",
+ "id": "8c2011e0",
"metadata": {},
"source": [
- "Compilation through Training Compiler changes the memory footprint of the model. Most commonly, this manifests as a reduction in memory utilization and a consequent increase in the largest batch size that can fit on the GPU. Note that if you want to change the batch size, you must adjust the learning rate appropriately.\n",
- "\n",
- "**Note:** We recommend you to turn the SageMaker Debugger's profiling and debugging tools off when you use compilation to avoid additional overheads.\n",
+ "Compilation through Training Compiler changes the memory footprint of the model. Most commonly, this manifests as a reduction in memory utilization and a consequent increase in the largest batch size that can fit on the GPU. Note that when you change the batch size, you must adjust the learning rate appropriately. Below, we have scaled the learning rate linearly with the increase in batch size.\n",
"\n",
- "Here, instead of using the `distribution` kwarg to launch a multi node training job, we use a wrapper script to set up an inter-node communication using `torch_xla.distributed.sm_dist`, which has been optimized to work with SageMaker Training Compiler."
+ "**Note:** We are using distribution mechanism `pytorchxla` which is a compiler aware method of distributed training.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "96d5450c",
- "metadata": {},
- "outputs": [],
- "source": [
- "!pygmentize ./scripts/launch_sm_training_compiler.py"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a1948135",
+ "id": "405d7bec",
"metadata": {},
"outputs": [],
"source": [
"from sagemaker.huggingface import HuggingFace, TrainingCompilerConfig\n",
"\n",
- "# with SageMaker Training Compiler we are able to fit a larger batch into memory\n",
- "hyperparameters[\"per_device_train_batch_size\"] = 22\n",
- "\n",
- "# The original LR was set for a batch of 32. Here we scale learning_rate with an adjusted batch size and the number of GPUs per instance.\n",
- "hyperparameters[\"learning_rate\"] = (\n",
- " float(\"5e-5\") / 32 * hyperparameters[\"per_device_train_batch_size\"] * num_gpus_per_instance\n",
+ "# The original learning rate was set for a batch of 32. Here we scale learning rate linearly with an adjusted batch size\n",
+ "new_per_device_train_batch_size = 20\n",
+ "global_batch_size = (\n",
+ " new_per_device_train_batch_size * num_gpus_per_instance * estimator_args[\"instance_count\"]\n",
")\n",
+ "learning_rate = float(\"5e-5\") / 32 * global_batch_size\n",
"\n",
- "# configure the training job\n",
+ "# Configure the training job\n",
"optimized_estimator = HuggingFace(\n",
- " entry_point=\"launch_sm_training_compiler.py\", # Wrapper around training script that enables multi GPU training\n",
- " compiler_config=TrainingCompilerConfig(), # We are enabling SageMaker Training Compiler here !\n",
- " source_dir=\"./scripts\",\n",
- " instance_type=\"ml.p3.8xlarge\",\n",
- " instance_count=1,\n",
- " role=role,\n",
- " volume_size=100,\n",
- " py_version=\"py38\",\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
- " hyperparameters=hyperparameters,\n",
- " disable_profiler=True, # Disabling SageMaker Profiler to avoid overheads during benchmarking\n",
- " debugger_hook_config=False, # Disabling SageMaker Debugger to avoid overheads during benchmarking\n",
+ " compiler_config=TrainingCompilerConfig(),\n",
+ " transformers_version=\"4.21\",\n",
+ " pytorch_version=\"1.11\",\n",
+ " hyperparameters=dict(\n",
+ " **hyperparameters,\n",
+ " **{\n",
+ " \"per_device_train_batch_size\": new_per_device_train_batch_size,\n",
+ " \"learning_rate\": learning_rate,\n",
+ " },\n",
+ " ),\n",
+ " distribution={\"pytorchxla\": {\"enabled\": True}},\n",
+ " **estimator_args,\n",
")\n",
"\n",
- "# start the training job\n",
+ "# Start the training job\n",
"optimized_estimator.fit(wait=False)\n",
+ "\n",
"optimized_estimator.latest_training_job.name"
]
},
{
"cell_type": "markdown",
- "id": "56f47e19",
+ "id": "acc95f44",
"metadata": {},
"source": [
"### Wait for training jobs to complete"
@@ -352,7 +348,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "5676eefc",
+ "id": "1b8ca2bd",
"metadata": {},
"outputs": [],
"source": [
@@ -360,15 +356,12 @@
" \"training_job_completed_or_stopped\"\n",
")\n",
"waiter.wait(TrainingJobName=native_estimator.latest_training_job.name)\n",
- "waiter = optimized_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
- " \"training_job_completed_or_stopped\"\n",
- ")\n",
"waiter.wait(TrainingJobName=optimized_estimator.latest_training_job.name)"
]
},
{
"cell_type": "markdown",
- "id": "78053474",
+ "id": "25f266b9",
"metadata": {},
"source": [
"## Analysis"
@@ -376,19 +369,31 @@
},
{
"cell_type": "markdown",
- "id": "7591a352",
+ "id": "85df1b04",
"metadata": {},
"source": [
"**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new HuggingFace estimator. For example:\n",
"\n",
"```python\n",
- "huggingface_estimator = HuggingFace.attach(\"your_huggingface_training_job_name\")\n",
+ "native_estimator = PyTorch.attach(\"your_huggingface_training_job_name\")\n",
+ "optimized_estimator = HuggingFace.attach(\"your_huggingface_training_job_name\")\n",
"```"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7a4195d5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "native_estimator = PyTorch.attach(native_estimator.latest_training_job.name)\n",
+ "optimized_estimator = HuggingFace.attach(optimized_estimator.latest_training_job.name)"
+ ]
+ },
{
"cell_type": "markdown",
- "id": "b5e54aca",
+ "id": "20bb89b1",
"metadata": {},
"source": [
"### Load logs of the training job *with* SageMaker Training Compiler"
@@ -397,7 +402,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "64b14de7",
+ "id": "b14fa522",
"metadata": {},
"outputs": [],
"source": [
@@ -409,7 +414,7 @@
},
{
"cell_type": "markdown",
- "id": "a9f687a0",
+ "id": "14944bde",
"metadata": {},
"source": [
"### Load logs of the training job *without* SageMaker Training Compiler"
@@ -418,7 +423,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "279602f2",
+ "id": "bb9f1be8",
"metadata": {},
"outputs": [],
"source": [
@@ -430,7 +435,7 @@
},
{
"cell_type": "markdown",
- "id": "a1d72507",
+ "id": "ba586740",
"metadata": {},
"source": [
"### Create helper functions for analysis"
@@ -439,7 +444,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "2a1b029c",
+ "id": "5376e667",
"metadata": {},
"outputs": [],
"source": [
@@ -480,7 +485,7 @@
},
{
"cell_type": "markdown",
- "id": "5800d165",
+ "id": "853afbef",
"metadata": {},
"source": [
"### Plot Optimized vs Native Training Throughput\n",
@@ -491,7 +496,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "01e672f5",
+ "id": "bd5f2774",
"metadata": {},
"outputs": [],
"source": [
@@ -510,7 +515,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "3d26ca26",
+ "id": "7eed5237",
"metadata": {},
"outputs": [],
"source": [
@@ -528,7 +533,7 @@
},
{
"cell_type": "markdown",
- "id": "4cdd3e80",
+ "id": "f17d5bbd",
"metadata": {},
"source": [
"### Convergence of Training Loss\n",
@@ -539,7 +544,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "2ce7fbd4",
+ "id": "dc92a294",
"metadata": {},
"outputs": [],
"source": [
@@ -558,7 +563,7 @@
},
{
"cell_type": "markdown",
- "id": "ee290661",
+ "id": "85bcad63",
"metadata": {},
"source": [
"### Training Stats\n",
@@ -569,7 +574,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "27462c06",
+ "id": "f34beb9b",
"metadata": {},
"outputs": [],
"source": [
@@ -581,7 +586,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "055d4fb2",
+ "id": "214e263f",
"metadata": {},
"outputs": [],
"source": [
@@ -598,7 +603,7 @@
},
{
"cell_type": "markdown",
- "id": "6fd0199c",
+ "id": "468541aa",
"metadata": {},
"source": [
"### Total Billable Time\n",
@@ -609,7 +614,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "612a6491",
+ "id": "0a9192ac",
"metadata": {},
"outputs": [],
"source": [
@@ -624,7 +629,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "12742c78",
+ "id": "7b30b9ed",
"metadata": {},
"outputs": [],
"source": [
@@ -637,7 +642,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "bfc2a8b4",
+ "id": "b696a714",
"metadata": {},
"outputs": [],
"source": [
@@ -647,7 +652,7 @@
},
{
"cell_type": "markdown",
- "id": "c580614f",
+ "id": "a0dfc123",
"metadata": {},
"source": [
"## Clean up\n",
@@ -658,7 +663,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "983acde4",
+ "id": "34367f18",
"metadata": {},
"outputs": [],
"source": [
@@ -679,7 +684,7 @@
},
{
"cell_type": "markdown",
- "id": "33bec82a",
+ "id": "7524e3ba",
"metadata": {},
"source": [
"Also, to find instructions on cleaning up resources, see [Clean Up](https://docs.aws.amazon.com/sagemaker/latest/dg/ex1-cleanup.html) in the *Amazon SageMaker Developer Guide*."
@@ -688,9 +693,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "conda_pytorch_latest_p36",
+ "display_name": "conda_pytorch_p38",
"language": "python",
- "name": "conda_pytorch_latest_p36"
+ "name": "conda_pytorch_p38"
},
"language_info": {
"codemirror_mode": {
@@ -702,7 +707,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.13"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/scripts/requirements.txt b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/scripts/requirements.txt
new file mode 100644
index 0000000000..c8f87cceeb
--- /dev/null
+++ b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/scripts/requirements.txt
@@ -0,0 +1,2 @@
+transformers==4.21.1
+datasets==1.18.4
\ No newline at end of file
diff --git a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/run_clm.py b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/scripts/run_clm.py
similarity index 100%
rename from sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/run_clm.py
rename to sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/language-modeling/scripts/run_clm.py
diff --git a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/launch_pt_dt_sm_native.py b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/launch_pt_dt_sm_native.py
deleted file mode 100644
index 28727d0074..0000000000
--- a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/launch_pt_dt_sm_native.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import argparse
-import os, subprocess
-from pdb import run
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser()
-
- # hyperparameters sent by the client are passed as command-line arguments to the script.
- parser.add_argument("--training_script", type=str, default="run_mlm.py")
- parser.add_argument("--n_gpus", type=str, default=os.environ["SM_NUM_GPUS"])
- parser.add_argument("--output_dir", type=str, default=os.environ["SM_OUTPUT_DIR"])
-
- args, rem_args = parser.parse_known_args()
- print("Parsed Arguments: ", vars(args), rem_args)
- os.environ["GPU_NUM_DEVICES"] = str(args.n_gpus)
-
- # native torch distributed as benchmark
- training_command = "python -m torch.distributed.launch "
- training_command += f"--nproc_per_node={args.n_gpus} "
- training_command += "--nnodes=1 --node_rank=0 --master_addr=127.0.0.1 --master_port=1234 "
-
- training_command += args.training_script + " "
-
- # output directory
- training_command += f"--output_dir {args.output_dir} "
- for i in range(0, len(rem_args), 2):
- arg, value = rem_args[i], rem_args[i + 1]
- if value == "True":
- training_command += f"{arg} "
- elif value != "False":
- training_command += f"{arg} {value} "
-
- print("Training Command: ", training_command)
- subprocess.check_call(training_command, shell=True)
diff --git a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/launch_sm_training_compiler.py b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/launch_sm_training_compiler.py
deleted file mode 100644
index 655af389a2..0000000000
--- a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/launch_sm_training_compiler.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import subprocess
-import sys
-
-if __name__ == "__main__":
- arguments_command = " ".join([arg for arg in sys.argv[1:]])
- """
- The following line will take care of setting up inter node communication as well as managing intra node workers for each GPU.
- """
- subprocess.check_call("python -m torch_xla.distributed.sm_dist " + arguments_command, shell=True)
diff --git a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/run_mlm.py b/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/run_mlm.py
deleted file mode 100644
index 3a1375bf86..0000000000
--- a/sagemaker-training-compiler/huggingface/pytorch_multiple_gpu_single_node/scripts/run_mlm.py
+++ /dev/null
@@ -1,600 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-# Copyright 2020 The HuggingFace Team All rights reserved.
-# Modifications Copyright 2021 Amazon.com, Inc. or its affiliates. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Fine-tuning the library models for masked language modeling (BERT, ALBERT, RoBERTa...) on a text file or a dataset.
-
-Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
-https://huggingface.co/models?filter=masked-lm
-"""
-# You can also adapt this script on your own masked language modeling task. Pointers for this are left as comments.
-
-import logging
-import math
-import os
-import sys
-from dataclasses import dataclass, field
-from typing import Optional
-
-import datasets
-from datasets import load_dataset
-
-import transformers
-from transformers import (
- CONFIG_MAPPING,
- MODEL_FOR_MASKED_LM_MAPPING,
- AutoConfig,
- AutoModelForMaskedLM,
- AutoTokenizer,
- DataCollatorForLanguageModeling,
- HfArgumentParser,
- Trainer,
- TrainingArguments,
- set_seed,
-)
-from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version
-from transformers.utils.versions import require_version
-
-
-# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
-check_min_version("4.10.0")
-
-require_version(
- "datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt"
-)
-
-logger = logging.getLogger(__name__)
-MODEL_CONFIG_CLASSES = list(MODEL_FOR_MASKED_LM_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-
-
-@dataclass
-class ModelArguments:
- """
- Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
- """
-
- model_name_or_path: Optional[str] = field(
- default=None,
- metadata={
- "help": "The model checkpoint for weights initialization."
- "Don't set if you want to train a model from scratch."
- },
- )
- model_type: Optional[str] = field(
- default=None,
- metadata={
- "help": "If training from scratch, pass a model type from the list: "
- + ", ".join(MODEL_TYPES)
- },
- )
- config_overrides: Optional[str] = field(
- default=None,
- metadata={
- "help": "Override some existing default config settings when a model is trained from scratch. Example: "
- "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
- },
- )
- config_name: Optional[str] = field(
- default=None,
- metadata={"help": "Pretrained config name or path if not the same as model_name"},
- )
- tokenizer_name: Optional[str] = field(
- default=None,
- metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"},
- )
- cache_dir: Optional[str] = field(
- default=None,
- metadata={
- "help": "Where do you want to store the pretrained models downloaded from huggingface.co"
- },
- )
- use_fast_tokenizer: bool = field(
- default=True,
- metadata={
- "help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."
- },
- )
- model_revision: str = field(
- default="main",
- metadata={
- "help": "The specific model version to use (can be a branch name, tag name or commit id)."
- },
- )
- use_auth_token: bool = field(
- default=False,
- metadata={
- "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
- "with private models)."
- },
- )
-
- def __post_init__(self):
- if self.config_overrides is not None and (
- self.config_name is not None or self.model_name_or_path is not None
- ):
- raise ValueError(
- "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
- )
-
-
-@dataclass
-class DataTrainingArguments:
- """
- Arguments pertaining to what data we are going to input our model for training and eval.
- """
-
- dataset_name: Optional[str] = field(
- default=None,
- metadata={"help": "The name of the dataset to use (via the datasets library)."},
- )
- dataset_config_name: Optional[str] = field(
- default=None,
- metadata={
- "help": "The configuration name of the dataset to use (via the datasets library)."
- },
- )
- train_file: Optional[str] = field(
- default=None, metadata={"help": "The input training data file (a text file)."}
- )
- validation_file: Optional[str] = field(
- default=None,
- metadata={
- "help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."
- },
- )
- overwrite_cache: bool = field(
- default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
- )
- validation_split_percentage: Optional[int] = field(
- default=5,
- metadata={
- "help": "The percentage of the train set used as validation set in case there's no validation split"
- },
- )
- max_seq_length: Optional[int] = field(
- default=None,
- metadata={
- "help": "The maximum total input sequence length after tokenization. Sequences longer "
- "than this will be truncated."
- },
- )
- preprocessing_num_workers: Optional[int] = field(
- default=None,
- metadata={"help": "The number of processes to use for the preprocessing."},
- )
- mlm_probability: float = field(
- default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
- )
- line_by_line: bool = field(
- default=False,
- metadata={
- "help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."
- },
- )
- pad_to_max_length: bool = field(
- default=False,
- metadata={
- "help": "Whether to pad all samples to `max_seq_length`. "
- "If False, will pad the samples dynamically when batching to the maximum length in the batch."
- },
- )
- max_train_samples: Optional[int] = field(
- default=None,
- metadata={
- "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
- "value if set."
- },
- )
- max_eval_samples: Optional[int] = field(
- default=None,
- metadata={
- "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
- "value if set."
- },
- )
-
- def __post_init__(self):
- if self.dataset_name is None and self.train_file is None and self.validation_file is None:
- raise ValueError("Need either a dataset name or a training/validation file.")
- else:
- if self.train_file is not None:
- extension = self.train_file.split(".")[-1]
- assert extension in [
- "csv",
- "json",
- "txt",
- ], "`train_file` should be a csv, a json or a txt file."
- if self.validation_file is not None:
- extension = self.validation_file.split(".")[-1]
- assert extension in [
- "csv",
- "json",
- "txt",
- ], "`validation_file` should be a csv, a json or a txt file."
-
-
-def main():
- # See all possible arguments in src/transformers/training_args.py
- # or by passing the --help flag to this script.
- # We now keep distinct sets of args, for a cleaner separation of concerns.
-
- parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
- if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
- # If we pass only one argument to the script and it's the path to a json file,
- # let's parse it to get our arguments.
- model_args, data_args, training_args = parser.parse_json_file(
- json_file=os.path.abspath(sys.argv[1])
- )
- else:
- model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-
- # Setup logging
- logging.basicConfig(
- format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
- datefmt="%m/%d/%Y %H:%M:%S",
- handlers=[logging.StreamHandler(sys.stdout)],
- )
-
- log_level = training_args.get_process_log_level()
- logger.setLevel(log_level)
- datasets.utils.logging.set_verbosity(log_level)
- transformers.utils.logging.set_verbosity(log_level)
- transformers.utils.logging.enable_default_handler()
- transformers.utils.logging.enable_explicit_format()
-
- # Log on each process the small summary:
- logger.warning(
- f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
- + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
- )
- # Set the verbosity to info of the Transformers logger (on main process only):
- logger.info(f"Training/evaluation parameters {training_args}")
-
- # Detecting last checkpoint.
- last_checkpoint = None
- if (
- os.path.isdir(training_args.output_dir)
- and training_args.do_train
- and not training_args.overwrite_output_dir
- ):
- last_checkpoint = get_last_checkpoint(training_args.output_dir)
- if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
- raise ValueError(
- f"Output directory ({training_args.output_dir}) already exists and is not empty. "
- "Use --overwrite_output_dir to overcome."
- )
- elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
- logger.info(
- f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
- "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
- )
-
- # Set seed before initializing model.
- set_seed(training_args.seed)
-
- # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
- # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
- # (the dataset will be downloaded automatically from the datasets Hub
- #
- # For CSV/JSON files, this script will use the column called 'text' or the first column. You can easily tweak this
- # behavior (see below)
- #
- # In distributed training, the load_dataset function guarantee that only one local process can concurrently
- # download the dataset.
- if data_args.dataset_name is not None:
- # Downloading and loading a dataset from the hub.
- raw_datasets = load_dataset(
- data_args.dataset_name, data_args.dataset_config_name, cache_dir=model_args.cache_dir
- )
- if "validation" not in raw_datasets.keys():
- raw_datasets["validation"] = load_dataset(
- data_args.dataset_name,
- data_args.dataset_config_name,
- split=f"train[:{data_args.validation_split_percentage}%]",
- cache_dir=model_args.cache_dir,
- )
- raw_datasets["train"] = load_dataset(
- data_args.dataset_name,
- data_args.dataset_config_name,
- split=f"train[{data_args.validation_split_percentage}%:]",
- cache_dir=model_args.cache_dir,
- )
- else:
- data_files = {}
- if data_args.train_file is not None:
- data_files["train"] = data_args.train_file
- extension = data_args.train_file.split(".")[-1]
- if data_args.validation_file is not None:
- data_files["validation"] = data_args.validation_file
- extension = data_args.validation_file.split(".")[-1]
- if extension == "txt":
- extension = "text"
- raw_datasets = load_dataset(
- extension, data_files=data_files, cache_dir=model_args.cache_dir
- )
-
- # If no validation data is there, validation_split_percentage will be used to divide the dataset.
- if "validation" not in raw_datasets.keys():
- raw_datasets["validation"] = load_dataset(
- extension,
- data_files=data_files,
- split=f"train[:{data_args.validation_split_percentage}%]",
- cache_dir=model_args.cache_dir,
- )
- raw_datasets["train"] = load_dataset(
- extension,
- data_files=data_files,
- split=f"train[{data_args.validation_split_percentage}%:]",
- cache_dir=model_args.cache_dir,
- )
-
- # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
- # https://huggingface.co/docs/datasets/loading_datasets.html.
-
- # Load pretrained model and tokenizer
- #
- # Distributed training:
- # The .from_pretrained methods guarantee that only one local process can concurrently
- # download model & vocab.
- config_kwargs = {
- "cache_dir": model_args.cache_dir,
- "revision": model_args.model_revision,
- "use_auth_token": True if model_args.use_auth_token else None,
- }
- if model_args.config_name:
- config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
- elif model_args.model_name_or_path:
- config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
- else:
- config = CONFIG_MAPPING[model_args.model_type]()
- logger.warning("You are instantiating a new config instance from scratch.")
- if model_args.config_overrides is not None:
- logger.info(f"Overriding config: {model_args.config_overrides}")
- config.update_from_string(model_args.config_overrides)
-
- tokenizer_kwargs = {
- "cache_dir": model_args.cache_dir,
- "use_fast": model_args.use_fast_tokenizer,
- "revision": model_args.model_revision,
- "use_auth_token": True if model_args.use_auth_token else None,
- }
- if model_args.tokenizer_name:
- tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
- elif model_args.model_name_or_path:
- tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
- else:
- raise ValueError(
- "You are instantiating a new tokenizer from scratch. This is not supported by this script."
- "You can do it from another script, save it, and load it from here, using --tokenizer_name."
- )
-
- if model_args.model_name_or_path:
- model = AutoModelForMaskedLM.from_pretrained(
- model_args.model_name_or_path,
- from_tf=bool(".ckpt" in model_args.model_name_or_path),
- config=config,
- cache_dir=model_args.cache_dir,
- revision=model_args.model_revision,
- use_auth_token=True if model_args.use_auth_token else None,
- )
- else:
- logger.info("Training new model from scratch")
- model = AutoModelForMaskedLM.from_config(config)
-
- model.resize_token_embeddings(len(tokenizer))
-
- # Preprocessing the datasets.
- # First we tokenize all the texts.
- if training_args.do_train:
- column_names = raw_datasets["train"].column_names
- else:
- column_names = raw_datasets["validation"].column_names
- text_column_name = "text" if "text" in column_names else column_names[0]
-
- if data_args.max_seq_length is None:
- max_seq_length = tokenizer.model_max_length
- if max_seq_length > 1024:
- logger.warning(
- f"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). "
- "Picking 1024 instead. You can change that default value by passing --max_seq_length xxx."
- )
- max_seq_length = 1024
- else:
- if data_args.max_seq_length > tokenizer.model_max_length:
- logger.warning(
- f"The max_seq_length passed ({data_args.max_seq_length}) is larger than the maximum length for the"
- f"model ({tokenizer.model_max_length}). Using max_seq_length={tokenizer.model_max_length}."
- )
- max_seq_length = min(data_args.max_seq_length, tokenizer.model_max_length)
-
- if data_args.line_by_line:
- # When using line_by_line, we just tokenize each nonempty line.
- padding = "max_length" if data_args.pad_to_max_length else False
-
- def tokenize_function(examples):
- # Remove empty lines
- examples[text_column_name] = [
- line for line in examples[text_column_name] if len(line) > 0 and not line.isspace()
- ]
- return tokenizer(
- examples[text_column_name],
- padding=padding,
- truncation=True,
- max_length=max_seq_length,
- # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
- # receives the `special_tokens_mask`.
- return_special_tokens_mask=True,
- )
-
- with training_args.main_process_first(desc="dataset map tokenization"):
- tokenized_datasets = raw_datasets.map(
- tokenize_function,
- batched=True,
- num_proc=data_args.preprocessing_num_workers,
- remove_columns=[text_column_name],
- load_from_cache_file=not data_args.overwrite_cache,
- desc="Running tokenizer on dataset line_by_line",
- )
- else:
- # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
- # We use `return_special_tokens_mask=True` because DataCollatorForLanguageModeling (see below) is more
- # efficient when it receives the `special_tokens_mask`.
- def tokenize_function(examples):
- return tokenizer(examples[text_column_name], return_special_tokens_mask=True)
-
- with training_args.main_process_first(desc="dataset map tokenization"):
- tokenized_datasets = raw_datasets.map(
- tokenize_function,
- batched=True,
- num_proc=data_args.preprocessing_num_workers,
- remove_columns=column_names,
- load_from_cache_file=not data_args.overwrite_cache,
- desc="Running tokenizer on every text in dataset",
- )
-
- # Main data processing function that will concatenate all texts from our dataset and generate chunks of
- # max_seq_length.
- def group_texts(examples):
- # Concatenate all texts.
- concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
- total_length = len(concatenated_examples[list(examples.keys())[0]])
- # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
- # customize this part to your needs.
- if total_length >= max_seq_length:
- total_length = (total_length // max_seq_length) * max_seq_length
- # Split by chunks of max_len.
- result = {
- k: [t[i : i + max_seq_length] for i in range(0, total_length, max_seq_length)]
- for k, t in concatenated_examples.items()
- }
- return result
-
- # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a
- # remainder for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value
- # might be slower to preprocess.
- #
- # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
- # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
-
- with training_args.main_process_first(desc="grouping texts together"):
- tokenized_datasets = tokenized_datasets.map(
- group_texts,
- batched=True,
- num_proc=data_args.preprocessing_num_workers,
- load_from_cache_file=not data_args.overwrite_cache,
- desc=f"Grouping texts in chunks of {max_seq_length}",
- )
-
- if training_args.do_train:
- if "train" not in tokenized_datasets:
- raise ValueError("--do_train requires a train dataset")
- train_dataset = tokenized_datasets["train"]
- if data_args.max_train_samples is not None:
- train_dataset = train_dataset.select(range(data_args.max_train_samples))
-
- if training_args.do_eval:
- if "validation" not in tokenized_datasets:
- raise ValueError("--do_eval requires a validation dataset")
- eval_dataset = tokenized_datasets["validation"]
- if data_args.max_eval_samples is not None:
- eval_dataset = eval_dataset.select(range(data_args.max_eval_samples))
-
- # Data collator
- # This one will take care of randomly masking the tokens.
- pad_to_multiple_of_8 = (
- data_args.line_by_line and training_args.fp16 and not data_args.pad_to_max_length
- )
- data_collator = DataCollatorForLanguageModeling(
- tokenizer=tokenizer,
- mlm_probability=data_args.mlm_probability,
- pad_to_multiple_of=8 if pad_to_multiple_of_8 else None,
- )
-
- # Initialize our Trainer
- trainer = Trainer(
- model=model,
- args=training_args,
- train_dataset=train_dataset if training_args.do_train else None,
- eval_dataset=eval_dataset if training_args.do_eval else None,
- tokenizer=tokenizer,
- data_collator=data_collator,
- )
-
- # Training
- if training_args.do_train:
- checkpoint = None
- if training_args.resume_from_checkpoint is not None:
- checkpoint = training_args.resume_from_checkpoint
- elif last_checkpoint is not None:
- checkpoint = last_checkpoint
- train_result = trainer.train(resume_from_checkpoint=checkpoint)
- trainer.save_model() # Saves the tokenizer too for easy upload
- metrics = train_result.metrics
-
- max_train_samples = (
- data_args.max_train_samples
- if data_args.max_train_samples is not None
- else len(train_dataset)
- )
- metrics["train_samples"] = min(max_train_samples, len(train_dataset))
-
- trainer.log_metrics("train", metrics)
- trainer.save_metrics("train", metrics)
- trainer.save_state()
-
- # Evaluation
- if training_args.do_eval:
- logger.info("*** Evaluate ***")
-
- metrics = trainer.evaluate()
-
- max_eval_samples = (
- data_args.max_eval_samples
- if data_args.max_eval_samples is not None
- else len(eval_dataset)
- )
- metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
- try:
- perplexity = math.exp(metrics["eval_loss"])
- except OverflowError:
- perplexity = float("inf")
- metrics["perplexity"] = perplexity
-
- trainer.log_metrics("eval", metrics)
- trainer.save_metrics("eval", metrics)
-
- if training_args.push_to_hub:
- kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "fill-mask"}
- if data_args.dataset_name is not None:
- kwargs["dataset_tags"] = data_args.dataset_name
- if data_args.dataset_config_name is not None:
- kwargs["dataset_args"] = data_args.dataset_config_name
- kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
- else:
- kwargs["dataset"] = data_args.dataset_name
-
- trainer.push_to_hub(**kwargs)
-
-
-def _mp_fn(index):
- # For xla_spawn (TPUs)
- main()
-
-
-if __name__ == "__main__":
- main()
diff --git a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/albert-base-v2.ipynb b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/albert-base-v2.ipynb
index c9b611aa3c..3eb085c4bb 100644
--- a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/albert-base-v2.ipynb
+++ b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/albert-base-v2.ipynb
@@ -38,11 +38,11 @@
"\n",
"## Introduction\n",
"\n",
- "This example notebook demonstrates how to compile and fine-tune a question and answering NLP task. We use Hugging Face's `transformers` and `datasets` libraries with Amazon Sagemaker Training Compiler to accelerate fine-tuning of a pre-trained transformer model on question and answering. In particular, the pre-trained model will be fine-tuned using the `SQuAD` dataset. To get started, we need to set up the environment with a few prerequisite steps to add permissions, configurations, and so on. \n",
+ "This example notebook demonstrates how to compile and fine-tune a question and answering NLP task. We use HuggingFace's transformers and datasets libraries with Amazon SageMaker Training Compiler to accelerate fine-tuning of a pre-trained transformer model on question and answering. In particular, the pre-trained model will be fine-tuned using the SQuAD dataset. To get started, we need to set up the environment with a few prerequisite steps to add permissions, configurations, and so on. \n",
"\n",
- "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, `Python 3 (PyTorch x.y Python 3.x CPU Optimized)` or `conda_pytorch_p36` respectively.\n",
+ "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, Python 3 (PyTorch x.y Python 3.x CPU Optimized) or conda_pytorch_p36 respectively.\n",
"\n",
- "**NOTE:** This notebook uses two `ml.p3.2xlarge` instances that have single GPU. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
+ "**NOTE:** This notebook uses two ml.p3.2xlarge instances that have single GPU. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
]
},
{
@@ -58,7 +58,7 @@
"source": [
"### Installation\n",
"\n",
- "This example notebook requires the **SageMaker Python SDK v2.70.0** and **transformers v4.11.0**."
+ "This example notebook requires the **SageMaker Python SDK v2.108.0** and **transformers v4.21**."
]
},
{
@@ -67,7 +67,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install sagemaker botocore boto3 awscli s3fs typing-extensions --upgrade"
+ "!pip install \"sagemaker>=2.108.0\" botocore boto3 awscli s3fs typing-extensions \"torch==1.11.0\" pandas numpy --upgrade"
]
},
{
@@ -76,7 +76,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install transformers datasets --upgrade"
+ "!pip install \"transformers==4.21\" datasets --upgrade"
]
},
{
@@ -98,7 +98,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Copy and run the following code if you need to upgrade ipywidgets for `datasets` library and restart kernel. This is only needed when prerpocessing is done in the notebook.\n",
+ "Copy and run the following code if you need to upgrade `ipywidgets` for `datasets` library and restart kernel. This is only needed when preprocessing is done in the notebook.\n",
"\n",
"```python\n",
"%%capture\n",
@@ -120,7 +120,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Note:** If you are going to use Sagemaker in a local environment. You need access to an IAM Role with the required permissions for SageMaker. To learn more, see [SageMaker Roles](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html)."
+ "**Note:** If you are going to use SageMaker in a local environment. You need access to an IAM Role with the required permissions for SageMaker. To learn more, see [SageMaker Roles](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html)."
]
},
{
@@ -322,7 +322,7 @@
"id": "Vl6IidfdIrJK"
},
"source": [
- "The following assertion ensures that our tokenizer is a fast tokenizers (backed by Rust) from the 🤗 Tokenizers library. Those fast tokenizers are available for almost all models, and we will need some of the special features they have for our preprocessing."
+ "The following assertion ensures that our tokenizer is a fast tokenizer (backed by Rust) from the 🤗 Tokenizers library. Those fast tokenizers are available for almost all models, and we will need some of the special features they have for our preprocessing."
]
},
{
@@ -515,7 +515,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Before we kick off our SageMaker training job we need to transfer our dataset to S3 so the training job can download it from S3."
+ "Before we kick off our SageMaker training job we need to transfer our dataset to S3, so the training job can download it from S3."
]
},
{
@@ -563,11 +563,11 @@
"source": [
"## SageMaker Training Job\n",
"\n",
- "To create a SageMaker training job, we use a `HuggingFace` estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through `entry_point`, which `instance_type` to use for training, which `hyperparameters` to pass, and so on.\n",
+ "To create a SageMaker training job, we use a HuggingFace/PyTorch estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through entry_point, which instance_type to use for training, which hyperparameters to pass, and so on.\n",
"\n",
- "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the `HuggingFace` Deep Learning Container, uploads your training script, and downloads the data from `sagemaker_session_bucket` into the container at `/opt/ml/input/data`.\n",
+ "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the HuggingFace Deep Learning Container, uploads your training script, and downloads the data from sagemaker_session_bucket into the container at /opt/ml/input/data.\n",
"\n",
- "In the following section, you learn how to set up two versions of the SageMaker `HuggingFace` estimator, a native one without the compiler and an optimized one with the compiler."
+ "In the following section, you learn how to set up two versions of the SageMaker HuggingFace/PyTorch estimator, a native one without the compiler and an optimized one with the compiler."
]
},
{
@@ -581,13 +581,13 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Below, we run a native PyTorch training job with the `HuggingFace` estimator on a `ml.p3.2xlarge` instance. \n",
+ "Below, we run a native PyTorch training job with the PyTorch estimator on a ml.p3.2xlarge instance. \n",
"\n",
- "We run a batch size of 28 on our native training job and 52 on our Training Compiler training job to make an apples to apples comparision. These batch sizes along with the max_length variable get us close to 100% GPU memory utilization.\n",
+ "We run a batch size of 28 on our native training job and 52 on our Training Compiler training job to make an apple to apple comparison. These batch sizes along with the max_length variable get us close to 100% GPU memory utilization.\n",
"\n",
"We recommend using the tested batch size that's provided at [Tested Models](https://docs.aws.amazon.com/sagemaker/latest/dg/training-compiler-support.html#training-compiler-tested-models) in the *SageMaker Training Compiler Developer Guide*.\n",
"\n",
- ""
+ ""
]
},
{
@@ -596,7 +596,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.huggingface import HuggingFace\n",
+ "from sagemaker.pytorch import PyTorch\n",
"\n",
"batch_size_native = 28\n",
"learning_rate_native = float(\"3e-5\") / 32 * batch_size_native\n",
@@ -622,15 +622,15 @@
"metadata": {},
"outputs": [],
"source": [
- "huggingface_estimator = HuggingFace(\n",
+ "native_estimator = PyTorch(\n",
" entry_point=\"qa_trainer_huggingface.py\",\n",
" source_dir=\"./scripts\",\n",
" instance_type=\"ml.p3.2xlarge\",\n",
" instance_count=1,\n",
" role=role,\n",
" py_version=\"py38\",\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
+ " transformers_version=\"4.21.1\",\n",
+ " framework_version=\"1.11.0\",\n",
" volume_size=volume_size,\n",
" hyperparameters=hyperparameters,\n",
" disable_profiler=True,\n",
@@ -638,10 +638,10 @@
")\n",
"\n",
"# starting the train job with our uploaded datasets as input\n",
- "huggingface_estimator.fit({\"train\": training_input_path, \"test\": eval_input_path}, wait=False)\n",
+ "native_estimator.fit({\"train\": training_input_path, \"test\": eval_input_path}, wait=False)\n",
"\n",
"# The name of the training job. You might need to note this down in case you lose connection to your notebook.\n",
- "huggingface_estimator.latest_training_job.name"
+ "native_estimator.latest_training_job.name"
]
},
{
@@ -697,7 +697,7 @@
"metadata": {},
"outputs": [],
"source": [
- "compile_estimator = HuggingFace(\n",
+ "optimized_estimator = HuggingFace(\n",
" entry_point=\"qa_trainer_huggingface.py\",\n",
" compiler_config=TrainingCompilerConfig(),\n",
" source_dir=\"./scripts\",\n",
@@ -705,8 +705,8 @@
" instance_count=1,\n",
" role=role,\n",
" py_version=\"py38\",\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
+ " transformers_version=\"4.21.1\",\n",
+ " pytorch_version=\"1.11.0\",\n",
" volume_size=volume_size,\n",
" hyperparameters=hyperparameters,\n",
" disable_profiler=True,\n",
@@ -714,10 +714,10 @@
")\n",
"\n",
"# starting the train job with our uploaded datasets as input\n",
- "compile_estimator.fit({\"train\": training_input_path, \"test\": eval_input_path}, wait=False)\n",
+ "optimized_estimator.fit({\"train\": training_input_path, \"test\": eval_input_path}, wait=False)\n",
"\n",
"# The name of the training job. You might need to note this down in case you lose connection to your notebook.\n",
- "compile_estimator.latest_training_job.name"
+ "optimized_estimator.latest_training_job.name"
]
},
{
@@ -728,14 +728,14 @@
"source": [
"# Wait for training jobs to complete.\n",
"\n",
- "waiter = huggingface_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
+ "waiter = native_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
" \"training_job_completed_or_stopped\"\n",
")\n",
- "waiter.wait(TrainingJobName=huggingface_estimator.latest_training_job.name)\n",
- "waiter = compile_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
+ "waiter.wait(TrainingJobName=native_estimator.latest_training_job.name)\n",
+ "waiter = optimized_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
" \"training_job_completed_or_stopped\"\n",
")\n",
- "waiter.wait(TrainingJobName=compile_estimator.latest_training_job.name)"
+ "waiter.wait(TrainingJobName=optimized_estimator.latest_training_job.name)"
]
},
{
@@ -759,14 +759,14 @@
"outputs": [],
"source": [
"# container image used for native training job\n",
- "print(f\"container image used for training job: \\n{huggingface_estimator.image_uri}\\n\")\n",
+ "print(f\"container image used for training job: \\n{native_estimator.image_uri}\\n\")\n",
"\n",
"# s3 uri where the native trained model is located\n",
- "print(f\"s3 uri where the trained model is located: \\n{huggingface_estimator.model_data}\\n\")\n",
+ "print(f\"s3 uri where the trained model is located: \\n{native_estimator.model_data}\\n\")\n",
"\n",
"# latest training job name for this estimator\n",
"print(\n",
- " f\"latest training job name for this estimator: \\n{huggingface_estimator.latest_training_job.name}\\n\"\n",
+ " f\"latest training job name for this estimator: \\n{native_estimator.latest_training_job.name}\\n\"\n",
")"
]
},
@@ -779,16 +779,16 @@
"%%capture native\n",
"\n",
"# access the logs of the native training job\n",
- "huggingface_estimator.sagemaker_session.logs_for_job(huggingface_estimator.latest_training_job.name)"
+ "native_estimator.sagemaker_session.logs_for_job(native_estimator.latest_training_job.name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new HuggingFace estimator. For example:\n",
+ "**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new PyTorch estimator. For example:\n",
"```python\n",
- "huggingface_estimator = HuggingFace.attach(\"your_huggingface_training_job_name\")\n",
+ "native_estimator = PyTorch.attach(\"your_native_training_job_name\")\n",
"```"
]
},
@@ -806,14 +806,14 @@
"outputs": [],
"source": [
"# container image used for optimized training job\n",
- "print(f\"container image used for training job: \\n{compile_estimator.image_uri}\\n\")\n",
+ "print(f\"container image used for training job: \\n{optimized_estimator.image_uri}\\n\")\n",
"\n",
"# s3 uri where the optimized trained model is located\n",
- "print(f\"s3 uri where the trained model is located: \\n{compile_estimator.model_data}\\n\")\n",
+ "print(f\"s3 uri where the trained model is located: \\n{optimized_estimator.model_data}\\n\")\n",
"\n",
"# latest training job name for this estimator\n",
"print(\n",
- " f\"latest training job name for this estimator: \\n{compile_estimator.latest_training_job.name}\\n\"\n",
+ " f\"latest training job name for this estimator: \\n{optimized_estimator.latest_training_job.name}\\n\"\n",
")"
]
},
@@ -826,7 +826,7 @@
"%%capture optimized\n",
"\n",
"# access the logs of the optimized training job\n",
- "compile_estimator.sagemaker_session.logs_for_job(compile_estimator.latest_training_job.name)"
+ "optimized_estimator.sagemaker_session.logs_for_job(optimized_estimator.latest_training_job.name)"
]
},
{
@@ -835,7 +835,7 @@
"source": [
"**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new HuggingFace estimator. For example:\n",
"```python\n",
- "optimized_est = HuggingFace.attach(\"your_compiled_huggingface_training_job_name\")\n",
+ "optimized_est = HuggingFace.attach(\"your_optimized_native_training_job_name\")\n",
"```"
]
},
@@ -925,16 +925,19 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- ""
+ "### Training Stats\n",
+ "Let's compare various training metrics with and without SageMaker Training Compiler. SageMaker Training Compiler provides an increase in training throughput which translates to a decrease in total training time."
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "### Training Time Plot\n",
+ "import pandas as pd\n",
"\n",
- "The following script creates a plot that compares the billable time of the two training jobs with and without SageMaker Training Compiler."
+ "pd.DataFrame([n[\"summary\"], o[\"summary\"]], index=[\"Native\", \"Optimized\"])"
]
},
{
@@ -943,34 +946,60 @@
"metadata": {},
"outputs": [],
"source": [
- "sm = boto3.client(\"sagemaker\")\n",
- "native_job = sm.describe_training_job(\n",
- " TrainingJobName=huggingface_estimator.latest_training_job.name\n",
+ "# calculate percentage speedup from SageMaker Training Compiler in terms of total training time reported by HF\n",
+ "speedup = (\n",
+ " (n[\"summary\"][\"train_runtime\"] - o[\"summary\"][\"train_runtime\"])\n",
+ " * 100\n",
+ " / n[\"summary\"][\"train_runtime\"]\n",
")\n",
- "\n",
- "compile_job = sm.describe_training_job(TrainingJobName=compile_estimator.latest_training_job.name)\n",
- "\n",
- "n_time = native_job[\"BillableTimeInSeconds\"]\n",
- "h_time = compile_job[\"BillableTimeInSeconds\"]\n",
- "\n",
- "time_decrease = round(((n_time - h_time) / n_time) * 100, 2)\n",
- "\n",
- "plt.title(\"Training Time\")\n",
- "plt.ylabel(\"Minutes\")\n",
- "\n",
- "plt.bar(x=[1], height=n_time / 60, label=\"Baseline PT\", width=0.35)\n",
- "plt.bar(x=[1.5], height=h_time / 60, label=\"Training Compiler PT\", width=0.35)\n",
- "\n",
- "plt.xlabel(f\"====> {time_decrease}% faster <====\")\n",
- "plt.xticks(ticks=[1, 1.5], labels=[\"Baseline PT\", \"Training Compiler PT\"])\n",
- "plt.show()"
+ "print(\n",
+ " f\"SageMaker Training Compiler integrated PyTorch is about {int(speedup)}% faster in terms of total training time as reported by HF.\"\n",
+ ")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- ""
+ "### Billable Time\n",
+ "\n",
+ "The following script creates a plot that compares the billable time of the two training jobs with and without SageMaker Training Compiler."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def BillableTimeInSeconds(name):\n",
+ " describe_training_job = (\n",
+ " optimized_estimator.sagemaker_session.sagemaker_client.describe_training_job\n",
+ " )\n",
+ " details = describe_training_job(TrainingJobName=name)\n",
+ " return details[\"BillableTimeInSeconds\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Billable = {}\n",
+ "Billable[\"Native\"] = BillableTimeInSeconds(native_estimator.latest_training_job.name)\n",
+ "Billable[\"Optimized\"] = BillableTimeInSeconds(optimized_estimator.latest_training_job.name)\n",
+ "pd.DataFrame(Billable, index=[\"BillableSecs\"])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "speedup = (Billable[\"Native\"] - Billable[\"Optimized\"]) * 100 / Billable[\"Native\"]\n",
+ "print(f\"SageMaker Training Compiler integrated PyTorch was {int(speedup)}% faster in summary.\")"
]
},
{
@@ -1002,20 +1031,13 @@
"plt.show()"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- ""
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Conclusion\n",
"\n",
- "In this example, we fine-tuned an [ALBERT model](https://huggingface.co/albert-base-v2) (`albert-base-v2`) with the SQuAD dataset and compared a native training job with a SageMaker Training Compiler training job. The Training Compiler job has `86% higher throughput` and `40% quicker training` time while training loss was equal with the native pytorch training job."
+ "In this example, we fine-tuned an [ALBERT model](https://huggingface.co/albert-base-v2) (albert-base-v2) with the SQuAD dataset and compared a native training job with a SageMaker Training Compiler training job. The Training Compiler job has 93% higher throughput and 38% quicker training time while training loss was equal with the native PyTorch training job."
]
},
{
@@ -1051,8 +1073,8 @@
" sm.stop_training_job(TrainingJobName=name)\n",
"\n",
"\n",
- "stop_training_job(huggingface_estimator.latest_training_job.name)\n",
- "stop_training_job(compile_estimator.latest_training_job.name)"
+ "stop_training_job(native_estimator.latest_training_job.name)\n",
+ "stop_training_job(optimized_estimator.latest_training_job.name)"
]
},
{
@@ -1070,9 +1092,9 @@
},
"instance_type": "ml.p3.2xlarge",
"kernelspec": {
- "display_name": "conda_pytorch_p36",
+ "display_name": "conda_pytorch_p38",
"language": "python",
- "name": "conda_pytorch_p36"
+ "name": "conda_pytorch_p38"
},
"language_info": {
"codemirror_mode": {
@@ -1084,7 +1106,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.13"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/scripts/requirements.txt b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/scripts/requirements.txt
new file mode 100644
index 0000000000..db6140254d
--- /dev/null
+++ b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/albert-base-v2/scripts/requirements.txt
@@ -0,0 +1,2 @@
+transformers == 4.21.1
+datasets == 1.18.4
\ No newline at end of file
diff --git a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/bert-base-cased-single-node-single-gpu.ipynb b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/bert-base-cased-single-node-single-gpu.ipynb
index 1650577653..2de9b89d3c 100644
--- a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/bert-base-cased-single-node-single-gpu.ipynb
+++ b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/bert-base-cased-single-node-single-gpu.ipynb
@@ -4,7 +4,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Compile and Train a Hugging Face Transformer ``BERT`` Model with the SST Dataset using SageMaker Training Compiler"
+ "# Compile and Train a Hugging Face Transformer BERT Model with the SST Dataset using SageMaker Training Compiler"
]
},
{
@@ -50,13 +50,13 @@
"source": [
"## Introduction\n",
"\n",
- "This notebooks is an end-to-end binary text classification example. In this demo, we use the Hugging Face's `transformers` and `datasets` libraries with SageMaker Training Compiler to compile and fine-tune a pre-trained transformer for binary text classification. In particular, the pre-trained model will be fine-tuned using the Stanford Sentiment Treebank (SST) dataset. To get started, you need to set up the environment with a few prerequisite steps, for permissions, configurations, and so on. \n",
+ "This notebook is an end-to-end binary text classification example. In this demo, we use the Hugging Face's transformers and datasets libraries with SageMaker Training Compiler to compile and fine-tune a pre-trained transformer for binary text classification. In particular, the pre-trained model will be fine-tuned using the Stanford Sentiment Treebank (SST) dataset. To get started, you need to set up the environment with a few prerequisite steps, for permissions, configurations, and so on. \n",
"\n",
"\n",
"\n",
- "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, `Python 3 (PyTorch x.y Python 3.x CPU Optimized)` or `conda_pytorch_p36` respectively.\n",
+ "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, Python 3 (PyTorch x.y Python 3.x CPU Optimized) or conda_pytorch_p36 respectively.\n",
"\n",
- "**NOTE:** This notebook uses two `ml.p3.2xlarge` instances that have single GPU. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
+ "**NOTE:** This notebook uses two ml.p3.2xlarge instances that have single GPU. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
]
},
{
@@ -72,7 +72,7 @@
"source": [
"### Installation\n",
"\n",
- "This example notebook requires the **SageMaker Python SDK v2.70.0** and **transformers v4.11.0**."
+ "This example notebook requires the **SageMaker Python SDK v2.108.0** and **transformers v4.21**."
]
},
{
@@ -81,7 +81,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install sagemaker botocore boto3 awscli s3fs typing-extensions --upgrade"
+ "!pip install \"sagemaker>=2.108.0\" botocore boto3 awscli s3fs typing-extensions \"torch==1.11.0\" --upgrade"
]
},
{
@@ -90,7 +90,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install transformers datasets --upgrade"
+ "!pip install \"transformers==4.21\" datasets --upgrade"
]
},
{
@@ -112,7 +112,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Copy and run the following code if you need to upgrade ipywidgets for `datasets` library and restart kernel. This is only needed when preprocessing is done in the notebook.\n",
+ "Copy and run the following code if you need to upgrade ipywidgets for datasets library and restart kernel. This is only needed when preprocessing is done in the notebook.\n",
"\n",
"```python\n",
"%%capture\n",
@@ -134,7 +134,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Note:** If you are going to use Sagemaker in a local environment. You need access to an IAM Role with the required permissions for SageMaker. To learn more, see [SageMaker Roles](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html)."
+ "**Note:** If you are going to use SageMaker in a local environment. You need access to an IAM Role with the required permissions for SageMaker. To learn more, see [SageMaker Roles](https://docs.aws.amazon.com/sagemaker/latest/dg/sagemaker-roles.html)."
]
},
{
@@ -343,7 +343,7 @@
"source": [
"### Uploading data to `sagemaker_session_bucket`\n",
"\n",
- "After we processed the `datasets` we are going to use the new `FileSystem` [integration](https://huggingface.co/docs/datasets/filesystems.html) to upload our dataset to S3."
+ "After we processed the datasets we are going to use the new FileSystem [integration](https://huggingface.co/docs/datasets/filesystems.html) to upload our dataset to S3."
]
},
{
@@ -375,11 +375,11 @@
"source": [
"## SageMaker Training Job\n",
"\n",
- "To create a SageMaker training job, we use a `HuggingFace` estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through `entry_point`, which `instance_type` to use for training, which `hyperparameters` to pass, and so on.\n",
+ "To create a SageMaker training job, we use a HuggingFace/PyTorch estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through entry_point, which instance_type to use for training, which hyperparameters to pass, and so on.\n",
"\n",
- "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the `HuggingFace` Deep Learning Container, uploads your training script, and downloads the data from `sagemaker_session_bucket` into the container at `/opt/ml/input/data`.\n",
+ "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the HuggingFace Deep Learning Container, uploads your training script, and downloads the data from sagemaker_session_bucket into the container at /opt/ml/input/data.\n",
"\n",
- "In the following section, you learn how to set up two versions of the SageMaker `HuggingFace` estimator, a native one without the compiler and an optimized one with the compiler."
+ "In the following section, you learn how to set up two versions of the SageMaker HuggingFace/PyTorch estimator, a native one without the compiler and an optimized one with the compiler."
]
},
{
@@ -404,9 +404,9 @@
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.huggingface import HuggingFace\n",
+ "from sagemaker.pytorch import PyTorch\n",
"\n",
- "hyperparameters = {\"epochs\": 5, \"train_batch_size\": 14, \"model_name\": \"bert-base-cased\"}\n",
+ "hyperparameters = {\"epochs\": 5, \"train_batch_size\": 16, \"model_name\": \"bert-base-cased\"}\n",
"\n",
"# Scale the learning rate by batch size, as original LR was using batch size of 32\n",
"hyperparameters[\"learning_rate\"] = float(\"5e-5\") / 32 * hyperparameters[\"train_batch_size\"]\n",
@@ -421,11 +421,11 @@
"metadata": {},
"outputs": [],
"source": [
- "# By setting the hyperparameters in the HuggingFace Estimator below\n",
+ "# By setting the hyperparameters in the PyTorch Estimator below\n",
"# and using the AutoModelForSequenceClassification class in the train.py script\n",
"# we can fine-tune the bert-base-cased pretrained Transformer for sequence classification\n",
"\n",
- "huggingface_estimator = HuggingFace(\n",
+ "native_estimator = PyTorch(\n",
" entry_point=\"train.py\",\n",
" source_dir=\"./scripts\",\n",
" instance_type=\"ml.p3.2xlarge\",\n",
@@ -434,18 +434,18 @@
" py_version=\"py38\",\n",
" base_job_name=\"native-sst-bert-base-cased-p3-2x-pytorch-190\",\n",
" volume_size=volume_size,\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
+ " transformers_version=\"4.21.1\",\n",
+ " framework_version=\"1.11.0\",\n",
" hyperparameters=hyperparameters,\n",
" disable_profiler=True,\n",
" debugger_hook_config=False,\n",
")\n",
"\n",
"# starting the train job with our uploaded datasets as input\n",
- "huggingface_estimator.fit({\"train\": training_input_path, \"test\": test_input_path}, wait=False)\n",
+ "native_estimator.fit({\"train\": training_input_path, \"test\": test_input_path}, wait=False)\n",
"\n",
"# The name of the training job. You might need to note this down in case your kernel crashes.\n",
- "huggingface_estimator.latest_training_job.name"
+ "native_estimator.latest_training_job.name"
]
},
{
@@ -493,7 +493,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# By setting the hyperparameters in the HuggingFace Estimator below\n",
+ "# By setting the hyperparameters in the PyTorch Estimator below\n",
"# and using the AutoModelForSequenceClassification class in the train.py script\n",
"# the bert-base-cased pretrained Transformer is fine-tuned for sequence classification\n",
"\n",
@@ -508,8 +508,8 @@
" py_version=\"py38\",\n",
" base_job_name=\"sm-compiled-sst-bert-base-cased-p3-2x-pytorch-190\",\n",
" volume_size=volume_size,\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
+ " transformers_version=\"4.21.1\",\n",
+ " pytorch_version=\"1.11.0\",\n",
" compiler_config=TrainingCompilerConfig(),\n",
" hyperparameters=hyperparameters,\n",
" disable_profiler=True,\n",
@@ -538,10 +538,10 @@
"metadata": {},
"outputs": [],
"source": [
- "waiter = huggingface_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
+ "waiter = native_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
" \"training_job_completed_or_stopped\"\n",
")\n",
- "waiter.wait(TrainingJobName=huggingface_estimator.latest_training_job.name)\n",
+ "waiter.wait(TrainingJobName=native_estimator.latest_training_job.name)\n",
"waiter = sm_training_compiler_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
" \"training_job_completed_or_stopped\"\n",
")\n",
@@ -569,14 +569,14 @@
"outputs": [],
"source": [
"# container image used for native training job\n",
- "print(f\"container image used for training job: \\n{huggingface_estimator.image_uri}\\n\")\n",
+ "print(f\"container image used for training job: \\n{native_estimator.image_uri}\\n\")\n",
"\n",
"# s3 uri where the native trained model is located\n",
- "print(f\"s3 uri where the trained model is located: \\n{huggingface_estimator.model_data}\\n\")\n",
+ "print(f\"s3 uri where the trained model is located: \\n{native_estimator.model_data}\\n\")\n",
"\n",
"# latest training job name for this estimator\n",
"print(\n",
- " f\"latest training job name for this estimator: \\n{huggingface_estimator.latest_training_job.name}\\n\"\n",
+ " f\"latest training job name for this estimator: \\n{native_estimator.latest_training_job.name}\\n\"\n",
")"
]
},
@@ -591,16 +591,16 @@
"%%capture native\n",
"\n",
"# access the logs of the native training job\n",
- "huggingface_estimator.sagemaker_session.logs_for_job(huggingface_estimator.latest_training_job.name)"
+ "native_estimator.sagemaker_session.logs_for_job(native_estimator.latest_training_job.name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new HuggingFace estimator. For example:\n",
+ "**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new PyTorch estimator. For example:\n",
"```python\n",
- "huggingface_estimator = HuggingFace.attach(\"your_huggingface_training_job_name\")\n",
+ "native_estimator = PyTorch.attach(\"your_native_training_job_name\")\n",
"```"
]
},
@@ -712,7 +712,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Plot and compare throughputs of compiled training and native training"
+ "### Plot and compare throughput of compiled training and native training"
]
},
{
@@ -765,7 +765,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "#### Example output for SageMaker Training Compiler traing job\n",
+ "#### Example output for SageMaker Training Compiler training job\n",
"\n",
"{'train_runtime': 3742.9028,\n",
" 'train_samples_per_second': 89.969,\n",
@@ -801,27 +801,6 @@
"plt.xticks(ticks=[1, 1.5], labels=[\"Baseline PT\", \"SM-Training-Compiler-enhanced PT\"])"
]
},
- {
- "attachments": {
- "throughput.png": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAElCAYAAAD+wXUWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de7xd853/8ddbQiOCiITGJaJiqrQoKUGrTFNKFTNl6FBpS7VTner82qpWR3Xqml4U0ypliLq2qbp1Ki6Dxl1CSAilLhFS4hLi0gr5/P74fresbN9zzt4nZ5+dnLyfj8d5nLXX+q61PmvfPvu7Lp+liMDMzKzeCu0OwMzMlk5OEGZmVuQEYWZmRU4QZmZW5ARhZmZFThBmZlbkBGFLHUn9JL0iaURPtu1JkkZJWirOEZfUX1JIGtnuWKxvcYKwJZa/oGt/CyW9Xnl8QLPLi4i3ImJQRMzqybbNkHRWZRvekLSg8vjKnlzX0mxpSoTW++QL5awnSXocOCQiruukTf+IeLP3oloyko4F1ouIz1XGjQIejgg1uIyWbbOk/sACYMOIeLyHl93Udlrf4h6EtZykYyVdIukiSfOBAyVtJ+l2SfMkzZF0qqQVc/vFdplIOj9P/6Ok+ZJuk7Rhs23z9N0k/VnSS5JOk3SLpM8twbYdJGm2pLmSjuximwfk2OZIekrSTyWtlNsfIunGyvz12zVM0h8kvSzpTknHV9tnu0p6RNKLkk6tLOsQSX+S9Iu83TMl7VyZPlvSTnWxn5sf/imPq/WePtTd58qWPU4Q1lv+CbgQWB24BHgTOBwYCuwAfAL4Uifz/yvwn8AQYBbww2bbSloL+A3wrbzex4BturtB2fbAKGBX4AeSNq5Mq9/mo4HRwObAB0nb/Z0G13M6MA9YG/gCMK7QZndg67zsAyWNrYvzQdJ2/xD4vaTBDax3R4C8G29QRNzVYLzWBzhBWG+5OSKujIiFEfF6RNwVEXdExJsR8ShwJvDRTuafGBFTImIBcAGwZTfa7gFMi4jL87STgeeWcLuOiYi/RcTdwP3AFpVpi20zcEBuPzcingX+C/hsVyvIPau9gaPzczcD+HWh6QkR8VLezXQjiz9Hc4DTImJBRFwIPArs1vTW2nLFCcJ6y5PVB5I2ybtM/irpZdKX5dBO5v9rZfg1YFA32q5TjSPSAbjZDcTeoYjoLK4n65oPB56oPH4CWLeB1awN9KtbXv2yofPnaHYsfsDxCdLzYdYhJwjrLfVnQ5wBzABGRcRqpN0vrT4QOgdYr/ZAkmjsC7q76rd5DrBB5fEI4Kk8/CowsDLt3ZXhZ4CFVGIH1m8ylvXqHo8Anm5g3T6LZTnmBGHtsirwEvCqpPfR+fGHnnIVsJWkT+Uzfw4HhvXCemsuAo6WNFTSMNJxkvPztHuBzSV9QNLKwPdrM+XdYZeRjnGsLGkz4MAm1z1c0lfzwe/9gY2Aq/O0acD+edo2wD9X5nsWCEnvaXJ91gc4QVi7fIN0oHU+qTdxSatXGBHPAPsBPwWeJ31J3gP8vdXrzn5ASgTTgfuAO4ATcmwPAMeTjh08RD57qOLfgDVJvYlzSMmmmbhvBTYDXgCOAT4dES/maUcBm5AOgv8n6cA6Oa75OcY78hlno5tYpy3jfB2ELbck9SPtZtknIia3O55mSPoJMDgiDm6g7SHAgRGxU8sDsz7FPQhbrkj6hKTVJb2L9Gv5TeDONofVJUmb5t1PkjQG+Dzw+3bHZX1b/3YHYNbLPkw69XUl0mmpe0dEb+1iWhKrkeIeTtrNdGJEXNXekKyv8y4mMzMr8i4mMzMrcoKwtsq1ivZuov0Bkq5psO3nJN3c/ei6XP53JZ3VjfkeryuD0VaS3iXpwVyKxOxtThDWNpI2J5WmuDw/Ln6hV79QI+KCiNildyMti4jjI+KQ3lxnfXKRNDIX9ev28cR8DOZ/gG/3RIzWdzhBWDt9CbggloEDYUvyBbw0q2zXhcC4fHaXGeAEYe21G3BTMzPU9zIk7SLpoVzG+heSbsrn/Vfn+XEugf2YpN0q41eXdHal/Pax+dqI2npukXSypNrFZfWxHCPp/Dw8QKnU+PP5grK7JK3dyaZ8SNIDOa5zJA2oLHcPSdPycm7NPS0k/ZpUIuPKXHr7CBZdUDcvj9sut/1CLuv9oqRJkjaoLD8kHSbpYeBhgIiYDbwIjOn6VbDlhROEtYWkVYANSVcNd3cZQ4GJpJLZa+ZlbV/XbNs8figwHjg712ACmEC6DmIUqUT2LsAhdfM+CqwFHNdFOONIZb3Xz7F8GXi9k/YHkEqEbwT8A/C9vE1bkXb3fCkv5wzgCknviojPksqXfyqX3h5PLsdNumhuUETclo/pfJdUMmMYMJl05XXV3nn7Nq2Mm8ni1WhtOecEYe1SuxfB/LrxY/Iv57f/SL+aS3YH7o+IS/Pd2k5l8YqmAE9ExK8i4i1SQhgOrJ1/3e8GfD0iXs3lt08G9q/M+3REnJZLknf2ZQ/pjm5rkooPvhURUyPi5U7a/3dEPBkRL5CSz2fy+C8CZ+RS6G9FxARSSY1mftl/iVT6e2Z+Xo4Htqz2IvL0F+q2az6LXhczJwhrm3n5/6p142+PiMHVP9Kv5pJGynf/tTL9tTw4iFRVdUVgTiURnUHqLdSUSmp35NfAJOBiSU9LGq98h7wOVJddLb29AfCNugS5Ps2V5t4AOKUy/wukSrnVyrWlbVuVRa+LmROEtUdEvAr8hbR7pbtK5bvry1p35EnSL/OhlWS0WkRsVg2z0UDyjXh+EBGbknZz7QEc1Mks1XLd1dLbTwLH1SXJgRFR20VUH1MpxieBL9UtY+WIuLWL+d5HKiZoBjhBWHv9L53fRa4rfwA+IGnvfDbOYSx+L4MORcQc4BrgJ5JWk7SCpI0kdSseSTvnWkn9gJdJu5ze6mSWwyStJ2kI6XhBrZrtr4AvS9o2111aRdInJdV6Ws8A1dLbc0n3iqiO+yXwHaWy4LWD8ft2Ef+6pFu03t7QBttywQnC2ulM4IDKQeOmRMRzwL6kg8/Pkw64TqHxMtgHkWoyPUA6g2ci6RhFd7w7z/8y6WDvTSy610PJhaQE9Wj+OxYgIqaQjkP8d47pEeBzlflOAL6Xdx99M+82Ow64JY8bExG/B04i7e56mXRjpq5uL/qvwIRlpC6V9RLXYrK2knQh8JuIuKwHlrUC6RjEARFxwxIHt5zI1z7cC+yYD9abAU4QtoyTtCvpxjuvA98i7WZ6TwNnHZlZF7yLyZZ125EOdj8HfIpUvtvJwawHuAdhZmZF7kGYmVnRMlGAbOjQoTFy5Mh2h2FmtkyZOnXqcxExrLvzLxMJYuTIkUyZMqXdYZiZLVMkPbEk83sXk5mZFTlBmJlZkROEmZkVOUGYmVmRE4SZmRU5QZiZWZEThJmZFTlBmJlZkROEmZkVLRNXUpsZcMzq7Y7AetsxL7V19e5BmJlZkROEmZkVOUGYmVmRE4SZmRU5QZiZWZEThJmZFTlBmJlZkROEmZkVOUGYmVmRE4SZmRU5QZiZWZEThJmZFTlBmJlZkROEmZkVOUGYmVmRE4SZmRU5QZiZWZEThJmZFTlBmJlZUUsThKT/kHS/pBmSLpI0QNKGku6Q9LCkSySt1MoYzMyse1qWICStC3wNGB0R7wf6AfsDJwEnR8TGwIvAwa2KwczMuq/Vu5j6AytL6g8MBOYA/whMzNMnAHu3OAYzM+uGliWIiHgK+DEwi5QYXgKmAvMi4s3cbDawbml+SYdKmiJpyty5c1sVppmZdaCVu5jWAPYCNgTWAVYBdis0jdL8EXFmRIyOiNHDhg1rVZhmZtaBVu5iGgs8FhFzI2IBcCmwPTA473ICWA94uoUxmJlZN7UyQcwCxkgaKEnAx4AHgBuAfXKbccDlLYzBzMy6qZXHIO4gHYy+G5ie13Um8G3g/0l6BFgTOLtVMZiZWff177pJ90XE94Hv141+FNimles1M7Ml5yupzcysyAnCzMyKnCDMzKzICcLMzIqcIMzMrMgJwszMipwgzMysyAnCzMyKnCDMzKzICcLMzIqcIMzMrMgJwszMipwgzMysyAnCzMyKnCDMzKzICcLMzIqcIMzMrMgJwszMipwgzMysyAnCzMyKnCDMzKzICcLMzIqcIMzMrMgJwszMipwgzMysyAnCzMyKnCDMzKzICcLMzIqcIMzMrMgJwszMivq3O4BWG3nkH9odgvWyx0/8ZLtDMOsT3IMwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrammCkDRY0kRJD0qaKWk7SUMkXSvp4fx/jVbGYGZm3dPqHsQpwNURsQmwBTATOBK4PiI2Bq7Pj83MbCnTsgQhaTVgR+BsgIh4IyLmAXsBE3KzCcDerYrBzMy6r6EEIWkVSStUHq8gaWAXs70HmAucI+keSWdJWgVYOyLmAOT/a3WwzkMlTZE0Ze7cuQ1tjJmZ9ZxGexDXA9WEMBC4rot5+gNbAadHxAeBV2lid1JEnBkRoyNi9LBhwxqdzczMekijCWJARLxSe5CHu+pBzAZmR8Qd+fFEUsJ4RtJwgPz/2eZCNjOz3tBognhV0la1B5K2Bl7vbIaI+CvwpKT35lEfAx4ArgDG5XHjgMubitjMzHpFo9Vcvw78VtLT+fFwYL8G5vt34AJJKwGPAp8nJaXfSDoYmAXs21zIZmbWGxpKEBFxl6RNgPcCAh6MiAUNzDcNGF2Y9LGmojQzs17X6FlMA4FvA4dHxHRgpKQ9WhqZmZm1VaPHIM4B3gC2y49nA8e2JCIzM1sqNJogNoqI8cACgIh4nbSryczM+qhGE8QbklYGAkDSRsDfWxaVmZm1XaNnMX0fuBpYX9IFwA7A51oVlJmZtV+jZzFdK+luYAxp19LhEfFcSyMzM7O2avQsph2Av0XEH4DBwHclbdDSyMzMrK0aPQZxOvCapC2AbwFPAOe1LCozM2u7RhPEmxERpFLdp0bEKcCqrQvLzMzardGD1PMlfQc4ENhRUj9gxdaFZWZm7dZoD2I/0mmtB+cifOsCP2pZVGZm1nad9iAkTSKd3vrHiPhpbXxEzMLHIMzM+rSuehDjgBeBYyTdLel0SXtJGtQLsZmZWRt12oPIu5POBc7NtxzdFtgNOELS68A1uQSHmZn1MY0epCYiFgK35b+jJQ0Fdm1VYGZm1l6NXig3XtJqklaUdL2k54BPRMQFLY7PzMzapNGzmHaJiJeBPUilvv+BdMGcmZn1UY0miNo1D7sDF0XECy2Kx8zMlhKNHoO4UtKDwOvAVyQNA/7WurDMzKzdGupBRMSRpLvJjc73on6NVHbDzMz6qGbuSX0YqWgfwDrA6FYFZWZm7dfsPam3z499T2ozsz7O96Q2M7Mi35PazMyKfE9qMzMr8j2pzcysqKty31vVjZqT/4+QNCIi7m5NWGZm1m5d9SB+0sm0AP6xB2MxM7OlSFflvnfurUDMzGzp0tAxCEkDgK8AHyb1HCYDv4wIl9swM+ujGj2L6TxgPnBafvwZ4NfAvq0IyszM2q/RBPHeiNii8vgGSfe2IiAzM1s6NHqh3D2SxtQeSNoWuKU1IZmZ2dKg0R7EtsBBkmblxyOAmZKmAxERm7ckOjMza5tGE8QnWhqFmZktdRq9kvoJSWsA61fn8YVyZmZ9V6Onuf6QVHvpL+SCffhCOTOzPq3RXUz/Qir5/UazK5DUD5gCPBURe0jaELgYGALcDXy2O8s1M7PWavQsphnA4G6u43BgZuXxScDJEbEx8CJwcDeXa2ZmLdRogjiBdKrrJElX1P66mknSesAngbPyY5F2S03MTSYAezcftpmZtVqju5gmkH75TwcWNrH8nwFHAKvmx2sC8yLizfx4NrBuaUZJhwKHAowYMaKJVZqZWU9oNEE8FxGnNrNgSXsAz0bEVEk71UYXmkZhHBFxJnAmwOjRo4ttzMysdRpNEFMlnQBcQeVWo12c5roDsKek3YEBwGqkHsVgSf1zL2I94OluRW5mZi3VaIL4YP4/pjKu09NcI+I7wHcAcg/imxFxgKTfAvuQzmQaB1zeZMxmZtYLGr1QrifvC/Ft4GJJxwL3AGf34LLNzKyHNNqDQNIngc1Iu4sAiIj/amTeiLgRuDEPPwps00yQZmbW+xo6zVXSL4H9gH8nHWjeF9ighXGZmVmbNXodxPYRcRDwYkT8ANiOVJfJzMz6qEYTxOv5/2uS1gHeBDZsTUhmZrY0aPQYxFWSBgPjgal53FmtCcnMzJYGnSYISR8CnoyIH+bHg0hXUz8InNz68MzMrF262sV0BvAGgKQdgRPzuJfIVzmbmVnf1NUupn4R8UIe3g84MyJ+B/xO0rTWhmZmZu3UVQ+in6RaEvkY8H+VaQ1fQ2FmZsuerr7kLwJukvQc6UymyQCSRpF2M5mZWR/VaYKIiOMkXQ8MB66JiFpV1RVIF82ZmVkf1eVuooi4vTDuz60Jx8zMlhaNXihnZmbLGScIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMralmCkLS+pBskzZR0v6TD8/ghkq6V9HD+v0arYjAzs+5rZQ/iTeAbEfE+YAxwmKRNgSOB6yNiY+D6/NjMzJYyLUsQETEnIu7Ow/OBmcC6wF7AhNxsArB3q2IwM7Pu65VjEJJGAh8E7gDWjog5kJIIsFYH8xwqaYqkKXPnzu2NMM3MrKLlCULSIOB3wNcj4uVG54uIMyNidESMHjZsWOsCNDOzopYmCEkrkpLDBRFxaR79jKThefpw4NlWxmBmZt3TyrOYBJwNzIyIn1YmXQGMy8PjgMtbFYOZmXVf/xYuewfgs8B0SdPyuO8CJwK/kXQwMAvYt4UxmJlZN7UsQUTEzYA6mPyxVq3XzMx6hq+kNjOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzIicIMzMrcoIwM7MiJwgzMytygjAzsyInCDMzK3KCMDOzorYkCEmfkPSQpEckHdmOGMzMrHO9niAk9QN+DuwGbAp8RtKmvR2HmZl1rh09iG2ARyLi0Yh4A7gY2KsNcZiZWSf6t2Gd6wJPVh7PBratbyTpUODQ/PAVSQ/1Qmx9zVDguXYH0dt0UrsjsBZYLt/L/EBLuoQNlmTmdiSI0hbHO0ZEnAmc2fpw+i5JUyJidLvjMFtSfi+3Rzt2Mc0G1q88Xg94ug1xmJlZJ9qRIO4CNpa0oaSVgP2BK9oQh5mZdaLXdzFFxJuSvgpMAvoB/xMR9/d2HMsJ76KzvsLv5TZQxDt2/5uZmflKajMzK3OCMDOzIieIXiTpLUnTJN0r6W5J2/fw8s+VtE8ePqsnrlCXtJOklyTdI2mmpO9L2jVvxzRJr+SyKdMknbfkW7HsknSUpPsl3Zefj23z+BslzZKkStvLJL3SwTJqz+1bleGvNRHHtpJO7qJNP0mTm9m+Lpa3oqTxuXzODEl3SNq1p5ZfWN/b8UsaJWlaq9bVwfr7S5rXm+vsiKTZkgZ3MH56fj9eLWktSVPy+2mWpLmV99f6pWW34zqI5dnrEbElQP7wnAB8tBUriohDenBxkyNiD0mrANOAqyrbcSPwzYiY0oPrW+ZI2g7YA9gqIv4uaSiwUqXJPGAH4Ob8YR5eWk5EHAccl5f5Su15Lqyvf0S82cEy7gDu6CzeiHgL+EjnW9WUE4AhwKYR8Yak4aTtbYklib+z564P+khEzJM0Hjiydi2JpEOA90fE1zub2T2I9lkNeBFA0iBJ1+dexXRJe+Xxq0j6Q+5xzJC0Xx6/taSbJE2VNCl/GBeTf7XW3gyvSDouL+d2SWvn8cMk/U7SXfmv0w90RLwKTAU26tFnom8YDjwXEX8HiIjnIqJ6fc/FpFO6Af4ZuLTZFUg6X9JPJN0AHC9pjKTbcu/uFkkb53ZjJV2Wh4+VdHZ+vzwq6bA8/u1fwLn99ZIuzb3B8yrr3DOPmyzptNpy6+JaFfgc8LVcPoeImBMRE/P0A/P7eoak46vrl/Sj/L6flHs+tTh3z+0OkfT7PP0hSd+rj78ulv6SfirpzvzL+ZDKNl4n6WLgnsJ8g5R64Hfm5/NTlfVPzOt/WNIJdfOdmD9Xt0laK4/bS6kHdY+kayrji69Fnvb5HO+9ks7J49bOr8mUHNeYPH6YpGvz83Y65YuP6/0JGNVAu8VFhP966Q94i/QL/EHgJWDrPL4/sFoeHgo8kl/0TwO/qsy/OrAicCswLI/bj3SqMMC5wD55+EZgdB4O4FN5eDzwvTx8IfDhPDwCmFmIeSdSjwFgTeBxYLPK9LfXszz/AYPya/tn4BfAR+ueo22B+0indl8DjARe6WKZr9Q9Ph+4DFih8n7ol4c/AVySh8cCl+XhY4HJpN7MWsDzOYb+wLxK+xdJSa4f6VqlMcBA0oWtG+T3429ry62Layvgrg62Yb38nhma37s3kXpa/fP78uO53ZXAH/P4rYEpefwhwFPAGsAqwAPAlnXxjwKm5eGvkH4pA7yLlAxG5G18BRjRQZzjgf3z8Br5dRyQ1/8wsCqwMqlM0DqV+HfL8/y0st41WHSG6JeBk7p4LbYgfScMye1q/y8BxuThkcCMPPwL4Lt5eK8cx+DCNs0GBufX7pfAcZVphwA/6+p97V1Mvau6i2k74DxJ7ye9gMdL2hFYSKpXtTYwHfixpJNIX9KTc/v3A9cq7dLuB8zpYr1vAFfl4anAx/PwWGBTLdo1vpqkVSNift38H5F0T47txPB1K+8QEa9I2pq022Nn4BJJR0bEubnJW8DNpIS+ckQ8Xnnem/HbiFiYhweT3kNd9eiuivTL/llJLwDDeGddo9sjYg6A0v78kcCbwEMR8UQefxFwUJPxbgv8X0Q8l5dxIbAjcDXp83BtbjcdeCnSdVLT8/prJkVErbd9GfBhYEYH69sFeJ+kWm9tdWDjPHxbRMzqZL7dtOj2AwNIiQXgutpnQtKDefyzOf4/5jZTWbTLawTwG0nvJiWpP1fWU3ot/pGU3F8AqP0nfT7fW3mfrCFpZdLzt3tue7mk+s9r1WTS53Ya0HSVMieINomI25T2Uw8jvdjDSD2KBZIeBwZExJ/zl87uwAmSrgF+D9wfEds1sboFkX82kL6oaq/7CsB2EfF6F/NPjog9mljfcinSfvEbgRvzl9w4Uq+u5mLS63dMdT5JxwGfzMsoHnOoeLUyfBzpy/MXkkaRvnRL/l4Zrr7+XbXpMINJuo7UK7gd+BawoaRVIu2GXKxpR8sg/XCpWViJYWFdjPUXa3V28ZaAr0TE9XXxjqXy3Ckd9P9CfrhLnm/viPhL3Xw70vHz90YH438OHB8R/5vXW73nTUfPc2mbBGyTE0o1JjpoX/KRiOj2wXQfg2gTSZuQfv0/T/qV82xODjuTKzBKWgd4LSLOB35M6so/BAzLPZDa2SObdTOMa4CvVmLq6svJOiDpvcrHALItgSfqmk0mHcy9qDoyIo6KiC0bSA71ViftfoF0DKCn3U/6Bbu+0rfSfrUJETE2x/zl/Ov6POBnklaE9N6VdAApgewsaU1J/UnHYW5qMo5dJA2WNJC0S+WWTtpOAr6S11V7XVaubxQRp9ae84h4Ns/39plikj7YZIxVqwNP5edsXAPtrwP2lzQkr3tIZXz1OEXt/fEn4IA87lOk3V8t4QTRu1ZWPq2MtH9xXP7VeQEwWtIU0gv/YG7/AeDO3P4o4Nj8a2If4CRJ95K6jt09XfZreb33SXqAtL/UumcQMEHSA5LuI90M65hqg0h+XNvd0gNOAn4kqbMvzG6LiNdIPyCuIyW3p0nHzkqOzNNm5t7TpaQfPbOBo0k9q2mkXVl/aDKUm0nHy+4BLoqIzk5pPYN0zGCapBnA6TS2p+QHwEClg+n3U/faNekYUk/xJuCZrhpHxH2kYyB/yp/1H+VJhwE7VD6fX8zjvw+MlXQ36RjhU7SIS22YWYckDcrHV0T68p0eEaf14vobOh3TWsM9CDPrzL/lX7UPkM7i+VWb47Fe5B6EmZkVuQdhZmZFThBmZlbkBGFmZkVOEMsxSVfkUwFrj4fkGi8P5/9r5PGfVqpSOlnSmnncRkp1bTpa9he0qJLkDC2qL3WupNeU6vfU2p4iKfKFg9VlfF6Lqk2+kZc3TdKJTWzj+pIuaaDdpGpMS0LJEUq1g2bkmA/oiWV3ss5JklbVUlJlVNJhrd5maz0fpO4DlKqsLqi/4rKLef6ZdD3F5hHx/jxuPPBCRJyoVHJgjYj4tqRbgV1JFzkNiIjTlMouHB0RDxeWvR7pHPCtIuIlSYNItaMek3Qu6YK/8RFxvqQVSOfHDwG27OgaAaWry0eXpmspq86pdEvdTwL/EhHzlaq37hkRLS+Hni8Qey4i3lH+uYP2In0PLOyy8TJA0pBKqQpbQu5B9A3/ADykVOnzfV01zl/Y/49UPKxqL2BCHp4A7J2HF5JqygwEFkj6CDCnlByytYD5pOJoRMQrEfFYZfpFLLoqdyfSlbFNfcErVcY8Q9K1wDm5RzNZqYLmVC26F8Pb9wpQJ5U5lWvq5/YzlKpu3i/pj5IG5DZjco/oVqUqpB1dsPVdoHaFMRExr5YcJH089yimS/qVpJUq6z9OqdruXZK2UqoE+hdJX8xtxkq6QeleEg9I+nn+gu/sngBHalFl06Mrz8kMSb8E7qau9HjetgfyPCflce+oUKp0T4YnJK2W20ipSunQ/Pp8PY+/Wanq6Z25V7V9Hr+KUjXheyVdpFS1dMvcC/q1FlWA7fReGJJWVqoYeyOpaJ71lK6q+fmvyyqeJ6FsGdAAAAXISURBVJN+Adf/1So77tzB9Fsry5jcQZuxefq3Oph+amUZq5IqNN5CuvL088AqncT8T1QqRObx8+ravZj/f5xUjOxKUhmBSaTeRUfPSb/cZhZwDrmSbJ52Lqnncjup6uWvSPfEeBwY2skyF5tOSm53kno0kJJXbXgT4I48XK30WazMmafVKl+OAhYAH8jjL2VRlc+ZpNo4kEqfTCvEuQYwt4NtGJjXuVF+fAHw1cr6v5iHTyNdNbwKqWjjX/P4scBr+XXrB/wfqX5QNf5qldPdSZU/RfoxeDXpqvtRpKT/oUKMa5NKbNT2LgyubFepQunPgc/m4R2Aqyuvz9fz8M2V9ntW2hwJ/DwPb0GqTbQlqcDfHysxvaNSaR7/wbz+v+TnbMvKtE0pf2amAavmNhM7mH5Ann5QB9NrVXMHd7KO97b7u6kn/toegP9a8KKmD8etwMuFaVsCV+bhkTSQIOrGjQMOJ5WDnkj6gh9YaCdgG+A7pPLlx+Tx55ISxBHAv5FKYK9A9xLEUZXHa5C+cGfkD+j8PL4+QZxemedaFpVTriaImZU2R+UvsqHAXyrjt6KcIIbQcYLYmlTZtPZ4V+A3lfWvnYcPrYvzaVIpj7F18x8K/Lgu/mqC+BnwWOVL6xFSzaZRwMMdxLgiqbLqWaQfESvm8Vvk52s6qTpprQT8jpXh04DPV16faoLYNg+vCzyYh68iFZOrrfs+0vtzTeBR4JT8HKkQ5xHA66Ty3u9q92eur/65musSUrq1486FSRdH2pe/M+kXe73XIqLW1Z5MueDWNyPiOknfIhfnqvOniKgWGNuA9AXwGeBeyvVktgO2zvv0+wNrSboxInYCnpE0PCLmKN2E6Nm6bR1IShC7kgr97QX8a45tsStsI32K7yTVkrqW1JOoxnMxaffGhIhYmPeUoHQTlVrNmd1j8Zvu1KtWDv0G6df5gaQvuXfczjNrRWXT84DNgVkRsaekBZJGxDtLS3dV37tazbQaQ7W6abOVTY+NiLPr4h3F4s/dooWlgpGjSb3G/UlJfBc6rlA6GThX6eSFPYH/7GLbqs958fmIiOclbQ7sRqoX9mlSMqyaQOpFHQZ8TOkmO1dHPhaldLvdCzuI5SORjg1NpHwTnR9FxAWSDiLtiq33UETsl3fp3djBOvaLiIc6mLbMcIJYQhHxH11Mv4H0q6izNp3eOjEifsSiAl7vIGkk6RffUNIX8Q4R8XwHyzqdVMCsNt9VOTkAXEFKACfm/5fXzX4EcEr+ElmZ9OW0kLTrpBrPOsC7I+LuPOodlU0jYpako0iF4Krjf076MmrW6sAjERGSxtHYXbYaFhFz8xf/6Ei3V92/Mq3+HgknAr+Q9JlYdJB6X1IPZ2NJ74mIR0nJrNnKpmMkjSAVaPsX0q/2jkwCvifp4oh4Venkgb91tnClM7kGRMRVku4gldiADiqU5uf7clJv5d5orrT0zXkbJkv6AKnni6RhwN8i4reSHiPd7GYxEfEMqTLuCZI+ChwMnCLp1Ig4JSJqNxbqUETs08X080hVajuaPq+rdSzrnCD6hrdId5i6cwmXcyLpRicHk44f7FubkL/0R0fEMXnUT0jHEeax6GB2zYqkGx2tQ/pCmkuhUmxEnLGE8Vb9NzBR0mdISefvXbTvji+QDojPJ5Vc7qiy6Wmk4wdTJb1BOqYxPiJey8/tpZL6ke4b3Wxto1tJz/1mpF+vV3TUMP/a3wS4PffQ5pN6fJ1ZPcf3LtKuv9ov6GNIFUpnk3qG1QPblwC3kRJeM04j3fDoPlJvcgbpOV0fODsnowC+3dlCIuIm4CZJqwOjm4zBOuHTXM0apFzZNA8fRbo15Dd6cf1jSQe16xPyMknplNz+EfE3pXtpXANsHEvRKcvLO/cgzBq3p6QjSJ+bx2nNTXqWJ4OA63OiEPAlJ4eli3sQZmZW5AvlzMysyAnCzMyKnCDMzKzICcLMzIqcIMzMrOj/A/rLJSMoV+ZEAAAAAElFTkSuQmCC"
- }
- },
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Training Throughput Example Plot\n",
- "\n",
- ""
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "**Note:** For this example, the compiler delivers higher throughput for an ML model as measured by samples per second. However, you might not see an improvement in the total training time for your model. The total training time depends on several other factors, such as key components of the Trainer and TFTrainer APIs."
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -916,7 +895,7 @@
" sm.stop_training_job(TrainingJobName=name)\n",
"\n",
"\n",
- "stop_training_job(huggingface_estimator.latest_training_job.name)\n",
+ "stop_training_job(native_estimator.latest_training_job.name)\n",
"stop_training_job(sm_training_compiler_estimator.latest_training_job.name)"
]
},
@@ -933,9 +912,9 @@
"hash": "c281c456f1b8161c8906f4af2c08ed2c40c50136979eaae69688b01f70e9f4a9"
},
"kernelspec": {
- "display_name": "conda_pytorch_p36",
+ "display_name": "conda_pytorch_p38",
"language": "python",
- "name": "conda_pytorch_p36"
+ "name": "conda_pytorch_p38"
},
"language_info": {
"codemirror_mode": {
@@ -947,7 +926,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.13"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/scripts/requirements.txt b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/scripts/requirements.txt
new file mode 100644
index 0000000000..db6140254d
--- /dev/null
+++ b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/bert-base-cased/scripts/requirements.txt
@@ -0,0 +1,2 @@
+transformers == 4.21.1
+datasets == 1.18.4
\ No newline at end of file
diff --git a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/roberta-base.ipynb b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/roberta-base.ipynb
index 5a41876fca..7e26318a9c 100644
--- a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/roberta-base.ipynb
+++ b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/roberta-base.ipynb
@@ -38,11 +38,11 @@
"\n",
"## Introduction\n",
"\n",
- "In this demo, you'll use Hugging Face's `transformers` and `datasets` libraries with Amazon SageMaker Training Compiler to train the `RoBERTa` model on the `Stanford Sentiment Treebank v2 (SST2)` dataset. To get started, we need to set up the environment with a few prerequisite steps, for permissions, configurations, and so on. \n",
+ "In this demo, you'll use Hugging Face's transformers and datasets libraries with Amazon SageMaker Training Compiler to train the RoBERTa model on the Stanford Sentiment Treebank v2 (SST2) dataset. To get started, we need to set up the environment with a few prerequisite steps, for permissions, configurations, and so on. \n",
"\n",
- "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, `Python 3 (PyTorch x.y Python 3.x CPU Optimized)` or `conda_pytorch_p36` respectively.\n",
+ "**NOTE:** You can run this demo in SageMaker Studio, SageMaker notebook instances, or your local machine with AWS CLI set up. If using SageMaker Studio or SageMaker notebook instances, make sure you choose one of the PyTorch-based kernels, Python 3 (PyTorch x.y Python 3.x CPU Optimized) or conda_pytorch_p36 respectively.\n",
"\n",
- "**NOTE:** This notebook uses two `ml.p3.2xlarge` instances that have single GPU. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
+ "**NOTE:** This notebook uses two ml.p3.2xlarge instances that have single GPU. If you don't have enough quota, see [Request a service quota increase for SageMaker resources](https://docs.aws.amazon.com/sagemaker/latest/dg/regions-quotas.html#service-limit-increase-request-procedure). "
]
},
{
@@ -58,7 +58,7 @@
"source": [
"### Installation\n",
"\n",
- "This example notebook requires the **SageMaker Python SDK v2.70.0** and **transformers v4.11.0**."
+ "This example notebook requires the **SageMaker Python SDK v2.108.0** and **transformers v4.21**."
]
},
{
@@ -67,7 +67,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install sagemaker botocore boto3 awscli --upgrade"
+ "!pip install \"sagemaker>=2.108.0\" botocore boto3 awscli \"torch==1.11.0\" --upgrade"
]
},
{
@@ -76,7 +76,7 @@
"metadata": {},
"outputs": [],
"source": [
- "!pip install -U transformers datasets --upgrade"
+ "!pip install -U \"transformers==4.21.1\" datasets --upgrade"
]
},
{
@@ -99,7 +99,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Copy and run the following code if you need to upgrade ipywidgets for `datasets` library and restart kernel. This is only needed when prerpocessing is done in the notebook.\n",
+ "Copy and run the following code if you need to upgrade ipywidgets for datasets library and restart kernel. This is only needed when preprocessing is done in the notebook.\n",
"\n",
"```python\n",
"%%capture\n",
@@ -164,7 +164,7 @@
"\n",
"If you'd like to try other training datasets later, you can simply use this method.\n",
"\n",
- "For this example notebook, we prepared the `SST2` dataset in the public SageMaker sample file S3 bucket. The following code cells show how you can directly load the dataset and convert to a HuggingFace DatasetDict."
+ "For this example notebook, we prepared the SST2 dataset in the public SageMaker sample file S3 bucket. The following code cells show how you can directly load the dataset and convert to a HuggingFace DatasetDict."
]
},
{
@@ -173,7 +173,7 @@
"source": [
"## Preprocessing\n",
"\n",
- "We download and preprocess the `SST2` dataset from the `s3://sagemaker-sample-files/datasets` bucket. After preprocessing, we'll upload the dataset to the `sagemaker_session_bucket`, which will be used as a data channel for the training job."
+ "We download and preprocess the SST2 dataset from the s3://sagemaker-sample-files/datasets bucket. After preprocessing, we'll upload the dataset to the sagemaker_session_bucket, which will be used as a data channel for the training job."
]
},
{
@@ -253,9 +253,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Uploading data to `sagemaker_session_bucket`\n",
+ "## Uploading data to sagemaker_session_bucket\n",
"\n",
- "We are going to use the new `FileSystem` [integration](https://huggingface.co/docs/datasets/filesystems.html) to upload our preprocessed dataset to S3."
+ "We are going to use the new FileSystem [integration](https://huggingface.co/docs/datasets/filesystems.html) to upload our preprocessed dataset to S3."
]
},
{
@@ -284,11 +284,11 @@
"source": [
"## SageMaker Training Job\n",
"\n",
- "To create a SageMaker training job, we use a `HuggingFace` estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through `entry_point`, which `instance_type` to use for training, which `hyperparameters` to pass, and so on.\n",
+ "To create a SageMaker training job, we use a HuggingFace/PyTorch estimator. Using the estimator, you can define which fine-tuning script should SageMaker use through entry_point, which instance_type to use for training, which hyperparameters to pass, and so on.\n",
"\n",
- "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the `HuggingFace` Deep Learning Container, uploads your training script, and downloads the data from `sagemaker_session_bucket` into the container at `/opt/ml/input/data`.\n",
+ "When a SageMaker training job starts, SageMaker takes care of starting and managing all the required machine learning instances, picks up the HuggingFace Deep Learning Container, uploads your training script, and downloads the data from sagemaker_session_bucket into the container at /opt/ml/input/data.\n",
"\n",
- "In the following section, you learn how to set up two versions of the SageMaker `HuggingFace` estimator, a native one without the compiler and an optimized one with the compiler."
+ "In the following section, you learn how to set up two versions of the SageMaker HuggingFace/PyTorch estimator, a native one without the compiler and an optimized one with the compiler."
]
},
{
@@ -302,7 +302,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Set up an option for fine-tuning or full training. Set `FINE_TUNING = 1` for fine-tuning and using `fine_tune_with_huggingface.py`. Set `FINE_TUNING = 0` for full training and using `full_train_roberta_with_huggingface.py`."
+ "Set up an option for fine-tuning or full training. `FINE_TUNING = 1` is for fine-tuning, and it will use fine_tune_with_huggingface.py. `FINE_TUNING = 0` is for full training, and it will use full_train_roberta_with_huggingface.py."
]
},
{
@@ -318,7 +318,7 @@
"FULL_TRAINING = not FINE_TUNING\n",
"\n",
"# Fine tuning is typically faster and is done for fewer epochs\n",
- "EPOCHS = 4 if FINE_TUNING else 100\n",
+ "EPOCHS = 7 if FINE_TUNING else 100\n",
"\n",
"TRAINING_SCRIPT = (\n",
" \"fine_tune_with_huggingface.py\" if FINE_TUNING else \"full_train_roberta_with_huggingface.py\"\n",
@@ -340,7 +340,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "The `train_batch_size` in the following code cell is the maximum batch that can fit into the memory of an `ml.p3.2xlarge` instance. If you change the model, instance type, sequence length, and other parameters, you need to do some experiments to find the largest batch size that will fit into GPU memory."
+ "The `train_batch_size` in the following code cell is the maximum batch that can fit into the memory of the ml.p3.2xlarge instance. If you change the model, instance type, sequence length, and other parameters, you need to do some experiments to find the largest batch size that will fit into GPU memory."
]
},
{
@@ -349,7 +349,7 @@
"metadata": {},
"outputs": [],
"source": [
- "from sagemaker.huggingface import HuggingFace, TrainingCompilerConfig\n",
+ "from sagemaker.pytorch import PyTorch\n",
"\n",
"# hyperparameters, which are passed into the training job\n",
"hyperparameters = {\"epochs\": EPOCHS, \"train_batch_size\": 18, \"model_name\": \"roberta-base\"}\n",
@@ -367,15 +367,15 @@
"outputs": [],
"source": [
"# configure the training job\n",
- "huggingface_estimator = HuggingFace(\n",
+ "native_estimator = PyTorch(\n",
" entry_point=TRAINING_SCRIPT,\n",
" source_dir=\"./scripts\",\n",
" instance_type=INSTANCE_TYPE,\n",
" instance_count=1,\n",
" role=role,\n",
" py_version=\"py38\",\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
+ " transformers_version=\"4.21.1\",\n",
+ " framework_version=\"1.11.0\",\n",
" volume_size=volume_size,\n",
" hyperparameters=hyperparameters,\n",
" disable_profiler=True,\n",
@@ -383,10 +383,10 @@
")\n",
"\n",
"# start training with our uploaded datasets as input\n",
- "huggingface_estimator.fit({\"train\": training_input_path, \"test\": test_input_path}, wait=False)\n",
+ "native_estimator.fit({\"train\": training_input_path, \"test\": test_input_path}, wait=False)\n",
"\n",
"# The name of the training job.\n",
- "huggingface_estimator.latest_training_job.name"
+ "native_estimator.latest_training_job.name"
]
},
{
@@ -417,7 +417,9 @@
"hyperparameters[\"learning_rate\"] = float(\"5e-5\") / 32 * hyperparameters[\"train_batch_size\"]\n",
"\n",
"# If checkpointing is enabled with higher epoch numbers, your disk requirements will increase as well\n",
- "volume_size = 60 + 2 * hyperparameters[\"epochs\"]"
+ "volume_size = 60 + 2 * hyperparameters[\"epochs\"]\n",
+ "\n",
+ "from sagemaker.huggingface import HuggingFace, TrainingCompilerConfig"
]
},
{
@@ -435,8 +437,8 @@
" instance_count=1,\n",
" role=role,\n",
" py_version=\"py38\",\n",
- " transformers_version=\"4.11.0\",\n",
- " pytorch_version=\"1.9.0\",\n",
+ " transformers_version=\"4.21.1\",\n",
+ " pytorch_version=\"1.11.0\",\n",
" volume_size=volume_size,\n",
" hyperparameters=hyperparameters,\n",
" disable_profiler=True,\n",
@@ -463,10 +465,10 @@
"metadata": {},
"outputs": [],
"source": [
- "waiter = huggingface_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
+ "waiter = native_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
" \"training_job_completed_or_stopped\"\n",
")\n",
- "waiter.wait(TrainingJobName=huggingface_estimator.latest_training_job.name)\n",
+ "waiter.wait(TrainingJobName=native_estimator.latest_training_job.name)\n",
"waiter = optimized_estimator.sagemaker_session.sagemaker_client.get_waiter(\n",
" \"training_job_completed_or_stopped\"\n",
")\n",
@@ -494,14 +496,14 @@
"outputs": [],
"source": [
"# container image used for native training job\n",
- "print(f\"container image used for training job: \\n{huggingface_estimator.image_uri}\\n\")\n",
+ "print(f\"container image used for training job: \\n{native_estimator.image_uri}\\n\")\n",
"\n",
"# s3 uri where the native trained model is located\n",
- "print(f\"s3 uri where the trained model is located: \\n{huggingface_estimator.model_data}\\n\")\n",
+ "print(f\"s3 uri where the trained model is located: \\n{native_estimator.model_data}\\n\")\n",
"\n",
"# latest training job name for this estimator\n",
"print(\n",
- " f\"latest training job name for this estimator: \\n{huggingface_estimator.latest_training_job.name}\\n\"\n",
+ " f\"latest training job name for this estimator: \\n{native_estimator.latest_training_job.name}\\n\"\n",
")"
]
},
@@ -514,16 +516,16 @@
"%%capture native\n",
"\n",
"# access the logs of the native training job\n",
- "huggingface_estimator.sagemaker_session.logs_for_job(huggingface_estimator.latest_training_job.name)"
+ "native_estimator.sagemaker_session.logs_for_job(native_estimator.latest_training_job.name)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new HuggingFace estimator. For example:\n",
+ "**Note:** If the estimator object is no longer available due to a kernel break or refresh, you need to directly use the training job name and manually attach the training job to a new PyTorch estimator. For example:\n",
"```python\n",
- "huggingface_estimator = HuggingFace.attach(\"your_huggingface_training_job_name\")\n",
+ "native_estimator = PyTorch.attach(\"your_huggingface_training_job_name\")\n",
"```"
]
},
@@ -626,9 +628,9 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Plot and compare throughputs of compiled training and native training\n",
+ "### Plot and compare throughput of compiled training and native training\n",
"\n",
- "Visualize average throughputs as reported by HuggingFace and see potential savings."
+ "Visualize average throughput as reported by HuggingFace and see potential savings."
]
},
{
@@ -782,7 +784,7 @@
"outputs": [],
"source": [
"Billable = {}\n",
- "Billable[\"Native\"] = BillableTimeInSeconds(huggingface_estimator.latest_training_job.name)\n",
+ "Billable[\"Native\"] = BillableTimeInSeconds(native_estimator.latest_training_job.name)\n",
"Billable[\"Optimized\"] = BillableTimeInSeconds(optimized_estimator.latest_training_job.name)\n",
"pd.DataFrame(Billable, index=[\"BillableSecs\"])"
]
@@ -823,7 +825,7 @@
" sm.stop_training_job(TrainingJobName=name)\n",
"\n",
"\n",
- "stop_training_job(huggingface_estimator.latest_training_job.name)\n",
+ "stop_training_job(native_estimator.latest_training_job.name)\n",
"stop_training_job(optimized_estimator.latest_training_job.name)"
]
},
@@ -841,9 +843,9 @@
"hash": "c281c456f1b8161c8906f4af2c08ed2c40c50136979eaae69688b01f70e9f4a9"
},
"kernelspec": {
- "display_name": "conda_pytorch_latest_p36",
+ "display_name": "conda_pytorch_p38",
"language": "python",
- "name": "conda_pytorch_latest_p36"
+ "name": "conda_pytorch_p38"
},
"language_info": {
"codemirror_mode": {
@@ -855,7 +857,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.13"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/scripts/requirements.txt b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/scripts/requirements.txt
new file mode 100644
index 0000000000..db6140254d
--- /dev/null
+++ b/sagemaker-training-compiler/huggingface/pytorch_single_gpu_single_node/roberta-base/scripts/requirements.txt
@@ -0,0 +1,2 @@
+transformers == 4.21.1
+datasets == 1.18.4
\ No newline at end of file
diff --git a/sagemaker-training-compiler/index.rst b/sagemaker-training-compiler/index.rst
index 220ff2e876..fa50581c8f 100644
--- a/sagemaker-training-compiler/index.rst
+++ b/sagemaker-training-compiler/index.rst
@@ -19,7 +19,7 @@ For single-node single-GPU training:
huggingface/pytorch_single_gpu_single_node/albert-base-v2/albert-base-v2
huggingface/pytorch_single_gpu_single_node/bert-base-cased/bert-base-cased-single-node-single-gpu
huggingface/pytorch_single_gpu_single_node/roberta-base/roberta-base
-
+ tensorflow/single_gpu_single_node/vision-transformer
For single-node multi-GPU training:
@@ -27,7 +27,7 @@ For single-node multi-GPU training:
:maxdepth: 1
huggingface/pytorch_multiple_gpu_single_node/language-modeling-multi-gpu-single-node
-
+ tensorflow/multiple_gpu_single_node/vision-transformer
For multi-node multi-GPU training:
diff --git a/sagemaker-triton/TensorFlow/Deploy-TensorFlow-Model-Using-NVIDIA-Triton.ipynb b/sagemaker-triton/TensorFlow/Deploy-TensorFlow-Model-Using-NVIDIA-Triton.ipynb
new file mode 100644
index 0000000000..18d3922922
--- /dev/null
+++ b/sagemaker-triton/TensorFlow/Deploy-TensorFlow-Model-Using-NVIDIA-Triton.ipynb
@@ -0,0 +1,461 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "6d6ee543",
+ "metadata": {},
+ "source": [
+ "# Deploy a TensorFlow Model using NVIDIA Triton on SageMaker"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7634d547",
+ "metadata": {},
+ "source": [
+ "Amazon SageMaker is a fully managed service for data science and machine learning workflows. It helps data scientists and developers to prepare, build, train, and deploy high-quality ML models quickly by bringing together a broad set of capabilities purpose-built for ML.\n",
+ "\n",
+ "Now, NVIDIA Triton Inference Server can be used to serve models for inference in Amazon SageMaker. Thanks to the new NVIDIA Triton container image, you can easily serve ML models and benefit from the performance optimizations, dynamic batching, and multi-framework support provided by NVIDIA Triton. Triton helps maximize the utilization of GPU and CPU, further lowering the cost of inference.\n",
+ "\n",
+ "This example will showcase how to deploy a pre-trained TensorFlow model using NVIDIA Triton on SageMaker.\n",
+ "\n",
+ "The model used here was pre-trained on the MNIST dataset. See this [Deploy a Trained TensorFlow V2 Model example](https://github.com/aws/amazon-sagemaker-examples/blob/1c5da8941bc933b176b56a93157073d5645d8cdf/frameworks/tensorflow/get_started_mnist_deploy.ipynb) for the training of the model. \n",
+ "\n",
+ "## Contents\n",
+ "1. [Introduction to NVIDIA Triton Server](#Introduction-to-NVIDIA-Triton-Server)\n",
+ "1. [Set up the environment](#Set-up-the-environment)\n",
+ "1. [Transform TensorFlow Model structure](#Transform-TensorFlow-Model-structure)\n",
+ " 1. [Inspect the model using the saved_model_cli](#Inspect-the-model-using-the-saved_model_cli)\n",
+ " 1. [Create the config.pbtxt](#Create-the-config.pbtxt)\n",
+ " 1. [Create the tar ball in the required Triton structure](#Create-the-tar-ball-in-the-required-Triton-structure)\n",
+ "1. [Deploy model to SageMaker Endpoint](#Deploy-model-to-SageMaker-Endpoint)\n",
+ "1. [Clean up](#Clean-up)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "51ab88ee",
+ "metadata": {},
+ "source": [
+ "## Introduction to NVIDIA Triton Server\n",
+ "\n",
+ "[NVIDIA Triton Inference Server](https://github.com/triton-inference-server/server/) was developed specifically to enable scalable, cost-effective, and easy deployment of models in production. NVIDIA Triton Inference Server is open-source inference serving software that simplifies the inference serving process and provides high inference performance.\n",
+ "\n",
+ "Some key features of Triton are:\n",
+ "* **Support for Multiple frameworks**: Triton can be used to deploy models from all major frameworks. Triton supports TensorFlow GraphDef, TensorFlow SavedModel, ONNX, PyTorch TorchScript, TensorRT, RAPIDS FIL for tree based models, and OpenVINO model formats. \n",
+ "* **Model pipelines**: Triton model ensemble represents a pipeline of one or more models or pre/post processing logic and the connection of input and output tensors between them. A single inference request to an ensemble will trigger the execution of the entire pipeline.\n",
+ "* **Concurrent model execution**: Multiple models (or multiple instances of the same model) can run simultaneously on the same GPU or on multiple GPUs for different model management needs.\n",
+ "* **Dynamic batching**: For models that support batching, Triton has multiple built-in scheduling and batching algorithms that combine individual inference requests together to improve inference throughput. These scheduling and batching decisions are transparent to the client requesting inference.\n",
+ "* **Diverse CPUs and GPUs**: The models can be executed on CPUs or GPUs for maximum flexibility and to support heterogeneous computing requirements.\n",
+ "\n",
+ "**Note**: This initial release of NVIDIA Triton on SageMaker will only support a single model. Future releases will have multi-model support. A minimal `config.pbtxt` configuration file is **required** in the model artifacts. This release doesn't support inferring the model config automatically.\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5cf5f5fc",
+ "metadata": {},
+ "source": [
+ "## Set up the environment\n",
+ "\n",
+ "Download the pre-trained TensorFlow model from a public S3 bucket.\n",
+ "Also define the IAM role that will give SageMaker access to the model artifacts and the NVIDIA Triton ECR image.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "13469557",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%time\n",
+ "import boto3\n",
+ "\n",
+ "# use the region-specific saved model object\n",
+ "region = boto3.Session().region_name\n",
+ "saved_model = \"s3://sagemaker-sample-files/datasets/image/MNIST/model/tensorflow-training-2020-11-20-23-57-13-077/model.tar.gz\"\n",
+ "!aws s3 cp $saved_model models/SavedModel/"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "76af8c28",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sagemaker\n",
+ "\n",
+ "sm_session = sagemaker.Session()\n",
+ "role = sagemaker.get_execution_role()\n",
+ "bucket_name = sm_session.default_bucket()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31b31768",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "account_id_map = {\n",
+ " \"us-east-1\": \"785573368785\",\n",
+ " \"us-east-2\": \"007439368137\",\n",
+ " \"us-west-1\": \"710691900526\",\n",
+ " \"us-west-2\": \"301217895009\",\n",
+ " \"eu-west-1\": \"802834080501\",\n",
+ " \"eu-west-2\": \"205493899709\",\n",
+ " \"eu-west-3\": \"254080097072\",\n",
+ " \"eu-north-1\": \"601324751636\",\n",
+ " \"eu-south-1\": \"966458181534\",\n",
+ " \"eu-central-1\": \"746233611703\",\n",
+ " \"ap-east-1\": \"110948597952\",\n",
+ " \"ap-south-1\": \"763008648453\",\n",
+ " \"ap-northeast-1\": \"941853720454\",\n",
+ " \"ap-northeast-2\": \"151534178276\",\n",
+ " \"ap-southeast-1\": \"324986816169\",\n",
+ " \"ap-southeast-2\": \"355873309152\",\n",
+ " \"cn-northwest-1\": \"474822919863\",\n",
+ " \"cn-north-1\": \"472730292857\",\n",
+ " \"sa-east-1\": \"756306329178\",\n",
+ " \"ca-central-1\": \"464438896020\",\n",
+ " \"me-south-1\": \"836785723513\",\n",
+ " \"af-south-1\": \"774647643957\",\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fd92a880",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if region not in account_id_map.keys():\n",
+ " raise (\"UNSUPPORTED REGION\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0cc3ddf8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "base = \"amazonaws.com.cn\" if region.startswith(\"cn-\") else \"amazonaws.com\"\n",
+ "triton_image_uri = \"{account_id}.dkr.ecr.{region}.{base}/sagemaker-tritonserver:21.08-py3\".format(\n",
+ " account_id=account_id_map[region], region=region, base=base\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "39414be7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!tar -xf models/SavedModel/model.tar.gz -C models/SavedModel/"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a1e5faca",
+ "metadata": {},
+ "source": [
+ "## Transform TensorFlow Model structure\n",
+ "\n",
+ "\n",
+ "The model that we want to deploy currently has the following structure:\n",
+ "\n",
+ "```\n",
+ "00000000\n",
+ " ├── saved_model.pb\n",
+ " ├── assets/\n",
+ " └── variables/\n",
+ " ├── variables.data-00000-of-00001\n",
+ " └── variables.index\n",
+ "```\n",
+ "For Triton, the model needs to have the following structure:\n",
+ "```\n",
+ "\n",
+ "├── config.pbtxt\n",
+ "└── 1/\n",
+ " └── model.savedmodel\n",
+ " ├── saved_model.pb\n",
+ " ├── assets/\n",
+ " └── variables/\n",
+ " ├── variables.data-00000-of-00001\n",
+ " └── variables.index\n",
+ " \n",
+ "\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "392b33db",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "! mkdir -p models/TritonModel/MNIST/1\n",
+ "! cp models/SavedModel/00000000 --recursive ./models/TritonModel/MNIST/1/model.savedmodel/"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4c21b8be",
+ "metadata": {},
+ "source": [
+ "### Inspect the model using the `saved_model_cli`\n",
+ "\n",
+ "In order to create the `config.pbtxt` we need to confirm the model inputs and outputs (Signature).\n",
+ "We use the `saved_model_cli` to inspect the model and take note of the input and output shape."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "42b58467",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!saved_model_cli show --all --dir {\"models/SavedModel/00000000\"}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1332701d",
+ "metadata": {},
+ "source": [
+ "### Create the config.pbtxt \n",
+ "\n",
+ "Triton requires a [Model Configuration file](https://github.com/triton-inference-server/server/blob/main/docs/model_configuration.md) known as a `config.pbtxt`. We create one below in the correct directory.\n",
+ "\n",
+ "The `name` in the `config.pbtxt` must match the name of our model directory. In this case we will use `MNIST`.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0f843f41",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%writefile models/TritonModel/MNIST/config.pbtxt\n",
+ "name: \"MNIST\"\n",
+ "platform: \"tensorflow_savedmodel\"\n",
+ "max_batch_size: 0\n",
+ "\n",
+ "instance_group {\n",
+ " count: 1\n",
+ " kind: KIND_GPU\n",
+ "}\n",
+ "\n",
+ "dynamic_batching {\n",
+ "\n",
+ "}\n",
+ "\n",
+ "input [\n",
+ " {\n",
+ " name: \"input_1\"\n",
+ " data_type: TYPE_FP32\n",
+ " dims: [-1, 28, 28, 1]\n",
+ " }\n",
+ "]\n",
+ "output [\n",
+ " {\n",
+ " name: \"output_1\"\n",
+ " data_type: TYPE_FP32\n",
+ " dims: [-1, 10]\n",
+ " }\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "af89bd5e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "model_location = f\"s3://{bucket_name}/TritonModel/TritonModel.tar.gz\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "311b6185",
+ "metadata": {},
+ "source": [
+ "### Create the tar ball in the required Triton structure"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e69c75c4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%sh\n",
+ "cd models/TritonModel/ \n",
+ "tar -czvf TritonModel.tar.gz MNIST/"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "32dfbd14",
+ "metadata": {},
+ "source": [
+ "### Upload the new tar ball containing the Triton model structure to s3"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1bec7fb4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!aws s3 cp models/TritonModel/TritonModel.tar.gz $model_location"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "bedf430a",
+ "metadata": {},
+ "source": [
+ "## Deploy model to SageMaker Endpoint\n",
+ "We start off by creating a sagemaker model from the model files we uploaded to s3 in the previous step.\n",
+ "\n",
+ "In this step we also provide an additional Environment Variable i.e. `SAGEMAKER_TRITON_DEFAULT_MODEL_NAME` which specifies the name of the model to be loaded by Triton. The value of this key should match the folder name in the model package uploaded to s3. This variable is optional in case of a single model. In case of ensemble models, this key has to be specified for Triton to startup in SageMaker.\n",
+ "\n",
+ "Additionally, customers can set `SAGEMAKER_TRITON_BUFFER_MANAGER_THREAD_COUNT` and `SAGEMAKER_TRITON_THREAD_COUNT` for optimizing the thread counts."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9097c998",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from sagemaker.model import Model\n",
+ "\n",
+ "tensorflow_model = Model(\n",
+ " model_data=model_location,\n",
+ " role=role,\n",
+ " env={\"SAGEMAKER_TRITON_DEFAULT_MODEL_NAME\": \"MNIST\"},\n",
+ " image_uri=triton_image_uri,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ea99bee4",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from datetime import datetime\n",
+ "\n",
+ "date = datetime.now().strftime(\"%Y-%m-%d-%H-%m-%S\")\n",
+ "\n",
+ "endpoint_name = f\"Triton-MNIST-{date}\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "376ca9af",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "predictor = tensorflow_model.deploy(\n",
+ " initial_instance_count=1,\n",
+ " instance_type=\"ml.g4dn.xlarge\",\n",
+ " endpoint_name=endpoint_name,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a1e48701",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import json\n",
+ "\n",
+ "payload = {\n",
+ " \"inputs\": [\n",
+ " {\n",
+ " \"name\": \"input_1\",\n",
+ " \"shape\": [4, 28, 28, 1],\n",
+ " \"datatype\": \"FP32\",\n",
+ " \"data\": np.random.rand(4, 28, 28, 1).tolist(),\n",
+ " }\n",
+ " ]\n",
+ "}\n",
+ "runtime_sm_client = boto3.client(\"sagemaker-runtime\")\n",
+ "response = runtime_sm_client.invoke_endpoint(\n",
+ " EndpointName=endpoint_name,\n",
+ " ContentType=\"application/octet-stream\",\n",
+ " Body=json.dumps(payload),\n",
+ ")\n",
+ "\n",
+ "predictions = json.loads(response[\"Body\"].read())[\"outputs\"][0][\"data\"]\n",
+ "predictions = np.array(predictions, dtype=np.float32)\n",
+ "predictions = np.argmax(predictions)\n",
+ "predictions"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f0f2a7d2",
+ "metadata": {},
+ "source": [
+ "## Clean up\n",
+ "We strongly recommend to delete the Real-time endpoint created to stop incurring cost when finished with the example"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8f9e893f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "sm_client = boto3.client(\"sagemaker\")\n",
+ "# Delete endpoint\n",
+ "sm_client.delete_endpoint(EndpointName=endpoint_name)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "conda_tensorflow2_p38",
+ "language": "python",
+ "name": "conda_tensorflow2_p38"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/sagemaker_batch_transform/index.rst b/sagemaker_batch_transform/index.rst
index cd6b035981..2a75b11501 100644
--- a/sagemaker_batch_transform/index.rst
+++ b/sagemaker_batch_transform/index.rst
@@ -5,7 +5,7 @@ Get started with Batch Transform
:maxdepth: 1
introduction_to_batch_transform/batch_transform_pca_dbscan_movie_clusters
-
+ batch_transform_associate_predictions_with_input/Batch Transform - breast cancer prediction with lowel level SDK
..
file name change required
batch_transform_associate_predictions_with_input/Batch Transform - breast cancer prediction with high level SDK
@@ -61,4 +61,5 @@ TensorFlow
.. toctree::
:maxdepth: 1
- tensorflow_cifar-10_with_inference_script/tensorflow-serving-cifar10-python-sdk
+ tensorflow_cifar-10_with_inference_script/tensorflow-serving-cifar10-python-sdk
+ custom_tensorflow_inference_script_csv_and_tfrecord/custom_tensorflow_inference_script_csv_and_tfrecord
diff --git a/sagemaker_batch_transform/introduction_to_batch_transform/Dockerfile b/sagemaker_batch_transform/introduction_to_batch_transform/Dockerfile
index c72b3e416f..9509a739c4 100644
--- a/sagemaker_batch_transform/introduction_to_batch_transform/Dockerfile
+++ b/sagemaker_batch_transform/introduction_to_batch_transform/Dockerfile
@@ -6,9 +6,33 @@ RUN apt-get -y update && apt-get install -y --no-install-recommends \
wget \
r-base \
r-base-dev \
- ca-certificates
+ ca-certificates
-RUN R -e "install.packages(c('dbscan', 'plumber'), repos='https://cloud.r-project.org')"
+RUN R -e "install.packages(c('Rcpp', 'BH', 'R6', 'jsonlite', 'crayon'), repos='https://cloud.r-project.org')"
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/stringi/stringi_1.2.4.tar.gz
+RUN R CMD INSTALL stringi_1.2.4.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/rlang/rlang_0.2.2.tar.gz
+RUN R CMD INSTALL rlang_0.2.2.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/magrittr/magrittr_1.5.tar.gz
+RUN R CMD INSTALL magrittr_1.5.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/later/later_0.7.5.tar.gz
+RUN R CMD INSTALL later_0.7.5.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/promises/promises_1.0.1.tar.gz
+RUN R CMD INSTALL promises_1.0.1.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/httpuv/httpuv_1.4.4.2.tar.gz
+RUN R CMD INSTALL httpuv_1.4.4.2.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/dbscan/dbscan_1.1-2.tar.gz
+RUN R CMD INSTALL dbscan_1.1-2.tar.gz
+
+RUN wget http://cran.r-project.org/src/contrib/Archive/plumber/plumber_0.4.6.tar.gz
+RUN R CMD INSTALL plumber_0.4.6.tar.gz
COPY dbscan.R /opt/ml/dbscan.R
COPY plumber.R /opt/ml/plumber.R
diff --git a/sagemaker_batch_transform/introduction_to_batch_transform/batch_transform_pca_dbscan_movie_clusters.ipynb b/sagemaker_batch_transform/introduction_to_batch_transform/batch_transform_pca_dbscan_movie_clusters.ipynb
index dab8931db9..420935fe05 100644
--- a/sagemaker_batch_transform/introduction_to_batch_transform/batch_transform_pca_dbscan_movie_clusters.ipynb
+++ b/sagemaker_batch_transform/introduction_to_batch_transform/batch_transform_pca_dbscan_movie_clusters.ipynb
@@ -261,7 +261,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Now, we'll setup to split our dataset into train and test. Dimensionality reduction and clustering don't always require a holdout set to test accuracy, but it will allow us to illustrate how batch prediction might be used when new data arrives. In this case, our test dataset will be a simple 10% sample of items."
+ "Now, we'll setup to split our dataset into train and test. Dimensionality reduction and clustering don't always require a holdout set to test accuracy, but it will allow us to illustrate how batch prediction might be used when new data arrives. In this case, our test dataset will be a simple 0.5% sample of items."
]
},
{
@@ -270,7 +270,7 @@
"metadata": {},
"outputs": [],
"source": [
- "test_products = products.sample(frac=0.1)\n",
+ "test_products = products.sample(frac=0.005)\n",
"train_products = products[~(products.index.isin(test_products.index))]"
]
},
diff --git a/sagemaker_batch_transform/introduction_to_batch_transform/dbscan.R b/sagemaker_batch_transform/introduction_to_batch_transform/dbscan.R
index c19cf3f5fe..7dba606c3f 100644
--- a/sagemaker_batch_transform/introduction_to_batch_transform/dbscan.R
+++ b/sagemaker_batch_transform/introduction_to_batch_transform/dbscan.R
@@ -69,7 +69,7 @@ parse_file <- function(file) {
# Second helper function for apply
parse_json <- function(line) {
if (validate(line)) {
- return(do.call(rbind, fromJSON(line)[['projections']][[1]]))}}
+ return(do.call(rbind, fromJSON(line)))}}
# Setup scoring function
diff --git a/sagemaker_batch_transform/introduction_to_batch_transform/plumber.R b/sagemaker_batch_transform/introduction_to_batch_transform/plumber.R
index 1a884f083c..6857c2e474 100644
--- a/sagemaker_batch_transform/introduction_to_batch_transform/plumber.R
+++ b/sagemaker_batch_transform/introduction_to_batch_transform/plumber.R
@@ -47,4 +47,4 @@ parse_file <- function(file) {
# Second helper function for apply
parse_json <- function(line) {
if (validate(line)) {
- return(do.call(rbind, fromJSON(line)[['projections']][[1]]))}}
+ return(do.call(rbind, fromJSON(line)))}}
diff --git a/sagemaker_batch_transform/tensorflow_open-images_tfrecord/tensorflow-serving-tfrecord.cli.ipynb b/sagemaker_batch_transform/tensorflow_open-images_tfrecord/tensorflow-serving-tfrecord.cli.ipynb
index 48f7427362..678a38c58c 100644
--- a/sagemaker_batch_transform/tensorflow_open-images_tfrecord/tensorflow-serving-tfrecord.cli.ipynb
+++ b/sagemaker_batch_transform/tensorflow_open-images_tfrecord/tensorflow-serving-tfrecord.cli.ipynb
@@ -270,7 +270,7 @@
"SPLIT_TYPE=\"TFRecord\"\n",
"BATCH_STRATEGY=\"SingleRecord\"\n",
"\n",
- "# Join outputs by newline characters. This will make JSONLines output, since each output is JSON.\n",
+ "# Join outputs by newline characters. This will make JSON Lines output, since each output is JSON.\n",
"ASSEMBLE_WITH=\"Line\"\n",
"\n",
"# The Data Source tells Batch to get all objects under the S3 prefix.\n",
diff --git a/sagemaker_model_monitor/index.rst b/sagemaker_model_monitor/index.rst
index 5f5efce4ec..740ae6db8b 100644
--- a/sagemaker_model_monitor/index.rst
+++ b/sagemaker_model_monitor/index.rst
@@ -6,7 +6,14 @@ Get started with Model Monitor
introduction/SageMaker-ModelMonitoring
+Model Quality
+==============================
+
+.. toctree::
+ :maxdepth: 1
+ model_quality/model_quality_churn_sdk
+
Model Monitor with Tensorflow
=============================
diff --git a/sagemaker_neo_compilation_jobs/index.rst b/sagemaker_neo_compilation_jobs/index.rst
index e91e483e3d..e22f7f8f53 100644
--- a/sagemaker_neo_compilation_jobs/index.rst
+++ b/sagemaker_neo_compilation_jobs/index.rst
@@ -16,7 +16,7 @@ Apache MXNet
gluoncv_yolo_darknet/gluoncv_yolo_darknet_neo
mxnet_mnist/mxnet_mnist_neo
-
+ gluoncv_ssd_mobilenet/gluoncv_ssd_mobilenet_neo
PyTorch
@@ -28,6 +28,15 @@ PyTorch
pytorch_torchvision/pytorch_torchvision_neo
pytorch_vgg19_bn/pytorch-vgg19-bn
+Inf1 Instance
+=======
+
+.. toctree::
+ :maxdepth: 1
+
+ deploy_pytorch_model_on_Inf1_instance/pytorch_torchvision_neo_on_Inf1
+ deploy_huggingface_model_on_Inf1_instance/inf1_bert_compile_and_deploy
+ deploy_mxnet_model_on_Inf1_instance/mxnet_distributed_mnist_neo_inf1
TensorFlow
==========
@@ -36,3 +45,5 @@ TensorFlow
:maxdepth: 1
tensorflow_distributed_mnist/tensorflow_distributed_mnist_neo
+ tensorflow_unet/sagemaker-neo-tf-unet
+
diff --git a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability.ipynb b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability.ipynb
index 7831728e60..99fa1fe0bc 100644
--- a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability.ipynb
+++ b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability.ipynb
@@ -50,7 +50,7 @@
"1. Explaining the importance of the various input features on the model's decision\n",
"1. Accessing the reports through SageMaker Studio if you have an instance set up.\n",
"\n",
- "In doing so, the notebook first trains a [SageMaker XGBoost](https://docs.aws.amazon.com/sagemaker/latest/dg/xgboost.html) model using training dataset, then use SageMaker Clarify to analyze a testing dataset in CSV format. SageMaker Clarify also supports analyzing dataset in [SageMaker JSONLines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats), which is illustrated in [another notebook](https://github.com/aws/amazon-sagemaker-examples/blob/master/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb)."
+ "In doing so, the notebook first trains a [SageMaker XGBoost](https://docs.aws.amazon.com/sagemaker/latest/dg/xgboost.html) model using training dataset, then use SageMaker Clarify to analyze a testing dataset in CSV format. SageMaker Clarify also supports analyzing dataset in [SageMaker JSON Lines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats), which is illustrated in [another notebook](https://github.com/aws/amazon-sagemaker-examples/blob/master/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb)."
]
},
{
diff --git a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_byoc.ipynb b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_byoc.ipynb
index cfa55ef88d..df31b3060e 100644
--- a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_byoc.ipynb
+++ b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_byoc.ipynb
@@ -422,9 +422,9 @@
"* At container startup, the script initializes an estimator using the model file provided by the client side deploy() method. The model directory and model file name are the same as in the `train` script.\n",
"* Once started, the server is ready to serve inference requests. The logic resides in the `predict` method,\n",
" * Input validation. The example container supports the same MIME types as Clarify job does, i.e., `text/csv` and `application/jsonlines`.\n",
- " * Parse payload. Clarify job may send **batch requests** to the container for better efficiency, i.e., the payload can have multiple lines and each is a sample. So, the method decodes request payload and then split lines, then loads the lines according to the content type. For JSONLines content, the method uses a key \"features\" to extract the list of features from a JSON line. The key shall be the same as the one defined in your Clarify job analysis configuration `predictor.content_template`. It is a **contract** between the Clarify job and the container, here you can change it to something else, like \"attributes\", but remember to update the `predictor.content_template` configuration accordingly.\n",
+ " * Parse payload. Clarify job may send **batch requests** to the container for better efficiency, i.e., the payload can have multiple lines and each is a sample. So, the method decodes request payload and then split lines, then loads the lines according to the content type. For JSON Lines content, the method uses a key \"features\" to extract the list of features from a JSON line. The key shall be the same as the one defined in your Clarify job analysis configuration `predictor.content_template`. It is a **contract** between the Clarify job and the container, here you can change it to something else, like \"attributes\", but remember to update the `predictor.content_template` configuration accordingly.\n",
" * Do prediction. The method gets the probability scores instead of binary labels, because scores are better for feature explainability.\n",
- " * Format output. For a **batch request**, Clarify job expects the same number of result lines as the number of samples in the request. So, the method encodes each prediction and then join them by line-break. For JSONLines accept type, the method uses two keys \"predicted_label\" and \"score\" to indicate the prediction. The keys shall be the same as your Clarify job analysis configuration `predictor.label` and `predictor.probability`, and they are used by the Clarify job to extract predictions from container response payload. The keys are **contracts** between the Clarify job and the container, here you can change them to something else, but remember to update the analysis configuration accordingly.\n",
+ " * Format output. For a **batch request**, Clarify job expects the same number of result lines as the number of samples in the request. So, the method encodes each prediction and then join them by line-break. For JSON Lines accept type, the method uses two keys \"predicted_label\" and \"score\" to indicate the prediction. The keys shall be the same as your Clarify job analysis configuration `predictor.label` and `predictor.probability`, and they are used by the Clarify job to extract predictions from container response payload. The keys are **contracts** between the Clarify job and the container, here you can change them to something else, but remember to update the analysis configuration accordingly.\n",
"\n",
"Similarly, the script is built from scratch for demonstration purpose. In a real project, you can utilize [SageMaker Inference Toolkit](https://github.com/aws/sagemaker-inference-toolkit) which implements a model serving stack built on [Multi Model Server](https://github.com/awslabs/multi-model-server), and it can serve your own models or those you trained on SageMaker using Machine Learning frameworks with native SageMaker support."
]
@@ -473,7 +473,7 @@
"python serve --model_dir \n",
"```\n",
"\n",
- "Upon successful execution, the script should be listening on local host port `8080` for inference requests. The following cell generates a few CURL commands to send inference requests (both CSV and JSONLines) to the port. You can copy&paste them to your local terminal for execution, to hit the port and trigger the inference code. For a single sample request, the command should output only one result, and for a batch request, the command should output the same number of results (lines) as the number of samples in the request."
+ "Upon successful execution, the script should be listening on local host port `8080` for inference requests. The following cell generates a few CURL commands to send inference requests (both CSV and JSON Lines) to the port. You can copy&paste them to your local terminal for execution, to hit the port and trigger the inference code. For a single sample request, the command should output only one result, and for a batch request, the command should output the same number of results (lines) as the number of samples in the request."
]
},
{
@@ -923,13 +923,13 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "There are three scenarios where Clarify handles data types, and they all support both CSV (`text/csv`) and JSONLines (`application/jsonlines`).\n",
+ "There are three scenarios where Clarify handles data types, and they all support both CSV (`text/csv`) and JSON Lines (`application/jsonlines`).\n",
"\n",
"* dataset type: the MIME type of the dataset and SHAP baseline.\n",
"* content type: the MIME type of the shadow endpoint request payload\n",
"* accept type: the MIME type of the shadow endpoint response payload\n",
"\n",
- "The Clarify jobs in this notebook always uses CSV for dataset type, but you can choose for the other two. The following code chose JSONLines for both, but it is fine if you change one of them or both of them to CSV, because CSV and JSONLines are supported by the customer container as well."
+ "The Clarify jobs in this notebook always uses CSV for dataset type, but you can choose for the other two. The following code chose JSON Lines for both, but it is fine if you change one of them or both of them to CSV, because CSV and JSON Lines are supported by the customer container as well."
]
},
{
@@ -991,7 +991,7 @@
"A [ModelConfig](https://sagemaker.readthedocs.io/en/stable/api/training/processing.html#sagemaker.clarify.ModelConfig) object communicates information about your trained model. To avoid additional traffic to your production models, SageMaker Clarify sets up and tears down a dedicated endpoint when processing.\n",
"* `instance_type` and `instance_count` specify your preferred instance type and instance count used to run your model on during SageMaker Clarify's processing. The testing dataset is small so a single standard instance is good enough to run this example. If you have a large complex dataset, you may want to use a better instance type to speed up, or add more instances to enable Spark parallelization.\n",
"* `accept_type` denotes the endpoint response payload format, and `content_type` denotes the payload format of request to the endpoint.\n",
- "* `content_template` is used by SageMaker Clarify to compose the request payload if the content type is JSONLines. To be more specific, the placeholder `$features` will be replaced by the features list from samples. For example, the first sample of the test dataset is `25,2,226802,1,7,4,6,3,2,1,0,0,40,37`, so the corresponding request payload is `'{\"features\":[25,2,226802,1,7,4,6,3,2,1,0,0,40,37]}'`, which conforms to [SageMaker JSONLines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats)."
+ "* `content_template` is used by SageMaker Clarify to compose the request payload if the content type is JSON Lines. To be more specific, the placeholder `$features` will be replaced by the features list from samples. For example, the first sample of the test dataset is `25,2,226802,1,7,4,6,3,2,1,0,0,40,37`, so the corresponding request payload is `'{\"features\":[25,2,226802,1,7,4,6,3,2,1,0,0,40,37]}'`, which conforms to [SageMaker JSON Lines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats)."
]
},
{
@@ -1017,7 +1017,7 @@
"#### Writing ModelPredictedLabelConfig\n",
"\n",
"A [ModelPredictedLabelConfig](https://sagemaker.readthedocs.io/en/stable/api/training/processing.html#sagemaker.clarify.ModelPredictedLabelConfig) provides information on the format of your predictions.\n",
- "* `probability` is used by SageMaker Clarify to locate the probability score in endpoint response if the accept type is JSONLines. In this case, the response payload for a single sample request looks like `'{\"predicted_label\": 0, \"score\": 0.026494730307781475}'`, so SageMaker Clarify can find the score `0.026494730307781475` by JSONPath `'score'`.\n",
+ "* `probability` is used by SageMaker Clarify to locate the probability score in endpoint response if the accept type is JSON Lines. In this case, the response payload for a single sample request looks like `'{\"predicted_label\": 0, \"score\": 0.026494730307781475}'`, so SageMaker Clarify can find the score `0.026494730307781475` by JSONPath `'score'`.\n",
"* `probability_threshold` is used by SageMaker Clarify to convert the probability to binary labels for bias analysis. Prediction above the threshold is interpreted as label value 1 and below or equal as label value 0."
]
},
diff --git a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb
index 4a7412b295..90b08ec30d 100644
--- a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb
+++ b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb
@@ -4,7 +4,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# Fairness and Explainability with SageMaker Clarify - JSONLines Format"
+ "# Fairness and Explainability with SageMaker Clarify - JSON Lines Format"
]
},
{
@@ -44,7 +44,7 @@
"1. Explaining the importance of the various input features on the model's decision\n",
"1. Accessing the reports through SageMaker Studio if you have an instance set up.\n",
"\n",
- "In doing so, the notebook will first train a [SageMaker Linear Learner](https://docs.aws.amazon.com/sagemaker/latest/dg/linear-learner.html) model using training dataset, then use SageMaker Clarify to analyze a testing dataset in [SageMaker JSONLines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats). SageMaker Clarify also supports analyzing CSV dataset, which is illustrated in [another notebook](https://github.com/aws/amazon-sagemaker-examples/blob/master/sagemaker_processing/fairness_and_explainability/fairness_and_explainability.ipynb)."
+ "In doing so, the notebook will first train a [SageMaker Linear Learner](https://docs.aws.amazon.com/sagemaker/latest/dg/linear-learner.html) model using training dataset, then use SageMaker Clarify to analyze a testing dataset in [SageMaker JSON Lines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats). SageMaker Clarify also supports analyzing CSV dataset, which is illustrated in [another notebook](https://github.com/aws/amazon-sagemaker-examples/blob/master/sagemaker_processing/fairness_and_explainability/fairness_and_explainability.ipynb)."
]
},
{
@@ -247,7 +247,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Then save the testing dataset to a JSONLines file. The file conforms to [SageMaker JSONLines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats), with an additional field to hold the ground truth label."
+ "Then save the testing dataset to a JSON Lines file. The file conforms to [SageMaker JSON Lines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats), with an additional field to hold the ground truth label."
]
},
{
@@ -392,14 +392,14 @@
"#### Writing DataConfig and ModelConfig\n",
"A `DataConfig` object communicates some basic information about data I/O to SageMaker Clarify. We specify where to find the input dataset, where to store the output, the target column (`label`), the header names, and the dataset type.\n",
"\n",
- "Some special things to note about this configuration for the JSONLines dataset,\n",
+ "Some special things to note about this configuration for the JSON Lines dataset,\n",
"* Argument `features` or `label` is **NOT** header string. Instead, it is a [JSONPath string](https://jmespath.org/specification.html) to locate the features list or label in the dataset. For example, for a sample like below, `features` should be 'data.features.values', and `label` should be 'data.label'. \n",
"\n",
"```\n",
"{\"data\": {\"features\": {\"values\": [25, 2, 226802, 1, 7, 4, 6, 3, 2, 1, 0, 0, 40, 37]}, \"label\": 0}}\n",
"```\n",
"\n",
- "* SageMaker Clarify will load the JSONLines dataset into tabular representation for further analysis, and argument `headers` is the list of column names. The label header shall be the last one in the headers list, and the order of feature headers shall be the same as the order of features in a sample."
+ "* SageMaker Clarify will load the JSON Lines dataset into tabular representation for further analysis, and argument `headers` is the list of column names. The label header shall be the last one in the headers list, and the order of feature headers shall be the same as the order of features in a sample."
]
},
{
@@ -426,7 +426,7 @@
"A `ModelConfig` object communicates information about your trained model. To avoid additional traffic to your production models, SageMaker Clarify sets up and tears down a dedicated endpoint when processing.\n",
"* `instance_type` and `instance_count` specify your preferred instance type and instance count used to run your model on during SageMaker Clarify's processing. The testing dataset is small so a single standard instance is good enough to run this example. If your have a large complex dataset, you may want to use a better instance type to speed up, or add more instances to enable Spark parallelization.\n",
"* `accept_type` denotes the endpoint response payload format, and `content_type` denotes the payload format of request to the endpoint.\n",
- "* `content_template` is used by SageMaker Clarify to compose the request payload if the content type is JSONLines. To be more specific, the placeholder `$features` will be replaced by the features list from samples. The request payload of a sample from the testing dataset happens to be similar to the sample itself, like `'{\"features\": [25, 2, 226802, 1, 7, 4, 6, 3, 2, 1, 0, 0, 40, 37]}'`, because both the dataset and the model input conform to [SageMaker JSONLines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats)."
+ "* `content_template` is used by SageMaker Clarify to compose the request payload if the content type is JSON Lines. To be more specific, the placeholder `$features` will be replaced by the features list from samples. The request payload of a sample from the testing dataset happens to be similar to the sample itself, like `'{\"features\": [25, 2, 226802, 1, 7, 4, 6, 3, 2, 1, 0, 0, 40, 37]}'`, because both the dataset and the model input conform to [SageMaker JSON Lines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats)."
]
},
{
@@ -465,7 +465,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "If you are building your own model, then you may choose a different JSONLines format, as long as it has the key elements like label and features list, and request payload built using `content_template` is supported by the model (you can customize the template but the placeholder of features list must be `$features`). Also, `dataset_type`, `accept_type` and `content_type` don't have to be the same, for example, a use case may use CSV dataset and content type, but JSONLines accept type."
+ "If you are building your own model, then you may choose a different JSON Lines format, as long as it has the key elements like label and features list, and request payload built using `content_template` is supported by the model (you can customize the template but the placeholder of features list must be `$features`). Also, `dataset_type`, `accept_type` and `content_type` don't have to be the same, for example, a use case may use CSV dataset and content type, but JSON Lines accept type."
]
},
{
diff --git a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_outputs.ipynb b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_outputs.ipynb
index 289d3b8e5b..ca4cd45863 100644
--- a/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_outputs.ipynb
+++ b/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_outputs.ipynb
@@ -70,7 +70,7 @@
"1. Explaining the importance of the various input features on the model's decision\n",
"1. Accessing the reports through SageMaker Studio if you have an instance set up.\n",
"\n",
- "In doing so, the notebook first trains a [SageMaker XGBoost](https://docs.aws.amazon.com/sagemaker/latest/dg/xgboost.html) model using training dataset, then use SageMaker Clarify to analyze a testing dataset in CSV format. SageMaker Clarify also supports analyzing dataset in [SageMaker JSONLines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats), which is illustrated in [another notebook](https://github.com/aws/amazon-sagemaker-examples/blob/master/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb)."
+ "In doing so, the notebook first trains a [SageMaker XGBoost](https://docs.aws.amazon.com/sagemaker/latest/dg/xgboost.html) model using training dataset, then use SageMaker Clarify to analyze a testing dataset in CSV format. SageMaker Clarify also supports analyzing dataset in [SageMaker JSON Lines dense format](https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#common-in-formats), which is illustrated in [another notebook](https://github.com/aws/amazon-sagemaker-examples/blob/master/sagemaker_processing/fairness_and_explainability/fairness_and_explainability_jsonlines_format.ipynb)."
]
},
{
diff --git a/sagemaker_processing/index.rst b/sagemaker_processing/index.rst
index 5ca65b46f4..4df9a18f64 100644
--- a/sagemaker_processing/index.rst
+++ b/sagemaker_processing/index.rst
@@ -16,4 +16,8 @@ Processing
fairness_and_explainability/text_explainability/text_explainability
fairness_and_explainability/text_explainability_sagemaker_algorithm/byo_blazingtext_model_hosting
computer_vision/explainability_image_classification
- local_pyspark/local_pyspark_example
\ No newline at end of file
+ local_pyspark/local_pyspark_example
+ pytorch_bert/deploy_bert
+ sklearn/sklearn_byom
+ sagemaker_processing/basic_sagemaker_data_processing/basic_sagemaker_processing
+
diff --git a/serverless-inference/index.rst b/serverless-inference/index.rst
new file mode 100644
index 0000000000..1dff26be0b
--- /dev/null
+++ b/serverless-inference/index.rst
@@ -0,0 +1,9 @@
+Serverless Inference
+--------------------------
+
+.. toctree::
+ :maxdepth: 1
+
+ Serverless-Inference-Walkthrough
+ serverless-model-registry
+ huggingface-serverless-inference/huggingface-text-classification-serverless-inference
diff --git a/training/frameworks.rst b/training/frameworks.rst
index e85645da92..7fde6b7099 100644
--- a/training/frameworks.rst
+++ b/training/frameworks.rst
@@ -17,7 +17,9 @@ Apache MXNet
../sagemaker-python-sdk/mxnet_onnx_export/mxnet_onnx_export
../sagemaker-python-sdk/mxnet_mnist/mxnet_mnist_with_batch_transform
../sagemaker-python-sdk/mxnet_mnist/mxnet_mnist_elastic_inference
-
+ ../sagemaker-python-sdk/mxnet_mnist/mxnet_mnist
+ ../sagemaker-python-sdk/mxnet_gluon_mnist/mxnet_mnist_with_gluon
+ ../sagemaker-python-sdk/dgl_gcmc/mxnet_gcmc
Deep Graph Library
====================================
@@ -47,7 +49,9 @@ PyTorch
../frameworks/pytorch/get_started_mnist_train
../frameworks/pytorch/get_started_mnist_deploy
../sagemaker-python-sdk/pytorch_lstm_word_language_model/pytorch_rnn
-
+ ../sagemaker-python-sdk/pytorch_horovod_mnist/pytorch_mnist_horovod
+ ../sagemaker-python-sdk/pytorch_mnist/pytorch_mnist_elastic_inference
+ ../sagemaker-python-sdk/pytorch_mnist/pytorch_mnist
R
====
@@ -66,7 +70,9 @@ Scikit-learn
../sagemaker-python-sdk/scikit_learn_randomforest/Sklearn_on_SageMaker_end2end
../sagemaker-python-sdk/scikit_learn_model_registry_batch_transform/scikit_learn_model_registry_batch_transform
-
+ ../sagemaker-python-sdk/scikit_learn_inference_pipeline/Inference Pipeline with Scikit-learn and Linear Learner
+ ../sagemaker-python-sdk/scikit_learn_iris/scikit_learn_estimator_example_with_batch_transform
+ ../aws_sagemaker_studio/sagemaker-python-sdk/scikit_learn_iris/Scikit-learn Estimator Example With Batch Transform
..
needs to be renamed (remove spaces)
../sagemaker-python-sdk/scikit_learn_iris/
@@ -89,6 +95,7 @@ TensorFlow
../sagemaker-python-sdk/tensorflow-eager-script-mode/tf-eager-sm-scriptmode
../sagemaker-python-sdk/tensorflow_script_mode_training_and_serving/tensorflow_script_mode_training_and_serving
../sagemaker-python-sdk/tensorboard_keras/tensorboard_keras
+ ../sagemaker-python-sdk/tensorflow_serving_using_elastic_inference_with_your_own_model/tensorflow_neo_compiled_model_elastic_inference
JAX
diff --git a/training/heterogeneous-clusters/.gitignore b/training/heterogeneous-clusters/.gitignore
new file mode 100644
index 0000000000..f6a49187cf
--- /dev/null
+++ b/training/heterogeneous-clusters/.gitignore
@@ -0,0 +1,11 @@
+.venv/
+.DS_Store
+data/MyMNIST
+pt.grpc.local/data/*
+pt.grpc.local/__pycache__
+pt.grpc.local/profile
+tf.data.service.sagemaker/data
+tf.data.service.sagemaker/code/__pycache__
+tf.data.service.local/data
+pt.grpc.sagemaker/data
+tf.data.service.sagemaker/__pycache__
diff --git a/training/heterogeneous-clusters/README.md b/training/heterogeneous-clusters/README.md
new file mode 100644
index 0000000000..bc3dee1a4b
--- /dev/null
+++ b/training/heterogeneous-clusters/README.md
@@ -0,0 +1,30 @@
+# SageMaker Heterogeneous Clusters for Model Training
+In July 2022, we [launched](https://aws.amazon.com/about-aws/whats-new/2022/07/announcing-heterogeneous-clusters-amazon-sagemaker-model-training/) heterogeneous clusters for Amazon SageMaker
+model training, which enables you to launch training jobs that use different instance types and
+families in a single job. A primary use case is offloading data preprocessing to
+compute-optimized instance types, whereas the deep neural network (DNN) process continues to
+run on GPU or ML accelerated instance types.
+
+In this repository, you'll find TensorFlow (tf.data.service) and PyTorch (a custom gRPC based distributed data loading) examples which demonstrates how to use heterogeneous clusters in your SageMaker training jobs. You can use these examples with minimal code changes in your existing training scripts.
+
+
+
+## Examples:
+
+### Hello world example
+- [**Heterogeneous Clusters - a hello world example**](hello.world.sagemaker/helloworld-example.ipynb):
+This basic example runs a heterogeneous training job consisting of two instance groups. Each group includes a different instance_type.
+Each instance prints its instance group information and exits.
+Note: This example only shows how to orchestrate the training job with instance type. For actual code to help with a distributed data loader, see the TensorFlow or PyTorch examples below.
+
+### TensorFlow examples
+- [**TensorFlow's tf.data.service based Amazon SageMaker Heterogeneous Clusters**](tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb):
+This TensorFlow example runs both Homogeneous and Heterogeneous clusters SageMaker training job, and compares their results. The heterogeneous cluster training job runs with two instance groups:
+ - `data_group` - this group has two ml.c5.18xlarge instances to which data preprocessing/augmentation is offloaded.
+ - `dnn_group` - this group has one ml.p4d.24xlarge instance (8GPUs) in a horovod/MPI distribution.
+
+### PyTorch examples
+- [**PyTorch and gRPC distributed dataloader based Amazon SageMaker Heterogeneous Clusters**](pt.grpc.sagemaker/hetero-pytorch-mnist.ipynb):
+This PyTorch example enables you to run both Homogeneous and Heterogeneous clusters SageMaker training job. We then compare their results, and understand price performance benefits.
+ - `data_group` - this group has one ml.c5.9xlarge instance for offloading data preprocessing job.
+ - `dnn_group` - this group has one ml.p3.2xlarge instance
diff --git a/training/heterogeneous-clusters/hello.world.sagemaker/helloworld-example.ipynb b/training/heterogeneous-clusters/hello.world.sagemaker/helloworld-example.ipynb
new file mode 100644
index 0000000000..e990cf22ce
--- /dev/null
+++ b/training/heterogeneous-clusters/hello.world.sagemaker/helloworld-example.ipynb
@@ -0,0 +1,467 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Heterogeneous Cluster - a hello world training job\n",
+ "\n",
+ "This basic example on how to run a Heterogeneous Clusters training job consisting of two instance groups. Each instance group includes a different instance type. Each instance prints its environment information including its instance group and exits.\n",
+ "\n",
+ "You can retrieve environment information in either of the following ways:\n",
+ " - **Option 1**: Read instance group information using the convenient `sagemaker_training.environment.Environment` class.\n",
+ " - **Option 2**: Read instance group information from `/opt/ml/input/config/resourceconfig.json`.\n",
+ " \n",
+ " \n",
+ "Note: This notebook does not demonstrate offloading of data preprocessing job to data group and deep neural network training to dnn_group. We will cover those examples in [TensorFlow's tf.data.service based Amazon SageMaker Heterogeneous Clusters for training](../tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb) and [PyTorch and gRPC distributed dataloader based Amazon SageMaker Heterogeneous Clusters for training](../pt.grpc.sagemaker/hetero-pytorch-mnist.ipynb) notebooks."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### A. Setting up SageMaker Studio notebook\n",
+ "#### Before you start\n",
+ "Ensure you have selected Python 3 (_TensorFlow 2.6 Python 3.8 CPU Optimized_) image for your SageMaker Studio Notebook instance, and running on _ml.t3.medium_ instance type.\n",
+ "\n",
+ "#### Step 1 - Upgrade SageMaker SDK and dependent packages\n",
+ "Heterogeneous Clusters for Amazon SageMaker model training was [announced](https://aws.amazon.com/about-aws/whats-new/2022/07/announcing-heterogeneous-clusters-amazon-sagemaker-model-training) on 07/08/2022. This feature release requires you to have updated SageMaker SDK and boto3 client libraries."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: boto3 in /usr/local/lib/python3.8/site-packages (1.24.72)\n",
+ "Collecting boto3\n",
+ " Downloading boto3-1.24.83-py3-none-any.whl (132 kB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 132.5/132.5 kB 2.5 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: botocore in /usr/local/lib/python3.8/site-packages (1.27.72)\n",
+ "Collecting botocore\n",
+ " Downloading botocore-1.27.83-py3-none-any.whl (9.2 MB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 9.2/9.2 MB 42.5 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: awscli in /usr/local/lib/python3.8/site-packages (1.25.73)\n",
+ "Collecting awscli\n",
+ " Downloading awscli-1.25.84-py3-none-any.whl (3.9 MB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.9/3.9 MB 35.4 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: sagemaker in /usr/local/lib/python3.8/site-packages (2.109.0)\n",
+ "Collecting sagemaker\n",
+ " Downloading sagemaker-2.110.0.tar.gz (576 kB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 576.0/576.0 kB 9.9 MB/s eta 0:00:00\n",
+ " Preparing metadata (setup.py): started\n",
+ " Preparing metadata (setup.py): finished with status 'done'\n",
+ "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/site-packages (from boto3) (0.6.0)\n",
+ "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.8/site-packages (from boto3) (0.10.0)\n",
+ "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/local/lib/python3.8/site-packages (from botocore) (1.25.11)\n",
+ "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.8/site-packages (from botocore) (2.8.2)\n",
+ "Requirement already satisfied: colorama<0.4.5,>=0.2.5 in /usr/local/lib/python3.8/site-packages (from awscli) (0.4.3)\n",
+ "Requirement already satisfied: PyYAML<5.5,>=3.10 in /usr/local/lib/python3.8/site-packages (from awscli) (5.4.1)\n",
+ "Requirement already satisfied: docutils<0.17,>=0.10 in /usr/local/lib/python3.8/site-packages (from awscli) (0.15.2)\n",
+ "Requirement already satisfied: rsa<4.8,>=3.1.2 in /usr/local/lib/python3.8/site-packages (from awscli) (4.7.2)\n",
+ "Requirement already satisfied: attrs<22,>=20.3.0 in /usr/local/lib/python3.8/site-packages (from sagemaker) (21.2.0)\n",
+ "Requirement already satisfied: google-pasta in /usr/local/lib/python3.8/site-packages (from sagemaker) (0.2.0)\n",
+ "Requirement already satisfied: numpy<2.0,>=1.9.0 in /usr/local/lib/python3.8/site-packages (from sagemaker) (1.19.5)\n",
+ "Requirement already satisfied: protobuf<4.0,>=3.1 in /usr/local/lib/python3.8/site-packages (from sagemaker) (3.19.1)\n",
+ "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /usr/local/lib/python3.8/site-packages (from sagemaker) (0.1.5)\n",
+ "Requirement already satisfied: smdebug_rulesconfig==1.0.1 in /usr/local/lib/python3.8/site-packages (from sagemaker) (1.0.1)\n",
+ "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /usr/local/lib/python3.8/site-packages (from sagemaker) (4.8.2)\n",
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/site-packages (from sagemaker) (21.3)\n",
+ "Requirement already satisfied: pandas in /usr/local/lib/python3.8/site-packages (from sagemaker) (1.2.5)\n",
+ "Requirement already satisfied: pathos in /usr/local/lib/python3.8/site-packages (from sagemaker) (0.2.8)\n",
+ "Collecting schema\n",
+ " Downloading schema-0.7.5-py2.py3-none-any.whl (17 kB)\n",
+ "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.8/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.6.0)\n",
+ "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/site-packages (from packaging>=20.0->sagemaker) (3.0.6)\n",
+ "Requirement already satisfied: six in /usr/local/lib/python3.8/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.16.0)\n",
+ "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.8/site-packages (from rsa<4.8,>=3.1.2->awscli) (0.4.8)\n",
+ "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/site-packages (from pandas->sagemaker) (2021.3)\n",
+ "Requirement already satisfied: dill>=0.3.4 in /usr/local/lib/python3.8/site-packages (from pathos->sagemaker) (0.3.4)\n",
+ "Requirement already satisfied: ppft>=1.6.6.4 in /usr/local/lib/python3.8/site-packages (from pathos->sagemaker) (1.6.6.4)\n",
+ "Requirement already satisfied: pox>=0.3.0 in /usr/local/lib/python3.8/site-packages (from pathos->sagemaker) (0.3.0)\n",
+ "Requirement already satisfied: multiprocess>=0.70.12 in /usr/local/lib/python3.8/site-packages (from pathos->sagemaker) (0.70.12.2)\n",
+ "Collecting contextlib2>=0.5.5\n",
+ " Downloading contextlib2-21.6.0-py2.py3-none-any.whl (13 kB)\n",
+ "Building wheels for collected packages: sagemaker\n",
+ " Building wheel for sagemaker (setup.py): started\n",
+ " Building wheel for sagemaker (setup.py): finished with status 'done'\n",
+ " Created wheel for sagemaker: filename=sagemaker-2.110.0-py2.py3-none-any.whl size=791666 sha256=5e4f859fef28f399b5eb60568410a22ddb2c42bbc357d0b3eae61587a14ca679\n",
+ " Stored in directory: /root/.cache/pip/wheels/ad/56/4f/4c5b1ed9fb3a725a634741aa293beb6fad882af965e2ccb6ae\n",
+ "Successfully built sagemaker\n",
+ "Installing collected packages: contextlib2, schema, botocore, boto3, awscli, sagemaker\n",
+ " Attempting uninstall: botocore\n",
+ " Found existing installation: botocore 1.27.72\n",
+ " Uninstalling botocore-1.27.72:\n",
+ " Successfully uninstalled botocore-1.27.72\n",
+ " Attempting uninstall: boto3\n",
+ " Found existing installation: boto3 1.24.72\n",
+ " Uninstalling boto3-1.24.72:\n",
+ " Successfully uninstalled boto3-1.24.72\n",
+ " Attempting uninstall: awscli\n",
+ " Found existing installation: awscli 1.25.73\n",
+ " Uninstalling awscli-1.25.73:\n",
+ " Successfully uninstalled awscli-1.25.73\n",
+ " Attempting uninstall: sagemaker\n",
+ " Found existing installation: sagemaker 2.109.0\n",
+ " Uninstalling sagemaker-2.109.0:\n",
+ " Successfully uninstalled sagemaker-2.109.0\n",
+ "Successfully installed awscli-1.25.84 boto3-1.24.83 botocore-1.27.83 contextlib2-21.6.0 sagemaker-2.110.0 schema-0.7.5\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%bash\n",
+ "python3 -m pip install --upgrade boto3 botocore awscli sagemaker"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Restart the notebook kernel "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#import IPython\n",
+ "#IPython.Application.instance().kernel.do_shutdown(True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 3 - Validate SageMaker Python SDK and TensorFlow versions\n",
+ "Ensure the output of the cell below reflects:\n",
+ "\n",
+ "- SageMaker Python SDK version 2.98.0 or above, \n",
+ "- boto3 1.24 or above \n",
+ "- botocore 1.27 or above \n",
+ "- TensorFlow 2.6 or above "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Name: sagemaker\n",
+ "Version: 2.110.0\n",
+ "---\n",
+ "Name: boto3\n",
+ "Version: 1.24.83\n",
+ "---\n",
+ "Name: botocore\n",
+ "Version: 1.27.83\n",
+ "---\n",
+ "Name: tensorflow\n",
+ "Version: 2.6.2\n",
+ "---\n",
+ "Name: protobuf\n",
+ "Version: 3.19.1\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip show sagemaker boto3 botocore tensorflow protobuf |egrep 'Name|Version|---'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### B. Run a heterogeneous cluster training job\n",
+ "\n",
+ "#### Step 1: Set up training environment\n",
+ "Import the required libraries that enable you to use Heterogeneous clusters for training. In this step, you are also inheriting this notebook's IAM role and SageMaker session. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import json\n",
+ "import datetime\n",
+ "\n",
+ "import sagemaker\n",
+ "from sagemaker import get_execution_role\n",
+ "from sagemaker.tensorflow import TensorFlow\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "\n",
+ "sess = sagemaker.Session()\n",
+ "role = get_execution_role()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 2: Define instance groups \n",
+ "Here we define instance groups. Each instance group includes a different instance type."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "data_group = InstanceGroup(\"data_group\", \"ml.c5.xlarge\", 1)\n",
+ "dnn_group = InstanceGroup(\"dnn_group\", \"ml.m4.xlarge\", 1) "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 3: Review the \"hello world\" training code"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mjson\u001b[39;49;00m\n",
+ "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36mos\u001b[39;49;00m\n",
+ "\u001b[34mimport\u001b[39;49;00m \u001b[04m\u001b[36msys\u001b[39;49;00m\n",
+ "\u001b[34mfrom\u001b[39;49;00m \u001b[04m\u001b[36msagemaker_training\u001b[39;49;00m \u001b[34mimport\u001b[39;49;00m environment \u001b[37m# This module is present on the DLC images, or you can install it with pip install sagemaker_training\u001b[39;49;00m\n",
+ "\n",
+ "\u001b[34mif\u001b[39;49;00m \u001b[31m__name__\u001b[39;49;00m == \u001b[33m\"\u001b[39;49;00m\u001b[33m__main__\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m:\n",
+ " \n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mOption-1: Read instance group information from the sagemaker_training.environment.Environment class\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " env = environment.Environment() \n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.is_hetero: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.is_hetero\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.current_host: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.current_host\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.current_instance_type: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.current_instance_type\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.current_instance_group: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.current_instance_group\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.current_instance_group_hosts: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.current_instance_group_hosts\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.instance_groups: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.instance_groups\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.instance_groups_dict: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.instance_groups_dict\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.distribution_hosts: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.distribution_hosts\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.distribution_instance_groups: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00menv.distribution_instance_groups\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \n",
+ "\n",
+ " file_path = \u001b[33m'\u001b[39;49;00m\u001b[33m/opt/ml/input/config/resourceconfig.json\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33m\"\u001b[39;49;00m\u001b[33mOption-2: Read instance group information from \u001b[39;49;00m\u001b[33m{file_path}\u001b[39;49;00m\u001b[33m.\u001b[39;49;00m\u001b[33m\\\u001b[39;49;00m\n",
+ "\u001b[33m You\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\u001b[33mll need to parse the json yourself. This doesn\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\u001b[33mt require an additional library.\u001b[39;49;00m\u001b[33m\\n\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \n",
+ " \u001b[34mwith\u001b[39;49;00m \u001b[36mopen\u001b[39;49;00m(file_path, \u001b[33m'\u001b[39;49;00m\u001b[33mr\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m) \u001b[34mas\u001b[39;49;00m f:\n",
+ " config = json.load(f)\n",
+ "\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\u001b[33m{\u001b[39;49;00mfile_path\u001b[33m}\u001b[39;49;00m\u001b[33m dump = \u001b[39;49;00m\u001b[33m{\u001b[39;49;00mjson.dumps(config, indent=\u001b[34m4\u001b[39;49;00m, sort_keys=\u001b[34mTrue\u001b[39;49;00m)\u001b[33m}\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m)\n",
+ " \n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.is_hetero: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\u001b[33minstance_groups\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m \u001b[35min\u001b[39;49;00m config\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33mcurrent_host=\u001b[39;49;00m\u001b[33m{\u001b[39;49;00mconfig[\u001b[33m'\u001b[39;49;00m\u001b[33mcurrent_host\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33mcurrent_instance_type=\u001b[39;49;00m\u001b[33m{\u001b[39;49;00mconfig[\u001b[33m'\u001b[39;49;00m\u001b[33mcurrent_instance_type\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.current_instance_group: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00mconfig[\u001b[33m'\u001b[39;49;00m\u001b[33mcurrent_group_name\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.current_instance_group_hosts: TODO\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.instance_groups: TODO\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.instance_groups_dict: \u001b[39;49;00m\u001b[33m{\u001b[39;49;00mconfig[\u001b[33m'\u001b[39;49;00m\u001b[33minstance_groups\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m]\u001b[33m}\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.distribution_hosts: TODO\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n",
+ " \u001b[36mprint\u001b[39;49;00m(\u001b[33mf\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m\u001b[33menv.distribution_instance_groups: TODO\u001b[39;49;00m\u001b[33m\"\u001b[39;49;00m)\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pygmentize source_dir/train.py"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 4: Configure the Estimator\n",
+ "In order to use SageMaker to fit our algorithm, we'll create an `Estimator` that defines how to use the container to train. This includes the configuration we need to invoke SageMaker training."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "estimator = TensorFlow(\n",
+ " entry_point='train.py',\n",
+ " source_dir='./source_dir',\n",
+ " #instance_type='ml.m4.xlarge',\n",
+ " #instance_count=1,\n",
+ " instance_groups = [data_group, dnn_group,],\n",
+ " framework_version='2.9.1',\n",
+ " py_version='py39',\n",
+ " role=role,\n",
+ " volume_size=10,\n",
+ " max_run=3600,\n",
+ " disable_profiler=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 5: Submit the training job\n",
+ "Here you are submitting the heterogeneous cluster training job. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2022-09-30 17:23:58 Starting - Starting the training job...\n",
+ "2022-09-30 17:24:26 Starting - Preparing the instances for training.........\n",
+ "2022-09-30 17:25:56 Downloading - Downloading input data...\n",
+ "2022-09-30 17:26:22 Training - Downloading the training image...............\n",
+ "2022-09-30 17:28:53 Training - Training image download completed. Training in progress....\n",
+ "2022-09-30 17:29:24 Uploading - Uploading generated training model\n",
+ "2022-09-30 17:29:24 Completed - Training job completed\n",
+ "..Training seconds: 0\n",
+ "Billable seconds: 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "estimator.fit(\n",
+ " job_name='hello-world-heterogenous' + \n",
+ " '-' + datetime.datetime.utcnow().strftime(\"%Y%m%dT%H%M%SZ\"),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 6: Review the logs for environment information\n",
+ "\n",
+ "Wait for the training job to finish, and review its logs in the AWS Console (click on **View logs** from the **Training Jobs** node in **Amazon SageMaker Console**) You'll find two logs: Algo1, Algo2. Examine the printouts on each node on how to retrieve instance group environment information. An example is shown here:\n",
+ "\n",
+ "```\n",
+ "Option-1: Read instance group information from the sagemaker_training.environment.Environment class\n",
+ "env.is_hetero: True\n",
+ "env.current_host: algo-1\n",
+ "env.current_instance_type: ml.c5.xlarge\n",
+ "env.current_instance_group: data_group\n",
+ "env.current_instance_group_hosts: ['algo-1']\n",
+ "env.instance_groups: ['data_group', 'dnn_group']\n",
+ "\n",
+ "Option-2: Read instance group information from {file_path}. You'll need to parse the json yourself. This doesn't require an additional library.\n",
+ "/opt/ml/input/config/resourceconfig.json dump = {\n",
+ " \"current_group_name\": \"data_group\",\n",
+ " \"current_host\": \"algo-1\",\n",
+ " \"current_instance_type\": \"ml.c5.xlarge\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\",\n",
+ " \"algo-2\"\n",
+ " ],\n",
+ " \"instance_groups\": [\n",
+ " {\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"instance_group_name\": \"data_group\",\n",
+ " \"instance_type\": \"ml.c5.xlarge\"\n",
+ " },\n",
+ " {\n",
+ " \"hosts\": [\n",
+ " \"algo-2\"\n",
+ " ],\n",
+ " \"instance_group_name\": \"dnn_group\",\n",
+ " \"instance_type\": \"ml.m4.xlarge\"\n",
+ " }\n",
+ " ],\n",
+ " \"network_interface_name\": \"eth0\"\n",
+ "}\n",
+ "env.is_hetero: True\n",
+ "current_host=algo-1\n",
+ "current_instance_type=ml.c5.xlarge\n",
+ "env.current_instance_group: data_group\n",
+ "env.current_instance_group_hosts: TODO\n",
+ "env.instance_groups: TODO\n",
+ "env.instance_groups_dict: [{'instance_group_name': 'data_group', 'instance_type': 'ml.c5.xlarge', 'hosts': ['algo-1']}, {'instance_group_name': 'dnn_group', 'instance_type': 'ml.m4.xlarge', 'hosts': ['algo-2']}]\n",
+ "env.distribution_hosts: TODO\n",
+ "env.distribution_instance_groups: TODO\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### C. Next steps\n",
+ "\n",
+ "In this notebook, we demonstrated how to retrieve the environment information, and differentiate which instance group an instance belongs to. Based on this, you can build logic to offload data processing tasks in your training job to a dedicated instance group. To understand how that can be done with a real-world example, we suggest going through the following notebook examples: \n",
+ "\n",
+ "- [TensorFlow's tf.data.service based Amazon SageMaker Heterogeneous Clusters for training](../tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb)\n",
+ "- [PyTorch and gRPC distributed dataloader based Amazon SageMaker Heterogeneous Clusters for training](../pt.grpc.sagemaker/hetero-pytorch-mnist.ipynb)"
+ ]
+ }
+ ],
+ "metadata": {
+ "instance_type": "ml.t3.medium",
+ "kernelspec": {
+ "display_name": "Python 3.9.7 ('.venv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "77c0de85c2cb739aa5100af7b92fb9d2075368f0e653f4148499a56c989df5f7"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/training/heterogeneous-clusters/hello.world.sagemaker/source_dir/train.py b/training/heterogeneous-clusters/hello.world.sagemaker/source_dir/train.py
new file mode 100644
index 0000000000..1884e20fab
--- /dev/null
+++ b/training/heterogeneous-clusters/hello.world.sagemaker/source_dir/train.py
@@ -0,0 +1,38 @@
+import json
+import os
+import sys
+from sagemaker_training import environment # This module is present on the DLC images, or you can install it with pip install sagemaker_training
+
+if __name__ == "__main__":
+
+ print("Option-1: Read instance group information from the sagemaker_training.environment.Environment class")
+ env = environment.Environment()
+ print(f"env.is_hetero: {env.is_hetero}")
+ print(f"env.current_host: {env.current_host}")
+ print(f"env.current_instance_type: {env.current_instance_type}")
+ print(f"env.current_instance_group: {env.current_instance_group}")
+ print(f"env.current_instance_group_hosts: {env.current_instance_group_hosts}")
+ print(f"env.instance_groups: {env.instance_groups}")
+ print(f"env.instance_groups_dict: {env.instance_groups_dict}")
+ print(f"env.distribution_hosts: {env.distribution_hosts}")
+ print(f"env.distribution_instance_groups: {env.distribution_instance_groups}")
+
+
+ file_path = '/opt/ml/input/config/resourceconfig.json'
+ print("Option-2: Read instance group information from {file_path}.\
+ You'll need to parse the json yourself. This doesn't require an additional library.\n")
+
+ with open(file_path, 'r') as f:
+ config = json.load(f)
+
+ print(f'{file_path} dump = {json.dumps(config, indent=4, sort_keys=True)}')
+
+ print(f"env.is_hetero: {'instance_groups' in config}")
+ print(f"current_host={config['current_host']}")
+ print(f"current_instance_type={config['current_instance_type']}")
+ print(f"env.current_instance_group: {config['current_group_name']}")
+ print(f"env.current_instance_group_hosts: TODO")
+ print(f"env.instance_groups: TODO")
+ print(f"env.instance_groups_dict: {config['instance_groups']}")
+ print(f"env.distribution_hosts: TODO")
+ print(f"env.distribution_instance_groups: TODO")
diff --git a/training/heterogeneous-clusters/index.rst b/training/heterogeneous-clusters/index.rst
new file mode 100644
index 0000000000..55b3125889
--- /dev/null
+++ b/training/heterogeneous-clusters/index.rst
@@ -0,0 +1,49 @@
+####################
+Heterogeneous Clusters
+####################
+
+SageMaker Training Heterogeneous Clusters allows you to run one training job
+that includes instances of different types. For example a GPU instance like
+ml.p4d.24xlarge and a CPU instance like c5.18xlarge.
+
+One primary use case is offloading CPU intensive tasks like image
+pre-processing (data augmentation) from the GPU instance to a dedicate
+CPU instance, so you can fully utilize the expensive GPUs, and arrive at
+an improved time and cost to train.
+
+.. admonition:: More resources:
+
+ - `SageMaker heterogeneous cluster developer guide `_
+
+
+See the following example notebooks:
+
+Hello World
+====================================
+This minimal example launches a Heterogeneous cluster training job, print environment information, and exit.
+
+.. toctree::
+ :maxdepth: 1
+
+ hello.world.sagemaker/helloworld-example
+
+
+TensorFlow
+====================================
+This example is a reusable implementation of Heterogeneous cluster with TensorFlow's tf.data.service
+
+.. toctree::
+ :maxdepth: 1
+
+ tf.data.service.sagemaker/hetero-tensorflow-restnet50
+
+
+PyTorch
+====================================
+This example is a reusable implementation of Heterogeneous cluster with gRPC based data loader
+
+.. toctree::
+ :maxdepth: 1
+
+ pt.grpc.sagemaker/hetero-pytorch-mnist
+
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed.proto b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed.proto
new file mode 100644
index 0000000000..94de2cd212
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed.proto
@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+service DatasetFeed {
+ rpc get_examples(Dummy) returns (stream Example) {}
+ rpc shutdown(Dummy) returns (Dummy) {}
+}
+
+message Dummy {
+}
+
+message Example {
+ bytes image = 1;
+ bytes label = 2;
+}
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed_pb2.py b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed_pb2.py
new file mode 100644
index 0000000000..78575b8888
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed_pb2.py
@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+# source: dataset_feed.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import message as _message
+from google.protobuf import reflection as _reflection
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12\x64\x61taset_feed.proto\"\x07\n\x05\x44ummy\"\'\n\x07\x45xample\x12\r\n\x05image\x18\x01 \x01(\x0c\x12\r\n\x05label\x18\x02 \x01(\x0c\x32Q\n\x0b\x44\x61tasetFeed\x12$\n\x0cget_examples\x12\x06.Dummy\x1a\x08.Example\"\x00\x30\x01\x12\x1c\n\x08shutdown\x12\x06.Dummy\x1a\x06.Dummy\"\x00\x62\x06proto3')
+
+
+
+_DUMMY = DESCRIPTOR.message_types_by_name['Dummy']
+_EXAMPLE = DESCRIPTOR.message_types_by_name['Example']
+Dummy = _reflection.GeneratedProtocolMessageType('Dummy', (_message.Message,), {
+ 'DESCRIPTOR' : _DUMMY,
+ '__module__' : 'dataset_feed_pb2'
+ # @@protoc_insertion_point(class_scope:Dummy)
+ })
+_sym_db.RegisterMessage(Dummy)
+
+Example = _reflection.GeneratedProtocolMessageType('Example', (_message.Message,), {
+ 'DESCRIPTOR' : _EXAMPLE,
+ '__module__' : 'dataset_feed_pb2'
+ # @@protoc_insertion_point(class_scope:Example)
+ })
+_sym_db.RegisterMessage(Example)
+
+_DATASETFEED = DESCRIPTOR.services_by_name['DatasetFeed']
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+ DESCRIPTOR._options = None
+ _DUMMY._serialized_start=22
+ _DUMMY._serialized_end=29
+ _EXAMPLE._serialized_start=31
+ _EXAMPLE._serialized_end=70
+ _DATASETFEED._serialized_start=72
+ _DATASETFEED._serialized_end=153
+# @@protoc_insertion_point(module_scope)
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed_pb2_grpc.py b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed_pb2_grpc.py
new file mode 100644
index 0000000000..b37fe7aad6
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/dataset_feed_pb2_grpc.py
@@ -0,0 +1,99 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+
+import dataset_feed_pb2 as dataset__feed__pb2
+
+
+class DatasetFeedStub(object):
+ """Missing associated documentation comment in .proto file."""
+
+ def __init__(self, channel):
+ """Constructor.
+
+ Args:
+ channel: A grpc.Channel.
+ """
+ self.get_examples = channel.unary_stream(
+ '/DatasetFeed/get_examples',
+ request_serializer=dataset__feed__pb2.Dummy.SerializeToString,
+ response_deserializer=dataset__feed__pb2.Example.FromString,
+ )
+ self.shutdown = channel.unary_unary(
+ '/DatasetFeed/shutdown',
+ request_serializer=dataset__feed__pb2.Dummy.SerializeToString,
+ response_deserializer=dataset__feed__pb2.Dummy.FromString,
+ )
+
+
+class DatasetFeedServicer(object):
+ """Missing associated documentation comment in .proto file."""
+
+ def get_examples(self, request, context):
+ """Missing associated documentation comment in .proto file."""
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+ context.set_details('Method not implemented!')
+ raise NotImplementedError('Method not implemented!')
+
+ def shutdown(self, request, context):
+ """Missing associated documentation comment in .proto file."""
+ context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+ context.set_details('Method not implemented!')
+ raise NotImplementedError('Method not implemented!')
+
+
+def add_DatasetFeedServicer_to_server(servicer, server):
+ rpc_method_handlers = {
+ 'get_examples': grpc.unary_stream_rpc_method_handler(
+ servicer.get_examples,
+ request_deserializer=dataset__feed__pb2.Dummy.FromString,
+ response_serializer=dataset__feed__pb2.Example.SerializeToString,
+ ),
+ 'shutdown': grpc.unary_unary_rpc_method_handler(
+ servicer.shutdown,
+ request_deserializer=dataset__feed__pb2.Dummy.FromString,
+ response_serializer=dataset__feed__pb2.Dummy.SerializeToString,
+ ),
+ }
+ generic_handler = grpc.method_handlers_generic_handler(
+ 'DatasetFeed', rpc_method_handlers)
+ server.add_generic_rpc_handlers((generic_handler,))
+
+
+ # This class is part of an EXPERIMENTAL API.
+class DatasetFeed(object):
+ """Missing associated documentation comment in .proto file."""
+
+ @staticmethod
+ def get_examples(request,
+ target,
+ options=(),
+ channel_credentials=None,
+ call_credentials=None,
+ insecure=False,
+ compression=None,
+ wait_for_ready=None,
+ timeout=None,
+ metadata=None):
+ return grpc.experimental.unary_stream(request, target, '/DatasetFeed/get_examples',
+ dataset__feed__pb2.Dummy.SerializeToString,
+ dataset__feed__pb2.Example.FromString,
+ options, channel_credentials,
+ insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
+
+ @staticmethod
+ def shutdown(request,
+ target,
+ options=(),
+ channel_credentials=None,
+ call_credentials=None,
+ insecure=False,
+ compression=None,
+ wait_for_ready=None,
+ timeout=None,
+ metadata=None):
+ return grpc.experimental.unary_unary(request, target, '/DatasetFeed/shutdown',
+ dataset__feed__pb2.Dummy.SerializeToString,
+ dataset__feed__pb2.Dummy.FromString,
+ options, channel_credentials,
+ insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/launcher.py b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/launcher.py
new file mode 100644
index 0000000000..371663f461
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/launcher.py
@@ -0,0 +1,101 @@
+import sys
+import time
+from typing import Optional
+
+# instance group names
+DATA_GROUP = 'data_group'
+DNN_GROUP = 'dnn_group'
+
+def start_child_process(name : str, additional_args=[]) -> int:
+ import subprocess
+ params = ["python", f"./{name}"] + sys.argv[1:] + additional_args
+ print(f'Opening process: {params}')
+ p = subprocess.run(params)
+ print(f'Process {name} closed with returncode={p.returncode}')
+ if p.returncode == -15 or p.returncode == -9:
+ print(f'Received SIGTERM|SIGKILL which is normal termination for pytorch data service to avoid hanging process')
+ return 0
+ return p.returncode
+
+
+def start_data_group(dispatcher_host : str) -> int:
+ return start_child_process('train_data.py', ["--dispatcher_host", dispatcher_host])
+
+
+def start_dnn_group(dispatcher_host : Optional[str]) -> int:
+ additional_args = [] if dispatcher_host is None else ["--dispatcher_host", dispatcher_host]
+ return start_child_process('train_dnn.py', additional_args)
+
+
+def get_group_first_host(instance_groups, target_group_name):
+ return instance_groups[target_group_name]['hosts'][0]
+
+def shutdown_pt_data_service_with_retries(dispatcher_host : str):
+ for i in range(0,12):
+ try:
+ if i>0:
+ sleeptime = 10
+ print(f'Will attempt {i} time to shutdown in {sleeptime} seconds')
+ time.sleep(sleeptime)
+ _shutdown_data_service(dispatcher_host)
+ break
+ except Exception as e:
+ print(f'Failed to shutdown dispatcher in {dispatcher_host} due to: {e}')
+
+
+def _shutdown_data_service(dispatcher_host : str):
+ SHUTDOWN_PORT = 16000
+ print(f'Shutting down data service dispatcher via: [{dispatcher_host}:{SHUTDOWN_PORT}]')
+ import socket
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.connect((dispatcher_host, SHUTDOWN_PORT))
+ print(f'Shutdown request sent to {dispatcher_host}:{SHUTDOWN_PORT}')
+
+
+def split_to_instance_group_train_script() -> int:
+ from sagemaker_training import environment
+ env = environment.Environment()
+ # try:
+ # from sagemaker_training import environment
+ # env = environment.Environment()
+ # except ImportError:
+ # class Object(object):
+ # pass
+
+ # env = Object()
+ # env.is_hetero = True
+ # env.current_host = 'dummyhost'
+ # env.instance_groups_dict = {DATA_GROUP : {'hosts': ['dummyhost']}}
+ # env.current_instance_group = DNN_GROUP
+ # env.current_instance_type = 'dummyinstance'
+
+ print(f'env.is_hetero={env.is_hetero}')
+ print(f'current_host={env.current_host}')
+
+ if env.is_hetero:
+ dispatcher_host = get_group_first_host(env.instance_groups_dict, DATA_GROUP)
+ first_host_in_dnn_group = get_group_first_host(env.instance_groups_dict, DNN_GROUP)
+ print(f'current_instance_type={env.current_instance_type}')
+ print(f'current_group_name={env.current_instance_group}')
+ print(f'dispatcher_host={dispatcher_host}')
+ if env.current_instance_group == DATA_GROUP:
+ return start_data_group(dispatcher_host)
+ elif env.current_instance_group == DNN_GROUP:
+ returncode = start_dnn_group(dispatcher_host)
+ # first host in DNN group takes care of shutting down the dispatcher
+ if env.current_host == first_host_in_dnn_group:
+ shutdown_pt_data_service_with_retries(dispatcher_host)
+ return returncode
+ else:
+ raise Exception(f'Unknown instance group: {env.current_instance_group}')
+
+ else: # not hetero
+ return start_dnn_group(dispatcher_host=None)
+
+if __name__ == "__main__":
+ try:
+ returncode = split_to_instance_group_train_script()
+ exit(returncode)
+ except Exception as e:
+ print(f'Failed due to {e}. exiting with returncode=1')
+ sys.exit(1)
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/requirements.txt b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/requirements.txt
new file mode 100644
index 0000000000..5d406f6b34
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/requirements.txt
@@ -0,0 +1,2 @@
+torchvision
+grpcio-tools
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train.py b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train.py
new file mode 100644
index 0000000000..1963d940bf
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train.py
@@ -0,0 +1,127 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torchvision import datasets, transforms
+import time
+import logging
+import sys
+import os
+import json
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
+class Net(nn.Module):
+ def __init__(self):
+ super(Net, self).__init__()
+ self.conv1 = nn.Conv2d(1, 32, 3, 1)
+ self.conv2 = nn.Conv2d(32, 64, 3, 1)
+ self.dropout1 = nn.Dropout(0.25)
+ self.dropout2 = nn.Dropout(0.5)
+ self.fc1 = nn.Linear(9216, 128)
+ self.fc2 = nn.Linear(128, 10)
+ def forward(self, x):
+ x = self.conv1(x)
+ x = F.relu(x)
+ x = self.conv2(x)
+ x = F.relu(x)
+ x = F.max_pool2d(x, 2)
+ x = self.dropout1(x)
+ x = torch.flatten(x, 1)
+ x = self.fc1(x)
+ x = F.relu(x)
+ x = self.fc2(x)
+ output = F.log_softmax(x, dim=1)
+ return output
+
+class MyMNIST(datasets.MNIST):
+ '''
+ A personalized extension of the MNIST class in which we
+ modify the __len__ operation to return the maximum value
+ of int32 so that we do not run out of data.
+ '''
+
+ def __init__(self, batch_size : int, iterations : int, **kwargs):
+
+ super().__init__(**kwargs)
+ self.batch_size = batch_size
+ self.iterations = iterations
+
+ def __len__(self) -> int:
+ size = self.batch_size * self.iterations
+ return size
+
+ def __getitem__(self, index: int):
+ return super(MyMNIST, self).__getitem__(index % len(self.data))
+
+def main(args):
+ use_cuda = torch.cuda.is_available()
+ device = torch.device("cuda" if use_cuda else "cpu")
+ train_kwargs = {'batch_size': args.batch_size,
+ 'num_workers': args.num_data_workers,
+ 'pin_memory': args.pin_memory
+ }
+ logger.info ('Training job started...')
+ transform=transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize((0.1307,), (0.3081,)),
+ transforms.GaussianBlur(11)
+ ])
+ dataset = MyMNIST(batch_size=args.batch_size, iterations=args.iterations, root='./data', train=True,
+ transform=transform, download=True)
+ train_loader = torch.utils.data.DataLoader(dataset,
+ **train_kwargs)
+ model = Net().to(device)
+ optimizer = optim.Adadelta(model.parameters())
+ model.train()
+ t = time.perf_counter()
+ for idx, (data, target) in enumerate(train_loader, start=1):
+ data, target = data.to(device), target.to(device)
+ optimizer.zero_grad()
+ output = model(data)
+ loss = F.nll_loss(output, target)
+ loss.backward()
+ optimizer.step()
+ if device=='cpu' or idx % 10 == 0:
+ logger.info(
+ f'{idx}: avg step time: {(time.perf_counter()-t)/idx}')
+ logger.info('Training completed!')
+ save_model(model, args.model_dir)
+
+def save_model(model, model_dir):
+ logger.info("Saving the model")
+ path = os.path.join(model_dir, "model.pth")
+ torch.save(model.cpu().state_dict(), path)
+ return
+
+def read_args():
+ import argparse
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("--batch-size", type=int, default=4,
+ help="Input batch size for training",)
+ parser.add_argument("--iterations", type=int, default=10,
+ help="Based on no. of cpu per training instance",)
+ parser.add_argument("--num-data-workers", type=int, default=1, metavar="N",
+ help="Based on no. of cpu per training instance type in data group",)
+ parser.add_argument("--num-dnn-workers", type=int, default=1, metavar="N",
+ help="Based on no. of cpu per training instance type in dnn group, ideally should match to grpc-workers",)
+ parser.add_argument("--grpc-workers", type=int, default=1, metavar="N",
+ help="No. of grpc server workers to start",)
+ parser.add_argument("--pin-memory", type=bool, default=1,
+ help="pin to GPU memory (default: True)",)
+ parser.add_argument("--seed", type=int, default=1,
+ help="random seed (default: 1)",)
+ parser.add_argument("--hosts", type=list, default=json.loads(os.environ["SM_HOSTS"]))
+ parser.add_argument("--current-host", type=str, default=os.environ["SM_CURRENT_HOST"])
+ parser.add_argument("--model-dir", type=str, default=os.environ["SM_MODEL_DIR"])
+ parser.add_argument("--train", type=str, default=os.environ["SM_CHANNEL_TRAINING"])
+ #parser.add_argument("--test", type=str, default=os.environ["SM_CHANNEL_TESTING"])
+ parser.add_argument("--num-gpus", type=int, default=os.environ["SM_NUM_GPUS"])
+ parser.add_argument("--dispatcher_host", type=str)
+ return parser.parse_args()
+
+if __name__ == '__main__':
+ main(read_args())
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train_data.py b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train_data.py
new file mode 100644
index 0000000000..2d35ba4b26
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train_data.py
@@ -0,0 +1,172 @@
+import multiprocessing as mp
+from concurrent import futures
+
+import grpc
+import torch
+from torchvision import datasets, transforms
+
+import dataset_feed_pb2
+import dataset_feed_pb2_grpc
+import logging
+import sys
+
+# Logging initialization
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
+# The following class implements the data feeding service
+class DatasetFeedService(dataset_feed_pb2_grpc.DatasetFeedServicer):
+ def __init__(self, q, kill_event):
+ '''
+ param q: A shared queue containing data batches
+ param kill: Kill event for graceful shutdown
+ '''
+ self.q = q
+ self.kill_event = kill_event
+
+
+ def get_examples(self, request, context):
+ while True:
+ #print('DEBUG: get_examples')
+ example = self.q.get()
+ yield dataset_feed_pb2.Example(image=example[0],
+ label=example[1])
+
+
+ def shutdown(self, request, context):
+ logger.info("Received shutdown request - Not implemented")
+ # from main_grpc_client import shutdown_data_service
+ # shutdown_data_service()
+ context.set_code(grpc.StatusCode.OK)
+ context.set_details('Shutting down')
+ return dataset_feed_pb2.Dummy()
+
+
+# The data loading and preprocessing logic.
+# We chose to keep the existing logic unchanged, just instead
+# of feeding the model, the dataloader feeds a shared queue
+class MyMNIST(datasets.MNIST):
+ '''
+ A personalized extension of the MNIST class in which we
+ modify the __len__ operation to return the maximum value
+ of int32 so that we do not run out of data.
+ '''
+
+ def __init__(self, batch_size : int, iterations : int, **kwargs):
+
+ super().__init__(**kwargs)
+ self.batch_size = batch_size
+ self.iterations = iterations
+
+ def __len__(self) -> int:
+ size = self.batch_size * self.iterations
+ return size
+
+ def __getitem__(self, index: int):
+ return super(MyMNIST, self).__getitem__(index % len(self.data))
+
+
+def fill_queue(q,kill, args):
+
+ MyMNIST.mirrors = ["https://sagemaker-sample-files.s3.amazonaws.com/datasets/image/MNIST/"]
+ train_kwargs = {'batch_size': args.batch_size,
+ 'num_workers': args.num_data_workers}
+ transform=transforms.Compose([
+ transforms.ToTensor(),
+ transforms.Normalize((0.1307,), (0.3081,)),
+ transforms.GaussianBlur(11)
+ ])
+ dataset = MyMNIST(batch_size=args.batch_size, iterations=args.iterations, root='./data', train=True,
+ transform=transform, download=True)
+ loader = torch.utils.data.DataLoader(dataset, **train_kwargs)
+ for batch_idx, (data, target) in enumerate(loader):
+ if kill.is_set():
+ logger.info('kill signal received, exiting fill_queue')
+ break
+ added = False
+ while not added and not kill.is_set():
+ try:
+ # convert the data to bytestrings and add to queue
+ q.put((data.numpy().tobytes(),
+ target.type(torch.int8).numpy().tobytes()),
+ timeout=1)
+ #print(f'DEBUG: Added example to queue')
+ added = True
+ except:
+ continue
+ logger.info('Finished filling queue with dataset.')
+
+
+def start(kill_event, args):
+ q = mp.Queue(maxsize=32)
+ queuing_process = mp.Process(target=fill_queue, args=(q, kill_event, args))
+ queuing_process.start()
+ logger.info('Started queuing process.')
+
+ server = grpc.server(futures.ThreadPoolExecutor(max_workers=args.grpc_workers))
+ dataset_feed_pb2_grpc.add_DatasetFeedServicer_to_server(
+ DatasetFeedService(q, kill_event), server)
+ server.add_insecure_port('[::]:6000')
+ server.start()
+ logger.info('gRPC Data Server started at port 6000.')
+ return queuing_process,server
+
+
+def shutdown(queuing_process, grpc_server):
+ logger.info('Shutting down...')
+ logger.info('Stopping gRPC server...')
+ grpc_server.stop(2).wait()
+ logger.info('Stopping queuing process...')
+ queuing_process.join(1)
+ queuing_process.terminate()
+ logger.info('Shutdown done.')
+ import os, time
+ os.system('kill -9 %d' % os.getpid())
+
+
+def wait_for_shutdown_signal():
+ SHUTDOWN_PORT = 16000
+ import socket
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind(('', SHUTDOWN_PORT))
+ s.listen(1)
+ logger.info('Awaiting shutdown signal on port {}'.format(SHUTDOWN_PORT))
+ conn, addr = s.accept()
+ print('Received shutdown signal from: ', addr)
+ try:
+ conn.close()
+ s.close()
+ except Exception as e:
+ logger.info(e)
+
+
+def serve(args):
+ kill_event = mp.Event() # an mp.Event for graceful shutdown
+ queue_data_loader_process, grpc_server = start(kill_event, args)
+ wait_for_shutdown_signal()
+ kill_event.set()
+ shutdown(queue_data_loader_process, grpc_server)
+
+def read_args():
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--batch-size", type=int, default=4, metavar="N",
+ help="input batch size for training",)
+ parser.add_argument("--num-data-workers", type=int, default=1, metavar="N",
+ help="based on no. of cpu per training instance",)
+ parser.add_argument("--num-dnn-workers", type=int, default=1,
+ help="based on no. of cpu per training instance",)
+ parser.add_argument("--iterations", type=int, default=10, metavar="N",
+ help="The number of iterations per epoch (multiply of 10)",)
+ parser.add_argument("--grpc-workers", type=int, default=1, metavar="N",
+ help="No. of gRPC server workers",)
+ parser.add_argument("--pin-memory", type=bool, default=1,
+ help="pin to GPU memory (default: True)",)
+ parser.add_argument("--first_data_host", type=str)
+ args, unknown = parser.parse_known_args()
+ return args
+
+
+if __name__ == "__main__":
+ serve(read_args())
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train_dnn.py b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train_dnn.py
new file mode 100644
index 0000000000..6dfa6f59f7
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/code/train_dnn.py
@@ -0,0 +1,179 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torchvision import datasets, transforms
+import time
+import grpc
+import dataset_feed_pb2_grpc
+import dataset_feed_pb2
+import logging
+import sys
+import json
+import os
+
+#Pass environment variables to detect heterogenous host names
+from sagemaker_training import environment
+
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+logger.addHandler(logging.StreamHandler(sys.stdout))
+
+# Based on https://github.com/pytorch/examples/blob/master/mnist/main.py
+class Net(nn.Module):
+ def __init__(self):
+ super(Net, self).__init__()
+ self.conv1 = nn.Conv2d(1, 32, 3, 1)
+ self.conv2 = nn.Conv2d(32, 64, 3, 1)
+ self.dropout1 = nn.Dropout(0.25)
+ self.dropout2 = nn.Dropout(0.5)
+ self.fc1 = nn.Linear(9216, 128)
+ self.fc2 = nn.Linear(128, 10)
+ def forward(self, x):
+ x = self.conv1(x)
+ x = F.relu(x)
+ x = self.conv2(x)
+ x = F.relu(x)
+ x = F.max_pool2d(x, 2)
+ x = self.dropout1(x)
+ x = torch.flatten(x, 1)
+ x = self.fc1(x)
+ x = F.relu(x)
+ x = self.fc2(x)
+ output = F.log_softmax(x, dim=1)
+ return output
+
+
+# Decode binary data from SM_CHANNEL_TRAINING
+# Decode and preprocess data
+# Create map dataset
+class RemoteDataset(torch.utils.data.IterableDataset):
+ '''
+ An iterable PyTorch dataset that opens a connection to the
+ gRPC server and reads from a stream of data batches
+ '''
+
+ def __init__(self, data_host, batch_size, iterations):
+ self.data_host = data_host
+ self.batch_size = batch_size
+ self.iterations = iterations
+
+
+ def __len__(self) -> int:
+ size = self.batch_size * self.iterations
+ return size
+
+ def get_stub(self):
+ channel = grpc.insecure_channel(f'{self.data_host}:6000',
+ # overwrite the default max message length
+ options=[('grpc.max_receive_message_length',
+ 200 * 1024 * 1024)])
+
+ try:
+ # print('Waiting for gRPC data server to be ready...')
+ grpc.channel_ready_future(channel).result(timeout=30)
+ except grpc.FutureTimeoutError:
+ logger.error('ERROR: Timeout connecting to gRPC data server. Check that it is running.')
+ raise
+ #print('Connected to gRPC data server.')
+
+ return dataset_feed_pb2_grpc.DatasetFeedStub(channel,)
+
+
+ def __iter__(self):
+ import numpy as np
+
+ examples = self.get_stub().get_examples(dataset_feed_pb2.Dummy())
+ for s in examples:
+ image = torch.tensor(np.frombuffer(s.image,
+ dtype=np.float32)).reshape(
+ [self.batch_size, 1, 28, 28])
+ label = torch.tensor(np.frombuffer(s.label,
+ dtype=np.int8)).reshape(
+ [self.batch_size]).type(torch.int64)
+ yield image, label
+
+
+ # def shutdown_remote(self):
+ # print('Calling remote server to shutdown')
+ # self.get_stub().shutdown(dataset_feed_pb2.Dummy())
+
+
+def main(args):
+ logger.info ('Training job started...')
+ use_cuda = args.num_gpus > 0
+ device = torch.device("cuda" if use_cuda > 0 else "cpu")
+
+ torch.manual_seed(args.seed)
+ if use_cuda:
+ torch.cuda.manual_seed(args.seed)
+
+ train_kwargs = {'batch_size': None, #the data is already batched
+ 'num_workers': args.num_dnn_workers,
+ 'pin_memory': args.pin_memory
+ }
+
+ dataset = RemoteDataset(args.dispatcher_host, args.batch_size, args.iterations)
+ train_loader = torch.utils.data.DataLoader(dataset,
+ **train_kwargs)
+ model = Net().to(device)
+ optimizer = optim.Adadelta(model.parameters())
+ model.train()
+ t = time.perf_counter()
+ for idx, (data, target) in enumerate(train_loader, start=1):
+ data, target = data.to(device), target.to(device)
+ optimizer.zero_grad()
+ output = model(data)
+ loss = F.nll_loss(output, target)
+ loss.backward()
+ optimizer.step()
+ if device.type == 'cpu' or idx % 10 == 0:
+ logger.info(
+ f'{idx}: avg step time: {(time.perf_counter()-t)/idx}')
+
+ # TODO: exit the loop through the iterator stopping by itself
+ if idx*args.batch_size==(dataset.__len__()):
+ break
+
+ save_model(model, args.model_dir)
+ logger.info ('Training job completed!')
+
+
+def save_model(model, model_dir):
+ logger.info("Saving the model")
+ path = os.path.join(model_dir, "model.pth")
+ torch.save(model.cpu().state_dict(), path)
+ return
+
+
+def read_args():
+ import argparse
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument("--batch-size", type=int, default=4,
+ help="Input batch size for training",)
+ parser.add_argument("--iterations", type=int, default=10,
+ help="Based on no. of cpu per training instance",)
+ parser.add_argument("--num-data-workers", type=int, default=1, metavar="N",
+ help="Based on no. of cpu per training instance type in data group",)
+ parser.add_argument("--num-dnn-workers", type=int, default=1, metavar="N",
+ help="Based on no. of cpu per training instance type in dnn group, ideally should match to grpc-workers",)
+ parser.add_argument("--grpc-workers", type=int, default=1, metavar="N",
+ help="No. of grpc server workers to start",)
+ parser.add_argument("--pin-memory", type=bool, default=1,
+ help="pin to GPU memory (default: True)",)
+ parser.add_argument("--seed", type=int, default=1,
+ help="random seed (default: 1)",)
+ parser.add_argument("--hosts", type=list, default=json.loads(os.environ["SM_HOSTS"]))
+ parser.add_argument("--current-host", type=str, default=os.environ["SM_CURRENT_HOST"])
+ parser.add_argument("--model-dir", type=str, default=os.environ["SM_MODEL_DIR"])
+ parser.add_argument("--train", type=str, default=os.environ["SM_CHANNEL_TRAINING"])
+ #parser.add_argument("--test", type=str, default=os.environ["SM_CHANNEL_TESTING"])
+ parser.add_argument("--num-gpus", type=int, default=os.environ["SM_NUM_GPUS"])
+ parser.add_argument("--dispatcher_host", type=str)
+ return parser.parse_args()
+
+
+if __name__ == '__main__':
+ main(read_args())
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/hetero-pytorch-mnist.ipynb b/training/heterogeneous-clusters/pt.grpc.sagemaker/hetero-pytorch-mnist.ipynb
new file mode 100644
index 0000000000..7146988464
--- /dev/null
+++ b/training/heterogeneous-clusters/pt.grpc.sagemaker/hetero-pytorch-mnist.ipynb
@@ -0,0 +1,520 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "85da9619",
+ "metadata": {},
+ "source": [
+ "# PyTorch's example to demonstrate Amazon SageMaker Heterogeneous Cluster for model training\n",
+ "\n",
+ "---\n",
+ "### Description\n",
+ "Heterogeneous clusters enable launching training jobs that use multiple instance types in a single job. This capability can improve your training cost and speed by running different parts of the model training on the most suitable instance type. This use case typically happens in computer vision DL training, where training is bottleneck on CPU resources needed for data augmentation, leaving the expensive GPU underutilized. Heterogeneous clusters enable you to add more CPU resources to fully utilize GPUs, thus increase training speed and cost-efficiency. For more details, you can find the documentation of this feature [here](https://docs.aws.amazon.com/sagemaker/latest/dg/train-heterogeneous-cluster.html).\n",
+ "\n",
+ "This notebook demonstrates how to use Heterogeneous Cluster feature of SageMaker Training with PyTorch. The notebook works on Python 3 (_PyTorch 1.11 Python 3.8 CPU Optimized_) image of SageMaker Studio Notebook instance, and runs on _ml.t3.medium_ instance type.\n",
+ "\n",
+ "The notebook covers:\n",
+ "- Setting up SageMaker Studio Notebook \n",
+ "- Setting up the Training environment \n",
+ "- Submit a Training job\n",
+ "- Monitor and visualize the CloudWatch metrics\n",
+ "- Comparing time-to-train and cost-to-train\n",
+ "- Conclusion \n",
+ "\n",
+ "In this sample notebook, we have taken the PyTorch model based on this [official MNIST example](https://github.com/pytorch/examples/tree/main/MNIST). We modified the training code to be heavy on data pre-processing. We are going to train this model in both Homogeneous and Heterogeneous Cluster modes. The flag to train on any of these modes can be set using `IS_HETERO = False or True` in section **B.2 Configure environment variables**. \n",
+ "\n",
+ "Homogeneous Training Job - In this baseline we observe an ml.p3.2xlarge with an under-utilized GPU due to a CPU bottleneck. \n",
+ "
\n",
+ "\n",
+ "Heterogeneous Training Job - Where we add ml.c5.9xlarge instance for extra CPU cores, to allow increased GPU usage of ml.p3.2xlarge instance, and improve cost-efficiency. Both the jobs runs the training code, train data set, pre-processing, and other relevant parameters.\n",
+ "
\n",
+ "\n",
+ "In homogeneous cluster training job, the data pre-processing and Deep Neural Network (DNN) training code runs on the same instance. However, in heterogeneous cluster training job, the data pre-processing code runs on the CPU nodes (here by referred as **data_group or data group**), whereas the Deep Neural Network (DNN) training code runs on the GPU nodes (here referred as **dnn_group or dnn group**). The inter-node communication between the data and dnn groups is handled by generic implementation of [gRPC client-server interface](https://grpc.io/docs/languages/python/basics/). \n",
+ "\n",
+ "The script (`launcher.py`) has the logic to detect (using SageMaker environment variables) whether the node it is running on belongs to data_group or dnn_group. If it is data_group, it spawns a separate process by executing `train_data.py`. This script runs grpc-server service for extracting processed training batches using [Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview). The gRPC server running on the data_group listens on a specific port (ex. 6000). In the code (`train_data.py`) documentation, we have chosen an implementation that keeps the data loading logic intact where data batches are entered into a shared queue. The `get_samples` function of the `DataFeedService` pulls batches from the same queue and sends them to the client in the form of a continuous data stream. While fetching the data, the main entrypoint script `launcher.py` listens on port 16000 for a shutdown request coming from gRPC client i.e. data group. The `train_data.py` waits for shutdown action from the parent process. \n",
+ "\n",
+ "If the node belongs to dnn_group, the main training script (`launcher.py`) spawns a separate set of processes by executing `train_dnn.py`. The script runs gRPC client code and DNN component of the training job. It consumes the processed training data from the gRPC server. We have defined an iterable PyTorch dataset, RemoteDataset, that opens a connection to the gRPC server, and reads from a stream of data batches. Once the model is trained with all the batches of training data, the gRPC client exits, and the parent process`launcher.py` sends a shutdown request on port 16000. This indicates the gRPC server to shutdown, and signals ends of the training job. \n",
+ "\n",
+ "Here is how the workflow looks like:\n",
+ "\n",
+ "
\n",
+ "\n",
+ "This example notebook runs a training job on 2 instances, 1 in each node group. The data_group uses ml.c5.9xlarge whereas dnn_group uses ml.p3.2xlarge.\n",
+ "\n",
+ "This notebook refers following files and folders:\n",
+ "\n",
+ "- Folders: \n",
+ " - `code`: this has the training (data pre-processing and dnn) scripts, and grpc client-server start and shutdown scripts\n",
+ " - `images`: contains images referred in notebook\n",
+ "- Files: \n",
+ " - `launcher.py`: entry point training script. This script is executed on all the nodes irrespective of which group it belongs to. This is a parent process that makes a decision on where to spawn a data pre-processing or dnn component of the training job. The script runs on all the nodes as entry point. It also handles the shutdown logic for gRPC server. \n",
+ " - `train_data.py`, `dataset_feed_pb2.py`, `dataset_feed_pb2_grpc.py`: these scripts run on the data_group nodes and responsible for setting up grpc-server, start and shutdown.\n",
+ " - `train_dnn.py`: this script runs dnn code on the training data set. It fetches preprocessed data from the data_group node as a stream using gRPC client-server communication. It also sends a shutdown request after all the iterations on the preprocessed training data set. \n",
+ " - `requirement.txt`: defines package required for gRPC \n",
+ " - `train.py`: this script is the entry point script for SageMaker homogeneous cluster training. This script is picked up when you choose IS_HETERO = False. This uses a local dataset and runs both data pre-processing and a dnn component on the same node. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1f98cde9",
+ "metadata": {},
+ "source": [
+ "### security groups update if running in private VPC\n",
+ "This section is relevant if you plan to [run in a private VPC](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html) (passing `subnets` and `security_group_ids` parameters when defining an Estimator). \n",
+ "SageMaker documentation recommends you [add](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html#train-vpc-vpc) a rule for your security group that allows inbound connections between members of the same security group, for all TCP communication. This will also cover for the gRPC related traffic between instances:\n",
+ "- the data_group instances will listen on port 6000 for connections from all nodes. This stream is not encrypted. You can change the code to encrypted the connection if needed.\n",
+ "- the data_group intances listen on port 16000 for a shutdown signal from all nodes."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fd1e5aca",
+ "metadata": {},
+ "source": [
+ "### A. Setting up SageMaker Studio notebook\n",
+ "\n",
+ "#### Step 1 - Upgrade SageMaker SDK and dependent packages \n",
+ "Heterogeneous Clusters for Amazon SageMaker model training was [announced](https://aws.amazon.com/about-aws/whats-new/2022/07/announcing-heterogeneous-clusters-amazon-sagemaker-model-training) on 07/08/2022. As a first step, ensure you have updated SageMaker SDK, PyTorch, and Boto3 client that enables this feature."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "id": "54ff1687",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "python3 -m pip install --upgrade boto3 botocore awscli sagemaker"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0d20b2f3",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Restart the notebook kernel "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "229e1b18",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#import IPython\n",
+ "#IPython.Application.instance().kernel.do_shutdown(True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a9592cda",
+ "metadata": {},
+ "source": [
+ "#### Step 3 - Validate SageMaker Python SDK and PyTorch versions\n",
+ "Ensure the output of the cell below reflects:\n",
+ "\n",
+ "- SageMaker Python SDK version 2.98.0 or above, \n",
+ "- boto3 1.24 or above \n",
+ "- botocore 1.27 or above \n",
+ "- PyTorch 1.10 or above "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "id": "0b0e3202",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Name: sagemaker\n",
+ "Version: 2.109.0\n",
+ "---\n",
+ "Name: torch\n",
+ "Version: 1.10.2+cpu\n",
+ "---\n",
+ "Name: boto3\n",
+ "Version: 1.24.72\n",
+ "---\n",
+ "Name: botocore\n",
+ "Version: 1.27.72\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip show sagemaker torch boto3 botocore |egrep 'Name|Version|---'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9176d868",
+ "metadata": {},
+ "source": [
+ "--------------\n",
+ "### B. Setting up the Training environment\n",
+ "\n",
+ "#### Step 1 - Import SageMaker components and set up the IAM role and Amazon S3 bucket"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "id": "594fce53",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "arn:aws:iam::776941257690:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole\n",
+ "s3://sagemaker-us-east-1-776941257690/DEMO-MNIST\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "import json\n",
+ "import datetime\n",
+ "import os\n",
+ "\n",
+ "import sagemaker\n",
+ "from sagemaker.pytorch import PyTorch\n",
+ "from sagemaker import get_execution_role\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "\n",
+ "\n",
+ "sess = sagemaker.Session()\n",
+ "\n",
+ "role = get_execution_role()\n",
+ "\n",
+ "output_path = \"s3://\" + sess.default_bucket() + \"/DEMO-MNIST\"\n",
+ "print(role)\n",
+ "print(output_path)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "165bca04",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Configure environment variables \n",
+ "This step defines whether you want to run training job in heterogeneous cluster mode or not. Also, defines instance groups, multiple nodes in group, and hyperparameter values. For baselining, run a homogeneous cluster training job by setting `IS_HETERO = False`. This will let both the data pre-processing and DNN code run on the same node i.e. `ml.p3.2xlarge`. \n",
+ "\n",
+ "\n",
+ "Test configuration (if running training on p3.2xl or g5.2xl as dnn_group instance type, and c5.2xl as data_group instance type: (training duration: 7-8 mins) \n",
+ "`num-data-workers: 4` \n",
+ "`grpc-workers: 4` \n",
+ "`num-dnn-workers: 4` \n",
+ "`pin-memory\": True` \n",
+ "`iterations : 100` \n",
+ "\n",
+ "Performance configuration (if running training on p3.2xl as dnn_group instance type, and c5.9xl as data_group instance type OR training in homogeneous cluster mode i.e. g5.8xl): (training duration - 30 mins) \n",
+ "`num-data-workers: 32` \n",
+ "`grpc-workers: 2` \n",
+ "`num-dnn-workers: 2` \n",
+ "`pin-memory\": True` \n",
+ "`iterations : 4800`\n",
+ "\n",
+ "Performance configuration (if running training on p3.2xl in homogeneous cluster mode): \n",
+ "`num-data-workers: 8` \n",
+ "`grpc-workers: 2` \n",
+ "`num-dnn-workers: 2` \n",
+ "`pin-memory\": True` \n",
+ "`iterations : 2400`\n",
+ "\n",
+ "Note: This PyTorch example has not been tested with multiple instances in an instance group. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "id": "0d65707b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "IS_CLOUD_JOB = True\n",
+ "IS_HETERO = True # if set to false, uses homogeneous cluster\n",
+ "PT_DATA_MODE = \"service\" if IS_HETERO else \"local\" # local | service\n",
+ "IS_DNN_DISTRIBUTION = False # Distributed Training with DNN nodes not tested, set it to False\n",
+ "\n",
+ "data_group = InstanceGroup(\n",
+ " \"data_group\", \"ml.c5.9xlarge\", 1\n",
+ ") # 36 vCPU #change the instance type if IS_HETERO=True\n",
+ "dnn_group = InstanceGroup(\n",
+ " \"dnn_group\", \"ml.p3.2xlarge\", 1\n",
+ ") # 8 vCPU #change the instance type if IS_HETERO=True\n",
+ "\n",
+ "kwargs = dict()\n",
+ "kwargs[\"hyperparameters\"] = {\n",
+ " \"batch-size\": 8192,\n",
+ " \"num-data-workers\": 4, # This number drives the avg. step time. More workers help parallel pre-processing of data. Recommendation: Total no. of cpu 'n' = 'num-data-wokers'+'grpc-workers'+ 2 (reserved)\n",
+ " \"grpc-workers\": 4, # No. of workers serving pre-processed data to DNN group (gRPC client). see above formula.\n",
+ " \"num-dnn-workers\": 4, # Modify this no. to be less than the cpu core of your training instances in dnn group\n",
+ " \"pin-memory\": True, # Pin to GPU memory\n",
+ " \"iterations\": 100, # No. of iterations in an epoch (must be multiple of 10).\n",
+ "}\n",
+ "\n",
+ "if IS_HETERO:\n",
+ " kwargs[\"instance_groups\"] = [data_group, dnn_group]\n",
+ " entry_point = \"launcher.py\"\n",
+ "else:\n",
+ " kwargs[\"instance_type\"] = (\n",
+ " \"ml.p3.2xlarge\" if IS_CLOUD_JOB else \"local\"\n",
+ " ) # change the instance type if IS_HETERO=False\n",
+ " kwargs[\"instance_count\"] = 1\n",
+ " entry_point = \"train.py\"\n",
+ "\n",
+ "if IS_DNN_DISTRIBUTION:\n",
+ " processes_per_host_dict = {\n",
+ " \"ml.g5.xlarge\": 1,\n",
+ " \"ml.g5.12xlarge\": 4,\n",
+ " \"ml.p3.8xlarge\": 4,\n",
+ " \"ml.p4d.24xlarge\": 8,\n",
+ " }\n",
+ " kwargs[\"distribution\"] = {\n",
+ " \"mpi\": {\n",
+ " \"enabled\": True,\n",
+ " \"processes_per_host\": processes_per_host_dict[dnn_instance_type],\n",
+ " \"custom_mpi_options\": \"--NCCL_DEBUG INFO\",\n",
+ " },\n",
+ " }\n",
+ " if IS_HETERO:\n",
+ " kwargs[\"distribution\"][\"instance_groups\"] = [dnn_group]\n",
+ "\n",
+ " print(f\"distribution={kwargs['distribution']}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4ff19e24",
+ "metadata": {},
+ "source": [
+ "#### Step 3: Set up the Estimator\n",
+ "In order to use SageMaker to fit our algorithm, we'll create `Estimator` that defines how to use the container to train. This includes the configuration we need to invoke SageMaker training."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "id": "94f4c8ce",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "estimator = PyTorch(\n",
+ " framework_version=\"1.11.0\", # 1.10.0 or later\n",
+ " py_version=\"py38\", # Python v3.8\n",
+ " role=role,\n",
+ " entry_point=entry_point,\n",
+ " source_dir=\"code\",\n",
+ " volume_size=10,\n",
+ " max_run=4800,\n",
+ " disable_profiler=True,\n",
+ " debugger_hook_config=False,\n",
+ " **kwargs,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a81dcab6",
+ "metadata": {},
+ "source": [
+ "#### Step 4: Download the MNIST Data and Upload it to S3 bucket\n",
+ "\n",
+ "This is an optional step for now. The training job downloads the data on its run directly from MNIST website to the data_group node (grpc server). "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "id": "d0534973",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import logging\n",
+ "import boto3\n",
+ "from botocore.exceptions import ClientError\n",
+ "\n",
+ "# Download training and testing data from a public S3 bucket\n",
+ "\n",
+ "\n",
+ "def download_from_s3(data_dir=\"./data\", train=True):\n",
+ " \"\"\"Download MNIST dataset and convert it to numpy array\n",
+ "\n",
+ " Args:\n",
+ " data_dir (str): directory to save the data\n",
+ " train (bool): download training set\n",
+ "\n",
+ " Returns:\n",
+ " None\n",
+ " \"\"\"\n",
+ "\n",
+ " if not os.path.exists(data_dir):\n",
+ " os.makedirs(data_dir)\n",
+ "\n",
+ " if train:\n",
+ " images_file = \"train-images-idx3-ubyte.gz\"\n",
+ " labels_file = \"train-labels-idx1-ubyte.gz\"\n",
+ " else:\n",
+ " images_file = \"t10k-images-idx3-ubyte.gz\"\n",
+ " labels_file = \"t10k-labels-idx1-ubyte.gz\"\n",
+ "\n",
+ " # download objects\n",
+ " s3 = boto3.client(\"s3\")\n",
+ " bucket = f\"sagemaker-sample-files\"\n",
+ " for obj in [images_file, labels_file]:\n",
+ " key = os.path.join(\"datasets/image/MNIST\", obj)\n",
+ " dest = os.path.join(data_dir, obj)\n",
+ " if not os.path.exists(dest):\n",
+ " s3.download_file(bucket, key, dest)\n",
+ " return\n",
+ "\n",
+ "\n",
+ "download_from_s3(\"./data\", True)\n",
+ "download_from_s3(\"./data\", False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "id": "2d699654",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Upload to the default bucket\n",
+ "\n",
+ "prefix = \"DEMO-MNIST\"\n",
+ "bucket = sess.default_bucket()\n",
+ "loc = sess.upload_data(path=\"./data\", bucket=bucket, key_prefix=prefix)\n",
+ "\n",
+ "channels = {\"training\": loc, \"testing\": loc}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "48352f04",
+ "metadata": {},
+ "source": [
+ "## C. Submit the training job\n",
+ "\n",
+ "The job runs for the predefined iterations. DNN instance group sends a shutdown request to data group after done with the training. You can see the following entries in the CloudWatch logs of dnn instance. A job with 4800 iterations finishes in 29 mins in a Heterogeneous cluster composed of 1x ml.c5.9xlarge as data node and 1x ml.p3.2xlarge as DNN node.\n",
+ "\n",
+ "Note: The console output of billing seconds can be ignored. See the AWS console > SageMaker > Training Job for the exact billing seconds.\n",
+ "\n",
+ "Log excerpt from algo-1 (DNN instance)\n",
+ "```\n",
+ "4780: avg step time: 0.19709917231025106\n",
+ "INFO:__main__:4780: avg step time: 0.19709917231025106\n",
+ "4790: avg step time: 0.19694106239373696\n",
+ "INFO:__main__:4790: avg step time: 0.19694106239373696\n",
+ "4800: avg step time: 0.196784295383125\n",
+ "Saving the model\n",
+ "INFO:__main__:4800: avg step time: 0.196784295383125\n",
+ "INFO:__main__:Saving the model\n",
+ "Training job completed!\n",
+ "INFO:__main__:Training job completed!\n",
+ "Process train_dnn.py closed with returncode=0\n",
+ "Shutting downdata service dispatcher via: [algo-2:16000]\n",
+ "shutdown request sent to algo-2:16000\n",
+ "2022-08-16 01:15:05,555 sagemaker-training-toolkit INFO Waiting for the process to finish and give a return code.\n",
+ "2022-08-16 01:15:05,555 sagemaker-training-toolkit INFO Done waiting for a return code. Received 0 from exiting process.\n",
+ "2022-08-16 01:15:05,556 sagemaker-training-toolkit INFO Reporting training SUCCESS\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31cb6cae",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2022-09-15 00:55:22 Starting - Starting the training job...\n",
+ "2022-09-15 00:55:50 Starting - Preparing the instances for training.........\n",
+ "2022-09-15 00:57:10 Downloading - Downloading input data.."
+ ]
+ }
+ ],
+ "source": [
+ "estimator.fit(\n",
+ " inputs=channels,\n",
+ " job_name=\"pt-hetero\"\n",
+ " + \"-\"\n",
+ " + \"H-\"\n",
+ " + str(IS_HETERO)[0]\n",
+ " + \"-\"\n",
+ " + datetime.datetime.utcnow().strftime(\"%Y%m%dT%H%M%SZ\"),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3ea2e092",
+ "metadata": {},
+ "source": [
+ "## D. Monitoring Instance Metrics for GPU and CPU utilization\n",
+ "\n",
+ "Click on **View instance metrics** from the **Training jobs** node in **Amazon SageMaker Console**. In the run above, all 30 vCPU of Data node (algo-1) is approx. 100% utilized, and the GPU utilization is at 100% at frequent intervals in the DNN node (algo-2). To rescale the CloudWatch Metrics to 100% on CPU utilization for algo-1 and algo-2, use CloudWatch \"Add Math\" feature and average it out by no. of cores on those instance types.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "430fb45e",
+ "metadata": {},
+ "source": [
+ "## E. Comparing time-to-train and cost-to-train\n",
+ "\n",
+ "Let's continue with the above example i.e. train a heavy data pre-processing (CPU intensive) model (MNIST) requiring only 1 GPU. We start with ml.p3.2xlarge (1xV100 GPU, 8x vCPU) in homogeneous cluster mode to get the baseline performance numbers. Due to the no. of CPU cores, we could not go beyond 8 data loader/workers for data pre-processing. The avg. step cost was `7.6 cents` and avg. step time is `1.19 seconds`. \n",
+ "\n",
+ "Our objective is to reduce the cost and speed up the model training time. The first choice here is to scale up the instance type in the same family. If we leverage the next instance type (4 GPU) in the P3 family, the GPUs would have gone underutilized. In this case, we needed more vCPU to GPU ratio. Assuming we haven't had any instance type in another instance family or the model is incompatible with the CPU/GPU architectures of other instance families, we are constrained to use ml.p3.2xlarge. The only way then to have more vCPUs to GPU ratio is to use SageMaker feature, Heterogeneous Cluster, which enables customers to offload data pre-processing logic to CPU only instance types example ml.c5. In the next test, we offloaded CPU intensive work i.e. data preprocessing to ml.c5.9xlarge (36 vCPU) and continued using ml.p3.2xlarge for DNN. The avg. step cost was `1.9 cents` and avg. step time is `0.18 seconds`. \n",
+ "\n",
+ "In summary, we reduced the training cost by 4.75 times, and the avg. step reduced by 6.5 times. This was possible because with higher cpu count, we could use 32 data loader workers (compared to 8 with p3.2xl) to preprocess the data, and kept GPU close to 100% utilized at frequent intervals. Note: These numbers are just taken as a sample, you have to do benchmarking with your own model and dataset to come up with the exact price-performance benefits. \n",
+ "\n",
+ "## F. Conclusion\n",
+ "In this notebook, we demonstrated how to leverage heterogeneous cluster feature of SageMaker Training to achieve better price performance. To get started you can copy this example project, and only change the `train_dnn.py` script.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.9.7 ('.venv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "77c0de85c2cb739aa5100af7b92fb9d2075368f0e653f4148499a56c989df5f7"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-cluster-diagram.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-cluster-diagram.png
new file mode 100644
index 0000000000..c84c185d31
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-cluster-diagram.png differ
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-instance-metrics.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-instance-metrics.png
new file mode 100644
index 0000000000..d812e2f46b
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-instance-metrics.png differ
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/homogeneous-cluster-diagram.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/homogeneous-cluster-diagram.png
new file mode 100644
index 0000000000..ae3119d4b6
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/homogeneous-cluster-diagram.png differ
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/pytorch-heterogeneous-workflow.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/pytorch-heterogeneous-workflow.png
new file mode 100644
index 0000000000..c40b09aeed
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/pytorch-heterogeneous-workflow.png differ
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/heterogenenous-workload.json b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/heterogenenous-workload.json
new file mode 100644
index 0000000000..5b3736fc35
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/heterogenenous-workload.json
@@ -0,0 +1,30 @@
+{
+ "metrics": [
+ [ { "expression": "100*(m1/9600)", "label": "DNN1 CPU/100%", "id": "e1" } ],
+ [ { "expression": "100*(m2/800)", "label": "DNN1 GPU/100%", "id": "e2" } ],
+ [ { "expression": "100*(m3/7200)", "label": "DATA1 CPU/100%", "id": "e3" } ],
+ [ { "expression": "100*(m4/7200)", "label": "DATA2 CPU/100%", "id": "e4" } ],
+ [ "/aws/sagemaker/TrainingJobs", "CPUUtilization", "Host", "hetero-tf-data-service-Dnode2-wrkrs-1-20220922T214326Z/algo-1", { "id": "m1", "yAxis": "left", "visible": false } ],
+ [ ".", "GPUUtilization", ".", ".", { "id": "m2", "visible": false } ],
+ [ ".", "CPUUtilization", ".", "hetero-tf-data-service-Dnode2-wrkrs-1-20220922T214326Z/algo-2", { "id": "m3", "visible": false } ],
+ [ "...", "hetero-tf-data-service-Dnode2-wrkrs-1-20220922T214326Z/algo-3", { "id": "m4", "visible": false } ]
+ ],
+ "sparkline": true,
+ "view": "timeSeries",
+ "stacked": false,
+ "region": "us-east-1",
+ "stat": "Average",
+ "period": 60,
+ "setPeriodToTimeRange": true,
+ "yAxis": {
+ "left": {
+ "min": 0,
+ "max": 100,
+ "label": "% Utilization",
+ "showUnits": false
+ }
+ },
+ "legend": {
+ "position": "bottom"
+ }
+}
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/homogenous-workload copy.json b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/homogenous-workload copy.json
new file mode 100644
index 0000000000..c514eced5a
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/homogenous-workload copy.json
@@ -0,0 +1,26 @@
+{
+ "sparkline": true,
+ "metrics": [
+ [ { "expression": "100*(m1/9600)", "label": "CPU/100%", "id": "e1" } ],
+ [ { "expression": "100*(m2/800)", "label": "GPU/100%", "id": "e2" } ],
+ [ "/aws/sagemaker/TrainingJobs", "CPUUtilization", "Host", "hetero-tf-data-local-Dnode1-wrkrs-1-20220921T213920Z/algo-1", { "id": "m1", "visible": false } ],
+ [ ".", "GPUUtilization", ".", ".", { "id": "m2", "visible": false } ]
+ ],
+ "view": "timeSeries",
+ "stacked": false,
+ "region": "us-east-1",
+ "stat": "Average",
+ "period": 60,
+ "setPeriodToTimeRange": true,
+ "yAxis": {
+ "left": {
+ "min": 0,
+ "max": 100,
+ "label": "% Utilization",
+ "showUnits": false
+ }
+ },
+ "legend": {
+ "position": "bottom"
+ }
+}
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/launcher.py b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/launcher.py
new file mode 100644
index 0000000000..d67b27af7d
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/launcher.py
@@ -0,0 +1,123 @@
+import sys
+import os
+import time
+from typing import Optional
+import subprocess
+
+# instance group names
+DATA_GROUP = 'data_group'
+DNN_GROUP = 'dnn_group'
+
+
+def start_child_process_async(name : str, additional_args=[]) -> int:
+ #TODO: Find a way to stream stdout and stderr to the parent process
+ params = ["python", f"./{name}"] + sys.argv[1:] + additional_args
+ print(f'Opening process async: {params}')
+ p = subprocess.Popen(params)
+ print(f'Process {name} started')
+ return p.pid
+
+
+def start_child_process(name : str, additional_args=[]) -> int:
+ params = ["python", f"./{name}"] + sys.argv[1:] + additional_args
+ print(f'Opening process: {params}')
+ p = subprocess.run(params)
+ print(f'Process {name} closed with returncode={p.returncode}')
+ return p.returncode
+
+
+def start_data_group(dispatcher_host : str) -> int:
+ return start_child_process('train_data.py', ["--dispatcher_host", dispatcher_host])
+
+
+def not_mpi_or_rank_0() -> bool:
+ return 'OMPI_COMM_WORLD_LOCAL_RANK' not in os.environ or os.environ['OMPI_COMM_WORLD_LOCAL_RANK'] == '0'
+
+
+def start_dnn_group(dispatcher_host : Optional[str]) -> int:
+ if dispatcher_host is not None:
+ args = ["--dispatcher_host", dispatcher_host]
+ # Start a tf.data.service worker processes for each host in the DNN group
+ # to take advantage of its CPU resources.
+ # Start once per instance, not per MPI process
+ if not_mpi_or_rank_0():
+ start_child_process_async('train_data.py', args)
+ else:
+ args = []
+ return start_child_process('train_dnn.py', args)
+
+
+def get_group_first_host(instance_groups, target_group_name):
+ return instance_groups[target_group_name]['hosts'][0]
+
+
+def is_not_mpi_or_world_rank_0() -> bool:
+ return 'OMPI_COMM_WORLD_RANK' in os.environ and os.environ['OMPI_COMM_WORLD_RANK'] != '0'
+
+
+def shutdown_tf_data_service_with_retries(hosts : list):
+ # only world rank 0 process should shutdown the dispatcher
+ if is_not_mpi_or_world_rank_0():
+ return
+
+ completed_hosts = []
+ for host in hosts:
+ for i in range(0,12):
+ try:
+ if i>0:
+ sleeptime = 10
+ print(f'Will attempt {i} time to shutdown in {sleeptime} seconds')
+ time.sleep(sleeptime)
+
+ if host not in completed_hosts:
+ _shutdown_data_service(host)
+ completed_hosts.append(host)
+ break
+ except Exception as e:
+ print(f'Failed to shutdown dispatcher in {host} due to: {e}')
+
+
+def _shutdown_data_service(dispatcher_host : str):
+ SHUTDOWN_PORT = 16000
+ print(f'Shutting down tf.data.service dispatcher via: [{dispatcher_host}:{SHUTDOWN_PORT}]')
+ import socket
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.connect((dispatcher_host, SHUTDOWN_PORT))
+ print(f'Shutdown request sent to {dispatcher_host}:{SHUTDOWN_PORT}')
+
+
+def split_to_instance_group_train_script() -> int:
+ from sagemaker_training import environment
+ env = environment.Environment()
+
+ print(f'env.is_hetero={env.is_hetero}')
+ print(f'current_host={env.current_host}')
+
+ if env.is_hetero:
+ dispatcher_host = get_group_first_host(env.instance_groups_dict, DATA_GROUP)
+ first_host_in_dnn_group = get_group_first_host(env.instance_groups_dict, DNN_GROUP)
+ print(f'current_instance_type={env.current_instance_type}')
+ print(f'current_group_name={env.current_instance_group}')
+ print(f'dispatcher_host={dispatcher_host}')
+ if env.current_instance_group == DATA_GROUP:
+ return start_data_group(dispatcher_host)
+ elif env.current_instance_group == DNN_GROUP:
+ returncode = start_dnn_group(dispatcher_host)
+ # first host in DNN group will take care of shutting down the dispatcher
+ if env.current_host == first_host_in_dnn_group:
+ hosts = env.instance_groups_dict[DATA_GROUP]['hosts'] + env.instance_groups_dict[DNN_GROUP]['hosts']
+ shutdown_tf_data_service_with_retries(hosts)
+ return returncode
+ else:
+ raise Exception(f'Unknown instance group: {env.current_instance_group}')
+
+ else: # not heterogenous
+ return start_dnn_group(dispatcher_host=None)
+
+if __name__ == "__main__":
+ try:
+ returncode = split_to_instance_group_train_script()
+ exit(returncode)
+ except Exception as e:
+ print(f'Failed due to {e}. exiting with returncode=1')
+ sys.exit(1)
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/requirements.txt b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/requirements.txt
new file mode 100644
index 0000000000..10aac59994
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/requirements.txt
@@ -0,0 +1,2 @@
+protobuf==3.20.2
+tensorflow-addons==0.17.0
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_data.py b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_data.py
new file mode 100644
index 0000000000..62248d1e68
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_data.py
@@ -0,0 +1,68 @@
+from tensorflow.data.experimental.service import DispatchServer, WorkerServer, DispatcherConfig, WorkerConfig
+
+def wait_for_shutdown_signal(dispatcher, workers):
+ SHUTDOWN_PORT = 16000
+ import socket
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind(('', SHUTDOWN_PORT))
+ s.listen(1)
+ print('Awaiting shutdown signal on port {}'.format(SHUTDOWN_PORT))
+ conn, addr = s.accept()
+ print('Received shutdown signal from: ', addr)
+ try:
+ conn.close()
+ s.close()
+ except Exception as e:
+ print(e)
+
+ if dispatcher is not None: # dispatcher runs only on the 1st data instance
+ print('Stopping dispatcher.')
+ dispatcher._stop()
+ print('Joining dispatcher')
+ dispatcher.join()
+
+ for i,worker in enumerate(workers, start=0):
+ print(f'Stopping worker {i}')
+ worker._stop()
+ print(f'Joining worker {i}')
+ worker.join()
+
+def create_worker(workerIndex : int, dispatcher_host : str, current_host : str) -> WorkerServer:
+ port = 6001 + workerIndex
+ w_config = WorkerConfig(port=port,
+ dispatcher_address=f'{dispatcher_host}:6000',
+ worker_address=f'{current_host}:{port}')
+ print(f'Starting tf.data.service WorkerServer {w_config}')
+ worker = WorkerServer(w_config)
+ return worker
+
+def start_dispatcher_and_worker(dispatcher_host : str, current_host : str, num_of_data_workers : int):
+ assert(dispatcher_host is not None)
+
+ if current_host == dispatcher_host:
+ print(f'starting Dispatcher (dispatcher_host={dispatcher_host})')
+ d_config = DispatcherConfig(port=6000)
+ dispatcher = DispatchServer(d_config)
+ else:
+ dispatcher = None
+
+ workers = [ create_worker(i, dispatcher_host, current_host) for i in range(num_of_data_workers) ]
+ print(f'Finished starting dispatcher and {num_of_data_workers} workers')
+
+ wait_for_shutdown_signal(dispatcher, workers)
+
+
+"This function read mode command line argument"
+def read_args():
+ import argparse, os
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--dispatcher_host", type=str)
+ parser.add_argument("--current_host", type=str, default=os.environ["SM_CURRENT_HOST"])
+ parser.add_argument("--num_of_data_workers", type=int)
+ args, unknown = parser.parse_known_args()
+ return args
+
+
+if __name__ == "__main__":
+ args = read_args()
+ start_dispatcher_and_worker(args.dispatcher_host, args.current_host, args.num_of_data_workers)
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_dnn.py b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_dnn.py
new file mode 100644
index 0000000000..cd938b46c7
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_dnn.py
@@ -0,0 +1,153 @@
+from tensorflow.keras.layers.experimental import preprocessing
+from tensorflow.keras.applications.resnet50 import ResNet50
+import tensorflow_addons as tfa
+import tensorflow as tf
+import os
+import horovod.tensorflow.keras as hvd
+
+
+# dilation filter
+def dilate(image, label):
+ dilateFilter = tf.zeros([3, 3, 3], tf.uint8)
+ image = tf.expand_dims(image, 0)
+ image = tf.nn.dilation2d(
+ image, dilateFilter, strides=[1, 1, 1, 1],
+ dilations=[1, 1, 1, 1],
+ padding='SAME',
+ data_format='NHWC')
+ image = tf.squeeze(image)
+ return image, label
+# blur filter
+
+
+def blur(image, label):
+ image = tfa.image.gaussian_filter2d(image=image,
+ filter_shape=(11, 11), sigma=0.8)
+ return image, label
+
+# rescale filter
+def rescale(image, label):
+ image = preprocessing.Rescaling(1.0 / 255)(image)
+ return image, label
+
+
+# augmentation filters
+def augment(image, label):
+ data_augmentation = tf.keras.Sequential(
+ [preprocessing.RandomFlip("horizontal"),
+ preprocessing.RandomRotation(0.1),
+ preprocessing.RandomZoom(0.1)])
+ image = data_augmentation(image)
+ return image, label
+
+
+# This function generates a dataset consisting 32x32x3 random images
+# And a corresponding random label representing 10 different classes.
+# As this dataset is randomly generated, you should not expect the model
+# to converge in a meaningful way, it doesn't matter as our intent is
+# only to measure data pipeline and DNN optimization throughput
+def generate_artificial_dataset():
+ import numpy as np
+ x_train = np.random.randint(0, 255, size=(32000, 32, 32, 3), dtype=np.uint8)
+ y_train = np.random.randint(0, 10, size=(32000,1))
+ train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
+ return train_dataset
+
+
+def get_dataset(batch_size : int, use_tf_data_service : bool, dispatcher_host : str):
+ autotune = tf.data.experimental.AUTOTUNE
+ options = tf.data.Options()
+ options.experimental_deterministic = False
+
+ ds = generate_artificial_dataset().shuffle(10000).repeat()
+
+ ds = ds.map(dilate, num_parallel_calls=autotune)
+ ds = ds.map(blur, num_parallel_calls=autotune)
+ ds = ds.map(rescale,num_parallel_calls=autotune)
+ ds = ds.map(augment, num_parallel_calls=autotune)
+ ds = ds.batch(batch_size)
+
+ if use_tf_data_service:
+ ds = ds.apply(tf.data.experimental.service.distribute(
+ processing_mode="parallel_epochs",
+ service=f'grpc://{dispatcher_host}:6000',),
+ )
+
+ #ds = ds.take(1).cache().repeat()
+ ds = ds.prefetch(autotune)
+ return ds
+
+"This function read mode command line argument"
+def read_args():
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--tf_data_mode', type=str, default='local',
+ help="'service' distributed dataset using tf.data.service. 'local' use standard tf.data")
+ parser.add_argument('--steps_per_epoch', type=int, default=1)
+ parser.add_argument('--batch_size', type=int)
+ parser.add_argument('--epochs', type=int, default=1)
+ parser.add_argument("--n_gpus", type=str,
+ default=os.environ.get("SM_NUM_GPUS"))
+ parser.add_argument("--dispatcher_host", type=str)
+ parser.add_argument("--num_of_data_workers", type=int, default=1)
+ parser.add_argument("--output_data_dir", type=str,
+ default=os.environ.get("SM_OUTPUT_DATA_DIR"))
+ parser.add_argument("--model_dir", type=str,
+ default=os.environ.get("SM_MODEL_DIR"))
+ parser.add_argument("--checkpoint-path",type=str,default="/opt/ml/checkpoints",help="Path where checkpoints are saved.")
+ args = parser.parse_args()
+ return args
+
+if __name__ == "__main__":
+ args = read_args()
+ hvd.init()
+ # Horovod: pin GPU to be used to process local rank (one GPU per process)
+ gpus = tf.config.experimental.list_physical_devices('GPU')
+ print(str(gpus))
+ for gpu in gpus:
+ tf.config.experimental.set_memory_growth(gpu, True)
+ if gpus:
+ print(f'hvd.local_rank() {hvd.local_rank()}')
+ tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], 'GPU')
+
+ model = ResNet50(weights=None,
+ input_shape=(32, 32, 3),
+ classes=10)
+
+ model.compile(loss=tf.losses.SparseCategoricalCrossentropy(),
+ optimizer=tf.optimizers.Adam())
+ # Horovod: adjust learning rate based on number of GPUs.
+ scaled_lr = 0.001 * hvd.size()
+ opt = tf.optimizers.Adam(scaled_lr)
+ opt = hvd.DistributedOptimizer(
+ opt, backward_passes_per_step=1, average_aggregated_gradients=True)
+
+ model.compile(loss=tf.losses.SparseCategoricalCrossentropy(),
+ optimizer=opt,
+ experimental_run_tf_function=False)
+
+ callbacks = [
+ hvd.callbacks.BroadcastGlobalVariablesCallback(0),
+ hvd.callbacks.MetricAverageCallback(),
+ hvd.callbacks.LearningRateWarmupCallback(initial_lr=scaled_lr, warmup_epochs=3, verbose=1),
+ ]
+ # Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them.
+ if hvd.rank() == 0:
+ path = os.path.join(args.checkpoint_path, './checkpoint-{epoch}.h5')
+ callbacks.append(tf.keras.callbacks.ModelCheckpoint(path))
+
+ # Horovod: write logs on worker 0.
+ verbose = 1 if hvd.rank() == 0 else 0
+
+ assert(args.tf_data_mode == 'local' or args.tf_data_mode == 'service')
+ print(f'Running in {args.tf_data_mode} tf_data_mode.')
+ dataset = get_dataset(batch_size = args.batch_size, use_tf_data_service=(args.tf_data_mode == 'service'), dispatcher_host = args.dispatcher_host)
+
+ model.fit( dataset,
+ steps_per_epoch=args.steps_per_epoch,
+ callbacks=callbacks,
+ epochs=args.epochs,
+ verbose=2,)
+
+ if hvd.rank() == 0:
+ model.save(os.path.join(args.model_dir, '000000001'), 'my_model.h5')
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb b/training/heterogeneous-clusters/tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb
new file mode 100644
index 0000000000..61cdc56cf5
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb
@@ -0,0 +1,1136 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# TensorFlow's tf.data.service with Amazon SageMaker Training Heterogeneous Clusters\n",
+ "\n",
+ "---\n",
+ "### Introduction\n",
+ "\n",
+ "Heterogeneous clusters enable launching training jobs that use multiple instance types in a single job. This capability can improve your training cost and speed by running different parts of the model training on the most suitable instance type. This use case typically happens in computer vision (CV) deep learning (DL) training, where training is bottleneck on CPU resources needed for data augmentation, leaving the expensive GPU underutilized. Heterogeneous clusters enable you to add more CPU resources to fully utilize GPUs to increase training speed and cost-efficiency. For more details, you can find the documentation of this feature [here](https://docs.aws.amazon.com/sagemaker/latest/dg/train-heterogeneous-cluster.html).\n",
+ "\n",
+ "This notebook demonstrates how to use Heterogeneous Clusters with TensorFlow's [tf.data.service](https://www.TensorFlow.org/api_docs/python/tf/data/experimental/service). It includes training a CPU intensive DL CV workload. Comparing cost and performance between homogeneous and heterogeneous training configurations. \n",
+ "\n",
+ "💡To get started quickly with heterogeneous clusters, we suggest you'll reuse the provided code as a quick way to migrate your workload from a local tf.data pipeline to a distributed tf.data.service pipeline. You'll need to change [code/train_dnn.py](./code/train_dnn.py), while keeping [code/train_data.py](./code/train_data.py) and [code/launcher.py](code/launcher.py) intact. This is explained below in the [Workload Details] section.\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This notebook covers:\n",
+ "- A guide to switching from a homogeneous job (single instance type) to a heterogeneous job (multiple instance types)\n",
+ "- Explaining to use Heterogeneous clusters with TensorFlow's tf.data.service\n",
+ "- Set up Amazon SageMaker Studio Notebook \n",
+ "- Run homogeneous cluster training job \n",
+ "- Run heterogeneous cluster training job \n",
+ "- Compare time and cost to train between homogeneous and heterogeneous clusters\n",
+ "- Conclusion\n",
+ "\n",
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### A guide to switching from a homogeneous to a heterogeneous job\n",
+ "\n",
+ "This notebook runs and compares these two workloads:\n",
+ "\n",
+ "Homogeneous Training Job - The image shows a ml.p4d.24xlarge instance GPUs is under-utilized due to a CPU bottleneck. \n",
+ "
\n",
+ " \n",
+ "Heterogeneous Training Job - The image shows two ml.c5.18xlarge instances with extra CPU cores, to reduce the CPU bottleneck and increase GPU usage, to improve training speed cost-efficiency. \n",
+ " 
\n",
+ "\n",
+ "In each workload: Training data is an artificially generated dataset consisting of 32x32x3 images with random pixel values, and a corresponding random label representing 10 different classes. As this dataset is randomly generated, you should not expect the model to converge in a meaningful way. This shouldn't matter as our intent is only to measure data pipeline and neural network optimization throughput expressed in epoch/step time. \n",
+ "The model we used is [Resnet50](https://www.TensorFlow.org/api_docs/python/tf/keras/applications/ResNet50). The workloads uses an 8 GPUs instance, ml.p4d.24xlarge, and uses Horovod for data parallelization. \n",
+ "\n",
+ "The heterogeneous job will include two instance groups:\n",
+ "- **data_group** - A group of CPU instances that will run data pre-processing code.\n",
+ "- **dnn_group** - A group of GPU instances that will run Deep Neural Network training code.\n",
+ "\n",
+ "In this example, the inter-node communication between CPU and GPU instance groups is implemented using [TensorFlow data service feature](https://www.TensorFlow.org/api_docs/python/tf/data/experimental/service). This feature allows offloading a configurable amount of preprocessing work to worker machines. Note that SageMaker's Heterogeneous cluster does not provide out-of-the-box support for inter-instance_group communication, and it is up to the user to implement (we provide reference implementation here).\n",
+ "\n",
+ "This notebook refers following files and folders:\n",
+ "- [code/train_dnn.py](./code/train_dnn.py) - this is standard TF training script, it has a single reference to tf.data.service when setting up the tf.data pipeline. This script will be executed on GPU instances belonging to the dnn_group.\n",
+ "- [code/train_data.py](./code/train_data.py) - this script starts tf.data.service services like a tf.data.service Dispatcher and tf.data.service Worker processes. You shouldn't edit this script when adjusting to your workload.\n",
+ "- [code/launcher.py](./code/launcher.py) - Entry point training script. This is the script that SageMaker Training will start on all instances (all instances groups must share the same entry point script in heterogeneous clusters). `launcher.py` is responsible for detecting the instance group the instance belong to, and start `train_dnn.py` and `train_data.py` accordingly. It is also responsible for shutting down tf.data.services the training script completes (`train_dnn.py`) so all instances exit allowing the SageMaker training job to complete. \n",
+ "In every instance `luncher.py` will use `train_data.py` to start a tf.data.service worker server (As all instance types have CPUs that could be used for preprocessing). `luncher.py` will start a single tf.data.service dispatcher server (on the first instance of the `data_group`). \n",
+ "`luncher.py` will start the `train_dnn.py` script in all GPU instances (`dnn_group` instances).\n",
+ "\n",
+ "#### Learn more about tf.data.service processes\n",
+ "`tf.data.service Dispatcher` - The dispatcher server acts as the control plain for tf.data.service; Being responsible for registering worker servers and assigning preprocessing tasks to them. Each training job has a single Dispatcher running in the first instance of the `data_group` and listens on port 6000.\n",
+ "`tf.data.service Workers` - Worker servers carry out the data processing. Each instance could have one or more workers (listen on port 6001/6002/...).\n",
+ "\n",
+ "##### Defining what part of your pipeline runs in which instance group\n",
+ " When you apply `tf.data.experimental.service.distribute()` on your dataset, all preprocessing operations defined up to the apply will run on the tf.data.service workers, and all dataset operations defined afterwords will run on the local process. All instances will need access to a dataset you'll make available through a SageMaker training data channel. You do have the option of limiting which instance group will see which training data channel.\n",
+ "\n",
+ "The below figure shows sequence of events of setting up and running in a tf.data.service based heterogeneous cluster training job.\n",
+ "\n",
+ "
\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### security groups update if running in private VPC\n",
+ "This section is relevant if you plan to [run in a private VPC](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html) (passing `subnets` and `security_group_ids` parameters when defining an Estimator). \n",
+ "SageMaker documentation recommends you [add](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html#train-vpc-vpc) a rule for your security group that allows inbound connections between members of the same security group, for all TCP communication. This will also cover for the tf.data.service related traffic between instances:\n",
+ "- tf.data.service Dispatcher node will listen for incoming connections on port 6000 (configurable) from all nodes.\n",
+ "- tf.data.service Workers will listen on ports 6001-6006 from all nodes.\n",
+ "- Each node listens on port 16000 for a tf.data.service shutdown signal from all nodes."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### A. Set up SageMaker Studio notebook\n",
+ "#### Before you start\n",
+ "Ensure you have selected Python 3 (_TensorFlow 2.6 Python 3.8 CPU Optimized_) image for your SageMaker Studio Notebook instance, and running on _ml.t3.medium_ instance type.\n",
+ "\n",
+ "#### Step 1 - Upgrade SageMaker SDK and dependent packages \n",
+ "Heterogeneous Clusters for Amazon SageMaker model training was [announced](https://aws.amazon.com/about-aws/whats-new/2022/07/announcing-heterogeneous-clusters-amazon-sagemaker-model-training) on 07/08/2022. This feature release requires you to have updated SageMaker SDK, Boto3 client."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: boto3 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (1.24.72)\n",
+ "Collecting boto3\n",
+ " Downloading boto3-1.24.80-py3-none-any.whl (132 kB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 132.5/132.5 kB 925.9 kB/s eta 0:00:00\n",
+ "Requirement already satisfied: botocore in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (1.27.72)\n",
+ "Collecting botocore\n",
+ " Downloading botocore-1.27.80-py3-none-any.whl (9.1 MB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 9.1/9.1 MB 16.4 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: awscli in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (1.25.73)\n",
+ "Collecting awscli\n",
+ " Downloading awscli-1.25.81-py3-none-any.whl (3.9 MB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.9/3.9 MB 38.4 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: sagemaker in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (2.109.0)\n",
+ "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from boto3) (0.6.0)\n",
+ "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/jmespath-1.0.0-py3.9.egg (from boto3) (1.0.0)\n",
+ "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/urllib3-1.26.9-py3.9.egg (from botocore) (1.26.9)\n",
+ "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/python_dateutil-2.8.2-py3.9.egg (from botocore) (2.8.2)\n",
+ "Requirement already satisfied: PyYAML<5.5,>=3.10 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (5.4.1)\n",
+ "Requirement already satisfied: docutils<0.17,>=0.10 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (0.16)\n",
+ "Requirement already satisfied: colorama<0.4.5,>=0.2.5 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (0.4.4)\n",
+ "Requirement already satisfied: rsa<4.8,>=3.1.2 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (4.7.2)\n",
+ "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/importlib_metadata-4.11.3-py3.9.egg (from sagemaker) (4.11.3)\n",
+ "Requirement already satisfied: pathos in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pathos-0.2.8-py3.9.egg (from sagemaker) (0.2.8)\n",
+ "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/protobuf3_to_dict-0.1.5-py3.9.egg (from sagemaker) (0.1.5)\n",
+ "Requirement already satisfied: pandas in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pandas-1.4.2-py3.9-macosx-10.9-x86_64.egg (from sagemaker) (1.4.2)\n",
+ "Requirement already satisfied: numpy<2.0,>=1.9.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from sagemaker) (1.22.4)\n",
+ "Requirement already satisfied: attrs<22,>=20.3.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/attrs-20.3.0-py3.9.egg (from sagemaker) (20.3.0)\n",
+ "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/smdebug_rulesconfig-1.0.1-py3.9.egg (from sagemaker) (1.0.1)\n",
+ "Requirement already satisfied: google-pasta in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/google_pasta-0.2.0-py3.9.egg (from sagemaker) (0.2.0)\n",
+ "Requirement already satisfied: protobuf<4.0,>=3.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from sagemaker) (3.20.1)\n",
+ "Requirement already satisfied: packaging>=20.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/packaging-21.3-py3.9.egg (from sagemaker) (21.3)\n",
+ "Requirement already satisfied: zipp>=0.5 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/zipp-3.7.0-py3.9.egg (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.7.0)\n",
+ "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pyparsing-3.0.7-py3.9.egg (from packaging>=20.0->sagemaker) (3.0.7)\n",
+ "Requirement already satisfied: six in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.15.0)\n",
+ "Requirement already satisfied: pyasn1>=0.1.3 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from rsa<4.8,>=3.1.2->awscli) (0.4.8)\n",
+ "Requirement already satisfied: pytz>=2020.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pytz-2022.1-py3.9.egg (from pandas->sagemaker) (2022.1)\n",
+ "Requirement already satisfied: dill>=0.3.4 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/dill-0.3.4-py3.9.egg (from pathos->sagemaker) (0.3.4)\n",
+ "Requirement already satisfied: multiprocess>=0.70.12 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/multiprocess-0.70.12.2-py3.9.egg (from pathos->sagemaker) (0.70.12.2)\n",
+ "Requirement already satisfied: pox>=0.3.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pox-0.3.0-py3.9.egg (from pathos->sagemaker) (0.3.0)\n",
+ "Requirement already satisfied: ppft>=1.6.6.4 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/ppft-1.6.6.4-py3.9.egg (from pathos->sagemaker) (1.6.6.4)\n",
+ "Installing collected packages: botocore, boto3, awscli\n",
+ " Attempting uninstall: botocore\n",
+ " Found existing installation: botocore 1.27.72\n",
+ " Uninstalling botocore-1.27.72:\n",
+ " Successfully uninstalled botocore-1.27.72\n",
+ " Attempting uninstall: boto3\n",
+ " Found existing installation: boto3 1.24.72\n",
+ " Uninstalling boto3-1.24.72:\n",
+ " Successfully uninstalled boto3-1.24.72\n",
+ " Attempting uninstall: awscli\n",
+ " Found existing installation: awscli 1.25.73\n",
+ " Uninstalling awscli-1.25.73:\n",
+ " Successfully uninstalled awscli-1.25.73\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+ "sagemaker-training 4.2.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Successfully installed awscli-1.25.81 boto3-1.24.80 botocore-1.27.80\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%bash\n",
+ "python3 -m pip install --upgrade boto3 botocore awscli sagemaker"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Restart the notebook kernel "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#import IPython\n",
+ "#IPython.Application.instance().kernel.do_shutdown(True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 3 - Validate SageMaker Python SDK and TensorFlow versions\n",
+ "Ensure the output of the cell below reflects:\n",
+ "\n",
+ "- SageMaker Python SDK version 2.98.0 or above, \n",
+ "- boto3 1.24 or above \n",
+ "- botocore 1.27 or above \n",
+ "- TensorFlow 2.6 or above "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Name: sagemaker\n",
+ "Version: 2.109.0\n",
+ "---\n",
+ "Name: boto3\n",
+ "Version: 1.24.80\n",
+ "---\n",
+ "Name: botocore\n",
+ "Version: 1.27.80\n",
+ "---\n",
+ "Name: tensorflow\n",
+ "Version: 2.8.0\n",
+ "---\n",
+ "Name: protobuf\n",
+ "Version: 3.20.1\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip show sagemaker boto3 botocore tensorflow protobuf |egrep 'Name|Version|---'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import json\n",
+ "import datetime\n",
+ "\n",
+ "import sagemaker\n",
+ "from sagemaker import get_execution_role\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "\n",
+ "sess = sagemaker.Session()\n",
+ "role = get_execution_role()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### C. Run a homogeneous training job\n",
+ "#### Step 1: Set up the training environment\n",
+ "In this step, we define and submit a homogeneous training job. It uses a single instance type (p4d.24xlarge) with 8 GPUs. The analysis of the job will shows that it is CPU bound and therefore its GPUs are underutilized."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "from sagemaker.tensorflow import TensorFlow\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "import os\n",
+ "\n",
+ "hyperparameters = {\n",
+ " \"epochs\": 10,\n",
+ " \"steps_per_epoch\": 500,\n",
+ " \"batch_size\": 1024,\n",
+ " \"tf_data_mode\": \"local\", # We won't be using tf.data.service ('service') for this homogeneous job\n",
+ " \"num_of_data_workers\": 0, # We won't be using tf.data.service ('service') for this homogeneous job\n",
+ "}\n",
+ "\n",
+ "estimator = TensorFlow(\n",
+ " entry_point=\"launcher.py\",\n",
+ " source_dir=\"code\",\n",
+ " framework_version=\"2.9.1\",\n",
+ " py_version=\"py39\",\n",
+ " role=role,\n",
+ " volume_size=10,\n",
+ " max_run=1800, # 30 minutes\n",
+ " disable_profiler=True,\n",
+ " instance_type=\"ml.p4d.24xlarge\",\n",
+ " instance_count=1,\n",
+ " hyperparameters=hyperparameters,\n",
+ " distribution={\n",
+ " \"mpi\": {\n",
+ " \"enabled\": True,\n",
+ " \"processes_per_host\": 8, # 8 GPUs per host\n",
+ " \"custom_mpi_options\": \"--NCCL_DEBUG WARN\",\n",
+ " },\n",
+ " },\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 2: Submit the training job\n",
+ "\n",
+ "Note: For the logs, click on **View logs** from the **Training Jobs** node in **Amazon SageMaker Console**. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2022-09-24 11:28:23 Starting - Starting the training job......\n",
+ "2022-09-24 11:29:08 Starting - Preparing the instances for training........................\n",
+ "2022-09-24 11:33:34 Downloading - Downloading input data\n",
+ "2022-09-24 11:33:34 Training - Downloading the training image..................\n",
+ "2022-09-24 11:37:00 Training - Training image download completed. Training in progress..2022-09-24 11:37:05.792579: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "2022-09-24 11:37:05.801314: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "2022-09-24 11:37:06.269740: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "2022-09-24 11:37:13,412 sagemaker-training-toolkit INFO Imported framework sagemaker_tensorflow_container.training\n",
+ "2022-09-24 11:37:14,075 sagemaker-training-toolkit INFO Installing dependencies from requirements.txt:\n",
+ "/usr/local/bin/python3.9 -m pip install -r requirements.txt\n",
+ "Collecting protobuf==3.20.1\n",
+ "Downloading protobuf-3.20.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB)\n",
+ "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.0/1.0 MB 39.6 MB/s eta 0:00:00\n",
+ "Collecting tensorflow-addons==0.17.0\n",
+ "Downloading tensorflow_addons-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
+ "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 51.6 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: packaging in /usr/local/lib/python3.9/site-packages (from tensorflow-addons==0.17.0->-r requirements.txt (line 2)) (21.3)\n",
+ "Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.9/site-packages (from tensorflow-addons==0.17.0->-r requirements.txt (line 2)) (2.13.3)\n",
+ "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.9/site-packages (from packaging->tensorflow-addons==0.17.0->-r requirements.txt (line 2)) (3.0.9)\n",
+ "Installing collected packages: protobuf, tensorflow-addons\n",
+ "Attempting uninstall: protobuf\n",
+ "Found existing installation: protobuf 3.19.4\n",
+ "Uninstalling protobuf-3.19.4:\n",
+ "Successfully uninstalled protobuf-3.19.4\n",
+ "Attempting uninstall: tensorflow-addons\n",
+ "Found existing installation: tensorflow-addons 0.17.1\n",
+ "Uninstalling tensorflow-addons-0.17.1:\n",
+ "Successfully uninstalled tensorflow-addons-0.17.1\n",
+ "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+ "tf-models-official 2.9.1 requires tensorflow~=2.9.0, which is not installed.\n",
+ "tensorflow-gpu 2.9.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n",
+ "tensorboard 2.9.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n",
+ "sagemaker-training 4.1.4.dev0 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n",
+ "Successfully installed protobuf-3.20.1 tensorflow-addons-0.17.0\n",
+ "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
+ "[notice] A new release of pip available: 22.1.2 -> 22.2.2\n",
+ "[notice] To update, run: pip install --upgrade pip\n",
+ "2022-09-24 11:37:24,079 sagemaker-training-toolkit INFO Waiting for the process to finish and give a return code.\n",
+ "2022-09-24 11:37:24,079 sagemaker-training-toolkit INFO Done waiting for a return code. Received 0 from exiting process.\n",
+ "2022-09-24 11:37:24,258 sagemaker-training-toolkit INFO Starting MPI run as worker node.\n",
+ "2022-09-24 11:37:24,258 sagemaker-training-toolkit INFO Creating SSH daemon.\n",
+ "2022-09-24 11:37:24,274 sagemaker-training-toolkit INFO Waiting for MPI workers to establish their SSH connections\n",
+ "2022-09-24 11:37:24,274 sagemaker-training-toolkit INFO Env Hosts: ['algo-1'] Hosts: ['algo-1:8'] process_per_hosts: 8 num_processes: 8\n",
+ "2022-09-24 11:37:24,276 sagemaker-training-toolkit INFO Network interface name: eth0\n",
+ "2022-09-24 11:37:24,368 sagemaker-training-toolkit INFO Invoking user script\n",
+ "Training Env:\n",
+ "{\n",
+ " \"additional_framework_parameters\": {\n",
+ " \"sagemaker_mpi_custom_mpi_options\": \"--NCCL_DEBUG WARN\",\n",
+ " \"sagemaker_mpi_enabled\": true,\n",
+ " \"sagemaker_mpi_num_of_processes_per_host\": 8\n",
+ " },\n",
+ " \"channel_input_dirs\": {},\n",
+ " \"current_host\": \"algo-1\",\n",
+ " \"current_instance_group\": \"homogeneousCluster\",\n",
+ " \"current_instance_group_hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"distribution_hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"distribution_instance_groups\": [\n",
+ " \"homogeneousCluster\"\n",
+ " ],\n",
+ " \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"hyperparameters\": {\n",
+ " \"batch_size\": 1024,\n",
+ " \"epochs\": 10,\n",
+ " \"model_dir\": \"/opt/ml/model\",\n",
+ " \"num_of_data_workers\": 0,\n",
+ " \"steps_per_epoch\": 500,\n",
+ " \"tf_data_mode\": \"local\"\n",
+ " },\n",
+ " \"input_config_dir\": \"/opt/ml/input/config\",\n",
+ " \"input_data_config\": {},\n",
+ " \"input_dir\": \"/opt/ml/input\",\n",
+ " \"instance_groups\": [\n",
+ " \"homogeneousCluster\"\n",
+ " ],\n",
+ " \"instance_groups_dict\": {\n",
+ " \"homogeneousCluster\": {\n",
+ " \"instance_group_name\": \"homogeneousCluster\",\n",
+ " \"instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ]\n",
+ " }\n",
+ " },\n",
+ " \"is_hetero\": false,\n",
+ " \"is_master\": true,\n",
+ " \"job_name\": \"homogeneous-20220924T112821Z-1\",\n",
+ " \"log_level\": 20,\n",
+ " \"master_hostname\": \"algo-1\",\n",
+ " \"model_dir\": \"/opt/ml/model\",\n",
+ " \"module_dir\": \"s3://sagemaker-us-east-1-331113010199/homogeneous-20220924T112821Z-1/source/sourcedir.tar.gz\",\n",
+ " \"module_name\": \"launcher\",\n",
+ " \"network_interface_name\": \"eth0\",\n",
+ " \"num_cpus\": 96,\n",
+ " \"num_gpus\": 8,\n",
+ " \"output_data_dir\": \"/opt/ml/output/data\",\n",
+ " \"output_dir\": \"/opt/ml/output\",\n",
+ " \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
+ " \"resource_config\": {\n",
+ " \"current_host\": \"algo-1\",\n",
+ " \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"current_group_name\": \"homogeneousCluster\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"instance_groups\": [\n",
+ " {\n",
+ " \"instance_group_name\": \"homogeneousCluster\",\n",
+ " \"instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ]\n",
+ " }\n",
+ " ],\n",
+ " \"network_interface_name\": \"eth0\"\n",
+ " },\n",
+ " \"user_entry_point\": \"launcher.py\"\n",
+ "}\n",
+ "Environment variables:\n",
+ "SM_HOSTS=[\"algo-1\"]\n",
+ "SM_NETWORK_INTERFACE_NAME=eth0\n",
+ "SM_HPS={\"batch_size\":1024,\"epochs\":10,\"model_dir\":\"/opt/ml/model\",\"num_of_data_workers\":0,\"steps_per_epoch\":500,\"tf_data_mode\":\"local\"}\n",
+ "SM_USER_ENTRY_POINT=launcher.py\n",
+ "SM_FRAMEWORK_PARAMS={\"sagemaker_mpi_custom_mpi_options\":\"--NCCL_DEBUG WARN\",\"sagemaker_mpi_enabled\":true,\"sagemaker_mpi_num_of_processes_per_host\":8}\n",
+ "SM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\n",
+ "SM_INPUT_DATA_CONFIG={}\n",
+ "SM_OUTPUT_DATA_DIR=/opt/ml/output/data\n",
+ "SM_CHANNELS=[]\n",
+ "SM_CURRENT_HOST=algo-1\n",
+ "SM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\n",
+ "SM_CURRENT_INSTANCE_GROUP=homogeneousCluster\n",
+ "SM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\n",
+ "SM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\n",
+ "SM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\n",
+ "SM_DISTRIBUTION_INSTANCE_GROUPS=[\"homogeneousCluster\"]\n",
+ "SM_IS_HETERO=false\n",
+ "SM_MODULE_NAME=launcher\n",
+ "SM_LOG_LEVEL=20\n",
+ "SM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\n",
+ "SM_INPUT_DIR=/opt/ml/input\n",
+ "SM_INPUT_CONFIG_DIR=/opt/ml/input/config\n",
+ "SM_OUTPUT_DIR=/opt/ml/output\n",
+ "SM_NUM_CPUS=96\n",
+ "SM_NUM_GPUS=8\n",
+ "SM_MODEL_DIR=/opt/ml/model\n",
+ "SM_MODULE_DIR=s3://sagemaker-us-east-1-331113010199/homogeneous-20220924T112821Z-1/source/sourcedir.tar.gz\n",
+ "SM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_mpi_custom_mpi_options\":\"--NCCL_DEBUG WARN\",\"sagemaker_mpi_enabled\":true,\"sagemaker_mpi_num_of_processes_per_host\":8},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[\"algo-1\"],\"distribution_instance_groups\":[\"homogeneousCluster\"],\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch_size\":1024,\"epochs\":10,\"model_dir\":\"/opt/ml/model\",\"num_of_data_workers\":0,\"steps_per_epoch\":500,\"tf_data_mode\":\"local\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"job_name\":\"homogeneous-20220924T112821Z-1\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-east-1-331113010199/homogeneous-20220924T112821Z-1/source/sourcedir.tar.gz\",\"module_name\":\"launcher\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"launcher.py\"}\n",
+ "SM_USER_ARGS=[\"--batch_size\",\"1024\",\"--epochs\",\"10\",\"--model_dir\",\"/opt/ml/model\",\"--num_of_data_workers\",\"0\",\"--steps_per_epoch\",\"500\",\"--tf_data_mode\",\"local\"]\n",
+ "SM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\n",
+ "SM_HP_BATCH_SIZE=1024\n",
+ "SM_HP_EPOCHS=10\n",
+ "SM_HP_MODEL_DIR=/opt/ml/model\n",
+ "SM_HP_NUM_OF_DATA_WORKERS=0\n",
+ "SM_HP_STEPS_PER_EPOCH=500\n",
+ "SM_HP_TF_DATA_MODE=local\n",
+ "PYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/local/lib/python39.zip:/usr/local/lib/python3.9:/usr/local/lib/python3.9/lib-dynload:/usr/local/lib/python3.9/site-packages:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg:/usr/local/lib/python3.9/site-packages/pyinstrument-3.4.2-py3.9.egg:/usr/local/lib/python3.9/site-packages/pyinstrument_cext-0.2.4-py3.9-linux-x86_64.egg\n",
+ "Invoking script with the following command:\n",
+ "mpirun --host algo-1:8 -np 8 --allow-run-as-root --display-map --tag-output -mca btl_tcp_if_include eth0 -mca oob_tcp_if_include eth0 -mca plm_rsh_no_tree_spawn 1 -bind-to none -map-by slot -mca pml ob1 -mca btl ^openib -mca orte_abort_on_non_zero_status 1 -mca btl_vader_single_copy_mechanism none -x NCCL_MIN_NRINGS=4 -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_DEBUG=WARN -x LD_LIBRARY_PATH -x PATH -x LD_PRELOAD=/usr/local/lib/python3.9/site-packages/gethostname.cpython-39-x86_64-linux-gnu.so -x SM_HOSTS -x SM_NETWORK_INTERFACE_NAME -x SM_HPS -x SM_USER_ENTRY_POINT -x SM_FRAMEWORK_PARAMS -x SM_RESOURCE_CONFIG -x SM_INPUT_DATA_CONFIG -x SM_OUTPUT_DATA_DIR -x SM_CHANNELS -x SM_CURRENT_HOST -x SM_CURRENT_INSTANCE_TYPE -x SM_CURRENT_INSTANCE_GROUP -x SM_CURRENT_INSTANCE_GROUP_HOSTS -x SM_INSTANCE_GROUPS -x SM_INSTANCE_GROUPS_DICT -x SM_DISTRIBUTION_INSTANCE_GROUPS -x SM_IS_HETERO -x SM_MODULE_NAME -x SM_LOG_LEVEL -x SM_FRAMEWORK_MODULE -x SM_INPUT_DIR -x SM_INPUT_CONFIG_DIR -x SM_OUTPUT_DIR -x SM_NUM_CPUS -x SM_NUM_GPUS -x SM_MODEL_DIR -x SM_MODULE_DIR -x SM_TRAINING_ENV -x SM_USER_ARGS -x SM_OUTPUT_INTERMEDIATE_DIR -x SM_HP_BATCH_SIZE -x SM_HP_EPOCHS -x SM_HP_MODEL_DIR -x SM_HP_NUM_OF_DATA_WORKERS -x SM_HP_STEPS_PER_EPOCH -x SM_HP_TF_DATA_MODE -x PYTHONPATH /usr/local/bin/python3.9 -m mpi4py launcher.py --batch_size 1024 --epochs 10 --model_dir /opt/ml/model --num_of_data_workers 0 --steps_per_epoch 500 --tf_data_mode local\n",
+ "Data for JOB [7555,1] offset 0 Total slots allocated 8\n",
+ " ======================== JOB MAP ========================\n",
+ " Data for node: ip-10-0-215-180#011Num slots: 8#011Max slots: 0#011Num procs: 8\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 0 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 1 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 2 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 3 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 4 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 5 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 6 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 7 Bound: N/A\n",
+ " =============================================================\n",
+ "[1,mpirank:1,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:1,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:1,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:4,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:4,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:4,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:5,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:5,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:5,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:7,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:7,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:7,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:0,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:0,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:0,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:6,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:6,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:6,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:3,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:3,algo-1]:current_host=algo-1\n",
+ "[1,mpirank:3,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:2,algo-1]:env.is_hetero=False\n",
+ "[1,mpirank:2,algo-1]:current_host=algo-1[1,mpirank:2,algo-1]:\n",
+ "[1,mpirank:2,algo-1]:Opening process: ['python', './train_dnn.py', '--batch_size', '1024', '--epochs', '10', '--model_dir', '/opt/ml/model', '--num_of_data_workers', '0', '--steps_per_epoch', '500', '--tf_data_mode', 'local']\n",
+ "[1,mpirank:1,algo-1]:2022-09-24 11:37:25.276381: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:2,algo-1]:2022-09-24 11:37:25.276382: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:4,algo-1]:2022-09-24 11:37:25.276384: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:1,algo-1]:2022-09-24 11:37:25.276524: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:2,algo-1]:2022-09-24 11:37:25.276524: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:4,algo-1]:2022-09-24 11:37:25.276524: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:0,algo-1]:2022-09-24 11:37:25.290991: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:7,algo-1]:2022-09-24 11:37:25.290987: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:3,algo-1]:2022-09-24 11:37:25.290990: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:5,algo-1]:2022-09-24 11:37:25.290991: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:6,algo-1]:2022-09-24 11:37:25.290990: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:0,algo-1]:2022-09-24 11:37:25.291121: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:7,algo-1]:2022-09-24 11:37:25.291122: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:3,algo-1]:2022-09-24 11:37:25.291124: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:5,algo-1]:2022-09-24 11:37:25.291124: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:6,algo-1]:2022-09-24 11:37:25.291121: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "[1,mpirank:4,algo-1]:2022-09-24 11:37:25.310966: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:2,algo-1]:2022-09-24 11:37:25.310966: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:1,algo-1]:2022-09-24 11:37:25.310966: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:0,algo-1]:2022-09-24 11:37:25.325878: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:7,algo-1]:2022-09-24 11:37:25.325873: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:3,algo-1]:2022-09-24 11:37:25.325878: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:6,algo-1]:2022-09-24 11:37:25.326012: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:5,algo-1]:2022-09-24 11:37:25.326064: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "[1,mpirank:6,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:0,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:0,algo-1]:hvd.local_rank() 0\n",
+ "[1,mpirank:1,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:6,algo-1]:hvd.local_rank() 6\n",
+ "[1,mpirank:1,algo-1]:hvd.local_rank() 1\n",
+ "[1,mpirank:5,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')][1,mpirank:5,algo-1]:\n",
+ "[1,mpirank:5,algo-1]:hvd.local_rank() 5\n",
+ "[1,mpirank:2,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:3,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:4,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:2,algo-1]:hvd.local_rank() 2\n",
+ "[1,mpirank:7,algo-1]:[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:2', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:3', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:4', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:5', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:6', device_type='GPU'), PhysicalDevice(name='/physical_device:GPU:7', device_type='GPU')]\n",
+ "[1,mpirank:3,algo-1]:hvd.local_rank() 3\n",
+ "[1,mpirank:4,algo-1]:hvd.local_rank() 4\n",
+ "[1,mpirank:7,algo-1]:hvd.local_rank() 7\n",
+ "[1,mpirank:3,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:6,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:5,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:0,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:7,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:2,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:1,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:4,algo-1]:Running in local tf_data_mode.\n",
+ "[1,mpirank:1,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:4,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:2,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:3,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:5,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:7,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:0,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:6,algo-1]:Epoch 1/10\n",
+ "[1,mpirank:5,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:5,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:5,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:0,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:0,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:0,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:7,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:7,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:7,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:3,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:3,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:3,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:1,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:1,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:1,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:2,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:2,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:2,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:4,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:4,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:4,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:6,algo-1]:Extension horovod.torch has not been built: /usr/local/lib/python3.9/site-packages/horovod/torch/mpi_lib/_mpi_lib.cpython-39-x86_64-linux-gnu.so not found\n",
+ "[1,mpirank:6,algo-1]:If this is not expected, reinstall Horovod with HOROVOD_WITH_PYTORCH=1 to debug the build error.\n",
+ "[1,mpirank:6,algo-1]:Warning! MPI libs are missing, but python applications are still available.\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.366 algo-1:177 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.366 algo-1:178 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.366 algo-1:179 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.366 algo-1:181 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.366 algo-1:183 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.366 algo-1:184 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.366 algo-1:182 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.366 algo-1:180 INFO utils.py:27] RULE_JOB_STOP_SIGNAL_FILENAME: None\n",
+ "[1,mpirank:1,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:2,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:4,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:5,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:6,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:0,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:3,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:1,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:2,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:5,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:4,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:6,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:7,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:0,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:3,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:7,algo-1]:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg/smdebug/profiler/system_metrics_reader.py:63: SyntaxWarning: \"is not\" with a literal. Did you mean \"!=\"?\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.581 algo-1:177 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.581 algo-1:184 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.582 algo-1:181 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.582 algo-1:179 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.582 algo-1:180 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.582 algo-1:182 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.582 algo-1:183 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.582 algo-1:178 INFO profiler_config_parser.py:111] Unable to find config at /opt/ml/input/config/profilerconfig.json. Profiler is disabled.\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.639 algo-1:184 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.639 algo-1:177 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.639 algo-1:178 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.639 algo-1:183 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.639 algo-1:180 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.639 algo-1:182 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.639 algo-1:181 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.639 algo-1:179 INFO json_config.py:91] Creating hook from json_config at /opt/ml/input/config/debughookconfig.json.\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.640 algo-1:177 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.640 algo-1:178 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.640 algo-1:184 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.640 algo-1:183 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.640 algo-1:180 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.640 algo-1:182 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.640 algo-1:181 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.640 algo-1:184 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.640 algo-1:177 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.640 algo-1:178 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.640 algo-1:184 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.640 algo-1:178 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.640 algo-1:177 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.640 algo-1:179 INFO hook.py:201] tensorboard_dir has not been set for the hook. SMDebug will not be exporting tensorboard summaries.\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.640 algo-1:183 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.640 algo-1:183 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:2,algo-1]:[2022-09-24 11:37:33.640 algo-1:184 INFO hook.py:421] Monitoring the collections: losses, sm_metrics, metrics\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.640 algo-1:180 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.640 algo-1:182 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:1,algo-1]:[2022-09-24 11:37:33.640 algo-1:177 INFO hook.py:421] Monitoring the collections: losses, metrics, sm_metrics\n",
+ "[1,mpirank:4,algo-1]:[2022-09-24 11:37:33.640 algo-1:178 INFO hook.py:421] Monitoring the collections: losses, metrics, sm_metrics\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.640 algo-1:181 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.640 algo-1:180 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.640 algo-1:182 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.641 algo-1:181 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:3,algo-1]:[2022-09-24 11:37:33.641 algo-1:183 INFO hook.py:421] Monitoring the collections: losses, sm_metrics, metrics\n",
+ "[1,mpirank:6,algo-1]:[2022-09-24 11:37:33.641 algo-1:182 INFO hook.py:421] Monitoring the collections: sm_metrics, metrics, losses\n",
+ "[1,mpirank:7,algo-1]:[2022-09-24 11:37:33.641 algo-1:180 INFO hook.py:421] Monitoring the collections: sm_metrics, losses, metrics\n",
+ "[1,mpirank:0,algo-1]:[2022-09-24 11:37:33.641 algo-1:181 INFO hook.py:421] Monitoring the collections: sm_metrics, metrics, losses\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.641 algo-1:179 INFO hook.py:254] Saving to /opt/ml/output/tensors\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.641 algo-1:179 INFO state_store.py:77] The checkpoint config file /opt/ml/input/config/checkpointconfig.json does not exist.\n",
+ "[1,mpirank:5,algo-1]:[2022-09-24 11:37:33.641 algo-1:179 INFO hook.py:421] Monitoring the collections: metrics, losses, sm_metrics\n",
+ "[1,mpirank:0,algo-1]:NCCL version 2.10.3+cuda11.2\n",
+ "[1,mpirank:2,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2246s vs `on_train_batch_end` time: 0.6465s). Check your callbacks.\n",
+ "[1,mpirank:2,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2246s vs `on_train_batch_end` time: 0.6465s). Check your callbacks.\n",
+ "[1,mpirank:3,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2247s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:3,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2247s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:0,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2237s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:0,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2237s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:5,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2247s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:5,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2247s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:7,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2241s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:6,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2247s vs `on_train_batch_end` time: 0.6465s). Check your callbacks.\n",
+ "[1,mpirank:7,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2241s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:6,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2247s vs `on_train_batch_end` time: 0.6465s). Check your callbacks.\n",
+ "[1,mpirank:1,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2255s vs `on_train_batch_end` time: 0.6463s). Check your callbacks.\n",
+ "[1,mpirank:1,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2255s vs `on_train_batch_end` time: 0.6463s). Check your callbacks.\n",
+ "[1,mpirank:4,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2250s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:4,algo-1]:WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.2250s vs `on_train_batch_end` time: 0.6464s). Check your callbacks.\n",
+ "[1,mpirank:7,algo-1]:500/500 - 121s - loss: 2.4081 - lr: 0.0033 - 121s/epoch - 242ms/step\n",
+ "[1,mpirank:1,algo-1]
+
+
+## Instructions to download the dateset
+You can use SageMaker Data Wrangler to import data from the following data sources: Amazon Simple Storage Service (Amazon S3), Amazon Athena, Amazon Redshift, and Snowflake. The dataset that you import can include up to 1000 columns. For this lab, we will be using Amazon S3 as the preferred data source. Before we import the dataset into SageMaker Data Wrangler, let's ensure we copy the dataset first from the publicly hosted location to our local S3 bucket in our own account.
+
+To copy the dataset, copy and execute the Python code below within SageMaker Studio. It is recommended to execute this code in a notebook setting. We also recommend to have your S3 bucket in the same region as SageMaker Data Wrangler.
+
+To create a SageMaker Studio notebook, from the launcher page, click on the ***Notebook Python 3*** options under ***Notebooks and compute resources*** as show in the figure below.
+
+
+
+Copy and paste the shared code snippet below into the launched notebook's cell (shown below) and execute it by clicking on the play icon on the top bar.
+
+
+
+
+```python
+import boto3
+import json
+
+
+# Setup
+REGION = 'us-east-1'
+account_id = boto3.client('sts').get_caller_identity().get('Account')
+bucket_name = f'{account_id}-{REGION}-dw-ts-lab'
+
+
+# Create S3 bucket to download dataset in your account
+s3 = boto3.resource('s3')
+
+if REGION == 'us-east-1':
+ s3.create_bucket(Bucket=bucket_name)
+else:
+ s3.create_bucket(Bucket=bucket_name,CreateBucketConfiguration={'LocationConstraint': REGION})
+ # Copy dataset from public hosted location to your S3 bucket
+trips = [
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-02.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-03.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-04.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-05.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-06.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-07.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-08.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-09.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-10.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-11.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2019-12.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2020-01.parquet'},
+ {'Bucket': 'nyc-tlc', 'Key': 'trip data/yellow_tripdata_2020-02.parquet'}
+]
+for trip in trips:
+ s3.meta.client.copy(trip, bucket_name, trip['Key'])
+```
+
+Now we have raw data in our S3 bucket and ready to explore it and build a training dataset
+
+## Dataset Import
+
+Our first step is to launch a new SageMaker Data Wrangler session and there are multiple ways how to do that. For example, use the following:
+
+Click File -> New -> Data Wrangler Flow
+
+
+
+Amazon SageMaker will start to provision a resources for you and you a could find a new Data Wrangler Flow file in a File Browser section
+
+
+
+Lets rename our new workflow: Right click on file -> Rename Data Wrangler Flow
+
+
+
+Put a new name, for example: `TS-Workshop-DataPreparation.flow`
+
+
+
+In few minutes Data Wrangler will finish to provision resources and you could see "Import Data" screen.
+SageMaker Data Wrangler supports many data sources: Amazon S3, Amazon Athena, Amazon Redshift, Snowflake, Databricks.
+Our data already in S3, let's import it by clicking "Amazon S3" button.
+
+
+
+
+You will see all your S3 buckets so please search for your bucket (if you used provided code the bucket will have a suffix dw-ts-lab)
+
+
+
+All the files required for this lab are in "trip data" folder, so let's select it. SageMaker Data Wrangler will import all files from a folder and sample up to 100 MB of data for an interactive preview. On a right side menu you could customize import job settings like Name, File type, Delimiter, etc. More information about import process could be found [here](https://docs.aws.amazon.com/sagemaker/latest/dg/data-wrangler-import.html).
+
+
+To finish setting up import step select "parquet" in "File type" drop down menu and press the orange button "Import"
+
+
+
+
+It will take a few minutes to import data and validate it. SageMaker Data Wrangler will automatically recognize data types. You should see "Validation complete 0 errors message"
+
+
+
+
+# Change data types
+
+First we will check the data types were correctly recognized. This might be necessary as Data Wrangler selects data types based on a sampled data which is limited to 50000 rows. Sampled data might potentially miss some variations.
+
+
+To add a data transformation step use the plus sign next to Data types and choose Edit data types as shown below.
+
+
+
+
+In our case several columns were incorrectly recognized:
+- `passenger_count` (must be `long` instead of `float`)
+- `RatecodeID` (must be `long` instead of `float`)
+- `airport_fee` (must be `float` instead of `long`)
+
+I know correct data types from dataset description. In real life you could also easily find such information. Let's correct data types by selecting a correct type from a drop down menu.
+
+
+
+
+Click Preview and then Apply button.
+
+Click Back to data flow.
+
+# Dataset preparation
+## Drop columns
+
+Before we analyze data and do feature engineering we have to clean dataset and below steps show how to remove unwanted data.
+
+To re-iterate our business goal:
+Predict the number of NY City yellow taxi pickups in the next 24 hour for each pickup per hour zones and provide some insights for drivers like average tips, average distance, etc.
+
+As we are interested in per hour forecast we have to aggregate some features and remove features which are impossible to aggregate. For this purpose we don't need the following columns:
+
+1. `VendorID` (A code indicating the TPEP provider that provided the record)
+2. `RatecodeID` (The final rate code in effect at the end of the trip)
+3. `Store_and_fwd_flag` (This flag indicates whether the trip record was held in vehicle memory before sending to the vendor, aka “store and forward,”because the vehicle did not have a connection to the server)
+4. `DOLocationID` (TLC Taxi Zone in which the taximeter was disengaged)
+5. `Payment_type` (A numeric code signifying how the passenger paid for the trip)
+6. `Fare_amount` (The time-and-distance fare calculated by the meter) - we will use total amount feature
+7. `Extra` (Miscellaneous extras and surcharges)
+8. `MTA_tax` (0.50 MTA tax that is automatically triggered based on the metered rate in use)
+9. `Tolls_amount` (Total amount of all tolls paid in trip)
+10. `Improvement_surcharge` (improvement surcharge assessed trips at the flag drop. The improvement surcharge began being levied in 2015)
+11. `Passenger_count` (This is a driver-entered value)
+12. `congestion_surcharge` (Total amount collected in trip for NYS congestion surcharge)
+13. `Airport_fee` (Only at LaGuardia and John F. Kennedy Airports)
+
+To remove those columns:
+1. Click the plus sign next to "Data types" element and choose Add transform.
+
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+
+3. Choose Manage columns. \
+
+
+
+
+4. For Transform, choose Drop column and for Column to drop, choose all mentioned above.
+
+
+
+
+5. Choose Preview
+6. Choose Add to save the step.
+
+Once transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset like show here.
+
+
+
+Click Back to data flow.
+
+## Handle missing and invalid data in timestamps
+Missing data is a common problem in real life, it could be a result of data corruption, data loss or issues in data ingestion. The best practice is to verify the presence of any missing or invalid values and handle them appropriately.
+
+There are many different strategies how missing or invalid data could be handled, for example dropping rows with missing values or filling the missing values with static or calculated values. Depending on dataset size you could choose what to do: fix values or just drop them. The **Time Series - Handle missing** transform allows you to choose from multiple strategies.
+
+All future aggregations will be based on time stamps, so we have to make sure that we don't have any rows with missing time stamps ( `tpep_pickup_datetime` and `tpep_dropoff_datetime` features). SageMaker Data Wrangler has several time series specific transformations, including **Validate timestamps** which checks for scenarios:
+1. Checking timestamp column for any missing values.
+2. Validate the timestamp columns for the desired timestamp format.
+
+To validate timestamps in `tpep_dropoff_datetime` and `tpep_pickup_datetime` columns:
+1. Click the plus sign next to "Drop columns" element and choose Add transform.
+
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series.
+
+
+
+4. For Transform choose Validate Timestamps, For TimeStamp columns choose `tpep_pickup_datetime`, for Policy select drop.
+
+
+
+5. Choose Preview
+6. Choose Add to save the step.
+7. Repeat same steps again for `tpep_dropoff_datetime` column
+
+When you apply a transformation a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+Click Back to data flow.
+
+
+## Feature engineering based on a timestamp with a custom transformation.
+
+At this stage we have pickup and drop-off timestamps, but we are more interested in pickup timestamp and ride duration. We have to create a new feature `ride duration` as a difference between pick up and drop off time in minutes. There is no built-in date difference transformation in a Data Wrangler, but we could create it with a custom transformation. The **Custom Transforms** allows you to use Pyspark, Pandas, or Pyspark (SQL) to define your own transformations. For all three options, you use the variable `df` to access a dataframe to which you want to apply the transform. You do not need to include a return statement.
+
+To create a custom transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom transform.
+
+
+
+4. Name the transformation as "Duration_Transformation" - (naming is optional but good to have a structure)
+5. In drop down menu select Python (PySpark) and use code below. This code will import functions, calculate difference between two timestamps by converting them to unix format (real number) and round result and drop tpep_dropoff_datetime column
+
+ ```Python
+ from pyspark.sql.functions import col, round
+ df = df.withColumn('duration', round((col("tpep_dropoff_datetime").cast("long")-col("tpep_pickup_datetime").cast("long"))/60,2))
+ df = df.drop("tpep_dropoff_datetime")
+ ```
+
+
+6. Choose Preview
+7. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset with a new column duration and without column tpep_dropoff_datetime
+
+
+
+Click Back to data flow.
+
+## Handling missing data in numeric attributes
+
+We already discussed what are missing values and why it is important to handle them. So far, we have been working with timestamps only. Now, we are going to handle missing values in the rest of attributes. We can exclude `duration` feature from this operation as it was calculated from timestamps in the previous step. As we discussed before, there are several ways to handle missing data: fill a static number or calculate a value (for example: median or mean for last 7 days). It might make sense to calculate a value if your time-series represents a continuous process like sensor reading or product sale quantity. In our case, all trips are independent from each other and we cannot calculate values based on previous trips as it might bring data bias and increase error. We can replace missing values with zeros or sometimes it might make sense to drop the entire row with missing values.
+
+Amazon Data Wrangler has two types of transformations to handle missing data: i) generic and ii) specifically designed for time series data. Here, we demonstrate how to use both of them and describe when to use each of these transformations.
+
+### Handle missing data with the generic "Handle missing values" transformations
+
+This transformation can be used if you want to:
+1. Replace missing values with a same static value for all time series
+2. Replace missing values with a calculated value and you have only one time series (for example: one sensor or one product in a shop)
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Handle Missing
+
+
+
+4. For "Transform" choose Fill missing
+
+5. For "inputs columns" choose `PULocationID`, `tip_amount`, and `total_amount`
+
+6. For "Fill value" put 0
+
+
+
+7. Choose Preview
+8. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+### Handle missing data with special Time Series transformation
+
+In real life datasets, we have many time-series in the same dataset and to separate them, we use some form of IDs. For example, sensor ID or item SKU. If we want to replace missing values with calculated values, for example mean for last 10 sensor observations, we must calculate it based on data for each time series independently. Instead of writing code, you could use the special Time Series transformation in Data Wrangler and get this easily done!.
+
+To create this transformation you have to:
+1. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+
+2. Choose Time Series
+
+
+
+3. For "Transform" choose Handle missing
+
+4. For "Time series input type" choose Along column
+
+5. For "Impute missing values for this column" choose `trip_distance`
+
+6. For "Timestamp column" choose tpep_pickup_datetime
+
+7. For "ID column" choose PULocationID
+
+8. For "Method for imputing values" choose Constant value
+
+9. For "Custom value" put 0.0
+
+
+
+10. Choose Preview
+11. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+### Filter rows with invalid data
+
+Based on our understanding of the dataset until this point, we could also apply several filters to remove invalid or corrupt data from a business point of view. This will improve data quality even further and ensure we feed only correct data to our model training process.
+
+We can filter data based on following rules:
+1. `tpep_pickup_datetime` - have to be in range from 1 Jan 2019 (included) till 1 March 2020 (excluded)
+2. `trip_distance` - have to be greater than or equal to 0 (only positive numbers)
+3. `tip_amount` - have to be greater than or equal to 0 (only positive numbers)
+4. `total_amount` - have to be greater than or equal to 0 (only positive numbers)
+5. `duration` - have to be greater than or equal to 1 (we are not interested in super short trips).
+6. `PULocationID` - have to be in the range (1 to 263). These are the assigned zones. For the sake of brevity, let's use only the 1st ten location IDs for this workshop (see image below).
+
+
+
+There is no built-in filter transformation in Data Wrangler to handle these various constraints. Hence, we will create a custom transformation.
+
+To create a custom transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom Transform.
+
+
+
+4. In drop down menu select Python (PySpark) and use code below. This code will filter rows based on the specified conditions.
+
+ ```Python
+ df = df.filter(df.trip_distance >= 0)
+ df = df.filter(df.tip_amount >= 0)
+ df = df.filter(df.total_amount >= 0)
+ df = df.filter(df.duration >= 1)
+ df = df.filter((1 <= df.PULocationID) & (df.PULocationID <= 263))
+ df = df.filter((df.tpep_pickup_datetime >= "2019-01-01 00:00:00") & (df.tpep_pickup_datetime < "2020-03-01 00:00:00"))
+ ```
+
+
+
+
+5. Choose Preview
+6. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+## Quick analysis of dataset
+
+Amazon SageMaker Data Wrangler includes built-in analysis that help you generate visualizations and data insights in a few clicks. You can either leverage the built-in analyses types we offer out of the box with the product or create your own custom analysis using your own code if needed. SageMaker Data Wrangler also provides automated insights by automatically performing exploratory and descriptive analyses behind the scenes on your data. It identifies hidden anomalies and red flags within your dataset and proposes prescriptive actions in the form of what transforms can be applied on what columns of your data to fix these issues.
+
+For this lab, let's use the Table Summary built-in analysis type to quickly summarize our existing dataset in its current form. For the numeric columns, including long and float data, table summary reports the number of entries (`count`), minimum (`min`), maximum (`max`), mean, and standard deviation (`stddev`) for each column. For columns with non-numerical data, including columns with String, Boolean, or DateTime data, table summary reports the number of entries (`count`), least frequent value (`min`), and most frequent value (`max`).
+
+To create this analysis, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose "Add analyses".
+
+
+
+2. In a "analyses type" drop down menu select "Table Summary" and provide a name for "Analysis name", for example: "Cleaned dataset summary"
+
+
+
+3. Choose Preview
+
+4. Choose Add to save the analyses.
+
+5. You could find your first analyses on a "Analysis" tab. All future visualizations will could be also found here.
+
+
+
+6. Click on analyses icon to open it.
+
+Let's take a look at our results. The most interesting part is the summary for duration column: maximum value is 1439 and this is in minutes! 1439 minutes = almost 24 hours and this is definitely an issue which will reduce the quality of our model if this dataset is used in its current form. This looks more like an issue due to the prevalence of outliers in our dataset. Next, let's see how to issue this issue using a built-in transform Data Wrangler offers.
+
+
+
+## Handling outliers in numeric attributes
+
+In statistics, an outlier is a data point that differs significantly from other observations in the same dataset. An outlier may be due to variability in the measurement or it may indicate experimental error. The latter are sometimes excluded from the dataset. For example, in our dataset we have the `tip_amount` feature and usually it is less than 10 dollars, but due to an error in a data collection, some values can show thousands of dollar as a tip. Such data errors will skew statistics and aggregated values which will lead to a lower model accuracy.
+
+An outlier can cause serious problems in statistical analysis. Machine learning models are sensitive to the distribution and range of feature values. Outliers, or rare values, can negatively impact model accuracy and lead to longer training times. When you define a Handle outliers transform step, the statistics used to detect outliers are generated on the data available in Data Wrangler when defining this step. These same statistics are used when running a Data Wrangler job.
+
+SageMaker Data Wrangler supports several outliers detection and handle methods. We are going to use **Standard Deviation Numeric Outliers** and we remove all outliers as our dataset is big enough. This transform detects and fixes outliers in numeric features using the mean and standard deviation. You specify the number of standard deviations a value must vary from the mean to be considered an outlier. For example, if you specify 3 for standard deviations, a value falling more than 3 standard deviations from the mean is considered an outlier.
+
+To create this transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose "Add transform".
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Handle Outliers.
+
+
+
+4. For "Transform" choose "Standard deviation numeric outliers"
+5. For "Inputs columns" choose `tip_amount`, `total_amount`, `duration`, and `trip_distance`
+6. For "Fix method" choose "Remove"
+7. For "Standard deviations" put 4
+
+
+
+8. Choose Preview
+9. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of resulted dataset.
+
+
+
+
+Optional: If you want, you could repeat the steps from our previous analysis ("Quick analysis of a current dataset") to create a new table summary and check for the new maximum for the `duration` column. You can see, the new max value for duration is 243 minutes = just over an hour. This is more realistic for long trips than what we previously had.
+
+
+
+## Grouping/Aggregating data
+At this moment we have cleaned dataset by removing outliers, invalid values, and added new features. There are few more steps before we start training our forecasting model.
+
+As we are interested in a hourly forecast we have to count number of trips per hour per station and also aggregate (with mean) all metrics such as distance, duration, tip, total amount.
+
+### Truncating timestamp
+We don't need minutes and seconds in out timestamp, so we remove them. There is no built-in filter transformation in SageMaker Data Wrangler, so we create a custom transformation.
+
+To create a custom transformation, follow the steps below::
+1. Click the plus sign next to a collection of transformation elements and choose "Add transform".
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom Transform.
+
+
+
+4. In drop down menu select Python (PySpark) and use code below. This code will create a new column with a truncated timestamp and then drop original pickup column.
+
+ ```Python
+ from pyspark.sql.functions import col, date_trunc
+ df = df.withColumn('pickup_time', date_trunc("hour",col("tpep_pickup_datetime")))
+ df = df.drop("tpep_pickup_datetime")
+ ```
+
+
+
+5. Choose Preview
+
+6. Choose Add to save the step
+
+When you apply the transformation on sampled data, you can see all the current steps until this point in time and get a preview of the resulting dataset with a new column `pickup_time` and without the old column `tpep_pickup_datetime`
+
+
+
+### Count number of trips per hour per station
+Currently, we have only piece of information about each trip, but we don't know how many trips were made from each station per hour. The simplest way to do that is count number of records per stationID per hourly timestamp. While Amazon Data Wrangler provides GroupBy transformation. The built-in transformation doesn't support grouping by multiple columns, so we use a custom transformation.
+
+To create a custom transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose "Add transform".
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Custom Transform.
+
+
+
+4. In drop down menu select Python (PySpark) and use code below. This code will create a new column with a number of trips from each location for each timestamp.
+
+ ```Python
+ from pyspark.sql import functions as f
+ from pyspark.sql import Window
+ df = df.withColumn('count', f.count('duration').over(Window.partitionBy([f.col("pickup_time"), f.col("PULocationID")])))
+ ```
+
+
+
+5. Choose Preview
+6. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset with a new column count.
+
+
+
+## Resample time series
+Now, we are ready to make a final aggregation! We want to aggregate all rows by a combination of `PULocationID` and `pickup_time` columns, while features should be replaced by mean value for each combination.
+
+We use special built-in Time Series transformation **Resample**. The Resample transformation changes the frequency of the time series observations to a specified granularity. It also comes with both upsampling and downsampling options. Applying upsampling increases the frequency of the observations, for example from daily to hourly, whereas downsampling decreases the frequency of the observations, for example from hourly to daily.
+
+To create this transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+
+3. Choose Time Series.
+
+
+
+4. For "Transform" choose "Resample"
+5. For "Timestamp" choose `pickup_time`
+6. For "ID column" choose `PULocationID`
+7. For "Frequency unit" choose "Hourly"
+8. For "Frequency quantity" put 1
+9. For "Method to aggregate numeric values" choose "mean"
+10. Use default values for the rest of parameters
+
+
+
+11. Choose Preview
+12. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+## Resample time series
+Now we are ready to make a final aggregation! We aggregate all rows by combination of `PULocationID` and `pickup_time` timestamp while features should be replaced by mean value for each combination.
+
+We use special built-in Time Series transformation **Resample**. The Resample transformation changes the frequency of the time series observations to a specified granularity. It also comes with both upsampling and downsampling options. Applying upsampling increases the frequency of the observations, for example from daily to hourly, whereas downsampling decreases the frequency of the observations, for example from hourly to daily.
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series.
+
+
+
+4. For "Transform" choose "Resample"
+5. For "Timestamp" choose pickup_time
+6. For "ID column" choose "PULocationID"
+7. For "Frequency unit" choose "Hourly"
+8. For "Frequency quantity" put 1
+9. For "Method to aggregate numeric values" choose "mean"
+10. Use default values for the rest of parameters
+
+
+
+11. Choose Preview
+12. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+[here](https://docs.aws.amazon.com/sagemaker/latest/dg/data-wrangler-import.html).
+
+## Featurize Date Time
+
+"Featurize datetime" time series transformation will add the month, day of the month, day of the year, week of the year, hour and quarter features to our dataset. Because we’re providing the date/time components as separate features, we enable ML algorithms to detect signals and patterns for improving prediction accuracy.
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu
+
+
+
+3. Choose Time Series
+
+
+
+ * For "Transform" choose "Featurize date/time"
+ * For "Input Column" choose `pickup_time`
+ * For "Output Column" enter "date"
+ * For "Output mode" choose "Ordinal"
+ * For "Output format" choose "Columns"
+ * For date/time features to extract, select Year, Month, Day, Hour, Week of year, Day of year, and Quarter.
+
+
+
+4. Choose Preview
+5. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+Click "Back to data flow" to head back to the block diagram editor window.
+
+## Lag feature
+Next let’s create lag features for the target column count. Lag features in time-series analysis are values at prior timestamps that are considered helpful in inferring future values. They also help identify **autocorrelation**, also known as serial correlation, patterns in the residual series by quantifying the relationship of the observation with observations at previous time steps. Autocorrelation is similar to regular correlation but between the values in a series and its past values. It forms the basis for the **autoregressive forecasting** models in the **ARIMA** series.
+
+With SageMaker Data Wrangler's Lag feature transform, you can easily create lag features n periods apart. Additionally, we often want to create multiple lag features at different lags and let the model decide the most meaningful features. For such a scenario, the **Lag features** transform helps create multiple lag columns over a specified window size.
+
+To create this transformation, follow the steps below:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform.
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series
+
+
+
+ * For "Transform" choose "Lag features"
+ * For "Generate lag features for this column" choose "count"
+ * For "ID column" enter "PULocationID"
+ * For "Timestamp Column" choose "pickup_time"
+ * For Lag, enter 8. You could try to use different values, maybe 24 hours in our case makes more sense.
+ * Because we’re interested in observing up to the previous 8 lag values, let’s select Include the entire lag window.
+ * To create a new column for each lag value, select Flatten the output
+
+
+
+4. Choose Preview
+5. Choose Add to save the step.
+
+When transformation is applied on a sampled data you should see all current steps and a preview of a resulted dataset.
+
+
+
+## Rolling window features
+We can also calculate meaningful statistical summaries across a range of values and include them as input features. Let’s extract common statistical time series features.
+
+Data Wrangler implements automatic time series feature extraction capabilities using the open source `tsfresh` package. With the time series feature extraction transforms, you can automate the feature extraction process. This eliminates the time and effort otherwise spent manually implementing signal processing libraries. We will extract features using the **Rolling window** features transform. This method computes statistical properties across a set of observations defined by the window size.
+
+To create this transformation you have to:
+1. Click the plus sign next to a collection of transformation elements and choose Add transform
+
+2. Click "+ Add step" orange button in the TRANSFORMS menu.
+
+
+
+3. Choose Time Series
+
+
+
+ * For "Transform" choose "Rolling window features"
+ * For "Generate rolling window features for this column" choose "count"
+ * For "Timestamp Column" choose "pickup_time"
+ * For "ID column" enter `PULocationID`
+ * For "Window size", enter 8. You could try to use different values, maybe 24 hours in our case makes more sense.
+ * Select Flatten to create a new column for each computed feature.
+ * Choose "Strategy" as "Minimal subset". This strategy extracts eight features that are useful in downstream analyses. Other strategies include Efficient Subset, Custom subset, and All features.
+
+
+
+4. Choose Preview
+5. Choose Add to save the step.
+
+When this transformation is applied on the dataset, you can see all current steps until this point in time and get a preview of the resulting dataset.
+
+
+
+Click "Back to data flow" to head back to the block diagram editor window.
+
+## Export Data
+At this stage, we have a new dataset that is cleaned and transformed with newly engineered features. This dataset can be used for forecasting either using open source libraries/frameworks or AWS services like [Amazon SageMaker Autopilot](https://aws.amazon.com/sagemaker/autopilot/), [Amazon SageMaker Canvas](https://aws.amazon.com/sagemaker/canvas/) or [Amazon Forecast](https://aws.amazon.com/forecast/).
+
+Given, we had only used a sample of the dataset for creating our data preparation and transformation recipe so far, what need to do next is to apply the same recipe (data flow) on our entire dataset and scale the whole process in a distributed fashion. Amazon Data Wrangler let's you do this in multiple ways. You can export your data flow: 1/ as a processing job, 2/ as a SageMaker pipeline step, or 3/ as a Python script. You can also kick-off these distributed jobs via the UI without writing any code using Data Wrangler's destination node option. The export options are also facilitated via SageMaker Studio notebooks (Jupyter). Additionally, the transformed features can also be ingested directly to SageMaker Feature Store.
+
+For this lab, we will see how to use the destination nodes option to export the transformed features to S3 via a distributed PySpark job powered by SageMaker Processing.
+
+### Exporting to S3 using Destination Nodes
+This option creates a SageMaker processing job which uses the data flow (recipe) we have created previously to kick-off a distributed processing job on the "entire" dataset saving the results to a specified S3 bucket.
+
+Additionally, you can also drop columns if needed right before the export step. For the sake of brevity and to simplify the prediction problem statement, let's drop all columns except three columns `pickup_time`, `count`, `PULocationID`. Here count is the target variable we will try to predict. `pickup_time` and `PULocationID` will be our feature columns used for modeling. To create the model, we will be using SageMaker Autopilot. This will be covered in the next 2 sections.
+
+Follow the next steps to setup export to S3.
+1. Click the plus sign next to a collection of transformation elements and choose **"Add destination" -> "Amazon S3"**
+
+
+
+2. Provide parameters for S3 destination:
+ * Dataset name - name for new dataset, for example used "NYC_export"
+ * File type - CSV
+ * Delimiter - Comma
+ * Compression - none
+ * Amazon S3 location - You can use the same bucket name which we created at the beginning
+
+3. Click "Add destination" orange button
+
+
+
+4. Now your dataflow has a final step and you see a new "Create job" orange button. Click it.
+
+
+
+5. Provide a "Job name" or keep autogenerated option and select "destination". We have only one "S3:NYC_export", but you might have multiple destinations from different steps in your workflow. Leave a "KMS key ARN" field empty and click "Next" orange button.
+
+
+
+6. Now your have to provide configuration for a compute capacity for a job. You can keep all defaults values:
+ * For Instance type use "ml.m5.4xlarge"
+ * For Instance count use "2"
+ * You can explore "Additional configuration", but keep them without change.
+ * Click "Run" orange button
+
+
+
+7. Now your job is started and it takes about 1 hour to process 6 GB of data according to our Data Wrangler processing flow. Cost for this job will be around 2 USD as "ml.m5.4xlarge" cost 0.922 USD per hour and we are using two of them.
+
+
+
+8. If you click on the job name you will be redirected to a new window with the job details. On the job details page you see all parameters from a previous steps.
+
+
+
+Approximately in one hour you should see that job status changed to "Completed" and you could also check "Processing time (seconds)" value.
+
+
+
+Now you could close job details page.
+
+## Check Processed output
+After the SageMaker Data Wrangler processing job is completed, we can check the results saved in our destination S3 bucket.
+
+At this stage, you have designed a data flow for data processing and feature engineering and successfully launched it. Of course it is not mandatory to always run a job by clicking on the "Run" button. You could also automate it, but this is a topic of another workshop in this series!
+
+ 💡
+Congratulations!
+ You reached the end of this part. Now you know how to use Amazon SageMaker Data Wrangler for time series dataset preparation!
+
+ You can now move to an optional advanced time series transformation exercise
+
+
+# Clean up (Only if not planning to do the Advanced part of the timeseries exercise)
+
+* Delete artifacts in S3.
+* Delete data flow file in SageMaker Studio.
+* Stop active SageMaker Data Wrangler instance.
+* Delete SageMaker user profile and domain (optional).
diff --git a/sagemaker-datawrangler/uploadflow.png b/sagemaker-datawrangler/uploadflow.png
new file mode 100644
index 0000000000..43b8fe2c03
Binary files /dev/null and b/sagemaker-datawrangler/uploadflow.png differ
diff --git a/sagemaker-debugger/index.rst b/sagemaker-debugger/index.rst
index 615494960d..b8b19f0019 100644
--- a/sagemaker-debugger/index.rst
+++ b/sagemaker-debugger/index.rst
@@ -25,6 +25,7 @@ Profiling
debugger_interactive_analysis_profiling/interactive_analysis_profiling_data
tensorflow_nlp_sentiment_analysis/sentiment-analysis-tf-distributed-training-bringyourownscript
+ pytorch_profiling/pt-resnet-profiling-single-gpu-single-node
----
@@ -77,6 +78,7 @@ TensorFlow 1.x
tensorflow_action_on_rule/detect_stalled_training_job_and_stop
tensorflow_action_on_rule/tf-mnist-stop-training-job
tensorflow_keras_custom_rule/tf-keras-custom-rule
+ tensorflow_action_on_rule/detect_stalled_training_job_and_actions
PyTorch
diff --git a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/cnn_class_activation_maps.ipynb b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/cnn_class_activation_maps.ipynb
index 0bd3a5f04f..653b95e4e4 100644
--- a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/cnn_class_activation_maps.ipynb
+++ b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/cnn_class_activation_maps.ipynb
@@ -80,7 +80,7 @@
" image.register_hook(self.backward_hook(\"image\"))\n",
" \n",
" def forward_hook(self, module, inputs, outputs):\n",
- " module_name = self.module_maps[module] \n",
+ " module_name = module._module_name\n",
" self._write_inputs(module_name, inputs)\n",
" \n",
" #register outputs for backward pass. this is expensive, so we will only do it during EVAL mode\n",
@@ -326,6 +326,16 @@
"Before starting the SageMaker training job, we need to install some libraries. We will use `smdebug` library to read, filter and analyze raw tensors that are stored in Amazon S3. We will use `opencv-python` library to plot saliency maps as heatmap."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fab25828",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!apt-get update && apt-get install -y python3-opencv"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -570,8 +580,8 @@
" role=role,\n",
" train_instance_type=\"ml.p3.2xlarge\",\n",
" train_instance_count=1,\n",
- " framework_version=\"1.3.1\",\n",
- " py_version=\"py3\",\n",
+ " framework_version=\"1.12.0\",\n",
+ " py_version=\"py38\",\n",
" hyperparameters={\n",
" \"epochs\": 5,\n",
" \"batch_size_train\": 64,\n",
@@ -1325,9 +1335,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Environment (conda_pytorch_p36)",
+ "display_name": "Python 3.8.11 64-bit ('3.8.11')",
"language": "python",
- "name": "conda_pytorch_p36"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -1339,7 +1349,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.10"
+ "version": "3.8.11"
},
"papermill": {
"default_parameters": {},
diff --git a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py
index 1c445eb15f..8e94b15a88 100644
--- a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py
+++ b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/custom_hook.py
@@ -10,7 +10,7 @@ def image_gradients(self, image):
image.register_hook(self.backward_hook("image"))
def forward_hook(self, module, inputs, outputs):
- module_name = self.module_maps[module]
+ module_name = module._module_name
self._write_inputs(module_name, inputs)
# register outputs for backward pass. this is expensive, so we will only do it during EVAL mode
diff --git a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py
index c5e57b9429..d45ab2a3a0 100644
--- a/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py
+++ b/sagemaker-debugger/model_specific_realtime_analysis/cnn_class_activation_maps/entry_point/train.py
@@ -81,6 +81,7 @@ def train_model(epochs, batch_size_train, batch_size_val):
# create custom hook that has a customized forward function, so that we can get gradients of outputs
hook = custom_hook.CustomHook.create_from_json_file()
hook.register_module(model)
+ hook.register_loss(loss_function)
# get the dataloaders for train and test data
train_loader, val_loader = get_dataloaders(batch_size_train, batch_size_val)
diff --git a/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_alexnet.ipynb b/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_alexnet.ipynb
index 37b1970e2c..ff5ab416fd 100644
--- a/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_alexnet.ipynb
+++ b/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_alexnet.ipynb
@@ -251,7 +251,7 @@
"# name of experiment\n",
"timestep = datetime.now()\n",
"timestep = timestep.strftime(\"%d-%m-%Y-%H-%M-%S\")\n",
- "experiment_name = timestep + \"-model-pruning-experiment\"\n",
+ "experiment_name = timestep + \"-alexnet-model-pruning-experiment\"\n",
"\n",
"# create experiment\n",
"Experiment.create(\n",
@@ -372,12 +372,12 @@
"estimator = PyTorch(\n",
" role=sagemaker.get_execution_role(),\n",
" instance_count=1,\n",
- " instance_type=\"ml.p2.xlarge\",\n",
+ " instance_type=\"ml.p3.2xlarge\",\n",
" volume_size=400,\n",
" source_dir=\"src\",\n",
" entry_point=\"train.py\",\n",
- " framework_version=\"1.6\",\n",
- " py_version=\"py3\",\n",
+ " framework_version=\"1.12\",\n",
+ " py_version=\"py38\",\n",
" metric_definitions=[\n",
" {\"Name\": \"train:loss\", \"Regex\": \"loss:(.*?)\"},\n",
" {\"Name\": \"eval:acc\", \"Regex\": \"acc:(.*?)\"},\n",
diff --git a/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_resnet.ipynb b/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_resnet.ipynb
index 9d4049371b..2c08e08870 100644
--- a/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_resnet.ipynb
+++ b/sagemaker-debugger/pytorch_iterative_model_pruning/iterative_model_pruning_resnet.ipynb
@@ -216,7 +216,7 @@
"# name of experiment\n",
"timestep = datetime.now()\n",
"timestep = timestep.strftime(\"%d-%m-%Y-%H-%M-%S\")\n",
- "experiment_name = timestep + \"-model-pruning-experiment\"\n",
+ "experiment_name = timestep + \"resnet-model-pruning-experiment\"\n",
"\n",
"# create experiment\n",
"Experiment.create(\n",
@@ -340,8 +340,8 @@
" volume_size=400,\n",
" source_dir=\"src\",\n",
" entry_point=\"train.py\",\n",
- " framework_version=\"1.6\",\n",
- " py_version=\"py3\",\n",
+ " framework_version=\"1.12\",\n",
+ " py_version=\"py38\",\n",
" metric_definitions=[\n",
" {\"Name\": \"train:loss\", \"Regex\": \"loss:(.*?)\"},\n",
" {\"Name\": \"eval:acc\", \"Regex\": \"acc:(.*?)\"},\n",
diff --git a/sagemaker-debugger/pytorch_model_debugging/pytorch_script_change_smdebug.ipynb b/sagemaker-debugger/pytorch_model_debugging/pytorch_script_change_smdebug.ipynb
index 46cfd0be7b..8ae6dcc438 100644
--- a/sagemaker-debugger/pytorch_model_debugging/pytorch_script_change_smdebug.ipynb
+++ b/sagemaker-debugger/pytorch_model_debugging/pytorch_script_change_smdebug.ipynb
@@ -98,13 +98,13 @@
"\n",
"Tensors that debug hook captures are stored in S3 location specified by you. There are two ways you can configure Amazon SageMaker Debugger for storage:\n",
"\n",
- " 1. **Zero code change**: If you use any of SageMaker provided [Deep Learning containers](https://docs.aws.amazon.com/sagemaker/latest/dg/pre-built-containers-frameworks-deep-learning.html) then you don't need to make any changes to your training script for tensors to be stored. Amazon SageMaker Debugger will use the configuration you provide in the framework `Estimator` to save tensors in the fashion you specify.\n",
+ " 1. **Zero code change (DEPRECATED for PyTorch versions >= 1.12)**: If you use any of SageMaker provided [Deep Learning containers](https://docs.aws.amazon.com/sagemaker/latest/dg/pre-built-containers-frameworks-deep-learning.html) then you don't need to make any changes to your training script for tensors to be stored. Amazon SageMaker Debugger will use the configuration you provide in the framework `Estimator` to save tensors in the fashion you specify.\n",
" \n",
" **Note**: In case of PyTorch training, Debugger collects output tensors in GLOBAL mode by default. In other words, this option does not distinguish output tensors from different phases within an epoch, such as training phase and validation phase.\n",
" \n",
" 2. **Script change**: Use the SageMaker Debugger client library, SMDebug, and customize training scripts to save the specific tensors you want at different frequencies and configurations. Refer to the [DeveloperGuide](https://github.com/awslabs/sagemaker-debugger/tree/master/docs) for details on how to use SageMaker Debugger with your choice of framework in your training script.\n",
" \n",
- "In this notebook, we choose the second option to properly save the output tensors from different training phases.\n",
+ "In this notebook, we choose the second option to properly save the output tensors from different training phases since we're using PyTorch=1.12\n",
"\n",
"### Analysis of tensors\n",
"\n",
@@ -289,19 +289,20 @@
" ```\n",
"\n",
"\n",
- "- **Step 4**: In the `main()` function, create the SMDebug hook and register to the model.\n",
+ "- **Step 4**: In the `main()` function, create the SMDebug hook and register to the model and loss function.\n",
"\n",
" ```python\n",
" hook = smd.Hook.create_from_json_file()\n",
" hook.register_hook(model)\n",
+ " hook.register_loss(loss_fn)\n",
" ```\n",
"\n",
"\n",
"- **Step 4**: In the `main()` function, pass the SMDebug hook to the `train()` and `test()` functions in the epoch loop.\n",
"\n",
" ```python\n",
- " train(args, model, device, train_loader, optimizer, epoch, hook)\n",
- " test(model, device, test_loader, hook)\n",
+ " train(args, model, loss_fn, device, train_loader, optimizer, epoch, hook)\n",
+ " test(model, device, loss_fn, test_loader, hook)\n",
" ```"
]
},
@@ -983,7 +984,7 @@
},
"outputs": [],
"source": [
- "len(trial.tensor(\"nll_loss_output_0\").steps(mode=ModeKeys.TRAIN))"
+ "len(trial.tensor(\"NLLLoss_output_0\").steps(mode=ModeKeys.TRAIN))"
]
},
{
@@ -1002,7 +1003,7 @@
},
"outputs": [],
"source": [
- "len(trial.tensor(\"nll_loss_output_0\").steps(mode=ModeKeys.EVAL))"
+ "len(trial.tensor(\"NLLLoss_output_0\").steps(mode=ModeKeys.EVAL))"
]
},
{
@@ -1116,7 +1117,7 @@
},
"outputs": [],
"source": [
- "plot_tensor(trial, \"nll_loss_output_0\")"
+ "plot_tensor(trial, \"NLLLoss_output_0\")"
]
},
{
@@ -1142,7 +1143,7 @@
"RuleEvaluationConditionMet: Evaluation of the rule Overfit at step 4000 resulted in the condition being met\n",
"```\n",
"\n",
- "Based on this rule evaluation and the plot above, we can conclude that the training job has an overfit issue. While the `nll_loss_output_0` line is decreasing, the `val_nll_loss_output_0` line is fluctuating and not decreasing. \n",
+ "Based on this rule evaluation and the plot above, we can conclude that the training job has an overfit issue. While the `NLLLoss_output_0` line is decreasing, the `val_NLLLoss_output_0` line is fluctuating and not decreasing. \n",
"\n",
"To resolve the overfit problem, you need to consider using or double-checking the following techniques:\n",
"\n",
@@ -1277,9 +1278,9 @@
"metadata": {
"instance_type": "ml.g4dn.xlarge",
"kernelspec": {
- "display_name": "Environment (conda_pytorch_p36)",
+ "display_name": "Python 3.8.11 64-bit ('3.8.11')",
"language": "python",
- "name": "conda_pytorch_p36"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -1291,7 +1292,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.10"
+ "version": "3.8.11"
},
"papermill": {
"default_parameters": {},
@@ -1310,4 +1311,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
-}
\ No newline at end of file
+}
diff --git a/sagemaker-debugger/pytorch_model_debugging/scripts/pytorch_mnist.py b/sagemaker-debugger/pytorch_model_debugging/scripts/pytorch_mnist.py
index d4342e3566..e9e43ffd08 100644
--- a/sagemaker-debugger/pytorch_model_debugging/scripts/pytorch_mnist.py
+++ b/sagemaker-debugger/pytorch_model_debugging/scripts/pytorch_mnist.py
@@ -74,7 +74,7 @@ def forward(self, x):
return output
-def train(args, model, device, train_loader, optimizer, epoch, hook):
+def train(args, model, loss_fn, device, train_loader, optimizer, epoch, hook):
model.train()
# =================================================#
# 2. Set the SMDebug hook for the training phase. #
@@ -84,12 +84,12 @@ def train(args, model, device, train_loader, optimizer, epoch, hook):
data, target = data.to(device), target.to(device)
optimizer.zero_grad()
output = model(data)
- loss = F.nll_loss(output, target)
+ loss = loss_fn(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
print(
- "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format(
+ "Train Epoch: {} [{}/{} ({:.0f}%)]\t Loss: {:.6f}".format(
epoch,
batch_idx * len(data),
len(train_loader.dataset),
@@ -101,7 +101,7 @@ def train(args, model, device, train_loader, optimizer, epoch, hook):
break
-def test(model, device, test_loader, hook):
+def test(model, loss_fn, device, test_loader, hook):
model.eval()
# ===================================================#
# 3. Set the SMDebug hook for the validation phase. #
@@ -113,7 +113,7 @@ def test(model, device, test_loader, hook):
for data, target in test_loader:
data, target = data.to(device), target.to(device)
output = model(data)
- test_loss += F.nll_loss(output, target, reduction="sum").item() # sum up batch loss
+ test_loss += loss_fn(output, target).item() # sum up batch loss
pred = output.argmax(dim=1, keepdim=True) # get the index of the max log-probability
correct += pred.eq(target.view_as(pred)).sum().item()
@@ -201,12 +201,14 @@ def main():
test_loader = torch.utils.data.DataLoader(dataset2, **test_kwargs)
model = Net().to(device)
+ loss_fn = nn.NLLLoss()
# ======================================================#
# 4. Register the SMDebug hook to save output tensors. #
# ======================================================#
hook = smd.Hook.create_from_json_file()
hook.register_hook(model)
+ hook.register_loss(loss_fn)
optimizer = optim.Adadelta(model.parameters(), lr=args.lr)
@@ -215,8 +217,8 @@ def main():
# ===========================================================#
# 5. Pass the SMDebug hook to the train and test functions. #
# ===========================================================#
- train(args, model, device, train_loader, optimizer, epoch, hook)
- test(model, device, test_loader, hook)
+ train(args, model, loss_fn, device, train_loader, optimizer, epoch, hook)
+ test(model, loss_fn, device, test_loader, hook)
scheduler.step()
if args.save_model:
diff --git a/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_distributed.py b/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_distributed.py
index 384333ce45..cc2e60f047 100644
--- a/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_distributed.py
+++ b/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_distributed.py
@@ -53,6 +53,7 @@ def train(batch_size, epoch, net, hook, device, local_rank):
epoch_times = []
if hook:
+ hook.register_module(net)
hook.register_loss(loss_optim)
# train the model
diff --git a/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_horovod_dataloader.py b/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_horovod_dataloader.py
index 0a46a6dabd..290baf6738 100644
--- a/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_horovod_dataloader.py
+++ b/sagemaker-debugger/pytorch_profiling/entry_point/pt_res50_cifar10_horovod_dataloader.py
@@ -101,6 +101,7 @@ def train(batch_size, epoch, net, hook, args, local_rank):
print("START VALIDATING")
if hook:
+ hook.register_module(net)
hook.set_mode(modes.EVAL)
test_sampler.set_epoch(i)
net.eval()
diff --git a/sagemaker-debugger/pytorch_profiling/entry_point/pytorch_res50_cifar10_dataloader.py b/sagemaker-debugger/pytorch_profiling/entry_point/pytorch_res50_cifar10_dataloader.py
index 095cd57032..6636b67979 100644
--- a/sagemaker-debugger/pytorch_profiling/entry_point/pytorch_res50_cifar10_dataloader.py
+++ b/sagemaker-debugger/pytorch_profiling/entry_point/pytorch_res50_cifar10_dataloader.py
@@ -75,6 +75,7 @@ def train(args, net, device):
epoch_times = []
if hook:
+ hook.register_module(net)
hook.register_loss(loss_optim)
# train the model
diff --git a/sagemaker-experiments/index.rst b/sagemaker-experiments/index.rst
index 6d08a5f809..4a9b522e2a 100644
--- a/sagemaker-experiments/index.rst
+++ b/sagemaker-experiments/index.rst
@@ -35,3 +35,11 @@ Search
:maxdepth: 1
../advanced_functionality/search/ml_experiment_management_using_search
+
+Hyperparameter Tuning Job
+=========================
+
+.. toctree::
+ :maxdepth: 1
+
+ associate-hyper-parameter-tuning-job/associate-hyperparameter-tuning-job
diff --git a/sagemaker-inference-deployment-guardrails/index.rst b/sagemaker-inference-deployment-guardrails/index.rst
new file mode 100644
index 0000000000..40bb041e28
--- /dev/null
+++ b/sagemaker-inference-deployment-guardrails/index.rst
@@ -0,0 +1,9 @@
+Deployment Guardrails
+==========================================
+
+.. toctree::
+ :maxdepth: 1
+
+ sagemaker-inference-deployment-guardrails/Update-SageMaker-Inference-endpoint-using-canary-traffic-shifting
+ sagemaker-inference-deployment-guardrails/Update-SageMaker-Inference-endpoint-using-linear-traffic-shifting
+~
diff --git a/sagemaker-inference-recommender/huggingface-inference-recommender/huggingface-inference-recommender.ipynb b/sagemaker-inference-recommender/huggingface-inference-recommender/huggingface-inference-recommender.ipynb
index cc7386cdbd..ef3ca53449 100644
--- a/sagemaker-inference-recommender/huggingface-inference-recommender/huggingface-inference-recommender.ipynb
+++ b/sagemaker-inference-recommender/huggingface-inference-recommender/huggingface-inference-recommender.ipynb
@@ -513,6 +513,79 @@
"print(model_package_version_response)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Alternative Option: ContainerConfig\n",
+ "\n",
+ "If you are missing mandatory fields to create an inference recommender job in your model package version like so (this `create_model_package` is missing `Domain`, `Task`, and `SamplePayloadUrl`):\n",
+ "\n",
+ "```\n",
+ "client.create_model_package(\n",
+ " ModelPackageGroupName=str(model_package_group_name),\n",
+ " ModelPackageDescription=\"HuggingFace PyTorch Inference Recommender Demo\",\n",
+ " InferenceSpecification={\n",
+ " \"Containers\": [\n",
+ " {\n",
+ " \"ContainerHostname\": \"huggingface-pytorch\",\n",
+ " \"Image\": inference_image,\n",
+ " \"ModelDataUrl\": model_url,\n",
+ " \"Framework\": ml_framework,\n",
+ " \"NearestModelName\": model,\n",
+ " \"Environment\": {\n",
+ " \"SAGEMAKER_CONTAINER_LOG_LEVEL\": \"20\",\n",
+ " \"SAGEMAKER_PROGRAM\": \"inference.py\",\n",
+ " \"SAGEMAKER_REGION\": region,\n",
+ " \"SAGEMAKER_SUBMIT_DIRECTORY\": model_url,\n",
+ " },\n",
+ " },\n",
+ " ],\n",
+ " \"SupportedRealtimeInferenceInstanceTypes\": [\n",
+ " \"ml.c5.large\",\n",
+ " \"ml.c5.xlarge\",\n",
+ " \"ml.c5.2xlarge\",\n",
+ " \"ml.m5.xlarge\",\n",
+ " \"ml.m5.2xlarge\",\n",
+ " ],\n",
+ " \"SupportedContentTypes\": [\"text/csv\"],\n",
+ " \"SupportedResponseMIMETypes\": [\"text/csv\"],\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "You may define the fields `Domain`, `Task`, and `SamplePayloadUrl` in the optional field `ContainerConfig` like so:\n",
+ "\n",
+ "```\n",
+ "payload_config = {\n",
+ " \"SamplePayloadUrl\": sample_payload_url,\n",
+ "}\n",
+ "\n",
+ "container_config = {\n",
+ " \"Domain\": ml_domain,\n",
+ " \"Task\": ml_task,\n",
+ " \"PayloadConfig\": payload_config,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "And then provide it directly within `create_inference_recommendations_job()` API like so:\n",
+ "\n",
+ "```\n",
+ "default_response = client.create_inference_recommendations_job(\n",
+ " JobName=str(default_job),\n",
+ " JobDescription=\"\",\n",
+ " JobType=\"Default\",\n",
+ " RoleArn=role,\n",
+ " InputConfig={\n",
+ " \"ModelPackageVersionArn\": model_package_arn,\n",
+ " \"ContainerConfig\": container_config\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "For more information on what else can be provided via `ContainerConfig` please refer to the `CreateInferenceRecommendationsJob` doc here: [CreateInferenceRecommendationsJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateInferenceRecommendationsJob.html)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -621,7 +694,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)"
]
},
diff --git a/sagemaker-inference-recommender/inference-recommender.ipynb b/sagemaker-inference-recommender/inference-recommender.ipynb
index bbbbf958fd..8c6fa80beb 100644
--- a/sagemaker-inference-recommender/inference-recommender.ipynb
+++ b/sagemaker-inference-recommender/inference-recommender.ipynb
@@ -24,7 +24,7 @@
"source": [
"## 2. Setup \n",
"\n",
- "Note that we are using the `conda_tensorflow2_p36` kernel in SageMaker Notebook Instances. This is running Python 3.6 and TensorFlow 2.1.3. If you'd like to use the same setup, in the AWS Management Console, go to the Amazon SageMaker console. Choose Notebook Instances, and click create a new notebook instance. Upload the current notebook and set the kernel. You can also run this in SageMaker Studio Notebooks with the `TensorFlow 2.1 Python 3.6 CPU Optimized` kernel.\n",
+ "Note that we are using the `conda_tensorflow2_p36` kernel in SageMaker Notebook Instances. This is running Python 3.6 and TensorFlow 2.1.3. If you'd like to use the same setup, in the AWS Management Console, go to the Amazon SageMaker console. Choose Notebook Instances, and click create a new notebook instance. Upload the current notebook and set the kernel. You can also run this in SageMaker Studio Notebooks with the `TensorFlow 2.6 Python 3.8 CPU Optimized` kernel.\n",
"\n",
"In the next steps, you'll import standard methods and libraries as well as set variables that will be used in this notebook. The `get_execution_role` function retrieves the AWS Identity and Access Management (IAM) role you created at the time of creating your notebook instance."
]
@@ -607,6 +607,54 @@
"sm_client.describe_model_package(ModelPackageName=model_package_arn)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Alternative Option: ContainerConfig\n",
+ "\n",
+ "If you are missing mandatory fields to create an inference recommender job in your model package version like so (this `create_model_package_input_dict` is missing `Domain`, `Task`, and `SamplePayloadUrl`):\n",
+ "\n",
+ "```\n",
+ "create_model_package_input_dict = {\n",
+ " \"ModelPackageGroupName\": model_package_group_name,\n",
+ " \"ModelPackageDescription\": model_package_description,\n",
+ " \"ModelApprovalStatus\": model_approval_status,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "You may define the fields `Domain`, `Task`, and `SamplePayloadUrl` in the optional field `ContainerConfig` like so:\n",
+ "\n",
+ "```\n",
+ "payload_config = {\n",
+ " \"SamplePayloadUrl\": sample_payload_url,\n",
+ "}\n",
+ "\n",
+ "container_config = {\n",
+ " \"Domain\": ml_domain.upper(),\n",
+ " \"Task\": ml_task.upper(),\n",
+ " \"PayloadConfig\": payload_config,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "And then provide it directly within `create_inference_recommendations_job()` API like so:\n",
+ "\n",
+ "```\n",
+ "default_response = client.create_inference_recommendations_job(\n",
+ " JobName=str(default_job),\n",
+ " JobDescription=\"\",\n",
+ " JobType=\"Default\",\n",
+ " RoleArn=role,\n",
+ " InputConfig={\n",
+ " \"ModelPackageVersionArn\": model_package_arn,\n",
+ " \"ContainerConfig\": container_config\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "For more information on what else can be provided via `ContainerConfig` please refer to the `CreateInferenceRecommendationsJob` doc here: [CreateInferenceRecommendationsJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateInferenceRecommendationsJob.html)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -628,11 +676,11 @@
"import uuid\n",
"from sagemaker import get_execution_role\n",
"\n",
- "client = boto3.client(\"sagemaker\", region)\n",
+ "inference_client = boto3.client(\"sagemaker\", region)\n",
"\n",
"role = get_execution_role()\n",
"default_job = uuid.uuid1()\n",
- "default_response = client.create_inference_recommendations_job(\n",
+ "default_response = inference_client.create_inference_recommendations_job(\n",
" JobName=str(default_job),\n",
" JobDescription=\"\",\n",
" JobType=\"Default\",\n",
@@ -673,20 +721,22 @@
"import pprint\n",
"import pandas as pd\n",
"\n",
- "finished = False\n",
- "while not finished:\n",
- " inference_recommender_job = sm_client.describe_inference_recommendations_job(\n",
+ "inference_client = boto3.client(\"sagemaker\", region)\n",
+ "\n",
+ "stopped = False\n",
+ "while not stopped:\n",
+ " inference_recommender_job = inference_client.describe_inference_recommendations_job(\n",
" JobName=str(default_job)\n",
" )\n",
" if inference_recommender_job[\"Status\"] in [\"COMPLETED\", \"STOPPED\", \"FAILED\"]:\n",
" finished = True\n",
" else:\n",
- " print(\"In progress\")\n",
+ " print(\"Inference recommender job in progress\")\n",
" time.sleep(300)\n",
"\n",
"if inference_recommender_job[\"Status\"] == \"FAILED\":\n",
" print(\"Inference recommender job failed \")\n",
- " print(\"Failed Reason: {}\".format(inference_recommender_job[\"FailureReason\"]))\n",
+ " print(\"Failed Reason: {}\".inference_recommender_job[\"FailureReason\"])\n",
"else:\n",
" print(\"Inference recommender job completed\")"
]
@@ -709,7 +759,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)\n",
"df.head()"
]
@@ -744,9 +795,14 @@
"metadata": {},
"outputs": [],
"source": [
+ "import boto3\n",
+ "import uuid\n",
+ "\n",
+ "inference_client = boto3.client(\"sagemaker\", region)\n",
+ "\n",
"role = get_execution_role()\n",
"advanced_job = uuid.uuid1()\n",
- "advanced_response = sm_client.create_inference_recommendations_job(\n",
+ "advanced_response = inference_client.create_inference_recommendations_job(\n",
" JobName=str(advanced_job),\n",
" JobDescription=\"\",\n",
" JobType=\"Advanced\",\n",
@@ -792,20 +848,27 @@
"metadata": {},
"outputs": [],
"source": [
- "finished = False\n",
- "while not finished:\n",
- " inference_recommender_job = sm_client.describe_inference_recommendations_job(\n",
+ "import boto3\n",
+ "import uuid\n",
+ "import pprint\n",
+ "import pandas as pd\n",
+ "\n",
+ "inference_client = boto3.client(\"sagemaker\", region)\n",
+ "\n",
+ "stopped = False\n",
+ "while not stopped:\n",
+ " inference_recommender_job = inference_client.describe_inference_recommendations_job(\n",
" JobName=str(advanced_job)\n",
" )\n",
" if inference_recommender_job[\"Status\"] in [\"COMPLETED\", \"STOPPED\", \"FAILED\"]:\n",
" finished = True\n",
" else:\n",
- " print(\"In progress\")\n",
+ " print(\"Inference recommender job in progress\")\n",
" time.sleep(300)\n",
"\n",
"if inference_recommender_job[\"Status\"] == \"FAILED\":\n",
" print(\"Inference recommender job failed \")\n",
- " print(\"Failed Reason: {}\".format(inference_recommender_job[\"FailureReason\"]))\n",
+ " print(\"Failed Reason: {}\".inference_recommender_job[\"FailureReason\"])\n",
"else:\n",
" print(\"Inference recommender job completed\")"
]
@@ -828,7 +891,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)\n",
"df.head()"
]
diff --git a/sagemaker-inference-recommender/sklearn-inference-recommender/sklearn-inference-recommender.ipynb b/sagemaker-inference-recommender/sklearn-inference-recommender/sklearn-inference-recommender.ipynb
index c0704e9d94..bda343d8ca 100644
--- a/sagemaker-inference-recommender/sklearn-inference-recommender/sklearn-inference-recommender.ipynb
+++ b/sagemaker-inference-recommender/sklearn-inference-recommender/sklearn-inference-recommender.ipynb
@@ -40,8 +40,7 @@
"execution_count": null,
"metadata": {
"collapsed": false,
- "jupyter": {
- },
+ "jupyter": {},
"pycharm": {
"name": "#%%\n"
}
@@ -56,8 +55,7 @@
"execution_count": null,
"metadata": {
"collapsed": false,
- "jupyter": {
- },
+ "jupyter": {},
"pycharm": {
"name": "#%%\n"
}
@@ -459,6 +457,79 @@
"print(model_package_version_response)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Alternative Option: ContainerConfig\n",
+ "\n",
+ "If you are missing mandatory fields to create an inference recommender job in your model package version like so (this `create_model_package` is missing `Domain`, `Task`, and `SamplePayloadUrl`):\n",
+ "\n",
+ "```\n",
+ "client.create_model_package(\n",
+ " ModelPackageGroupName=str(model_package_group_name),\n",
+ " ModelPackageDescription=\"scikit-learn Inference Recommender Demo\",\n",
+ " InferenceSpecification={\n",
+ " \"Containers\": [\n",
+ " {\n",
+ " \"ContainerHostname\": \"scikit-learn\",\n",
+ " \"Image\": inference_image,\n",
+ " \"ModelDataUrl\": model_url,\n",
+ " \"Framework\": ml_framework,\n",
+ " \"NearestModelName\": model,\n",
+ " \"Environment\": {\n",
+ " \"SAGEMAKER_CONTAINER_LOG_LEVEL\": \"20\",\n",
+ " \"SAGEMAKER_PROGRAM\": sagemaker_program,\n",
+ " \"SAGEMAKER_REGION\": region,\n",
+ " \"SAGEMAKER_SUBMIT_DIRECTORY\": sourcedir_url,\n",
+ " },\n",
+ " },\n",
+ " ],\n",
+ " \"SupportedRealtimeInferenceInstanceTypes\": [\n",
+ " \"ml.c5.large\",\n",
+ " \"ml.c5.xlarge\",\n",
+ " \"ml.c5.2xlarge\",\n",
+ " \"ml.m5.xlarge\",\n",
+ " \"ml.m5.2xlarge\",\n",
+ " ],\n",
+ " \"SupportedContentTypes\": [\"text/csv\"],\n",
+ " \"SupportedResponseMIMETypes\": [\"text/csv\"],\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "You may define the fields `Domain`, `Task`, and `SamplePayloadUrl` in the optional field `ContainerConfig` like so:\n",
+ "\n",
+ "```\n",
+ "payload_config = {\n",
+ " \"SamplePayloadUrl\": sample_payload_url,\n",
+ "}\n",
+ "\n",
+ "container_config = {\n",
+ " \"Domain\": ml_domain,\n",
+ " \"Task\": ml_task,\n",
+ " \"PayloadConfig\": payload_config,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "And then provide it directly within `create_inference_recommendations_job()` API like so:\n",
+ "\n",
+ "```\n",
+ "default_response = client.create_inference_recommendations_job(\n",
+ " JobName=str(default_job),\n",
+ " JobDescription=\"\",\n",
+ " JobType=\"Default\",\n",
+ " RoleArn=role,\n",
+ " InputConfig={\n",
+ " \"ModelPackageVersionArn\": model_package_arn,\n",
+ " \"ContainerConfig\": container_config\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "For more information on what else can be provided via `ContainerConfig` please refer to the `CreateInferenceRecommendationsJob` doc here: [CreateInferenceRecommendationsJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateInferenceRecommendationsJob.html)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -567,7 +638,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)"
]
},
@@ -717,7 +789,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)\n",
"df.head()"
]
diff --git a/sagemaker-inference-recommender/tensorflow-cloudwatch/tf-cloudwatch-inference-recommender.ipynb b/sagemaker-inference-recommender/tensorflow-cloudwatch/tf-cloudwatch-inference-recommender.ipynb
index eed4f4db0c..ec97f091a0 100644
--- a/sagemaker-inference-recommender/tensorflow-cloudwatch/tf-cloudwatch-inference-recommender.ipynb
+++ b/sagemaker-inference-recommender/tensorflow-cloudwatch/tf-cloudwatch-inference-recommender.ipynb
@@ -484,6 +484,54 @@
"print(\"ModelPackage Version ARN : {}\".format(model_package_arn))"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Alternative Option: ContainerConfig\n",
+ "\n",
+ "If you are missing mandatory fields to create an inference recommender job in your model package version like so (this `create_model_package_input_dict` is missing `Domain`, `Task`, and `SamplePayloadUrl`):\n",
+ "\n",
+ "```\n",
+ "create_model_package_input_dict = {\n",
+ " \"ModelPackageGroupName\": model_package_group_name,\n",
+ " \"ModelPackageDescription\": model_package_description,\n",
+ " \"ModelApprovalStatus\": model_approval_status,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "You may define the fields `Domain`, `Task`, and `SamplePayloadUrl` in the optional field `ContainerConfig` like so:\n",
+ "\n",
+ "```\n",
+ "payload_config = {\n",
+ " \"SamplePayloadUrl\": sample_payload_url,\n",
+ "}\n",
+ "\n",
+ "container_config = {\n",
+ " \"Domain\": ml_domain.upper(),\n",
+ " \"Task\": ml_task.upper(),\n",
+ " \"PayloadConfig\": payload_config,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "And then provide it directly within `create_inference_recommendations_job()` API like so:\n",
+ "\n",
+ "```\n",
+ "default_response = client.create_inference_recommendations_job(\n",
+ " JobName=str(default_job),\n",
+ " JobDescription=\"\",\n",
+ " JobType=\"Default\",\n",
+ " RoleArn=role,\n",
+ " InputConfig={\n",
+ " \"ModelPackageVersionArn\": model_package_arn,\n",
+ " \"ContainerConfig\": container_config\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "For more information on what else can be provided via `ContainerConfig` please refer to the `CreateInferenceRecommendationsJob` doc here: [CreateInferenceRecommendationsJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateInferenceRecommendationsJob.html)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -583,7 +631,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)\n",
"df.head()"
]
@@ -778,7 +827,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)\n",
"df.head()"
]
diff --git a/sagemaker-inference-recommender/xgboost/xgboost-inference-recommender.ipynb b/sagemaker-inference-recommender/xgboost/xgboost-inference-recommender.ipynb
index 3ae7e16924..a0dfe49986 100644
--- a/sagemaker-inference-recommender/xgboost/xgboost-inference-recommender.ipynb
+++ b/sagemaker-inference-recommender/xgboost/xgboost-inference-recommender.ipynb
@@ -509,6 +509,54 @@
"print(\"ModelPackage Version ARN : {}\".format(model_package_arn))"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Alternative Option: ContainerConfig\n",
+ "\n",
+ "If you are missing mandatory fields to create an inference recommender job in your model package version like so (this `create_model_package_input_dict` is missing `Domain`, `Task`, and `SamplePayloadUrl`):\n",
+ "\n",
+ "```\n",
+ "create_model_package_input_dict = {\n",
+ " \"ModelPackageGroupName\": model_package_group_name,\n",
+ " \"ModelPackageDescription\": model_package_description,\n",
+ " \"ModelApprovalStatus\": model_approval_status,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "You may define the fields `Domain`, `Task`, and `SamplePayloadUrl` in the optional field `ContainerConfig` like so:\n",
+ "\n",
+ "```\n",
+ "payload_config = {\n",
+ " \"SamplePayloadUrl\": sample_payload_url,\n",
+ "}\n",
+ "\n",
+ "container_config = {\n",
+ " \"Domain\": ml_domain.upper(),\n",
+ " \"Task\": ml_task.upper(),\n",
+ " \"PayloadConfig\": payload_config,\n",
+ "}\n",
+ "```\n",
+ "\n",
+ "And then provide it directly within `create_inference_recommendations_job()` API like so:\n",
+ "\n",
+ "```\n",
+ "default_response = client.create_inference_recommendations_job(\n",
+ " JobName=str(default_job),\n",
+ " JobDescription=\"\",\n",
+ " JobType=\"Default\",\n",
+ " RoleArn=role,\n",
+ " InputConfig={\n",
+ " \"ModelPackageVersionArn\": model_package_arn,\n",
+ " \"ContainerConfig\": container_config\n",
+ " },\n",
+ ")\n",
+ "```\n",
+ "\n",
+ "For more information on what else can be provided via `ContainerConfig` please refer to the `CreateInferenceRecommendationsJob` doc here: [CreateInferenceRecommendationsJob](https://docs.aws.amazon.com/sagemaker/latest/APIReference/API_CreateInferenceRecommendationsJob.html)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -606,7 +654,8 @@
" for x in inference_recommender_job[\"InferenceRecommendations\"]\n",
"]\n",
"df = pd.DataFrame(data)\n",
- "df.drop(\"VariantName\", inplace=True, axis=1)\n",
+ "dropFilter = df.filter([\"VariantName\"])\n",
+ "df.drop(dropFilter, inplace=True, axis=1)\n",
"pd.set_option(\"max_colwidth\", 400)\n",
"df.head()"
]
diff --git a/sagemaker-jumpstart/index.rst b/sagemaker-jumpstart/index.rst
new file mode 100644
index 0000000000..b6073d7431
--- /dev/null
+++ b/sagemaker-jumpstart/index.rst
@@ -0,0 +1,10 @@
+SageMaker JumpStart
+==========================================
+
+.. toctree::
+ :maxdepth: 1
+
+ nlp_score_dashboard_sec/sec-dashboard/SEC_Section_Extraction_Functions
+ financial_tabtext_construction/SEC_Retrieval_Summarizer_Scoring
+ multimodal_tabtext/PPP_tabtext_ML
+ multicategory_sec/SEC_MNIST_ML
diff --git a/sagemaker-pipelines/index.rst b/sagemaker-pipelines/index.rst
index 1a84f55923..7ec0f925c2 100644
--- a/sagemaker-pipelines/index.rst
+++ b/sagemaker-pipelines/index.rst
@@ -11,3 +11,34 @@ Amazon SageMaker Model Building Pipelines is a tool for building machine learnin
tabular/tensorflow2-california-housing-sagemaker-pipelines-deploy-endpoint/tensorflow2-california-housing-sagemaker-pipelines-deploy-endpoint
nlp/amazon_comprehend_sagemaker_pipeline/sm_pipeline_with_comprehend
time_series_forecasting/amazon_forecast_pipeline/sm_pipeline_with_amazon_forecast
+ sagemaker-pipelines/tabular/tuning-step/sagemaker-pipelines-tuning-step
+ sagemaker-pipelines/tabular/lambda-step/sagemaker-pipelines-lambda-step
+ sagemaker-pipelines/tabular/model-monitor-clarify-pipelines/sagemaker-pipeline-model-monitor-clarify-steps
+ sagemaker-pipelines/tabular/customizing_build_train_deploy_project/sagemaker-pipelines-customized-project
+ sagemaker-pipelines/tabular/train-register-deploy-pipeline-model/train register and deploy a pipeline model
+
+
+Pipeline Parameterization
+============================
+
+.. toctree::
+ :maxdepth: 1
+
+ ../sagemaker-pipeline-parameterization/parameterized-pipeline
+
+
+SageMaker Pipeline Multi-Model
+================================
+
+.. toctree::
+ :maxdepth: 1
+
+ ../sagemaker-pipeline-multi-model/restate-project
+
+Pipeline Compare
+============================
+
+.. toctree::
+ :maxdepth: 1
+
+ ../sagemaker-pipeline-compare-model-versions/notebook
diff --git a/sagemaker-pipelines/tabular/local-mode/sagemaker-pipelines-local-mode.ipynb b/sagemaker-pipelines/tabular/local-mode/sagemaker-pipelines-local-mode.ipynb
new file mode 100644
index 0000000000..242a34de5e
--- /dev/null
+++ b/sagemaker-pipelines/tabular/local-mode/sagemaker-pipelines-local-mode.ipynb
@@ -0,0 +1,1489 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Use SageMaker Pipelines to Run Your Jobs Locally\n",
+ "\n",
+ "This notebook demonstrates how to orchestrate SageMaker jobs locally using SageMaker Pipelines. \n",
+ "\n",
+ "Using a `LocalPipelineSession` object, you can now run your pipelines on your local machine before running them in the cloud. \n",
+ "\n",
+ "The `LocalPipelineSession` object is used while defining each pipeline step and when defining the complete Pipeline object. To run this pipeline in the cloud, each step along with the Pipeline object must be redefined using `PipelineSession`.\n",
+ "\n",
+ "**Note**: This notebook will not run in SageMaker Studio. You can run this on SageMaker Classic Notebook instances OR your local IDE."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## SageMaker Pipelines Local Mode\n",
+ "\n",
+ "SageMaker Pipelines Local Mode supports the following activities, which are demonstrated in this notebook:\n",
+ "\n",
+ "* ProcessingStep\n",
+ "* TrainingStep\n",
+ "* ConditionStep\n",
+ "* ModelStep\n",
+ "* TransformStep\n",
+ "* FailStep"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "## Dataset\n",
+ "\n",
+ "The dataset you use is the [UCI Machine Learning Abalone Dataset](https://archive.ics.uci.edu/ml/datasets/abalone) [1]. The aim for this task is to determine the age of an abalone snail from its physical measurements. At the core, this is a regression problem.\n",
+ "\n",
+ "The dataset contains several features: length (the longest shell measurement), diameter (the diameter perpendicular to length), height (the height with meat in the shell), whole_weight (the weight of whole abalone), shucked_weight (the weight of meat), viscera_weight (the gut weight after bleeding), shell_weight (the weight after being dried), sex ('M', 'F', 'I' where 'I' is Infant), and rings (integer).\n",
+ "\n",
+ "The number of rings turns out to be a good approximation for age (age is rings + 1.5). However, to obtain this number requires cutting the shell through the cone, staining the section, and counting the number of rings through a microscope, which is a time-consuming task. However, the other physical measurements are easier to determine. You use the dataset to build a predictive model of the variable rings through these other physical measurements.\n",
+ "\n",
+ "Before you upload the data to an S3 bucket, install the SageMaker Python SDK and gather some constants you can use later in this notebook.\n",
+ "\n",
+ "> [1] Dua, D. and Graff, C. (2019). [UCI Machine Learning Repository](http://archive.ics.uci.edu/ml). Irvine, CA: University of California, School of Information and Computer Science."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Install the latest version of the SageMaker Python SDK. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install 'sagemaker' --upgrade"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import sys\n",
+ "\n",
+ "import boto3\n",
+ "import sagemaker\n",
+ "from sagemaker.workflow.pipeline_context import LocalPipelineSession\n",
+ "\n",
+ "# Create a `LocalPipelineSession` object so that each pipeline step will run locally\n",
+ "# To run this pipeline in the cloud, you must change `LocalPipelineSession()` to `PipelineSession()`\n",
+ "local_pipeline_session = LocalPipelineSession()\n",
+ "\n",
+ "region = local_pipeline_session.boto_region_name\n",
+ "\n",
+ "default_bucket = local_pipeline_session.default_bucket()\n",
+ "prefix = \"sagemaker-pipelines-local-mode-example\"\n",
+ "\n",
+ "role = None # Role is set below"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Please Note: Provide SageMaker Execution Role ARN if not running on SageMaker Notebook environment\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ " 💡 Set Execution Role for Permissions \n",
+ "If you are running this notebook from a local machine, as opposed to within the SageMaker Jupyter environment, you will need to add the code below, after filling in the name for a valid SageMaker Execution Role.
\n", + " \n", + " Click here to lookup IAM SageMaker Execution Roles \n", + " The except block below will lookup the ARN from the role name.\n", + "\n", + "
"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# try:\n",
+ "# role = sagemaker.get_execution_role()\n",
+ "# except ValueError:\n",
+ "# iam = boto3.client('iam')\n",
+ "# role = iam.get_role(RoleName='\n", + " \n", + " Click here to lookup IAM SageMaker Execution Roles \n", + " The except block below will lookup the ARN from the role name.\n", + "\n", + "
\n",
+ "\n",
+ "Heterogeneous Training Job - Where we add ml.c5.9xlarge instance for extra CPU cores, to allow increased GPU usage of ml.p3.2xlarge instance, and improve cost-efficiency. Both the jobs runs the training code, train data set, pre-processing, and other relevant parameters.\n",
+ "\n",
+ "\n",
+ "In homogeneous cluster training job, the data pre-processing and Deep Neural Network (DNN) training code runs on the same instance. However, in heterogeneous cluster training job, the data pre-processing code runs on the CPU nodes (here by referred as **data_group or data group**), whereas the Deep Neural Network (DNN) training code runs on the GPU nodes (here referred as **dnn_group or dnn group**). The inter-node communication between the data and dnn groups is handled by generic implementation of [gRPC client-server interface](https://grpc.io/docs/languages/python/basics/). \n",
+ "\n",
+ "The script (`launcher.py`) has the logic to detect (using SageMaker environment variables) whether the node it is running on belongs to data_group or dnn_group. If it is data_group, it spawns a separate process by executing `train_data.py`. This script runs grpc-server service for extracting processed training batches using [Protocol Buffers](https://developers.google.com/protocol-buffers/docs/overview). The gRPC server running on the data_group listens on a specific port (ex. 6000). In the code (`train_data.py`) documentation, we have chosen an implementation that keeps the data loading logic intact where data batches are entered into a shared queue. The `get_samples` function of the `DataFeedService` pulls batches from the same queue and sends them to the client in the form of a continuous data stream. While fetching the data, the main entrypoint script `launcher.py` listens on port 16000 for a shutdown request coming from gRPC client i.e. data group. The `train_data.py` waits for shutdown action from the parent process. \n",
+ "\n",
+ "If the node belongs to dnn_group, the main training script (`launcher.py`) spawns a separate set of processes by executing `train_dnn.py`. The script runs gRPC client code and DNN component of the training job. It consumes the processed training data from the gRPC server. We have defined an iterable PyTorch dataset, RemoteDataset, that opens a connection to the gRPC server, and reads from a stream of data batches. Once the model is trained with all the batches of training data, the gRPC client exits, and the parent process`launcher.py` sends a shutdown request on port 16000. This indicates the gRPC server to shutdown, and signals ends of the training job. \n",
+ "\n",
+ "Here is how the workflow looks like:\n",
+ "\n",
+ "\n",
+ "\n",
+ "This example notebook runs a training job on 2 instances, 1 in each node group. The data_group uses ml.c5.9xlarge whereas dnn_group uses ml.p3.2xlarge.\n",
+ "\n",
+ "This notebook refers following files and folders:\n",
+ "\n",
+ "- Folders: \n",
+ " - `code`: this has the training (data pre-processing and dnn) scripts, and grpc client-server start and shutdown scripts\n",
+ " - `images`: contains images referred in notebook\n",
+ "- Files: \n",
+ " - `launcher.py`: entry point training script. This script is executed on all the nodes irrespective of which group it belongs to. This is a parent process that makes a decision on where to spawn a data pre-processing or dnn component of the training job. The script runs on all the nodes as entry point. It also handles the shutdown logic for gRPC server. \n",
+ " - `train_data.py`, `dataset_feed_pb2.py`, `dataset_feed_pb2_grpc.py`: these scripts run on the data_group nodes and responsible for setting up grpc-server, start and shutdown.\n",
+ " - `train_dnn.py`: this script runs dnn code on the training data set. It fetches preprocessed data from the data_group node as a stream using gRPC client-server communication. It also sends a shutdown request after all the iterations on the preprocessed training data set. \n",
+ " - `requirement.txt`: defines package required for gRPC \n",
+ " - `train.py`: this script is the entry point script for SageMaker homogeneous cluster training. This script is picked up when you choose IS_HETERO = False. This uses a local dataset and runs both data pre-processing and a dnn component on the same node. "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1f98cde9",
+ "metadata": {},
+ "source": [
+ "### security groups update if running in private VPC\n",
+ "This section is relevant if you plan to [run in a private VPC](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html) (passing `subnets` and `security_group_ids` parameters when defining an Estimator). \n",
+ "SageMaker documentation recommends you [add](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html#train-vpc-vpc) a rule for your security group that allows inbound connections between members of the same security group, for all TCP communication. This will also cover for the gRPC related traffic between instances:\n",
+ "- the data_group instances will listen on port 6000 for connections from all nodes. This stream is not encrypted. You can change the code to encrypted the connection if needed.\n",
+ "- the data_group intances listen on port 16000 for a shutdown signal from all nodes."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fd1e5aca",
+ "metadata": {},
+ "source": [
+ "### A. Setting up SageMaker Studio notebook\n",
+ "\n",
+ "#### Step 1 - Upgrade SageMaker SDK and dependent packages \n",
+ "Heterogeneous Clusters for Amazon SageMaker model training was [announced](https://aws.amazon.com/about-aws/whats-new/2022/07/announcing-heterogeneous-clusters-amazon-sagemaker-model-training) on 07/08/2022. As a first step, ensure you have updated SageMaker SDK, PyTorch, and Boto3 client that enables this feature."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "id": "54ff1687",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%%bash\n",
+ "python3 -m pip install --upgrade boto3 botocore awscli sagemaker"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0d20b2f3",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Restart the notebook kernel "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "229e1b18",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#import IPython\n",
+ "#IPython.Application.instance().kernel.do_shutdown(True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a9592cda",
+ "metadata": {},
+ "source": [
+ "#### Step 3 - Validate SageMaker Python SDK and PyTorch versions\n",
+ "Ensure the output of the cell below reflects:\n",
+ "\n",
+ "- SageMaker Python SDK version 2.98.0 or above, \n",
+ "- boto3 1.24 or above \n",
+ "- botocore 1.27 or above \n",
+ "- PyTorch 1.10 or above "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "id": "0b0e3202",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Name: sagemaker\n",
+ "Version: 2.109.0\n",
+ "---\n",
+ "Name: torch\n",
+ "Version: 1.10.2+cpu\n",
+ "---\n",
+ "Name: boto3\n",
+ "Version: 1.24.72\n",
+ "---\n",
+ "Name: botocore\n",
+ "Version: 1.27.72\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip show sagemaker torch boto3 botocore |egrep 'Name|Version|---'"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9176d868",
+ "metadata": {},
+ "source": [
+ "--------------\n",
+ "### B. Setting up the Training environment\n",
+ "\n",
+ "#### Step 1 - Import SageMaker components and set up the IAM role and Amazon S3 bucket"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "id": "594fce53",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "arn:aws:iam::776941257690:role/service-role/AmazonSageMakerServiceCatalogProductsUseRole\n",
+ "s3://sagemaker-us-east-1-776941257690/DEMO-MNIST\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "import json\n",
+ "import datetime\n",
+ "import os\n",
+ "\n",
+ "import sagemaker\n",
+ "from sagemaker.pytorch import PyTorch\n",
+ "from sagemaker import get_execution_role\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "\n",
+ "\n",
+ "sess = sagemaker.Session()\n",
+ "\n",
+ "role = get_execution_role()\n",
+ "\n",
+ "output_path = \"s3://\" + sess.default_bucket() + \"/DEMO-MNIST\"\n",
+ "print(role)\n",
+ "print(output_path)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "165bca04",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Configure environment variables \n",
+ "This step defines whether you want to run training job in heterogeneous cluster mode or not. Also, defines instance groups, multiple nodes in group, and hyperparameter values. For baselining, run a homogeneous cluster training job by setting `IS_HETERO = False`. This will let both the data pre-processing and DNN code run on the same node i.e. `ml.p3.2xlarge`. \n",
+ "\n",
+ "\n",
+ "Test configuration (if running training on p3.2xl or g5.2xl as dnn_group instance type, and c5.2xl as data_group instance type: (training duration: 7-8 mins) \n",
+ "`num-data-workers: 4` \n",
+ "`grpc-workers: 4` \n",
+ "`num-dnn-workers: 4` \n",
+ "`pin-memory\": True` \n",
+ "`iterations : 100` \n",
+ "\n",
+ "Performance configuration (if running training on p3.2xl as dnn_group instance type, and c5.9xl as data_group instance type OR training in homogeneous cluster mode i.e. g5.8xl): (training duration - 30 mins) \n",
+ "`num-data-workers: 32` \n",
+ "`grpc-workers: 2` \n",
+ "`num-dnn-workers: 2` \n",
+ "`pin-memory\": True` \n",
+ "`iterations : 4800`\n",
+ "\n",
+ "Performance configuration (if running training on p3.2xl in homogeneous cluster mode): \n",
+ "`num-data-workers: 8` \n",
+ "`grpc-workers: 2` \n",
+ "`num-dnn-workers: 2` \n",
+ "`pin-memory\": True` \n",
+ "`iterations : 2400`\n",
+ "\n",
+ "Note: This PyTorch example has not been tested with multiple instances in an instance group. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "id": "0d65707b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "IS_CLOUD_JOB = True\n",
+ "IS_HETERO = True # if set to false, uses homogeneous cluster\n",
+ "PT_DATA_MODE = \"service\" if IS_HETERO else \"local\" # local | service\n",
+ "IS_DNN_DISTRIBUTION = False # Distributed Training with DNN nodes not tested, set it to False\n",
+ "\n",
+ "data_group = InstanceGroup(\n",
+ " \"data_group\", \"ml.c5.9xlarge\", 1\n",
+ ") # 36 vCPU #change the instance type if IS_HETERO=True\n",
+ "dnn_group = InstanceGroup(\n",
+ " \"dnn_group\", \"ml.p3.2xlarge\", 1\n",
+ ") # 8 vCPU #change the instance type if IS_HETERO=True\n",
+ "\n",
+ "kwargs = dict()\n",
+ "kwargs[\"hyperparameters\"] = {\n",
+ " \"batch-size\": 8192,\n",
+ " \"num-data-workers\": 4, # This number drives the avg. step time. More workers help parallel pre-processing of data. Recommendation: Total no. of cpu 'n' = 'num-data-wokers'+'grpc-workers'+ 2 (reserved)\n",
+ " \"grpc-workers\": 4, # No. of workers serving pre-processed data to DNN group (gRPC client). see above formula.\n",
+ " \"num-dnn-workers\": 4, # Modify this no. to be less than the cpu core of your training instances in dnn group\n",
+ " \"pin-memory\": True, # Pin to GPU memory\n",
+ " \"iterations\": 100, # No. of iterations in an epoch (must be multiple of 10).\n",
+ "}\n",
+ "\n",
+ "if IS_HETERO:\n",
+ " kwargs[\"instance_groups\"] = [data_group, dnn_group]\n",
+ " entry_point = \"launcher.py\"\n",
+ "else:\n",
+ " kwargs[\"instance_type\"] = (\n",
+ " \"ml.p3.2xlarge\" if IS_CLOUD_JOB else \"local\"\n",
+ " ) # change the instance type if IS_HETERO=False\n",
+ " kwargs[\"instance_count\"] = 1\n",
+ " entry_point = \"train.py\"\n",
+ "\n",
+ "if IS_DNN_DISTRIBUTION:\n",
+ " processes_per_host_dict = {\n",
+ " \"ml.g5.xlarge\": 1,\n",
+ " \"ml.g5.12xlarge\": 4,\n",
+ " \"ml.p3.8xlarge\": 4,\n",
+ " \"ml.p4d.24xlarge\": 8,\n",
+ " }\n",
+ " kwargs[\"distribution\"] = {\n",
+ " \"mpi\": {\n",
+ " \"enabled\": True,\n",
+ " \"processes_per_host\": processes_per_host_dict[dnn_instance_type],\n",
+ " \"custom_mpi_options\": \"--NCCL_DEBUG INFO\",\n",
+ " },\n",
+ " }\n",
+ " if IS_HETERO:\n",
+ " kwargs[\"distribution\"][\"instance_groups\"] = [dnn_group]\n",
+ "\n",
+ " print(f\"distribution={kwargs['distribution']}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4ff19e24",
+ "metadata": {},
+ "source": [
+ "#### Step 3: Set up the Estimator\n",
+ "In order to use SageMaker to fit our algorithm, we'll create `Estimator` that defines how to use the container to train. This includes the configuration we need to invoke SageMaker training."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "id": "94f4c8ce",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "estimator = PyTorch(\n",
+ " framework_version=\"1.11.0\", # 1.10.0 or later\n",
+ " py_version=\"py38\", # Python v3.8\n",
+ " role=role,\n",
+ " entry_point=entry_point,\n",
+ " source_dir=\"code\",\n",
+ " volume_size=10,\n",
+ " max_run=4800,\n",
+ " disable_profiler=True,\n",
+ " debugger_hook_config=False,\n",
+ " **kwargs,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a81dcab6",
+ "metadata": {},
+ "source": [
+ "#### Step 4: Download the MNIST Data and Upload it to S3 bucket\n",
+ "\n",
+ "This is an optional step for now. The training job downloads the data on its run directly from MNIST website to the data_group node (grpc server). "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "id": "d0534973",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import logging\n",
+ "import boto3\n",
+ "from botocore.exceptions import ClientError\n",
+ "\n",
+ "# Download training and testing data from a public S3 bucket\n",
+ "\n",
+ "\n",
+ "def download_from_s3(data_dir=\"./data\", train=True):\n",
+ " \"\"\"Download MNIST dataset and convert it to numpy array\n",
+ "\n",
+ " Args:\n",
+ " data_dir (str): directory to save the data\n",
+ " train (bool): download training set\n",
+ "\n",
+ " Returns:\n",
+ " None\n",
+ " \"\"\"\n",
+ "\n",
+ " if not os.path.exists(data_dir):\n",
+ " os.makedirs(data_dir)\n",
+ "\n",
+ " if train:\n",
+ " images_file = \"train-images-idx3-ubyte.gz\"\n",
+ " labels_file = \"train-labels-idx1-ubyte.gz\"\n",
+ " else:\n",
+ " images_file = \"t10k-images-idx3-ubyte.gz\"\n",
+ " labels_file = \"t10k-labels-idx1-ubyte.gz\"\n",
+ "\n",
+ " # download objects\n",
+ " s3 = boto3.client(\"s3\")\n",
+ " bucket = f\"sagemaker-sample-files\"\n",
+ " for obj in [images_file, labels_file]:\n",
+ " key = os.path.join(\"datasets/image/MNIST\", obj)\n",
+ " dest = os.path.join(data_dir, obj)\n",
+ " if not os.path.exists(dest):\n",
+ " s3.download_file(bucket, key, dest)\n",
+ " return\n",
+ "\n",
+ "\n",
+ "download_from_s3(\"./data\", True)\n",
+ "download_from_s3(\"./data\", False)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "id": "2d699654",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Upload to the default bucket\n",
+ "\n",
+ "prefix = \"DEMO-MNIST\"\n",
+ "bucket = sess.default_bucket()\n",
+ "loc = sess.upload_data(path=\"./data\", bucket=bucket, key_prefix=prefix)\n",
+ "\n",
+ "channels = {\"training\": loc, \"testing\": loc}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "48352f04",
+ "metadata": {},
+ "source": [
+ "## C. Submit the training job\n",
+ "\n",
+ "The job runs for the predefined iterations. DNN instance group sends a shutdown request to data group after done with the training. You can see the following entries in the CloudWatch logs of dnn instance. A job with 4800 iterations finishes in 29 mins in a Heterogeneous cluster composed of 1x ml.c5.9xlarge as data node and 1x ml.p3.2xlarge as DNN node.\n",
+ "\n",
+ "Note: The console output of billing seconds can be ignored. See the AWS console > SageMaker > Training Job for the exact billing seconds.\n",
+ "\n",
+ "Log excerpt from algo-1 (DNN instance)\n",
+ "```\n",
+ "4780: avg step time: 0.19709917231025106\n",
+ "INFO:__main__:4780: avg step time: 0.19709917231025106\n",
+ "4790: avg step time: 0.19694106239373696\n",
+ "INFO:__main__:4790: avg step time: 0.19694106239373696\n",
+ "4800: avg step time: 0.196784295383125\n",
+ "Saving the model\n",
+ "INFO:__main__:4800: avg step time: 0.196784295383125\n",
+ "INFO:__main__:Saving the model\n",
+ "Training job completed!\n",
+ "INFO:__main__:Training job completed!\n",
+ "Process train_dnn.py closed with returncode=0\n",
+ "Shutting downdata service dispatcher via: [algo-2:16000]\n",
+ "shutdown request sent to algo-2:16000\n",
+ "2022-08-16 01:15:05,555 sagemaker-training-toolkit INFO Waiting for the process to finish and give a return code.\n",
+ "2022-08-16 01:15:05,555 sagemaker-training-toolkit INFO Done waiting for a return code. Received 0 from exiting process.\n",
+ "2022-08-16 01:15:05,556 sagemaker-training-toolkit INFO Reporting training SUCCESS\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "31cb6cae",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2022-09-15 00:55:22 Starting - Starting the training job...\n",
+ "2022-09-15 00:55:50 Starting - Preparing the instances for training.........\n",
+ "2022-09-15 00:57:10 Downloading - Downloading input data.."
+ ]
+ }
+ ],
+ "source": [
+ "estimator.fit(\n",
+ " inputs=channels,\n",
+ " job_name=\"pt-hetero\"\n",
+ " + \"-\"\n",
+ " + \"H-\"\n",
+ " + str(IS_HETERO)[0]\n",
+ " + \"-\"\n",
+ " + datetime.datetime.utcnow().strftime(\"%Y%m%dT%H%M%SZ\"),\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3ea2e092",
+ "metadata": {},
+ "source": [
+ "## D. Monitoring Instance Metrics for GPU and CPU utilization\n",
+ "\n",
+ "Click on **View instance metrics** from the **Training jobs** node in **Amazon SageMaker Console**. In the run above, all 30 vCPU of Data node (algo-1) is approx. 100% utilized, and the GPU utilization is at 100% at frequent intervals in the DNN node (algo-2). To rescale the CloudWatch Metrics to 100% on CPU utilization for algo-1 and algo-2, use CloudWatch \"Add Math\" feature and average it out by no. of cores on those instance types.\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "430fb45e",
+ "metadata": {},
+ "source": [
+ "## E. Comparing time-to-train and cost-to-train\n",
+ "\n",
+ "Let's continue with the above example i.e. train a heavy data pre-processing (CPU intensive) model (MNIST) requiring only 1 GPU. We start with ml.p3.2xlarge (1xV100 GPU, 8x vCPU) in homogeneous cluster mode to get the baseline performance numbers. Due to the no. of CPU cores, we could not go beyond 8 data loader/workers for data pre-processing. The avg. step cost was `7.6 cents` and avg. step time is `1.19 seconds`. \n",
+ "\n",
+ "Our objective is to reduce the cost and speed up the model training time. The first choice here is to scale up the instance type in the same family. If we leverage the next instance type (4 GPU) in the P3 family, the GPUs would have gone underutilized. In this case, we needed more vCPU to GPU ratio. Assuming we haven't had any instance type in another instance family or the model is incompatible with the CPU/GPU architectures of other instance families, we are constrained to use ml.p3.2xlarge. The only way then to have more vCPUs to GPU ratio is to use SageMaker feature, Heterogeneous Cluster, which enables customers to offload data pre-processing logic to CPU only instance types example ml.c5. In the next test, we offloaded CPU intensive work i.e. data preprocessing to ml.c5.9xlarge (36 vCPU) and continued using ml.p3.2xlarge for DNN. The avg. step cost was `1.9 cents` and avg. step time is `0.18 seconds`. \n",
+ "\n",
+ "In summary, we reduced the training cost by 4.75 times, and the avg. step reduced by 6.5 times. This was possible because with higher cpu count, we could use 32 data loader workers (compared to 8 with p3.2xl) to preprocess the data, and kept GPU close to 100% utilized at frequent intervals. Note: These numbers are just taken as a sample, you have to do benchmarking with your own model and dataset to come up with the exact price-performance benefits. \n",
+ "\n",
+ "## F. Conclusion\n",
+ "In this notebook, we demonstrated how to leverage heterogeneous cluster feature of SageMaker Training to achieve better price performance. To get started you can copy this example project, and only change the `train_dnn.py` script.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.9.7 ('.venv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.7"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "77c0de85c2cb739aa5100af7b92fb9d2075368f0e653f4148499a56c989df5f7"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-cluster-diagram.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-cluster-diagram.png
new file mode 100644
index 0000000000..c84c185d31
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-cluster-diagram.png differ
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-instance-metrics.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-instance-metrics.png
new file mode 100644
index 0000000000..d812e2f46b
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/heterogeneous-instance-metrics.png differ
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/homogeneous-cluster-diagram.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/homogeneous-cluster-diagram.png
new file mode 100644
index 0000000000..ae3119d4b6
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/homogeneous-cluster-diagram.png differ
diff --git a/training/heterogeneous-clusters/pt.grpc.sagemaker/images/pytorch-heterogeneous-workflow.png b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/pytorch-heterogeneous-workflow.png
new file mode 100644
index 0000000000..c40b09aeed
Binary files /dev/null and b/training/heterogeneous-clusters/pt.grpc.sagemaker/images/pytorch-heterogeneous-workflow.png differ
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/heterogenenous-workload.json b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/heterogenenous-workload.json
new file mode 100644
index 0000000000..5b3736fc35
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/heterogenenous-workload.json
@@ -0,0 +1,30 @@
+{
+ "metrics": [
+ [ { "expression": "100*(m1/9600)", "label": "DNN1 CPU/100%", "id": "e1" } ],
+ [ { "expression": "100*(m2/800)", "label": "DNN1 GPU/100%", "id": "e2" } ],
+ [ { "expression": "100*(m3/7200)", "label": "DATA1 CPU/100%", "id": "e3" } ],
+ [ { "expression": "100*(m4/7200)", "label": "DATA2 CPU/100%", "id": "e4" } ],
+ [ "/aws/sagemaker/TrainingJobs", "CPUUtilization", "Host", "hetero-tf-data-service-Dnode2-wrkrs-1-20220922T214326Z/algo-1", { "id": "m1", "yAxis": "left", "visible": false } ],
+ [ ".", "GPUUtilization", ".", ".", { "id": "m2", "visible": false } ],
+ [ ".", "CPUUtilization", ".", "hetero-tf-data-service-Dnode2-wrkrs-1-20220922T214326Z/algo-2", { "id": "m3", "visible": false } ],
+ [ "...", "hetero-tf-data-service-Dnode2-wrkrs-1-20220922T214326Z/algo-3", { "id": "m4", "visible": false } ]
+ ],
+ "sparkline": true,
+ "view": "timeSeries",
+ "stacked": false,
+ "region": "us-east-1",
+ "stat": "Average",
+ "period": 60,
+ "setPeriodToTimeRange": true,
+ "yAxis": {
+ "left": {
+ "min": 0,
+ "max": 100,
+ "label": "% Utilization",
+ "showUnits": false
+ }
+ },
+ "legend": {
+ "position": "bottom"
+ }
+}
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/homogenous-workload copy.json b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/homogenous-workload copy.json
new file mode 100644
index 0000000000..c514eced5a
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/cloudwatch-metric-definitions/homogenous-workload copy.json
@@ -0,0 +1,26 @@
+{
+ "sparkline": true,
+ "metrics": [
+ [ { "expression": "100*(m1/9600)", "label": "CPU/100%", "id": "e1" } ],
+ [ { "expression": "100*(m2/800)", "label": "GPU/100%", "id": "e2" } ],
+ [ "/aws/sagemaker/TrainingJobs", "CPUUtilization", "Host", "hetero-tf-data-local-Dnode1-wrkrs-1-20220921T213920Z/algo-1", { "id": "m1", "visible": false } ],
+ [ ".", "GPUUtilization", ".", ".", { "id": "m2", "visible": false } ]
+ ],
+ "view": "timeSeries",
+ "stacked": false,
+ "region": "us-east-1",
+ "stat": "Average",
+ "period": 60,
+ "setPeriodToTimeRange": true,
+ "yAxis": {
+ "left": {
+ "min": 0,
+ "max": 100,
+ "label": "% Utilization",
+ "showUnits": false
+ }
+ },
+ "legend": {
+ "position": "bottom"
+ }
+}
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/launcher.py b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/launcher.py
new file mode 100644
index 0000000000..d67b27af7d
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/launcher.py
@@ -0,0 +1,123 @@
+import sys
+import os
+import time
+from typing import Optional
+import subprocess
+
+# instance group names
+DATA_GROUP = 'data_group'
+DNN_GROUP = 'dnn_group'
+
+
+def start_child_process_async(name : str, additional_args=[]) -> int:
+ #TODO: Find a way to stream stdout and stderr to the parent process
+ params = ["python", f"./{name}"] + sys.argv[1:] + additional_args
+ print(f'Opening process async: {params}')
+ p = subprocess.Popen(params)
+ print(f'Process {name} started')
+ return p.pid
+
+
+def start_child_process(name : str, additional_args=[]) -> int:
+ params = ["python", f"./{name}"] + sys.argv[1:] + additional_args
+ print(f'Opening process: {params}')
+ p = subprocess.run(params)
+ print(f'Process {name} closed with returncode={p.returncode}')
+ return p.returncode
+
+
+def start_data_group(dispatcher_host : str) -> int:
+ return start_child_process('train_data.py', ["--dispatcher_host", dispatcher_host])
+
+
+def not_mpi_or_rank_0() -> bool:
+ return 'OMPI_COMM_WORLD_LOCAL_RANK' not in os.environ or os.environ['OMPI_COMM_WORLD_LOCAL_RANK'] == '0'
+
+
+def start_dnn_group(dispatcher_host : Optional[str]) -> int:
+ if dispatcher_host is not None:
+ args = ["--dispatcher_host", dispatcher_host]
+ # Start a tf.data.service worker processes for each host in the DNN group
+ # to take advantage of its CPU resources.
+ # Start once per instance, not per MPI process
+ if not_mpi_or_rank_0():
+ start_child_process_async('train_data.py', args)
+ else:
+ args = []
+ return start_child_process('train_dnn.py', args)
+
+
+def get_group_first_host(instance_groups, target_group_name):
+ return instance_groups[target_group_name]['hosts'][0]
+
+
+def is_not_mpi_or_world_rank_0() -> bool:
+ return 'OMPI_COMM_WORLD_RANK' in os.environ and os.environ['OMPI_COMM_WORLD_RANK'] != '0'
+
+
+def shutdown_tf_data_service_with_retries(hosts : list):
+ # only world rank 0 process should shutdown the dispatcher
+ if is_not_mpi_or_world_rank_0():
+ return
+
+ completed_hosts = []
+ for host in hosts:
+ for i in range(0,12):
+ try:
+ if i>0:
+ sleeptime = 10
+ print(f'Will attempt {i} time to shutdown in {sleeptime} seconds')
+ time.sleep(sleeptime)
+
+ if host not in completed_hosts:
+ _shutdown_data_service(host)
+ completed_hosts.append(host)
+ break
+ except Exception as e:
+ print(f'Failed to shutdown dispatcher in {host} due to: {e}')
+
+
+def _shutdown_data_service(dispatcher_host : str):
+ SHUTDOWN_PORT = 16000
+ print(f'Shutting down tf.data.service dispatcher via: [{dispatcher_host}:{SHUTDOWN_PORT}]')
+ import socket
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+ s.connect((dispatcher_host, SHUTDOWN_PORT))
+ print(f'Shutdown request sent to {dispatcher_host}:{SHUTDOWN_PORT}')
+
+
+def split_to_instance_group_train_script() -> int:
+ from sagemaker_training import environment
+ env = environment.Environment()
+
+ print(f'env.is_hetero={env.is_hetero}')
+ print(f'current_host={env.current_host}')
+
+ if env.is_hetero:
+ dispatcher_host = get_group_first_host(env.instance_groups_dict, DATA_GROUP)
+ first_host_in_dnn_group = get_group_first_host(env.instance_groups_dict, DNN_GROUP)
+ print(f'current_instance_type={env.current_instance_type}')
+ print(f'current_group_name={env.current_instance_group}')
+ print(f'dispatcher_host={dispatcher_host}')
+ if env.current_instance_group == DATA_GROUP:
+ return start_data_group(dispatcher_host)
+ elif env.current_instance_group == DNN_GROUP:
+ returncode = start_dnn_group(dispatcher_host)
+ # first host in DNN group will take care of shutting down the dispatcher
+ if env.current_host == first_host_in_dnn_group:
+ hosts = env.instance_groups_dict[DATA_GROUP]['hosts'] + env.instance_groups_dict[DNN_GROUP]['hosts']
+ shutdown_tf_data_service_with_retries(hosts)
+ return returncode
+ else:
+ raise Exception(f'Unknown instance group: {env.current_instance_group}')
+
+ else: # not heterogenous
+ return start_dnn_group(dispatcher_host=None)
+
+if __name__ == "__main__":
+ try:
+ returncode = split_to_instance_group_train_script()
+ exit(returncode)
+ except Exception as e:
+ print(f'Failed due to {e}. exiting with returncode=1')
+ sys.exit(1)
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/requirements.txt b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/requirements.txt
new file mode 100644
index 0000000000..10aac59994
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/requirements.txt
@@ -0,0 +1,2 @@
+protobuf==3.20.2
+tensorflow-addons==0.17.0
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_data.py b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_data.py
new file mode 100644
index 0000000000..62248d1e68
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_data.py
@@ -0,0 +1,68 @@
+from tensorflow.data.experimental.service import DispatchServer, WorkerServer, DispatcherConfig, WorkerConfig
+
+def wait_for_shutdown_signal(dispatcher, workers):
+ SHUTDOWN_PORT = 16000
+ import socket
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind(('', SHUTDOWN_PORT))
+ s.listen(1)
+ print('Awaiting shutdown signal on port {}'.format(SHUTDOWN_PORT))
+ conn, addr = s.accept()
+ print('Received shutdown signal from: ', addr)
+ try:
+ conn.close()
+ s.close()
+ except Exception as e:
+ print(e)
+
+ if dispatcher is not None: # dispatcher runs only on the 1st data instance
+ print('Stopping dispatcher.')
+ dispatcher._stop()
+ print('Joining dispatcher')
+ dispatcher.join()
+
+ for i,worker in enumerate(workers, start=0):
+ print(f'Stopping worker {i}')
+ worker._stop()
+ print(f'Joining worker {i}')
+ worker.join()
+
+def create_worker(workerIndex : int, dispatcher_host : str, current_host : str) -> WorkerServer:
+ port = 6001 + workerIndex
+ w_config = WorkerConfig(port=port,
+ dispatcher_address=f'{dispatcher_host}:6000',
+ worker_address=f'{current_host}:{port}')
+ print(f'Starting tf.data.service WorkerServer {w_config}')
+ worker = WorkerServer(w_config)
+ return worker
+
+def start_dispatcher_and_worker(dispatcher_host : str, current_host : str, num_of_data_workers : int):
+ assert(dispatcher_host is not None)
+
+ if current_host == dispatcher_host:
+ print(f'starting Dispatcher (dispatcher_host={dispatcher_host})')
+ d_config = DispatcherConfig(port=6000)
+ dispatcher = DispatchServer(d_config)
+ else:
+ dispatcher = None
+
+ workers = [ create_worker(i, dispatcher_host, current_host) for i in range(num_of_data_workers) ]
+ print(f'Finished starting dispatcher and {num_of_data_workers} workers')
+
+ wait_for_shutdown_signal(dispatcher, workers)
+
+
+"This function read mode command line argument"
+def read_args():
+ import argparse, os
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--dispatcher_host", type=str)
+ parser.add_argument("--current_host", type=str, default=os.environ["SM_CURRENT_HOST"])
+ parser.add_argument("--num_of_data_workers", type=int)
+ args, unknown = parser.parse_known_args()
+ return args
+
+
+if __name__ == "__main__":
+ args = read_args()
+ start_dispatcher_and_worker(args.dispatcher_host, args.current_host, args.num_of_data_workers)
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_dnn.py b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_dnn.py
new file mode 100644
index 0000000000..cd938b46c7
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/code/train_dnn.py
@@ -0,0 +1,153 @@
+from tensorflow.keras.layers.experimental import preprocessing
+from tensorflow.keras.applications.resnet50 import ResNet50
+import tensorflow_addons as tfa
+import tensorflow as tf
+import os
+import horovod.tensorflow.keras as hvd
+
+
+# dilation filter
+def dilate(image, label):
+ dilateFilter = tf.zeros([3, 3, 3], tf.uint8)
+ image = tf.expand_dims(image, 0)
+ image = tf.nn.dilation2d(
+ image, dilateFilter, strides=[1, 1, 1, 1],
+ dilations=[1, 1, 1, 1],
+ padding='SAME',
+ data_format='NHWC')
+ image = tf.squeeze(image)
+ return image, label
+# blur filter
+
+
+def blur(image, label):
+ image = tfa.image.gaussian_filter2d(image=image,
+ filter_shape=(11, 11), sigma=0.8)
+ return image, label
+
+# rescale filter
+def rescale(image, label):
+ image = preprocessing.Rescaling(1.0 / 255)(image)
+ return image, label
+
+
+# augmentation filters
+def augment(image, label):
+ data_augmentation = tf.keras.Sequential(
+ [preprocessing.RandomFlip("horizontal"),
+ preprocessing.RandomRotation(0.1),
+ preprocessing.RandomZoom(0.1)])
+ image = data_augmentation(image)
+ return image, label
+
+
+# This function generates a dataset consisting 32x32x3 random images
+# And a corresponding random label representing 10 different classes.
+# As this dataset is randomly generated, you should not expect the model
+# to converge in a meaningful way, it doesn't matter as our intent is
+# only to measure data pipeline and DNN optimization throughput
+def generate_artificial_dataset():
+ import numpy as np
+ x_train = np.random.randint(0, 255, size=(32000, 32, 32, 3), dtype=np.uint8)
+ y_train = np.random.randint(0, 10, size=(32000,1))
+ train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
+ return train_dataset
+
+
+def get_dataset(batch_size : int, use_tf_data_service : bool, dispatcher_host : str):
+ autotune = tf.data.experimental.AUTOTUNE
+ options = tf.data.Options()
+ options.experimental_deterministic = False
+
+ ds = generate_artificial_dataset().shuffle(10000).repeat()
+
+ ds = ds.map(dilate, num_parallel_calls=autotune)
+ ds = ds.map(blur, num_parallel_calls=autotune)
+ ds = ds.map(rescale,num_parallel_calls=autotune)
+ ds = ds.map(augment, num_parallel_calls=autotune)
+ ds = ds.batch(batch_size)
+
+ if use_tf_data_service:
+ ds = ds.apply(tf.data.experimental.service.distribute(
+ processing_mode="parallel_epochs",
+ service=f'grpc://{dispatcher_host}:6000',),
+ )
+
+ #ds = ds.take(1).cache().repeat()
+ ds = ds.prefetch(autotune)
+ return ds
+
+"This function read mode command line argument"
+def read_args():
+ import argparse
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--tf_data_mode', type=str, default='local',
+ help="'service' distributed dataset using tf.data.service. 'local' use standard tf.data")
+ parser.add_argument('--steps_per_epoch', type=int, default=1)
+ parser.add_argument('--batch_size', type=int)
+ parser.add_argument('--epochs', type=int, default=1)
+ parser.add_argument("--n_gpus", type=str,
+ default=os.environ.get("SM_NUM_GPUS"))
+ parser.add_argument("--dispatcher_host", type=str)
+ parser.add_argument("--num_of_data_workers", type=int, default=1)
+ parser.add_argument("--output_data_dir", type=str,
+ default=os.environ.get("SM_OUTPUT_DATA_DIR"))
+ parser.add_argument("--model_dir", type=str,
+ default=os.environ.get("SM_MODEL_DIR"))
+ parser.add_argument("--checkpoint-path",type=str,default="/opt/ml/checkpoints",help="Path where checkpoints are saved.")
+ args = parser.parse_args()
+ return args
+
+if __name__ == "__main__":
+ args = read_args()
+ hvd.init()
+ # Horovod: pin GPU to be used to process local rank (one GPU per process)
+ gpus = tf.config.experimental.list_physical_devices('GPU')
+ print(str(gpus))
+ for gpu in gpus:
+ tf.config.experimental.set_memory_growth(gpu, True)
+ if gpus:
+ print(f'hvd.local_rank() {hvd.local_rank()}')
+ tf.config.experimental.set_visible_devices(gpus[hvd.local_rank()], 'GPU')
+
+ model = ResNet50(weights=None,
+ input_shape=(32, 32, 3),
+ classes=10)
+
+ model.compile(loss=tf.losses.SparseCategoricalCrossentropy(),
+ optimizer=tf.optimizers.Adam())
+ # Horovod: adjust learning rate based on number of GPUs.
+ scaled_lr = 0.001 * hvd.size()
+ opt = tf.optimizers.Adam(scaled_lr)
+ opt = hvd.DistributedOptimizer(
+ opt, backward_passes_per_step=1, average_aggregated_gradients=True)
+
+ model.compile(loss=tf.losses.SparseCategoricalCrossentropy(),
+ optimizer=opt,
+ experimental_run_tf_function=False)
+
+ callbacks = [
+ hvd.callbacks.BroadcastGlobalVariablesCallback(0),
+ hvd.callbacks.MetricAverageCallback(),
+ hvd.callbacks.LearningRateWarmupCallback(initial_lr=scaled_lr, warmup_epochs=3, verbose=1),
+ ]
+ # Horovod: save checkpoints only on worker 0 to prevent other workers from corrupting them.
+ if hvd.rank() == 0:
+ path = os.path.join(args.checkpoint_path, './checkpoint-{epoch}.h5')
+ callbacks.append(tf.keras.callbacks.ModelCheckpoint(path))
+
+ # Horovod: write logs on worker 0.
+ verbose = 1 if hvd.rank() == 0 else 0
+
+ assert(args.tf_data_mode == 'local' or args.tf_data_mode == 'service')
+ print(f'Running in {args.tf_data_mode} tf_data_mode.')
+ dataset = get_dataset(batch_size = args.batch_size, use_tf_data_service=(args.tf_data_mode == 'service'), dispatcher_host = args.dispatcher_host)
+
+ model.fit( dataset,
+ steps_per_epoch=args.steps_per_epoch,
+ callbacks=callbacks,
+ epochs=args.epochs,
+ verbose=2,)
+
+ if hvd.rank() == 0:
+ model.save(os.path.join(args.model_dir, '000000001'), 'my_model.h5')
\ No newline at end of file
diff --git a/training/heterogeneous-clusters/tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb b/training/heterogeneous-clusters/tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb
new file mode 100644
index 0000000000..61cdc56cf5
--- /dev/null
+++ b/training/heterogeneous-clusters/tf.data.service.sagemaker/hetero-tensorflow-restnet50.ipynb
@@ -0,0 +1,1136 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# TensorFlow's tf.data.service with Amazon SageMaker Training Heterogeneous Clusters\n",
+ "\n",
+ "---\n",
+ "### Introduction\n",
+ "\n",
+ "Heterogeneous clusters enable launching training jobs that use multiple instance types in a single job. This capability can improve your training cost and speed by running different parts of the model training on the most suitable instance type. This use case typically happens in computer vision (CV) deep learning (DL) training, where training is bottleneck on CPU resources needed for data augmentation, leaving the expensive GPU underutilized. Heterogeneous clusters enable you to add more CPU resources to fully utilize GPUs to increase training speed and cost-efficiency. For more details, you can find the documentation of this feature [here](https://docs.aws.amazon.com/sagemaker/latest/dg/train-heterogeneous-cluster.html).\n",
+ "\n",
+ "This notebook demonstrates how to use Heterogeneous Clusters with TensorFlow's [tf.data.service](https://www.TensorFlow.org/api_docs/python/tf/data/experimental/service). It includes training a CPU intensive DL CV workload. Comparing cost and performance between homogeneous and heterogeneous training configurations. \n",
+ "\n",
+ "💡To get started quickly with heterogeneous clusters, we suggest you'll reuse the provided code as a quick way to migrate your workload from a local tf.data pipeline to a distributed tf.data.service pipeline. You'll need to change [code/train_dnn.py](./code/train_dnn.py), while keeping [code/train_data.py](./code/train_data.py) and [code/launcher.py](code/launcher.py) intact. This is explained below in the [Workload Details] section.\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This notebook covers:\n",
+ "- A guide to switching from a homogeneous job (single instance type) to a heterogeneous job (multiple instance types)\n",
+ "- Explaining to use Heterogeneous clusters with TensorFlow's tf.data.service\n",
+ "- Set up Amazon SageMaker Studio Notebook \n",
+ "- Run homogeneous cluster training job \n",
+ "- Run heterogeneous cluster training job \n",
+ "- Compare time and cost to train between homogeneous and heterogeneous clusters\n",
+ "- Conclusion\n",
+ "\n",
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### A guide to switching from a homogeneous to a heterogeneous job\n",
+ "\n",
+ "This notebook runs and compares these two workloads:\n",
+ "\n",
+ "Homogeneous Training Job - The image shows a ml.p4d.24xlarge instance GPUs is under-utilized due to a CPU bottleneck. \n",
+ "\n",
+ " \n",
+ "Heterogeneous Training Job - The image shows two ml.c5.18xlarge instances with extra CPU cores, to reduce the CPU bottleneck and increase GPU usage, to improve training speed cost-efficiency. \n",
+ " \n",
+ "\n",
+ "In each workload: Training data is an artificially generated dataset consisting of 32x32x3 images with random pixel values, and a corresponding random label representing 10 different classes. As this dataset is randomly generated, you should not expect the model to converge in a meaningful way. This shouldn't matter as our intent is only to measure data pipeline and neural network optimization throughput expressed in epoch/step time. \n",
+ "The model we used is [Resnet50](https://www.TensorFlow.org/api_docs/python/tf/keras/applications/ResNet50). The workloads uses an 8 GPUs instance, ml.p4d.24xlarge, and uses Horovod for data parallelization. \n",
+ "\n",
+ "The heterogeneous job will include two instance groups:\n",
+ "- **data_group** - A group of CPU instances that will run data pre-processing code.\n",
+ "- **dnn_group** - A group of GPU instances that will run Deep Neural Network training code.\n",
+ "\n",
+ "In this example, the inter-node communication between CPU and GPU instance groups is implemented using [TensorFlow data service feature](https://www.TensorFlow.org/api_docs/python/tf/data/experimental/service). This feature allows offloading a configurable amount of preprocessing work to worker machines. Note that SageMaker's Heterogeneous cluster does not provide out-of-the-box support for inter-instance_group communication, and it is up to the user to implement (we provide reference implementation here).\n",
+ "\n",
+ "This notebook refers following files and folders:\n",
+ "- [code/train_dnn.py](./code/train_dnn.py) - this is standard TF training script, it has a single reference to tf.data.service when setting up the tf.data pipeline. This script will be executed on GPU instances belonging to the dnn_group.\n",
+ "- [code/train_data.py](./code/train_data.py) - this script starts tf.data.service services like a tf.data.service Dispatcher and tf.data.service Worker processes. You shouldn't edit this script when adjusting to your workload.\n",
+ "- [code/launcher.py](./code/launcher.py) - Entry point training script. This is the script that SageMaker Training will start on all instances (all instances groups must share the same entry point script in heterogeneous clusters). `launcher.py` is responsible for detecting the instance group the instance belong to, and start `train_dnn.py` and `train_data.py` accordingly. It is also responsible for shutting down tf.data.services the training script completes (`train_dnn.py`) so all instances exit allowing the SageMaker training job to complete. \n",
+ "In every instance `luncher.py` will use `train_data.py` to start a tf.data.service worker server (As all instance types have CPUs that could be used for preprocessing). `luncher.py` will start a single tf.data.service dispatcher server (on the first instance of the `data_group`). \n",
+ "`luncher.py` will start the `train_dnn.py` script in all GPU instances (`dnn_group` instances).\n",
+ "\n",
+ "#### Learn more about tf.data.service processes\n",
+ "`tf.data.service Dispatcher` - The dispatcher server acts as the control plain for tf.data.service; Being responsible for registering worker servers and assigning preprocessing tasks to them. Each training job has a single Dispatcher running in the first instance of the `data_group` and listens on port 6000.\n",
+ "`tf.data.service Workers` - Worker servers carry out the data processing. Each instance could have one or more workers (listen on port 6001/6002/...).\n",
+ "\n",
+ "##### Defining what part of your pipeline runs in which instance group\n",
+ " When you apply `tf.data.experimental.service.distribute()` on your dataset, all preprocessing operations defined up to the apply will run on the tf.data.service workers, and all dataset operations defined afterwords will run on the local process. All instances will need access to a dataset you'll make available through a SageMaker training data channel. You do have the option of limiting which instance group will see which training data channel.\n",
+ "\n",
+ "The below figure shows sequence of events of setting up and running in a tf.data.service based heterogeneous cluster training job.\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### security groups update if running in private VPC\n",
+ "This section is relevant if you plan to [run in a private VPC](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html) (passing `subnets` and `security_group_ids` parameters when defining an Estimator). \n",
+ "SageMaker documentation recommends you [add](https://docs.aws.amazon.com/sagemaker/latest/dg/train-vpc.html#train-vpc-vpc) a rule for your security group that allows inbound connections between members of the same security group, for all TCP communication. This will also cover for the tf.data.service related traffic between instances:\n",
+ "- tf.data.service Dispatcher node will listen for incoming connections on port 6000 (configurable) from all nodes.\n",
+ "- tf.data.service Workers will listen on ports 6001-6006 from all nodes.\n",
+ "- Each node listens on port 16000 for a tf.data.service shutdown signal from all nodes."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### A. Set up SageMaker Studio notebook\n",
+ "#### Before you start\n",
+ "Ensure you have selected Python 3 (_TensorFlow 2.6 Python 3.8 CPU Optimized_) image for your SageMaker Studio Notebook instance, and running on _ml.t3.medium_ instance type.\n",
+ "\n",
+ "#### Step 1 - Upgrade SageMaker SDK and dependent packages \n",
+ "Heterogeneous Clusters for Amazon SageMaker model training was [announced](https://aws.amazon.com/about-aws/whats-new/2022/07/announcing-heterogeneous-clusters-amazon-sagemaker-model-training) on 07/08/2022. This feature release requires you to have updated SageMaker SDK, Boto3 client."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Requirement already satisfied: boto3 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (1.24.72)\n",
+ "Collecting boto3\n",
+ " Downloading boto3-1.24.80-py3-none-any.whl (132 kB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 132.5/132.5 kB 925.9 kB/s eta 0:00:00\n",
+ "Requirement already satisfied: botocore in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (1.27.72)\n",
+ "Collecting botocore\n",
+ " Downloading botocore-1.27.80-py3-none-any.whl (9.1 MB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 9.1/9.1 MB 16.4 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: awscli in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (1.25.73)\n",
+ "Collecting awscli\n",
+ " Downloading awscli-1.25.81-py3-none-any.whl (3.9 MB)\n",
+ " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.9/3.9 MB 38.4 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: sagemaker in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (2.109.0)\n",
+ "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from boto3) (0.6.0)\n",
+ "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/jmespath-1.0.0-py3.9.egg (from boto3) (1.0.0)\n",
+ "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/urllib3-1.26.9-py3.9.egg (from botocore) (1.26.9)\n",
+ "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/python_dateutil-2.8.2-py3.9.egg (from botocore) (2.8.2)\n",
+ "Requirement already satisfied: PyYAML<5.5,>=3.10 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (5.4.1)\n",
+ "Requirement already satisfied: docutils<0.17,>=0.10 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (0.16)\n",
+ "Requirement already satisfied: colorama<0.4.5,>=0.2.5 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (0.4.4)\n",
+ "Requirement already satisfied: rsa<4.8,>=3.1.2 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from awscli) (4.7.2)\n",
+ "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/importlib_metadata-4.11.3-py3.9.egg (from sagemaker) (4.11.3)\n",
+ "Requirement already satisfied: pathos in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pathos-0.2.8-py3.9.egg (from sagemaker) (0.2.8)\n",
+ "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/protobuf3_to_dict-0.1.5-py3.9.egg (from sagemaker) (0.1.5)\n",
+ "Requirement already satisfied: pandas in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pandas-1.4.2-py3.9-macosx-10.9-x86_64.egg (from sagemaker) (1.4.2)\n",
+ "Requirement already satisfied: numpy<2.0,>=1.9.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from sagemaker) (1.22.4)\n",
+ "Requirement already satisfied: attrs<22,>=20.3.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/attrs-20.3.0-py3.9.egg (from sagemaker) (20.3.0)\n",
+ "Requirement already satisfied: smdebug-rulesconfig==1.0.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/smdebug_rulesconfig-1.0.1-py3.9.egg (from sagemaker) (1.0.1)\n",
+ "Requirement already satisfied: google-pasta in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/google_pasta-0.2.0-py3.9.egg (from sagemaker) (0.2.0)\n",
+ "Requirement already satisfied: protobuf<4.0,>=3.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from sagemaker) (3.20.1)\n",
+ "Requirement already satisfied: packaging>=20.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/packaging-21.3-py3.9.egg (from sagemaker) (21.3)\n",
+ "Requirement already satisfied: zipp>=0.5 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/zipp-3.7.0-py3.9.egg (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.7.0)\n",
+ "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pyparsing-3.0.7-py3.9.egg (from packaging>=20.0->sagemaker) (3.0.7)\n",
+ "Requirement already satisfied: six in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.15.0)\n",
+ "Requirement already satisfied: pyasn1>=0.1.3 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages (from rsa<4.8,>=3.1.2->awscli) (0.4.8)\n",
+ "Requirement already satisfied: pytz>=2020.1 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pytz-2022.1-py3.9.egg (from pandas->sagemaker) (2022.1)\n",
+ "Requirement already satisfied: dill>=0.3.4 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/dill-0.3.4-py3.9.egg (from pathos->sagemaker) (0.3.4)\n",
+ "Requirement already satisfied: multiprocess>=0.70.12 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/multiprocess-0.70.12.2-py3.9.egg (from pathos->sagemaker) (0.70.12.2)\n",
+ "Requirement already satisfied: pox>=0.3.0 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/pox-0.3.0-py3.9.egg (from pathos->sagemaker) (0.3.0)\n",
+ "Requirement already satisfied: ppft>=1.6.6.4 in /Users/gili/dev/hetro-training/.venv/lib/python3.9/site-packages/ppft-1.6.6.4-py3.9.egg (from pathos->sagemaker) (1.6.6.4)\n",
+ "Installing collected packages: botocore, boto3, awscli\n",
+ " Attempting uninstall: botocore\n",
+ " Found existing installation: botocore 1.27.72\n",
+ " Uninstalling botocore-1.27.72:\n",
+ " Successfully uninstalled botocore-1.27.72\n",
+ " Attempting uninstall: boto3\n",
+ " Found existing installation: boto3 1.24.72\n",
+ " Uninstalling boto3-1.24.72:\n",
+ " Successfully uninstalled boto3-1.24.72\n",
+ " Attempting uninstall: awscli\n",
+ " Found existing installation: awscli 1.25.73\n",
+ " Uninstalling awscli-1.25.73:\n",
+ " Successfully uninstalled awscli-1.25.73\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+ "sagemaker-training 4.2.2 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Successfully installed awscli-1.25.81 boto3-1.24.80 botocore-1.27.80\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%bash\n",
+ "python3 -m pip install --upgrade boto3 botocore awscli sagemaker"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 2 - Restart the notebook kernel "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#import IPython\n",
+ "#IPython.Application.instance().kernel.do_shutdown(True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 3 - Validate SageMaker Python SDK and TensorFlow versions\n",
+ "Ensure the output of the cell below reflects:\n",
+ "\n",
+ "- SageMaker Python SDK version 2.98.0 or above, \n",
+ "- boto3 1.24 or above \n",
+ "- botocore 1.27 or above \n",
+ "- TensorFlow 2.6 or above "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Name: sagemaker\n",
+ "Version: 2.109.0\n",
+ "---\n",
+ "Name: boto3\n",
+ "Version: 1.24.80\n",
+ "---\n",
+ "Name: botocore\n",
+ "Version: 1.27.80\n",
+ "---\n",
+ "Name: tensorflow\n",
+ "Version: 2.8.0\n",
+ "---\n",
+ "Name: protobuf\n",
+ "Version: 3.20.1\n"
+ ]
+ }
+ ],
+ "source": [
+ "!pip show sagemaker boto3 botocore tensorflow protobuf |egrep 'Name|Version|---'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import json\n",
+ "import datetime\n",
+ "\n",
+ "import sagemaker\n",
+ "from sagemaker import get_execution_role\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "\n",
+ "sess = sagemaker.Session()\n",
+ "role = get_execution_role()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### C. Run a homogeneous training job\n",
+ "#### Step 1: Set up the training environment\n",
+ "In this step, we define and submit a homogeneous training job. It uses a single instance type (p4d.24xlarge) with 8 GPUs. The analysis of the job will shows that it is CPU bound and therefore its GPUs are underutilized."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "from sagemaker.tensorflow import TensorFlow\n",
+ "from sagemaker.instance_group import InstanceGroup\n",
+ "import os\n",
+ "\n",
+ "hyperparameters = {\n",
+ " \"epochs\": 10,\n",
+ " \"steps_per_epoch\": 500,\n",
+ " \"batch_size\": 1024,\n",
+ " \"tf_data_mode\": \"local\", # We won't be using tf.data.service ('service') for this homogeneous job\n",
+ " \"num_of_data_workers\": 0, # We won't be using tf.data.service ('service') for this homogeneous job\n",
+ "}\n",
+ "\n",
+ "estimator = TensorFlow(\n",
+ " entry_point=\"launcher.py\",\n",
+ " source_dir=\"code\",\n",
+ " framework_version=\"2.9.1\",\n",
+ " py_version=\"py39\",\n",
+ " role=role,\n",
+ " volume_size=10,\n",
+ " max_run=1800, # 30 minutes\n",
+ " disable_profiler=True,\n",
+ " instance_type=\"ml.p4d.24xlarge\",\n",
+ " instance_count=1,\n",
+ " hyperparameters=hyperparameters,\n",
+ " distribution={\n",
+ " \"mpi\": {\n",
+ " \"enabled\": True,\n",
+ " \"processes_per_host\": 8, # 8 GPUs per host\n",
+ " \"custom_mpi_options\": \"--NCCL_DEBUG WARN\",\n",
+ " },\n",
+ " },\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Step 2: Submit the training job\n",
+ "\n",
+ "Note: For the logs, click on **View logs** from the **Training Jobs** node in **Amazon SageMaker Console**. \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "2022-09-24 11:28:23 Starting - Starting the training job......\n",
+ "2022-09-24 11:29:08 Starting - Preparing the instances for training........................\n",
+ "2022-09-24 11:33:34 Downloading - Downloading input data\n",
+ "2022-09-24 11:33:34 Training - Downloading the training image..................\n",
+ "2022-09-24 11:37:00 Training - Training image download completed. Training in progress..2022-09-24 11:37:05.792579: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "2022-09-24 11:37:05.801314: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:105] SageMaker Profiler is not enabled. The timeline writer thread will not be started, future recorded events will be dropped.\n",
+ "2022-09-24 11:37:06.269740: W tensorflow/core/profiler/internal/smprofiler_timeline.cc:460] Initializing the SageMaker Profiler.\n",
+ "2022-09-24 11:37:13,412 sagemaker-training-toolkit INFO Imported framework sagemaker_tensorflow_container.training\n",
+ "2022-09-24 11:37:14,075 sagemaker-training-toolkit INFO Installing dependencies from requirements.txt:\n",
+ "/usr/local/bin/python3.9 -m pip install -r requirements.txt\n",
+ "Collecting protobuf==3.20.1\n",
+ "Downloading protobuf-3.20.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl (1.0 MB)\n",
+ "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.0/1.0 MB 39.6 MB/s eta 0:00:00\n",
+ "Collecting tensorflow-addons==0.17.0\n",
+ "Downloading tensorflow_addons-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
+ "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.1/1.1 MB 51.6 MB/s eta 0:00:00\n",
+ "Requirement already satisfied: packaging in /usr/local/lib/python3.9/site-packages (from tensorflow-addons==0.17.0->-r requirements.txt (line 2)) (21.3)\n",
+ "Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.9/site-packages (from tensorflow-addons==0.17.0->-r requirements.txt (line 2)) (2.13.3)\n",
+ "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.9/site-packages (from packaging->tensorflow-addons==0.17.0->-r requirements.txt (line 2)) (3.0.9)\n",
+ "Installing collected packages: protobuf, tensorflow-addons\n",
+ "Attempting uninstall: protobuf\n",
+ "Found existing installation: protobuf 3.19.4\n",
+ "Uninstalling protobuf-3.19.4:\n",
+ "Successfully uninstalled protobuf-3.19.4\n",
+ "Attempting uninstall: tensorflow-addons\n",
+ "Found existing installation: tensorflow-addons 0.17.1\n",
+ "Uninstalling tensorflow-addons-0.17.1:\n",
+ "Successfully uninstalled tensorflow-addons-0.17.1\n",
+ "ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
+ "tf-models-official 2.9.1 requires tensorflow~=2.9.0, which is not installed.\n",
+ "tensorflow-gpu 2.9.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n",
+ "tensorboard 2.9.1 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n",
+ "sagemaker-training 4.1.4.dev0 requires protobuf<3.20,>=3.9.2, but you have protobuf 3.20.1 which is incompatible.\n",
+ "Successfully installed protobuf-3.20.1 tensorflow-addons-0.17.0\n",
+ "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n",
+ "[notice] A new release of pip available: 22.1.2 -> 22.2.2\n",
+ "[notice] To update, run: pip install --upgrade pip\n",
+ "2022-09-24 11:37:24,079 sagemaker-training-toolkit INFO Waiting for the process to finish and give a return code.\n",
+ "2022-09-24 11:37:24,079 sagemaker-training-toolkit INFO Done waiting for a return code. Received 0 from exiting process.\n",
+ "2022-09-24 11:37:24,258 sagemaker-training-toolkit INFO Starting MPI run as worker node.\n",
+ "2022-09-24 11:37:24,258 sagemaker-training-toolkit INFO Creating SSH daemon.\n",
+ "2022-09-24 11:37:24,274 sagemaker-training-toolkit INFO Waiting for MPI workers to establish their SSH connections\n",
+ "2022-09-24 11:37:24,274 sagemaker-training-toolkit INFO Env Hosts: ['algo-1'] Hosts: ['algo-1:8'] process_per_hosts: 8 num_processes: 8\n",
+ "2022-09-24 11:37:24,276 sagemaker-training-toolkit INFO Network interface name: eth0\n",
+ "2022-09-24 11:37:24,368 sagemaker-training-toolkit INFO Invoking user script\n",
+ "Training Env:\n",
+ "{\n",
+ " \"additional_framework_parameters\": {\n",
+ " \"sagemaker_mpi_custom_mpi_options\": \"--NCCL_DEBUG WARN\",\n",
+ " \"sagemaker_mpi_enabled\": true,\n",
+ " \"sagemaker_mpi_num_of_processes_per_host\": 8\n",
+ " },\n",
+ " \"channel_input_dirs\": {},\n",
+ " \"current_host\": \"algo-1\",\n",
+ " \"current_instance_group\": \"homogeneousCluster\",\n",
+ " \"current_instance_group_hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"distribution_hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"distribution_instance_groups\": [\n",
+ " \"homogeneousCluster\"\n",
+ " ],\n",
+ " \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"hyperparameters\": {\n",
+ " \"batch_size\": 1024,\n",
+ " \"epochs\": 10,\n",
+ " \"model_dir\": \"/opt/ml/model\",\n",
+ " \"num_of_data_workers\": 0,\n",
+ " \"steps_per_epoch\": 500,\n",
+ " \"tf_data_mode\": \"local\"\n",
+ " },\n",
+ " \"input_config_dir\": \"/opt/ml/input/config\",\n",
+ " \"input_data_config\": {},\n",
+ " \"input_dir\": \"/opt/ml/input\",\n",
+ " \"instance_groups\": [\n",
+ " \"homogeneousCluster\"\n",
+ " ],\n",
+ " \"instance_groups_dict\": {\n",
+ " \"homogeneousCluster\": {\n",
+ " \"instance_group_name\": \"homogeneousCluster\",\n",
+ " \"instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ]\n",
+ " }\n",
+ " },\n",
+ " \"is_hetero\": false,\n",
+ " \"is_master\": true,\n",
+ " \"job_name\": \"homogeneous-20220924T112821Z-1\",\n",
+ " \"log_level\": 20,\n",
+ " \"master_hostname\": \"algo-1\",\n",
+ " \"model_dir\": \"/opt/ml/model\",\n",
+ " \"module_dir\": \"s3://sagemaker-us-east-1-331113010199/homogeneous-20220924T112821Z-1/source/sourcedir.tar.gz\",\n",
+ " \"module_name\": \"launcher\",\n",
+ " \"network_interface_name\": \"eth0\",\n",
+ " \"num_cpus\": 96,\n",
+ " \"num_gpus\": 8,\n",
+ " \"output_data_dir\": \"/opt/ml/output/data\",\n",
+ " \"output_dir\": \"/opt/ml/output\",\n",
+ " \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
+ " \"resource_config\": {\n",
+ " \"current_host\": \"algo-1\",\n",
+ " \"current_instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"current_group_name\": \"homogeneousCluster\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ],\n",
+ " \"instance_groups\": [\n",
+ " {\n",
+ " \"instance_group_name\": \"homogeneousCluster\",\n",
+ " \"instance_type\": \"ml.p4d.24xlarge\",\n",
+ " \"hosts\": [\n",
+ " \"algo-1\"\n",
+ " ]\n",
+ " }\n",
+ " ],\n",
+ " \"network_interface_name\": \"eth0\"\n",
+ " },\n",
+ " \"user_entry_point\": \"launcher.py\"\n",
+ "}\n",
+ "Environment variables:\n",
+ "SM_HOSTS=[\"algo-1\"]\n",
+ "SM_NETWORK_INTERFACE_NAME=eth0\n",
+ "SM_HPS={\"batch_size\":1024,\"epochs\":10,\"model_dir\":\"/opt/ml/model\",\"num_of_data_workers\":0,\"steps_per_epoch\":500,\"tf_data_mode\":\"local\"}\n",
+ "SM_USER_ENTRY_POINT=launcher.py\n",
+ "SM_FRAMEWORK_PARAMS={\"sagemaker_mpi_custom_mpi_options\":\"--NCCL_DEBUG WARN\",\"sagemaker_mpi_enabled\":true,\"sagemaker_mpi_num_of_processes_per_host\":8}\n",
+ "SM_RESOURCE_CONFIG={\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"}\n",
+ "SM_INPUT_DATA_CONFIG={}\n",
+ "SM_OUTPUT_DATA_DIR=/opt/ml/output/data\n",
+ "SM_CHANNELS=[]\n",
+ "SM_CURRENT_HOST=algo-1\n",
+ "SM_CURRENT_INSTANCE_TYPE=ml.p4d.24xlarge\n",
+ "SM_CURRENT_INSTANCE_GROUP=homogeneousCluster\n",
+ "SM_CURRENT_INSTANCE_GROUP_HOSTS=[\"algo-1\"]\n",
+ "SM_INSTANCE_GROUPS=[\"homogeneousCluster\"]\n",
+ "SM_INSTANCE_GROUPS_DICT={\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}}\n",
+ "SM_DISTRIBUTION_INSTANCE_GROUPS=[\"homogeneousCluster\"]\n",
+ "SM_IS_HETERO=false\n",
+ "SM_MODULE_NAME=launcher\n",
+ "SM_LOG_LEVEL=20\n",
+ "SM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\n",
+ "SM_INPUT_DIR=/opt/ml/input\n",
+ "SM_INPUT_CONFIG_DIR=/opt/ml/input/config\n",
+ "SM_OUTPUT_DIR=/opt/ml/output\n",
+ "SM_NUM_CPUS=96\n",
+ "SM_NUM_GPUS=8\n",
+ "SM_MODEL_DIR=/opt/ml/model\n",
+ "SM_MODULE_DIR=s3://sagemaker-us-east-1-331113010199/homogeneous-20220924T112821Z-1/source/sourcedir.tar.gz\n",
+ "SM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_mpi_custom_mpi_options\":\"--NCCL_DEBUG WARN\",\"sagemaker_mpi_enabled\":true,\"sagemaker_mpi_num_of_processes_per_host\":8},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"current_instance_group\":\"homogeneousCluster\",\"current_instance_group_hosts\":[\"algo-1\"],\"current_instance_type\":\"ml.p4d.24xlarge\",\"distribution_hosts\":[\"algo-1\"],\"distribution_instance_groups\":[\"homogeneousCluster\"],\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch_size\":1024,\"epochs\":10,\"model_dir\":\"/opt/ml/model\",\"num_of_data_workers\":0,\"steps_per_epoch\":500,\"tf_data_mode\":\"local\"},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"instance_groups\":[\"homogeneousCluster\"],\"instance_groups_dict\":{\"homogeneousCluster\":{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}},\"is_hetero\":false,\"is_master\":true,\"job_name\":\"homogeneous-20220924T112821Z-1\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://sagemaker-us-east-1-331113010199/homogeneous-20220924T112821Z-1/source/sourcedir.tar.gz\",\"module_name\":\"launcher\",\"network_interface_name\":\"eth0\",\"num_cpus\":96,\"num_gpus\":8,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_group_name\":\"homogeneousCluster\",\"current_host\":\"algo-1\",\"current_instance_type\":\"ml.p4d.24xlarge\",\"hosts\":[\"algo-1\"],\"instance_groups\":[{\"hosts\":[\"algo-1\"],\"instance_group_name\":\"homogeneousCluster\",\"instance_type\":\"ml.p4d.24xlarge\"}],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"launcher.py\"}\n",
+ "SM_USER_ARGS=[\"--batch_size\",\"1024\",\"--epochs\",\"10\",\"--model_dir\",\"/opt/ml/model\",\"--num_of_data_workers\",\"0\",\"--steps_per_epoch\",\"500\",\"--tf_data_mode\",\"local\"]\n",
+ "SM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\n",
+ "SM_HP_BATCH_SIZE=1024\n",
+ "SM_HP_EPOCHS=10\n",
+ "SM_HP_MODEL_DIR=/opt/ml/model\n",
+ "SM_HP_NUM_OF_DATA_WORKERS=0\n",
+ "SM_HP_STEPS_PER_EPOCH=500\n",
+ "SM_HP_TF_DATA_MODE=local\n",
+ "PYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/local/lib/python39.zip:/usr/local/lib/python3.9:/usr/local/lib/python3.9/lib-dynload:/usr/local/lib/python3.9/site-packages:/usr/local/lib/python3.9/site-packages/smdebug-1.0.17b20220701-py3.9.egg:/usr/local/lib/python3.9/site-packages/pyinstrument-3.4.2-py3.9.egg:/usr/local/lib/python3.9/site-packages/pyinstrument_cext-0.2.4-py3.9-linux-x86_64.egg\n",
+ "Invoking script with the following command:\n",
+ "mpirun --host algo-1:8 -np 8 --allow-run-as-root --display-map --tag-output -mca btl_tcp_if_include eth0 -mca oob_tcp_if_include eth0 -mca plm_rsh_no_tree_spawn 1 -bind-to none -map-by slot -mca pml ob1 -mca btl ^openib -mca orte_abort_on_non_zero_status 1 -mca btl_vader_single_copy_mechanism none -x NCCL_MIN_NRINGS=4 -x NCCL_SOCKET_IFNAME=eth0 -x NCCL_DEBUG=WARN -x LD_LIBRARY_PATH -x PATH -x LD_PRELOAD=/usr/local/lib/python3.9/site-packages/gethostname.cpython-39-x86_64-linux-gnu.so -x SM_HOSTS -x SM_NETWORK_INTERFACE_NAME -x SM_HPS -x SM_USER_ENTRY_POINT -x SM_FRAMEWORK_PARAMS -x SM_RESOURCE_CONFIG -x SM_INPUT_DATA_CONFIG -x SM_OUTPUT_DATA_DIR -x SM_CHANNELS -x SM_CURRENT_HOST -x SM_CURRENT_INSTANCE_TYPE -x SM_CURRENT_INSTANCE_GROUP -x SM_CURRENT_INSTANCE_GROUP_HOSTS -x SM_INSTANCE_GROUPS -x SM_INSTANCE_GROUPS_DICT -x SM_DISTRIBUTION_INSTANCE_GROUPS -x SM_IS_HETERO -x SM_MODULE_NAME -x SM_LOG_LEVEL -x SM_FRAMEWORK_MODULE -x SM_INPUT_DIR -x SM_INPUT_CONFIG_DIR -x SM_OUTPUT_DIR -x SM_NUM_CPUS -x SM_NUM_GPUS -x SM_MODEL_DIR -x SM_MODULE_DIR -x SM_TRAINING_ENV -x SM_USER_ARGS -x SM_OUTPUT_INTERMEDIATE_DIR -x SM_HP_BATCH_SIZE -x SM_HP_EPOCHS -x SM_HP_MODEL_DIR -x SM_HP_NUM_OF_DATA_WORKERS -x SM_HP_STEPS_PER_EPOCH -x SM_HP_TF_DATA_MODE -x PYTHONPATH /usr/local/bin/python3.9 -m mpi4py launcher.py --batch_size 1024 --epochs 10 --model_dir /opt/ml/model --num_of_data_workers 0 --steps_per_epoch 500 --tf_data_mode local\n",
+ "Data for JOB [7555,1] offset 0 Total slots allocated 8\n",
+ " ======================== JOB MAP ========================\n",
+ " Data for node: ip-10-0-215-180#011Num slots: 8#011Max slots: 0#011Num procs: 8\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 0 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 1 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 2 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 3 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 4 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 5 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 6 Bound: N/A\n",
+ " #011Process OMPI jobid: [7555,1] App: 0 Process rank: 7 Bound: N/A\n",
+ " =============================================================\n",
+ "[1,mpirank:1,algo-1]