diff --git a/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist-handwritten-digits-classification-experiment.ipynb b/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist-handwritten-digits-classification-experiment.ipynb index 9b3c0bdf8c..51bfd3db54 100644 --- a/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist-handwritten-digits-classification-experiment.ipynb +++ b/sagemaker-experiments/mnist-handwritten-digits-classification-experiment/mnist-handwritten-digits-classification-experiment.ipynb @@ -65,7 +65,7 @@ "!{sys.executable} -m pip install torch==1.1.0\n", "!{sys.executable} -m pip install torchvision==0.3.0\n", "!{sys.executable} -m pip install pillow==6.2.2 ", - "!{sys.executable} -m pip install --upgrade sagemaker", + "!{sys.executable} -m pip install --upgrade sagemaker" ] }, { diff --git a/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py b/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py new file mode 100644 index 0000000000..280fc5ed9c --- /dev/null +++ b/sagemaker_neo_compilation_jobs/pytorch_torchvision/code/resnet18.py @@ -0,0 +1,78 @@ +import io +import json +import logging +import os +import pickle + +import numpy as np +import torch +import torchvision.transforms as transforms +from PIL import Image # Training container doesn't have this package + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def transform_fn(model, payload, request_content_type, + response_content_type): + + logger.info('Invoking user-defined transform function') + + if request_content_type != 'application/octet-stream': + raise RuntimeError( + 'Content type must be application/octet-stream. Provided: {0}'.format(request_content_type)) + + # preprocess + decoded = Image.open(io.BytesIO(payload)) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize( + mean=[ + 0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + normalized = preprocess(decoded) + batchified = normalized.unsqueeze(0) + + # predict + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + batchified = batchified.to(device) + result = model.forward(batchified) + + # Softmax (assumes batch size 1) + result = np.squeeze(result.cpu().numpy()) + result_exp = np.exp(result - np.max(result)) + result = result_exp / np.sum(result_exp) + + response_body = json.dumps(result.tolist()) + content_type = 'application/json' + + return response_body, content_type + + +def model_fn(model_dir): + + logger.info('model_fn') + with torch.neo.config(model_dir=model_dir, neo_runtime=True): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # The compiled model is saved as "compiled.pt" + model = torch.jit.load(os.path.join(model_dir, 'compiled.pt')) + model = model.to(device) + + # It is recommended to run warm-up inference during model load + sample_input_path = os.path.join(model_dir, 'sample_input.pkl') + with open(sample_input_path, 'rb') as input_file: + model_input = pickle.load(input_file) + if torch.is_tensor(model_input): + model_input = model_input.to(device) + model(model_input) + elif isinstance(model_input, tuple): + model_input = (inp.to(device) + for inp in model_input if torch.is_tensor(inp)) + model(*model_input) + else: + print("Only supports a torch tensor or a tuple of torch tensors") + + return model diff --git a/sagemaker_neo_compilation_jobs/pytorch_torchvision/pytorch_torchvision_neo.ipynb b/sagemaker_neo_compilation_jobs/pytorch_torchvision/pytorch_torchvision_neo.ipynb index 3deac240b8..691647bf9d 100644 --- a/sagemaker_neo_compilation_jobs/pytorch_torchvision/pytorch_torchvision_neo.ipynb +++ b/sagemaker_neo_compilation_jobs/pytorch_torchvision/pytorch_torchvision_neo.ipynb @@ -11,7 +11,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Amazon SageMaker Neo is API to compile machine learning models to optimize them for our choice of hardward targets. Currently, Neo supports pre-trained PyTorch models from [TorchVision](https://pytorch.org/docs/stable/torchvision/models.html). General support for other PyTorch models is forthcoming." + "Amazon SageMaker Neo is an API to compile machine learning models to optimize them for our choice of hardward targets. Currently, Neo supports pre-trained PyTorch models from [TorchVision](https://pytorch.org/docs/stable/torchvision/models.html). General support for other PyTorch models is forthcoming." ] }, { @@ -20,7 +20,16 @@ "metadata": {}, "outputs": [], "source": [ - "!~/anaconda3/envs/pytorch_p36/bin/pip install torch==1.2.0 torchvision==0.4.0" + "!~/anaconda3/envs/pytorch_p36/bin/pip install torch==1.4.0 torchvision==0.5.0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!~/anaconda3/envs/pytorch_p36/bin/pip install --upgrade sagemaker" ] }, { @@ -34,7 +43,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We'll import [ResNet18](https://arxiv.org/abs/1512.03385) model from TorchVision and create a model artifact `model.tar.gz`:" + "We'll import [ResNet18](https://arxiv.org/abs/1512.03385) model from TorchVision and create a model artifact `model.tar.gz`." ] }, { @@ -60,14 +69,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Invoke Neo Compilation API" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We then forward the model artifact to Neo Compilation API:" + "### Upload the model archive to S3" ] }, { @@ -87,111 +89,29 @@ "bucket = sess.default_bucket()\n", "\n", "compilation_job_name = name_from_base('TorchVision-ResNet18-Neo')\n", + "prefix = compilation_job_name+'/model'\n", "\n", - "model_key = '{}/model/model.tar.gz'.format(compilation_job_name)\n", - "model_path = 's3://{}/{}'.format(bucket, model_key)\n", - "boto3.resource('s3').Bucket(bucket).upload_file('model.tar.gz', model_key)\n", + "model_path = sess.upload_data(path='model.tar.gz', key_prefix=prefix)\n", "\n", - "sm_client = boto3.client('sagemaker')\n", "data_shape = '{\"input0\":[1,3,224,224]}'\n", "target_device = 'ml_c5'\n", "framework = 'PYTORCH'\n", - "framework_version = '1.2.0'\n", + "framework_version = '1.4.0'\n", "compiled_model_path = 's3://{}/{}/output'.format(bucket, compilation_job_name)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "response = sm_client.create_compilation_job(\n", - " CompilationJobName=compilation_job_name,\n", - " RoleArn=role,\n", - " InputConfig={\n", - " 'S3Uri': model_path,\n", - " 'DataInputConfig': data_shape,\n", - " 'Framework': framework\n", - " },\n", - " OutputConfig={\n", - " 'S3OutputLocation': compiled_model_path,\n", - " 'TargetDevice': target_device\n", - " },\n", - " StoppingCondition={\n", - " 'MaxRuntimeInSeconds': 300\n", - " }\n", - ")\n", - "print(response)\n", - "\n", - "# Poll every 30 sec\n", - "while True:\n", - " response = sm_client.describe_compilation_job(CompilationJobName=compilation_job_name)\n", - " if response['CompilationJobStatus'] == 'COMPLETED':\n", - " break\n", - " elif response['CompilationJobStatus'] == 'FAILED':\n", - " raise RuntimeError('Compilation failed')\n", - " print('Compiling ...')\n", - " time.sleep(30)\n", - "print('Done!')\n", - "\n", - "# Extract compiled model artifact\n", - "compiled_model_path = response['ModelArtifacts']['S3ModelArtifacts']" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create prediction endpoint" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To create a prediction endpoint, we first specify two additional functions, to be used with Neo Deep Learning Runtime:\n", - "\n", - "* `neo_preprocess(payload, content_type)`: Function that takes in the payload and Content-Type of each incoming request and returns a NumPy array. Here, the payload is byte-encoded NumPy array, so the function simply decodes the bytes to obtain the NumPy array.\n", - "* `neo_postprocess(result)`: Function that takes the prediction results produced by Deep Learining Runtime and returns the response body" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize resnet18.py" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Upload the Python script containing the two functions to S3:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "source_key = '{}/source/sourcedir.tar.gz'.format(compilation_job_name)\n", - "source_path = 's3://{}/{}'.format(bucket, source_key)\n", - "\n", - "with tarfile.open('sourcedir.tar.gz', 'w:gz') as f:\n", - " f.add('resnet18.py')\n", - "\n", - "boto3.resource('s3').Bucket(bucket).upload_file('sourcedir.tar.gz', source_key)" + "## Invoke Neo Compilation API" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We then create a SageMaker model record:" + "### Create a PyTorch SageMaker model" ] }, { @@ -200,31 +120,26 @@ "metadata": {}, "outputs": [], "source": [ - "from sagemaker.model import NEO_IMAGE_ACCOUNT\n", - "from sagemaker.fw_utils import create_image_uri\n", - "\n", - "model_name = name_from_base('TorchVision-ResNet18-Neo')\n", + "from sagemaker.pytorch.model import PyTorchModel\n", + "from sagemaker.predictor import Predictor\n", "\n", - "image_uri = create_image_uri(region, 'neo-' + framework.lower(), target_device.replace('_', '.'),\n", - " framework_version, py_version='py3', account=NEO_IMAGE_ACCOUNT[region])\n", - "\n", - "response = sm_client.create_model(\n", - " ModelName=model_name,\n", - " PrimaryContainer={\n", - " 'Image': image_uri,\n", - " 'ModelDataUrl': compiled_model_path,\n", - " 'Environment': { 'SAGEMAKER_SUBMIT_DIRECTORY': source_path }\n", - " },\n", - " ExecutionRoleArn=role\n", - ")\n", - "print(response)" + "sagemaker_model = PyTorchModel(model_data=model_path,\n", + " predictor_cls=Predictor,\n", + " framework_version = framework_version,\n", + " role=role,\n", + " sagemaker_session=sess,\n", + " entry_point='resnet18.py',\n", + " source_dir='code',\n", + " py_version='py3',\n", + " env={'MMS_DEFAULT_RESPONSE_TIMEOUT': '500'}\n", + " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Then we create an Endpoint Configuration:" + "### Use Neo compiler to compile the model" ] }, { @@ -233,28 +148,21 @@ "metadata": {}, "outputs": [], "source": [ - "config_name = model_name\n", - "\n", - "response = sm_client.create_endpoint_config(\n", - " EndpointConfigName=config_name,\n", - " ProductionVariants=[\n", - " {\n", - " 'VariantName': 'default-variant-name',\n", - " 'ModelName': model_name,\n", - " 'InitialInstanceCount': 1,\n", - " 'InstanceType': 'ml.c5.xlarge',\n", - " 'InitialVariantWeight': 1.0\n", - " },\n", - " ],\n", - ")\n", - "print(response)" + "compiled_model = sagemaker_model.compile(target_instance_family=target_device, \n", + " input_shape=data_shape,\n", + " job_name=compilation_job_name,\n", + " role=role,\n", + " framework=framework.lower(),\n", + " framework_version=framework_version,\n", + " output_path=compiled_model_path\n", + " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Finally, we create an Endpoint:" + "## Deploy the model" ] }, { @@ -263,20 +171,9 @@ "metadata": {}, "outputs": [], "source": [ - "endpoint_name = model_name + '-Endpoint'\n", - "\n", - "response = sm_client.create_endpoint(\n", - " EndpointName=endpoint_name,\n", - " EndpointConfigName=config_name,\n", - ")\n", - "print(response)\n", - "\n", - "print('Creating endpoint ...')\n", - "waiter = sm_client.get_waiter('endpoint_in_service')\n", - "waiter.wait(EndpointName=endpoint_name)\n", - "\n", - "response = sm_client.describe_endpoint(EndpointName=endpoint_name)\n", - "print(response)" + "predictor = compiled_model.deploy(initial_instance_count = 1,\n", + " instance_type = 'ml.c5.9xlarge'\n", + " )" ] }, { @@ -301,19 +198,15 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", "import numpy as np\n", - "\n", - "sm_runtime = boto3.Session().client('sagemaker-runtime')\n", + "import json\n", "\n", "with open('cat.jpg', 'rb') as f:\n", " payload = f.read()\n", + " payload = bytearray(payload) \n", "\n", - "response = sm_runtime.invoke_endpoint(EndpointName=endpoint_name,\n", - " ContentType='application/x-image',\n", - " Body=payload)\n", - "print(response)\n", - "result = json.loads(response['Body'].read().decode())\n", + "response = predictor.predict(payload)\n", + "result = json.loads(response.decode())\n", "print('Most likely class: {}'.format(np.argmax(result)))" ] }, @@ -346,7 +239,7 @@ "metadata": {}, "outputs": [], "source": [ - "sess.delete_endpoint(endpoint_name)" + "sess.delete_endpoint(predictor.endpoint_name)" ] } ], @@ -366,9 +259,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.10" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/sagemaker_neo_compilation_jobs/pytorch_torchvision/resnet18.py b/sagemaker_neo_compilation_jobs/pytorch_torchvision/resnet18.py deleted file mode 100644 index 2f7018f46d..0000000000 --- a/sagemaker_neo_compilation_jobs/pytorch_torchvision/resnet18.py +++ /dev/null @@ -1,46 +0,0 @@ -def neo_preprocess(payload, content_type): - import PIL.Image # Training container doesn't have this package - import logging - import numpy as np - import io - - logging.info('Invoking user-defined pre-processing function') - - if content_type != 'application/x-image': - raise RuntimeError('Content type must be application/x-image') - - f = io.BytesIO(payload) - # Load image and convert to RGB space - image = PIL.Image.open(f).convert('RGB') - # Resize - image = np.asarray(image.resize((224, 224))) - - # Normalize - mean_vec = np.array([0.485, 0.456, 0.406]) - stddev_vec = np.array([0.229, 0.224, 0.225]) - image = (image/255- mean_vec)/stddev_vec - - # Transpose - if len(image.shape) == 2: # for greyscale image - image = np.expand_dims(image, axis=2) - - image = np.rollaxis(image, axis=2, start=0)[np.newaxis, :] - - return image - -def neo_postprocess(result): - import logging - import numpy as np - import json - - logging.info('Invoking user-defined post-processing function') - - # Softmax (assumes batch size 1) - result = np.squeeze(result) - result_exp = np.exp(result - np.max(result)) - result = result_exp / np.sum(result_exp) - - response_body = json.dumps(result.tolist()) - content_type = 'application/json' - - return response_body, content_type \ No newline at end of file diff --git a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py new file mode 100644 index 0000000000..8e39f2003b --- /dev/null +++ b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_compiled.py @@ -0,0 +1,78 @@ +import io +import json +import logging +import os +import pickle + +import numpy as np +import torch +import torchvision.transforms as transforms +from PIL import Image # Training container doesn't have this package + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def transform_fn(model, payload, request_content_type, + response_content_type): + + logger.info('Invoking user-defined transform function') + + if request_content_type != 'application/octet-stream': + raise RuntimeError( + 'Content type must be application/octet-stream. Provided: {0}'.format(request_content_type)) + + # preprocess + decoded = Image.open(io.BytesIO(payload)) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize( + mean=[ + 0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + normalized = preprocess(decoded) + batchified = normalized.unsqueeze(0) + + # predict + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + batchified = batchified.to(device) + result = model.forward(batchified) + + # Softmax (assumes batch size 1) + result = np.squeeze(result.cpu().detach().numpy()) + result_exp = np.exp(result - np.max(result)) + result = result_exp / np.sum(result_exp) + + response_body = json.dumps(result.tolist()) + content_type = 'application/json' + + return response_body, content_type + + +def model_fn(model_dir): + + logger.info('model_fn') + with torch.neo.config(model_dir=model_dir, neo_runtime=True): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # The compiled model is saved as "compiled.pt" + model = torch.jit.load(os.path.join(model_dir, 'compiled.pt')) + model = model.to(device) + + # It is recommended to run warm-up inference during model load + sample_input_path = os.path.join(model_dir, 'sample_input.pkl') + with open(sample_input_path, 'rb') as input_file: + model_input = pickle.load(input_file) + if torch.is_tensor(model_input): + model_input = model_input.to(device) + model(model_input) + elif isinstance(model_input, tuple): + model_input = (inp.to(device) + for inp in model_input if torch.is_tensor(inp)) + model(*model_input) + else: + print("Only supports a torch tensor or a tuple of torch tensors") + + return model diff --git a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py new file mode 100644 index 0000000000..164f534c11 --- /dev/null +++ b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/code/vgg19_bn_uncompiled.py @@ -0,0 +1,64 @@ +import io +import json +import logging +import os +import pickle + +import numpy as np +import torch +import torchvision.transforms as transforms +from PIL import Image # Training container doesn't have this package + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +def transform_fn(model, payload, request_content_type, + response_content_type): + + logger.info('Invoking user-defined transform function') + + if request_content_type != 'application/octet-stream': + raise RuntimeError( + 'Content type must be application/octet-stream. Provided: {0}'.format(request_content_type)) + + # preprocess + decoded = Image.open(io.BytesIO(payload)) + preprocess = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize( + mean=[ + 0.485, 0.456, 0.406], std=[ + 0.229, 0.224, 0.225]), + ]) + normalized = preprocess(decoded) + batchified = normalized.unsqueeze(0) + + # predict + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + batchified = batchified.to(device) + result = model.forward(batchified) + + # Softmax (assumes batch size 1) + result = np.squeeze(result.cpu().detach().numpy()) + result_exp = np.exp(result - np.max(result)) + result = result_exp / np.sum(result_exp) + + response_body = json.dumps(result.tolist()) + content_type = 'application/json' + + return response_body, content_type + + +def model_fn(model_dir): + + logger.info('model_fn') + with torch.neo.config(model_dir=model_dir, neo_runtime=True): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + # The compiled model is saved as "compiled.pt" + model = torch.jit.load(os.path.join(model_dir, 'model.pth')) + model = model.to(device) + + return model diff --git a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/pytorch-vgg19-bn.ipynb b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/pytorch-vgg19-bn.ipynb index 76d511d2d9..0631520344 100644 --- a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/pytorch-vgg19-bn.ipynb +++ b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/pytorch-vgg19-bn.ipynb @@ -24,7 +24,16 @@ "metadata": {}, "outputs": [], "source": [ - "!~/anaconda3/envs/pytorch_p36/bin/pip install torch==1.2.0 torchvision==0.4.0" + "!~/anaconda3/envs/pytorch_p36/bin/pip install torch==1.4.0 torchvision==0.5.0" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!~/anaconda3/envs/pytorch_p36/bin/pip install --upgrade sagemaker" ] }, { @@ -84,6 +93,7 @@ "import sagemaker\n", "import time\n", "from sagemaker.utils import name_from_base\n", + "from sagemaker import image_uris\n", "\n", "role = sagemaker.get_execution_role()\n", "sess = sagemaker.Session()\n", @@ -96,10 +106,12 @@ "model_path = sess.upload_data(path='model.tar.gz', key_prefix=prefix)\n", "\n", "data_shape = '{\"input0\":[1,3,224,224]}'\n", - "target_device = 'ml_m5'\n", - "framework = 'PYTORCH'\n", - "framework_version = '1.2.0'\n", - "compiled_model_path = 's3://{}/{}/output'.format(bucket, compilation_job_name)" + "target_device = 'ml_c5'\n", + "framework = 'pytorch'\n", + "framework_version = '1.4.0'\n", + "compiled_model_path = 's3://{}/{}/output'.format(bucket, compilation_job_name)\n", + "\n", + "inference_image_uri = image_uris.retrieve(f'neo-{framework}', region, framework_version, instance_type=target_device)" ] }, { @@ -116,13 +128,17 @@ "outputs": [], "source": [ "from sagemaker.pytorch.model import PyTorchModel\n", + "from sagemaker.predictor import Predictor\n", "\n", "pt_vgg = PyTorchModel(model_data=model_path,\n", " framework_version=framework_version,\n", - " role=role, \n", - " entry_point='vgg19_bn.py',\n", + " predictor_cls=Predictor,\n", + " role=role, \n", " sagemaker_session=sess,\n", - " py_version='py3'\n", + " entry_point='vgg19_bn_uncompiled.py',\n", + " source_dir='code',\n", + " py_version='py3',\n", + " image_uri=inference_image_uri\n", " )" ] }, @@ -140,7 +156,7 @@ "outputs": [], "source": [ "vgg_predictor = pt_vgg.deploy(initial_instance_count = 1,\n", - " instance_type = 'ml.m5.12xlarge'\n", + " instance_type = 'ml.c5.9xlarge'\n", " )" ] }, @@ -167,7 +183,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Image Pre-processing" + "#### Read the image payload" ] }, { @@ -176,19 +192,11 @@ "metadata": {}, "outputs": [], "source": [ - "import torch\n", - "from PIL import Image\n", - "from torchvision import transforms\n", - "import numpy as np\n", - "input_image = Image.open('cat.jpg')\n", - "preprocess = transforms.Compose([\n", - " transforms.Resize(256),\n", - " transforms.CenterCrop(224),\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n", - "])\n", - "input_tensor = preprocess(input_image)\n", - "input_batch = input_tensor.unsqueeze(0) # create a mini-batch as expected by the model" + "import json\n", + "\n", + "with open('cat.jpg', 'rb') as f:\n", + " payload = f.read()\n", + " payload = bytearray(payload) " ] }, { @@ -207,7 +215,7 @@ "import time\n", "start = time.time()\n", "for _ in range(1000):\n", - " output = vgg_predictor.predict(input_batch)\n", + " output = vgg_predictor.predict(payload)\n", "inference_time = (time.time()-start)\n", "print('Inference time is ' + str(inference_time) + 'millisecond')" ] @@ -218,7 +226,9 @@ "metadata": {}, "outputs": [], "source": [ - "_, predicted = torch.max(torch.from_numpy(np.array(output)), 1)" + "import numpy as np\n", + "result = json.loads(output.decode())\n", + "predicted = np.argmax(result)" ] }, { @@ -241,7 +251,7 @@ "metadata": {}, "outputs": [], "source": [ - "print(\"Result: label - \" + object_categories[str(predicted.item())])" + "print(\"Result: label - \" + object_categories[str(predicted)])" ] }, { @@ -258,7 +268,7 @@ "metadata": {}, "outputs": [], "source": [ - "sess.delete_endpoint(vgg_predictor.endpoint)" + "sess.delete_endpoint(vgg_predictor.endpoint_name)" ] }, { @@ -272,20 +282,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Fetch neo container image for PyTorch" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sagemaker.model import NEO_IMAGE_ACCOUNT\n", - "from sagemaker.fw_utils import create_image_uri\n", - "\n", - "image_uri = create_image_uri(region, 'neo-' + framework.lower(), target_device.replace('_', '.'),\n", - " framework_version, py_version='py3', account=NEO_IMAGE_ACCOUNT[region])" + "### Create a PyTorch SageMaker model" ] }, { @@ -295,16 +292,17 @@ "outputs": [], "source": [ "from sagemaker.pytorch.model import PyTorchModel\n", - "from sagemaker.predictor import RealTimePredictor\n", + "from sagemaker.predictor import Predictor\n", "\n", "sagemaker_model = PyTorchModel(model_data=model_path,\n", - " image=image_uri,\n", - " predictor_cls=RealTimePredictor,\n", + " predictor_cls=Predictor,\n", " framework_version = framework_version,\n", " role=role,\n", " sagemaker_session=sess,\n", - " entry_point='vgg19_bn.py',\n", - " py_version='py3'\n", + " entry_point='vgg19_bn_compiled.py',\n", + " source_dir='code',\n", + " py_version='py3',\n", + " env={'MMS_DEFAULT_RESPONSE_TIMEOUT': '500'}\n", " )" ] }, @@ -312,7 +310,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Use Neo compiler to compile the model" + "### Use Neo compiler to compile the model" ] }, { @@ -325,7 +323,7 @@ " input_shape=data_shape,\n", " job_name=compilation_job_name,\n", " role=role,\n", - " framework=framework,\n", + " framework=framework.lower(),\n", " framework_version=framework_version,\n", " output_path=compiled_model_path\n", " )" @@ -338,32 +336,10 @@ "outputs": [], "source": [ "predictor = compiled_model.deploy(initial_instance_count = 1,\n", - " instance_type = 'ml.m5.12xlarge'\n", + " instance_type = 'ml.c5.9xlarge'\n", " )" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import json\n", - "\n", - "with open('cat.jpg', 'rb') as f:\n", - " payload = f.read()\n", - " payload = bytearray(payload) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predictor.content_type = 'application/x-image'" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -402,8 +378,15 @@ "metadata": {}, "outputs": [], "source": [ - "sess.delete_endpoint(predictor.endpoint)" + "sess.delete_endpoint(predictor.endpoint_name)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -422,9 +405,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.6.10" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/vgg19_bn.py b/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/vgg19_bn.py deleted file mode 100644 index 00eab4f2f6..0000000000 --- a/sagemaker_neo_compilation_jobs/pytorch_vgg19_bn/vgg19_bn.py +++ /dev/null @@ -1,70 +0,0 @@ -# ------------------------------------------------------------ # -# Neo host methods # -# ------------------------------------------------------------ # - -def neo_preprocess(payload, content_type): - import PIL.Image # Training container doesn't have this package - import logging - import numpy as np - import io - - logging.info('Invoking user-defined pre-processing function') - - if content_type != 'application/x-image': - raise RuntimeError('Content type must be application/x-image') - - f = io.BytesIO(payload) - # Load image and convert to RGB space - image = PIL.Image.open(f).convert('RGB') - # Resize - image = np.asarray(image.resize((224, 224))) - - # Normalize - mean_vec = np.array([0.485, 0.456, 0.406]) - stddev_vec = np.array([0.229, 0.224, 0.225]) - image = (image/255- mean_vec)/stddev_vec - - # Transpose - if len(image.shape) == 2: # for greyscale image - image = np.expand_dims(image, axis=2) - - image = np.rollaxis(image, axis=2, start=0)[np.newaxis, :] - - return image - -def neo_postprocess(result): - import logging - import numpy as np - import json - - logging.info('Invoking user-defined post-processing function') - - # Softmax (assumes batch size 1) - result = np.squeeze(result) - result_exp = np.exp(result - np.max(result)) - result = result_exp / np.sum(result_exp) - - response_body = json.dumps(result.tolist()) - content_type = 'application/json' - - return response_body, content_type - -# ------------------------------------------------------------ # -# Sagemaker Hosting methods # -# ------------------------------------------------------------ # - -import os -import logging - -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - -def model_fn(model_dir): - import torch - - logger.info('model_fn') - device = "cuda" if torch.cuda.is_available() else "cpu" - with open(os.path.join(model_dir, 'model.pth'), 'rb') as f: - model = torch.jit.load(f) - return model.to(device) - diff --git a/template.ipynb b/template.ipynb new file mode 100644 index 0000000000..8f42b0a07c --- /dev/null +++ b/template.ipynb @@ -0,0 +1,333 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Title\n", + "The title should be similar to the filename, but the filename should be very concise and compact, so people can read what it is when displayed in a list view in JupyterLab.\n", + "\n", + "Example title - **Amazon SageMaker Processing: pre-processing images with PyTorch using a GPU instance type**\n", + "\n", + "* Bad example filename: *amazon_sagemaker-processing-images_with_pytorch_on_GPU.ipynb* (too long & mixes case, dashes, and underscores)\n", + "* Good example filename: *processing_images_pytorch_gpu.ipynb* (succinct, all lowercase, all underscores)\n", + "\n", + "**IMPORTANT:** Use only one maining heading with `#`, so your next subheading is `##` or `###` and so on.\n", + "\n", + "## Overview\n", + "1. What does this notebook do?\n", + " - What will the user learn how to do?\n", + "1. Is this an end-to-end tutorial or it is a how-to (procedural) example?\n", + " - Tutorial: add conceptual information, flowcharts, images\n", + " - How to: notebook should be lean. More of a list of steps. No conceptual info, but links to resources for more info.\n", + "1. Who is the audience? \n", + " - What should the user be familiar with before running this? \n", + " - Link to other examples they should have run first.\n", + "1. How much will this cost?\n", + " - Some estimate of both time and money is recommended.\n", + " - List the instance types and other resources that are created.\n", + "\n", + "\n", + "## Prerequisites\n", + "1. Which environments does this notebook work in? Select all that apply.\n", + " - Notebook Instances: Jupyter?\n", + " - Notebook Instances: JupyterLab?\n", + " - Studio?\n", + "1. Which conda kernel is required?\n", + "1. Is there a previous notebook that is required?\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup \n", + "\n", + "### Setup Dependencies\n", + "\n", + "1. Describe any pip or conda or apt installs or setup scripts that are needed.\n", + "1. Pin sagemaker if version <2 is required.\n", + "\n", + " `%pip install \"sagemaker>=1.14.2,<2\"`\n", + " \n", + " \n", + "1. Upgrade sagemaker if version 2 is required, but rollback upgrades to packages that might taint the user's kernel and make other notebooks break. Do this at the end of the notebook in the cleanup cell.\n", + "\n", + " ```python\n", + " # setup\n", + " import sagemaker\n", + " version = sagemaker.__version__\n", + " %pip install 'sagemaker>=2.0.0'\n", + " ...\n", + " # cleanup\n", + " %pip install 'sagemaker=={}'.format(version)\n", + " ```\n", + " \n", + "\n", + "1. Use flags that facilitate automatic, end-to-end running without a user prompt, so that the notebook can run in CI without any updates or special configuration." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# SageMaker Python SDK version 1.x is required\n", + "import sys\n", + "%pip install \"sagemaker>=1.14.2,<2\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# SageMaker Python SDK version 2.x is required\n", + "import sagemaker\n", + "import sys\n", + "original_version = sagemaker.__version__\n", + "%pip install 'sagemaker>=2.0.0'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setup Python Modules\n", + "1. Import modules, set options, and activate extensions." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2019-06-16T14:44:50.874881Z", + "start_time": "2019-06-16T14:44:38.616867Z" + } + }, + "outputs": [], + "source": [ + "# imports\n", + "import sagemaker\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "# options\n", + "pd.options.display.max_columns = 50\n", + "pd.options.display.max_rows = 30\n", + "\n", + "# visualizations\n", + "import plotly\n", + "import plotly.graph_objs as go\n", + "import plotly.offline as ply\n", + "plotly.offline.init_notebook_mode(connected=True)\n", + "\n", + "# extensions\n", + "if 'autoreload' not in get_ipython().extension_manager.loaded:\n", + " %load_ext autoreload\n", + " \n", + "%autoreload 2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Parameters\n", + "1. Setup user supplied parameters like custom bucket names and roles in a separated cell and call out what their options are.\n", + "1. Use defaults, so the notebook will still run end-to-end without any user modification.\n", + "\n", + "For example, the following description & code block prompts the user to select the preferred dataset.\n", + "\n", + "~~~\n", + "\n", + "To do select a particular dataset, assign choosen_data_set below to be one of 'diabetes', 'california', or 'boston' where each name corresponds to the it's respective dataset.\n", + "\n", + "'boston' : boston house data\n", + "'california' : california house data\n", + "'diabetes' : diabetes data\n", + "\n", + "~~~\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_sets = {'diabetes': 'load_diabetes()', 'california': 'fetch_california_housing()', 'boston' : 'load_boston()'}\n", + "\n", + "# Change choosen_data_set variable to one of the data sets above. \n", + "choosen_data_set = 'california'\n", + "assert choosen_data_set in data_sets.keys()\n", + "print(\"I selected the '{}' dataset!\".format(choosen_data_set))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Data import\n", + "1. Look for the data that was stored by a previous notebook run `%store -r variableName`\n", + "1. If that doesn't exist, look in S3 in their default bucket\n", + "1. If that doesn't exist, download it from the [SageMaker dataset bucket](https://sagemaker-sample-files.s3.amazonaws.com/) \n", + "1. If that doesn't exist, download it from origin\n", + "\n", + "For example, the following code block will pull training and validation data that was created in a previous notebook. This allows the customer to experiment with features, re-run the notebook, and not have it pull the dataset over and over." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load relevant dataframes and variables from preprocessing_tabular_data.ipynb required for this notebook\n", + "%store -r X_train\n", + "%store -r X_test\n", + "%store -r X_val\n", + "%store -r Y_train\n", + "%store -r Y_test\n", + "%store -r Y_val\n", + "%store -r choosen_data_set" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Procedure or tutorial\n", + "1. Break up processes with Markdown blocks to explain what's going on.\n", + "1. Make use of visualizations to better demonstrate each step." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Cleanup\n", + "1. If you upgraded their `sagemaker` SDK, roll it back.\n", + "1. Delete any endpoints or other resources that linger and might cost the user money.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# rollback the SageMaker Python SDK to the kernel's original version\n", + "print(\"Original version: {}\".format(original_version))\n", + "print(\"Current version: {}\".format(sagemaker.__version__))\n", + "s = 'sagemaker=={}'.format(version)\n", + "print(\"Rolling back to... {}\".format(s))\n", + "%pip install {s}\n", + "import sagemaker\n", + "print(\"{} installed!\".format(sagemaker.__version__))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next steps\n", + "\n", + "1. Wrap up with some conclusion or overview of what was accomplished.\n", + "1. Offer another notebook or more resources or some other call to action." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## References\n", + "1. author1, article1, journal1, year1, url1\n", + "2. author2, article2, journal2, year2, url2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "conda_python3", + "language": "python", + "name": "conda_python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + }, + "pycharm": { + "stem_cell": { + "cell_type": "raw", + "metadata": { + "collapsed": false + }, + "source": [] + } + }, + "toc": { + "base_numbering": 1, + "nav_menu": {}, + "number_sections": true, + "sideBar": true, + "skip_h1_title": false, + "title_cell": "Table of Contents", + "title_sidebar": "Contents", + "toc_cell": false, + "toc_position": {}, + "toc_section_display": true, + "toc_window_display": false + }, + "varInspector": { + "cols": { + "lenName": 16, + "lenType": 16, + "lenVar": 40 + }, + "kernels_config": { + "python": { + "delete_cmd_postfix": "", + "delete_cmd_prefix": "del ", + "library": "var_list.py", + "varRefreshCmd": "print(var_dic_list())" + }, + "r": { + "delete_cmd_postfix": ") ", + "delete_cmd_prefix": "rm(", + "library": "var_list.r", + "varRefreshCmd": "cat(var_dic_list()) " + } + }, + "types_to_exclude": [ + "module", + "function", + "builtin_function_or_method", + "instance", + "_Feature" + ], + "window_display": false + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}