From ce14e728a25c16d1ace5cb9b62e0e70471c8e7a3 Mon Sep 17 00:00:00 2001 From: Alexander Shirkov <10080307+gradientsky@users.noreply.github.com> Date: Tue, 26 Oct 2021 13:37:41 -0700 Subject: [PATCH] AutoGluon deep learning containers - removed URI resolution code (#2993) * AutoGluon deep learning containers - removed URI resolution code * Formatting --- ...toGluon_Tabular_SageMaker_Containers.ipynb | 77 +++++++++++++++---- .../autogluon-tabular-containers/ag_model.py | 45 +++++++---- 2 files changed, 91 insertions(+), 31 deletions(-) diff --git a/advanced_functionality/autogluon-tabular-containers/AutoGluon_Tabular_SageMaker_Containers.ipynb b/advanced_functionality/autogluon-tabular-containers/AutoGluon_Tabular_SageMaker_Containers.ipynb index c717e35c94..aa41c8a258 100644 --- a/advanced_functionality/autogluon-tabular-containers/AutoGluon_Tabular_SageMaker_Containers.ipynb +++ b/advanced_functionality/autogluon-tabular-containers/AutoGluon_Tabular_SageMaker_Containers.ipynb @@ -25,6 +25,22 @@ "# Prerequisites" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe1a048b", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "# Ensure autogluon images information is available in SageMaker Python SDK\n", + "!pip install -q -U 'sagemaker>=2.66'" + ] + }, { "cell_type": "code", "execution_count": null, @@ -35,7 +51,11 @@ "import sagemaker\n", "import pandas as pd\n", "import numpy as np\n", - "from ag_model import AutoGluonTraining, AutoGluonInferenceModel, AutoGluonTabularPredictor\n", + "from ag_model import (\n", + " AutoGluonTraining,\n", + " AutoGluonInferenceModel,\n", + " AutoGluonTabularPredictor,\n", + ")\n", "from sagemaker import utils\n", "from sagemaker.serializers import CSVSerializer\n", "import os\n", @@ -117,7 +137,9 @@ "outputs": [], "source": [ "df_test = pd.read_csv(\n", - " \"s3://sagemaker-sample-files/datasets/tabular/uci_adult/adult.test\", header=None, skiprows=1\n", + " \"s3://sagemaker-sample-files/datasets/tabular/uci_adult/adult.test\",\n", + " header=None,\n", + " skiprows=1,\n", ")\n", "df_test.columns = columns\n", "df_test[\"class\"] = df_test[\"class\"].map(\n", @@ -212,7 +234,10 @@ "outputs": [], "source": [ "job_name = utils.unique_name_from_base(\"test-autogluon-image\")\n", - "ag.fit({\"config\": config_input, \"train\": train_input, \"test\": eval_input}, job_name=job_name)" + "ag.fit(\n", + " {\"config\": config_input, \"train\": train_input, \"test\": eval_input},\n", + " job_name=job_name,\n", + ")" ] }, { @@ -304,6 +329,22 @@ "Deploy remote or local endpoint" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d19344e", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "instance_type = \"ml.m5.2xlarge\"\n", + "# instance_type = 'local'" + ] + }, { "cell_type": "code", "execution_count": null, @@ -324,22 +365,12 @@ " role=role,\n", " region=region,\n", " framework_version=\"0.3.1\",\n", + " instance_type=instance_type,\n", " source_dir=\"scripts\",\n", " entry_point=\"tabular_serve.py\",\n", ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "8659a03f", - "metadata": {}, - "outputs": [], - "source": [ - "instance_type = \"ml.m5.2xlarge\"\n", - "# instance_type = 'local'" - ] - }, { "cell_type": "code", "execution_count": null, @@ -506,6 +537,21 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e744653", + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "instance_type = \"ml.m5.2xlarge\"" + ] + }, { "cell_type": "code", "execution_count": null, @@ -526,6 +572,7 @@ " role=role,\n", " region=region,\n", " framework_version=\"0.3.1\",\n", + " instance_type=instance_type,\n", " entry_point=\"tabular_serve-batch.py\",\n", " source_dir=\"scripts\",\n", " predictor_cls=AutoGluonTabularPredictor,\n", @@ -549,7 +596,7 @@ "source": [ "transformer = model.transformer(\n", " instance_count=1,\n", - " instance_type=\"ml.m5.2xlarge\",\n", + " instance_type=instance_type,\n", " strategy=\"MultiRecord\",\n", " max_payload=6,\n", " max_concurrent_transforms=1,\n", diff --git a/advanced_functionality/autogluon-tabular-containers/ag_model.py b/advanced_functionality/autogluon-tabular-containers/ag_model.py index 8ffd607c77..a2cb288f6d 100644 --- a/advanced_functionality/autogluon-tabular-containers/ag_model.py +++ b/advanced_functionality/autogluon-tabular-containers/ag_model.py @@ -3,33 +3,38 @@ from sagemaker.mxnet import MXNetModel from sagemaker.mxnet.model import MXNetPredictor from sagemaker import utils +from sagemaker import image_uris from sagemaker.serializers import CSVSerializer from sagemaker.deserializers import StringDeserializer -ACCOUNT = 763104351884 -ECR_TRAINING_REPO = "autogluon-training" -ECR_INFERENCE_REPO = "autogluon-inference" -TRAINING_IMAGE_CPU = "cpu-py37-ubuntu18.04" -TRAINING_IMAGE_GPU = "gpu-py37-cu102-ubuntu18.04" -INFERENCE_IMAGE_CPU = "cpu-py37-ubuntu16.04" - - class AutoGluonTraining(Framework): def __init__( self, entry_point, region, framework_version, - image_type="cpu", + instance_type, source_dir=None, hyperparameters=None, **kwargs, ): - image = TRAINING_IMAGE_GPU if image_type == "gpu" else TRAINING_IMAGE_CPU - image = f"{framework_version}-{image}" - image_uri = f"{ACCOUNT}.dkr.ecr.{region}.amazonaws.com/{ECR_TRAINING_REPO}:{image}" - super().__init__(entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs) + image_uri = image_uris.retrieve( + "autogluon", + region=region, + version=framework_version, + py_version="py37", + image_scope="training", + instance_type=instance_type, + ) + super().__init__( + entry_point, + source_dir, + hyperparameters, + instance_type=instance_type, + image_uri=image_uri, + **kwargs, + ) def _configure_distribution(self, distributions): return @@ -56,9 +61,17 @@ def __init__(self, *args, **kwargs): class AutoGluonInferenceModel(MXNetModel): - def __init__(self, model_data, role, entry_point, region, framework_version, **kwargs): - image = f"{framework_version}-{INFERENCE_IMAGE_CPU}" - image_uri = f"{ACCOUNT}.dkr.ecr.{region}.amazonaws.com/{ECR_INFERENCE_REPO}:{image}" + def __init__( + self, model_data, role, entry_point, region, framework_version, instance_type, **kwargs + ): + image_uri = image_uris.retrieve( + "autogluon", + region=region, + version=framework_version, + py_version="py37", + image_scope="inference", + instance_type=instance_type, + ) super().__init__( model_data, role, entry_point, image_uri=image_uri, framework_version="1.8.0", **kwargs )