AutoGluon deep learning containers - removed URI resolution code (aws…

…#2993) * AutoGluon deep learning containers - removed URI resolution code * Formatting
hchings · Oct 26, 2021 · ce14e72 · ce14e72
1 parent 0e9eb3d
commit ce14e72
Show file tree

Hide file tree

Showing 2 changed files with 91 additions and 31 deletions.
diff --git a/...d_functionality/autogluon-tabular-containers/AutoGluon_Tabular_SageMaker_Containers.ipynb b/...d_functionality/autogluon-tabular-containers/AutoGluon_Tabular_SageMaker_Containers.ipynb
@@ -25,6 +25,22 @@
     "# Prerequisites"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe1a048b",
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# Ensure autogluon images information is available in SageMaker Python SDK\n",
+    "!pip install -q -U 'sagemaker>=2.66'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -35,7 +51,11 @@
     "import sagemaker\n",
     "import pandas as pd\n",
     "import numpy as np\n",
-    "from ag_model import AutoGluonTraining, AutoGluonInferenceModel, AutoGluonTabularPredictor\n",
+    "from ag_model import (\n",
+    "    AutoGluonTraining,\n",
+    "    AutoGluonInferenceModel,\n",
+    "    AutoGluonTabularPredictor,\n",
+    ")\n",
     "from sagemaker import utils\n",
     "from sagemaker.serializers import CSVSerializer\n",
     "import os\n",
@@ -117,7 +137,9 @@
    "outputs": [],
    "source": [
     "df_test = pd.read_csv(\n",
-    "    \"s3://sagemaker-sample-files/datasets/tabular/uci_adult/adult.test\", header=None, skiprows=1\n",
+    "    \"s3://sagemaker-sample-files/datasets/tabular/uci_adult/adult.test\",\n",
+    "    header=None,\n",
+    "    skiprows=1,\n",
     ")\n",
     "df_test.columns = columns\n",
     "df_test[\"class\"] = df_test[\"class\"].map(\n",
@@ -212,7 +234,10 @@
    "outputs": [],
    "source": [
     "job_name = utils.unique_name_from_base(\"test-autogluon-image\")\n",
-    "ag.fit({\"config\": config_input, \"train\": train_input, \"test\": eval_input}, job_name=job_name)"
+    "ag.fit(\n",
+    "    {\"config\": config_input, \"train\": train_input, \"test\": eval_input},\n",
+    "    job_name=job_name,\n",
+    ")"
    ]
   },
   {
@@ -304,6 +329,22 @@
     "Deploy remote or local endpoint"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1d19344e",
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "instance_type = \"ml.m5.2xlarge\"\n",
+    "# instance_type = 'local'"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -324,22 +365,12 @@
     "    role=role,\n",
     "    region=region,\n",
     "    framework_version=\"0.3.1\",\n",
+    "    instance_type=instance_type,\n",
     "    source_dir=\"scripts\",\n",
     "    entry_point=\"tabular_serve.py\",\n",
     ")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8659a03f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "instance_type = \"ml.m5.2xlarge\"\n",
-    "# instance_type = 'local'"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -506,6 +537,21 @@
     ")"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2e744653",
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "instance_type = \"ml.m5.2xlarge\""
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -526,6 +572,7 @@
     "    role=role,\n",
     "    region=region,\n",
     "    framework_version=\"0.3.1\",\n",
+    "    instance_type=instance_type,\n",
     "    entry_point=\"tabular_serve-batch.py\",\n",
     "    source_dir=\"scripts\",\n",
     "    predictor_cls=AutoGluonTabularPredictor,\n",
@@ -549,7 +596,7 @@
    "source": [
     "transformer = model.transformer(\n",
     "    instance_count=1,\n",
-    "    instance_type=\"ml.m5.2xlarge\",\n",
+    "    instance_type=instance_type,\n",
     "    strategy=\"MultiRecord\",\n",
     "    max_payload=6,\n",
     "    max_concurrent_transforms=1,\n",

diff --git a/advanced_functionality/autogluon-tabular-containers/ag_model.py b/advanced_functionality/autogluon-tabular-containers/ag_model.py
@@ -3,33 +3,38 @@
 from sagemaker.mxnet import MXNetModel
 from sagemaker.mxnet.model import MXNetPredictor
 from sagemaker import utils
+from sagemaker import image_uris
 from sagemaker.serializers import CSVSerializer
 from sagemaker.deserializers import StringDeserializer
 
 
-ACCOUNT = 763104351884
-ECR_TRAINING_REPO = "autogluon-training"
-ECR_INFERENCE_REPO = "autogluon-inference"
-TRAINING_IMAGE_CPU = "cpu-py37-ubuntu18.04"
-TRAINING_IMAGE_GPU = "gpu-py37-cu102-ubuntu18.04"
-INFERENCE_IMAGE_CPU = "cpu-py37-ubuntu16.04"
-
-
 class AutoGluonTraining(Framework):
     def __init__(
         self,
         entry_point,
         region,
         framework_version,
-        image_type="cpu",
+        instance_type,
         source_dir=None,
         hyperparameters=None,
         **kwargs,
     ):
-        image = TRAINING_IMAGE_GPU if image_type == "gpu" else TRAINING_IMAGE_CPU
-        image = f"{framework_version}-{image}"
-        image_uri = f"{ACCOUNT}.dkr.ecr.{region}.amazonaws.com/{ECR_TRAINING_REPO}:{image}"
-        super().__init__(entry_point, source_dir, hyperparameters, image_uri=image_uri, **kwargs)
+        image_uri = image_uris.retrieve(
+            "autogluon",
+            region=region,
+            version=framework_version,
+            py_version="py37",
+            image_scope="training",
+            instance_type=instance_type,
+        )
+        super().__init__(
+            entry_point,
+            source_dir,
+            hyperparameters,
+            instance_type=instance_type,
+            image_uri=image_uri,
+            **kwargs,
+        )
 
     def _configure_distribution(self, distributions):
         return
@@ -56,9 +61,17 @@ def __init__(self, *args, **kwargs):
 
 
 class AutoGluonInferenceModel(MXNetModel):
-    def __init__(self, model_data, role, entry_point, region, framework_version, **kwargs):
-        image = f"{framework_version}-{INFERENCE_IMAGE_CPU}"
-        image_uri = f"{ACCOUNT}.dkr.ecr.{region}.amazonaws.com/{ECR_INFERENCE_REPO}:{image}"
+    def __init__(
+        self, model_data, role, entry_point, region, framework_version, instance_type, **kwargs
+    ):
+        image_uri = image_uris.retrieve(
+            "autogluon",
+            region=region,
+            version=framework_version,
+            py_version="py37",
+            image_scope="inference",
+            instance_type=instance_type,
+        )
         super().__init__(
             model_data, role, entry_point, image_uri=image_uri, framework_version="1.8.0", **kwargs
         )