From 0944c717fc5d22375fb32597725dda03f012260b Mon Sep 17 00:00:00 2001
From: Cristian Magherusan-Stanciu <cristi.magherusan@gmail.com>
Date: Thu, 31 Oct 2024 11:44:05 +0100
Subject: [PATCH] Fix crash in OpenSearch scraper caused by missing instance
 type key for Serverless and other SKU types

---
 opensearch.py | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/opensearch.py b/opensearch.py
index d743f14..031a2ca 100755
--- a/opensearch.py
+++ b/opensearch.py
@@ -114,16 +114,18 @@ def scrape(output_file, input_file=None):
 
     # loop through products, and only fetch available instances for now
     for sku, product in tqdm(six.iteritems(data["products"])):
-        if (
-            product.get("productFamily", None) == "Amazon OpenSearch Service Instance"
+        if (product.get("productFamily", None) == "Amazon OpenSearch Service Instance"
             and product.get("attributes", {}).get("operation", None)
-            != "DirectQueryAmazonS3GDCOCU"
-        ):
-            attributes = product["attributes"]
+            != "DirectQueryAmazonS3GDCOCU"):
+
+            attributes = product.get("attributes", {})
+            if "instanceType" not in attributes:
+                continue
+
+            instance_type = attributes["instanceType"]
 
             # map the region
             location = ec2.canonicalize_location(attributes["location"])
-            instance_type = attributes["instanceType"]
             if location == "Any":
                 region = "us-east-1"
             elif location == "Asia Pacific (Osaka-Local)":
@@ -149,9 +151,7 @@ def scrape(output_file, input_file=None):
 
             if instance_type not in instances.keys():
                 # delete some attributes that are inconsistent among skus
-                new_attributes = (
-                    attributes.copy()
-                )  # make copy so we can keep these attributes with the sku
+                new_attributes = attributes.copy()  # make copy so we can keep these attributes with the sku
                 new_attributes.pop("location", None)
                 new_attributes.pop("locationType", None)
                 new_attributes.pop("operation", None)
@@ -291,9 +291,7 @@ def scrape(output_file, input_file=None):
                             "yrTerm3.noUpfront-hrs"
                         ]
 
-                    instances[instance_type]["pricing"][region][
-                        "reserved"
-                    ] = reserved_prices
+                    instances[instance_type]["pricing"][region]["reserved"] = reserved_prices
                 except Exception as e:
                     print(
                         "ERROR: Trouble generating Cache reserved price for {}: {!r}".format(
@@ -316,4 +314,4 @@ def scrape(output_file, input_file=None):
         input_file = sys.argv[1]
 
     output_file = "./www/opensearch/instances.json"
-    scrape(output_file, input_file)
+    scrape(output_file, input_file)
\ No newline at end of file