diff --git a/requirements.txt b/requirements.txt index 624d45c..59f5ecd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ lxml requests six boto3 -pyyaml +pyyaml \ No newline at end of file diff --git a/scrape.py b/scrape.py index 5a8c988..7fd8bb8 100755 --- a/scrape.py +++ b/scrape.py @@ -3,6 +3,7 @@ import re import json import locale +import gzip import ec2 import os import requests @@ -252,7 +253,13 @@ def add_pricing_info(instances): def fetch_data(url): - content = urllib2.urlopen(url).read().decode() + response = urllib2.urlopen(url).read() + + try: + content = response.decode() + except UnicodeDecodeError: + content = gzip.decompress(response).decode() + try: pricing = json.loads(content) except ValueError: