diff --git a/siphon/simplewebservice/spc.py b/siphon/simplewebservice/spc.py old mode 100755 new mode 100644 index 705fc97a4..6b7d64815 --- a/siphon/simplewebservice/spc.py +++ b/siphon/simplewebservice/spc.py @@ -1,18 +1,16 @@ # Copyright (c) 2019 Siphon Contributors. # Distributed under the terms of the BSD 3-Clause License. # SPDX-License-Identifier: BSD-3-Clause -"""Reading Storm Prediction Center Data. +"""Read storm reports from the Storm Prediction Center. -====================================== -This program pulls data from the Storm Prediction -Center's data from 12/31/2011 in one day increments. -Weather events that are available are -hail, wind, and tornados. +Facilities are here to access the daily sets of storm reports (from 2012 onward) as well +as the full, assembled SPC tornado, hail, and wind report databases. """ from datetime import datetime, timedelta -from io import StringIO +from io import BytesIO, StringIO +from zipfile import ZipFile import pandas as pd import requests @@ -110,7 +108,6 @@ def parse_report_time(s): return reports - def _get_data_raw(self, stormtype, date_time): """Download the storm report file for a given type and date.""" if date_time.year < 2012: @@ -127,10 +124,10 @@ def _get_data_raw(self, stormtype, date_time): class SPCArchive(HTTPEndPoint): """ - Pulls data from the SPC archive. + Retrieve the Storm Prediction Center storm report databases. - This class gets data on tornados, hail, and severe wind events. - This will return a pandas dataframe for each of these storm events. + This class gets the entire report databases for tornadoes, hail, and severe wind events. + Data are returned as `pandas.DataFrame` instances. """ @@ -138,139 +135,96 @@ def __init__(self): """Set up the endpoint.""" super(SPCArchive, self).__init__('https://www.spc.noaa.gov/wcm/data') - def _get_data_raw(self): - - - def storm_type_selection(self): + @classmethod + def get_tornado_database(cls, filename='1950-2018_torn.csv.zip'): """ - Split http requests based on storm type. + Download and parse the SPC tornado database. + + This contains information for all tornadoes from 1950 to (roughly) present. This not + a realtime database, so there is usually a lag between when a year ends and when + updated data are available. The ``filename`` parameter is to allow pointing to + an updated database file on the server. Parameters ---------- - self: - The date_time string attribute will be used for year identification + filename : str + Filename for the database file on the SPC server, optional. Returns ------- - (torn/wind/hail)_reports: pandas DataFrame - This dataframe has the data about the specific SPC data type for either one day - or a 60+ year period based on what year is chosen. + `pandas.DataFrame` containing the entire SPC database """ - # Place holder string 'mag' will be replaced by event type (tornado, hail or wind) - mag = str - # Append columns with appropriate names for the archival data. - self.columns = ['Num', 'Year', 'Month', 'Day', 'Time', 'Time Zone', - 'State', mag, 'Injuries', 'Fatalities', 'Property Loss', - 'Crop Loss', 'Start Lat', 'Start Lon', 'End Lat', - 'End Lon', 'Length (mi)', 'Width (yd)', 'Ns', 'SN', 'SG', - 'County Code 1', 'County Code 2', 'County Code 3', - 'County Code 4'] - - if self.storm_type == 'tornado': - torn_reports = self.tornado_selection() - return(torn_reports) - elif self.storm_type == 'hail': - hail_reports = self.hail_selection() - return(hail_reports) - elif self.storm_type == 'wind': - wind_reports = self.wind_selection() - return(wind_reports) - else: - raise ValueError('Not a valid event type: enter either tornado, wind or hail.') + return cls()._get_data(filename) - def tornado_selection(self): + @classmethod + def get_wind_database(cls, filename='1955-2018_wind.csv.zip'): """ - Request tornado data from 1950 until this year. - - Parameters - ---------- - self: - Year attributes, endpoints, and column names are all used - - Returns - ------- - torn_reports: - Archive of tornado reports from 1950-2017 + Download and parse the SPC wind report database. - """ - columns = self.columns - columns[7] = 'F-Scale' - archive_path = 'wcm/data/1950-2017_torn.csv' - try: - resp = self.get_path(archive_path) - resp.raise_for_status() - storm_list = StringIO(resp.text) - torn_reports = pd.read_csv(storm_list, names=columns, - header=0, index_col=False, - usecols=[0, 1, 2, 3, 5, 6, 7, 10, 11, 12, 13, - 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27]) - except requests.exceptions.HTTPError as http_error: - raise ValueError(http_error, 'Tornado archive url not working.') - - return(torn_reports) - - def hail_selection(self): - """ - Request hail data from 1955 until this year. + This contains information for all wind reports from 1950 to (roughly) present. This + not a realtime database, so there is usually a lag between when a year ends and when + updated data are available. The ``filename`` parameter is to allow pointing to + an updated database file on the server. Parameters ---------- - self: - Year attributes, endpoints, and column names are all used + filename : str + Filename for the database file on the SPC server, optional. Returns ------- - hail_reports: - Archive of hail reports from 1955-2017 + `pandas.DataFrame` containing the entire SPC database """ - columns = self.columns - columns[7] = 'Size (hundredth in)' - archive_path = 'wcm/data/1955-2017_hail.csv' - try: - resp = self.get_path(archive_path) - resp.raise_for_status() - storm_list = StringIO(resp.text) - hail_reports = pd.read_csv(storm_list, names=columns, - header=0, index_col=False, - usecols=[0, 1, 2, 3, 5, 6, 7, 10, 11, 12, 13, - 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27]) - except requests.exceptions.HTTPError as http_error: - raise ValueError(http_error, 'Hail archive url not working.') - - return(hail_reports) - - def wind_selection(self): + return cls()._get_data(filename) + + @classmethod + def get_hail_database(cls, filename='1955-2018_hail.csv.zip'): """ - Request wind data from 1955 until this year. + Download and parse the SPC hail report database. + + This contains information for all hail report from 1955 to (roughly) present. This not + a realtime database, so there is usually a lag between when a year ends and when + updated data are available. The ``filename`` parameter is to allow pointing to + an updated database file on the server. Parameters ---------- - self: - Year attributes, endpoints, and column names are all used + filename : str + Filename for the database file on the SPC server, optional. Returns ------- - wind_reports: - Archive of wind reports from 1955-2017 + `pandas.DataFrame` containing the entire SPC database """ - columns = self.columns - columns[7] = 'Speed (kt)' - archive_path = 'wcm/data/1955-2017_wind.csv' - try: - resp = self.get_path(archive_path) - resp.raise_for_status() - storm_list = StringIO(resp.text) - wind_reports = pd.read_csv(storm_list, names=columns, - header=0, index_col=False, - usecols=[0, 1, 2, 3, 5, 6, 7, 10, 11, 12, 13, - 14, 15, 16, 17, 18, 19, 20, 21, 22, - 23, 24, 25, 26, 27]) - except requests.exceptions.HTTPError as http_error: - raise ValueError(http_error, 'Wind archive url not working.') - - return(wind_reports) + return cls()._get_data(filename) + + def _get_data(self, path): + if 'hail' in path: + mag_col = 'Size' + elif 'wind' in path: + mag_col = 'Speed' + else: + mag_col = 'F-Scale' + db = pd.read_csv(self._get_data_raw(path), index_col=False, parse_dates=[[4, 5]]) + db = db.drop(columns=['yr', 'mo', 'dy']) + db = db.rename(columns={'date_time': 'Date', 'tz': 'Time Zone', 'st': 'State', + 'stf': 'State FIPS', 'stn': 'State Number', 'mag': mag_col, + 'inj': 'Injuries', 'fat': 'Fatalities', + 'loss': 'Property Loss', 'closs': 'Crop Loss', + 'slat': 'Start Lat', 'slon': 'Start Lon', 'elat': 'End Lat', + 'elon': 'End Lon', 'len': 'Length (mi)', 'wid': 'Width (yd)', + 'f1': 'County FIPS 1', 'f2': 'County FIPS 2', + 'f3': 'County FIPS 3', 'f4': 'County FIPS 4', + 'mt': 'Magnitude Type'}) + return db + + def _get_data_raw(self, path): + resp = self.get_path(path) + if path.endswith('.zip'): + file_info = ZipFile(BytesIO(resp.content)).infolist()[0] + return ZipFile(BytesIO(resp.content)).open(file_info) + else: + return StringIO(resp.text) diff --git a/siphon/tests/fixtures/spc_after_2018_archive b/siphon/tests/fixtures/spc_after_2018_archive deleted file mode 100644 index 07634f851..000000000 --- a/siphon/tests/fixtures/spc_after_2018_archive +++ /dev/null @@ -1,110 +0,0 @@ -interactions: -- request: - body: null - headers: - Connection: - - close - Host: - - www.spc.noaa.gov - User-Agent: - - Python-urllib/3.7 - method: GET - uri: https://www.spc.noaa.gov/climo/reports/180615_rpts_filtered_hail.csv - response: - body: - string: 'Time,Size,Location,County,State,Lat,Lon,Comments - - 1335,100,MOSINEE,MARATHON,WI,44.78,-89.69,(GRB) - - 1431,100,WAUSAU,MARATHON,WI,44.96,-89.63,(GRB) - - 1510,175,MOOSE LAKE,CARLTON,MN,46.45,-92.77,TIME ESTIMATED BY RADAR. (DLH) - - 1600,100,NICKERSON,PINE,MN,46.41,-92.5,LARGEST HAILSTONES WERE QUARTER SIZED. - (DLH) - - 1632,100,NICKERSON,PINE,MN,46.41,-92.5,(DLH) - - 1724,100,5 N MCGREGOR,AITKIN,MN,46.68,-93.32,LARGEST HAILSTONES WERE QUARTER - SIZED. (DLH) - - 1942,100,2 SW LELAND,BRUNSWICK,NC,34.21,-78.03,HAIL STONE FOUND AT MAGNOLIA - GREENS GOLF PLANTATION IN LELAND AND REPORTED VIA TWITTER. (ILM) - - 1946,100,3 SSW SOCASTEE,HORRY,SC,33.64,-79.03,TWITTER REPORTS WITH PICTURES - SHOW QUARTER-SIZED HAIL. HAIL WAS REPORTED FOR 5 MINUTES AT THE LOCATION. - (ILM) - - 2130,100,SPRINGFIELD,ROBERTSON,TN,36.5,-86.88,PICTURE OF QUARTER SIZE HAIL. - (OHX) - - 2137,100,5 NE CHEYENNE,LARAMIE,WY,41.2,-104.72,(CYS) - - 2245,100,NASHVILLE,DAVIDSON,TN,36.17,-86.78,PICTURE OF QUARTER SIZE HAIL NEAR - TRINITY AND DICKERSON. (OHX) - - 2316,150,MOUNT VERNON,JEFFERSON,IL,38.32,-88.91,(PAH) - - 2323,100,DAYTON,SHERIDAN,WY,44.87,-107.26,PEA TO QUARTER SIZED HAIL BETWEEN - PARKMAN AND DAYTON. (BYZ) - - 2326,175,MIDWEST,NATRONA,WY,43.41,-106.28,(RIW) - - 2347,100,COLUMBIA,MAURY,TN,35.62,-87.05,EM PASSED ALONG QUARTER SIZE HAIL - ALONG OLD SUNNY SIDE IN COLUMBIA. (OHX) - - 0000,100,17 NE RENO JUNCTION,CAMPBELL,WY,43.97,-105.28,(UNR) - - 0011,125,7 NNE DOWNTOWN GILLETTE,CAMPBELL,WY,44.39,-105.46,(UNR) - - 0318,125,OAKDALE,MONROE,WI,43.97,-90.38,RELAYED VIA WKBT. (ARX) - - 0559,100,17 SSE MISSION,TODD,SD,43.09,-100.52,DAMAGED THE GARDEN... APPLES - TREES AND FLATTENED SOME PASTURES (UNR) - - 1020,100,1 NNW ISHPEMING,MARQUETTE,MI,46.5,-87.67,OBSERVED ONE INCH HAIL JUST - TO THE WEST OF THE ISHPEMING ROUND ABOUT THIS MORNING. (MQT) - -' - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Cache-Control: - - max-age=300 - Connection: - - close - Content-Length: - - '1739' - Content-Type: - - text/csv - Date: - - Wed, 03 Jul 2019 18:40:51 GMT - ETag: - - '"a8a7222e-6cb-5729ae78f1fc0"' - Expires: - - Wed, 03 Jul 2019 18:45:51 GMT - Last-Modified: - - Sat, 04 Aug 2018 12:08:39 GMT - Server: - - Apache - Strict-Transport-Security: - - max-age=31536000 - Vary: - - Accept-Encoding - Via: - - 1.1 c4.w3.woc (squid), 1.1 627bd3ccf992ce79999b331b1a9d283d.cloudfront.net - (CloudFront) - X-Amz-Cf-Id: - - yDlytKxjSI365f2V7l1KjfGsQj9lSGcyEv_RwpvoygWy-a_WxLV6Bg== - X-Amz-Cf-Pop: - - DEN50-C1 - X-Cache: - - Miss from cloudfront - X-XSS-Protection: - - 1; mode=block - status: - code: 200 - message: OK -version: 1 diff --git a/siphon/tests/fixtures/spc_hail_after_2018_archive b/siphon/tests/fixtures/spc_hail_after_2018_archive deleted file mode 100644 index 0b87a397d..000000000 --- a/siphon/tests/fixtures/spc_hail_after_2018_archive +++ /dev/null @@ -1,110 +0,0 @@ -interactions: -- request: - body: null - headers: - Connection: - - close - Host: - - www.spc.noaa.gov - User-Agent: - - Python-urllib/3.7 - method: GET - uri: https://www.spc.noaa.gov/climo/reports/180615_rpts_filtered_hail.csv - response: - body: - string: 'Time,Size,Location,County,State,Lat,Lon,Comments - - 1335,100,MOSINEE,MARATHON,WI,44.78,-89.69,(GRB) - - 1431,100,WAUSAU,MARATHON,WI,44.96,-89.63,(GRB) - - 1510,175,MOOSE LAKE,CARLTON,MN,46.45,-92.77,TIME ESTIMATED BY RADAR. (DLH) - - 1600,100,NICKERSON,PINE,MN,46.41,-92.5,LARGEST HAILSTONES WERE QUARTER SIZED. - (DLH) - - 1632,100,NICKERSON,PINE,MN,46.41,-92.5,(DLH) - - 1724,100,5 N MCGREGOR,AITKIN,MN,46.68,-93.32,LARGEST HAILSTONES WERE QUARTER - SIZED. (DLH) - - 1942,100,2 SW LELAND,BRUNSWICK,NC,34.21,-78.03,HAIL STONE FOUND AT MAGNOLIA - GREENS GOLF PLANTATION IN LELAND AND REPORTED VIA TWITTER. (ILM) - - 1946,100,3 SSW SOCASTEE,HORRY,SC,33.64,-79.03,TWITTER REPORTS WITH PICTURES - SHOW QUARTER-SIZED HAIL. HAIL WAS REPORTED FOR 5 MINUTES AT THE LOCATION. - (ILM) - - 2130,100,SPRINGFIELD,ROBERTSON,TN,36.5,-86.88,PICTURE OF QUARTER SIZE HAIL. - (OHX) - - 2137,100,5 NE CHEYENNE,LARAMIE,WY,41.2,-104.72,(CYS) - - 2245,100,NASHVILLE,DAVIDSON,TN,36.17,-86.78,PICTURE OF QUARTER SIZE HAIL NEAR - TRINITY AND DICKERSON. (OHX) - - 2316,150,MOUNT VERNON,JEFFERSON,IL,38.32,-88.91,(PAH) - - 2323,100,DAYTON,SHERIDAN,WY,44.87,-107.26,PEA TO QUARTER SIZED HAIL BETWEEN - PARKMAN AND DAYTON. (BYZ) - - 2326,175,MIDWEST,NATRONA,WY,43.41,-106.28,(RIW) - - 2347,100,COLUMBIA,MAURY,TN,35.62,-87.05,EM PASSED ALONG QUARTER SIZE HAIL - ALONG OLD SUNNY SIDE IN COLUMBIA. (OHX) - - 0000,100,17 NE RENO JUNCTION,CAMPBELL,WY,43.97,-105.28,(UNR) - - 0011,125,7 NNE DOWNTOWN GILLETTE,CAMPBELL,WY,44.39,-105.46,(UNR) - - 0318,125,OAKDALE,MONROE,WI,43.97,-90.38,RELAYED VIA WKBT. (ARX) - - 0559,100,17 SSE MISSION,TODD,SD,43.09,-100.52,DAMAGED THE GARDEN... APPLES - TREES AND FLATTENED SOME PASTURES (UNR) - - 1020,100,1 NNW ISHPEMING,MARQUETTE,MI,46.5,-87.67,OBSERVED ONE INCH HAIL JUST - TO THE WEST OF THE ISHPEMING ROUND ABOUT THIS MORNING. (MQT) - -' - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Cache-Control: - - max-age=300 - Connection: - - close - Content-Length: - - '1739' - Content-Type: - - text/csv - Date: - - Wed, 10 Jul 2019 17:00:58 GMT - ETag: - - '"6cb-5729ae78f1fc0"' - Expires: - - Wed, 10 Jul 2019 17:05:58 GMT - Last-Modified: - - Sat, 04 Aug 2018 12:08:39 GMT - Server: - - Apache/2.4.6 (CentOS) PHP/5.4.16 mod_perl/2.0.10 Perl/v5.16.3 - Strict-Transport-Security: - - max-age=31536000 - Vary: - - Accept-Encoding - Via: - - 1.1 c0.w1.woc (squid), 1.1 5e1c6e4ae68d5fdcd68291d459e4483c.cloudfront.net - (CloudFront) - X-Amz-Cf-Id: - - Y1NKLcpCd6vQ4-6CRUOByZonfzjOhZQELe2AwiBAOwP5uBgxB4hZ9g== - X-Amz-Cf-Pop: - - DEN50-C1 - X-Cache: - - Miss from cloudfront - X-XSS-Protection: - - 1; mode=block - status: - code: 200 - message: OK -version: 1 diff --git a/siphon/tests/fixtures/spc_torn_after_2018_archive b/siphon/tests/fixtures/spc_torn_after_2018_archive deleted file mode 100644 index 034b612f6..000000000 --- a/siphon/tests/fixtures/spc_torn_after_2018_archive +++ /dev/null @@ -1,63 +0,0 @@ -interactions: -- request: - body: null - headers: - Connection: - - close - Host: - - www.spc.noaa.gov - User-Agent: - - Python-urllib/3.7 - method: GET - uri: https://www.spc.noaa.gov/climo/reports/180615_rpts_filtered_torn.csv - response: - body: - string: 'Time,F_Scale,Location,County,State,Lat,Lon,Comments - - 2225,UNK,5 W WYNNE,CROSS,AR,35.23,-90.88,A WEAK LANDSPOUT TORNADO OCCURRED - NEAR THE L''ANGUILLE RIVER WEST OF WYNNE. NO DAMAGE WAS REPORTED. TIME IS - ESTIMATED. (MEG) - -' - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Age: - - '145' - Cache-Control: - - max-age=300 - Connection: - - close - Content-Length: - - '217' - Content-Type: - - text/csv - Date: - - Wed, 10 Jul 2019 16:58:33 GMT - ETag: - - '"ac607ca5-d9-5729ae78f1fc0"' - Expires: - - Wed, 10 Jul 2019 17:03:33 GMT - Last-Modified: - - Sat, 04 Aug 2018 12:08:39 GMT - Server: - - Apache - Strict-Transport-Security: - - max-age=31536000 - Via: - - 1.1 c2.w5.woc (squid), 1.1 b78abe87b75ea37372a6d2e80c96b9e4.cloudfront.net - (CloudFront) - X-Amz-Cf-Id: - - DEbR_ZV2iyQE1A_QYGksLz-rj8KykGZMwNGbs9yTTW8HJwiL7DsWTg== - X-Amz-Cf-Pop: - - DEN50-C1 - X-Cache: - - Hit from cloudfront - X-XSS-Protection: - - 1; mode=block - status: - code: 200 - message: OK -version: 1 diff --git a/siphon/tests/fixtures/spc_torn_archive b/siphon/tests/fixtures/spc_torn_archive index b89b2ffa2..3b3a856e8 100644 --- a/siphon/tests/fixtures/spc_torn_archive +++ b/siphon/tests/fixtures/spc_torn_archive @@ -9,117 +9,120 @@ interactions: Connection: - keep-alive User-Agent: - - Siphon (0.8.0+47.gd17796a.dirty) + - Siphon (0.8.0+61.ga4fea13.dirty) method: GET - uri: https://www.spc.noaa.gov/wcm/data/1950-2017_torn.csv + uri: https://www.spc.noaa.gov/wcm/data/1950-2018_torn.csv.zip response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//rJtbj1y3Dcff97NoDkTd5TcjbdEirgPUBZo+GrE3cOELUBso0k9fkrqQ - R2dkr4NivY7OemZ+S4r6k6KUTx/Mb/82Hz6ZN7+ZN6+/vDVf3n3Av/5rPn/BP4/4/dF8eP2reffx - X+bx9Rfz/tPnz+YX/vvze3z+/P7TR/OWRm9p9P7tR/Ofd2/MR/znj+bzr+YRzKMzj948BvP4ywMY - qNEaC8Z6Ht4s3GgIz6zFP8abv/5kXDWAI2+sSQe+Gr99OXI2t2oP5+iheH7At9YjGsDPdPhq/Njx - 9X9EOX4Ah69wHQX0XepTYNBhf3lhIDOM3hIFJp9/sssf+HY7YSAwt4fFqGGuWzZh9QBzKxU/mYaO - x45QyYAn9/GXjxPld6g0nfjTn/HD+lutCR0V7FEKfn44YuEftG980cSkOVsPYWJAMDi08ZnrFj3/ - m4nsPXqvHwaFI6C/wuGzgqTmaP4CPyFxQlycEB4+86eAcPgVtdsyfiTNSio8blOUonHkSDsMqoJK - G5ST2Gsz5DlsBRXgwJ/eSj68X9w2MeK2LJgkGByWifn7zyYUjiAG9U90iWenFgoCebAR5w8/1PvO - CnmySmdRKHaWu9HQzwhvrE6akeDqESjC40EOwNXaHoBWbsU4GXZ5CfC6Z8UTi8IbaIEoWMr8+ZZh - vJzIueDoLztpzoobKSTv8tR0NV5YIwMjIDbbaDzsxF8nqRkLTiIDYMfyc/E2Fv7+Jp1ZtXCwYwB6 - WgE0zsiKJo14x4EPfrCcsJywcHrsjPjGSsgqyo348d7z56NJ+BDieMAgMWXGh5f4AL+DoQSeDMtd - poMYZlv8oS/xAdrY8/JykwVBeTHsYGmJxrJEo4cWgOmIkR/yfEApkTlzajVD3NBIcLOmcTJxetLg - aHPmeNjG+M5My0wCRIk7pB3MSd56ZdxY1F5Mo6gjRSc/3FXcIMIO+akYN7LqVzBtcd3DlG9jukTR - ED3td75rrsMXomrEu56re9YId84iQ6F0FqGPxwW80dwoweDsjuKnRS+eG8eqZLCQwF9ZWVQz22HH - CqYsX9xJLjxw/oot5z842APdCUhf6MaoJcMxA3MwyUfmB/zwWE7BjsQyoAjcyQaExcJA+StqILTs - 6Ck7IjFGfsCCIZNMCbGgZlWG+m8QzyaOfCnTdOebA/aGeVzUye3UiaVJxYZf9cIfLnNs1MiyGMdD - PC1hUOnL7dTJrSESr1JIlQacsz+Bx0zJ0nJKlbxQPAnuXFovKYWzw3W8RzYDl3DWZZNTSzgq16Ud - xy0cjj/UwCCc3Dih3OdUhRFB6kPC0NDOGPjpR6wXWPdOTostzqhK04I0RQKsmpshSJ6rTcb4GxUD - IhJN9/yqexjOaAyW6PG+vGY1N3VSIE0KDm2dyfBPL1rpctYiKmAIA7T5GBjoW5yrOd4KqAoIX5Cn - OS3UUkdNr9mW1Tm5a7dJwlUY2GCo+osak9fJsZzPQa1P0HOjthruqYzS36OKvrZfg1H/YcjhPBf6 - waiMokzQg/cT5WR6eHjOFIETrNUpFpqiWiq0Du9YDTCvQyZFlTnCHFUHLXyFBooWe8xNNU0HRHYe - lrE+sZYDLSmSHjelJykfxg3K2bNhVPBljQpHUzkqwcLRsiHJMkaJSaJyOibSjiUFH7N4l2PPMK5k - 0RZPq9e2B/zd8R/Qk0lwog8+73D+bFpZ9joIaBEYeBiHvkb8QVHVpUKJRkwp8iRFVvS7SdGdSqK0 - /UA+i0SeG1In+w5fd5y41GCpa+u0ibM4hXzciJGqWILdYdKSjuqqEa6lcxQjLiBye7CkRtSdGEap - rBRgB8uLvuYllZNNnpcuJh9Kuz11UJlNYTFlVlhuzwpRs8L3lA21/k7CaUE58hk3kfLYupEqYc6n - 9osUsJVDArBAyVTtBf9UYvwem1Ru+j5C+opNodmUCxe2d20CyqwPIeyQZclSMHp4Kn/UpntwLomm - sKs9aIh7ztm0eg0+zhuFrfFHdu3Bt4bI3BUqN6YNS/cXmVXWVRV4IeHHu3x/8UJQrbG848Sl/BoF - iyrzEs1Q3Jd5WfluJ3qQ5hy1nlVYRC9Y3kWjPYnGiYe8odY1mGrEhaF7gYtKRoWb5WL8JOUVB+Q9 - ZVFu+u303jOKRUkMinZHKYuKX2r+2Hqy+bC6BrN6/yITFOHbnNa8tNcJ6rJQ9Y696Gafal864Xjh - cC925NtmT1g5iZUVs1LxJ3vu9lain5y+eycODnFfMWfnRVt6vGHUa7UvIkfj0spl+vST71TRH8Oe - NRYssy6J1h6tTVqXJsQMOGVQnJC+MyMIDpXgdfWBq/rA6Mefex3X+jUmwWTBZN0rf/WDIY1aKVxO - 3spZvTMp29iOCSULRTxGw/AMso6C2Hvlquri/lPlZB5bAYEPWJthyEkfNlMdNmrXWHa4PI368RWV - dxejSiuVI8WBL6NPmii6OMTHPKmVVHewSz+beivx3Ong5isdBsmYXnnqIUZJ7cnuYPWZPS2nRH0O - p2G9ZE29pPQ89pFbsaouV+cBsIf5kxZd9k+hbW6p+r8vEU7ZJBLhqmDqnd6hXQ4eMIO7vnHy9OB7 - bqfiy98/fUh+R4tLexncGhxUL1CXN9GQy0oaA5wcqA9UhlBE2bZH3rZLtu1reLCUZbk32u4v4aQc - GHeYtSK/SgXM8niT1XU8pB2nrpw7rot9D7M7WOsYeEh5gwG71l3+6rXaWl102uCo5m8NUojnja5q - sqWyw8E8xuu4S7ncrcJCeXblHSvHqTWqp6ruYYttcS3AWni3/ef9/K4WVbZPBqX+Jh0TzSoXNyCl - SBkE5ATkdKOtg3KfZt13zcN9FN00ZCUsaq6o8h/T9ZDdjnbpYxfTOpXrZHE3+/6aUoVy9gIKAgqG - u3o6hVxCvXABi0LudycaqveRw56zyBEVfBzxqtMSzA1/ffQXjV17oF273th41RbNcYdTZ7zPuZXV - zRo9MKyRa28SrGe8c1GpqEh7zjl1XI66+MSVc2Lknu9IkJiF+Xyrq6yg8lNR9VrJxjZTsBE/1R7P - ZY8Jp8x7zVKpSQP3CO4vKFCdj1x3JDm0bqHne9GsrmVY3wziaxncKm2bw0D5cIZEleOuh6KUIgou - 0pLyJ1xo59anmxmpVUv4H3zoXSSq+TPyQlAHJ2Mn9VCUYGTh0ZHTMl8zW1lxZDNvk0PUEV5xG4pb - u2Hg1nZpajVg7g1MLjkzeZGdOCtOmbHinwy7bHcT614+FdF6UaneZVFaUQSDzgjTc6/+YGiPv1CC - b7kja+1rEThylLozETcYdVbYjgHcgnG1NcOAykoxh66b3Em8Je04ZQm8uC6o2grzQrUyPoTcDt/p - vkQySTqx6gpDyTtaXQQ9dbuC0OIAUMS3uo/uVakNgbpVVcqe5E4a65Z+ToC2fiNVtfTQb2Y4jgkp - MUF1j0rd085ezJ2n7HLNFjoknA+8eaKmyFy9urFT7QZH/fOscaVHocKFdt61af6q9FFhTzkbNTTd - CiW3xZt3bSqVDqvbcdyU2pd/5LPtJSgwHfKKirwLbR1G7lwV+sG82aLjvSqVqAKrdLHFwmlXeukn - 1qNf//jmtTR4qGHDcZczybByQrtEmHTPhcoLFXtqTVVRin4biUg0FKX4xz9NHKd4qpoAuhRxAzqp - OfdC5BaXCjpRinn9KPLVqkvKGGfTTvS19Loo6nIsyuGGmiHRCJDq8nyH4OUPo8Nxclzr8eW8mqO6 - o4pTdpy8cC7XWLj8uuWiLn2QT9X8FDU/9amc+Z7fxeE6ZoC8gPyd3UZdPQfjdKueT3Rnq1zdlbGi - CyClEQ2lW9Wr5bWdSHf4mraqk+O1CFMXw6zboVbfXRqXdMp0y1Sl0LCNM20Ki+os69t11u9Z/sSK - /SqfmifLPbhUNmGXtftEGUBKPerTiQJ198WlLHd8YkL5Aed93FcslBihaJ6Lmhc3PHXa0CoJv0SF - 4zzElcRGxqu6rmXTUzlhNasc8ekYJQ9JMKebq13A1wsSPrfNZ9H6YLmNeZdUdqS18waj2FPRV9zc - A+BD23lg+j91xPS1RKtkQrIgDePs0bcsuO51A982Z3VNOmHQUah06ZVhYHeotQqDcW1KbW6gJcHC - mxvvx0M7gJdlrFYxwJN5wyHTkX0/U47NfldfpAelFpJzqXZfCxZc1XYNjdLt2hRGoB0oUuFE0l27 - LlhPkTFurCqtSL17rhtI+oKb1yaJWDjRdBpeon38fqpcyfHaAsFoj/fSIYDohBP9o+G6BYVRnCvv - uTza5vTQNvR0ARhr2Jnjvb7LnHa0vNIul35rq/oS7aOooG1NbuoNUNBLxaz+LwGAvOFdGhRQlhL9 - fwAAAP//jFxLbiUhDNznLMkTHzfQs4s00mg22c79bzJtm7YLaJTsmsWTBRi77Co/7uroQYYdB3bg - zjxsJI+D/FnmZlKZk1YS57gyLPMtvDr6SinR1rwiRXf00JHc7/mTflEb9nYnfkBmHUGn+rw3oNxi - CjtDdc78i993/WNh2o0X9n2BmwRQHe4sxY25RQypJNLQP1AxeL6C0gCdzBfRTvqxnQVy9u5pftVn - Ow3uKXnYyI7U+dOR7Z9PbU9MjkEqh6SR7gX2OgNJHpNHjexQnT+PCcosHnhIB/OC0GkXBzFmmJC0 - OLFcPsIgJO1khMxbvCd0iKBE0TGiJotOaKe4HXI7hDokPbk0B/asSobr5EYSx3M+pkaTki6G2mTo - viRgYpteUTpw0YexHtKV6UmL9/zKh8CnaZ4jsqR0kNMEqdyURgzaoigyt8IOb6+JKp7huTO3wPZb - zHWPtaVTODe2lnppX17SWUOFC4YkE5cWb8YV6S+6B2pReqMZK0rVLy6Y/EIPBOkJVPPR1KWLHU8j - el9LgSDacr6iOIlYAcNAkDCNafEuRZEuRZgawrEJwxwRNAlOquyDvuBZM9b3QelzutjlLZrUtHgL - oXALAQp7nTpb+plcDXQZJn+r1JSn+lj9mTxBgjea1HSx1oyB68rFMJsrL+1rLjHDnjIepMeM6DGD - t+3bUrXQLHEnJRZn8fkGpOWyscP1yA2of79nUQnJj2yS7t7PrBB5Are57sxA2Si4XaijMPSvVLNf - OduTZvtTgCgNmwpozoNG9AA1Ft/6sPL8sEj6wvyw4kZTj5EwnztDUHrL8aV+fLarJrq7q6DmeyKh - PmTFE0FDxMDymzxiRA9PfKHz0O3yks8uQB7Y84SUYoCNUdwYuk6QUjf0pRmca1Vo4Belqgq/YF4k - W3DH2zAMtPEjpY01oDC1D80pBbsK7O1NB2LkW2EGT/ex9CE8PmLykGEla5GS9ZhR9dICagLbGW2y - yKe9OhCNWcO/X1rAW/OoET1qRGlz0jjbGe5fedTQQJ+4JwSrJBrhJuf4dH8eP6z4Kh8Cztwx5cHR - ZJF61hSCjqtmfeQ6F/bIoEbyIJL8+vjTZaGdgb73B0VRVdKKdtGq4knWnSUnLP79fT+Otf5XzDYM - SgzY0ANIvOy0jR3gutXOMmB8KAXDXJJ/Rx7sJOjV4OmdO1t+VWor9hl88mDVx9lZG0Xq/dLhr68G - M9qnMYBv8Qg/tUYcQ2RUG7ZmYJSOexCEM3C+bszrLoTzR/zeXveNOPsGc+lcUZaNOqGgnbSzk6cb - O+ZEljVhCg== + UEsDBBQAAAAAAHaydU/PKQCUOhAAADoQAAASAAAAMTk1MC0yMDE4X3Rvcm4uY3N2b20seXIsbW8s + ZHksZGF0ZSx0aW1lLHR6LHN0LHN0ZixzdG4sbWFnLGluaixmYXQsbG9zcyxjbG9zcyxzbGF0LHNs + b24sZWxhdCxlbG9uLGxlbix3aWQsbnMsc24sc2csZjEsZjIsZjMsZjQsZmMKMSwxOTUwLDAxLDAz + LDE5NTAtMDEtMDMsMTE6MDA6MDAsMyxNTywyOSwxLDMsMywwLDYuMCwwLjAsMzguNzcsLTkwLjIy + LDM4LjgzLC05MC4wMyw5LjUsMTUwLDIsMCwxLDAsMCwwLDAsMAoxLDE5NTAsMDEsMDMsMTk1MC0w + MS0wMywxMTowMDowMCwzLE1PLDI5LDEsMywzLDAsNi4wLDAuMCwzOC43NywtOTAuMjIsMzguODIs + LTkwLjEyLDYuMiwxNTAsMiwxLDIsMTg5LDAsMCwwLDAKMSwxOTUwLDAxLDAzLDE5NTAtMDEtMDMs + MTE6MTA6MDAsMyxJTCwxNywxLDMsMCwwLDUuMCwwLjAsMzguODIsLTkwLjEyLDM4LjgzLC05MC4w + MywzLjMsMTAwLDIsMSwyLDExOSwwLDAsMCwwCjIsMTk1MCwwMSwwMywxOTUwLTAxLTAzLDExOjU1 + OjAwLDMsSUwsMTcsMiwzLDMsMCw1LjAsMC4wLDM5LjEsLTg5LjMsMzkuMTIsLTg5LjIzLDMuNiwx + MzAsMSwxLDEsMTM1LDAsMCwwLDAKMywxOTUwLDAxLDAzLDE5NTAtMDEtMDMsMTY6MDA6MDAsMyxP + SCwzOSwxLDEsMSwwLDQuMCwwLjAsNDAuODgsLTg0LjU4LDAuMCwwLjAsMC4xLDEwLDEsMSwxLDE2 + MSwwLDAsMCwwCjQsMTk1MCwwMSwxMywxOTUwLTAxLTEzLDA1OjI1OjAwLDMsQVIsNSwxLDMsMSwx + LDMuMCwwLjAsMzQuNCwtOTQuMzcsMC4wLDAuMCwwLjYsMTcsMSwxLDEsMTEzLDAsMCwwLDAKNSwx + OTUwLDAxLDI1LDE5NTAtMDEtMjUsMTk6MzA6MDAsMyxNTywyOSwyLDIsNSwwLDUuMCwwLjAsMzcu + NiwtOTAuNjgsMzcuNjMsLTkwLjY1LDIuMywzMDAsMSwxLDEsOTMsMCwwLDAsMAo2LDE5NTAsMDEs + MjUsMTk1MC0wMS0yNSwyMTowMDowMCwzLElMLDE3LDMsMiwwLDAsNS4wLDAuMCw0MS4xNywtODcu + MzMsMC4wLDAuMCwwLjEsMTAwLDEsMSwxLDkxLDAsMCwwLDAKNywxOTUwLDAxLDI2LDE5NTAtMDEt + MjYsMTg6MDA6MDAsMyxUWCw0OCwxLDIsMiwwLDAuMCwwLjAsMjYuODgsLTk4LjEyLDI2Ljg4LC05 + OC4wNSw0LjcsMTMzLDEsMSwxLDQ3LDAsMCwwLDAKOCwxOTUwLDAyLDExLDE5NTAtMDItMTEsMTM6 + MTA6MDAsMyxUWCw0OCwyLDIsMCwwLDQuMCwwLjAsMjkuNDIsLTk1LjI1LDI5LjUyLC05NS4xMyw5 + LjksNDAwLDEsMSwxLDM5LDAsMCwwLDAKOSwxOTUwLDAyLDExLDE5NTAtMDItMTEsMTM6NTA6MDAs + MyxUWCw0OCwzLDMsMTIsMSw0LjAsMC4wLDI5LjY3LC05NS4wNSwyOS44MywtOTUuMCwxMi4wLDEw + MDAsMSwxLDEsMjAxLDAsMCwwLDAKMTAsMTk1MCwwMiwxMSwxOTUwLTAyLTExLDIxOjAwOjAwLDMs + VFgsNDgsNCwyLDUsMCw1LjAsMC4wLDMyLjM1LC05NS4yLDMyLjQyLC05NS4yLDQuNiwxMDAsMSwx + LDEsNDIzLDAsMCwwLDAKMTEsMTk1MCwwMiwxMSwxOTUwLTAyLTExLDIzOjU1OjAwLDMsVFgsNDgs + NSwyLDYsMCw1LjAsMC4wLDMyLjk4LC05NC42MywzMy4wLC05NC43LDQuNSw2NywxLDEsMSw2Nywz + NDMsMCwwLDAKMTIsMTk1MCwwMiwxMiwxOTUwLTAyLTEyLDAwOjMwOjAwLDMsVFgsNDgsNiwyLDgs + MSw0LjAsMC4wLDMzLjMzLC05NC40MiwzMy40NSwtOTQuNDIsOC4wLDgzMywxLDEsMSwzNywwLDAs + MCwwCjEzLDE5NTAsMDIsMTIsMTk1MC0wMi0xMiwwMToxNTowMCwzLFRYLDQ4LDcsMSwwLDAsNC4w + LDAuMCwzMi4wOCwtOTguMzUsMzIuMSwtOTguMzMsMi4zLDIzMywxLDEsMSwxNDMsMCwwLDAsMAox + NCwxOTUwLDAyLDEyLDE5NTAtMDItMTIsMDY6MTA6MDAsMyxUWCw0OCw4LDIsMCwwLDQuMCwwLjAs + MzEuNTIsLTk2LjU1LDMxLjU3LC05Ni41NSwzLjQsMTAwLDEsMSwxLDI5MywwLDAsMCwwCjE1LDE5 + NTAsMDIsMTIsMTk1MC0wMi0xMiwxMTo1NzowMCwzLFRYLDQ4LDksMSwzMiwwLDUuMCwwLjAsMzEu + OCwtOTQuMiwzMS44OCwtOTQuMTIsNy43LDEwMCwxLDEsMSw0MTksMCwwLDAsMAoxNiwxOTUwLDAy + LDEyLDE5NTAtMDItMTIsMTI6MDA6MDAsMyxNUywyOCwxLDIsMiwzLDQuMCwwLjAsMzQuNiwtODku + MTIsMC4wLDAuMCwwLjEsMTAsMSwxLDEsMTQ1LDAsMCwwLDAKMTcsMTk1MCwwMiwxMiwxOTUwLTAy + LTEyLDEyOjAwOjAwLDMsTVMsMjgsMiwxLDAsMCwwLjAsMC4wLDM0LjYsLTg5LjEyLDAuMCwwLjAs + Mi4wLDEwLDEsMSwxLDE0NSwwLDAsMCwwCjE4LDE5NTAsMDIsMTIsMTk1MC0wMi0xMiwxMjowMDow + MCwzLFRYLDQ4LDEwLDMsMTUsMyw1LjAsMC4wLDMxLjgsLTk0LjIsMzEuOCwtOTQuMTgsMS45LDUw + LDEsMSwxLDQxOSwwLDAsMCwwCjE5LDE5NTAsMDIsMTIsMTk1MC0wMi0xMiwxMjozMDowMCwzLEFS + LDUsMiwyLDAsMCwzLjAsMC4wLDM0LjQ4LC05Mi40LDAuMCwwLjAsMC4xLDEwMCwxLDEsMSw1Myww + LDAsMCwwCjIwLDE5NTAsMDIsMTIsMTk1MC0wMi0xMiwxMzowMDowMCwzLExBLDIyLDEsNCw3Nywx + OCw1LjAsMC4wLDMxLjk3LC05NC4wLDMzLjAsLTkzLjMsODIuNiwxMDAsMSwxLDEsMzEsMTcsMTUs + MTE5LDAKMjEsMTk1MCwwMiwxMiwxOTUwLTAyLTEyLDEzOjIwOjAwLDMsTEEsMjIsMiwyLDEwLDUs + NS4wLDAuMCwzMi4yLC05My41OCwzMi45NywtOTMuMTcsNTguNCwxMDAsMSwxLDEsMzEsODEsMTcs + MTUsMAoyMiwxOTUwLDAyLDEyLDE5NTAtMDItMTIsMTQ6MDA6MDAsMyxMQSwyMiw0LDMsMjUsNSw1 + LjAsMC4wLDMxLjYzLC05My42NSwzMi41NSwtOTMuMDMsNzQuNSwxMDAsMSwxLDEsODUsNjksODEs + MTMsMAoyMiwxOTUwLDAyLDEyLDE5NTAtMDItMTIsMTQ6MjA6MDAsMyxMQSwyMiwzLDIsMCwwLDAu + MCwwLjAsMC4wLDAuMCwwLjAsMC4wLDAuMCwwLDEsMCwtOSwyNywwLDAsMCwwCjIzLDE5NTAsMDIs + MTIsMTk1MC0wMi0xMiwxNTowMDowMCwzLEFSLDUsMywyLDAsMCw0LjAsMC4wLDMzLjI3LC05Mi45 + NSwzMy4zNSwtOTIuOTUsNS43LDEwMCwxLDEsMSwxMzksMCwwLDAsMAoyNCwxOTUwLDAyLDEyLDE5 + NTAtMDItMTIsMjM6MDA6MDAsMyxMQSwyMiw1LDEsMCwwLDQuMCwwLjAsMzIuNiwtOTEuMzMsMC4w + LDAuMCwwLjUsMzMsMSwxLDEsMzUsMCwwLDAsMAoyNSwxOTUwLDAyLDEzLDE5NTAtMDItMTMsMDE6 + MDA6MDAsMyxUTiw0NywxLDEsOCwwLDMuMCwwLjAsMzUuMzUsLTg5Ljc3LDAuMCwwLjAsMC4yLDEw + LDEsMSwxLDE1NywwLDAsMCwwCjI2LDE5NTAsMDIsMTMsMTk1MC0wMi0xMywwMjowMDowMCwzLFRO + LDQ3LDIsMiwxLDksNC4wLDAuMCwzNS43NSwtODkuNDgsMC4wLDAuMCwwLjIsMTAsMSwxLDEsOTcs + MCwwLDAsMAoyNywxOTUwLDAyLDI3LDE5NTAtMDItMjcsMTA6MjA6MDAsMyxPSyw0MCwxLDIsMCww + LDQuMCwwLjAsMzUuNTUsLTk3LjYsMC4wLDAuMCwyLjAsNTAsMSwxLDEsMTA5LDAsMCwwLDAKMjgs + MTk1MCwwMywwMSwxOTUwLTAzLTAxLDAyOjMwOjAwLDMsTVMsMjgsMywxLDAsMCwwLjAsMC4wLDMy + LjUsLTg4Ljg1LDAuMCwwLjAsMC4xLDEwLDEsMSwxLDc1LDAsMCwwLDAKMjksMTk1MCwwMywxNiwx + OTUwLTAzLTE2LDA5OjE1OjAwLDMsRkwsMTIsMSwyLDAsMCwzLjAsMC4wLDI5LjY1LC04MS4yMiww + LjAsMC4wLDEuNSwxNTAsMSwxLDEsMTA5LDAsMCwwLDAKMzAsMTk1MCwwMywxOSwxOTUwLTAzLTE5 + LDA3OjMwOjAwLDMsTEEsMjIsNiwxLDIsMCw0LjAsMC4wLDMwLjQ1LC05My40NSwwLjAsMC4wLDIu + MCwzMywxLDEsMSwxOSwwLDAsMCwwCjMxLDE5NTAsMDMsMTksMTk1MC0wMy0xOSwxMzoxNTowMCwz + LExBLDIyLDcsMiwwLDAsNC4wLDAuMCwzMC4xLC05MS4wLDAuMCwwLjAsMS4wLDUwLDEsMSwxLDUs + MCwwLDAsMAozMiwxOTUwLDAzLDE5LDE5NTAtMDMtMTksMTM6MTU6MDAsMyxMQSwyMiw4LDAsMCww + LDQuMCwwLjAsMjkuNywtOTAuMSwyOS42NywtODkuOCwxOC4xLDI3LDEsMSwxLDUxLDc1LDAsMCww + CjMzLDE5NTAsMDMsMjYsMTk1MC0wMy0yNiwxOTozMDowMCwzLEFSLDUsNCwyLDMsMCw0LjAsMC4w + LDM0LjEyLC05My4wNywzNC4zMiwtOTIuODgsMTcuNCwxNTAsMSwxLDEsMTksNTksMCwwLDAKMzQs + MTk1MCwwMywyNiwxOTUwLTAzLTI2LDE5OjMxOjAwLDMsQVIsNSw1LDMsMSwwLDUuMCwwLjAsMzYu + MTUsLTkxLjgzLDM2LjIsLTkxLjc1LDUuNywyMDAsMSwxLDEsNjUsMCwwLDAsMAozNSwxOTUwLDAz + LDI2LDE5NTAtMDMtMjYsMjA6MzA6MDAsMyxBUiw1LDYsMiw3LDAsNS4wLDAuMCwzNC43LC05Mi4z + NSwzNC44LC05Mi4yMiwxMC40LDYwMCwxLDEsMSwxMTksMCwwLDAsMApQSwECFAMUAAAAAAB2snVP + zykAlDoQAAA6EAAAEgAAAAAAAAAAAAAAgAEAAAAAMTk1MC0yMDE4X3Rvcm4uY3N2UEsFBgAAAAAB + AAEAQAAAAGoQAAAAAA== headers: + Accept-Ranges: + - bytes Access-Control-Allow-Origin: - '*' + Age: + - '213' Cache-Control: - max-age=300 Connection: - keep-alive - Content-Encoding: - - gzip + Content-Length: + - '1572057' Content-Type: - - text/csv + - application/zip Date: - - Thu, 21 Nov 2019 23:25:37 GMT + - Fri, 22 Nov 2019 05:08:36 GMT + ETag: + - '"17fcd9-593bb49731700"' Expires: - - Thu, 21 Nov 2019 23:30:37 GMT + - Fri, 22 Nov 2019 05:13:36 GMT Last-Modified: - - Sun, 09 Sep 2018 11:42:14 GMT + - Mon, 30 Sep 2019 01:38:04 GMT Server: - Apache/2.4.6 (CentOS) PHP/5.4.16 mod_perl/2.0.10 Perl/v5.16.3 Strict-Transport-Security: - max-age=31536000 - Transfer-Encoding: - - chunked - Vary: - - Accept-Encoding Via: - - 1.1 c0.w1.woc (squid), 1.1 6211d31dba7b1c097ae4459c62ae1440.cloudfront.net + - 1.1 c5.w3.woc (squid), 1.1 94302df9990bc95bcddeb66c4a58718c.cloudfront.net (CloudFront) X-Amz-Cf-Id: - - Icnc2S0oH9gS-0tEcbjdSXC_sbtIesK0bnzmAu0xwqX3abxKKox1iw== + - XjMS5Cq6oILBU_ucsm2qZ0Be4D5j6vK7b4fEAROhXRqDr3S-qkIrrg== X-Amz-Cf-Pop: - DEN50-C1 X-Cache: diff --git a/siphon/tests/fixtures/spc_wind_after_2018_archive b/siphon/tests/fixtures/spc_wind_after_2018_archive deleted file mode 100644 index 997083837..000000000 --- a/siphon/tests/fixtures/spc_wind_after_2018_archive +++ /dev/null @@ -1,139 +0,0 @@ -interactions: -- request: - body: null - headers: - Connection: - - close - Host: - - www.spc.noaa.gov - User-Agent: - - Python-urllib/3.7 - method: GET - uri: https://www.spc.noaa.gov/climo/reports/180615_rpts_filtered_wind.csv - response: - body: - string: 'Time,Speed,Location,County,State,Lat,Lon,Comments - - 1215,78,13 SSE LITTLE MARAIS,LSZ162,MN,47.25,-90.96,REPORTED BY THE JAMES - R. BARKER. (DLH) - - 1228,UNK,3 N IRONWOOD,GOGEBIC,MI,46.49,-90.14,SOCIAL MEDIA REPORT RECEIVED - OF SMALL POPLAR TREES DOWN ON SUNSET RD IN IRONWOOD TOWNSHIP. (MQT) - - 1730,UNK,1 ESE LAKE NEBAGAMON,DOUGLAS,WI,46.5,-91.68,10 INCH DIAMETER POPLAR - TREE DOWNED ONTO A POWER LINE. TIME ESTIMATED BY RADAR. (DLH) - - 1740,UNK,DULUTH,ST. LOUIS,MN,46.78,-92.12,SEVERAL REPORTS OF TREES AND POWER - LINES DOWN IN THE CITY OF DULUTH. TIME ESTIMATED BY RADAR. (DLH) - - 1755,UNK,1 ESE APALACHEE REGIONA,LEON,FL,30.41,-84.14,REPORT RELAYED THROUGH - SOCIAL MEDIA OF TREE DOWN ON POWERLINES AT OLD ST AUGUSTINE RD AND LOUVINIA - DRIVE. TIME ESTIMATED BY RADAR. (TAE) - - 1800,UNK,OCHLOCKNEE,THOMAS,GA,30.97,-84.06,THOMAS CO DISPATCH REPORTS TREE - DOWN ON POWER LINE ON STEWART RD. (TAE) - - 1920,UNK,1 NNW ZEPHYRHILLS,PASCO,FL,28.25,-82.18,LARGE TREE LIMBS DOWNED AND - MINOR DAMAGE TO SEVERAL AWNINGS. (TBW) - - 1920,UNK,1 ENE MADISON,MADISON,FL,30.47,-83.41,SPOTTER REPORTS TREE DOWN ACROSS - DUVAL ST. NORTH OF MADISON. ALSO REPORTS TREE DOWN ON POWERLINE ON DUVAL ST. - SOUTH OF MADISON. (TAE) - - 2130,UNK,1 SE SPRINGFIELD,ROBERTSON,TN,36.49,-86.87,TWITTER PHOTOS SHOWED - LARGE TREE BLOWN DOWN IN SOUTHEAST SPRINGFIELD (OHX) - - 2205,UNK,1 SE WEST NASHVILLE,DAVIDSON,TN,36.15,-86.83,TREE DOWN IN THE ROAD - ON SENTINEL DRIVE AND HALF OF A TREE DOWN IN THE YARD OF A HOME ON 37TH AVENUE - IN SYLVAN HEIGHTS (OHX) - - 2210,UNK,BLACKVILLE,BARNWELL,SC,33.35,-81.28,DISPATCH REPORTED TREES DOWN - ON DEXTER STREET. TIME ESTIMATED BY RADAR. (CAE) - - 2231,UNK,NASHVILLE,DAVIDSON,TN,36.17,-86.78,PICTURE OF A LIMB DOWN ON MUSIC - ROW TOWARDS WEDGEWOOD. (OHX) - - 2259,UNK,4 NNW THOMPSON''S STATIO,WILLIAMSON,TN,35.87,-86.93,TWITTER PHOTOS - SHOWED SEVERAL TREES SNAPPED AND UPROOTED ALONG KING LANE. TWO CARS REPORTEDLY - TOTALED DUE TO FALLEN TREES. (OHX) - - 2305,UNK,4 SSW WHITEHALL,TREMPEALEAU,WI,44.31,-91.34,TREES DOWN. (ARX) - - 2310,UNK,1 N BONNIE,JEFFERSON,IL,38.22,-88.91,LATE REPORT. SEVERAL LARGE TREES - DOWN... METAL ROOF PEELED BACK ALONG WITH SOFFET AND SIDING BLOWN DOWN OFF - A HOME. TIME ESTIMATED BY RADAR. (PAH) - - 2315,UNK,3 S TAYLOR,JACKSON,WI,44.27,-91.13,TREE DOWN. TIME ESTIMATED FROM - RADAR. (ARX) - - 2357,63,2 WNW AUGUSTA,RICHMOND,GA,33.47,-82.04,63 MPH WIND GUST RECORDED BY - THE AUGUSTA DANIEL FIELD (KDNL) ASOS. (CAE) - - 0010,UNK,4 SSW KNIGHTSVILLE,DORCHESTER,SC,32.95,-80.26,HIGHWAY PATROL REPORTED - TREE DOWN AT THE INTERSECTION OF SUMMERS DRIVE AND HIGHWAY 61. (CHS) - - 0028,UNK,COTTAGEVILLE,COLLETON,SC,32.94,-80.48,HIGHWAY PATROL REPORTED A TREE - DOWN AT THE 1000 BLOCK OF GRIFFITH ACRES DRIVE. (CHS) - - 0043,UNK,9 ESE EAST SUMTER,SUMTER,SC,33.85,-80.17,COUNTY SHERIFF REPORTED - TREES IN ROADWAY ON E BREWINGTON RD. TREES ALSO DOWN ON FOXWORTH MILL ROAD... - ASBURY LANE AND QUEEN CHAPELS ROAD. TIME ESTIMATED BY RADAR. (CAE) - - 0133,UNK,2 ESE SOUTH SUMTER,SUMTER,SC,33.88,-80.3,SCHP REPORTED TREE IN ROADWAY - ON COWBY LANE AT BOULEVARD ROAD. TIME ESTIMATED BY RADAR. (CAE) - - 0141,60,1 NW CHERRYVALE,SUMTER,SC,33.97,-80.48,52-KT (60 MPH WIND GUST REPORTED - BY THE SHAW AFB ASOS (KSSC). (CAE) - - 0211,UNK,7 ENE SAINT MATTHEWS,CALHOUN,SC,33.71,-80.67,TREE IN ROADWAY ON COL. - THOMSON HIGHWAY. TIME ESTIMATED BY RADAR. (CAE) - - 0349,UNK,CAMP DOUGLAS,JUNEAU,WI,43.92,-90.27,TREE REPORTED DOWN IN CAMP DOUGLAS. - TIME ESTIMATED. (ARX) - - 0350,UNK,1 W DE FOREST,DANE,WI,43.25,-89.36,LARGE OAK TREE DOWN ABOUT 1 MILE - WEST OF DOWNTOWN DE FOREST. REPORTED VIA SOCIAL MEDIA. TIME ESTIMATED. (MKX) - -' - headers: - Accept-Ranges: - - bytes - Access-Control-Allow-Origin: - - '*' - Cache-Control: - - max-age=300 - Connection: - - close - Content-Length: - - '3504' - Content-Type: - - text/csv - Date: - - Wed, 10 Jul 2019 16:59:32 GMT - ETag: - - '"a8b45e43-db0-5729ae78f1fc0"' - Expires: - - Wed, 10 Jul 2019 17:04:32 GMT - Last-Modified: - - Sat, 04 Aug 2018 12:08:39 GMT - Server: - - Apache - Strict-Transport-Security: - - max-age=31536000 - Vary: - - Accept-Encoding - Via: - - 1.1 c4.w1.woc (squid), 1.1 969f35f01b6eddd92239a3e818fc1e0d.cloudfront.net - (CloudFront) - X-Amz-Cf-Id: - - PWdIn0DiONpYnL9iMtxHT4G1ID9a3mcS1hpe7Z8RoddQyuYBdbYctQ== - X-Amz-Cf-Pop: - - DEN50-C1 - X-Cache: - - Miss from cloudfront - X-XSS-Protection: - - 1; mode=block - status: - code: 200 - message: OK -version: 1 diff --git a/siphon/tests/test_spc.py b/siphon/tests/test_spc.py index 0c1517e2e..5eeb37064 100644 --- a/siphon/tests/test_spc.py +++ b/siphon/tests/test_spc.py @@ -26,112 +26,134 @@ def subset_records(response): # Grab whole lines up to this maximum size while index < 4096: index = data.find(b'\n', index + 1) + if index == -1: + index = 4096 response['body']['string'] = data[:index + 1] return response +def subset_zipped_records(response): + """Filter to subset SPC zipped archive and only save a subset of records.""" + from io import BytesIO + from zipfile import ZipFile + + data = response['body']['string'] + + # Unzip and pull out enough bytes to process + with ZipFile(BytesIO(data)) as zip_product: + target_filename = zip_product.namelist()[0] + target_file = zip_product.open(target_filename, 'r') + data = target_file.read(4200) + + # Grab whole lines up to this maximum size + index = 0 + while index < 4096: + index = data.find(b'\n', index + 1) + if index == -1: + index = 4096 + + # Rezip our truncated file + with BytesIO() as modified_bytes: + with ZipFile(modified_bytes, 'w') as modified_file: + modified_file.writestr(target_filename, data[:index + 1]) + response['body']['string'] = modified_bytes.getvalue() + + return response + + @recorder.use_cassette('spc_wind_archive', before_record_response=subset_records) def test_wind_archive(): """Test that we are properly parsing wind data from the SPC archive.""" # Testing wind events for random day in may 20th, 2010 - spc_wind_archive = SPCArchive('wind') - - assert(spc_wind_archive.storm_type == 'wind') - - assert(spc_wind_archive.storms['Num'].iloc[0] == 1) - assert(spc_wind_archive.storms['Year'].iloc[0] == 1955) - assert(spc_wind_archive.storms['Month'].iloc[0] == 2) - assert(spc_wind_archive.storms['Day'].iloc[0] == 1) - assert(spc_wind_archive.storms['Time'].iloc[0] == '13:45:00') - assert(spc_wind_archive.storms['Time Zone'].iloc[0] == 3) - assert(spc_wind_archive.storms['State'].iloc[0] == 'AR') - assert(spc_wind_archive.storms['Speed (kt)'].iloc[0] == 0) - assert(spc_wind_archive.storms['Injuries'].iloc[0] == 0) - assert(spc_wind_archive.storms['Fatalities'].iloc[0] == 0) - assert(spc_wind_archive.storms['Crop Loss'].iloc[0] == 0) - assert(spc_wind_archive.storms['Length (mi)'].iloc[0] == 0) - assert(spc_wind_archive.storms['Width (yd)'].iloc[0] == 0) - assert(spc_wind_archive.storms['Ns'].iloc[0] == 0) - assert(spc_wind_archive.storms['SN'].iloc[0] == 0) - assert(spc_wind_archive.storms['SG'].iloc[0] == 0) - assert(spc_wind_archive.storms['County Code 1'].iloc[0] == 77) - assert(spc_wind_archive.storms['County Code 2'].iloc[0] == 0) - assert(spc_wind_archive.storms['County Code 3'].iloc[0] == 0) - assert(spc_wind_archive.storms['County Code 4'].iloc[0] == 0) - assert_almost_equal(spc_wind_archive.storms['Property Loss'].iloc[0], 0) - assert_almost_equal(spc_wind_archive.storms['Start Lat'].iloc[0], 34.78, 3) - assert_almost_equal(spc_wind_archive.storms['Start Lon'].iloc[0], -90.78, 3) - assert_almost_equal(spc_wind_archive.storms['End Lat'].iloc[0], 0) - assert_almost_equal(spc_wind_archive.storms['End Lon'].iloc[0], 0) - - -@recorder.use_cassette('spc_torn_archive', before_record_response=subset_records) + storms = SPCArchive.get_wind_database(filename='1955-2017_wind.csv') + + assert storms['om'].iloc[0] == 1 + assert storms['Date'].iloc[0] == datetime(1955, 2, 1, 13, 45) + assert storms['Time Zone'].iloc[0] == 3 + assert storms['State'].iloc[0] == 'AR' + assert storms['State FIPS'].iloc[0] == 5 + assert storms['State Number'].iloc[0] == 1 + assert storms['Speed'].iloc[0] == 0 + assert storms['Injuries'].iloc[0] == 0 + assert storms['Fatalities'].iloc[0] == 0 + assert storms['Crop Loss'].iloc[0] == 0 + assert storms['Length (mi)'].iloc[0] == 0 + assert storms['Width (yd)'].iloc[0] == 0 + assert storms['ns'].iloc[0] == 0 + assert storms['sn'].iloc[0] == 0 + assert storms['sg'].iloc[0] == 0 + assert storms['County FIPS 1'].iloc[0] == 77 + assert storms['County FIPS 2'].iloc[0] == 0 + assert storms['County FIPS 3'].iloc[0] == 0 + assert storms['County FIPS 4'].iloc[0] == 0 + assert_almost_equal(storms['Property Loss'].iloc[0], 0) + assert_almost_equal(storms['Start Lat'].iloc[0], 34.78, 3) + assert_almost_equal(storms['Start Lon'].iloc[0], -90.78, 3) + assert_almost_equal(storms['End Lat'].iloc[0], 0) + assert_almost_equal(storms['End Lon'].iloc[0], 0) + + +@recorder.use_cassette('spc_torn_archive', before_record_response=subset_zipped_records) def test_torn_archive(): """Test that we are properly parsing tornado data from the SPC archive.""" - spc_torn_archive = SPCArchive('tornado') - - assert(spc_torn_archive.storm_type == 'tornado') - - assert(spc_torn_archive.storms['Num'].iloc[0] == 1) - assert(spc_torn_archive.storms['Year'].iloc[0] == 1950) - assert(spc_torn_archive.storms['Month'].iloc[0] == 1) - assert(spc_torn_archive.storms['Day'].iloc[0] == 3) - assert(spc_torn_archive.storms['Time'].iloc[0] == '11:00:00') - assert(spc_torn_archive.storms['Time Zone'].iloc[0] == 3) - assert(spc_torn_archive.storms['State'].iloc[0] == 'MO') - assert(spc_torn_archive.storms['F-Scale'].iloc[0] == 3) - assert(spc_torn_archive.storms['Injuries'].iloc[0] == 3) - assert(spc_torn_archive.storms['Fatalities'].iloc[0] == 0) - assert(spc_torn_archive.storms['Property Loss'].iloc[0] == 6) - assert(spc_torn_archive.storms['Crop Loss'].iloc[0] == 0) - assert(spc_torn_archive.storms['Length (mi)'].iloc[0] == 9.5) - assert(spc_torn_archive.storms['Width (yd)'].iloc[0] == 150) - assert(spc_torn_archive.storms['Ns'].iloc[0] == 2) - assert(spc_torn_archive.storms['SN'].iloc[0] == 0) - assert(spc_torn_archive.storms['SG'].iloc[0] == 1) - assert(spc_torn_archive.storms['County Code 1'].iloc[0] == 0) - assert(spc_torn_archive.storms['County Code 2'].iloc[0] == 0) - assert(spc_torn_archive.storms['County Code 3'].iloc[0] == 0) - assert(spc_torn_archive.storms['County Code 4'].iloc[0] == 0) - assert_almost_equal(spc_torn_archive.storms['Start Lat'].iloc[0], 38.77, 3) - assert_almost_equal(spc_torn_archive.storms['Start Lon'].iloc[0], -90.22, 3) - assert_almost_equal(spc_torn_archive.storms['End Lat'].iloc[0], 38.83, 3) - assert_almost_equal(spc_torn_archive.storms['End Lon'].iloc[0], -90.03, 3) + storms = SPCArchive.get_tornado_database() + + assert storms['om'].iloc[0] == 1 + assert storms['Date'].iloc[0] == datetime(1950, 1, 3, 11, 0, 0) + assert storms['Time Zone'].iloc[0] == 3 + assert storms['State'].iloc[0] == 'MO' + assert storms['State FIPS'].iloc[0] == 29 + assert storms['State Number'].iloc[0] == 1 + assert storms['F-Scale'].iloc[0] == 3 + assert storms['Injuries'].iloc[0] == 3 + assert storms['Fatalities'].iloc[0] == 0 + assert storms['Property Loss'].iloc[0] == 6 + assert storms['Crop Loss'].iloc[0] == 0 + assert storms['Length (mi)'].iloc[0] == 9.5 + assert storms['Width (yd)'].iloc[0] == 150 + assert storms['ns'].iloc[0] == 2 + assert storms['sn'].iloc[0] == 0 + assert storms['sg'].iloc[0] == 1 + assert storms['County FIPS 1'].iloc[0] == 0 + assert storms['County FIPS 2'].iloc[0] == 0 + assert storms['County FIPS 3'].iloc[0] == 0 + assert storms['County FIPS 4'].iloc[0] == 0 + assert_almost_equal(storms['Start Lat'].iloc[0], 38.77, 3) + assert_almost_equal(storms['Start Lon'].iloc[0], -90.22, 3) + assert_almost_equal(storms['End Lat'].iloc[0], 38.83, 3) + assert_almost_equal(storms['End Lon'].iloc[0], -90.03, 3) @recorder.use_cassette('spc_hail_archive', before_record_response=subset_records) def test_hail_archive(): """Test that we are properly parsing hail data from the SPC archive.""" - spc_hail_archive = SPCArchive('hail') - - assert(spc_hail_archive.storm_type == 'hail') - - assert(spc_hail_archive.storms['Num'].iloc[0] == 1) - assert(spc_hail_archive.storms['Year'].iloc[0] == 1955) - assert(spc_hail_archive.storms['Month'].iloc[0] == 1) - assert(spc_hail_archive.storms['Day'].iloc[0] == 17) - assert(spc_hail_archive.storms['Time'].iloc[0] == '16:39:00') - assert(spc_hail_archive.storms['Time Zone'].iloc[0] == 3) - assert(spc_hail_archive.storms['State'].iloc[0] == 'TX') - assert(spc_hail_archive.storms['Size (hundredth in)'].iloc[0] == 0.75) - assert(spc_hail_archive.storms['Injuries'].iloc[0] == 0) - assert(spc_hail_archive.storms['Fatalities'].iloc[0] == 0) - assert(spc_hail_archive.storms['Property Loss'].iloc[0] == 0) - assert(spc_hail_archive.storms['Crop Loss'].iloc[0] == 0) - assert(spc_hail_archive.storms['Length (mi)'].iloc[0] == 0) - assert(spc_hail_archive.storms['Width (yd)'].iloc[0] == 0) - assert(spc_hail_archive.storms['Ns'].iloc[0] == 0) - assert(spc_hail_archive.storms['SN'].iloc[0] == 0) - assert(spc_hail_archive.storms['SG'].iloc[0] == 0) - assert(spc_hail_archive.storms['County Code 1'].iloc[0] == 227) - assert(spc_hail_archive.storms['County Code 2'].iloc[0] == 0) - assert(spc_hail_archive.storms['County Code 3'].iloc[0] == 0) - assert(spc_hail_archive.storms['County Code 4'].iloc[0] == 0) - assert_almost_equal(spc_hail_archive.storms['Start Lat'].iloc[0], 32.2, 3) - assert_almost_equal(spc_hail_archive.storms['Start Lon'].iloc[0], -101.5, 3) - assert_almost_equal(spc_hail_archive.storms['End Lat'].iloc[0], 0.0, 3) - assert_almost_equal(spc_hail_archive.storms['End Lon'].iloc[0], 0.0, 3) + storms = SPCArchive.get_hail_database('1955-2017_hail.csv') + + assert storms['om'].iloc[0] == 1 + assert storms['Date'].iloc[0] == datetime(1955, 1, 17, 16, 39) + assert storms['Time Zone'].iloc[0] == 3 + assert storms['State'].iloc[0] == 'TX' + assert storms['State FIPS'].iloc[0] == 48 + assert storms['State Number'].iloc[0] == 1 + assert storms['Size'].iloc[0] == 0.75 + assert storms['Injuries'].iloc[0] == 0 + assert storms['Fatalities'].iloc[0] == 0 + assert storms['Property Loss'].iloc[0] == 0 + assert storms['Crop Loss'].iloc[0] == 0 + assert storms['Length (mi)'].iloc[0] == 0 + assert storms['Width (yd)'].iloc[0] == 0 + assert storms['ns'].iloc[0] == 0 + assert storms['sn'].iloc[0] == 0 + assert storms['sg'].iloc[0] == 0 + assert storms['County FIPS 1'].iloc[0] == 227 + assert storms['County FIPS 2'].iloc[0] == 0 + assert storms['County FIPS 3'].iloc[0] == 0 + assert storms['County FIPS 4'].iloc[0] == 0 + assert_almost_equal(storms['Start Lat'].iloc[0], 32.2, 3) + assert_almost_equal(storms['Start Lon'].iloc[0], -101.5, 3) + assert_almost_equal(storms['End Lat'].iloc[0], 0.0, 3) + assert_almost_equal(storms['End Lon'].iloc[0], 0.0, 3) @recorder.use_cassette('spc_wind_after_2011_archive') @@ -224,9 +246,3 @@ def test_hail_before_1955(): """Test error retrieval of hail data before 1955.""" with pytest.raises(ValueError): SPC.get_hail_reports(datetime(1949, 5, 20)) - - -def test_no_data_spc_archive(): - """Test error data when passed an invalid storm type.""" - with pytest.raises(ValueError): - SPCArchive('tornado and wind')