Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Added web server for deployment #27

Merged
merged 25 commits into from
Dec 13, 2020
Merged
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
7914cb5
refactor: Updated package import hierarchy
frgfm Dec 13, 2020
35210b7
chore: Removed unused dependencies
frgfm Dec 13, 2020
bef3893
feat: Added dummy cache mechanism
frgfm Dec 13, 2020
2b4a42c
feat: Added extra information logging
frgfm Dec 13, 2020
e6d853b
feat: Added FastAPI web server for deployment
frgfm Dec 13, 2020
b30a4c4
chore: Added docker orchestration
frgfm Dec 13, 2020
573a316
feat: Added date as a route argument
frgfm Dec 13, 2020
964a207
refactor: Reversed cache to prevent system running OOM
frgfm Dec 13, 2020
2b008b5
refactor: Silenced urllib warnings
frgfm Dec 13, 2020
092c045
chore: Added Heroku setup file
frgfm Dec 13, 2020
50b9107
chore: Build debug
frgfm Dec 13, 2020
83edeb1
chore: Updated heroku setup file
frgfm Dec 13, 2020
ab1eca6
chore: Added apt prebuild
frgfm Dec 13, 2020
40200df
chore: Build debug
frgfm Dec 13, 2020
b83f799
chore: Fixed procfile
frgfm Dec 13, 2020
7bdaf67
chore: Added back requirements
frgfm Dec 13, 2020
47bc0dc
refactor: Reflected package import fix
frgfm Dec 13, 2020
d28f42f
refactor: Removed unnecessary dependencies
frgfm Dec 13, 2020
8951aa1
chore: Added workflow to check web server sanity
frgfm Dec 13, 2020
bc57416
style: Fixed lint
frgfm Dec 13, 2020
62466a9
refactor: Removed unused import
frgfm Dec 13, 2020
d977e87
chore: Fixed CI config
frgfm Dec 13, 2020
014074e
chore: Fixed workflows
frgfm Dec 13, 2020
3c87ee4
feat: Added possibility to load env variables from .env
frgfm Dec 13, 2020
3791f1e
docs: Updated README
frgfm Dec 13, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: Added dummy cache mechanism
frgfm committed Dec 13, 2020
commit bef38930000e159bc180eb8e8661d5083c96b3f6
4 changes: 4 additions & 0 deletions pyro_risks/config.py
Original file line number Diff line number Diff line change
@@ -67,3 +67,7 @@
'objective': 'binary:logistic',
'eval_metric': ['logloss', 'aucpr']
}

CACHE_FOLDER: str = ".cache"
if not os.path.exists(CACHE_FOLDER):
os.makedirs(CACHE_FOLDER)
8 changes: 4 additions & 4 deletions pyro_risks/datasets/ERA5.py
Original file line number Diff line number Diff line change
@@ -33,25 +33,25 @@ def get_data_era5land_for_predict(date: str) -> pd.DataFrame:
year, month, day = date.split("-")
call_era5land(tmp, year, month, day)
# TODO: make sure that the directory works when on server
data = ERA5Land(source_path=os.path.join(tmp, f"era5land_{year}_{month}_{day}.nc"))
data = ERA5Land(source_path=os.path.join(cfg.CACHE_FOLDER, f"era5land_{year}_{month}_{day}.nc"))

# Lag J-1
lag = np.datetime64(date) - np.timedelta64(1, "D")
year, month, day = str(lag).split("-")
call_era5land(tmp, year, month, day)
dataJ1 = ERA5Land(source_path=os.path.join(tmp, f"era5land_{year}_{month}_{day}.nc"))
dataJ1 = ERA5Land(source_path=os.path.join(cfg.CACHE_FOLDER, f"era5land_{year}_{month}_{day}.nc"))

# Lag J-3
lag = np.datetime64(date) - np.timedelta64(3, "D")
year, month, day = str(lag).split("-")
call_era5land(tmp, year, month, day)
dataJ3 = ERA5Land(source_path=os.path.join(tmp, f"era5land_{year}_{month}_{day}.nc"))
dataJ3 = ERA5Land(source_path=os.path.join(cfg.CACHE_FOLDER, f"era5land_{year}_{month}_{day}.nc"))

# Lag J-7
lag = np.datetime64(date) - np.timedelta64(7, "D")
year, month, day = str(lag).split("-")
call_era5land(tmp, year, month, day)
dataJ7 = ERA5Land(source_path=os.path.join(tmp, f"era5land_{year}_{month}_{day}.nc"))
dataJ7 = ERA5Land(source_path=os.path.join(cfg.CACHE_FOLDER, f"era5land_{year}_{month}_{day}.nc"))

merged_data = pd.concat([data, dataJ1, dataJ3, dataJ7], ignore_index=True)
return merged_data
2 changes: 1 addition & 1 deletion pyro_risks/datasets/fwi.py
Original file line number Diff line number Diff line change
@@ -72,7 +72,7 @@ def get_fwi_from_api(date: str):
with tempfile.TemporaryDirectory() as tmp:
call_fwi(tmp, year, month, day)

file = zipfile.ZipFile(os.path.join(tmp, f"fwi_{year}_{month}_{day}.zip"))
file = zipfile.ZipFile(os.path.join(cfg.CACHE_FOLDER, f"fwi_{year}_{month}_{day}.zip"))
file.extractall(path=os.path.join(tmp, f"fwi_{year}_{month}_{day}"))

df0 = pd.DataFrame({})
28 changes: 25 additions & 3 deletions pyro_risks/datasets/queries_api.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import cdsapi
import os
import logging

from pyro_risks import config as cfg

logger = logging.getLogger("uvicorn.warning")


def call_era5land(output_path: str, year: str, month: str, day: str) -> None:
"""Call cdpaspi to get ERA5Land data as file nc format for given date.
@@ -16,6 +19,12 @@ def call_era5land(output_path: str, year: str, month: str, day: str) -> None:
month: str
day: str
"""
file_path = os.path.join(cfg.CACHE_FOLDER, f"era5land_{year}_{month}_{day}.nc")

if os.path.exists(file_path):
logger.info(f"Using cached {file_path}")
return

chloeskt marked this conversation as resolved.
Show resolved Hide resolved
c = cdsapi.Client(url=cfg.CDS_URL, key=f"{cfg.CDS_UID}:{cfg.CDS_API_KEY}", verify=0)

c.retrieve(
@@ -85,7 +94,7 @@ def call_era5land(output_path: str, year: str, month: str, day: str) -> None:
],
"format": "netcdf",
},
os.path.join(output_path, f"era5land_{year}_{month}_{day}.nc"),
file_path,
)


@@ -102,6 +111,12 @@ def call_era5t(output_path: str, year: str, month: str, day: str) -> None:
month: str
day: str
"""
file_path = os.path.join(cfg.CACHE_FOLDER, f"era5t_{year}_{month}_{day}.nc")

if os.path.exists(file_path):
logger.info(f"Using cached {file_path}")
return

c = cdsapi.Client(url=cfg.CDS_URL, key=f"{cfg.CDS_UID}:{cfg.CDS_API_KEY}", verify=0)

c.retrieve(
@@ -387,7 +402,7 @@ def call_era5t(output_path: str, year: str, month: str, day: str) -> None:
],
"format": "netcdf",
},
os.path.join(output_path, f"era5t_{year}_{month}_{day}.nc"),
file_path,
)
# TODO : take only needed variables for the model

@@ -407,6 +422,13 @@ def call_fwi(output_path, year, month, day):
month: str
day: str
"""

file_path = os.path.join(cfg.CACHE_FOLDER, f"fwi_{year}_{month}_{day}.zip")

if os.path.exists(file_path):
logger.info(f"Using cached {file_path}")
return

c = cdsapi.Client(url=cfg.CDS_URL, key=f"{cfg.CDS_UID}:{cfg.CDS_API_KEY}", verify=0)

c.retrieve(
@@ -430,4 +452,4 @@ def call_fwi(output_path, year, month, day):
'product_type': 'reanalysis',
'day': day,
},
os.path.join(output_path, f"fwi_{year}_{month}_{day}.zip"))
file_path)