Skip to content

Commit

Permalink
fix coding norms issues
Browse files Browse the repository at this point in the history
  • Loading branch information
ypwang19 committed May 28, 2024
1 parent c224002 commit 8beebee
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 50 deletions.
5 changes: 2 additions & 3 deletions scripts/exglobal_prep_aero_obs.py
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
# exglobal_prep_aero_obs.py
# This script collect available viirs
# This script collect available viirs
# obs files, combine and preprocess
# them.
# them.
import os

from wxflow import Logger, cast_strdict_as_dtypedict
Expand All @@ -12,7 +12,6 @@
logger = Logger(level='DEBUG', colored_log=True)



if __name__ == '__main__':

# Take configuration from environment and cast it as python dictionary
Expand Down
68 changes: 23 additions & 45 deletions ush/python/pygfs/task/aero_prepobs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,43 +50,38 @@ def initialize(self) -> None:
Generate corrosponding YMAL file.
Run IODA converter.
"""

self.task_config.COM_OBS_CHEM = os.path.join(self.task_config.COM_OBS,'chem')
self.task_config.COM_OBS_CHEM = os.path.join(self.task_config.COM_OBS, 'chem')
if os.path.exists(self.task_config.COM_OBS_CHEM):
rmdir(self.task_config.COM_OBS_CHEM)
rmdir(self.task_config.COM_OBS_CHEM)
FileHandler({'mkdir': [self.task_config.COM_OBS_CHEM]}).sync()

self.task_config.DATA_OBS = os.path.join(self.task_config.DATA,'obs')
self.task_config.DATA_OBS = os.path.join(self.task_config.DATA, 'obs')
if os.path.exists(self.task_config.DATA_OBS):
rmdir(self.task_config.DATA_OBS)
rmdir(self.task_config.DATA_OBS)
FileHandler({'mkdir': [self.task_config.DATA_OBS]}).sync()

self.task_config.prepaero_yaml = []
for sensor in self.task_config.sensors:
# print(sensor,'file_list',file_list)
raw_files = self.list_raw_files(sensor)
print('raw_files',raw_files)
self.task_config.input_files = self.copy_obs(raw_files)
print('raw files in obs_chem',self.task_config.input_files)
self.link_obsconvexe()
self.task_config.prepaero_config = self.get_obsproc_config(sensor)

# generate converter YAML file
_prepaero_yaml = os.path.join(self.runtime_config.DATA, f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml")
self.task_config.prepaero_yaml.append( _prepaero_yaml)
# generate converter YAML file
template = f"{self.runtime_config.CDUMP}.t{self.runtime_config['cyc']:02d}z.prepaero_viirs_{sensor}.yaml"
_prepaero_yaml = os.path.join(self.runtime_config.DATA, template)
self.task_config.prepaero_yaml.append(_prepaero_yaml)
logger.debug(f"Generate PrepAeroObs YAML file: {_prepaero_yaml}")
save_as_yaml(self.task_config.prepaero_config, _prepaero_yaml)
logger.info(f"Wrote PrepAeroObs YAML to: {_prepaero_yaml}")



@logit(logger)
def list_raw_files(self,sensor) -> List[str]:
def list_raw_files(self, sensor) -> List[str]:
"""
List all files in the predefined directory that match the predefined sensor and within the time window.
"""
if sensor == 'n20':
sensor = 'j01'
if sensor == 'n20':
sensor = 'j01'
dir1 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_begin)}"
dir2 = f"{self.task_config.data_dir}/{datetime_to_YMD(self.task_config.window_end)}"

Expand All @@ -99,8 +94,7 @@ def list_raw_files(self,sensor) -> List[str]:
allfiles_1 = [os.path.join(dir1, file) for file in files_1]
files_2 = os.listdir(dir2)
allfiles_2 = [os.path.join(dir2, file) for file in files_2]
allfiles = sorted(allfiles_1,allfiles_2)

allfiles = sorted(allfiles_1, allfiles_2)
matching_files = []
try:
for file in allfiles:
Expand All @@ -110,26 +104,22 @@ def list_raw_files(self,sensor) -> List[str]:
dd = fshort[0][24:26]
HH = fshort[0][26:28]
MM = fshort[0][28:30]
fstart = to_datetime(yyyy+'-'+mm+'-'+dd+'T'+HH+':'+MM+'Z')

fstart = to_datetime(yyyy + '-' + mm + '-' + dd + 'T' + HH + ':' + MM + 'Z')
if sensor in file:
# temporally select obs files based on time stamp inthe filename.
if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end):
# print('time window',sensor,fstart, self.task_config.window_begin, self.task_config.window_end)
matching_files.append(os.path.join(self.task_config.data_dir, file))
# temporally select obs files based on time stamp inthe filename.
if (fstart > self.task_config.window_begin) and (fstart < self.task_config.window_end):
matching_files.append(os.path.join(self.task_config.data_dir, file))
logger.info("Found %d matching files.", len(matching_files))
except FileNotFoundError:
logger.error("The specified file/directory does not exist.")
raise
return matching_files


@logit(logger)
def copy_obs(self,inputfiles) -> Dict[str, Any]:
def copy_obs(self, inputfiles) -> Dict[str, Any]:
"""
Copy the raw obs files to $DATA/obs.
"""

copylist = []
destlist = []
for filename in inputfiles:
Expand All @@ -141,27 +131,24 @@ def copy_obs(self,inputfiles) -> Dict[str, Any]:

return destlist


@logit(logger)
def get_obsproc_config(self,sensor) -> Dict[str, Any]:
"""Compile a dictionary of obs proc configuration from OBSPROCYAML template file
def get_obsproc_config(self, sensor) -> Dict[str, Any]:
"""
Compile a dictionary of obs proc configuration from OBSPROCYAML template file
Parameters
----------
Returns
----------
obsproc_config : Dict
a dictionary containing the fully rendered obs proc yaml configuration
"""

self.task_config.sensor = sensor
# generate JEDI YAML file
logger.info(f"Generate gdas_obsprovider2ioda YAML config: {self.task_config.OBSPROCYAML}")
prepaero_config = parse_j2yaml(self.task_config.OBSPROCYAML, self.task_config)
# logger.debug(f"OBSPROC config:\n{format(obsproc_config)}")

return prepaero_config


@logit(logger)
def link_obsconvexe(self) -> None:
"""
Expand Down Expand Up @@ -205,7 +192,6 @@ def runConverter(self) -> None:

pass


@logit(logger)
def finalize(self) -> None:
"""
Expand All @@ -218,7 +204,7 @@ def finalize(self) -> None:
copylist = []
for obsfile in obsfiles:
basename = os.path.basename(obsfile)
src = os.path.join(self.task_config['DATA'],basename)
src = os.path.join(self.task_config['DATA'], basename)
dest = os.path.join(self.task_config.COM_OBS, basename)
copylist.append([src, dest])
FileHandler({'copy': copylist}).sync()
Expand All @@ -234,8 +220,6 @@ def finalize(self) -> None:
for obsfile in obsfiles:
aeroobsgzip = f"{obsfile}.gz"
archive.add(aeroobsgzip, arcname=os.path.basename(aeroobsgzip))


# get list of raw viirs L2 files
rawfiles = glob.glob(os.path.join(self.task_config.DATA_OBS, 'JRR-AOD*'))
# gzip the raw L2 files first
Expand All @@ -249,17 +233,11 @@ def finalize(self) -> None:
for rawfile in rawfiles:
aerorawobsgzip = f"{rawfile}.gz"
archive.add(aerorawobsgzip, arcname=os.path.basename(aerorawobsgzip))

copylist = []
for prepaero_yaml in self.task_config.prepaero_yaml:
basename = os.path.basename(prepaero_yaml)
dest = os.path.join(self.task_config.COM_OBS,basename)
copylist.append([prepaero_yaml,dest])
dest = os.path.join(self.task_config.COM_OBS, basename)
copylist.append([prepaero_yaml, dest])
FileHandler({'copy': copylist}).sync()

pass





3 changes: 1 addition & 2 deletions workflow/applications/gfs_cycled.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ def _get_app_configs(self):
if self.do_aero:
configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal']
if self.do_prep_aero_obs:
configs += ['prepaeroobs']
configs += ['prepaeroobs']

if self.do_jedisnowda:
configs += ['prepsnowobs', 'snowanl']
Expand Down Expand Up @@ -157,7 +157,6 @@ def get_task_names(self):
if self.do_prep_aero_obs:
gdas_gfs_common_tasks_before_fcst += ['prepaeroobs']


if self.do_jedisnowda:
gdas_gfs_common_tasks_before_fcst += ['prepsnowobs', 'snowanl']

Expand Down

0 comments on commit 8beebee

Please sign in to comment.