Skip to content

Commit

Permalink
check for empty data
Browse files Browse the repository at this point in the history
  • Loading branch information
garciampred committed Feb 23, 2024
1 parent 4678f4d commit 346d8ee
Showing 1 changed file with 6 additions and 0 deletions.
6 changes: 6 additions & 0 deletions cdsobs/ingestion/readers/cuon.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,6 +272,9 @@ def read_cuon_netcdfs(
else:
# Do not use threads as HDF5 is not yet thread safe.
scheduler = "processes"
# Check for emptiness
if len(files_and_slices) == 0:
raise EmptyBatchException
# Use dask to speed up the process
for file_and_slices in files_and_slices:
denormalized_table_future = dask.delayed(_get_denormalized_table_file)(
Expand All @@ -284,6 +287,9 @@ def read_cuon_netcdfs(
scheduler=scheduler,
num_workers=min(len(files_and_slices), 32),
)
# Check for emptiness
if all([dt is None for dt in denormalized_tables]):
raise EmptyBatchException
return pandas.concat(denormalized_tables)


Expand Down

0 comments on commit 346d8ee

Please sign in to comment.