Drop events with no L1/L2/L3 impacts (or NULL impacts) (#195)

VUB-HYDR · Nov 27, 2024 · 3ddd749 · 3ddd749
1 parent 0280d9e
commit 3ddd749
Show file tree

Hide file tree

Showing 4,187 changed files with 1,312,770 additions and 2 deletions.
diff --git a/Database/fill_data_gap.py b/Database/fill_data_gap.py
@@ -56,7 +56,7 @@
                 logger.info(
                     f"Dropping any records of {e} in {name} for impact {impact}. Shape before: {level[impact].shape}"
                 )
-                level[impact] = level[impact][~(level[impact]["Event_ID"] == e)]
+                level[impact] = level[impact][~(level[impact][dg_utils.event_id] == e)]
                 logger.info(
                     f"Dropped any records of {e} in {name} for impact {impact}. Shape after: {level[impact].shape}"
                 )
@@ -322,6 +322,46 @@
         for impact in level.keys():
             level[impact].replace(float("nan"), None, inplace=True)
 
+    null_mask_total = l1[[x for x in l1.columns if "Total_" in x]].isnull().all(axis=1)
+    e_ids_missing_l1_impacts: list[str] = list(l1[null_mask_total].Event_ID.unique())
+    missing_event_ids_to_drop: list[str] = []
+    logger.info(f"Found {len(e_ids_missing_l1_impacts)} Event IDs without impacts")
+    for e_id in e_ids_missing_l1_impacts:
+        drop_l2, drop_l3 = True, True
+        for impact in l2.keys():
+            null_mask_l2 = l2[impact][[dg_utils.num_min, dg_utils.num_max]].isnull().all(axis=1)
+            l2_df = l2[impact][(~null_mask_l2) & (l2[impact][dg_utils.event_id] == e_id)]
+            if not l2_df.empty:
+                logger.warning(f"L2 {e_id} contains impacts not propagated to L1!\n{l2_df}")
+                drop_l2 = False
+            del l2_df
+
+            null_mask_l3 = l3[impact][[dg_utils.num_min, dg_utils.num_max]].isnull().all(axis=1)
+            l3_df = l3[impact][(~null_mask_l3) & (l3[impact][dg_utils.event_id] == e_id)]
+            if not l3_df.empty:
+                logger.warning(f"L3 {e_id} contains impacts not propagated to L1!\n{l3_df}")
+                drop_l3 = False
+            del l3_df
+
+        if drop_l3 and drop_l2:
+            missing_event_ids_to_drop.append(e_id)
+
+    missing_event_ids_to_drop = list(set(missing_event_ids_to_drop))
+
+    logger.warning(
+        f"Dropping the following {len(missing_event_ids_to_drop)} IDs from L1/L2/L3: {missing_event_ids_to_drop}"
+    )
+    l1 = l1[~l1[dg_utils.event_id].isin(missing_event_ids_to_drop)]
+    for level in [l2, l3]:
+        for impact in level.keys():
+            for e in missing_event_ids_to_drop:
+                level[impact] = level[impact][~(level[impact][dg_utils.event_id] == e)]
+
+    # Replace NaNs will NoneType
+    for level in [l2, l3]:
+        for impact in level.keys():
+            level[impact].replace(float("nan"), None, inplace=True)
+
     logger.info(f"Storing results in {args.output_dir}")
     pathlib.Path(args.output_dir).mkdir(parents=True, exist_ok=True)
 

diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0000.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0000.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0001.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0001.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0002.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0002.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0003.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0003.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0004.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0004.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0005.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0005.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0006.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0006.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0007.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0007.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0008.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0008.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0009.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0009.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0010.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0010.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0011.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0011.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0012.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0012.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0013.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0013.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0014.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0014.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0015.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0015.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0016.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0016.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0017.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0017.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0018.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0018.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0019.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0019.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0020.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0020.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0021.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0021.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0022.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0022.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0023.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0023.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0024.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0024.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0025.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0025.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0026.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0026.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0027.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0027.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0028.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0028.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0029.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0029.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0030.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0030.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0031.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0031.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0032.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0032.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0033.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0033.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0034.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0034.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0035.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0035.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0036.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0036.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0037.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0037.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0038.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0038.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0039.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0039.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0040.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0040.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0041.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0041.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0042.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0042.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0043.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0043.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0044.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0044.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0045.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0045.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0046.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0046.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0047.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0047.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0048.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0048.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0049.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0049.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0050.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0050.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0051.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0051.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0052.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0052.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0053.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0053.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0054.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0054.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0055.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0055.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0056.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0056.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0057.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0057.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0058.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0058.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0059.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0059.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0060.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0060.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0061.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0061.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0062.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0062.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0063.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0063.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0064.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0064.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0065.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0065.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0066.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0066.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0067.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0067.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0068.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0068.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0069.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0069.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0070.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0070.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0071.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0071.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0072.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0072.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0073.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0073.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0074.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0074.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0075.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0075.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0076.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0076.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0077.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0077.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0078.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0078.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0079.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0079.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0080.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0080.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0081.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0081.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0082.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0082.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0083.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0083.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0084.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0084.parquet
diff --git a/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0085.parquet b/..._run_25_deduplicated_data_gap_inflation_adjustment_drop_missing_event_ids/l1/0085.parquet