Merge pull request #18 from ludwiglierhammer/fixxing_tests

fixxing level1e

CHANGES.rst

@@ -13,7 +13,8 @@ Breaking changes
 
 Bug fixes
 ^^^^^^^^^
-* fixing observation suite level1e tests (:pull:`17`) 
+* fixing observation suite level1e tests (:pull:`17`)
+* level1e: change QC mapping from ``v7.0.0`` is now running by setting values of ``location_quality`` and ``report_time_quality`` to ``str`` (:pull:`18`)
 
 v7.0.0 (2024-06-13)
 -------------------
@@ -36,7 +37,7 @@ Breaking changes
 * change QC mapping in obs_suite level1e (:issue:`7`, :pull:`8`):
 
   * if ``location_quality`` is equal ``2`` set both ``report_quality`` and ``quality_flag`` to ``1``
-  * if ```report_time_quality`` is equal ``4`` or ``5`` set both ``report_quality`` and ``quality_flag`` to ``1``
+  * if ``report_time_quality`` is equal ``4`` or ``5`` set both ``report_quality`` and ``quality_flag`` to ``1``
 
 New features and enhancements
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

glamod_marine_processing/obs_suite/scripts/level1e.py

@@ -211,9 +211,9 @@ def add_report_quality(qc_df_full):
 
 def compare_quality_checks(df):
     """Compare entries with location_quality and report_time_quality."""
-    df = df.mask(location_quality == 2, 1)
-    df = df.mask(report_time_quality == 4, 1)
-    df = df.mask(report_time_quality == 5, 1)
+    df = df.mask(location_quality == "2", "1")
+    df = df.mask(report_time_quality == "4", "1")
+    df = df.mask(report_time_quality == "5", "1")
     return df
 
 
@@ -283,13 +283,6 @@ def process_table(table_df, table_name):
         table_df["quality_flag"] = compare_quality_checks(table_df["quality_flag"])
 
     if table_name == "header":
-        # set report quality to 2 for ids with partial match to TEST
-        if params.year >= "2015":
-            loc = (table_df.primary_station_id.str.contains("TEST")) & (
-                table_df.duplicate_status == "4"
-            )
-            table_df.report_quality.loc[loc] = "2"
-
         table_df["report_quality"] = compare_quality_checks(table_df["report_quality"])
 
     cdm_columns = cdm_tables.get(table_name).keys()
@@ -523,8 +516,8 @@ def clean_level(file_id):
 #    header.location_quality = default not-checked ('3') to not-checked('3')
 
 # First header, then rest.
-location_quality = header_df["location_quality"]
-report_time_quality = header_df["report_time_quality"]
+location_quality = header_df["location_quality"].copy()
+report_time_quality = header_df["report_time_quality"].copy()
 
 flag = True if qc_avail else False
 process_table(header_df, "header")

tests/_load_data.py

@@ -55,26 +55,28 @@ def load_input(level):
     """Load level input data data from cdm-testdata."""
     p_level = prev_level[level]
     leveli = level_input[level]
+    cache_dir = f"./T{level}/{leveli}/ICOADS_R3.0.2T/{p_level}/114-992"
     if level == "level1a":
-        load_imma(level, leveli, p_level)
+        load_imma(cache_dir)
     else:
-        load_cdms(level, leveli, p_level)
+        load_cdms(cache_dir)
+    return cache_dir
 
 
-def load_cdms(level, leveli, p_level):
+def load_cdms(cache_dir):
     """Load level CDM input data from cdm-testdata."""
     for table_name in table_names:
         load_file(
             f"imma1_992/cdm_tables/{table_name}-114-992_2022-01-01_subset.psv",
-            cache_dir=f"./T{level}/{leveli}/ICOADS_R3.0.2T/{p_level}/114-992",
+            cache_dir=cache_dir,
             within_drs=False,
         )
 
 
-def load_imma(level, leveli, p_level):
+def load_imma(cache_dir):
     """Load level IMMA input data from cdm-testdata."""
     load_file(
         "imma1_992/input/114-992_2022-01-01_subset.imma",
-        cache_dir=f"./T{level}/{leveli}/ICOADS_R3.0.2T/{p_level}/114-992",
+        cache_dir=cache_dir,
         within_drs=False,
     )

tests/_settings.py

@@ -60,20 +60,18 @@
 }
 
 manipulation = {
-    "level1a": {},
-    "level1b": {},
     "level1c": {},
     "level1d": {
         ("header", "station_name"): [
             "null",
             "FF HELMER HANSEN",
             "WAVERIDER TFSTD",
-            "NORNE",
+            "null",
             "WAVERIDER TFDRN",
         ],
-        ("header", "platform_sub_type"): ["null", "RV", "OT", "MI", "OT"],
-        ("header", "station_record_number"): ["1", "1", "0", "13", "0"],
-        ("header", "report_duration"): ["11", "HLY", "11", "HLY", "11"],
+        ("header", "platform_sub_type"): ["null", "RV", "OT", "null", "OT"],
+        ("header", "station_record_number"): ["1", "1", "0", "1", "0"],
+        ("header", "report_duration"): ["11", "HLY", "11", "11", "11"],
         ("observations-at", "sensor_id"): ["null", "AT", np.nan, "null", np.nan],
         ("observations-dpt", "sensor_id"): [np.nan, "HUM", np.nan, "null", np.nan],
         ("observations-slp", "sensor_id"): ["null", "SLP", np.nan, "null", np.nan],
@@ -132,5 +130,9 @@
         ("observations-wd", "quality_flag"): ["0", "0", np.nan, "0", np.nan],
         ("observations-ws", "quality_flag"): ["0", "0", np.nan, "0", np.nan],
     },
-    "level2": {},
+}
+
+drops = {
+    "level1a": [0],
+    "level1c": [0, 3],
 }

tests/_testing_suite.py

@@ -20,8 +20,11 @@
 
 def manipulate_expected(expected, level):
     """Manipulate expected result data."""
-    for index, values in _settings.manipulation[level].items():
-        expected[index] = values
+    if level in _settings.manipulation.keys():
+        for index, values in _settings.manipulation[level].items():
+            expected[index] = values
+    if level in _settings.drops.keys():
+        expected = expected.drop(_settings.drops[level]).reset_index(drop=True)
     return expected
 
 
@@ -33,7 +36,7 @@ def _obs_testing(level, capsys):
             cache_dir=f"./T{level}/release_7.0",
         )
 
-    _load_data.load_input(level)
+    cache_dir = _load_data.load_input(level)
 
     s = (
         "obs_suite "