Refactor test directories

disdrodb/__init__.py

@@ -7,7 +7,7 @@
     available_stations,
 )
 from disdrodb.api.metadata import read_station_metadata
-from disdrodb.l0.standards import available_sensor_name
+from disdrodb.l0.standards import available_sensor_names
 
 __root_path__ = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
@@ -16,7 +16,7 @@
     "available_stations",
     "available_campaigns",
     "available_data_sources",
-    "available_sensor_name",
+    "available_sensor_names",
     "read_station_metadata",
 ]
 

disdrodb/l0/check_configs.py

@@ -7,12 +7,12 @@
 from pydantic import BaseModel, ValidationError, field_validator, model_validator
 
 from disdrodb.l0.standards import (
-    available_sensor_name,
-    get_configs_dir,
+    available_sensor_names,
     get_diameter_bin_center,
     get_diameter_bin_lower,
     get_diameter_bin_upper,
     get_diameter_bin_width,
+    get_sensor_configs_dir,
     get_velocity_bin_center,
     get_velocity_bin_lower,
     get_velocity_bin_upper,
@@ -41,7 +41,7 @@ def check_yaml_files_exists(sensor_name: str) -> None:
     sensor_name : str
         Name of the sensor.
     """
-    config_dir = get_configs_dir(sensor_name)
+    config_dir = get_sensor_configs_dir(sensor_name)
 
     list_of_file_names = [os.path.split(i)[-1] for i in glob.glob(f"{config_dir}/*.yml")]
 
@@ -375,5 +375,5 @@ def check_sensor_configs(sensor_name: str) -> None:
 
 def check_all_sensors_configs() -> None:
     """Check all sensors configs."""
-    for sensor_name in available_sensor_name():
+    for sensor_name in available_sensor_names():
         check_sensor_configs(sensor_name=sensor_name)

disdrodb/l0/check_readers.py

@@ -4,12 +4,13 @@
 
 import pandas as pd
 
+from disdrodb import __root_path__
 from disdrodb.api.io import _get_disdrodb_directory
 from disdrodb.l0.l0_reader import get_station_reader
 
 # current file path
 PACKAGE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-RAW_DIR = os.path.join(PACKAGE_DIR, "tests", "pytest_files", "check_readers", "DISDRODB")
+RAW_DIR = os.path.join(__root_path__, "disdrodb", "tests", "pytest_files", "check_readers", "DISDRODB")
 
 
 def get_list_test_data_sources() -> list:

disdrodb/l0/check_standards.py

@@ -131,13 +131,13 @@ def check_sensor_name(sensor_name: str) -> None:
     ValueError
         Error if the input sensor name has not been found in the list of available sensors.
     """
-    from disdrodb.l0.standards import available_sensor_name
+    from disdrodb.l0.standards import available_sensor_names
 
-    available_sensor_name = available_sensor_name()
+    available_sensor_names = available_sensor_names()
     if not isinstance(sensor_name, str):
         raise TypeError("'sensor_name' must be a string.")
-    if sensor_name not in available_sensor_name:
-        msg = f"{sensor_name} not valid {sensor_name}. Valid values are {available_sensor_name}."
+    if sensor_name not in available_sensor_names:
+        msg = f"{sensor_name} not valid {sensor_name}. Valid values are {available_sensor_names}."
         logger.error(msg)
         raise ValueError(msg)
 

disdrodb/l0/io.py

@@ -562,7 +562,7 @@ def _remove_if_exists(fpath: str, force: bool = False) -> None:
                         msg = f"Can not delete file {f}, error: {e.strerror}"
                         logger.exception(msg)
                 os.rmdir(fpath)
-            except:
+            except Exception:
                 msg = f"Something wrong with: {fpath}"
                 logger.error(msg)
                 raise ValueError(msg)

disdrodb/l0/l0_processing.py

@@ -238,7 +238,6 @@ def _generate_l0b(
         error_type = str(type(e).__name__)
         msg = f"{error_type}: {e}"
         log_error(logger, msg, verbose=verbose)
-        pass
 
     # Close the file logger
     close_logger(logger)
@@ -314,7 +313,6 @@ def _generate_l0b_from_nc(
         error_type = str(type(e).__name__)
         msg = f"{error_type}: {e}"
         log_error(logger, msg, verbose=verbose)
-        pass
 
     # Close the file logger
     close_logger(logger)

disdrodb/l0/l0_reader.py

@@ -28,11 +28,9 @@
 
 def _get_readers_directory() -> str:
     """Returns the path to the disdrodb.l0.readers directory within the disdrodb package."""
-    # Current file path
-    l0_folder_path = os.path.dirname(__file__)
+    from disdrodb import __root_path__
 
-    # Readers folder path
-    reader_folder_path = os.path.join(l0_folder_path, "readers")
+    reader_folder_path = os.path.join(__root_path__, "disdrodb", "l0", "readers")
     return reader_folder_path
 
 

disdrodb/l0/readers/UK/DIVEN.py

@@ -63,8 +63,6 @@ def reader(
 
     # Define dataset sanitizer
     def ds_sanitizer_fun(ds):
-        # Drop coordinates not DISDRODB-compliants
-        pass
         # Return dataset
         return ds
 

disdrodb/l0/standards.py

@@ -58,7 +58,7 @@ def read_config_yml(sensor_name: str, filename: str) -> dict:
     """
 
     # Get config path
-    config_sensor_dir_path = get_configs_dir(sensor_name)
+    config_sensor_dir_path = get_sensor_configs_dir(sensor_name)
     fpath = os.path.join(config_sensor_dir_path, filename)
     # Check yaml file exists
     if not os.path.exists(fpath):
@@ -71,7 +71,14 @@ def read_config_yml(sensor_name: str, filename: str) -> dict:
     return d
 
 
-def get_configs_dir(sensor_name: str) -> str:
+def _get_config_dir():
+    from disdrodb import __root_path__
+
+    config_dir_path = os.path.join(__root_path__, "disdrodb", "l0", "configs")
+    return config_dir_path
+
+
+def get_sensor_configs_dir(sensor_name: str) -> str:
     """Retrieve configs directory.
 
     Parameters
@@ -89,9 +96,7 @@ def get_configs_dir(sensor_name: str) -> str:
     ValueError
         Error if the config directory does not exist.
     """
-
-    dir_path = os.path.dirname(__file__)
-    config_dir_path = os.path.join(dir_path, "configs")
+    config_dir_path = _get_config_dir()
     config_sensor_dir_path = os.path.join(config_dir_path, sensor_name)
     if not os.path.exists(config_sensor_dir_path):
         list_sensors = sorted(os.listdir(config_dir_path))
@@ -103,7 +108,7 @@ def get_configs_dir(sensor_name: str) -> str:
 ####--------------------------------------------------------------------------.
 
 
-def available_sensor_name() -> sorted:
+def available_sensor_names() -> sorted:
     """Get available names of sensors.
 
     Returns
@@ -112,8 +117,7 @@ def available_sensor_name() -> sorted:
         Sorted list of the available sensors
     """
 
-    dir_path = os.path.dirname(__file__)
-    config_dir_path = os.path.join(dir_path, "configs")
+    config_dir_path = _get_config_dir()
     return sorted(os.listdir(config_dir_path))
 
 

disdrodb/tests/test_check_configs.py → disdrodb/tests/test_l0/test_check_configs.py

@@ -13,51 +13,51 @@
     check_variable_consistency,
     check_yaml_files_exists,
 )
-from disdrodb.l0.standards import available_sensor_name
+from disdrodb.l0.standards import available_sensor_names
 
 PATH_TEST_FOLDERS_FILES = os.path.join(__root_path__, "disdrodb", "tests", "pytest_files")
 
 
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_yaml_files_exists(sensor_name):
     check_yaml_files_exists(sensor_name)
 
 
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_variable_consistency(sensor_name):
     check_variable_consistency(sensor_name)
     assert True
 
 
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_l0b_encoding(sensor_name):
     check_l0b_encoding(sensor_name)
 
 
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_l0a_encoding(sensor_name):
     check_l0b_encoding(sensor_name)
 
 
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_raw_data_format(sensor_name):
     check_raw_data_format(sensor_name)
 
 
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_cf_attributes(sensor_name):
     check_cf_attributes(sensor_name)
 
 
 # Test that all instruments config files are valid regarding the bin consistency
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_bin_consistency(sensor_name):
     # Test that no error is raised
     assert check_bin_consistency(sensor_name) is None
 
 
 # Test that all instruments config files are valid regarding the bin consistency
-@pytest.mark.parametrize("sensor_name", available_sensor_name())
+@pytest.mark.parametrize("sensor_name", available_sensor_names())
 def test_check_raw_array(sensor_name):
     # Test that no error is raised
     assert check_raw_array(sensor_name) is None

disdrodb/tests/test_check_metadata.py → disdrodb/tests/test_l0/test_check_metadata.py

@@ -1,5 +1,4 @@
 import os
-import random
 
 import pytest
 import yaml
@@ -21,6 +20,8 @@
     identify_missing_metadata_coords,
     read_yaml,
 )
+from disdrodb.l0.l0_reader import available_readers
+from disdrodb.l0.standards import available_sensor_names
 
 PATH_TEST_FOLDERS_FILES = os.path.join(__root_path__, "disdrodb", "tests", "pytest_files")
 
@@ -217,15 +218,14 @@ def test_check_archive_metadata_data_source(tmp_path):
 
 
 def test_check_archive_metadata_sensor_name(tmp_path):
-    from disdrodb.l0.standards import available_sensor_name
-
-    available_sensor_name = available_sensor_name()
+    sensor_names = available_sensor_names()
 
     # Test 1 : create a correct metadata file
     yaml_file_name = "station_1.yml"
     data_source = "data_source"
     campaign_name = "campaign_name"
-    yaml_dict = {"sensor_name": random.choice(available_sensor_name)}
+    sensor_name = sensor_names[0]
+    yaml_dict = {"sensor_name": sensor_name}
     create_fake_metadata_file(tmp_path, yaml_file_name, yaml_dict, data_source, campaign_name)
     result = check_archive_metadata_sensor_name(os.path.join(tmp_path, "DISDRODB"))
     assert result is True
@@ -262,13 +262,12 @@ def test_check_archive_metadata_station_name(tmp_path):
 
 def test_check_archive_metadata_reader(tmp_path):
     # Test 1 : create a correct metadata file
-    from disdrodb.l0.l0_reader import available_readers
-
     list_readers = available_readers()
     yaml_file_name = "station_1.yml"
     campaign_name = "campaign_name"
-    data_source = random.choice(list(list_readers.keys()))
-    reader_name = random.choice(list_readers[data_source])
+    data_source = list(list_readers.keys())[0]
+    reader_names = list_readers[data_source]
+    reader_name = reader_names[0]
     yaml_dict = {"reader": f"{data_source}/{reader_name}"}
     create_fake_metadata_file(tmp_path, yaml_file_name, yaml_dict, data_source, campaign_name)
     result = check_archive_metadata_reader(os.path.join(tmp_path, "DISDRODB"))

disdrodb/tests/test_check_standards.py → disdrodb/tests/test_l0/test_check_standards.py

@@ -1,5 +1,4 @@
 import os
-import random
 
 import numpy as np
 import pandas as pd
@@ -14,6 +13,7 @@
     check_l0a_standards,
     check_sensor_name,
 )
+from disdrodb.l0.standards import available_sensor_names, get_raw_array_nvalues
 
 RAW_DIR = os.path.join(__root_path__, "disdrodb", "tests", "pytest_files", "check_readers", "DISDRODB")
 
@@ -89,16 +89,13 @@ def test_check_raw_fields_available():
         _check_raw_fields_available(df, sensor_name)
 
     # Test case 2: All required columns present
-    from disdrodb.l0.standards import available_sensor_name, get_raw_array_nvalues
-
-    available_sensor_name = available_sensor_name()
-    sensor_name = random.choice(available_sensor_name)
-    n_bins_dict = get_raw_array_nvalues(sensor_name=sensor_name)
-    raw_vars = np.array(list(n_bins_dict.keys()))
-    dict_data = {i: [1, 2] for i in raw_vars}
-    df = pd.DataFrame.from_dict(dict_data)
-
-    assert _check_raw_fields_available(df, sensor_name) is None
+    sensor_names = available_sensor_names()
+    for sensor_name in sensor_names:
+        n_bins_dict = get_raw_array_nvalues(sensor_name=sensor_name)
+        raw_vars = np.array(list(n_bins_dict.keys()))
+        dict_data = {i: [1, 2] for i in raw_vars}
+        df = pd.DataFrame.from_dict(dict_data)
+        assert _check_raw_fields_available(df, sensor_name) is None
 
 
 def test_check_sensor_name():
@@ -114,10 +111,10 @@ def test_check_sensor_name():
 
 
 def test_check_l0a_column_names(capsys):
-    from disdrodb.l0.standards import available_sensor_name, get_sensor_variables
+    from disdrodb.l0.standards import available_sensor_names, get_sensor_variables
 
-    available_sensor_name = available_sensor_name()
-    sensor_name = random.choice(available_sensor_name)
+    sensor_names = available_sensor_names()
+    sensor_name = sensor_names[0]
 
     # Test 1 : All columns are present
     list_column_names = get_sensor_variables(sensor_name) + ["time", "latitude", "longitude"]

disdrodb/tests/test_l0a_processing.py → disdrodb/tests/test_l0/test_l0a_processing.py

@@ -38,8 +38,7 @@ def create_dummy_config_file(request: list) -> None:
     """
     content = request.param[0]
     file_name = request.param[1]
-    root_folder = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-    config_folder = os.path.join(root_folder, "l0", "configs")
+    config_folder = os.path.join(__root_path__, "disdrodb", "l0", "configs")
 
     test_folder = os.path.join(config_folder, "test")
     if not os.path.exists(test_folder):
@@ -597,8 +596,6 @@ def mock_process_raw_file(filepath, column_names, reader_kwargs, df_sanitizer_fu
             return df2
 
     # Monkey patch the function
-    from disdrodb.l0 import l0a_processing
-
     l0a_processing.process_raw_file = mock_process_raw_file
 
     # Call the function

disdrodb/tests/test_l0b_processing.py → disdrodb/tests/test_l0/test_l0b_processing.py

@@ -27,8 +27,7 @@ def create_dummy_config_file(request):
 
     for i, content in enumerate(list_content):
         file_name = list_file_name[i]
-        root_folder = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-        config_folder = os.path.join(root_folder, "l0", "configs")
+        config_folder = os.path.join(__root_path__, "disdrodb", "l0", "configs")
 
         test_folder = os.path.join(config_folder, "test")
         if not os.path.exists(test_folder):

disdrodb/utils/netcdf.py

@@ -213,7 +213,7 @@ def ensure_unique_dimension_values(list_ds: list, fpaths: str, dim: str = "time"
     sorted_list_ds, sorted_fpaths = _sort_datasets_by_dim(list_ds=list_ds, fpaths=fpaths, dim=dim)
 
     # Get the datasets dimension values array (and associated list_ds/xr.Dataset indices)
-    dim_values, list_index, ds_index = _get_dim_values_index(list_ds, dim=dim)
+    dim_values, list_index, ds_index = _get_dim_values_index(sorted_list_ds, dim=dim)
 
     # Get duplicated indices
     idx_duplicated = _get_duplicated_indices(dim_values, keep="first")
@@ -273,7 +273,7 @@ def ensure_monotonic_dimension(list_ds: list, fpaths: str, dim: str = "time", ve
     """
     # Reorder the files and filepaths by the starting dimension value (time)
     # TODO: should maybe also split by non-continuous time ...
-    sorted_list_ds, sorted_fpaths = _sort_datasets_by_dim(list_ds=list_ds, fpaths=fpaths, dim=dim)
+    list_ds, fpaths = _sort_datasets_by_dim(list_ds=list_ds, fpaths=fpaths, dim=dim)
 
     # Get the datasets dimension values array (and associated list_ds/xr.Dataset indices)
     dim_values, list_index, ds_index = _get_dim_values_index(list_ds, dim=dim)