Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[pre-commit.ci] pre-commit autoupdate #209

Closed
wants to merge 2 commits into from
Closed

[pre-commit.ci] pre-commit autoupdate #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a529099
[pre-commit.ci] pre-commit autoupdate
pre-commit-ci[bot] Mar 25, 2024
dfe61fe
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Mar 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .pre-commit-config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ repos:
- id: end-of-file-fixer

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.1.8
rev: v0.3.4
hooks:
- id: ruff
args: [--fix]

- repo: https://github.com/psf/black
rev: 23.12.0
rev: 24.3.0
hooks:
- id: black
language_version: python3
Expand Down
6 changes: 4 additions & 2 deletions disdrodb/issue/writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@

def _write_issue_docs(f):
"""Provide template for issue.yml"""
f.write("""# This file is used to store timesteps/time periods with wrong/corrupted observation.
f.write(
"""# This file is used to store timesteps/time periods with wrong/corrupted observation.
# The specified timesteps are dropped during the L0 processing.
# The time format used is the isoformat : YYYY-mm-dd HH:MM:SS.
# The 'timesteps' key enable to specify the list of timesteps to be discarded.
Expand All @@ -46,7 +47,8 @@ def _write_issue_docs(f):
# - ['2018-08-01 12:00:00', '2018-08-01 14:00:00']
# - ['2018-08-01 15:44:30', '2018-08-01 15:59:31']
# - ['2018-08-02 12:44:30', '2018-08-02 12:59:31'] \n
""")
"""
)
return None


Expand Down
106 changes: 56 additions & 50 deletions disdrodb/l0/routines.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,24 +201,26 @@ def run_disdrodb_l0a_station(
):
"""Run the L0A processing of a station calling the disdrodb_l0a_station in the terminal."""
# Define command
cmd = " ".join([
"disdrodb_run_l0a_station",
# Station arguments
data_source,
campaign_name,
station_name,
# Processing options
"--force",
str(force),
"--verbose",
str(verbose),
"--debugging_mode",
str(debugging_mode),
"--parallel",
str(parallel),
"--base_dir",
str(base_dir),
])
cmd = " ".join(
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ℹ Getting worse: Lines of Code in a Single File
The lines of code increases from 601 to 607, improve code health by reducing it to 600

Why does this problem occur?

The number of Lines of Code in a single file. More Lines of Code lowers the code health. Read more.

[
"disdrodb_run_l0a_station",
# Station arguments
data_source,
campaign_name,
station_name,
# Processing options
"--force",
str(force),
"--verbose",
str(verbose),
"--debugging_mode",
str(debugging_mode),
"--parallel",
str(parallel),
"--base_dir",
str(base_dir),
]
)
# Execute command
_execute_cmd(cmd)
return None
Expand All @@ -239,26 +241,28 @@ def run_disdrodb_l0b_station(
):
"""Run the L0B processing of a station calling disdrodb_run_l0b_station in the terminal."""
# Define command
cmd = " ".join([
"disdrodb_run_l0b_station",
# Station arguments
data_source,
campaign_name,
station_name,
# Processing options
"--force",
str(force),
"--verbose",
str(verbose),
"--debugging_mode",
str(debugging_mode),
"--parallel",
str(parallel),
"--remove_l0a",
str(remove_l0a),
"--base_dir",
str(base_dir),
])
cmd = " ".join(
[
"disdrodb_run_l0b_station",
# Station arguments
data_source,
campaign_name,
station_name,
# Processing options
"--force",
str(force),
"--verbose",
str(verbose),
"--debugging_mode",
str(debugging_mode),
"--parallel",
str(parallel),
"--remove_l0a",
str(remove_l0a),
"--base_dir",
str(base_dir),
]
)
# Execute command
_execute_cmd(cmd)
return None
Expand All @@ -276,18 +280,20 @@ def run_disdrodb_l0b_concat_station(

This function runs the ``disdrodb_run_l0b_concat_station`` script in the terminal.
"""
cmd = " ".join([
"disdrodb_run_l0b_concat_station",
data_source,
campaign_name,
station_name,
"--remove_l0b",
str(remove_l0b),
"--verbose",
str(verbose),
"--base_dir",
str(base_dir),
])
cmd = " ".join(
[
"disdrodb_run_l0b_concat_station",
data_source,
campaign_name,
station_name,
"--remove_l0b",
str(remove_l0b),
"--verbose",
str(verbose),
"--base_dir",
str(base_dir),
]
)
_execute_cmd(cmd)


Expand Down
12 changes: 7 additions & 5 deletions disdrodb/tests/test_l0/test_l0a_processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -336,11 +336,13 @@ def test_is_not_corrupted():

def test_cast_column_dtypes():
# Create a test dataframe with object columns
df = pd.DataFrame({
"time": ["2022-01-01 00:00:00", "2022-01-01 00:05:00", "2022-01-01 00:10:00"],
"station_number": "station_number",
"altitude": "8849",
})
df = pd.DataFrame(
{
"time": ["2022-01-01 00:00:00", "2022-01-01 00:05:00", "2022-01-01 00:10:00"],
"station_number": "station_number",
"altitude": "8849",
}
)
# Call the function
sensor_name = "OTT_Parsivel"
df_out = cast_column_dtypes(df, sensor_name, verbose=False)
Expand Down
86 changes: 49 additions & 37 deletions disdrodb/tests/test_l0/test_l0b_nc_processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,13 +85,15 @@ def test_replace_nan_flags(create_test_config_files):
Function that creates and removes the dummy config file.
"""
# Mock xarray Dataset
ds = xr.Dataset({
"key_1": xr.DataArray([0, 1, 2, 3, 4]),
"key_2": xr.DataArray([1, -9999, 2, 3, 89]),
"key_3": xr.DataArray([1.0, -9999.0, 2.0, 3.0, 89.0]),
"key_4": xr.DataArray([1, -9999, -8888, 2, 3]),
"key_not_in_dict": xr.DataArray([10, 20, 30, 40, 50]),
})
ds = xr.Dataset(
{
"key_1": xr.DataArray([0, 1, 2, 3, 4]),
"key_2": xr.DataArray([1, -9999, 2, 3, 89]),
"key_3": xr.DataArray([1.0, -9999.0, 2.0, 3.0, 89.0]),
"key_4": xr.DataArray([1, -9999, -8888, 2, 3]),
"key_not_in_dict": xr.DataArray([10, 20, 30, 40, 50]),
}
)

# Call the replace_nan_flags function
result_ds = replace_nan_flags(ds, sensor_name=TEST_SENSOR_NAME, verbose=True)
Expand All @@ -108,13 +110,15 @@ def test_replace_nan_flags(create_test_config_files):

@pytest.mark.parametrize("create_test_config_files", [config_dict], indirect=True)
def test_set_nan_outside_data_range(create_test_config_files):
ds = xr.Dataset({
"key_1": xr.DataArray([0, 1, 2, 3, 4]),
"key_2": xr.DataArray([9, 10, 50, 51, 30]),
"key_3": xr.DataArray([-11, -10, 0, 10, 11]),
"key_4": xr.DataArray([99, 100, 150, 200, 201]),
"key_not_in_dict": xr.DataArray([0, 1, 2, 3, 4]),
})
ds = xr.Dataset(
{
"key_1": xr.DataArray([0, 1, 2, 3, 4]),
"key_2": xr.DataArray([9, 10, 50, 51, 30]),
"key_3": xr.DataArray([-11, -10, 0, 10, 11]),
"key_4": xr.DataArray([99, 100, 150, 200, 201]),
"key_not_in_dict": xr.DataArray([0, 1, 2, 3, 4]),
}
)

result_ds = set_nan_outside_data_range(ds, TEST_SENSOR_NAME, verbose=True)

Expand All @@ -131,13 +135,15 @@ def test_set_nan_outside_data_range(create_test_config_files):

@pytest.mark.parametrize("create_test_config_files", [config_dict], indirect=True)
def test_set_nan_invalid_values(create_test_config_files):
ds = xr.Dataset({
"key_1": xr.DataArray([0, 1, 2, 3, 4]),
"key_2": xr.DataArray([9, 10, 20, 30, 40]),
"key_3": xr.DataArray([0, 0.1, 0.2, 0.3, 1.0]),
"key_4": xr.DataArray([0, 0, 0, 1, 1]),
"key_not_in_dict": xr.DataArray([0, 1, 2, 3, 4]),
})
ds = xr.Dataset(
{
"key_1": xr.DataArray([0, 1, 2, 3, 4]),
"key_2": xr.DataArray([9, 10, 20, 30, 40]),
"key_3": xr.DataArray([0, 0.1, 0.2, 0.3, 1.0]),
"key_4": xr.DataArray([0, 0, 0, 1, 1]),
"key_not_in_dict": xr.DataArray([0, 1, 2, 3, 4]),
}
)

result_ds = set_nan_invalid_values(ds, TEST_SENSOR_NAME, verbose=True)

Expand All @@ -159,12 +165,14 @@ def test_replace_custom_nan_flags():
dict_nan_flags = {"key_1": [-999], "key_2": [-9999, -8888], "key_3": [0]}

# Mock xarray Dataset
ds = xr.Dataset({
"key_1": xr.DataArray([1, -999, 2, 3, 4]),
"key_2": xr.DataArray([1, -9999, -8888, 2, 3]),
"key_3": xr.DataArray([0, 1, 0, 2, 3]),
"key_not_in_flags": xr.DataArray([10, 20, 30, 40, 50]),
})
ds = xr.Dataset(
{
"key_1": xr.DataArray([1, -999, 2, 3, 4]),
"key_2": xr.DataArray([1, -9999, -8888, 2, 3]),
"key_3": xr.DataArray([0, 1, 0, 2, 3]),
"key_not_in_flags": xr.DataArray([10, 20, 30, 40, 50]),
}
)

# Call the replace_custom_nan_flags function
result_ds = replace_custom_nan_flags(ds, dict_nan_flags=dict_nan_flags)
Expand Down Expand Up @@ -253,12 +261,14 @@ def test_rename_dataset():
@pytest.mark.parametrize("create_test_config_files", [config_dict], indirect=True)
def test_subset_dataset(create_test_config_files):
# Define xarray Dataset with extra variables (assumed to be renamed)
ds = xr.Dataset({
"var1": xr.DataArray([1, 2, 3]),
"var2": xr.DataArray([4, 5, 6]),
"var3": xr.DataArray([7, 8, 9]),
"var_not_needed": xr.DataArray([10, 11, 12]),
})
ds = xr.Dataset(
{
"var1": xr.DataArray([1, 2, 3]),
"var2": xr.DataArray([4, 5, 6]),
"var3": xr.DataArray([7, 8, 9]),
"var_not_needed": xr.DataArray([10, 11, 12]),
}
)

# Define dict_names mapping
# - Key are used to rename (the values are used for subsetting)
Expand All @@ -277,10 +287,12 @@ def test_subset_dataset(create_test_config_files):
@pytest.mark.parametrize("create_test_config_files", [config_dict], indirect=True)
def test_get_missing_variables(create_test_config_files):
# Define xarray Dataset with some variables (assumed to be renamed and subsetted)
ds = xr.Dataset({
"var1": xr.DataArray([1, 2, 3]),
"var2": xr.DataArray([4, 5, 6]),
})
ds = xr.Dataset(
{
"var1": xr.DataArray([1, 2, 3]),
"var2": xr.DataArray([4, 5, 6]),
}
)

# Define dict_names mapping
# - Key are used to rename (the values are used for subsetting)
Expand Down
44 changes: 25 additions & 19 deletions disdrodb/tests/test_l0/test_l0b_processing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,15 +96,17 @@ def define_test_dummy_configs():
@pytest.mark.parametrize("create_test_config_files", [define_test_dummy_configs()], indirect=True)
def test_create_l0b_from_l0a(create_test_config_files):
# Create a sample DataFrame
df = pd.DataFrame({
"time": pd.date_range("2022-01-01", periods=10, freq="H"),
"raw_drop_concentration": np.random.rand(10),
"raw_drop_average_velocity": np.random.rand(10),
"raw_drop_number": np.random.rand(10),
"latitude": np.random.rand(10),
"longitude": np.random.rand(10),
"altitude": np.random.rand(10),
})
df = pd.DataFrame(
{
"time": pd.date_range("2022-01-01", periods=10, freq="H"),
"raw_drop_concentration": np.random.rand(10),
"raw_drop_average_velocity": np.random.rand(10),
"raw_drop_number": np.random.rand(10),
"latitude": np.random.rand(10),
"longitude": np.random.rand(10),
"altitude": np.random.rand(10),
}
)
# Create a sample attrs dictionary
attrs = {
"sensor_name": "test",
Expand Down Expand Up @@ -152,11 +154,13 @@ def test_create_l0b_from_l0a(create_test_config_files):

def test_add_dataset_crs_coords():
# Create example dataset
ds = xr.Dataset({
"var1": xr.DataArray([1, 2, 3], dims="time"),
"lat": xr.DataArray([0, 1, 2], dims="time"),
"lon": xr.DataArray([0, 1, 2], dims="time"),
})
ds = xr.Dataset(
{
"var1": xr.DataArray([1, 2, 3], dims="time"),
"lat": xr.DataArray([0, 1, 2], dims="time"),
"lon": xr.DataArray([0, 1, 2], dims="time"),
}
)

# Call the function and check the output
ds_out = add_dataset_crs_coords(ds)
Expand All @@ -182,11 +186,13 @@ def test_set_attrs_dict():

def test__set_coordinate_attributes():
# Create example dataset
ds = xr.Dataset({
"var1": xr.DataArray([1, 2, 3], dims="time"),
"lat": xr.DataArray([0, 1, 2], dims="time"),
"lon": xr.DataArray([0, 1, 2], dims="time"),
})
ds = xr.Dataset(
{
"var1": xr.DataArray([1, 2, 3], dims="time"),
"lat": xr.DataArray([0, 1, 2], dims="time"),
"lon": xr.DataArray([0, 1, 2], dims="time"),
}
)
ds.lat.attrs["units"] = "degrees_north"
ds.lon.attrs["units"] = "degrees_east"

Expand Down
Loading
Loading