Lint: Utils module

.github/workflows/pylint.yml

@@ -28,3 +28,7 @@ jobs:
       - name: Run Pylint on mhkit/power/
         run: |
           pylint mhkit/power/
+
+      - name: Run Pylint on mhkit/utils/
+        run: |
+          pylint mhkit/utils/

mhkit/river/io/usgs.py

@@ -120,7 +120,10 @@ def request_usgs_data(
 
     # Use handle_caching to manage cache
     cached_data, metadata, cache_filepath = handle_caching(
-        hash_params, cache_dir, write_json, clear_cache
+        hash_params,
+        cache_dir,
+        cache_content={"data": None, "metadata": None, "write_json": write_json},
+        clear_cache_file=clear_cache,
     )
 
     if cached_data is not None:
@@ -165,7 +168,12 @@ def request_usgs_data(
 
     # After making the API request and processing the response, write the
     #  response to a cache file
-    handle_caching(hash_params, cache_dir, data=data, clear_cache_file=clear_cache)
+    handle_caching(
+        hash_params,
+        cache_dir,
+        cache_content={"data": data, "metadata": None, "write_json": None},
+        clear_cache_file=clear_cache,
+    )
 
     if write_json:
         shutil.copy(cache_filepath, write_json)

mhkit/tests/utils/test_cache.py

@@ -93,7 +93,11 @@ def test_handle_caching_creates_cache(self):
         Asserts:
         - The cache file is successfully created at the expected file path.
         """
-        handle_caching(self.hash_params, self.cache_dir, data=self.data)
+        handle_caching(
+            self.hash_params,
+            self.cache_dir,
+            cache_content={"data": self.data, "metadata": None, "write_json": None},
+        )
 
         cache_filename = (
             hashlib.md5(self.hash_params.encode("utf-8")).hexdigest() + ".json"
@@ -114,8 +118,18 @@ def test_handle_caching_retrieves_data(self):
         Asserts:
         - The retrieved data matches the original sample DataFrame.
         """
-        handle_caching(self.hash_params, self.cache_dir, data=self.data)
-        retrieved_data, _, _ = handle_caching(self.hash_params, self.cache_dir)
+        handle_caching(
+            self.hash_params,
+            self.cache_dir,
+            cache_content={"data": self.data, "metadata": None, "write_json": None},
+        )
+
+        retrieved_data, _, _ = handle_caching(
+            self.hash_params,
+            self.cache_dir,
+            cache_content={"data": None, "metadata": None, "write_json": None},
+        )
+
         pd.testing.assert_frame_equal(self.data, retrieved_data, check_freq=False)
 
     def test_handle_caching_cdip_file_extension(self):
@@ -132,7 +146,11 @@ def test_handle_caching_cdip_file_extension(self):
         - The cache file with a ".pkl" extension is successfully created at the expected file path.
         """
         cache_dir = os.path.join(self.cache_dir, "cdip")
-        handle_caching(self.hash_params, cache_dir, data=self.data)
+        handle_caching(
+            self.hash_params,
+            cache_dir,
+            cache_content={"data": self.data, "metadata": None, "write_json": None},
+        )
 
         cache_filename = (
             hashlib.md5(self.hash_params.encode("utf-8")).hexdigest() + ".pkl"

mhkit/tests/utils/test_utils.py

@@ -29,10 +29,10 @@ def test_get_statistics(self):
         # load in file
         df = self.data["loads"]
         df.Timestamp = pd.to_datetime(df.Timestamp)
-        df.set_index("Timestamp", inplace=True)
+        test_df = df.set_index("Timestamp")
         # run function
         means, maxs, mins, stdevs = utils.get_statistics(
-            df,
+            test_df,
             self.freq,
             period=self.period,
             vector_channels=["WD_Nacelle", "WD_NacelleMod"],
@@ -57,6 +57,37 @@ def test_get_statistics(self):
         time = pd.to_datetime(string_time)
         self.assertTrue(means.index[0] == time)
 
+    def test__calculate_statistics(self):
+        # load in file
+        df = self.data["loads"]
+        df.Timestamp = pd.to_datetime(df.Timestamp)
+        test_df = df.set_index("Timestamp")
+
+        # Select a specific data chunk (the first 10 rows)
+        datachunk = test_df.iloc[:10]
+
+        # Run the calculate_statistics function
+        stats = utils._calculate_statistics(
+            datachunk, vector_channels=["WD_Nacelle", "WD_NacelleMod"]
+        )
+
+        means = stats["means"]
+        maxs = stats["maxs"]
+        mins = stats["mins"]
+        stdevs = stats["stdevs"]
+
+        # check statistics for a specific column ('uWind_80m')
+        self.assertAlmostEqual(means["uWind_80m"], 3.226, 2)  # mean
+        self.assertAlmostEqual(maxs["uWind_80m"], 3.234, 2)  # max
+        self.assertAlmostEqual(mins["uWind_80m"], 3.221, 2)  # min
+        self.assertAlmostEqual(stdevs["uWind_80m"], 0.005049, 2)  # standard deviation
+
+        # check vector statistics for 'WD_Nacelle'
+        self.assertAlmostEqual(means["WD_Nacelle"], 157.302, 2)  # vector mean
+        self.assertAlmostEqual(
+            stdevs["WD_Nacelle"], 0.000, 2
+        )  # vector standard deviation
+
     def test_vector_statistics(self):
         # load in vector variable
         df = self.data["loads"]

mhkit/tidal/io/noaa.py

@@ -124,7 +124,10 @@ def request_noaa_data(
 
     # Use handle_caching to manage cache
     cached_data, cached_metadata, cache_filepath = handle_caching(
-        hash_params, cache_dir, write_json=write_json, clear_cache_file=clear_cache
+        hash_params,
+        cache_dir,
+        cache_content={"data": None, "metadata": None, "write_json": write_json},
+        clear_cache_file=clear_cache,
     )
 
     if cached_data is not None:
@@ -205,8 +208,7 @@ def request_noaa_data(
         handle_caching(
             hash_params,
             cache_dir,
-            data=data,
-            metadata=metadata,
+            cache_content={"data": data, "metadata": metadata, "write_json": None},
             clear_cache_file=clear_cache,
         )
 

mhkit/utils/__init__.py

@@ -1,5 +1,12 @@
+"""
+This module initializes and imports the essential utility functions for data 
+conversion, statistical analysis, caching, and event detection for the 
+MHKiT library.
+"""
+
 from .time_utils import matlab_to_datetime, excel_to_datetime, index_to_datetime
 from .stat_utils import (
+    _calculate_statistics,
     get_statistics,
     vector_statistics,
     unwrap_vector,
@@ -15,4 +22,5 @@
     convert_nested_dict_and_pandas,
 )
 
+# pylint: disable=invalid-name
 _matlab = False  # Private variable indicating if mhkit is run through matlab

mhkit/utils/cache.py

@@ -42,7 +42,6 @@
 import hashlib
 import json
 import os
-import re
 import shutil
 import pickle
 import pandas as pd
@@ -51,146 +50,101 @@
 def handle_caching(
     hash_params,
     cache_dir,
-    data=None,
-    metadata=None,
-    write_json=None,
+    cache_content=None,
     clear_cache_file=False,
 ):
     """
     Handles caching of data to avoid redundant network requests or
     computations.
-
-    The function checks if a cache file exists for the given parameters.
-    If it does, the function will load data from the cache file, unless
-    the `clear_cache_file` parameter is set to `True`, in which case the
-    cache file is cleared. If the cache file does not exist and the
-    `data` parameter is not `None`, the function will store the
-    provided data in a cache file.
-
-    Parameters
-    ----------
-    hash_params : str
-        The parameters to be hashed and used as the filename for the cache file.
-    cache_dir : str
-        The directory where the cache files are stored.
-    data : pandas DataFrame or None
-        The data to be stored in the cache file. If `None`, the function
-        will attempt to load data from the cache file.
-    metadata : dict or None
-        Metadata associated with the data. This will be stored in the
-        cache file along with the data.
-    write_json : str or None
-        If specified, the cache file will be copied to a file with this name.
-    clear_cache_file : bool
-        If `True`, the cache file for the given parameters will be cleared.
-
-    Returns
-    -------
-    data : pandas DataFrame or None
-        The data loaded from the cache file. If data was provided as a
-        parameter, the same data will be returned. If the cache file
-        does not exist and no data was provided, `None` will be returned.
-    metadata : dict or None
-        The metadata loaded from the cache file. If metadata was provided
-        as a parameter, the same metadata will be returned. If the cache
-        file does not exist and no metadata was provided, `None` will be
-        returned.
-    cache_filepath : str
-        The path to the cache file.
     """
 
-    # Check if 'cdip' is in cache_dir, then use .pkl instead of .json
-    file_extension = (
-        ".pkl"
-        if "cdip" in cache_dir or "hindcast" in cache_dir or "ndbc" in cache_dir
-        else ".json"
-    )
-
-    # Make cache directory if it doesn't exist
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
-
-    # Create a unique filename based on the function parameters
-    cache_filename = (
-        hashlib.md5(hash_params.encode("utf-8")).hexdigest() + file_extension
-    )
-    cache_filepath = os.path.join(cache_dir, cache_filename)
-
-    # If clear_cache_file is True, remove the cache file for this request
-    if clear_cache_file and os.path.isfile(cache_filepath):
-        os.remove(cache_filepath)
-        print(f"Cleared cache for {cache_filepath}")
-
-    # If a cached file exists, load and return the data from the file
-    if os.path.isfile(cache_filepath) and data is None:
+    data = None
+    metadata = None
+
+    def _generate_cache_filepath():
+        """Generates the cache file path based on the hashed parameters."""
+        file_extension = (
+            ".pkl"
+            if "cdip" in cache_dir or "hindcast" in cache_dir or "ndbc" in cache_dir
+            else ".json"
+        )
+        cache_filename = (
+            hashlib.md5(hash_params.encode("utf-8")).hexdigest() + file_extension
+        )
+        return os.path.join(cache_dir, cache_filename), file_extension
+
+    def _clear_cache(cache_filepath):
+        """Clear the cache file if requested."""
+        if clear_cache_file and os.path.isfile(cache_filepath):
+            os.remove(cache_filepath)
+            print(f"Cleared cache for {cache_filepath}")
+
+    def _load_cache(file_extension, cache_filepath):
+        """Load data from the cache file based on its extension."""
+        nonlocal data, metadata  # Specify that these are outer variables
         if file_extension == ".json":
             with open(cache_filepath, encoding="utf-8") as f:
-                jsonData = json.load(f)
-
-            # Extract metadata if it exists
-            if "metadata" in jsonData:
-                metadata = jsonData.pop("metadata", None)
-
-            # Check if index is datetime formatted
-            if all(
-                re.match(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}", str(dt))
-                for dt in jsonData["index"]
-            ):
-                data = pd.DataFrame(
-                    jsonData["data"],
-                    index=pd.to_datetime(jsonData["index"]),
-                    columns=jsonData["columns"],
-                )
-            else:
-                data = pd.DataFrame(
-                    jsonData["data"],
-                    index=jsonData["index"],
-                    columns=jsonData["columns"],
-                )
+                json_data = json.load(f)
+
+            metadata = json_data.pop("metadata", None)
 
-            # Convert the rest to DataFrame
             data = pd.DataFrame(
-                jsonData["data"],
-                index=pd.to_datetime(jsonData["index"]),
-                columns=jsonData["columns"],
+                json_data["data"],
+                index=pd.to_datetime(json_data["index"]),
+                columns=json_data["columns"],
             )
-
         elif file_extension == ".pkl":
             with open(cache_filepath, "rb") as f:
                 data, metadata = pickle.load(f)
 
-        if write_json:
-            shutil.copy(cache_filepath, write_json)
+        return data, metadata
 
-        return data, metadata, cache_filepath
-
-    # If a cached file does not exist and data is provided,
-    # store the data in a cache file
-    elif data is not None:
+    def _write_cache(data, metadata, file_extension, cache_filepath):
+        """Store data in the cache file based on the extension."""
         if file_extension == ".json":
-            # Convert DataFrame to python dict
-            pyData = data.to_dict(orient="split")
-            # Add metadata to pyData
-            pyData["metadata"] = metadata
-            # Check if index is datetime indexed
+            py_data = data.to_dict(orient="split")
+            py_data["metadata"] = metadata
             if isinstance(data.index, pd.DatetimeIndex):
-                pyData["index"] = [
-                    dt.strftime("%Y-%m-%d %H:%M:%S") for dt in pyData["index"]
+                py_data["index"] = [
+                    dt.strftime("%Y-%m-%d %H:%M:%S") for dt in py_data["index"]
                 ]
             else:
-                pyData["index"] = list(data.index)
+                py_data["index"] = list(data.index)
             with open(cache_filepath, "w", encoding="utf-8") as f:
-                json.dump(pyData, f)
-
+                json.dump(py_data, f)
         elif file_extension == ".pkl":
             with open(cache_filepath, "wb") as f:
                 pickle.dump((data, metadata), f)
 
-        if write_json:
-            shutil.copy(cache_filepath, write_json)
+    # Create the cache directory if it doesn't exist
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+
+    # Generate cache filepath and extension
+    cache_filepath, file_extension = _generate_cache_filepath()
+
+    # Clear cache if requested
+    _clear_cache(cache_filepath)
+
+    # If cache file exists and cache_content["data"] is None, load from cache
+    if os.path.isfile(cache_filepath) and (
+        cache_content is None or cache_content["data"] is None
+    ):
+        return _load_cache(file_extension, cache_filepath) + (cache_filepath,)
+
+    # Store data in cache if provided
+    if cache_content and cache_content["data"] is not None:
+        _write_cache(
+            cache_content["data"],
+            cache_content["metadata"],
+            file_extension,
+            cache_filepath,
+        )
+        if cache_content["write_json"]:
+            shutil.copy(cache_filepath, cache_content["write_json"])
+
+        return cache_content["data"], cache_content["metadata"], cache_filepath
 
-        return data, metadata, cache_filepath
-    # If data is not provided and the cache file doesn't exist, return cache_filepath
     return None, None, cache_filepath