RoyalHaskoningDHV · abontsema · Aug 25, 2022 · Aug 2, 2022 · Aug 2, 2022 · Aug 2, 2022
diff --git a/.gitignore b/.gitignore
@@ -64,7 +64,6 @@ wheels/
 MANIFEST
 .idea
 
-
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -145,4 +144,4 @@ venv.bak/
 
 # Exceptions
 !docs/requirements.txt
-!data/rainbow_beach.parquet
+!sam/datasets/data/*.csv
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -12,11 +12,29 @@ Version X.Y.Z stands for:
 ## Version 3.1.0
 
 ### New features
+
 - New class `sam.models.LassoTimeseriesRegressor` to create a Lasso regression model for time series data incl. quantile predictions.
+- New class `sam.preprocessing.ClipTransformer` to clip input values to the range from the train set, making models more robust again
+- New abstract base class `sam.validation.BaseValidator` for all validators.
+- Renamed `sam.validation.RemoveFlatlines` to `sam.validation.FlatlineValidator`. `sam.validation.RemoveFlatlines` is still available, but removed in future versions.
+- Renamed `sam.validation.RemoveExtremeValues` to `sam.validation.MADValidator`. `sam.validation.RemoveExtremeValues` is still available, but removed in future versions.
+- New class `sam.validation.OutsideRangeValidator` for checking / removing data outside of a range. 
+- New function `datetime_train_test_split` to split pandas dataframes and series based on a datetime.
+- New `sam.datasets` module containing functions for loading read-to-use datasets: `sam.datasets.load_rainbow_beach` and `sam.datasets.load_sewage_data`.
+st outliers.
+
+## Version 3.0.3
+
+### New features
+- Data collection function `sam.data_sources.read_regenradar` does now accept `batch_size` and collects data in batches to avoid timeouts.
+
+## Version 3.0.2
+
+No changes, version bump only.
 
 ## Version 3.0.1
 
-No changes, bumped number for release.
+No changes, version bump only.
 
 ## Version 3.0.0
 

diff --git a/data/rainbow_beach.parquet b/data/rainbow_beach.parquet
diff --git a/docs/source/datasets.rst b/docs/source/datasets.rst
@@ -0,0 +1,18 @@
+.. _datasets:
+
+=============
+Data Sets
+=============
+
+This is the documentation for available datasets.
+
+Rainbow Beach
+-------------
+.. autofunction:: sam.datasets.load_rainbow_beach
+
+
+Sewage data
+-----------
+.. autofunction:: sam.datasets.load_sewage_data
+
+
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -16,6 +16,7 @@ Welcome to SAM's documentation!
    data
    examples
    data_sources
+   datasets
    preprocessing
    exploration
    feature_engineering

diff --git a/docs/source/preprocessing.rst b/docs/source/preprocessing.rst
@@ -6,6 +6,13 @@ Preprocessing
 
 This is the documentation for preprocessing functions.
 
+Clipping data
+-------------
+.. autoclass:: sam.preprocessing.ClipTransformer
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 Normalize timestamps
 --------------------
 .. warning::

diff --git a/docs/source/validation.rst b/docs/source/validation.rst
@@ -6,9 +6,23 @@ Data Validation
 
 This is the documentation for the validation functions.
 
+Base Validation class
+---------------------
+.. autoclass:: sam.validation.BaseValidator
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+Detect Outside Range
+--------------------
+.. autoclass:: sam.validation.OutsideRangeValidator
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 Detect Extreme Values
 ---------------------------
-.. autoclass:: sam.validation.RemoveExtremeValues
+.. autoclass:: sam.validation.MADValidator
     :members:
     :undoc-members:
     :show-inheritance:
@@ -23,7 +37,7 @@ Testset image:
 
 Detect Flatlines
 ---------------------------
-.. autoclass:: sam.validation.RemoveFlatlines
+.. autoclass:: sam.validation.FlatlineValidator
     :members:
     :undoc-members:
     :show-inheritance:

diff --git a/examples/feature_engineering.ipynb b/examples/feature_engineering.ipynb
@@ -130,10 +130,9 @@
    ],
    "source": [
     "import pandas as pd\n",
+    "from sam.datasets import load_rainbow_beach\n",
     "\n",
-    "data = pd.read_parquet('../data/rainbow_beach.parquet')\n",
-    "\n",
-    "data.head()"
+    "data = load_rainbow_beach()"
    ]
   },
   {

diff --git a/examples/lasso.ipynb b/examples/lasso.ipynb
diff --git a/examples/mlp.ipynb b/examples/mlp.ipynb
diff --git a/pyproject.toml b/pyproject.toml
@@ -6,6 +6,7 @@ build-backend = "setuptools.build_meta"
 packages = [
     "sam",
     "sam.data_sources",
+    "sam.datasets",
     "sam.exploration",
     "sam.feature_engineering",
     "sam.logging_functions",

diff --git a/sam/__init__.py b/sam/__init__.py
@@ -25,6 +25,7 @@
 
 __all__ = [
     "data_sources",
+    "datasets",
     "exploration",
     "feature_engineering",
     "logging_functions",

diff --git a/sam/datasets/__init__.py b/sam/datasets/__init__.py
@@ -0,0 +1,7 @@
+from .datasets import load_rainbow_beach, load_sewage_data
+
+
+__all__ = [
+    "load_rainbow_beach",
+    "load_sewage_data",
+]