Add unit tests

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit = */tests/*, slkspec/_version.py
+concurrency = multiprocessing
+
+[report]
+exclude_lines =
+    pragma: no cover

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/.github/dependabot/"
+    schedule:
+      interval: "daily"

.github/dependabot/constraints.txt

@@ -0,0 +1,2 @@
+fsspec==2022.11.0
+pyslk==1.4.1

.github/workflows/lint_job.yml

@@ -0,0 +1,23 @@
+name: Linter
+run-name: ${{ github.actor }} is linting the code
+
+on: [push, pull_request]
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    -
+      name: Ceckout
+      uses: actions/checkout@v3
+    -
+      name: Set up Python 3.11
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.11"
+    -
+      name: Install packages
+      run: |
+        python3 -m pip install -e .[tests]
+    -
+      name: Linting
+      run: make lint

.github/workflows/tests.yml

@@ -0,0 +1,45 @@
+name: Tests
+run-name: ${{ github.actor }} is testing the code
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+
+jobs:
+  tests:
+
+    runs-on: ubuntu-latest
+    strategy:
+        max-parallel: 5
+        matrix:
+          python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version:  ${{ matrix.python-version }}
+
+    - name: Add conda to system path
+      run: |
+        # $CONDA is an environment variable pointing to the root of the miniconda directory
+        echo $CONDA/bin >> $GITHUB_PATH
+
+    - name: Creating conda environment for ${{ matrix.python-version }}
+      run: |
+        conda create -n test -y python=${{matrix.python-version}} pip make
+
+    - name: Install dependencies
+      run: |
+        conda run -n test python3 -m pip install -e .[tests]
+
+    - name: Test with pytest
+      run: |
+        conda run -n test make test_coverage
+
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1

.gitignore

@@ -25,3 +25,8 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
+
+# Coverage reports
+.coverage
+coverage.xml
+report.xml

Makefile

@@ -0,0 +1,24 @@
+# makefile used for testing
+#
+#
+all: install test
+
+.PHONY: docs
+install:
+	python3 -m pip install -e .[tests]
+
+test:
+	python3 -m pytest -vv $(PWD)/slkspec/tests
+
+test_coverage:
+	python3 -m pytest -vv \
+	    --cov=$(PWD)/slkspec --cov-report html:coverage_report \
+		--cov-report=xml --junitxml report.xml
+	rm -rf '='
+	python3 -m coverage report
+
+
+lint:
+	mypy --install-types --non-interactive
+	black --check -t py310 .
+	flake8 slkspec --count --max-complexity=10 --max-line-length=88 --statistics --doctests

setup.py

@@ -30,6 +30,15 @@
         "tests": [
             "mypy",
             "black",
+            "flake8",
+            "mock",
+            "pandas",
+            "pytest",
+            "pytest-env",
+            "pytest-cov",
+            "testpath",
+            "flake8",
+            "xarray",
         ],
         "preffs": [
             "preffs @ git+https://github.com/observingClouds/preffs.git@slkspec_patch",

slkspec/__init__.py

@@ -7,4 +7,4 @@
 
 register_implementation(SLKFileSystem.protocol, SLKFileSystem)
 
-__all__ = ["__version__", "SLKFileSystem"]
+__all__ = ["__version__", "SLKFileSystem", "logger"]

slkspec/core.py

@@ -33,7 +33,9 @@
 
 MAX_RETRIES = 2
 FileQueue: Queue[Tuple[str, str]] = Queue(maxsize=-1)
-FileInfo = TypedDict("FileInfo", {"name": str, "size": Literal[None], "type": str})
+FileInfo = TypedDict(
+    "FileInfo", {"name": str, "size": Literal[None], "type": str}
+)
 _retrieval_lock = threading.Lock()
 
 
@@ -73,7 +75,8 @@ class SLKFile(io.IOBase):
         import ffspec
         import xarray as xr
 
-        url = fsspec.open("slk:////arch/bb1203/data.nc", slk_cache="/scratch/b/b12346").open()
+        url = fsspec.open("slk:////arch/bb1203/data.nc",
+                          slk_cache="/scratch/b/b12346").open()
         dset = xr.open_dataset(url)
 
     """
@@ -113,6 +116,7 @@ def __init__(
         self.encoding = kwargs.get("encoding")
         self.write_through = False
         self._file_queue = _file_queue
+        print(self._file)
         with _lock:
             if not Path(self._file).exists() or override:
                 self._file_queue.put((self._url, str(Path(self._file).parent)))
@@ -136,7 +140,9 @@ def _retrieve_items(self, retrieve_files: list[tuple[str, str]]) -> None:
         for inp_file, out_dir in retrieve_files:
             retrieval_requests[Path(out_dir)].append(inp_file)
         for output_dir, inp_files in retrieval_requests.items():
-            output_dir.mkdir(parents=True, exist_ok=True, mode=self.file_permissions)
+            output_dir.mkdir(
+                parents=True, exist_ok=True, mode=self.file_permissions
+            )
             logger.critical("Creating slk query for %i files", len(inp_files))
             search_str = pyslk.slk_search(pyslk.slk_gen_file_query(inp_files))
             search_id_re = re.search("Search ID: [0-9]*", search_str)
@@ -269,16 +275,16 @@ def __init__(
         )
         slk_cache = slk_cache or os.environ.get("SLK_CACHE")
         if not slk_cache:
-            local_cache = f"/scratch/{getuser()[0]}/{getuser()}"
+            slk_cache = f"/scratch/{getuser()[0]}/{getuser()}"
             warnings.warn(
                 "The slk_cache variable nor the SLK_CACHE environment"
                 "variable wasn't set. Falling back to default "
-                f"{local_cache}",
+                f"{slk_cache}",
                 UserWarning,
                 stacklevel=2,
             )
         self.touch = touch
-        self.slk_cache = Path(local_cache)
+        self.slk_cache = Path(slk_cache)
         self.override = override
         self.file_permissions = file_permissions
 

slkspec/tests/conftest.py

@@ -0,0 +1,122 @@
+"""pytest definitions to run the unittests."""
+from __future__ import annotations
+from pathlib import Path
+from subprocess import run, PIPE
+from tempfile import TemporaryDirectory
+from typing import Generator
+import shutil
+
+import pytest
+import mock
+import numpy as np
+import pandas as pd
+import xarray as xr
+
+
+class SLKMock:
+    """A mock that emulates what pyslk is doing."""
+
+    def __init__(self, _cache: dict[int, list[str]] = {}) -> None:
+
+        self._cache = _cache
+
+    def slk_list(self, inp_path: str) -> str:
+        """Mock the slk_list method."""
+        res = (
+            run(["ls", "-l", inp_path], stdout=PIPE, stderr=PIPE)
+            .stdout.decode()
+            .split("\n")
+        )
+        return "\n".join(res[1:] + [res[0]])
+
+    def slk_search(self, inp_f: list[str]) -> str:
+        """Mock slk_search."""
+        if not inp_f:
+            return ""
+        hash_value = abs(hash(",".join(inp_f)))
+        self._cache[hash_value] = inp_f
+        return f"Search ID: {hash_value} foo"
+
+    def slk_gen_file_query(self, inp_files: list[str]) -> list[str]:
+        """Mock slk_gen_file_qeury."""
+        return [f for f in inp_files if Path(f).exists()]
+
+    def slk_retrieve(self, search_id: int, out_dir: str) -> None:
+        """Mock slk_retrieve."""
+        for inp_file in map(Path, self._cache[search_id]):
+            shutil.copy(inp_file, Path(out_dir) / inp_file.name)
+
+
+def create_data(variable_name: str, size: int) -> xr.Dataset:
+    """Create a netcdf dataset."""
+    coords: dict[str, np.ndarray] = {}
+    coords["x"] = np.linspace(-10, -5, size)
+    coords["y"] = np.linspace(120, 125, size)
+    lat, lon = np.meshgrid(coords["y"], coords["x"])
+    lon_vec = xr.DataArray(lon, name="Lg", coords=coords, dims=("y", "x"))
+    lat_vec = xr.DataArray(lat, name="Lt", coords=coords, dims=("y", "x"))
+    coords["time"] = np.array(
+        [
+            np.datetime64("2020-01-01T00:00"),
+            np.datetime64("2020-01-01T12:00"),
+            np.datetime64("2020-01-02T00:00"),
+            np.datetime64("2020-01-02T12:00"),
+        ]
+    )
+    dims = (4, size, size)
+    data_array = np.empty(dims)
+    for time in range(dims[0]):
+        data_array[time] = np.zeros((size, size))
+    dset = xr.DataArray(
+        data_array,
+        dims=("time", "y", "x"),
+        coords=coords,
+        name=variable_name,
+    )
+    data_array = np.zeros(dims)
+    return xr.Dataset({variable_name: dset, "Lt": lon_vec, "Lg": lat_vec}).set_coords(
+        list(coords.keys())
+    )
+
+
+@pytest.fixture(scope="session")
+def patch_dir() -> Generator[Path, None, None]:
+    with TemporaryDirectory() as temp_dir:
+        with mock.patch("slkspec.core.pyslk", SLKMock()):
+            yield Path(temp_dir)
+
+
+@pytest.fixture(scope="session")
+def save_dir() -> Generator[Path, None, None]:
+    """Crate a temporary directory."""
+    with TemporaryDirectory() as td:
+        yield Path(td)
+
+
+@pytest.fixture(scope="session")
+def data() -> Generator[xr.Dataset, None, None]:
+    """Define a simple dataset with a blob in the middle."""
+    dset = create_data("precip", 100)
+    yield dset
+
+
+@pytest.fixture(scope="session")
+def netcdf_files(
+    data: xr.Dataset,
+) -> Generator[Path, None, None]:
+    """Save data with a blob to file."""
+
+    with TemporaryDirectory() as td:
+        for time in (data.time[:2], data.time[2:]):
+            time1 = pd.Timestamp(time.values[0]).strftime("%Y%m%d%H%M")
+            time2 = pd.Timestamp(time.values[1]).strftime("%Y%m%d%H%M")
+            out_file = (
+                Path(td)
+                / "the_project"
+                / "test1"
+                / "precip"
+                / f"precip_{time1}-{time2}.nc"
+            )
+            out_file.parent.mkdir(exist_ok=True, parents=True)
+            data.sel(time=time).to_netcdf(out_file)
+        yield Path(td)