Move on refactoring

argopy/data_fetchers/gdac_index.py

@@ -9,7 +9,6 @@
 from abc import ABC, abstractmethod
 import warnings
 import logging
-import importlib
 
 from ..utils.format import format_oneline
 from ..options import OPTIONS, check_gdac_option
@@ -19,15 +18,6 @@
 
 log = logging.getLogger("argopy.gdac.index")
 
-# has_pyarrow = importlib.util.find_spec('pyarrow') is not None
-# if has_pyarrow:
-#     from argopy.stores.argo_index_pa import indexstore_pyarrow as indexstore
-#     log.debug("Using pyarrow indexstore")
-# else:
-#     from argopy.stores.argo_index_pd import indexstore_pandas as indexstore
-#     # warnings.warn("Consider installing pyarrow in order to improve performances when fetching GDAC data")
-#     log.debug("Using pandas indexstore")
-
 access_points = ["wmo", "box"]
 exit_formats = ["xarray"]
 dataset_ids = ["phy", "bgc"]  # First is default

argopy/stores/__init__.py

@@ -1,6 +1,4 @@
-from .filesystems import argo_store_proto
-from .filesystems import has_distributed, distributed  # noqa: F401
-
+from .spec import ArgoStoreProto
 from .implementations.local import filestore
 from .implementations.memory import memorystore
 from .implementations.http import httpstore
@@ -10,9 +8,10 @@
 from .implementations.gdac import gdacfs
 
 from .index.argo_index import ArgoIndex
-from .index.argo_index_pa import indexstore_pyarrow as indexstore_pa
-from .index.argo_index_pd import indexstore_pandas as indexstore_pd
+from .index.implementations.index_pyarrow import indexstore as indexstore_pa
+from .index.implementations.index_pandas import indexstore as indexstore_pd
 
+from .filesystems import has_distributed, distributed  # noqa: F401
 from .kerchunker import ArgoKerchunker
 
 

argopy/stores/filesystems.py

@@ -20,7 +20,7 @@
 try:
     from tqdm import tqdm
 except ModuleNotFoundError:
-    log.debug("argopy needs tqdm installed to display progress bars")
+    log.debug("argopy needs 'tqdm' to display progress bars")
 
     def tqdm(fct, **kw):
         return fct
@@ -31,7 +31,7 @@ def tqdm(fct, **kw):
 
     has_distributed = True
 except ModuleNotFoundError:
-    log.debug("argopy needs distributed to use Dask cluster/client")
+    log.debug("argopy needs 'distributed' to use Dask cluster/client")
     has_distributed = False
     distributed = None
 

argopy/stores/implementations/gdac.py

@@ -1,3 +1,12 @@
+from typing import Union
+from pathlib import Path
+from fsspec.core import split_protocol
+from urllib.parse import urlparse
+from socket import gaierror
+
+from ...options import OPTIONS
+from ...errors import GdacPathError
+from .. import filestore, httpstore, ftpstore, s3store
 
 
 class gdacfs:
@@ -11,7 +20,7 @@ class gdacfs:
 
     Returns
     -------
-    A file system based on :class:`argopy.stores.argo_store_proto`
+    A file system based on :class:`argopy.stores.ArgoStoreProto`
 
     Examples
     --------

argopy/stores/implementations/http.py

@@ -23,15 +23,15 @@
     fill_variables_not_in_all_datasets,
 )
 from ...utils.monitored_threadpool import MyThreadPoolExecutor as MyExecutor
-from ..filesystems import argo_store_proto
+from ..spec import ArgoStoreProto
 from ..filesystems import has_distributed, distributed
 from ..filesystems import tqdm
 
 
 log = logging.getLogger("argopy.stores.implementation.http")
 
 
-class httpstore(argo_store_proto):
+class httpstore(ArgoStoreProto):
     """Argo http file system
 
     Relies on :class:`fsspec.implementations.http.HTTPFileSystem`

argopy/stores/implementations/local.py

@@ -7,7 +7,7 @@
 import logging
 
 
-from ..filesystems import argo_store_proto
+from ..spec import ArgoStoreProto
 from ..filesystems import has_distributed, distributed
 from ..filesystems import tqdm
 from ...errors import InvalidMethod, DataNotFound
@@ -16,7 +16,7 @@
 log = logging.getLogger("argopy.stores.implementation.local")
 
 
-class filestore(argo_store_proto):
+class filestore(ArgoStoreProto):
     """Argo local file system
 
     Relies on :class:`fsspec.implementations.local.LocalFileSystem`

argopy/stores/implementations/memory.py

@@ -5,7 +5,7 @@
 class memorystore(filestore):
     """Argo in-memory file system (global)
 
-    Note that this inherits from :class:`argopy.stores.filestore`, not the:class:`argopy.stores.argo_store_proto`.
+    Note that this inherits from :class:`argopy.stores.filestore`, not the:class:`argopy.stores.ArgoStoreProto`.
 
     Relies on :class:`fsspec.implementations.memory.MemoryFileSystem`
     """

argopy/stores/index/argo_index.py

@@ -2,9 +2,9 @@
 
 
 if importlib.util.find_spec("pyarrow") is not None:
-    from .argo_index_pa import indexstore_pyarrow as indexstore
+    from .implementations.index_pyarrow import indexstore
 else:
-    from .argo_index_pd import indexstore_pandas as indexstore
+    from .implementations.index_pandas import indexstore
 
 
 class ArgoIndex(indexstore):

argopy/stores/index/argo_index_pd.py → argopy/stores/index/implementations/index_pandas.py

@@ -5,17 +5,17 @@
 from pathlib import Path
 from typing import List
 
-from ...errors import DataNotFound, InvalidDatasetStructure
-from ...utils.checkers import check_index_cols, is_indexbox, check_wmo, check_cyc
-from ...utils.casting import to_list
-from .argo_index_proto import ArgoIndexStoreProto
-from .argo_index_proto_s3 import search_s3
+from ....errors import DataNotFound, InvalidDatasetStructure
+from ....utils.checkers import check_index_cols, is_indexbox, check_wmo, check_cyc
+from ....utils.casting import to_list
+from ..spec import ArgoIndexStoreProto
+from .index_s3 import search_s3
 
 
 log = logging.getLogger("argopy.stores.index.pd")
 
 
-class indexstore_pandas(ArgoIndexStoreProto):
+class indexstore(ArgoIndexStoreProto):
     """Argo GDAC index store using :class:`pandas.DataFrame` as internal storage format.
 
     With this store, index and search results are saved as pickle files in cache

argopy/stores/index/argo_index_pa.py → argopy/stores/index/implementations/index_pyarrow.py

@@ -15,17 +15,17 @@
 except ModuleNotFoundError:
     pass
 
-from ...errors import DataNotFound, InvalidDatasetStructure
-from ...utils.checkers import check_index_cols, is_indexbox, check_wmo, check_cyc
-from ...utils.casting import to_list
-from .argo_index_proto import ArgoIndexStoreProto
-from .argo_index_proto_s3 import search_s3
+from ....errors import DataNotFound, InvalidDatasetStructure
+from ....utils.checkers import check_index_cols, is_indexbox, check_wmo, check_cyc
+from ....utils.casting import to_list
+from ..spec import ArgoIndexStoreProto
+from .index_s3 import search_s3
 
 
 log = logging.getLogger("argopy.stores.index.pa")
 
 
-class indexstore_pyarrow(ArgoIndexStoreProto):
+class indexstore(ArgoIndexStoreProto):
     """Argo GDAC index store using :class:`pyarrow.Table` as internal storage format.
 
     With this store, index and search results are saved as pyarrow/parquet files in cache

argopy/stores/index/argo_index_proto_s3.py → argopy/stores/index/implementations/index_s3.py

@@ -4,16 +4,16 @@
 from decorator import decorator
 import warnings
 
-from ...utils.checkers import (
+from ....utils.checkers import (
     check_index_cols,
     check_wmo,
     check_cyc,
     is_list_of_strings,
     has_aws_credentials,
     HAS_BOTO3,
 )
-from ...utils.format import redact
-from .. import s3store
+from ....utils.format import redact
+from ... import s3store
 
 
 try:

argopy/stores/index/argo_index_proto.py → argopy/stores/index/spec.py

@@ -21,7 +21,7 @@
 from ...utils.checkers import isconnected, has_aws_credentials
 from ...utils.accessories import Registry
 from .. import httpstore, memorystore, filestore, ftpstore, s3store
-from .argo_index_proto_s3 import get_a_s3index
+from .implementations.index_s3 import get_a_s3index
 
 try:
     import pyarrow.csv as csv  # noqa: F401
@@ -296,11 +296,12 @@ def _format(self, x, typ: str) -> str:
 
     @property
     def index_path(self):
+        """Absolute path to the index file"""
         return self.fs["src"].fs.sep.join([self.host, self.index_file])
 
     @property
     def cname(self) -> str:
-        """Return the search constraint(s) as a pretty formatted string
+        """Search constraint(s) as a pretty formatted string
 
         Return 'full' if a search was not yet performed on the indexstore instance
 
@@ -624,7 +625,7 @@ def get_filename(s, index):
         else:
             log.debug("Converting [%s] to dataframe from scratch ..." % src)
             # Post-processing for user:
-            from ..related import load_dict, mapp_dict
+            from ...related import load_dict, mapp_dict
 
             if nrows is not None:
                 df = df.loc[0 : nrows - 1].copy()

argopy/stores/spec.py

@@ -16,7 +16,7 @@
 from .filesystems import new_fs
 
 
-class argo_store_proto(ABC):
+class ArgoStoreProto(ABC):
     """Argo Abstract File System
 
     Provide a prototype for Argo file systems

argopy/tests/test_stores_index.py

@@ -17,7 +17,7 @@
     InvalidDatasetStructure,
 )
 from argopy.utils.checkers import is_list_of_strings
-from argopy.stores.argo_index_pd import indexstore_pandas
+from argopy.stores import indexstore_pd
 from mocked_http import mocked_httpserver, mocked_server_address
 
 log = logging.getLogger("argopy.tests.indexstores")
@@ -471,23 +471,23 @@ def test_to_indexfile(self):
 @skip_CORE
 class Test_IndexStore_pandas_CORE(IndexStore_test_proto):
     network = "core"
-    indexstore = indexstore_pandas
+    indexstore = indexstore_pd
     index_file = "ar_index_global_prof.txt"
 
 
 @skip_pandas
 @skip_BGCs
 class Test_IndexStore_pandas_BGC_synthetic(IndexStore_test_proto):
     network = "bgc"
-    indexstore = indexstore_pandas
+    indexstore = indexstore_pd
     index_file = "argo_synthetic-profile_index.txt"
 
 
 @skip_pandas
 @skip_BGCb
 class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto):
     network = "bgc"
-    indexstore = indexstore_pandas
+    indexstore = indexstore_pd
     index_file = "argo_bio-profile_index.txt"
 
 
@@ -500,9 +500,9 @@ class Test_IndexStore_pandas_BGC_bio(IndexStore_test_proto):
 @skip_CORE
 class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto):
     network = "core"
-    from argopy.stores.argo_index_pa import indexstore_pyarrow
+    from argopy.stores import indexstore_pa
 
-    indexstore = indexstore_pyarrow
+    indexstore = indexstore_pa
     index_file = "ar_index_global_prof.txt"
 
 
@@ -511,9 +511,9 @@ class Test_IndexStore_pyarrow_CORE(IndexStore_test_proto):
 @skip_BGCs
 class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto):
     network = "bgc"
-    from argopy.stores.argo_index_pa import indexstore_pyarrow
+    from argopy.stores import indexstore_pa
 
-    indexstore = indexstore_pyarrow
+    indexstore = indexstore_pa
     index_file = "argo_bio-profile_index.txt"
 
 
@@ -522,7 +522,7 @@ class Test_IndexStore_pyarrow_BGC_bio(IndexStore_test_proto):
 @skip_BGCb
 class Test_IndexStore_pyarrow_BGC_synthetic(IndexStore_test_proto):
     network = "bgc"
-    from argopy.stores.argo_index_pa import indexstore_pyarrow
+    from argopy.stores import indexstore_pa
 
-    indexstore = indexstore_pyarrow
+    indexstore = indexstore_pa
     index_file = "argo_synthetic-profile_index.txt"