REFACTOR-#2957: Restructure project files

.github/workflows/ci.yml

@@ -55,28 +55,29 @@ jobs:
             modin/pandas/series_utils.py modin/pandas/general.py \
             modin/pandas/plotting.py modin/pandas/utils.py \
             modin/pandas/iterator.py modin/pandas/indexing.py \
-      - run: python scripts/doc_checker.py modin/engines/base/frame
-      - run: python scripts/doc_checker.py modin/engines/dask
+      - run: python scripts/doc_checker.py modin/core/dataframe
+      - run: python scripts/doc_checker.py modin/core/execution/dask
       - run: |
           python scripts/doc_checker.py \
             modin/pandas/accessor.py modin/pandas/general.py \
             modin/pandas/groupby.py modin/pandas/indexing.py \
             modin/pandas/iterator.py modin/pandas/plotting.py \
             modin/pandas/series_utils.py modin/pandas/utils.py \
             modin/pandas/base.py \
-            modin/pandas/io.py modin/engines/base/io/io.py \
-            modin/engines/base/frame asv_bench/benchmarks/utils \
+            modin/pandas/io.py \
+            asv_bench/benchmarks/utils \
             asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \
             asv_bench/benchmarks/scalability/__init__.py \
-            modin/engines/base/io/column_stores \
-            modin/engines/base/io/sql \
-            modin/engines/base/io/text \
-            modin/engines/base/io/__init__.py \
-            modin/engines/base/io/file_dispatcher.py \
-            modin/experimental/engines/pandas_on_ray \
-            modin/experimental/engines/pyarrow_on_ray \
+            modin/core/io/io.py \
+            modin/core/io/column_stores \
+            modin/core/io/sql \
+            modin/core/io/text \
+            modin/core/io/__init__.py \
+            modin/core/io/file_dispatcher.py \
+            modin/experimental/core/execution/ray/implementations/pandas_on_ray \
+            modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \
             modin/pandas/series.py \
-            modin/engines/python \
+            modin/core/execution/python \
             modin/pandas/dataframe.py \
             modin/config/__init__.py \
             modin/config/__main__.py \
@@ -89,24 +90,23 @@ jobs:
           python scripts/doc_checker.py modin/experimental/xgboost/__init__.py \
             modin/experimental/xgboost/utils.py modin/experimental/xgboost/xgboost.py \
             modin/experimental/xgboost/xgboost_ray.py
-      - run: python scripts/doc_checker.py modin/engines/ray
+      - run: python scripts/doc_checker.py modin/core/execution/ray
       - run: |
-          python scripts/doc_checker.py modin/data_management/functions \
-            modin/data_management/factories/factories.py                \
-            modin/data_management/factories/dispatcher.py               \
-            modin/data_management/utils.py
+          python scripts/doc_checker.py modin/core/execution/dispatching/factories/factories.py \
+            modin/core/execution/dispatching/factories/dispatcher.py                            \
       - run: python scripts/doc_checker.py scripts/doc_checker.py
       - run: |
-          python scripts/doc_checker.py modin/experimental/pandas/io_exp.py \
+          python scripts/doc_checker.py modin/experimental/pandas/io.py \
             modin/experimental/pandas/numpy_wrap.py modin/experimental/pandas/__init__.py
-      - run: python scripts/doc_checker.py modin/backends/base
-      - run: python scripts/doc_checker.py modin/backends/pyarrow
-      - run: python scripts/doc_checker.py modin/backends/pandas
+      - run: python scripts/doc_checker.py modin/core/storage_formats/base
+      - run: python scripts/doc_checker.py modin/core/storage_formats/pyarrow
+      - run: python scripts/doc_checker.py modin/core/storage_formats/pandas
       - run: |
           python scripts/doc_checker.py \
-          modin/experimental/engines/omnisci_on_native/frame \
-          modin/experimental/engines/omnisci_on_native/io.py
-      - run: python scripts/doc_checker.py modin/experimental/backends/omnisci
+            modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe \
+            modin/experimental/core/execution/native/implementations/omnisci_on_native/io \
+            modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning \
+      - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/omnisci
 
   lint-flake8:
     name: lint (flake8)
@@ -239,11 +239,11 @@ jobs:
           conda info
           conda list
       - name: Internals tests
-        run: python -m pytest modin/data_management/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py
+        run: python -m pytest modin/core/execution/dispatching/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py
       - run: python -m pytest modin/config/test
       - run: python -m pytest modin/test/test_envvar_catcher.py
-      - run: python -m pytest modin/test/backends/base/test_internals.py
-      - run: python -m pytest modin/test/backends/pandas/test_internals.py
+      - run: python -m pytest modin/test/storage_formats/base/test_internals.py
+      - run: python -m pytest modin/test/storage_formats/pandas/test_internals.py
       - run: python -m pytest modin/test/test_envvar_npartitions.py
       - run: python -m pytest -n 2 modin/test/test_partition_api.py
       - run: python -m pytest modin/test/test_utils.py
@@ -280,7 +280,7 @@ jobs:
       - name: Install HDF5
         run: sudo apt update && sudo apt install -y libhdf5-dev
       - run: pytest modin/experimental/xgboost/test/test_default.py --backend=${{ matrix.backend }}
-      - run: python -m pytest -n 2 modin/test/backends/base/test_internals.py --backend=${{ matrix.backend }}
+      - run: python -m pytest -n 2 modin/test/storage_formats/base/test_internals.py --backend=${{ matrix.backend }}
       - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py --backend=${{ matrix.backend }}
       - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py --backend=${{ matrix.backend }}
       - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py --backend=${{ matrix.backend }}
@@ -337,7 +337,7 @@ jobs:
       - name: Install HDF5
         run: sudo apt update && sudo apt install -y libhdf5-dev
       - run: MODIN_BENCHMARK_MODE=True pytest modin/pandas/test/internals/test_benchmark_mode.py
-      - run: pytest modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
+      - run: pytest modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
       - run: pytest modin/pandas/test/test_io.py::TestCsv --verbose
       - run: |
           curl -o codecov https://codecov.io/bash
@@ -666,6 +666,7 @@ jobs:
         python-version: [ "3.7", "3.8" ]
         engine: ["ray", "dask"]
     env:
+      MODIN_EXPERIMENTAL: "True"
       MODIN_ENGINE: ${{matrix.engine}}
     name: test-spreadsheet (engine ${{matrix.engine}}, python ${{matrix.python-version}})
     steps:
@@ -683,4 +684,4 @@ jobs:
         run: |
           conda info
           conda list
-      - run: python -m pytest modin/spreadsheet/test/test_general.py
+      - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py

.github/workflows/push.yml

@@ -23,10 +23,10 @@ jobs:
           conda info
           conda list
       - name: Internals tests
-        run: python -m pytest modin/data_management/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py
+        run: python -m pytest modin/core/execution/dispatching/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py
       - run: python -m pytest modin/config/test
       - run: python -m pytest modin/test/test_envvar_catcher.py
-      - run: python -m pytest modin/test/backends/pandas/test_internals.py
+      - run: python -m pytest modin/test/storage_formats/pandas/test_internals.py
       - run: python -m pytest modin/test/test_envvar_npartitions.py
       - run: python -m pytest modin/test/test_partition_api.py
 
@@ -114,7 +114,7 @@ jobs:
           conda list
       - name: Install HDF5
         run: sudo apt update && sudo apt install -y libhdf5-dev
-      - run: pytest modin/experimental/engines/omnisci_on_native/test/test_dataframe.py
+      - run: pytest modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py
       - run: pytest modin/pandas/test/test_io.py::TestCsv
       - run: |
           curl -o codecov https://codecov.io/bash
@@ -287,6 +287,7 @@ jobs:
         python-version: [ "3.7", "3.8" ]
         engine: ["ray", "dask"]
     env:
+      MODIN_EXPERIMENTAL: "True"
       MODIN_ENGINE: ${{matrix.engine}}
     name: test-spreadsheet (engine ${{matrix.engine}}, python ${{matrix.python-version}})
     steps:
@@ -304,4 +305,4 @@ jobs:
         run: |
           conda info
           conda list
-      - run: python -m pytest modin/spreadsheet/test/test_general.py
+      - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py

asv_bench/benchmarks/utils/common.py

@@ -458,7 +458,7 @@ def trigger_import(*dfs):
     """
     assert ASV_USE_BACKEND == "omnisci"
 
-    from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import (
+    from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import (
         OmnisciServer,
     )
 

examples/docker/modin-omnisci/census-omnisci.py

@@ -14,7 +14,7 @@
 import sys
 import time
 import modin.pandas as pd
-from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer
+from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import OmnisciServer
 
 from sklearn import config_context
 import sklearnex

examples/docker/modin-omnisci/nyc-taxi-omnisci.py

@@ -14,7 +14,7 @@
 import sys
 import time
 import modin.pandas as pd
-from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer
+from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import OmnisciServer
 
 
 def read(filename):

examples/docker/modin-omnisci/plasticc-omnisci.py

@@ -16,7 +16,7 @@
 from collections import OrderedDict
 from functools import partial
 import modin.pandas as pd
-from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer
+from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import OmnisciServer
 
 import numpy as np
 import xgboost as xgb

modin/conftest.py

@@ -44,9 +44,14 @@ def _saving_make_api_url(token, _make_api_url=modin.utils._make_api_url):
 import modin.config  # noqa: E402
 from modin.config import IsExperimental, TestRayClient  # noqa: E402
 
-from modin.backends import PandasQueryCompiler, BaseQueryCompiler  # noqa: E402
-from modin.engines.python.pandas_on_python.io import PandasOnPythonIO  # noqa: E402
-from modin.data_management.factories import factories  # noqa: E402
+from modin.core.storage_formats import (  # noqa: E402
+    PandasQueryCompiler,
+    BaseQueryCompiler,
+)
+from modin.core.execution.python.implementations.pandas_on_python.io import (  # noqa: E402
+    PandasOnPythonIO,
+)
+from modin.core.execution.dispatching.factories import factories  # noqa: E402
 from modin.utils import get_current_backend  # noqa: E402
 from modin.pandas.test.utils import (  # noqa: E402
     _make_csv_file,

modin/engines/python/__init__.py → modin/core/__init__.py

@@ -10,3 +10,5 @@
 # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
+
+"""Modin's core functionality."""

modin/engines/base/__init__.py → modin/core/dataframe/__init__.py

@@ -10,3 +10,5 @@
 # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
+
+"""Base Modin Dataframe functionality."""

modin/data_management/functions/__init__.py → modin/core/dataframe/algebra/__init__.py

@@ -11,15 +11,18 @@
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
-"""Function module provides template for a query compiler methods for a set of common operations."""
+"""Modin Dataframe algebra (core operators)."""
 
 from .function import Function
 from .mapfunction import MapFunction
 from .mapreducefunction import MapReduceFunction
 from .reductionfunction import ReductionFunction
 from .foldfunction import FoldFunction
 from .binary_function import BinaryFunction
-from .groupby_function import GroupbyReduceFunction, groupby_reduce_functions
+from .groupby_function import (
+    GroupbyReduceFunction,
+    groupby_reduce_functions,
+)
 
 __all__ = [
     "Function",

modin/data_management/functions/default_methods/__init__.py → modin/core/dataframe/algebra/default2pandas/__init__.py

@@ -11,19 +11,19 @@
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
-"""Function module provides templates for a query compiler default-to-pandas methods."""
+"""Module default2pandas provides templates for a query compiler default-to-pandas methods."""
 
-from .dataframe_default import DataFrameDefault
-from .datetime_default import DateTimeDefault
-from .series_default import SeriesDefault
-from .str_default import StrDefault
-from .binary_default import BinaryDefault
-from .any_default import AnyDefault
-from .resample_default import ResampleDefault
-from .rolling_default import RollingDefault
+from .dataframe import DataFrameDefault
+from .datetime import DateTimeDefault
+from .series import SeriesDefault
+from .str import StrDefault
+from .binary import BinaryDefault
+from .any import AnyDefault
+from .resample import ResampleDefault
+from .rolling import RollingDefault
 from .default import DefaultMethod
-from .cat_default import CatDefault
-from .groupby_default import GroupByDefault
+from .cat import CatDefault
+from .groupby import GroupByDefault
 
 __all__ = [
     "DataFrameDefault",

modin/data_management/functions/default_methods/binary_default.py → modin/core/dataframe/algebra/default2pandas/binary.py

@@ -13,7 +13,7 @@
 
 """Module houses default binary functions builder class."""
 
-from .any_default import AnyDefault
+from .any import AnyDefault
 
 import pandas
 from pandas.core.dtypes.common import is_list_like

modin/data_management/functions/default_methods/cat_default.py → modin/core/dataframe/algebra/default2pandas/cat.py

@@ -13,7 +13,7 @@
 
 """Module houses default applied-on-category functions builder class."""
 
-from .series_default import SeriesDefault
+from .series import SeriesDefault
 
 
 class CatDefault(SeriesDefault):

modin/data_management/functions/default_methods/datetime_default.py → modin/core/dataframe/algebra/default2pandas/datetime.py

@@ -13,7 +13,7 @@
 
 """Module houses default applied-on-datetime functions builder class."""
 
-from .series_default import SeriesDefault
+from .series import SeriesDefault
 
 
 class DateTimeDefault(SeriesDefault):

modin/data_management/functions/default_methods/default.py → modin/core/dataframe/algebra/default2pandas/default.py

@@ -13,7 +13,7 @@
 
 """Module houses default functions builder class."""
 
-from modin.data_management.functions.function import Function
+from modin.core.dataframe.algebra.function import Function
 from modin.utils import try_cast_to_pandas
 
 from pandas.core.dtypes.common import is_list_like

modin/data_management/functions/default_methods/series_default.py → modin/core/dataframe/algebra/default2pandas/series.py

@@ -13,7 +13,7 @@
 
 """Module houses default Series functions builder class."""
 
-from .any_default import AnyDefault
+from .any import AnyDefault
 
 
 class SeriesDefault(AnyDefault):

modin/data_management/functions/default_methods/str_default.py → modin/core/dataframe/algebra/default2pandas/str.py

@@ -13,7 +13,7 @@
 
 """Module houses default applied-on-str functions builder class."""
 
-from .series_default import SeriesDefault
+from .series import SeriesDefault
 
 
 class StrDefault(SeriesDefault):

modin/data_management/functions/groupby_function.py → modin/core/dataframe/algebra/groupby_function.py

@@ -16,7 +16,7 @@
 import pandas
 
 from .mapreducefunction import MapReduceFunction
-from .default_methods.groupby_default import GroupBy
+from .default2pandas.groupby import GroupBy
 from modin.utils import try_cast_to_pandas, hashable
 
 

modin/engines/ray/__init__.py → modin/core/dataframe/base/__init__.py

@@ -10,3 +10,5 @@
 # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
+
+"""Base Modin Dataframe classes."""

modin/engines/base/io/pickle/__init__.py → modin/core/dataframe/base/dataframe/__init__.py

@@ -10,3 +10,5 @@
 # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
+
+"""Base Modin Dataframe class."""

modin/core/dataframe/base/partitioning/__init__.py

@@ -0,0 +1,14 @@
+# Licensed to Modin Development Team under one or more contributor license agreements.
+# See the NOTICE file distributed with this work for additional information regarding
+# copyright ownership.  The Modin Development Team licenses this file to you under the
+# Apache License, Version 2.0 (the "License"); you may not use this file except in
+# compliance with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific language
+# governing permissions and limitations under the License.
+
+"""Base Modin Dataframe classes related to its partitioning."""

modin/engines/base/README.md → modin/core/dataframe/pandas/README.md

@@ -2,10 +2,10 @@
 
 ### Object Hierarchy
 
-- `frame/partition.py` contains `PandasFramePartition` interface and its implementations.
-- `frame/partition_manager.py` contains `PandasFramePartitionManager` interface and its implementations.
+- `partitioning/partition.py` contains `PandasFramePartition` interface and its implementations.
+- `partitioning/partition_manager.py` contains `PandasFramePartitionManager` interface and its implementations.
 	- `PandasFramePartitionManager` manages 2D-array of `PandasFramePartition` object
-- `frame/axis_partition.py` contains `BaseFrameAxisPartition` and with the following hierarchy:
+- `partitioning/axis_partition.py` contains `BaseFrameAxisPartition` and with the following hierarchy:
 	```
 	BaseFrameAxisPartition -> PandasOnRayFrameAxisPartition -> {PandasOnRayFrameColumnPartition, PandasOnRayFrameRowPartition}
 	```

modin/engines/base/frame/__init__.py → modin/core/dataframe/pandas/__init__.py

@@ -11,4 +11,4 @@
 # ANY KIND, either express or implied. See the License for the specific language
 # governing permissions and limitations under the License.
 
-"""Package holds key base classes to making efficient scale of data."""
+"""Base Modin Dataframe classes optimized for pandas storage format."""