From ae07c0052b24c00e64f6d69fddd10b52dd4a00d5 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Fri, 2 Jul 2021 08:25:17 -0500 Subject: [PATCH 01/18] REFACTOR-#2957: Restructure project files Signed-off-by: Devin Petersohn --- project_structure.txt | 119 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 project_structure.txt diff --git a/project_structure.txt b/project_structure.txt new file mode 100644 index 00000000000..e4e38bb6a54 --- /dev/null +++ b/project_structure.txt @@ -0,0 +1,119 @@ +.modin/ + .config/ + .distributed/ + .pandas/ + .test/ + .rest files + .core/ + .io/ + .column_stores/ + . # the same as it is now + .sql/ + . # the same as it is now + .text/ + . # the same as it is now + .file_dispatcher.py # the same as it is now + .io.py # the same as it is now + .dispatching/ + .factories/ + . # the same as it is now + .formats/ + .base/ + .query_compiler.py # BaseQueryCompiler + .pandas/ + .base/ # or default (DefaultPandasQueryCompiler) + .query_compiler.py # BasePandasQueryCompiler + .kernels.py + .query_compiler.py # PandasQueryCompiler + .parsers.py + .utils.py # data_management/utils moved here + .execution/ + .ray/ + .frame/ + .dataframe.py # GenericRayDataframe + .metadata.py # GenericRayDataframeMetadata + .partition_management/ # or `partitioning` + .axis_partition.py # GenericRayDataframeAxisPartition + .partition_manager.py # GenericRayDataframeManager + .partition.py # GenericRayDataframePartition + .io/ + .io.py # GenericRayIO + .task_wrapper.py # GenericRayTask + .utils.py # init_ray + .dask/ + . # the same as for ray + .python/ + . # the same as for ray + .dataframe/ # implementations go here + .algebra/ + .operators/ # dispatching/functions here + . # the same as it is in dispatching/functions + .implementations/ # all of the implementations go here + .base/ + .frame/ + .dataframe.py # BaseDataframe + .metadata.py # BaseDataframeMetaData + .partitioning/ + .axis_partition.py # BaseDataframeAxisPartition + .partition_manager.py # BaseDataframeManager + .partition.py # BaseDataframePartition + .pandas_base/ # make distinct from pandas format + .frame/ + .dataframe.py # PandasDataframe + .metadata.py # PandasDataframeMetaData + .partitioning/ + .axis_partition.py # PandasDataframeAxisPartition + .partition_manager.py # PandasDataframeManager + .partition.py # PandasDataframePartition + .pandas_on_ray/ + .frame/ + .dataframe.py # PandasOnRayDataframe + .metadata.py # PandasOnRayDataframeMetadata + .partition_management/ # or `partitioning` + .axis_partition.py # PandasOnRayDataframeAxisPartition + .partition_manager.py # PandasOnRayDataframeManager + .partition.py # PandasOnRayDataframePartition + .io/ + .io.py # PandasOnRayIO + .pandas_on_dask/ + . # the same as for pandas_on_ray + .pandas_on_python/ + . # the same as for pandas_on_ray + .experimental/ + ./cloud + . # the same as it is now + ./pandas + . # the same as it is now + ./sklearn + . # the same as it is now + ./sql + . # the same as it is now + .spreadsheet/ + . # from modin/spreadsheet + ./xgboost + . # the same as it is now + ./core + ./formats + ./omnisci + .kernels.py + .query_compiler.py + ./pyarrow + .kernels.py + .query_compiler.py + .dataframes/ + .omnisci_on_ray/ + .frame/ + .dataframe.py # OmniSciOnRayDataframe + .metadata.py # OmniSciOnRayFrameMetaData + .partition_management/ # or `partitioning` + .axis_partition.py # OmniSciOnRayFrameAxisPartition + .partition_manager.py # OmniSciOnRayFrameManager + .partition.py # OmniSciOnRayFramePartition + .pyarrow_on_ray/ + .frame/ + .dataframe.py # PyarrowOnRayDataframe + .metadata.py # PyarrowOnRayDataframe + .partition_management/ # or `partitioning` + .axis_partition.py # PyarrowOnRayDataframeAxisPartition + .partition_manager.py # PyarrowOnRayDataframeManager + .partition.py # PyarrowOnRayDataframePartition From 0450762885eb9f23fcef7aa78fad3dbbfd16020f Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Thu, 2 Sep 2021 20:10:12 +0300 Subject: [PATCH 02/18] REFACTOR-#2957: Stabilize the structure Signed-off-by: Igoshev, Yaroslav --- project_structure.txt | 150 +++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 84 deletions(-) diff --git a/project_structure.txt b/project_structure.txt index e4e38bb6a54..d213a9631b5 100644 --- a/project_structure.txt +++ b/project_structure.txt @@ -5,6 +5,16 @@ .test/ .rest files .core/ + .storage_formats/ + .base/ + .query_compiler.py # BaseQueryCompiler + .pandas/ + .base/ # or default (DefaultPandasQueryCompiler) + .query_compiler.py # DefaultPandasQueryCompiler + .kernels.py + .query_compiler.py # PandasQueryCompiler + .parsers.py + .utils.py # data_management/utils moved here .io/ .column_stores/ . # the same as it is now @@ -14,71 +24,41 @@ . # the same as it is now .file_dispatcher.py # the same as it is now .io.py # the same as it is now - .dispatching/ - .factories/ - . # the same as it is now - .formats/ - .base/ - .query_compiler.py # BaseQueryCompiler - .pandas/ - .base/ # or default (DefaultPandasQueryCompiler) - .query_compiler.py # BasePandasQueryCompiler - .kernels.py - .query_compiler.py # PandasQueryCompiler - .parsers.py - .utils.py # data_management/utils moved here - .execution/ - .ray/ - .frame/ - .dataframe.py # GenericRayDataframe - .metadata.py # GenericRayDataframeMetadata - .partition_management/ # or `partitioning` - .axis_partition.py # GenericRayDataframeAxisPartition - .partition_manager.py # GenericRayDataframeManager - .partition.py # GenericRayDataframePartition - .io/ - .io.py # GenericRayIO - .task_wrapper.py # GenericRayTask - .utils.py # init_ray - .dask/ - . # the same as for ray - .python/ - . # the same as for ray - .dataframe/ # implementations go here + .dataframe/ .algebra/ - .operators/ # dispatching/functions here - . # the same as it is in dispatching/functions - .implementations/ # all of the implementations go here - .base/ - .frame/ - .dataframe.py # BaseDataframe - .metadata.py # BaseDataframeMetaData - .partitioning/ - .axis_partition.py # BaseDataframeAxisPartition - .partition_manager.py # BaseDataframeManager - .partition.py # BaseDataframePartition - .pandas_base/ # make distinct from pandas format - .frame/ - .dataframe.py # PandasDataframe - .metadata.py # PandasDataframeMetaData - .partitioning/ - .axis_partition.py # PandasDataframeAxisPartition - .partition_manager.py # PandasDataframeManager - .partition.py # PandasDataframePartition - .pandas_on_ray/ - .frame/ - .dataframe.py # PandasOnRayDataframe - .metadata.py # PandasOnRayDataframeMetadata - .partition_management/ # or `partitioning` - .axis_partition.py # PandasOnRayDataframeAxisPartition - .partition_manager.py # PandasOnRayDataframeManager - .partition.py # PandasOnRayDataframePartition - .io/ - .io.py # PandasOnRayIO - .pandas_on_dask/ - . # the same as for pandas_on_ray - .pandas_on_python/ - . # the same as for pandas_on_ray + ./d2p + . # the same as it is now in data_management/functions/default_methods + .operators/ + .map.py + .reduce.py + .tree_reduce.py + .explode.py + .groupby.py + .etc. + .execution/ + .ray/ + .generic/ + .dataframe/ + .dataframe.py # GenericRayDataframe + .metadata.py # GenericRayDataframeMetadata + .partition_management/ + .axis_partition.py # GenericRayDataframeAxisPartition + .partition_manager.py # GenericRayDataframeManager + .partition.py # GenericRayDataframePartition + .io/ + .io.py # GenericRayIO + .task_wrapper.py # GenericRayTask + .utils.py # init_ray + .implementations/ + .pandas_on_ray/ + . # the same as it is now + .dask/ + . # the same as it is for ray + .python/ + . # the same as it is for ray + .dispatching/ + .factories/ + . # the same as it is now .experimental/ ./cloud . # the same as it is now @@ -93,27 +73,29 @@ ./xgboost . # the same as it is now ./core - ./formats - ./omnisci + .storage_formats/ + .omnisci/ .kernels.py .query_compiler.py - ./pyarrow + .pyarrow/ .kernels.py .query_compiler.py - .dataframes/ - .omnisci_on_ray/ - .frame/ - .dataframe.py # OmniSciOnRayDataframe - .metadata.py # OmniSciOnRayFrameMetaData - .partition_management/ # or `partitioning` - .axis_partition.py # OmniSciOnRayFrameAxisPartition - .partition_manager.py # OmniSciOnRayFrameManager - .partition.py # OmniSciOnRayFramePartition - .pyarrow_on_ray/ - .frame/ - .dataframe.py # PyarrowOnRayDataframe - .metadata.py # PyarrowOnRayDataframe - .partition_management/ # or `partitioning` - .axis_partition.py # PyarrowOnRayDataframeAxisPartition - .partition_manager.py # PyarrowOnRayDataframeManager - .partition.py # PyarrowOnRayDataframePartition + .dataframe/ + .execution/ + .ray/ + .omnisci_on_ray/ # this should go to omnisci_on_native in execution/native folder after #3352 is merged + .dataframe/ + .dataframe.py # OmniSciOnRayDataframe + .metadata.py # OmniSciOnRayFrameMetaData + .partition_management/ + .axis_partition.py # OmniSciOnRayFrameAxisPartition + .partition_manager.py # OmniSciOnRayFrameManager + .partition.py # OmniSciOnRayFramePartition + .pyarrow_on_ray/ + .frame/ + .dataframe.py # PyarrowOnRayDataframe + .metadata.py # PyarrowOnRayDataframe + .partition_management/ # or `partitioning` + .axis_partition.py # PyarrowOnRayDataframeAxisPartition + .partition_manager.py # PyarrowOnRayDataframeManager + .partition.py # PyarrowOnRayDataframePartition From a31833e9c4e381a26de1eef1b5d4558a90276222 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 11:23:42 +0300 Subject: [PATCH 03/18] REFACTOR-#2957: Apply the changes to the project structure Signed-off-by: Igoshev, Yaroslav --- modin/conftest.py | 11 +- .../ray/generic/frame => core}/__init__.py | 2 + .../backends => core/dataframe}/__init__.py | 2 + modin/core/dataframe/algebra/__init__.py | 14 ++ .../dataframe/algebra/d2p}/__init__.py | 22 +-- .../dataframe/algebra/d2p}/any_default.py | 2 +- .../dataframe/algebra/d2p}/binary_default.py | 2 +- .../dataframe/algebra/d2p}/cat_default.py | 2 +- .../algebra/d2p}/dataframe_default.py | 2 +- .../algebra/d2p}/datetime_default.py | 2 +- .../dataframe/algebra/d2p}/default.py | 2 +- .../dataframe/algebra/d2p}/groupby_default.py | 2 +- .../algebra/d2p}/resample_default.py | 2 +- .../dataframe/algebra/d2p}/rolling_default.py | 2 +- .../dataframe/algebra/d2p}/series_default.py | 2 +- .../dataframe/algebra/d2p}/str_default.py | 2 +- .../dataframe/algebra/operators}/__init__.py | 17 ++- .../algebra/operators}/binary_function.py | 2 +- .../algebra/operators}/foldfunction.py | 2 +- .../dataframe/algebra/operators}/function.py | 0 .../algebra/operators}/groupby_function.py | 2 +- .../algebra/operators}/mapfunction.py | 2 +- .../algebra/operators}/mapreducefunction.py | 2 +- .../algebra/operators}/reductionfunction.py | 2 +- .../frame => core/dataframe/base}/__init__.py | 2 + .../dataframe/base/dataframe}/__init__.py | 2 + .../dataframe/base/partitioning/__init__.py | 14 ++ .../base => core/dataframe/pandas}/README.md | 6 +- .../dataframe/pandas}/__init__.py | 2 +- .../dataframe/pandas/dataframe/__init__.py | 14 ++ .../dataframe/pandas/dataframe/dataframe.py} | 6 +- .../dataframe/pandas/partitioning/__init__.py | 14 ++ .../pandas/partitioning}/axis_partition.py | 2 +- .../pandas/partitioning}/partition.py | 0 .../pandas/partitioning}/partition_manager.py | 2 +- modin/core/execution/__init__.py | 14 ++ .../execution}/dask/__init__.py | 0 modin/core/execution/dask/common/__init__.py | 14 ++ .../execution/dask/common}/task_wrapper.py | 0 .../execution/dask/common}/utils.py | 0 .../dask/implementations/__init__.py | 14 ++ .../pandas_on_dask/__init__.py | 0 .../pandas_on_dask/dataframe}/__init__.py | 0 .../pandas_on_dask/dataframe/dataframe.py} | 6 +- .../pandas_on_dask/io/__init__.py | 20 +++ .../implementations/pandas_on_dask/io}/io.py | 18 ++- .../pandas_on_dask/partitioning/__init__.py | 14 ++ .../partitioning}/axis_partition.py | 8 +- .../pandas_on_dask/partitioning}/partition.py | 4 +- .../partitioning}/partition_manager.py | 10 +- modin/core/execution/dispatching/__init__.py | 14 ++ .../dispatching}/factories/__init__.py | 4 +- .../dispatching}/factories/dispatcher.py | 2 +- .../dispatching}/factories/factories.py | 28 ++-- .../dispatching/factories/test}/__init__.py | 0 .../factories/test/test_dispatcher.py | 0 modin/core/execution/python/__init__.py | 14 ++ .../python/implementations/__init__.py | 14 ++ .../pandas_on_python/__init__.py | 0 .../pandas_on_python/dataframe}/__init__.py | 0 .../pandas_on_python/dataframe/dataframe.py} | 6 +- .../pandas_on_python/io/__init__.py | 22 +++ .../pandas_on_python/io}/io.py | 8 +- .../pandas_on_python/partitioning/__init__.py | 14 ++ .../partitioning}/axis_partition.py | 8 +- .../partitioning}/partition.py | 4 +- .../partitioning}/partition_manager.py | 10 +- modin/core/execution/ray/__init__.py | 14 ++ modin/core/execution/ray/common/__init__.py | 14 ++ .../execution/ray/common}/task_wrapper.py | 0 .../execution/ray/common}/utils.py | 6 +- .../execution/ray/generic}/__init__.py | 4 +- .../core/execution/ray/generic/io/__init__.py | 16 +++ .../execution/ray/generic/io}/io.py | 2 +- .../ray/generic/partitioning/__init__.py | 14 ++ .../partitioning}/partition_manager.py | 4 +- .../execution/ray/implementations/__init__.py | 0 .../implementations}/cudf_on_ray/__init__.py | 0 .../cudf_on_ray/dataframe}/__init__.py | 0 .../cudf_on_ray/dataframe/dataframe.py} | 12 +- .../cudf_on_ray/io/__init__.py | 6 +- .../ray/implementations}/cudf_on_ray/io/io.py | 18 +-- .../cudf_on_ray/io/text/__init__.py | 0 .../cudf_on_ray/io/text/csv_dispatcher.py | 6 +- .../cudf_on_ray/partitioning/__init__.py | 14 ++ .../partitioning}/axis_partition.py | 4 +- .../cudf_on_ray/partitioning}/gpu_manager.py | 0 .../cudf_on_ray/partitioning}/partition.py | 2 +- .../partitioning}/partition_manager.py | 10 +- .../pandas_on_ray}/__init__.py | 0 .../pandas_on_ray/dataframe}/__init__.py | 0 .../pandas_on_ray/dataframe/dataframe.py} | 10 +- .../pandas_on_ray/io/__init__.py | 16 +++ .../implementations/pandas_on_ray/io}/io.py | 18 ++- .../pandas_on_ray}/modin_aqp.py | 0 .../pandas_on_ray/partitioning/__init__.py | 0 .../partitioning}/axis_partition.py | 8 +- .../pandas_on_ray/partitioning}/partition.py | 4 +- .../partitioning}/partition_manager.py | 12 +- modin/{engines/base => core}/io/__init__.py | 26 ++-- .../io/column_stores/__init__.py | 0 .../column_stores/column_store_dispatcher.py | 4 +- .../io/column_stores/feather_dispatcher.py | 2 +- .../io/column_stores/hdf_dispatcher.py | 2 +- .../io/column_stores/parquet_dispatcher.py | 2 +- .../base => core}/io/file_dispatcher.py | 0 modin/{engines/base => core}/io/io.py | 2 +- modin/{engines => core/io/pickle}/__init__.py | 0 .../io/pickle/pickle_dispatcher.py | 2 +- .../{engines/base => core}/io/sql/__init__.py | 0 .../base => core}/io/sql/sql_dispatcher.py | 2 +- .../base => core}/io/text/__init__.py | 0 .../base => core}/io/text/csv_dispatcher.py | 2 +- .../io/text/csv_glob_dispatcher.py | 4 +- .../base => core}/io/text/excel_dispatcher.py | 2 +- .../base => core}/io/text/fwf_dispatcher.py | 2 +- .../base => core}/io/text/json_dispatcher.py | 2 +- .../io/text/text_file_dispatcher.py | 4 +- .../storage_formats}/__init__.py | 8 +- .../storage_formats}/base/__init__.py | 2 +- .../storage_formats}/base/doc_utils.py | 0 .../storage_formats}/base/query_compiler.py | 4 +- modin/core/storage_formats/cudf/__init__.py | 18 +++ .../storage_formats}/cudf/parser.py | 8 +- .../storage_formats}/cudf/query_compiler.py | 2 +- .../storage_formats}/pandas/__init__.py | 2 +- .../storage_formats}/pandas/parsers.py | 4 +- .../storage_formats}/pandas/query_compiler.py | 4 +- .../storage_formats/pandas}/utils.py | 0 .../storage_formats}/pyarrow/__init__.py | 0 .../storage_formats}/pyarrow/parsers.py | 2 +- .../pyarrow/query_compiler.py | 2 +- .../dataframe/pandas/partitions.py | 4 +- .../experimental/backends/omnisci/__init__.py | 12 -- modin/experimental/cloud/meta_magic.py | 2 +- modin/experimental/core/__init__.py | 14 ++ modin/experimental/core/execution/__init__.py | 14 ++ .../core/execution/ray}/__init__.py | 0 .../omnisci_on_ray}/__init__.py | 0 .../omnisci_on_ray/dataframe}/__init__.py | 0 .../dataframe}/calcite_algebra.py | 4 +- .../dataframe}/calcite_builder.py | 6 +- .../dataframe}/calcite_serializer.py | 4 +- .../omnisci_on_ray/dataframe/dataframe.py} | 10 +- .../omnisci_on_ray/dataframe}/df_algebra.py | 4 +- .../omnisci_on_ray/dataframe}/expr.py | 0 .../dataframe}/omnisci_worker.py | 0 .../omnisci_on_ray/io/__init__.py | 18 +++ .../implementations/omnisci_on_ray/io}/io.py | 0 .../omnisci_on_ray/partitioning/__init__.py | 0 .../partitioning}/axis_partition.py | 8 +- .../omnisci_on_ray/partitioning}/partition.py | 2 +- .../partitioning}/partition_manager.py | 18 ++- .../omnisci_on_ray/test}/__init__.py | 0 .../omnisci_on_ray/test/test_dataframe.py | 0 .../pandas_on_ray/__init__.py | 0 .../pandas_on_ray/io/__init__.py | 20 +++ .../implementations/pandas_on_ray/io/io.py} | 18 ++- .../implementations/pandas_on_ray/io}/sql.py | 0 .../pyarrow_on_ray/__init__.py | 0 .../pyarrow_on_ray/dataframe}/__init__.py | 0 .../pyarrow_on_ray/dataframe/dataframe.py} | 6 +- .../pyarrow_on_ray/io/__init__.py | 5 + .../implementations/pyarrow_on_ray/io}/io.py | 16 ++- .../pyarrow_on_ray/partitioning}/__init__.py | 0 .../partitioning}/axis_partition.py | 8 +- .../pyarrow_on_ray/partitioning}/partition.py | 4 +- .../partitioning}/partition_manager.py | 8 +- .../core/storage_formats/__init__.py | 14 ++ .../core/storage_formats/omnisci/__init__.py | 14 ++ .../omnisci/query_compiler.py | 4 +- modin/experimental/engines/__init__.py | 12 -- .../engines/omnisci_on_ray/__init__.py | 12 -- .../engines/omnisci_on_ray/test/__init__.py | 12 -- .../spreadsheet/__init__.py | 0 .../{ => experimental}/spreadsheet/general.py | 0 .../spreadsheet/test}/__init__.py | 0 .../spreadsheet/test/test_general.py | 2 +- modin/pandas/__init__.py | 8 +- modin/spreadsheet/test/__init__.py | 12 -- modin/sql/connection.py | 64 --------- project_structure.txt | 132 ++++++------------ setup.cfg | 21 ++- 183 files changed, 827 insertions(+), 449 deletions(-) rename modin/{engines/ray/generic/frame => core}/__init__.py (95%) rename modin/{experimental/backends => core/dataframe}/__init__.py (94%) create mode 100644 modin/core/dataframe/algebra/__init__.py rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/__init__.py (58%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/any_default.py (97%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/binary_default.py (97%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/cat_default.py (94%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/dataframe_default.py (96%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/datetime_default.py (94%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/default.py (99%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/groupby_default.py (99%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/resample_default.py (97%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/rolling_default.py (97%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/series_default.py (95%) rename modin/{data_management/functions/default_methods => core/dataframe/algebra/d2p}/str_default.py (94%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/__init__.py (64%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/binary_function.py (98%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/foldfunction.py (96%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/function.py (100%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/groupby_function.py (99%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/mapfunction.py (96%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/mapreducefunction.py (97%) rename modin/{data_management/functions => core/dataframe/algebra/operators}/reductionfunction.py (96%) rename modin/{engines/ray/pandas_on_ray/frame => core/dataframe/base}/__init__.py (95%) rename modin/{engines/ray/pandas_on_ray => core/dataframe/base/dataframe}/__init__.py (95%) create mode 100644 modin/core/dataframe/base/partitioning/__init__.py rename modin/{engines/base => core/dataframe/pandas}/README.md (57%) rename modin/{engines/base/frame => core/dataframe/pandas}/__init__.py (91%) create mode 100644 modin/core/dataframe/pandas/dataframe/__init__.py rename modin/{engines/base/frame/data.py => core/dataframe/pandas/dataframe/dataframe.py} (99%) create mode 100644 modin/core/dataframe/pandas/partitioning/__init__.py rename modin/{engines/base/frame => core/dataframe/pandas/partitioning}/axis_partition.py (99%) rename modin/{engines/base/frame => core/dataframe/pandas/partitioning}/partition.py (100%) rename modin/{engines/base/frame => core/dataframe/pandas/partitioning}/partition_manager.py (99%) create mode 100644 modin/core/execution/__init__.py rename modin/{engines => core/execution}/dask/__init__.py (100%) create mode 100644 modin/core/execution/dask/common/__init__.py rename modin/{engines/dask => core/execution/dask/common}/task_wrapper.py (100%) rename modin/{engines/dask => core/execution/dask/common}/utils.py (100%) create mode 100644 modin/core/execution/dask/implementations/__init__.py rename modin/{engines/dask => core/execution/dask/implementations}/pandas_on_dask/__init__.py (100%) rename modin/{engines/dask/pandas_on_dask/frame => core/execution/dask/implementations/pandas_on_dask/dataframe}/__init__.py (100%) rename modin/{engines/dask/pandas_on_dask/frame/data.py => core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py} (92%) create mode 100644 modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py rename modin/{engines/dask/pandas_on_dask => core/execution/dask/implementations/pandas_on_dask/io}/io.py (81%) create mode 100644 modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py rename modin/{engines/dask/pandas_on_dask/frame => core/execution/dask/implementations/pandas_on_dask/partitioning}/axis_partition.py (97%) rename modin/{engines/dask/pandas_on_dask/frame => core/execution/dask/implementations/pandas_on_dask/partitioning}/partition.py (98%) rename modin/{engines/dask/pandas_on_dask/frame => core/execution/dask/implementations/pandas_on_dask/partitioning}/partition_manager.py (93%) create mode 100644 modin/core/execution/dispatching/__init__.py rename modin/{data_management => core/execution/dispatching}/factories/__init__.py (90%) rename modin/{data_management => core/execution/dispatching}/factories/dispatcher.py (99%) rename modin/{data_management => core/execution/dispatching}/factories/factories.py (94%) rename modin/{backends/cudf => core/execution/dispatching/factories/test}/__init__.py (100%) rename modin/{data_management => core/execution/dispatching}/factories/test/test_dispatcher.py (100%) create mode 100644 modin/core/execution/python/__init__.py create mode 100644 modin/core/execution/python/implementations/__init__.py rename modin/{engines/python => core/execution/python/implementations}/pandas_on_python/__init__.py (100%) rename modin/{engines/python/pandas_on_python/frame => core/execution/python/implementations/pandas_on_python/dataframe}/__init__.py (100%) rename modin/{engines/python/pandas_on_python/frame/data.py => core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py} (89%) create mode 100644 modin/core/execution/python/implementations/pandas_on_python/io/__init__.py rename modin/{engines/python/pandas_on_python => core/execution/python/implementations/pandas_on_python/io}/io.py (82%) create mode 100644 modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py rename modin/{engines/python/pandas_on_python/frame => core/execution/python/implementations/pandas_on_python/partitioning}/axis_partition.py (91%) rename modin/{engines/python/pandas_on_python/frame => core/execution/python/implementations/pandas_on_python/partitioning}/partition.py (98%) rename modin/{engines/python/pandas_on_python/frame => core/execution/python/implementations/pandas_on_python/partitioning}/partition_manager.py (79%) create mode 100644 modin/core/execution/ray/__init__.py create mode 100644 modin/core/execution/ray/common/__init__.py rename modin/{engines/ray => core/execution/ray/common}/task_wrapper.py (100%) rename modin/{engines/ray => core/execution/ray/common}/utils.py (96%) rename modin/{sql => core/execution/ray/generic}/__init__.py (93%) create mode 100644 modin/core/execution/ray/generic/io/__init__.py rename modin/{engines/ray/generic => core/execution/ray/generic/io}/io.py (99%) create mode 100644 modin/core/execution/ray/generic/partitioning/__init__.py rename modin/{engines/ray/generic/frame => core/execution/ray/generic/partitioning}/partition_manager.py (94%) create mode 100644 modin/core/execution/ray/implementations/__init__.py rename modin/{engines/ray => core/execution/ray/implementations}/cudf_on_ray/__init__.py (100%) rename modin/{engines/ray/cudf_on_ray/frame => core/execution/ray/implementations/cudf_on_ray/dataframe}/__init__.py (100%) rename modin/{engines/ray/cudf_on_ray/frame/data.py => core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py} (97%) rename modin/{engines/ray => core/execution/ray/implementations}/cudf_on_ray/io/__init__.py (82%) rename modin/{engines/ray => core/execution/ray/implementations}/cudf_on_ray/io/io.py (69%) rename modin/{engines/ray => core/execution/ray/implementations}/cudf_on_ray/io/text/__init__.py (100%) rename modin/{engines/ray => core/execution/ray/implementations}/cudf_on_ray/io/text/csv_dispatcher.py (95%) create mode 100644 modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py rename modin/{engines/ray/cudf_on_ray/frame => core/execution/ray/implementations/cudf_on_ray/partitioning}/axis_partition.py (96%) rename modin/{engines/ray/cudf_on_ray/frame => core/execution/ray/implementations/cudf_on_ray/partitioning}/gpu_manager.py (100%) rename modin/{engines/ray/cudf_on_ray/frame => core/execution/ray/implementations/cudf_on_ray/partitioning}/partition.py (99%) rename modin/{engines/ray/cudf_on_ray/frame => core/execution/ray/implementations/cudf_on_ray/partitioning}/partition_manager.py (94%) rename modin/{data_management => core/execution/ray/implementations/pandas_on_ray}/__init__.py (100%) rename modin/{data_management/factories/test => core/execution/ray/implementations/pandas_on_ray/dataframe}/__init__.py (100%) rename modin/{engines/ray/pandas_on_ray/frame/data.py => core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py} (90%) create mode 100644 modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py rename modin/{engines/ray/pandas_on_ray => core/execution/ray/implementations/pandas_on_ray/io}/io.py (81%) rename modin/{engines/ray/pandas_on_ray/frame => core/execution/ray/implementations/pandas_on_ray}/modin_aqp.py (100%) create mode 100644 modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py rename modin/{engines/ray/pandas_on_ray/frame => core/execution/ray/implementations/pandas_on_ray/partitioning}/axis_partition.py (96%) rename modin/{engines/ray/pandas_on_ray/frame => core/execution/ray/implementations/pandas_on_ray/partitioning}/partition.py (98%) rename modin/{engines/ray/pandas_on_ray/frame => core/execution/ray/implementations/pandas_on_ray/partitioning}/partition_manager.py (97%) rename modin/{engines/base => core}/io/__init__.py (54%) rename modin/{engines/base => core}/io/column_stores/__init__.py (100%) rename modin/{engines/base => core}/io/column_stores/column_store_dispatcher.py (98%) rename modin/{engines/base => core}/io/column_stores/feather_dispatcher.py (96%) rename modin/{engines/base => core}/io/column_stores/hdf_dispatcher.py (97%) rename modin/{engines/base => core}/io/column_stores/parquet_dispatcher.py (98%) rename modin/{engines/base => core}/io/file_dispatcher.py (100%) rename modin/{engines/base => core}/io/io.py (99%) rename modin/{engines => core/io/pickle}/__init__.py (100%) rename modin/{engines/base => core}/io/pickle/pickle_dispatcher.py (97%) rename modin/{engines/base => core}/io/sql/__init__.py (100%) rename modin/{engines/base => core}/io/sql/sql_dispatcher.py (98%) rename modin/{engines/base => core}/io/text/__init__.py (100%) rename modin/{engines/base => core}/io/text/csv_dispatcher.py (99%) rename modin/{engines/base => core}/io/text/csv_glob_dispatcher.py (99%) rename modin/{engines/base => core}/io/text/excel_dispatcher.py (99%) rename modin/{engines/base => core}/io/text/fwf_dispatcher.py (99%) rename modin/{engines/base => core}/io/text/json_dispatcher.py (98%) rename modin/{engines/base => core}/io/text/text_file_dispatcher.py (99%) rename modin/{backends => core/storage_formats}/__init__.py (76%) rename modin/{backends => core/storage_formats}/base/__init__.py (92%) rename modin/{backends => core/storage_formats}/base/doc_utils.py (100%) rename modin/{backends => core/storage_formats}/base/query_compiler.py (99%) create mode 100644 modin/core/storage_formats/cudf/__init__.py rename modin/{backends => core/storage_formats}/cudf/parser.py (95%) rename modin/{backends => core/storage_formats}/cudf/query_compiler.py (97%) rename modin/{backends => core/storage_formats}/pandas/__init__.py (91%) rename modin/{backends => core/storage_formats}/pandas/parsers.py (99%) rename modin/{backends => core/storage_formats}/pandas/query_compiler.py (99%) rename modin/{data_management => core/storage_formats/pandas}/utils.py (100%) rename modin/{backends => core/storage_formats}/pyarrow/__init__.py (100%) rename modin/{backends => core/storage_formats}/pyarrow/parsers.py (97%) rename modin/{backends => core/storage_formats}/pyarrow/query_compiler.py (99%) delete mode 100644 modin/experimental/backends/omnisci/__init__.py create mode 100644 modin/experimental/core/__init__.py create mode 100644 modin/experimental/core/execution/__init__.py rename modin/{engines/base => experimental/core/execution/ray}/__init__.py (100%) rename modin/{engines/base/io/pickle => experimental/core/execution/ray/implementations/omnisci_on_ray}/__init__.py (100%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/__init__.py (100%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/calcite_algebra.py (98%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/calcite_builder.py (99%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/calcite_serializer.py (97%) rename modin/experimental/{engines/omnisci_on_ray/frame/data.py => core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py} (99%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/df_algebra.py (99%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/expr.py (100%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/dataframe}/omnisci_worker.py (100%) create mode 100644 modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py rename modin/experimental/{engines/omnisci_on_ray => core/execution/ray/implementations/omnisci_on_ray/io}/io.py (100%) create mode 100644 modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/partitioning}/axis_partition.py (91%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/partitioning}/partition.py (98%) rename modin/experimental/{engines/omnisci_on_ray/frame => core/execution/ray/implementations/omnisci_on_ray/partitioning}/partition_manager.py (94%) rename modin/{engines/python => experimental/core/execution/ray/implementations/omnisci_on_ray/test}/__init__.py (100%) rename modin/experimental/{engines => core/execution/ray/implementations}/omnisci_on_ray/test/test_dataframe.py (100%) rename modin/experimental/{engines => core/execution/ray/implementations}/pandas_on_ray/__init__.py (100%) create mode 100644 modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py rename modin/experimental/{engines/pandas_on_ray/io_exp.py => core/execution/ray/implementations/pandas_on_ray/io/io.py} (95%) rename modin/experimental/{engines/pandas_on_ray => core/execution/ray/implementations/pandas_on_ray/io}/sql.py (100%) rename modin/experimental/{engines => core/execution/ray/implementations}/pyarrow_on_ray/__init__.py (100%) rename modin/experimental/{engines/pyarrow_on_ray/frame => core/execution/ray/implementations/pyarrow_on_ray/dataframe}/__init__.py (100%) rename modin/experimental/{engines/pyarrow_on_ray/frame/data.py => core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py} (94%) create mode 100644 modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py rename modin/experimental/{engines/pyarrow_on_ray => core/execution/ray/implementations/pyarrow_on_ray/io}/io.py (74%) rename modin/{engines/ray => experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning}/__init__.py (100%) rename modin/experimental/{engines/pyarrow_on_ray/frame => core/execution/ray/implementations/pyarrow_on_ray/partitioning}/axis_partition.py (97%) rename modin/experimental/{engines/pyarrow_on_ray/frame => core/execution/ray/implementations/pyarrow_on_ray/partitioning}/partition.py (96%) rename modin/experimental/{engines/pyarrow_on_ray/frame => core/execution/ray/implementations/pyarrow_on_ray/partitioning}/partition_manager.py (82%) create mode 100644 modin/experimental/core/storage_formats/__init__.py create mode 100644 modin/experimental/core/storage_formats/omnisci/__init__.py rename modin/experimental/{backends => core/storage_formats}/omnisci/query_compiler.py (99%) delete mode 100644 modin/experimental/engines/__init__.py delete mode 100644 modin/experimental/engines/omnisci_on_ray/__init__.py delete mode 100644 modin/experimental/engines/omnisci_on_ray/test/__init__.py rename modin/{ => experimental}/spreadsheet/__init__.py (100%) rename modin/{ => experimental}/spreadsheet/general.py (100%) rename modin/{engines/ray/generic => experimental/spreadsheet/test}/__init__.py (100%) rename modin/{ => experimental}/spreadsheet/test/test_general.py (98%) delete mode 100644 modin/spreadsheet/test/__init__.py delete mode 100644 modin/sql/connection.py diff --git a/modin/conftest.py b/modin/conftest.py index f96a17a24c0..82b3f031bed 100644 --- a/modin/conftest.py +++ b/modin/conftest.py @@ -44,9 +44,14 @@ def _saving_make_api_url(token, _make_api_url=modin.utils._make_api_url): import modin.config # noqa: E402 from modin.config import IsExperimental, TestRayClient # noqa: E402 -from modin.backends import PandasQueryCompiler, BaseQueryCompiler # noqa: E402 -from modin.engines.python.pandas_on_python.io import PandasOnPythonIO # noqa: E402 -from modin.data_management.factories import factories # noqa: E402 +from modin.core.storage_formats import ( + PandasQueryCompiler, + BaseQueryCompiler, +) # noqa: E402 +from modin.core.execution.python.implementations.pandas_on_python.io import ( + PandasOnPythonIO, +) # noqa: E402 +from modin.core.execution.dispatching.factories import factories # noqa: E402 from modin.utils import get_current_backend # noqa: E402 from modin.pandas.test.utils import ( # noqa: E402 _make_csv_file, diff --git a/modin/engines/ray/generic/frame/__init__.py b/modin/core/__init__.py similarity index 95% rename from modin/engines/ray/generic/frame/__init__.py rename to modin/core/__init__.py index cae6413e559..ccdac6cf80d 100644 --- a/modin/engines/ray/generic/frame/__init__.py +++ b/modin/core/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Modin's core functionality.""" diff --git a/modin/experimental/backends/__init__.py b/modin/core/dataframe/__init__.py similarity index 94% rename from modin/experimental/backends/__init__.py rename to modin/core/dataframe/__init__.py index cae6413e559..55e522da120 100644 --- a/modin/experimental/backends/__init__.py +++ b/modin/core/dataframe/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Base Modin Dataframe functionality.""" diff --git a/modin/core/dataframe/algebra/__init__.py b/modin/core/dataframe/algebra/__init__.py new file mode 100644 index 00000000000..d315ac1b758 --- /dev/null +++ b/modin/core/dataframe/algebra/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Modin Dataframe algebra (core operators).""" diff --git a/modin/data_management/functions/default_methods/__init__.py b/modin/core/dataframe/algebra/d2p/__init__.py similarity index 58% rename from modin/data_management/functions/default_methods/__init__.py rename to modin/core/dataframe/algebra/d2p/__init__.py index 082fe553b66..8575590ad20 100644 --- a/modin/data_management/functions/default_methods/__init__.py +++ b/modin/core/dataframe/algebra/d2p/__init__.py @@ -13,17 +13,17 @@ """Function module provides templates for a query compiler default-to-pandas methods.""" -from .dataframe_default import DataFrameDefault -from .datetime_default import DateTimeDefault -from .series_default import SeriesDefault -from .str_default import StrDefault -from .binary_default import BinaryDefault -from .any_default import AnyDefault -from .resample_default import ResampleDefault -from .rolling_default import RollingDefault -from .default import DefaultMethod -from .cat_default import CatDefault -from .groupby_default import GroupByDefault +from modin.core.dataframe.algebra.d2p.dataframe_default import DataFrameDefault +from modin.core.dataframe.algebra.d2p.datetime_default import DateTimeDefault +from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault +from modin.core.dataframe.algebra.d2p.str_default import StrDefault +from modin.core.dataframe.algebra.d2p.binary_default import BinaryDefault +from modin.core.dataframe.algebra.d2p.any_default import AnyDefault +from modin.core.dataframe.algebra.d2p.resample_default import ResampleDefault +from modin.core.dataframe.algebra.d2p.rolling_default import RollingDefault +from modin.core.dataframe.algebra.d2p.default import DefaultMethod +from modin.core.dataframe.algebra.d2p.cat_default import CatDefault +from modin.core.dataframe.algebra.d2p.groupby_default import GroupByDefault __all__ = [ "DataFrameDefault", diff --git a/modin/data_management/functions/default_methods/any_default.py b/modin/core/dataframe/algebra/d2p/any_default.py similarity index 97% rename from modin/data_management/functions/default_methods/any_default.py rename to modin/core/dataframe/algebra/d2p/any_default.py index 80ce5b07f1f..73a2b7eb399 100644 --- a/modin/data_management/functions/default_methods/any_default.py +++ b/modin/core/dataframe/algebra/d2p/any_default.py @@ -15,7 +15,7 @@ """Module houses default functions builder class.""" -from .default import DefaultMethod +from modin.core.dataframe.algebra.d2p.default import DefaultMethod class ObjTypeDeterminer: diff --git a/modin/data_management/functions/default_methods/binary_default.py b/modin/core/dataframe/algebra/d2p/binary_default.py similarity index 97% rename from modin/data_management/functions/default_methods/binary_default.py rename to modin/core/dataframe/algebra/d2p/binary_default.py index e337f7df558..a9b2a2e8ee6 100644 --- a/modin/data_management/functions/default_methods/binary_default.py +++ b/modin/core/dataframe/algebra/d2p/binary_default.py @@ -13,7 +13,7 @@ """Module houses default binary functions builder class.""" -from .any_default import AnyDefault +from modin.core.dataframe.algebra.d2p.any_default import AnyDefault import pandas from pandas.core.dtypes.common import is_list_like diff --git a/modin/data_management/functions/default_methods/cat_default.py b/modin/core/dataframe/algebra/d2p/cat_default.py similarity index 94% rename from modin/data_management/functions/default_methods/cat_default.py rename to modin/core/dataframe/algebra/d2p/cat_default.py index 172fc22502b..e5c9366d7dc 100644 --- a/modin/data_management/functions/default_methods/cat_default.py +++ b/modin/core/dataframe/algebra/d2p/cat_default.py @@ -13,7 +13,7 @@ """Module houses default applied-on-category functions builder class.""" -from .series_default import SeriesDefault +from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault class CatDefault(SeriesDefault): diff --git a/modin/data_management/functions/default_methods/dataframe_default.py b/modin/core/dataframe/algebra/d2p/dataframe_default.py similarity index 96% rename from modin/data_management/functions/default_methods/dataframe_default.py rename to modin/core/dataframe/algebra/d2p/dataframe_default.py index 95746d4b580..05bc87ee85d 100644 --- a/modin/data_management/functions/default_methods/dataframe_default.py +++ b/modin/core/dataframe/algebra/d2p/dataframe_default.py @@ -15,7 +15,7 @@ # FIXME: This whole module is duplicating the logic of `default.py` and should be removed. -from .default import DefaultMethod +from modin.core.dataframe.algebra.d2p.default import DefaultMethod from modin.utils import _inherit_docstrings import pandas diff --git a/modin/data_management/functions/default_methods/datetime_default.py b/modin/core/dataframe/algebra/d2p/datetime_default.py similarity index 94% rename from modin/data_management/functions/default_methods/datetime_default.py rename to modin/core/dataframe/algebra/d2p/datetime_default.py index c81a655012e..17879d1cc1e 100644 --- a/modin/data_management/functions/default_methods/datetime_default.py +++ b/modin/core/dataframe/algebra/d2p/datetime_default.py @@ -13,7 +13,7 @@ """Module houses default applied-on-datetime functions builder class.""" -from .series_default import SeriesDefault +from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault class DateTimeDefault(SeriesDefault): diff --git a/modin/data_management/functions/default_methods/default.py b/modin/core/dataframe/algebra/d2p/default.py similarity index 99% rename from modin/data_management/functions/default_methods/default.py rename to modin/core/dataframe/algebra/d2p/default.py index 43e6b99f95e..88b60153f59 100644 --- a/modin/data_management/functions/default_methods/default.py +++ b/modin/core/dataframe/algebra/d2p/default.py @@ -13,7 +13,7 @@ """Module houses default functions builder class.""" -from modin.data_management.functions.function import Function +from modin.core.dataframe.algebra.operators.function import Function from modin.utils import try_cast_to_pandas from pandas.core.dtypes.common import is_list_like diff --git a/modin/data_management/functions/default_methods/groupby_default.py b/modin/core/dataframe/algebra/d2p/groupby_default.py similarity index 99% rename from modin/data_management/functions/default_methods/groupby_default.py rename to modin/core/dataframe/algebra/d2p/groupby_default.py index 4999283e18f..6d56532eb7e 100644 --- a/modin/data_management/functions/default_methods/groupby_default.py +++ b/modin/core/dataframe/algebra/d2p/groupby_default.py @@ -13,7 +13,7 @@ """Module houses default GroupBy functions builder class.""" -from .default import DefaultMethod +from modin.core.dataframe.algebra.d2p.default import DefaultMethod import pandas diff --git a/modin/data_management/functions/default_methods/resample_default.py b/modin/core/dataframe/algebra/d2p/resample_default.py similarity index 97% rename from modin/data_management/functions/default_methods/resample_default.py rename to modin/core/dataframe/algebra/d2p/resample_default.py index b047a08d840..f455e02fec5 100644 --- a/modin/data_management/functions/default_methods/resample_default.py +++ b/modin/core/dataframe/algebra/d2p/resample_default.py @@ -13,7 +13,7 @@ """Module houses default Resamle functions builder class.""" -from .default import DefaultMethod +from modin.core.dataframe.algebra.d2p.default import DefaultMethod # FIXME: there is no sence of keeping `Resampler` and `ResampleDefault` logic in a different diff --git a/modin/data_management/functions/default_methods/rolling_default.py b/modin/core/dataframe/algebra/d2p/rolling_default.py similarity index 97% rename from modin/data_management/functions/default_methods/rolling_default.py rename to modin/core/dataframe/algebra/d2p/rolling_default.py index 4c3debadc6c..6d84a810d20 100644 --- a/modin/data_management/functions/default_methods/rolling_default.py +++ b/modin/core/dataframe/algebra/d2p/rolling_default.py @@ -13,7 +13,7 @@ """Module houses default Rolling functions builder class.""" -from .default import DefaultMethod +from modin.core.dataframe.algebra.d2p.default import DefaultMethod # FIXME: there is no sence of keeping `Rolling` and `RollingDefault` logic in a different diff --git a/modin/data_management/functions/default_methods/series_default.py b/modin/core/dataframe/algebra/d2p/series_default.py similarity index 95% rename from modin/data_management/functions/default_methods/series_default.py rename to modin/core/dataframe/algebra/d2p/series_default.py index 16514967951..449521ad27a 100644 --- a/modin/data_management/functions/default_methods/series_default.py +++ b/modin/core/dataframe/algebra/d2p/series_default.py @@ -13,7 +13,7 @@ """Module houses default Series functions builder class.""" -from .any_default import AnyDefault +from modin.core.dataframe.algebra.d2p.any_default import AnyDefault class SeriesDefault(AnyDefault): diff --git a/modin/data_management/functions/default_methods/str_default.py b/modin/core/dataframe/algebra/d2p/str_default.py similarity index 94% rename from modin/data_management/functions/default_methods/str_default.py rename to modin/core/dataframe/algebra/d2p/str_default.py index 74e744f8d76..6e9a32c004d 100644 --- a/modin/data_management/functions/default_methods/str_default.py +++ b/modin/core/dataframe/algebra/d2p/str_default.py @@ -13,7 +13,7 @@ """Module houses default applied-on-str functions builder class.""" -from .series_default import SeriesDefault +from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault class StrDefault(SeriesDefault): diff --git a/modin/data_management/functions/__init__.py b/modin/core/dataframe/algebra/operators/__init__.py similarity index 64% rename from modin/data_management/functions/__init__.py rename to modin/core/dataframe/algebra/operators/__init__.py index c546ae7400f..ccc3b6402cd 100644 --- a/modin/data_management/functions/__init__.py +++ b/modin/core/dataframe/algebra/operators/__init__.py @@ -13,13 +13,16 @@ """Function module provides template for a query compiler methods for a set of common operations.""" -from .function import Function -from .mapfunction import MapFunction -from .mapreducefunction import MapReduceFunction -from .reductionfunction import ReductionFunction -from .foldfunction import FoldFunction -from .binary_function import BinaryFunction -from .groupby_function import GroupbyReduceFunction, groupby_reduce_functions +from modin.core.dataframe.algebra.operators.function import Function +from modin.core.dataframe.algebra.operators.mapfunction import MapFunction +from modin.core.dataframe.algebra.operators.mapreducefunction import MapReduceFunction +from modin.core.dataframe.algebra.operators.reductionfunction import ReductionFunction +from modin.core.dataframe.algebra.operators.foldfunction import FoldFunction +from modin.core.dataframe.algebra.operators.binary_function import BinaryFunction +from modin.core.dataframe.algebra.operators.groupby_function import ( + GroupbyReduceFunction, + groupby_reduce_functions, +) __all__ = [ "Function", diff --git a/modin/data_management/functions/binary_function.py b/modin/core/dataframe/algebra/operators/binary_function.py similarity index 98% rename from modin/data_management/functions/binary_function.py rename to modin/core/dataframe/algebra/operators/binary_function.py index 975f714b33d..cc6bbd6d94b 100644 --- a/modin/data_management/functions/binary_function.py +++ b/modin/core/dataframe/algebra/operators/binary_function.py @@ -16,7 +16,7 @@ import numpy as np import pandas -from .function import Function +from modin.core.dataframe.algebra.operators.function import Function class BinaryFunction(Function): diff --git a/modin/data_management/functions/foldfunction.py b/modin/core/dataframe/algebra/operators/foldfunction.py similarity index 96% rename from modin/data_management/functions/foldfunction.py rename to modin/core/dataframe/algebra/operators/foldfunction.py index 077a7a6b46b..58f4f422214 100644 --- a/modin/data_management/functions/foldfunction.py +++ b/modin/core/dataframe/algebra/operators/foldfunction.py @@ -13,7 +13,7 @@ """Module houses Fold functions builder class.""" -from .function import Function +from modin.core.dataframe.algebra.operators.function import Function class FoldFunction(Function): diff --git a/modin/data_management/functions/function.py b/modin/core/dataframe/algebra/operators/function.py similarity index 100% rename from modin/data_management/functions/function.py rename to modin/core/dataframe/algebra/operators/function.py diff --git a/modin/data_management/functions/groupby_function.py b/modin/core/dataframe/algebra/operators/groupby_function.py similarity index 99% rename from modin/data_management/functions/groupby_function.py rename to modin/core/dataframe/algebra/operators/groupby_function.py index c4046cc0058..5794cfb5962 100644 --- a/modin/data_management/functions/groupby_function.py +++ b/modin/core/dataframe/algebra/operators/groupby_function.py @@ -15,7 +15,7 @@ import pandas -from .mapreducefunction import MapReduceFunction +from modin.core.dataframe.algebra.operators.mapreducefunction import MapReduceFunction from modin.utils import try_cast_to_pandas, hashable diff --git a/modin/data_management/functions/mapfunction.py b/modin/core/dataframe/algebra/operators/mapfunction.py similarity index 96% rename from modin/data_management/functions/mapfunction.py rename to modin/core/dataframe/algebra/operators/mapfunction.py index 134b05aea0a..576c0b1a9eb 100644 --- a/modin/data_management/functions/mapfunction.py +++ b/modin/core/dataframe/algebra/operators/mapfunction.py @@ -13,7 +13,7 @@ """Module houses Map functions builder class.""" -from .function import Function +from modin.core.dataframe.algebra.operators.function import Function class MapFunction(Function): diff --git a/modin/data_management/functions/mapreducefunction.py b/modin/core/dataframe/algebra/operators/mapreducefunction.py similarity index 97% rename from modin/data_management/functions/mapreducefunction.py rename to modin/core/dataframe/algebra/operators/mapreducefunction.py index 16364d5fdb2..9d95bf7c5d1 100644 --- a/modin/data_management/functions/mapreducefunction.py +++ b/modin/core/dataframe/algebra/operators/mapreducefunction.py @@ -13,7 +13,7 @@ """Module houses MapReduce functions builder class.""" -from .function import Function +from modin.core.dataframe.algebra.operators.function import Function class MapReduceFunction(Function): diff --git a/modin/data_management/functions/reductionfunction.py b/modin/core/dataframe/algebra/operators/reductionfunction.py similarity index 96% rename from modin/data_management/functions/reductionfunction.py rename to modin/core/dataframe/algebra/operators/reductionfunction.py index 3843caeb4db..e30780f3865 100644 --- a/modin/data_management/functions/reductionfunction.py +++ b/modin/core/dataframe/algebra/operators/reductionfunction.py @@ -13,7 +13,7 @@ """Module houses Reduction functions builder class.""" -from .function import Function +from modin.core.dataframe.algebra.operators.function import Function class ReductionFunction(Function): diff --git a/modin/engines/ray/pandas_on_ray/frame/__init__.py b/modin/core/dataframe/base/__init__.py similarity index 95% rename from modin/engines/ray/pandas_on_ray/frame/__init__.py rename to modin/core/dataframe/base/__init__.py index cae6413e559..fddab056042 100644 --- a/modin/engines/ray/pandas_on_ray/frame/__init__.py +++ b/modin/core/dataframe/base/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Base Modin Dataframe classes.""" diff --git a/modin/engines/ray/pandas_on_ray/__init__.py b/modin/core/dataframe/base/dataframe/__init__.py similarity index 95% rename from modin/engines/ray/pandas_on_ray/__init__.py rename to modin/core/dataframe/base/dataframe/__init__.py index cae6413e559..cf62f44e1f5 100644 --- a/modin/engines/ray/pandas_on_ray/__init__.py +++ b/modin/core/dataframe/base/dataframe/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Base Modin Dataframe class.""" diff --git a/modin/core/dataframe/base/partitioning/__init__.py b/modin/core/dataframe/base/partitioning/__init__.py new file mode 100644 index 00000000000..a7992787753 --- /dev/null +++ b/modin/core/dataframe/base/partitioning/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Base Modin Dataframe classes related to its partitioning.""" diff --git a/modin/engines/base/README.md b/modin/core/dataframe/pandas/README.md similarity index 57% rename from modin/engines/base/README.md rename to modin/core/dataframe/pandas/README.md index 4376506bf58..522bf4822ff 100644 --- a/modin/engines/base/README.md +++ b/modin/core/dataframe/pandas/README.md @@ -2,10 +2,10 @@ ### Object Hierarchy -- `frame/partition.py` contains `PandasFramePartition` interface and its implementations. -- `frame/partition_manager.py` contains `PandasFramePartitionManager` interface and its implementations. +- `partitioning/partition.py` contains `PandasFramePartition` interface and its implementations. +- `partitioning/partition_manager.py` contains `PandasFramePartitionManager` interface and its implementations. - `PandasFramePartitionManager` manages 2D-array of `PandasFramePartition` object -- `frame/axis_partition.py` contains `BaseFrameAxisPartition` and with the following hierarchy: +- `partitioning/axis_partition.py` contains `BaseFrameAxisPartition` and with the following hierarchy: ``` BaseFrameAxisPartition -> PandasOnRayFrameAxisPartition -> {PandasOnRayFrameColumnPartition, PandasOnRayFrameRowPartition} ``` diff --git a/modin/engines/base/frame/__init__.py b/modin/core/dataframe/pandas/__init__.py similarity index 91% rename from modin/engines/base/frame/__init__.py rename to modin/core/dataframe/pandas/__init__.py index 4035880c987..5a2add6dc81 100644 --- a/modin/engines/base/frame/__init__.py +++ b/modin/core/dataframe/pandas/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Package holds key base classes to making efficient scale of data.""" +"""Base Modin Dataframe classes designed for pandas storage format.""" diff --git a/modin/core/dataframe/pandas/dataframe/__init__.py b/modin/core/dataframe/pandas/dataframe/__init__.py new file mode 100644 index 00000000000..556229b52a4 --- /dev/null +++ b/modin/core/dataframe/pandas/dataframe/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Base Modin Dataframe class designed for pandas storage format.""" diff --git a/modin/engines/base/frame/data.py b/modin/core/dataframe/pandas/dataframe/dataframe.py similarity index 99% rename from modin/engines/base/frame/data.py rename to modin/core/dataframe/pandas/dataframe/dataframe.py index ead5418c9f3..fc69bd788c5 100644 --- a/modin/engines/base/frame/data.py +++ b/modin/core/dataframe/pandas/dataframe/dataframe.py @@ -24,9 +24,11 @@ from pandas.core.dtypes.common import is_numeric_dtype from typing import List, Hashable -from modin.backends.pandas.query_compiler import PandasQueryCompiler +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler from modin.error_message import ErrorMessage -from modin.backends.pandas.parsers import find_common_type_cat as find_common_type +from modin.core.storage_formats.pandas.parsers import ( + find_common_type_cat as find_common_type, +) class PandasFrame(object): diff --git a/modin/core/dataframe/pandas/partitioning/__init__.py b/modin/core/dataframe/pandas/partitioning/__init__.py new file mode 100644 index 00000000000..f0d5bc50a65 --- /dev/null +++ b/modin/core/dataframe/pandas/partitioning/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Base Modin Dataframe classes related to its partitioning designed for pandas storage format.""" diff --git a/modin/engines/base/frame/axis_partition.py b/modin/core/dataframe/pandas/partitioning/axis_partition.py similarity index 99% rename from modin/engines/base/frame/axis_partition.py rename to modin/core/dataframe/pandas/partitioning/axis_partition.py index 57a2f508651..95b7aa2a2e9 100644 --- a/modin/engines/base/frame/axis_partition.py +++ b/modin/core/dataframe/pandas/partitioning/axis_partition.py @@ -16,7 +16,7 @@ from abc import ABC import pandas import numpy as np -from modin.data_management.utils import split_result_of_axis_func_pandas +from modin.core.storage_formats.pandas.utils import split_result_of_axis_func_pandas class BaseFrameAxisPartition(ABC): # pragma: no cover diff --git a/modin/engines/base/frame/partition.py b/modin/core/dataframe/pandas/partitioning/partition.py similarity index 100% rename from modin/engines/base/frame/partition.py rename to modin/core/dataframe/pandas/partitioning/partition.py diff --git a/modin/engines/base/frame/partition_manager.py b/modin/core/dataframe/pandas/partitioning/partition_manager.py similarity index 99% rename from modin/engines/base/frame/partition_manager.py rename to modin/core/dataframe/pandas/partitioning/partition_manager.py index 7c0f2dd02c1..3719cf8357d 100644 --- a/modin/engines/base/frame/partition_manager.py +++ b/modin/core/dataframe/pandas/partitioning/partition_manager.py @@ -24,7 +24,7 @@ import warnings from modin.error_message import ErrorMessage -from modin.data_management.utils import compute_chunksize +from modin.core.storage_formats.pandas.utils import compute_chunksize from modin.config import NPartitions, ProgressBar, BenchmarkMode from pandas.api.types import union_categoricals diff --git a/modin/core/execution/__init__.py b/modin/core/execution/__init__.py new file mode 100644 index 00000000000..893ed97fd73 --- /dev/null +++ b/modin/core/execution/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Modin's functionality related to execution engines supported.""" diff --git a/modin/engines/dask/__init__.py b/modin/core/execution/dask/__init__.py similarity index 100% rename from modin/engines/dask/__init__.py rename to modin/core/execution/dask/__init__.py diff --git a/modin/core/execution/dask/common/__init__.py b/modin/core/execution/dask/common/__init__.py new file mode 100644 index 00000000000..55121e95ee5 --- /dev/null +++ b/modin/core/execution/dask/common/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Common utilities for Dask execution engine.""" diff --git a/modin/engines/dask/task_wrapper.py b/modin/core/execution/dask/common/task_wrapper.py similarity index 100% rename from modin/engines/dask/task_wrapper.py rename to modin/core/execution/dask/common/task_wrapper.py diff --git a/modin/engines/dask/utils.py b/modin/core/execution/dask/common/utils.py similarity index 100% rename from modin/engines/dask/utils.py rename to modin/core/execution/dask/common/utils.py diff --git a/modin/core/execution/dask/implementations/__init__.py b/modin/core/execution/dask/implementations/__init__.py new file mode 100644 index 00000000000..ac0f9b75e15 --- /dev/null +++ b/modin/core/execution/dask/implementations/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" diff --git a/modin/engines/dask/pandas_on_dask/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/__init__.py similarity index 100% rename from modin/engines/dask/pandas_on_dask/__init__.py rename to modin/core/execution/dask/implementations/pandas_on_dask/__init__.py diff --git a/modin/engines/dask/pandas_on_dask/frame/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/__init__.py similarity index 100% rename from modin/engines/dask/pandas_on_dask/frame/__init__.py rename to modin/core/execution/dask/implementations/pandas_on_dask/dataframe/__init__.py diff --git a/modin/engines/dask/pandas_on_dask/frame/data.py b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py similarity index 92% rename from modin/engines/dask/pandas_on_dask/frame/data.py rename to modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py index a8475fdd9b3..be4385331c0 100644 --- a/modin/engines/dask/pandas_on_dask/frame/data.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py @@ -13,8 +13,10 @@ """Module houses class that implements ``PandasFrame``.""" -from modin.engines.base.frame.data import PandasFrame -from .partition_manager import PandasOnDaskFramePartitionManager +from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame +from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition_manager import ( + PandasOnDaskFramePartitionManager, +) from distributed.client import default_client diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py new file mode 100644 index 00000000000..c5aed1a1599 --- /dev/null +++ b/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py @@ -0,0 +1,20 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" + +from modin.core.execution.dask.implementations.pandas_on_dask.io import PandasOnDaskIO + +__all__ = [ + "PandasOnDaskIO", +] diff --git a/modin/engines/dask/pandas_on_dask/io.py b/modin/core/execution/dask/implementations/pandas_on_dask/io/io.py similarity index 81% rename from modin/engines/dask/pandas_on_dask/io.py rename to modin/core/execution/dask/implementations/pandas_on_dask/io/io.py index a0bfda112ea..49f74b46a1f 100644 --- a/modin/engines/dask/pandas_on_dask/io.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/io/io.py @@ -13,11 +13,15 @@ """Module houses class that implements ``BaseIO`` using Dask as an execution engine.""" -from modin.engines.base.io import BaseIO -from modin.backends.pandas.query_compiler import PandasQueryCompiler -from modin.engines.dask.pandas_on_dask.frame.data import PandasOnDaskFrame -from modin.engines.dask.pandas_on_dask.frame.partition import PandasOnDaskFramePartition -from modin.engines.base.io import ( +from modin.core.io import BaseIO +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler +from modin.core.execution.dask.implementations.pandas_on_dask.dataframe.dataframe import ( + PandasOnDaskFrame, +) +from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition import ( + PandasOnDaskFramePartition, +) +from modin.core.io import ( CSVDispatcher, JSONDispatcher, ParquetDispatcher, @@ -25,7 +29,7 @@ SQLDispatcher, ExcelDispatcher, ) -from modin.backends.pandas.parsers import ( +from modin.core.storage_formats.pandas.parsers import ( PandasCSVParser, PandasJSONParser, PandasParquetParser, @@ -33,7 +37,7 @@ PandasSQLParser, PandasExcelParser, ) -from modin.engines.dask.task_wrapper import DaskTask +from modin.core.execution.dask.common.task_wrapper import DaskTask class PandasOnDaskIO(BaseIO): diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py new file mode 100644 index 00000000000..ac0f9b75e15 --- /dev/null +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" diff --git a/modin/engines/dask/pandas_on_dask/frame/axis_partition.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py similarity index 97% rename from modin/engines/dask/pandas_on_dask/frame/axis_partition.py rename to modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py index 727db68e252..7c77c3601af 100644 --- a/modin/engines/dask/pandas_on_dask/frame/axis_partition.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py @@ -13,8 +13,12 @@ """Module houses classes responsible for storing an axis partition and applying a function to it.""" -from modin.engines.base.frame.axis_partition import PandasFrameAxisPartition -from .partition import PandasOnDaskFramePartition +from modin.core.dataframe.pandas.partitioning.axis_partition import ( + PandasFrameAxisPartition, +) +from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition import ( + PandasOnDaskFramePartition, +) from distributed.client import default_client from distributed import Future diff --git a/modin/engines/dask/pandas_on_dask/frame/partition.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py similarity index 98% rename from modin/engines/dask/pandas_on_dask/frame/partition.py rename to modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py index ed6b8781d68..b3a10664bb9 100644 --- a/modin/engines/dask/pandas_on_dask/frame/partition.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition.py @@ -15,8 +15,8 @@ import pandas -from modin.data_management.utils import length_fn_pandas, width_fn_pandas -from modin.engines.base.frame.partition import PandasFramePartition +from modin.core.storage_formats.pandas.utils import length_fn_pandas, width_fn_pandas +from modin.core.dataframe.pandas.partitioning.partition import PandasFramePartition from distributed.client import default_client from distributed import Future diff --git a/modin/engines/dask/pandas_on_dask/frame/partition_manager.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py similarity index 93% rename from modin/engines/dask/pandas_on_dask/frame/partition_manager.py rename to modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py index 70ac53c2d94..8031d30970c 100644 --- a/modin/engines/dask/pandas_on_dask/frame/partition_manager.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py @@ -15,12 +15,16 @@ import numpy as np -from modin.engines.base.frame.partition_manager import PandasFramePartitionManager -from .axis_partition import ( +from modin.core.dataframe.pandas.partitioning.partition_manager import ( + PandasFramePartitionManager, +) +from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.axis_partition import ( PandasOnDaskFrameColumnPartition, PandasOnDaskFrameRowPartition, ) -from .partition import PandasOnDaskFramePartition +from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition import ( + PandasOnDaskFramePartition, +) from modin.error_message import ErrorMessage import pandas diff --git a/modin/core/execution/dispatching/__init__.py b/modin/core/execution/dispatching/__init__.py new file mode 100644 index 00000000000..28756e6c759 --- /dev/null +++ b/modin/core/execution/dispatching/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Modin's functionality related to dispatching to specific execution.""" diff --git a/modin/data_management/factories/__init__.py b/modin/core/execution/dispatching/factories/__init__.py similarity index 90% rename from modin/data_management/factories/__init__.py rename to modin/core/execution/dispatching/factories/__init__.py index faac60edcc4..bdcad80fae7 100644 --- a/modin/data_management/factories/__init__.py +++ b/modin/core/execution/dispatching/factories/__init__.py @@ -11,7 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -from . import factories +"""Factories responsible for dispatching to specific execution.""" + +from modin.core.execution.dispatching import factories def _get_remote_engines(): diff --git a/modin/data_management/factories/dispatcher.py b/modin/core/execution/dispatching/factories/dispatcher.py similarity index 99% rename from modin/data_management/factories/dispatcher.py rename to modin/core/execution/dispatching/factories/dispatcher.py index 8579485ce60..4c1963c54bf 100644 --- a/modin/data_management/factories/dispatcher.py +++ b/modin/core/execution/dispatching/factories/dispatcher.py @@ -18,7 +18,7 @@ """ from modin.config import Engine, Backend, IsExperimental -from modin.data_management.factories import factories +from modin.core.execution.dispatching.factories import factories from modin.utils import get_current_backend, _inherit_docstrings diff --git a/modin/data_management/factories/factories.py b/modin/core/execution/dispatching/factories/factories.py similarity index 94% rename from modin/data_management/factories/factories.py rename to modin/core/execution/dispatching/factories/factories.py index 3664cff120f..ef087248b36 100644 --- a/modin/data_management/factories/factories.py +++ b/modin/core/execution/dispatching/factories/factories.py @@ -25,7 +25,7 @@ from modin.config import Engine from modin.utils import _inherit_docstrings -from modin.engines.base.io import BaseIO +from modin.core.io import BaseIO from pandas.util._decorators import doc import pandas @@ -403,7 +403,7 @@ class CudfOnRayFactory(BaseFactory): @classmethod @doc(_doc_factory_prepare_method, io_module_name="``cuDFOnRayIO``") def prepare(cls): - from modin.engines.ray.cudf_on_ray.io import cuDFOnRayIO + from modin.core.execution.ray.implementations.cudf_on_ray.io import cuDFOnRayIO cls.io_cls = cuDFOnRayIO @@ -413,7 +413,9 @@ class PandasOnRayFactory(BaseFactory): @classmethod @doc(_doc_factory_prepare_method, io_module_name="``PandasOnRayIO``") def prepare(cls): - from modin.engines.ray.pandas_on_ray.io import PandasOnRayIO + from modin.core.execution.ray.implementations.pandas_on_ray.io import ( + PandasOnRayIO, + ) cls.io_cls = PandasOnRayIO @@ -423,7 +425,9 @@ class PandasOnPythonFactory(BaseFactory): @classmethod @doc(_doc_factory_prepare_method, io_module_name="``PandasOnPythonIO``") def prepare(cls): - from modin.engines.python.pandas_on_python.io import PandasOnPythonIO + from modin.core.execution.python.implementations.pandas_on_python.io import ( + PandasOnPythonIO, + ) cls.io_cls = PandasOnPythonIO @@ -433,7 +437,9 @@ class PandasOnDaskFactory(BaseFactory): @classmethod @doc(_doc_factory_prepare_method, io_module_name="``PandasOnDaskIO``") def prepare(cls): - from modin.engines.dask.pandas_on_dask.io import PandasOnDaskIO + from modin.core.execution.dask.implementations.pandas_on_dask.io import ( + PandasOnDaskIO, + ) cls.io_cls = PandasOnDaskIO @@ -476,7 +482,7 @@ class ExperimentalPandasOnRayFactory(ExperimentalBaseFactory, PandasOnRayFactory @classmethod @doc(_doc_factory_prepare_method, io_module_name="``ExperimentalPandasOnRayIO``") def prepare(cls): - from modin.experimental.engines.pandas_on_ray.io_exp import ( + from modin.experimental.core.execution.ray.implementations.pandas_on_ray.io import ( ExperimentalPandasOnRayIO, ) @@ -525,7 +531,9 @@ class ExperimentalPyarrowOnRayFactory(BaseFactory): # pragma: no cover @classmethod @doc(_doc_factory_prepare_method, io_module_name="experimental ``PyarrowOnRayIO``") def prepare(cls): - from modin.experimental.engines.pyarrow_on_ray.io import PyarrowOnRayIO + from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.io import ( + PyarrowOnRayIO, + ) cls.io_cls = PyarrowOnRayIO @@ -542,7 +550,7 @@ def prepare(cls): # if there isn't such counterpart rpyc generates some bogus # class type which raises TypeError() # upon checking its isinstance() or issubclass() - import modin.backends.pandas.query_compiler # noqa: F401 + import modin.core.storage_formats.pandas.query_compiler # noqa: F401 from modin.experimental.cloud import get_connection # import a numpy overrider if it wasn't already imported @@ -594,7 +602,9 @@ class ExperimentalOmnisciOnRayFactory(BaseFactory): @classmethod @doc(_doc_factory_prepare_method, io_module_name="experimental ``OmnisciOnRayIO``") def prepare(cls): - from modin.experimental.engines.omnisci_on_ray.io import OmnisciOnRayIO + from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.io import ( + OmnisciOnRayIO, + ) cls.io_cls = OmnisciOnRayIO diff --git a/modin/backends/cudf/__init__.py b/modin/core/execution/dispatching/factories/test/__init__.py similarity index 100% rename from modin/backends/cudf/__init__.py rename to modin/core/execution/dispatching/factories/test/__init__.py diff --git a/modin/data_management/factories/test/test_dispatcher.py b/modin/core/execution/dispatching/factories/test/test_dispatcher.py similarity index 100% rename from modin/data_management/factories/test/test_dispatcher.py rename to modin/core/execution/dispatching/factories/test/test_dispatcher.py diff --git a/modin/core/execution/python/__init__.py b/modin/core/execution/python/__init__.py new file mode 100644 index 00000000000..d7466faa6f9 --- /dev/null +++ b/modin/core/execution/python/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Key base classes to making efficient scale of data with ``Python`` as an execution engine.""" diff --git a/modin/core/execution/python/implementations/__init__.py b/modin/core/execution/python/implementations/__init__.py new file mode 100644 index 00000000000..d7466faa6f9 --- /dev/null +++ b/modin/core/execution/python/implementations/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Key base classes to making efficient scale of data with ``Python`` as an execution engine.""" diff --git a/modin/engines/python/pandas_on_python/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/__init__.py similarity index 100% rename from modin/engines/python/pandas_on_python/__init__.py rename to modin/core/execution/python/implementations/pandas_on_python/__init__.py diff --git a/modin/engines/python/pandas_on_python/frame/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/dataframe/__init__.py similarity index 100% rename from modin/engines/python/pandas_on_python/frame/__init__.py rename to modin/core/execution/python/implementations/pandas_on_python/dataframe/__init__.py diff --git a/modin/engines/python/pandas_on_python/frame/data.py b/modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py similarity index 89% rename from modin/engines/python/pandas_on_python/frame/data.py rename to modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py index 6c3fb0664b8..79e859c9c98 100644 --- a/modin/engines/python/pandas_on_python/frame/data.py +++ b/modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py @@ -17,8 +17,10 @@ ``PandasOnPythonFrame`` is dataframe class with pandas backend and Python engine. """ -from modin.engines.base.frame.data import PandasFrame -from .partition_manager import PandasOnPythonFramePartitionManager +from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame +from modin.core.execution.python.implementations.pandas_on_python.partitioning.partition_manager import ( + PandasOnPythonFramePartitionManager, +) class PandasOnPythonFrame(PandasFrame): diff --git a/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py new file mode 100644 index 00000000000..c2ecb0cff2e --- /dev/null +++ b/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py @@ -0,0 +1,22 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Functionality implementations with pandas backend and Python engine.""" + +from modin.core.execution.python.implementations.pandas_on_python.io import ( + PandasOnPythonIO, +) + +__all__ = [ + "PandasOnPythonIO", +] diff --git a/modin/engines/python/pandas_on_python/io.py b/modin/core/execution/python/implementations/pandas_on_python/io/io.py similarity index 82% rename from modin/engines/python/pandas_on_python/io.py rename to modin/core/execution/python/implementations/pandas_on_python/io/io.py index 7f3f7867953..0596906fe8b 100644 --- a/modin/engines/python/pandas_on_python/io.py +++ b/modin/core/execution/python/implementations/pandas_on_python/io/io.py @@ -13,9 +13,11 @@ """Module for housing IO classes with pandas backend and Python engine.""" -from modin.engines.base.io import BaseIO -from modin.backends.pandas.query_compiler import PandasQueryCompiler -from modin.engines.python.pandas_on_python.frame.data import PandasOnPythonFrame +from modin.core.io import BaseIO +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler +from modin.core.execution.python.implementations.pandas_on_python.dataframe.dataframe import ( + PandasOnPythonFrame, +) class PandasOnPythonIO(BaseIO): diff --git a/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py new file mode 100644 index 00000000000..b6600935d72 --- /dev/null +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Functionality implementations with pandas backend and Python engine.""" diff --git a/modin/engines/python/pandas_on_python/frame/axis_partition.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py similarity index 91% rename from modin/engines/python/pandas_on_python/frame/axis_partition.py rename to modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py index 50758933998..5d52b21cd26 100644 --- a/modin/engines/python/pandas_on_python/frame/axis_partition.py +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py @@ -15,8 +15,12 @@ import pandas -from modin.engines.base.frame.axis_partition import PandasFrameAxisPartition -from .partition import PandasOnPythonFramePartition +from modin.core.dataframe.pandas.partitioning.axis_partition import ( + PandasFrameAxisPartition, +) +from modin.core.execution.python.implementations.pandas_on_python.partitioning.partition import ( + PandasOnPythonFramePartition, +) class PandasOnPythonFrameAxisPartition(PandasFrameAxisPartition): diff --git a/modin/engines/python/pandas_on_python/frame/partition.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py similarity index 98% rename from modin/engines/python/pandas_on_python/frame/partition.py rename to modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py index 0aee2e43294..1faecb1e6d0 100644 --- a/modin/engines/python/pandas_on_python/frame/partition.py +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition.py @@ -15,8 +15,8 @@ import pandas -from modin.data_management.utils import length_fn_pandas, width_fn_pandas -from modin.engines.base.frame.partition import PandasFramePartition +from modin.core.storage_formats.pandas.utils import length_fn_pandas, width_fn_pandas +from modin.core.dataframe.pandas.partitioning.partition import PandasFramePartition class PandasOnPythonFramePartition(PandasFramePartition): diff --git a/modin/engines/python/pandas_on_python/frame/partition_manager.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py similarity index 79% rename from modin/engines/python/pandas_on_python/frame/partition_manager.py rename to modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py index 554938eaeef..4ba7aaa2072 100644 --- a/modin/engines/python/pandas_on_python/frame/partition_manager.py +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py @@ -13,12 +13,16 @@ """Module houses class for managing partitions with pandas backend and Python engine.""" -from modin.engines.base.frame.partition_manager import PandasFramePartitionManager -from .axis_partition import ( +from modin.core.dataframe.pandas.partitioning.partition_manager import ( + PandasFramePartitionManager, +) +from modin.core.execution.python.implementations.pandas_on_python.partitioning.axis_partition import ( PandasOnPythonFrameColumnPartition, PandasOnPythonFrameRowPartition, ) -from .partition import PandasOnPythonFramePartition +from modin.core.execution.python.implementations.pandas_on_python.partitioning.partition import ( + PandasOnPythonFramePartition, +) class PandasOnPythonFramePartitionManager(PandasFramePartitionManager): diff --git a/modin/core/execution/ray/__init__.py b/modin/core/execution/ray/__init__.py new file mode 100644 index 00000000000..721a6ac6dde --- /dev/null +++ b/modin/core/execution/ray/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Modin's functionality related to Ray execution engine.""" diff --git a/modin/core/execution/ray/common/__init__.py b/modin/core/execution/ray/common/__init__.py new file mode 100644 index 00000000000..348a3436f59 --- /dev/null +++ b/modin/core/execution/ray/common/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Common utilities for Ray execution engine.""" diff --git a/modin/engines/ray/task_wrapper.py b/modin/core/execution/ray/common/task_wrapper.py similarity index 100% rename from modin/engines/ray/task_wrapper.py rename to modin/core/execution/ray/common/task_wrapper.py diff --git a/modin/engines/ray/utils.py b/modin/core/execution/ray/common/utils.py similarity index 96% rename from modin/engines/ray/utils.py rename to modin/core/execution/ray/common/utils.py index 44fb3683887..db82c67e167 100644 --- a/modin/engines/ray/utils.py +++ b/modin/core/execution/ray/common/utils.py @@ -174,8 +174,10 @@ def initialize_ray( ray.init(**ray_init_kwargs) if Backend.get() == "Cudf": - from modin.engines.ray.cudf_on_ray.frame.gpu_manager import GPUManager - from modin.engines.ray.cudf_on_ray.frame.partition_manager import ( + from modin.core.execution.ray.implementations.cudf_on_ray.frame.gpu_manager import ( + GPUManager, + ) + from modin.core.execution.ray.implementations.cudf_on_ray.frame.partition_manager import ( GPU_MANAGERS, ) diff --git a/modin/sql/__init__.py b/modin/core/execution/ray/generic/__init__.py similarity index 93% rename from modin/sql/__init__.py rename to modin/core/execution/ray/generic/__init__.py index fc0b2287f70..113208f3e35 100644 --- a/modin/sql/__init__.py +++ b/modin/core/execution/ray/generic/__init__.py @@ -11,6 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -from .connection import connect - -__all__ = ["connect"] +"""Generic functionality for Ray execution engine.""" diff --git a/modin/core/execution/ray/generic/io/__init__.py b/modin/core/execution/ray/generic/io/__init__.py new file mode 100644 index 00000000000..a67ce9089ca --- /dev/null +++ b/modin/core/execution/ray/generic/io/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +from modin.core.execution.ray.generic.io import RayIO + +__all__ = ["RayIO"] diff --git a/modin/engines/ray/generic/io.py b/modin/core/execution/ray/generic/io/io.py similarity index 99% rename from modin/engines/ray/generic/io.py rename to modin/core/execution/ray/generic/io/io.py index 9d3799bdfbc..71bf502419b 100644 --- a/modin/engines/ray/generic/io.py +++ b/modin/core/execution/ray/generic/io/io.py @@ -16,7 +16,7 @@ import io import pandas -from modin.engines.base.io import BaseIO +from modin.core.io import BaseIO from ray.util.queue import Queue from ray import wait diff --git a/modin/core/execution/ray/generic/partitioning/__init__.py b/modin/core/execution/ray/generic/partitioning/__init__.py new file mode 100644 index 00000000000..9e272f9b09e --- /dev/null +++ b/modin/core/execution/ray/generic/partitioning/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Generic partitioning functionality for Ray execution engine.""" diff --git a/modin/engines/ray/generic/frame/partition_manager.py b/modin/core/execution/ray/generic/partitioning/partition_manager.py similarity index 94% rename from modin/engines/ray/generic/frame/partition_manager.py rename to modin/core/execution/ray/generic/partitioning/partition_manager.py index 41da352af4e..64a1702e5f7 100644 --- a/modin/engines/ray/generic/frame/partition_manager.py +++ b/modin/core/execution/ray/generic/partitioning/partition_manager.py @@ -15,7 +15,9 @@ import numpy as np -from modin.engines.base.frame.partition_manager import PandasFramePartitionManager +from modin.core.dataframe.pandas.partitioning.partition_manager import ( + PandasFramePartitionManager, +) import ray diff --git a/modin/core/execution/ray/implementations/__init__.py b/modin/core/execution/ray/implementations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/modin/engines/ray/cudf_on_ray/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/__init__.py similarity index 100% rename from modin/engines/ray/cudf_on_ray/__init__.py rename to modin/core/execution/ray/implementations/cudf_on_ray/__init__.py diff --git a/modin/engines/ray/cudf_on_ray/frame/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/__init__.py similarity index 100% rename from modin/engines/ray/cudf_on_ray/frame/__init__.py rename to modin/core/execution/ray/implementations/cudf_on_ray/dataframe/__init__.py diff --git a/modin/engines/ray/cudf_on_ray/frame/data.py b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py similarity index 97% rename from modin/engines/ray/cudf_on_ray/frame/data.py rename to modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py index d288df5dd06..15f7bf8bbac 100644 --- a/modin/engines/ray/cudf_on_ray/frame/data.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py @@ -16,10 +16,16 @@ import numpy as np import ray -from .partition import cuDFOnRayFramePartition -from .partition_manager import cuDFOnRayFramePartitionManager +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( + cuDFOnRayFramePartition, +) +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition_manager import ( + cuDFOnRayFramePartitionManager, +) -from modin.engines.ray.pandas_on_ray.frame.data import PandasOnRayFrame +from modin.core.execution.ray.implementations.pandas_on_ray.dataframe.dataframe import ( + PandasOnRayFrame, +) from modin.error_message import ErrorMessage diff --git a/modin/engines/ray/cudf_on_ray/io/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py similarity index 82% rename from modin/engines/ray/cudf_on_ray/io/__init__.py rename to modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py index 08159711780..6937ac9606e 100644 --- a/modin/engines/ray/cudf_on_ray/io/__init__.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py @@ -13,8 +13,10 @@ """Package holds implementations of key IO classes using bunch of cuDF and Ray.""" -from modin.engines.ray.cudf_on_ray.io.text.csv_dispatcher import cuDFCSVDispatcher -from modin.engines.ray.cudf_on_ray.io.io import cuDFOnRayIO +from modin.core.execution.ray.implementations.cudf_on_ray.io.text.csv_dispatcher import ( + cuDFCSVDispatcher, +) +from modin.core.execution.ray.implementations.cudf_on_ray.io import cuDFOnRayIO __all__ = [ diff --git a/modin/engines/ray/cudf_on_ray/io/io.py b/modin/core/execution/ray/implementations/cudf_on_ray/io/io.py similarity index 69% rename from modin/engines/ray/cudf_on_ray/io/io.py rename to modin/core/execution/ray/implementations/cudf_on_ray/io/io.py index b3074f1d313..4e1f8e22467 100644 --- a/modin/engines/ray/cudf_on_ray/io/io.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/io/io.py @@ -13,20 +13,22 @@ """Module holds implementation of ``BaseIO`` using cuDF.""" -from modin.engines.base.io import BaseIO -from modin.engines.ray.cudf_on_ray.io import cuDFCSVDispatcher -from modin.backends.cudf.query_compiler import cuDFQueryCompiler -from modin.engines.ray.cudf_on_ray.frame.data import cuDFOnRayFrame -from modin.engines.ray.cudf_on_ray.frame.partition_manager import ( +from modin.core.io import BaseIO +from modin.core.execution.ray.implementations.cudf_on_ray.io import cuDFCSVDispatcher +from modin.core.storage_formats.cudf.query_compiler import cuDFQueryCompiler +from modin.core.execution.ray.implementations.cudf_on_ray.dataframe.dataframe import ( + cuDFOnRayFrame, +) +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition_manager import ( cuDFOnRayFramePartitionManager, ) -from modin.engines.ray.cudf_on_ray.frame.partition import ( +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( cuDFOnRayFramePartition, ) -from modin.engines.ray.task_wrapper import RayTask -from modin.backends.cudf.parser import cuDFCSVParser +from modin.core.execution.ray.common.task_wrapper import RayTask +from modin.core.storage_formats.cudf.parser import cuDFCSVParser class cuDFOnRayIO(BaseIO): diff --git a/modin/engines/ray/cudf_on_ray/io/text/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/io/text/__init__.py similarity index 100% rename from modin/engines/ray/cudf_on_ray/io/text/__init__.py rename to modin/core/execution/ray/implementations/cudf_on_ray/io/text/__init__.py diff --git a/modin/engines/ray/cudf_on_ray/io/text/csv_dispatcher.py b/modin/core/execution/ray/implementations/cudf_on_ray/io/text/csv_dispatcher.py similarity index 95% rename from modin/engines/ray/cudf_on_ray/io/text/csv_dispatcher.py rename to modin/core/execution/ray/implementations/cudf_on_ray/io/text/csv_dispatcher.py index 861b7d9c5a4..73dbd25a869 100644 --- a/modin/engines/ray/cudf_on_ray/io/text/csv_dispatcher.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/io/text/csv_dispatcher.py @@ -15,8 +15,10 @@ import numpy as np -from modin.engines.base.io import CSVDispatcher -from modin.engines.ray.cudf_on_ray.frame.partition_manager import GPU_MANAGERS +from modin.core.io import CSVDispatcher +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition_manager import ( + GPU_MANAGERS, +) from typing import Tuple diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py new file mode 100644 index 00000000000..174c5c4b6ce --- /dev/null +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Package holds implementations of key classes to making efficient scale of data using cuDF on Ray.""" diff --git a/modin/engines/ray/cudf_on_ray/frame/axis_partition.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py similarity index 96% rename from modin/engines/ray/cudf_on_ray/frame/axis_partition.py rename to modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py index b359ee4508a..d1a831d6460 100644 --- a/modin/engines/ray/cudf_on_ray/frame/axis_partition.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py @@ -16,7 +16,9 @@ import cudf import ray -from .partition import cuDFOnRayFramePartition +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( + cuDFOnRayFramePartition, +) class cuDFOnRayFrameAxisPartition(object): diff --git a/modin/engines/ray/cudf_on_ray/frame/gpu_manager.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/gpu_manager.py similarity index 100% rename from modin/engines/ray/cudf_on_ray/frame/gpu_manager.py rename to modin/core/execution/ray/implementations/cudf_on_ray/partitioning/gpu_manager.py diff --git a/modin/engines/ray/cudf_on_ray/frame/partition.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition.py similarity index 99% rename from modin/engines/ray/cudf_on_ray/frame/partition.py rename to modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition.py index 29cc1bbc5c0..33ac0833131 100644 --- a/modin/engines/ray/cudf_on_ray/frame/partition.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition.py @@ -18,7 +18,7 @@ import cupy import numpy as np import cupy as cp -from modin.engines.base.frame.partition import PandasFramePartition +from modin.core.dataframe.pandas.partitioning.partition import PandasFramePartition from pandas.core.dtypes.common import is_list_like diff --git a/modin/engines/ray/cudf_on_ray/frame/partition_manager.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py similarity index 94% rename from modin/engines/ray/cudf_on_ray/frame/partition_manager.py rename to modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py index aec06b2831c..67ab790aa59 100644 --- a/modin/engines/ray/cudf_on_ray/frame/partition_manager.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py @@ -16,15 +16,17 @@ import numpy as np import ray -from .axis_partition import ( +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.axis_partition import ( cuDFOnRayFrameColumnPartition, cuDFOnRayFrameRowPartition, ) -from .partition import cuDFOnRayFramePartition +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( + cuDFOnRayFramePartition, +) -from modin.data_management.utils import split_result_of_axis_func_pandas +from modin.core.storage_formats.pandas.utils import split_result_of_axis_func_pandas from modin.config import GpuCount -from modin.engines.ray.generic.frame.partition_manager import ( +from modin.core.execution.ray.generic.partitioning.partition_manager import ( GenericRayFramePartitionManager, ) diff --git a/modin/data_management/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/__init__.py similarity index 100% rename from modin/data_management/__init__.py rename to modin/core/execution/ray/implementations/pandas_on_ray/__init__.py diff --git a/modin/data_management/factories/test/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/__init__.py similarity index 100% rename from modin/data_management/factories/test/__init__.py rename to modin/core/execution/ray/implementations/pandas_on_ray/dataframe/__init__.py diff --git a/modin/engines/ray/pandas_on_ray/frame/data.py b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py similarity index 90% rename from modin/engines/ray/pandas_on_ray/frame/data.py rename to modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py index ea9ef0d87a1..30c288e69e9 100644 --- a/modin/engines/ray/pandas_on_ray/frame/data.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py @@ -15,9 +15,13 @@ import pandas -from .partition_manager import PandasOnRayFramePartitionManager -from modin.engines.base.frame.data import PandasFrame -from modin.backends.pandas.parsers import find_common_type_cat as find_common_type +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition_manager import ( + PandasOnRayFramePartitionManager, +) +from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame +from modin.core.storage_formats.pandas.parsers import ( + find_common_type_cat as find_common_type, +) import ray diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py new file mode 100644 index 00000000000..5788f2c389f --- /dev/null +++ b/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -0,0 +1,16 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +from modin.core.execution.ray.implementations.pandas_on_ray.io import PandasOnRayIO + +__all__ = ["PandasOnRayIO"] diff --git a/modin/engines/ray/pandas_on_ray/io.py b/modin/core/execution/ray/implementations/pandas_on_ray/io/io.py similarity index 81% rename from modin/engines/ray/pandas_on_ray/io.py rename to modin/core/execution/ray/implementations/pandas_on_ray/io/io.py index 3b49dbd171e..b0d611fe889 100644 --- a/modin/engines/ray/pandas_on_ray/io.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/io/io.py @@ -13,9 +13,9 @@ """The module holds the factory which performs I/O using pandas on Ray.""" -from modin.backends.pandas.query_compiler import PandasQueryCompiler -from modin.engines.ray.generic.io import RayIO -from modin.engines.base.io import ( +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler +from modin.core.execution.ray.generic.io import RayIO +from modin.core.io import ( CSVDispatcher, FWFDispatcher, JSONDispatcher, @@ -24,7 +24,7 @@ SQLDispatcher, ExcelDispatcher, ) -from modin.backends.pandas.parsers import ( +from modin.core.storage_formats.pandas.parsers import ( PandasCSVParser, PandasFWFParser, PandasJSONParser, @@ -33,9 +33,13 @@ PandasSQLParser, PandasExcelParser, ) -from modin.engines.ray.task_wrapper import RayTask -from modin.engines.ray.pandas_on_ray.frame.partition import PandasOnRayFramePartition -from modin.engines.ray.pandas_on_ray.frame.data import PandasOnRayFrame +from modin.core.execution.ray.common.task_wrapper import RayTask +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( + PandasOnRayFramePartition, +) +from modin.core.execution.ray.implementations.pandas_on_ray.dataframe.dataframe import ( + PandasOnRayFrame, +) class PandasOnRayIO(RayIO): diff --git a/modin/engines/ray/pandas_on_ray/frame/modin_aqp.py b/modin/core/execution/ray/implementations/pandas_on_ray/modin_aqp.py similarity index 100% rename from modin/engines/ray/pandas_on_ray/frame/modin_aqp.py rename to modin/core/execution/ray/implementations/pandas_on_ray/modin_aqp.py diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/modin/engines/ray/pandas_on_ray/frame/axis_partition.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py similarity index 96% rename from modin/engines/ray/pandas_on_ray/frame/axis_partition.py rename to modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py index 926fe49b27f..2fdcbbe1c79 100644 --- a/modin/engines/ray/pandas_on_ray/frame/axis_partition.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py @@ -15,8 +15,12 @@ import pandas -from modin.engines.base.frame.axis_partition import PandasFrameAxisPartition -from .partition import PandasOnRayFramePartition +from modin.core.dataframe.pandas.partitioning.axis_partition import ( + PandasFrameAxisPartition, +) +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( + PandasOnRayFramePartition, +) import ray from ray.services import get_node_ip_address diff --git a/modin/engines/ray/pandas_on_ray/frame/partition.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py similarity index 98% rename from modin/engines/ray/pandas_on_ray/frame/partition.py rename to modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py index 839c3968565..92c76ef5575 100644 --- a/modin/engines/ray/pandas_on_ray/frame/partition.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition.py @@ -15,8 +15,8 @@ import pandas -from modin.data_management.utils import length_fn_pandas, width_fn_pandas -from modin.engines.base.frame.partition import PandasFramePartition +from modin.core.storage_formats.pandas.utils import length_fn_pandas, width_fn_pandas +from modin.core.dataframe.pandas.partitioning.partition import PandasFramePartition from modin.pandas.indexing import compute_sliced_len import ray diff --git a/modin/engines/ray/pandas_on_ray/frame/partition_manager.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py similarity index 97% rename from modin/engines/ray/pandas_on_ray/frame/partition_manager.py rename to modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py index b3e983e68d2..4c8d85493f1 100644 --- a/modin/engines/ray/pandas_on_ray/frame/partition_manager.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py @@ -18,15 +18,19 @@ import threading from modin.config import ProgressBar -from modin.engines.ray.generic.frame.partition_manager import ( +from modin.core.execution.ray.generic.partitioning.partition_manager import ( GenericRayFramePartitionManager, ) -from .axis_partition import ( +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.axis_partition import ( PandasOnRayFrameColumnPartition, PandasOnRayFrameRowPartition, ) -from .partition import PandasOnRayFramePartition -from .modin_aqp import call_progress_bar +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( + PandasOnRayFramePartition, +) +from modin.core.execution.ray.implementations.pandas_on_ray.modin_aqp import ( + call_progress_bar, +) from modin.error_message import ErrorMessage import pandas diff --git a/modin/engines/base/io/__init__.py b/modin/core/io/__init__.py similarity index 54% rename from modin/engines/base/io/__init__.py rename to modin/core/io/__init__.py index 764f8957865..945c100ddc9 100644 --- a/modin/engines/base/io/__init__.py +++ b/modin/core/io/__init__.py @@ -13,19 +13,19 @@ """IO functions implementations.""" -from modin.engines.base.io.io import BaseIO -from modin.engines.base.io.text.csv_dispatcher import CSVDispatcher -from modin.engines.base.io.text.csv_glob_dispatcher import CSVGlobDispatcher -from modin.engines.base.io.text.fwf_dispatcher import FWFDispatcher -from modin.engines.base.io.text.json_dispatcher import JSONDispatcher -from modin.engines.base.io.text.excel_dispatcher import ExcelDispatcher -from modin.engines.base.io.file_dispatcher import FileDispatcher -from modin.engines.base.io.text.text_file_dispatcher import TextFileDispatcher -from modin.engines.base.io.column_stores.parquet_dispatcher import ParquetDispatcher -from modin.engines.base.io.column_stores.hdf_dispatcher import HDFDispatcher -from modin.engines.base.io.column_stores.feather_dispatcher import FeatherDispatcher -from modin.engines.base.io.sql.sql_dispatcher import SQLDispatcher -from modin.engines.base.io.pickle.pickle_dispatcher import PickleExperimentalDispatcher +from modin.core.io import BaseIO +from modin.core.io.text.csv_dispatcher import CSVDispatcher +from modin.core.io.text.csv_glob_dispatcher import CSVGlobDispatcher +from modin.core.io.text.fwf_dispatcher import FWFDispatcher +from modin.core.io.text.json_dispatcher import JSONDispatcher +from modin.core.io.text.excel_dispatcher import ExcelDispatcher +from modin.core.io.file_dispatcher import FileDispatcher +from modin.core.io.text.text_file_dispatcher import TextFileDispatcher +from modin.core.io.column_stores.parquet_dispatcher import ParquetDispatcher +from modin.core.io.column_stores.hdf_dispatcher import HDFDispatcher +from modin.core.io.column_stores.feather_dispatcher import FeatherDispatcher +from modin.core.io.sql.sql_dispatcher import SQLDispatcher +from modin.core.io.pickle.pickle_dispatcher import PickleExperimentalDispatcher __all__ = [ "BaseIO", diff --git a/modin/engines/base/io/column_stores/__init__.py b/modin/core/io/column_stores/__init__.py similarity index 100% rename from modin/engines/base/io/column_stores/__init__.py rename to modin/core/io/column_stores/__init__.py diff --git a/modin/engines/base/io/column_stores/column_store_dispatcher.py b/modin/core/io/column_stores/column_store_dispatcher.py similarity index 98% rename from modin/engines/base/io/column_stores/column_store_dispatcher.py rename to modin/core/io/column_stores/column_store_dispatcher.py index 8cd89e7b918..188d0d02b2f 100644 --- a/modin/engines/base/io/column_stores/column_store_dispatcher.py +++ b/modin/core/io/column_stores/column_store_dispatcher.py @@ -22,8 +22,8 @@ import numpy as np import pandas -from modin.data_management.utils import compute_chunksize -from modin.engines.base.io.file_dispatcher import FileDispatcher +from modin.core.storage_formats.pandas.utils import compute_chunksize +from modin.core.io.file_dispatcher import FileDispatcher from modin.config import NPartitions diff --git a/modin/engines/base/io/column_stores/feather_dispatcher.py b/modin/core/io/column_stores/feather_dispatcher.py similarity index 96% rename from modin/engines/base/io/column_stores/feather_dispatcher.py rename to modin/core/io/column_stores/feather_dispatcher.py index 31fdbfca944..e965f11316a 100644 --- a/modin/engines/base/io/column_stores/feather_dispatcher.py +++ b/modin/core/io/column_stores/feather_dispatcher.py @@ -13,7 +13,7 @@ """Module houses `FeatherDispatcher` class, that is used for reading `.feather` files.""" -from modin.engines.base.io.column_stores.column_store_dispatcher import ( +from modin.core.io.column_stores.column_store_dispatcher import ( ColumnStoreDispatcher, ) diff --git a/modin/engines/base/io/column_stores/hdf_dispatcher.py b/modin/core/io/column_stores/hdf_dispatcher.py similarity index 97% rename from modin/engines/base/io/column_stores/hdf_dispatcher.py rename to modin/core/io/column_stores/hdf_dispatcher.py index b62c1271f77..3dd0d5b7bc0 100644 --- a/modin/engines/base/io/column_stores/hdf_dispatcher.py +++ b/modin/core/io/column_stores/hdf_dispatcher.py @@ -15,7 +15,7 @@ import pandas -from modin.engines.base.io.column_stores.column_store_dispatcher import ( +from modin.core.io.column_stores.column_store_dispatcher import ( ColumnStoreDispatcher, ) from modin.error_message import ErrorMessage diff --git a/modin/engines/base/io/column_stores/parquet_dispatcher.py b/modin/core/io/column_stores/parquet_dispatcher.py similarity index 98% rename from modin/engines/base/io/column_stores/parquet_dispatcher.py rename to modin/core/io/column_stores/parquet_dispatcher.py index 6ecad423633..cfcc7385195 100644 --- a/modin/engines/base/io/column_stores/parquet_dispatcher.py +++ b/modin/core/io/column_stores/parquet_dispatcher.py @@ -15,7 +15,7 @@ import os -from modin.engines.base.io.column_stores.column_store_dispatcher import ( +from modin.core.io.column_stores.column_store_dispatcher import ( ColumnStoreDispatcher, ) from modin.error_message import ErrorMessage diff --git a/modin/engines/base/io/file_dispatcher.py b/modin/core/io/file_dispatcher.py similarity index 100% rename from modin/engines/base/io/file_dispatcher.py rename to modin/core/io/file_dispatcher.py diff --git a/modin/engines/base/io/io.py b/modin/core/io/io.py similarity index 99% rename from modin/engines/base/io/io.py rename to modin/core/io/io.py index 341ba09d4ec..450eb6d9c9b 100644 --- a/modin/engines/base/io/io.py +++ b/modin/core/io/io.py @@ -27,7 +27,7 @@ from pandas.util._decorators import doc from modin.error_message import ErrorMessage -from modin.backends.base.query_compiler import BaseQueryCompiler +from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler from modin.utils import _inherit_docstrings _doc_default_io_method = """ diff --git a/modin/engines/__init__.py b/modin/core/io/pickle/__init__.py similarity index 100% rename from modin/engines/__init__.py rename to modin/core/io/pickle/__init__.py diff --git a/modin/engines/base/io/pickle/pickle_dispatcher.py b/modin/core/io/pickle/pickle_dispatcher.py similarity index 97% rename from modin/engines/base/io/pickle/pickle_dispatcher.py rename to modin/core/io/pickle/pickle_dispatcher.py index cf71a8c979f..513b758123e 100644 --- a/modin/engines/base/io/pickle/pickle_dispatcher.py +++ b/modin/core/io/pickle/pickle_dispatcher.py @@ -14,7 +14,7 @@ import glob import warnings -from modin.engines.base.io.file_dispatcher import FileDispatcher +from modin.core.io.file_dispatcher import FileDispatcher from modin.config import NPartitions diff --git a/modin/engines/base/io/sql/__init__.py b/modin/core/io/sql/__init__.py similarity index 100% rename from modin/engines/base/io/sql/__init__.py rename to modin/core/io/sql/__init__.py diff --git a/modin/engines/base/io/sql/sql_dispatcher.py b/modin/core/io/sql/sql_dispatcher.py similarity index 98% rename from modin/engines/base/io/sql/sql_dispatcher.py rename to modin/core/io/sql/sql_dispatcher.py index 511dc72520e..13de4d3b889 100644 --- a/modin/engines/base/io/sql/sql_dispatcher.py +++ b/modin/core/io/sql/sql_dispatcher.py @@ -24,7 +24,7 @@ import pandas import warnings -from modin.engines.base.io.file_dispatcher import FileDispatcher +from modin.core.io.file_dispatcher import FileDispatcher from modin.config import NPartitions diff --git a/modin/engines/base/io/text/__init__.py b/modin/core/io/text/__init__.py similarity index 100% rename from modin/engines/base/io/text/__init__.py rename to modin/core/io/text/__init__.py diff --git a/modin/engines/base/io/text/csv_dispatcher.py b/modin/core/io/text/csv_dispatcher.py similarity index 99% rename from modin/engines/base/io/text/csv_dispatcher.py rename to modin/core/io/text/csv_dispatcher.py index e0dfffdf1cc..1131fff6f52 100644 --- a/modin/engines/base/io/text/csv_dispatcher.py +++ b/modin/core/io/text/csv_dispatcher.py @@ -13,7 +13,7 @@ """Module houses `CSVDispatcher` class, that is used for reading `.csv` files.""" -from modin.engines.base.io.text.text_file_dispatcher import ( +from modin.core.io.text.text_file_dispatcher import ( TextFileDispatcher, ColumnNamesTypes, ) diff --git a/modin/engines/base/io/text/csv_glob_dispatcher.py b/modin/core/io/text/csv_glob_dispatcher.py similarity index 99% rename from modin/engines/base/io/text/csv_glob_dispatcher.py rename to modin/core/io/text/csv_glob_dispatcher.py index ffde411c1a7..7e1ce13d66e 100644 --- a/modin/engines/base/io/text/csv_glob_dispatcher.py +++ b/modin/core/io/text/csv_glob_dispatcher.py @@ -25,8 +25,8 @@ import pandas._libs.lib as lib from modin.config import NPartitions -from modin.engines.base.io.file_dispatcher import S3_ADDRESS_REGEX -from modin.engines.base.io.text.csv_dispatcher import CSVDispatcher +from modin.core.io.file_dispatcher import S3_ADDRESS_REGEX +from modin.core.io.text.csv_dispatcher import CSVDispatcher class CSVGlobDispatcher(CSVDispatcher): diff --git a/modin/engines/base/io/text/excel_dispatcher.py b/modin/core/io/text/excel_dispatcher.py similarity index 99% rename from modin/engines/base/io/text/excel_dispatcher.py rename to modin/core/io/text/excel_dispatcher.py index 898b05bb7ab..72a6e6caa92 100644 --- a/modin/engines/base/io/text/excel_dispatcher.py +++ b/modin/core/io/text/excel_dispatcher.py @@ -18,7 +18,7 @@ import sys import warnings -from modin.engines.base.io.text.text_file_dispatcher import TextFileDispatcher +from modin.core.io.text.text_file_dispatcher import TextFileDispatcher from modin.config import NPartitions EXCEL_READ_BLOCK_SIZE = 4096 diff --git a/modin/engines/base/io/text/fwf_dispatcher.py b/modin/core/io/text/fwf_dispatcher.py similarity index 99% rename from modin/engines/base/io/text/fwf_dispatcher.py rename to modin/core/io/text/fwf_dispatcher.py index da765c0a985..fc5604369e4 100644 --- a/modin/engines/base/io/text/fwf_dispatcher.py +++ b/modin/core/io/text/fwf_dispatcher.py @@ -13,7 +13,7 @@ """Module houses `FWFDispatcher` class, that is used for reading of tables with fixed-width formatted lines.""" -from modin.engines.base.io.text.text_file_dispatcher import TextFileDispatcher +from modin.core.io.text.text_file_dispatcher import TextFileDispatcher import pandas._libs.lib as lib import pandas from csv import QUOTE_NONE diff --git a/modin/engines/base/io/text/json_dispatcher.py b/modin/core/io/text/json_dispatcher.py similarity index 98% rename from modin/engines/base/io/text/json_dispatcher.py rename to modin/core/io/text/json_dispatcher.py index 6f4e3a6174d..656feaa7f0b 100644 --- a/modin/engines/base/io/text/json_dispatcher.py +++ b/modin/core/io/text/json_dispatcher.py @@ -13,7 +13,7 @@ """Module houses `JSONDispatcher` class, that is used for reading `.json` files.""" -from modin.engines.base.io.text.text_file_dispatcher import TextFileDispatcher +from modin.core.io.text.text_file_dispatcher import TextFileDispatcher from io import BytesIO import pandas import numpy as np diff --git a/modin/engines/base/io/text/text_file_dispatcher.py b/modin/core/io/text/text_file_dispatcher.py similarity index 99% rename from modin/engines/base/io/text/text_file_dispatcher.py rename to modin/core/io/text/text_file_dispatcher.py index 6fcc15b13ab..db2ee8633c4 100644 --- a/modin/engines/base/io/text/text_file_dispatcher.py +++ b/modin/core/io/text/text_file_dispatcher.py @@ -18,8 +18,8 @@ files from `FileDispatcher` class and can be used as base class for dipatchers of SQL queries. """ -from modin.engines.base.io.file_dispatcher import FileDispatcher -from modin.data_management.utils import compute_chunksize +from modin.core.io.file_dispatcher import FileDispatcher +from modin.core.storage_formats.pandas.utils import compute_chunksize from modin.utils import _inherit_docstrings import numpy as np import warnings diff --git a/modin/backends/__init__.py b/modin/core/storage_formats/__init__.py similarity index 76% rename from modin/backends/__init__.py rename to modin/core/storage_formats/__init__.py index b3bca8a6f37..13698a88065 100644 --- a/modin/backends/__init__.py +++ b/modin/core/storage_formats/__init__.py @@ -11,12 +11,14 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -from .base import BaseQueryCompiler -from .pandas import PandasQueryCompiler +"""Modin's functionality related to storage formats supported.""" + +from modin.core.storage_formats.base import BaseQueryCompiler +from modin.core.storage_formats.pandas import PandasQueryCompiler __all__ = ["BaseQueryCompiler", "PandasQueryCompiler"] try: - from .pyarrow import PyarrowQueryCompiler # noqa: F401 + from modin.core.storage_formats.pyarrow import PyarrowQueryCompiler # noqa: F401 except ImportError: pass else: diff --git a/modin/backends/base/__init__.py b/modin/core/storage_formats/base/__init__.py similarity index 92% rename from modin/backends/base/__init__.py rename to modin/core/storage_formats/base/__init__.py index 8d0a4f40fa3..1a4c7a3cdbf 100644 --- a/modin/backends/base/__init__.py +++ b/modin/core/storage_formats/base/__init__.py @@ -13,6 +13,6 @@ """The module represents the base query compiler that defines the common query compiler API.""" -from .query_compiler import BaseQueryCompiler +from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler __all__ = ["BaseQueryCompiler"] diff --git a/modin/backends/base/doc_utils.py b/modin/core/storage_formats/base/doc_utils.py similarity index 100% rename from modin/backends/base/doc_utils.py rename to modin/core/storage_formats/base/doc_utils.py diff --git a/modin/backends/base/query_compiler.py b/modin/core/storage_formats/base/query_compiler.py similarity index 99% rename from modin/backends/base/query_compiler.py rename to modin/core/storage_formats/base/query_compiler.py index 82b5c38689a..44672547549 100644 --- a/modin/backends/base/query_compiler.py +++ b/modin/core/storage_formats/base/query_compiler.py @@ -19,7 +19,7 @@ import abc -from modin.data_management.functions.default_methods import ( +from modin.core.dataframe.algebra.d2p import ( DataFrameDefault, SeriesDefault, DateTimeDefault, @@ -31,7 +31,7 @@ GroupByDefault, ) from modin.error_message import ErrorMessage -import modin.backends.base.doc_utils as doc_utils +import modin.core.storage_formats.base.doc_utils as doc_utils from pandas.core.dtypes.common import is_scalar import pandas.core.resample diff --git a/modin/core/storage_formats/cudf/__init__.py b/modin/core/storage_formats/cudf/__init__.py new file mode 100644 index 00000000000..4175d9c9119 --- /dev/null +++ b/modin/core/storage_formats/cudf/__init__.py @@ -0,0 +1,18 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""The module represents the query compiler level for the cuDF backend.""" + +from modin.core.storage_formats.cudf.query_compiler import cuDFQueryCompiler + +__all__ = ["cuDFQueryCompiler"] diff --git a/modin/backends/cudf/parser.py b/modin/core/storage_formats/cudf/parser.py similarity index 95% rename from modin/backends/cudf/parser.py rename to modin/core/storage_formats/cudf/parser.py index b9b79ba2858..7902cda9537 100644 --- a/modin/backends/cudf/parser.py +++ b/modin/core/storage_formats/cudf/parser.py @@ -20,9 +20,11 @@ from pandas.io.common import infer_compression import warnings -from modin.engines.base.io import FileDispatcher -from modin.engines.ray.cudf_on_ray.frame.partition_manager import GPU_MANAGERS -from modin.data_management.utils import split_result_of_axis_func_pandas +from modin.core.io import FileDispatcher +from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition_manager import ( + GPU_MANAGERS, +) +from modin.core.storage_formats.pandas.utils import split_result_of_axis_func_pandas from modin.error_message import ErrorMessage diff --git a/modin/backends/cudf/query_compiler.py b/modin/core/storage_formats/cudf/query_compiler.py similarity index 97% rename from modin/backends/cudf/query_compiler.py rename to modin/core/storage_formats/cudf/query_compiler.py index f8bb478442a..640d2eddd54 100644 --- a/modin/backends/cudf/query_compiler.py +++ b/modin/core/storage_formats/cudf/query_compiler.py @@ -14,7 +14,7 @@ import numpy as np import pandas -from modin.backends.pandas.query_compiler import PandasQueryCompiler +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler class cuDFQueryCompiler(PandasQueryCompiler): diff --git a/modin/backends/pandas/__init__.py b/modin/core/storage_formats/pandas/__init__.py similarity index 91% rename from modin/backends/pandas/__init__.py rename to modin/core/storage_formats/pandas/__init__.py index 0a285af52dd..06c82683346 100644 --- a/modin/backends/pandas/__init__.py +++ b/modin/core/storage_formats/pandas/__init__.py @@ -13,6 +13,6 @@ """The module represents the query compiler level for the pandas backend.""" -from .query_compiler import PandasQueryCompiler +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler __all__ = ["PandasQueryCompiler"] diff --git a/modin/backends/pandas/parsers.py b/modin/core/storage_formats/pandas/parsers.py similarity index 99% rename from modin/backends/pandas/parsers.py rename to modin/core/storage_formats/pandas/parsers.py index 82fbb7e1837..58c913290cd 100644 --- a/modin/backends/pandas/parsers.py +++ b/modin/core/storage_formats/pandas/parsers.py @@ -49,8 +49,8 @@ from pandas.util._decorators import doc import warnings -from modin.engines.base.io import FileDispatcher -from modin.data_management.utils import split_result_of_axis_func_pandas +from modin.core.io import FileDispatcher +from modin.core.storage_formats.pandas.utils import split_result_of_axis_func_pandas from modin.error_message import ErrorMessage _doc_pandas_parser_class = """ diff --git a/modin/backends/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py similarity index 99% rename from modin/backends/pandas/query_compiler.py rename to modin/core/storage_formats/pandas/query_compiler.py index ef5f25d385a..b2cee6e6c27 100644 --- a/modin/backends/pandas/query_compiler.py +++ b/modin/core/storage_formats/pandas/query_compiler.py @@ -34,7 +34,7 @@ import warnings -from modin.backends.base.query_compiler import BaseQueryCompiler +from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler from modin.error_message import ErrorMessage from modin.utils import ( try_cast_to_pandas, @@ -42,7 +42,7 @@ hashable, _inherit_docstrings, ) -from modin.data_management.functions import ( +from modin.core.dataframe.algebra.operators import ( FoldFunction, MapFunction, MapReduceFunction, diff --git a/modin/data_management/utils.py b/modin/core/storage_formats/pandas/utils.py similarity index 100% rename from modin/data_management/utils.py rename to modin/core/storage_formats/pandas/utils.py diff --git a/modin/backends/pyarrow/__init__.py b/modin/core/storage_formats/pyarrow/__init__.py similarity index 100% rename from modin/backends/pyarrow/__init__.py rename to modin/core/storage_formats/pyarrow/__init__.py diff --git a/modin/backends/pyarrow/parsers.py b/modin/core/storage_formats/pyarrow/parsers.py similarity index 97% rename from modin/backends/pyarrow/parsers.py rename to modin/core/storage_formats/pyarrow/parsers.py index 595d0a1076a..8150f245ba6 100644 --- a/modin/backends/pyarrow/parsers.py +++ b/modin/core/storage_formats/pyarrow/parsers.py @@ -13,7 +13,7 @@ """Module houses Modin parser classes, that are used for data parsing on the workers.""" -from modin.data_management.utils import get_default_chunksize +from modin.core.storage_formats.pandas.utils import get_default_chunksize from io import BytesIO import pandas diff --git a/modin/backends/pyarrow/query_compiler.py b/modin/core/storage_formats/pyarrow/query_compiler.py similarity index 99% rename from modin/backends/pyarrow/query_compiler.py rename to modin/core/storage_formats/pyarrow/query_compiler.py index 019ce1b4295..ca205a25864 100644 --- a/modin/backends/pyarrow/query_compiler.py +++ b/modin/core/storage_formats/pyarrow/query_compiler.py @@ -18,7 +18,7 @@ queries for the ``PyarrowOnRayFrame``. """ -from modin.backends.pandas.query_compiler import PandasQueryCompiler +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler from modin.utils import _inherit_docstrings import pandas from pandas.core.computation.expr import Expr diff --git a/modin/distributed/dataframe/pandas/partitions.py b/modin/distributed/dataframe/pandas/partitions.py index c5e26a59731..0e3ec5ec873 100644 --- a/modin/distributed/dataframe/pandas/partitions.py +++ b/modin/distributed/dataframe/pandas/partitions.py @@ -15,7 +15,7 @@ import numpy as np -from modin.backends.pandas.query_compiler import PandasQueryCompiler +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler from modin.pandas.dataframe import DataFrame @@ -129,7 +129,7 @@ def from_partitions( Pass `index`, `columns`, `row_lengths` and `column_widths` to avoid triggering extra computations of the metadata when creating a DataFrame. """ - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher factory = FactoryDispatcher.get_factory() diff --git a/modin/experimental/backends/omnisci/__init__.py b/modin/experimental/backends/omnisci/__init__.py deleted file mode 100644 index cae6413e559..00000000000 --- a/modin/experimental/backends/omnisci/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. diff --git a/modin/experimental/cloud/meta_magic.py b/modin/experimental/cloud/meta_magic.py index fef74e58b49..2513fa83e7e 100644 --- a/modin/experimental/cloud/meta_magic.py +++ b/modin/experimental/cloud/meta_magic.py @@ -16,7 +16,7 @@ import types from modin.config import Engine -from modin.data_management.factories import REMOTE_ENGINES +from modin.core.execution.dispatching.factories import REMOTE_ENGINES # the attributes that must be alwasy taken from a local part of dual-nature class, # never going to remote end diff --git a/modin/experimental/core/__init__.py b/modin/experimental/core/__init__.py new file mode 100644 index 00000000000..e367d7c1ef7 --- /dev/null +++ b/modin/experimental/core/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Experimental Modin's core functionality.""" diff --git a/modin/experimental/core/execution/__init__.py b/modin/experimental/core/execution/__init__.py new file mode 100644 index 00000000000..8b1d42c0f9c --- /dev/null +++ b/modin/experimental/core/execution/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Experimental Modin's functionality related to execution engines supported.""" diff --git a/modin/engines/base/__init__.py b/modin/experimental/core/execution/ray/__init__.py similarity index 100% rename from modin/engines/base/__init__.py rename to modin/experimental/core/execution/ray/__init__.py diff --git a/modin/engines/base/io/pickle/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py similarity index 100% rename from modin/engines/base/io/pickle/__init__.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py diff --git a/modin/experimental/engines/omnisci_on_ray/frame/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/__init__.py similarity index 100% rename from modin/experimental/engines/omnisci_on_ray/frame/__init__.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/__init__.py diff --git a/modin/experimental/engines/omnisci_on_ray/frame/calcite_algebra.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py similarity index 98% rename from modin/experimental/engines/omnisci_on_ray/frame/calcite_algebra.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py index bf4570307db..62f9e446a98 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/calcite_algebra.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py @@ -19,7 +19,9 @@ """ import abc -from .expr import BaseExpr +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( + BaseExpr, +) class CalciteInputRefExpr(BaseExpr): diff --git a/modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py similarity index 99% rename from modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py index 43cf83f6619..9fc834f068f 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/calcite_builder.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py @@ -13,14 +13,14 @@ """Module provides ``CalciteBuilder`` class.""" -from .expr import ( +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( InputRefExpr, LiteralExpr, AggregateExpr, build_if_then_else, build_row_idx_filter_expr, ) -from .calcite_algebra import ( +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_algebra import ( CalciteBaseNode, CalciteInputRefExpr, CalciteInputIdxExpr, @@ -33,7 +33,7 @@ CalciteJoinNode, CalciteUnionNode, ) -from .df_algebra import ( +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.df_algebra import ( FrameNode, MaskNode, GroupbyAggNode, diff --git a/modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py similarity index 97% rename from modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py index b73e67932d6..745d4270a86 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/calcite_serializer.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py @@ -13,13 +13,13 @@ """Module provides ``CalciteSerializer`` class.""" -from .expr import ( +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( BaseExpr, LiteralExpr, OpExpr, AggregateExpr, ) -from .calcite_algebra import ( +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_algebra import ( CalciteBaseNode, CalciteInputRefExpr, CalciteInputIdxExpr, diff --git a/modin/experimental/engines/omnisci_on_ray/frame/data.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py similarity index 99% rename from modin/experimental/engines/omnisci_on_ray/frame/data.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py index b24fc4d8bdd..014524ec8c4 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/data.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py @@ -13,9 +13,13 @@ """Module provides ``OmnisciOnRayFrame`` class implementing lazy frame.""" -from modin.engines.base.frame.data import PandasFrame -from modin.experimental.backends.omnisci.query_compiler import DFAlgQueryCompiler -from .partition_manager import OmnisciOnRayFramePartitionManager +from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame +from modin.experimental.core.storage_formats.omnisci.query_compiler import ( + DFAlgQueryCompiler, +) +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition_manager import ( + OmnisciOnRayFramePartitionManager, +) from pandas.core.index import ensure_index, Index, MultiIndex, RangeIndex from pandas.core.dtypes.common import get_dtype, is_list_like, is_bool_dtype diff --git a/modin/experimental/engines/omnisci_on_ray/frame/df_algebra.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py similarity index 99% rename from modin/experimental/engines/omnisci_on_ray/frame/df_algebra.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py index 63440b65655..f5b4c2e4683 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/df_algebra.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py @@ -14,7 +14,9 @@ """Module provides classes for lazy DataFrame algebra operations.""" import abc -from .expr import InputRefExpr +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( + InputRefExpr, +) from collections import OrderedDict diff --git a/modin/experimental/engines/omnisci_on_ray/frame/expr.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/expr.py similarity index 100% rename from modin/experimental/engines/omnisci_on_ray/frame/expr.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/expr.py diff --git a/modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/omnisci_worker.py similarity index 100% rename from modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/omnisci_worker.py diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py new file mode 100644 index 00000000000..a98d57f5845 --- /dev/null +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py @@ -0,0 +1,18 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.io import ( + OmnisciOnRayIO, +) + +__all__ = ["OmnisciOnRayIO"] diff --git a/modin/experimental/engines/omnisci_on_ray/io.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/io.py similarity index 100% rename from modin/experimental/engines/omnisci_on_ray/io.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/io.py diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/modin/experimental/engines/omnisci_on_ray/frame/axis_partition.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py similarity index 91% rename from modin/experimental/engines/omnisci_on_ray/frame/axis_partition.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py index 20fc144b138..e61ecc88eaf 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/axis_partition.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py @@ -17,8 +17,12 @@ These classes are intended to perform operations on the data rows and columns. """ -from modin.engines.base.frame.axis_partition import PandasFrameAxisPartition -from .partition import OmnisciOnRayFramePartition +from modin.core.dataframe.pandas.partitioning.axis_partition import ( + PandasFrameAxisPartition, +) +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition import ( + OmnisciOnRayFramePartition, +) import ray diff --git a/modin/experimental/engines/omnisci_on_ray/frame/partition.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition.py similarity index 98% rename from modin/experimental/engines/omnisci_on_ray/frame/partition.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition.py index 5b9985fff9c..b248843e2d6 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/partition.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition.py @@ -15,7 +15,7 @@ import pandas -from modin.engines.base.frame.partition import PandasFramePartition +from modin.core.dataframe.pandas.partitioning.partition import PandasFramePartition import pyarrow import ray diff --git a/modin/experimental/engines/omnisci_on_ray/frame/partition_manager.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py similarity index 94% rename from modin/experimental/engines/omnisci_on_ray/frame/partition_manager.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py index aac1e7cef0c..2394638a3be 100644 --- a/modin/experimental/engines/omnisci_on_ray/frame/partition_manager.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py @@ -16,17 +16,25 @@ from modin.pandas.utils import is_scalar import numpy as np -from modin.engines.ray.generic.frame.partition_manager import ( +from modin.core.execution.ray.generic.partitioning.partition_manager import ( GenericRayFramePartitionManager, ) from .axis_partition import ( OmnisciOnRayFrameColumnPartition, OmnisciOnRayFrameRowPartition, ) -from .partition import OmnisciOnRayFramePartition -from .omnisci_worker import OmnisciServer -from .calcite_builder import CalciteBuilder -from .calcite_serializer import CalciteSerializer +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition import ( + OmnisciOnRayFramePartition, +) +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.omnisci_worker import ( + OmnisciServer, +) +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_builder import ( + CalciteBuilder, +) +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_serializer import ( + CalciteSerializer, +) from modin.config import DoUseCalcite import pyarrow diff --git a/modin/engines/python/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/__init__.py similarity index 100% rename from modin/engines/python/__init__.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/__init__.py diff --git a/modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py similarity index 100% rename from modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py rename to modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py diff --git a/modin/experimental/engines/pandas_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py similarity index 100% rename from modin/experimental/engines/pandas_on_ray/__init__.py rename to modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py diff --git a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py new file mode 100644 index 00000000000..428d5330180 --- /dev/null +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -0,0 +1,20 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Experimental functionality implementations with pandas backend and Ray engine.""" + +from modin.experimental.core.execution.ray.implementations.pandas_on_ray.io import ( + ExperimentalPandasOnRayIO, +) + +__all__ = ["ExperimentalPandasOnRayIO"] diff --git a/modin/experimental/engines/pandas_on_ray/io_exp.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/io.py similarity index 95% rename from modin/experimental/engines/pandas_on_ray/io_exp.py rename to modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/io.py index cd0fdc30513..7d76c4b5d99 100644 --- a/modin/experimental/engines/pandas_on_ray/io_exp.py +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/io.py @@ -22,17 +22,21 @@ import pandas import warnings -from modin.backends.pandas.parsers import ( +from modin.core.storage_formats.pandas.parsers import ( _split_result_for_readers, PandasCSVGlobParser, PandasPickleExperimentalParser, ) -from modin.backends.pandas.query_compiler import PandasQueryCompiler -from modin.engines.ray.pandas_on_ray.io import PandasOnRayIO -from modin.engines.base.io import CSVGlobDispatcher, PickleExperimentalDispatcher -from modin.engines.ray.pandas_on_ray.frame.data import PandasOnRayFrame -from modin.engines.ray.pandas_on_ray.frame.partition import PandasOnRayFramePartition -from modin.engines.ray.task_wrapper import RayTask +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler +from modin.core.execution.ray.implementations.pandas_on_ray.io import PandasOnRayIO +from modin.core.io import CSVGlobDispatcher, PickleExperimentalDispatcher +from modin.core.execution.ray.implementations.pandas_on_ray.dataframe.dataframe import ( + PandasOnRayFrame, +) +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( + PandasOnRayFramePartition, +) +from modin.core.execution.ray.common.task_wrapper import RayTask from modin.config import NPartitions import ray diff --git a/modin/experimental/engines/pandas_on_ray/sql.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/sql.py similarity index 100% rename from modin/experimental/engines/pandas_on_ray/sql.py rename to modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/sql.py diff --git a/modin/experimental/engines/pyarrow_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py similarity index 100% rename from modin/experimental/engines/pyarrow_on_ray/__init__.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py diff --git a/modin/experimental/engines/pyarrow_on_ray/frame/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py similarity index 100% rename from modin/experimental/engines/pyarrow_on_ray/frame/__init__.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py diff --git a/modin/experimental/engines/pyarrow_on_ray/frame/data.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py similarity index 94% rename from modin/experimental/engines/pyarrow_on_ray/frame/data.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py index 80ad803e06a..aa726f6d29e 100644 --- a/modin/experimental/engines/pyarrow_on_ray/frame/data.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py @@ -20,8 +20,10 @@ import pandas from pandas.core.dtypes.cast import find_common_type -from .partition_manager import PyarrowOnRayFramePartitionManager -from modin.engines.base.frame.data import PandasFrame +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition_manager import ( + PyarrowOnRayFramePartitionManager, +) +from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame import ray diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py new file mode 100644 index 00000000000..8595d91ed5a --- /dev/null +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py @@ -0,0 +1,5 @@ +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.io import ( + PyarrowOnRayIO, +) + +__all__ = ["PyarrowOnRayIO"] diff --git a/modin/experimental/engines/pyarrow_on_ray/io.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/io.py similarity index 74% rename from modin/experimental/engines/pyarrow_on_ray/io.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/io.py index 00baf438ab1..30a7cae0a9b 100644 --- a/modin/experimental/engines/pyarrow_on_ray/io.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/io.py @@ -13,15 +13,17 @@ """Module for housing IO classes with PyArrow backend and Ray engine.""" -from modin.backends.pyarrow.query_compiler import PyarrowQueryCompiler -from modin.engines.ray.generic.io import RayIO -from modin.experimental.engines.pyarrow_on_ray.frame.data import PyarrowOnRayFrame -from modin.experimental.engines.pyarrow_on_ray.frame.partition import ( +from modin.core.storage_formats.pyarrow.query_compiler import PyarrowQueryCompiler +from modin.core.execution.ray.generic.io import RayIO +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.dataframe.dataframe import ( + PyarrowOnRayFrame, +) +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition import ( PyarrowOnRayFramePartition, ) -from modin.backends.pyarrow.parsers import PyarrowCSVParser -from modin.engines.ray.task_wrapper import RayTask -from modin.engines.base.io import CSVDispatcher +from modin.core.storage_formats.pyarrow.parsers import PyarrowCSVParser +from modin.core.execution.ray.common.task_wrapper import RayTask +from modin.core.io import CSVDispatcher class PyarrowOnRayCSVDispatcher(RayTask, PyarrowCSVParser, CSVDispatcher): diff --git a/modin/engines/ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py similarity index 100% rename from modin/engines/ray/__init__.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py diff --git a/modin/experimental/engines/pyarrow_on_ray/frame/axis_partition.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py similarity index 97% rename from modin/experimental/engines/pyarrow_on_ray/frame/axis_partition.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py index ffe9d5fbd82..89c4a231ee6 100644 --- a/modin/experimental/engines/pyarrow_on_ray/frame/axis_partition.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py @@ -13,8 +13,12 @@ """The module defines interface for an axis partition with PyArrow backend and Ray engine.""" -from modin.engines.base.frame.axis_partition import BaseFrameAxisPartition -from .partition import PyarrowOnRayFramePartition +from modin.core.dataframe.pandas.partitioning.axis_partition import ( + BaseFrameAxisPartition, +) +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition import ( + PyarrowOnRayFramePartition, +) import ray import pyarrow diff --git a/modin/experimental/engines/pyarrow_on_ray/frame/partition.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition.py similarity index 96% rename from modin/experimental/engines/pyarrow_on_ray/frame/partition.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition.py index 0693b828bb8..c0d9626faa4 100644 --- a/modin/experimental/engines/pyarrow_on_ray/frame/partition.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition.py @@ -14,7 +14,9 @@ """The module defines interface for a partition with PyArrow backend and Ray engine.""" import pandas -from modin.engines.ray.pandas_on_ray.frame.partition import PandasOnRayFramePartition +from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( + PandasOnRayFramePartition, +) import ray import pyarrow diff --git a/modin/experimental/engines/pyarrow_on_ray/frame/partition_manager.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py similarity index 82% rename from modin/experimental/engines/pyarrow_on_ray/frame/partition_manager.py rename to modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py index ce491b995e5..8153b9874f4 100644 --- a/modin/experimental/engines/pyarrow_on_ray/frame/partition_manager.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py @@ -13,14 +13,16 @@ """Module houses class for tracking partitions with PyArrow backend and Ray engine.""" -from modin.engines.ray.generic.frame.partition_manager import ( +from modin.core.execution.ray.generic.partitioning.partition_manager import ( GenericRayFramePartitionManager, ) -from .axis_partition import ( +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.axis_partition import ( PyarrowOnRayFrameColumnPartition, PyarrowOnRayFrameRowPartition, ) -from .partition import PyarrowOnRayFramePartition +from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition import ( + PyarrowOnRayFramePartition, +) class PyarrowOnRayFramePartitionManager(GenericRayFramePartitionManager): diff --git a/modin/experimental/core/storage_formats/__init__.py b/modin/experimental/core/storage_formats/__init__.py new file mode 100644 index 00000000000..d7956bac537 --- /dev/null +++ b/modin/experimental/core/storage_formats/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Experimental Modin's functionality related to storage formats supported.""" diff --git a/modin/experimental/core/storage_formats/omnisci/__init__.py b/modin/experimental/core/storage_formats/omnisci/__init__.py new file mode 100644 index 00000000000..7cb97f9f5db --- /dev/null +++ b/modin/experimental/core/storage_formats/omnisci/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""The module represents the query compiler level for the omnisci backend.""" diff --git a/modin/experimental/backends/omnisci/query_compiler.py b/modin/experimental/core/storage_formats/omnisci/query_compiler.py similarity index 99% rename from modin/experimental/backends/omnisci/query_compiler.py rename to modin/experimental/core/storage_formats/omnisci/query_compiler.py index 90d79bc356b..e28572ef339 100644 --- a/modin/experimental/backends/omnisci/query_compiler.py +++ b/modin/experimental/core/storage_formats/omnisci/query_compiler.py @@ -17,12 +17,12 @@ ``DFAlgQueryCompiler`` is used for lazy DataFrame Algebra based engine. """ -from modin.backends.base.query_compiler import ( +from modin.core.storage_formats.base.query_compiler import ( BaseQueryCompiler, _set_axis as default_axis_setter, _get_axis as default_axis_getter, ) -from modin.backends.pandas.query_compiler import PandasQueryCompiler +from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler from modin.utils import ( _inherit_docstrings, ) diff --git a/modin/experimental/engines/__init__.py b/modin/experimental/engines/__init__.py deleted file mode 100644 index cae6413e559..00000000000 --- a/modin/experimental/engines/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. diff --git a/modin/experimental/engines/omnisci_on_ray/__init__.py b/modin/experimental/engines/omnisci_on_ray/__init__.py deleted file mode 100644 index cae6413e559..00000000000 --- a/modin/experimental/engines/omnisci_on_ray/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. diff --git a/modin/experimental/engines/omnisci_on_ray/test/__init__.py b/modin/experimental/engines/omnisci_on_ray/test/__init__.py deleted file mode 100644 index cae6413e559..00000000000 --- a/modin/experimental/engines/omnisci_on_ray/test/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. diff --git a/modin/spreadsheet/__init__.py b/modin/experimental/spreadsheet/__init__.py similarity index 100% rename from modin/spreadsheet/__init__.py rename to modin/experimental/spreadsheet/__init__.py diff --git a/modin/spreadsheet/general.py b/modin/experimental/spreadsheet/general.py similarity index 100% rename from modin/spreadsheet/general.py rename to modin/experimental/spreadsheet/general.py diff --git a/modin/engines/ray/generic/__init__.py b/modin/experimental/spreadsheet/test/__init__.py similarity index 100% rename from modin/engines/ray/generic/__init__.py rename to modin/experimental/spreadsheet/test/__init__.py diff --git a/modin/spreadsheet/test/test_general.py b/modin/experimental/spreadsheet/test/test_general.py similarity index 98% rename from modin/spreadsheet/test/test_general.py rename to modin/experimental/spreadsheet/test/test_general.py index a09b57f4df3..081380cc40d 100644 --- a/modin/spreadsheet/test/test_general.py +++ b/modin/experimental/spreadsheet/test/test_general.py @@ -14,7 +14,7 @@ import pandas import pytest import modin.pandas as pd -import modin.spreadsheet as mss +import modin.experimental.spreadsheet as mss import numpy as np from modin_spreadsheet import SpreadsheetWidget diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py index 3144363c368..055fae7a85d 100644 --- a/modin/pandas/__init__.py +++ b/modin/pandas/__init__.py @@ -106,7 +106,7 @@ def _update_engine(publisher: Parameter): from modin.config import Backend, CpuCount if publisher.get() == "Ray": - from modin.engines.ray.utils import initialize_ray + from modin.core.execution.ray.common.utils import initialize_ray # With OmniSci backend there is only a single worker per node # and we allow it to work on all cores. @@ -118,7 +118,7 @@ def _update_engine(publisher: Parameter): elif publisher.get() == "Dask": if _is_first_update.get("Dask", True): - from modin.engines.dask.utils import initialize_dask + from modin.core.execution.dask.common.utils import initialize_dask initialize_dask() elif publisher.get() == "Cloudray": @@ -131,7 +131,7 @@ def _update_engine(publisher: Parameter): def init_remote_ray(partition): from ray import ray_constants import modin - from modin.engines.ray.utils import initialize_ray + from modin.core.execution.ray.common.utils import initialize_ray modin.set_backends("Ray", partition) initialize_ray( @@ -143,7 +143,7 @@ def init_remote_ray(partition): init_remote_ray(Backend.get()) # import FactoryDispatcher here to initialize IO class # so it doesn't skew read_csv() timings later on - import modin.data_management.factories.dispatcher # noqa: F401 + import modin.core.execution.dispatching.factories.dispatcher # noqa: F401 else: get_connection().modules["modin"].set_backends("Ray", Backend.get()) elif publisher.get() == "Cloudpython": diff --git a/modin/spreadsheet/test/__init__.py b/modin/spreadsheet/test/__init__.py deleted file mode 100644 index cae6413e559..00000000000 --- a/modin/spreadsheet/test/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. diff --git a/modin/sql/connection.py b/modin/sql/connection.py deleted file mode 100644 index 63f8946cc96..00000000000 --- a/modin/sql/connection.py +++ /dev/null @@ -1,64 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. - -from ..pandas import Series, DataFrame - - -class Connection(object): - def __init__(self, name): - self._name = name - self._cursor = None - - def cursor(self): - self._cursor = Cursor() - return self._cursor - - def commit(self): - pass - - def close(self): - self._cursor = None - - -class Cursor(object): - def __init__(self): - self._tables = {} - - def execute(self, query): - split_query = query.split(" ") - if " ".join(split_query[:2]) == "CREATE TABLE": - self._create_table(split_query) - - elif " ".join(split_query[:2]) == "INSERT INTO": - self._insert_into(split_query) - else: - raise NotImplementedError( - "This API is for demonstration purposes only. Coming Soon!" - ) - - def _create_table(self, split_query): - column_names = ( - " ".join(split_query[3:]).replace("(", "").replace(")", "").split(", ") - ) - columns = Series(column_names) - self._tables[split_query[2]] = DataFrame(columns=columns) - - def _insert_into(self, split_query): - table = self._tables[split_query[2]] - values = " ".join(split_query[4:]).replace("(", "").replace(")", "").split(", ") - to_append = Series([eval(i) for i in values], index=table.columns) - self._tables[split_query[2]] = table.append(to_append, ignore_index=True) - - -def connect(name): - return Connection(name) diff --git a/project_structure.txt b/project_structure.txt index d213a9631b5..04f6df01678 100644 --- a/project_structure.txt +++ b/project_structure.txt @@ -1,29 +1,6 @@ .modin/ - .config/ - .distributed/ - .pandas/ - .test/ - .rest files + # left `core` folder only since it mostly has changes .core/ - .storage_formats/ - .base/ - .query_compiler.py # BaseQueryCompiler - .pandas/ - .base/ # or default (DefaultPandasQueryCompiler) - .query_compiler.py # DefaultPandasQueryCompiler - .kernels.py - .query_compiler.py # PandasQueryCompiler - .parsers.py - .utils.py # data_management/utils moved here - .io/ - .column_stores/ - . # the same as it is now - .sql/ - . # the same as it is now - .text/ - . # the same as it is now - .file_dispatcher.py # the same as it is now - .io.py # the same as it is now .dataframe/ .algebra/ ./d2p @@ -35,67 +12,48 @@ .explode.py .groupby.py .etc. - .execution/ - .ray/ - .generic/ - .dataframe/ - .dataframe.py # GenericRayDataframe - .metadata.py # GenericRayDataframeMetadata - .partition_management/ - .axis_partition.py # GenericRayDataframeAxisPartition - .partition_manager.py # GenericRayDataframeManager - .partition.py # GenericRayDataframePartition - .io/ - .io.py # GenericRayIO - .task_wrapper.py # GenericRayTask - .utils.py # init_ray - .implementations/ - .pandas_on_ray/ - . # the same as it is now - .dask/ - . # the same as it is for ray - .python/ - . # the same as it is for ray - .dispatching/ - .factories/ + .base/ # will contain base (abstract) classes + .pandas/ # contains PandasDataframe classes + .io/ + .column_stores/ + . # the same as it is now + .sql/ + . # the same as it is now + .text/ + . # the same as it is now + .file_dispatcher.py + .io.py + .storage_formats/ + .base/ + .query_compiler.py # BaseQueryCompiler + .pandas/ + .base/ # or default (DefaultPandasQueryCompiler) + .query_compiler.py # DefaultPandasQueryCompiler + .kernels.py + .query_compiler.py # PandasQueryCompiler + .parsers.py + .utils.py # data_management/utils moved here + .execution/ + .ray/ + .common/ + .task_wrapper.py # RayTask + .utils.py # init_ray + .generic/ + .dataframe/ + .dataframe.py # GenericRayDataframe + .partition_management/ + .axis_partition.py # GenericRayDataframeAxisPartition + .partition_manager.py # GenericRayDataframeManager + .partition.py # GenericRayDataframePartition + .io/ + .io.py # GenericRayIO + .implementations/ + .pandas_on_ray/ . # the same as it is now - .experimental/ - ./cloud - . # the same as it is now - ./pandas - . # the same as it is now - ./sklearn - . # the same as it is now - ./sql - . # the same as it is now - .spreadsheet/ - . # from modin/spreadsheet - ./xgboost - . # the same as it is now - ./core - .storage_formats/ - .omnisci/ - .kernels.py - .query_compiler.py - .pyarrow/ - .kernels.py - .query_compiler.py - .dataframe/ - .execution/ - .ray/ - .omnisci_on_ray/ # this should go to omnisci_on_native in execution/native folder after #3352 is merged - .dataframe/ - .dataframe.py # OmniSciOnRayDataframe - .metadata.py # OmniSciOnRayFrameMetaData - .partition_management/ - .axis_partition.py # OmniSciOnRayFrameAxisPartition - .partition_manager.py # OmniSciOnRayFrameManager - .partition.py # OmniSciOnRayFramePartition - .pyarrow_on_ray/ - .frame/ - .dataframe.py # PyarrowOnRayDataframe - .metadata.py # PyarrowOnRayDataframe - .partition_management/ # or `partitioning` - .axis_partition.py # PyarrowOnRayDataframeAxisPartition - .partition_manager.py # PyarrowOnRayDataframeManager - .partition.py # PyarrowOnRayDataframePartition + .dask/ + . # the same as it is for ray + .python/ + . # the same as it is for ray + .dispatching/ + .factories/ + . # the same as it is now \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 5cb66cac1d7..529149ae514 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,8 +24,8 @@ per-file-ignores = stress_tests/kaggle/*:E402 modin/experimental/pandas/__init__.py:E402 modin/_version.py:T001 - modin/experimental/engines/omnisci_on_ray/frame/omnisci_worker.py:E402 - modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py:E402 + modin/experimental/core/execution/ray/implementations/omnisci_on_ray/omnisci_worker.py:E402 + modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py:E402 [coverage:run] source = @@ -41,21 +41,20 @@ omit = modin/test/* modin/config/test/* modin/pandas/test/* - modin/data_management/test/* - modin/data_management/factories/test/* + modin/core/execution/dispatching/factories/test/* modin/experimental/cloud/test/* modin/experimental/pandas/test/* - modin/experimental/engines/omnisci_on_ray/test/* + modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/* # Plotting is not tested modin/pandas/plotting.py # Skip Gandiva because it is experimental - modin/experimental/engines/pyarrow_on_ray/* - modin/backends/pyarrow/* + modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/* + modin/core/storage_formats/pyarrow/* # Skip CUDF tests - modin/backends/cudf/* - modin/engines/ray/cudf_on_ray/* - modin/engines/ray/cudf_on_ray/frame/* - modin/engines/ray/cudf_on_ray/series/* + modin/storage_formats/cudf/* + modin/core/execution/ray/implementations/cudf_on_ray/* + modin/core/execution/ray/implementations/cudf_on_ray/frame/* + modin/core/execution/ray/implementations/cudf_on_ray/series/* parallel = True [coverage:report] From b72d1faa0b0d72d78fc8d663c962e89d68d0abe4 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 11:52:17 +0300 Subject: [PATCH 04/18] REFACTOR-#2957: Fix flake8 job Signed-off-by: Igoshev, Yaroslav --- modin/conftest.py | 8 ++++---- modin/core/dataframe/algebra/d2p/__init__.py | 2 +- modin/core/dataframe/algebra/operators/__init__.py | 2 +- .../core/execution/ray/implementations/__init__.py | 14 ++++++++++++++ .../pandas_on_ray/partitioning/__init__.py | 12 ++++++++++++ modin/experimental/core/execution/ray/__init__.py | 2 ++ .../core/execution/ray/implementations/__init__.py | 14 ++++++++++++++ .../omnisci_on_ray/partitioning/__init__.py | 12 ++++++++++++ .../implementations/pyarrow_on_ray/io/__init__.py | 13 +++++++++++++ 9 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 modin/experimental/core/execution/ray/implementations/__init__.py diff --git a/modin/conftest.py b/modin/conftest.py index 82b3f031bed..fa732d0a356 100644 --- a/modin/conftest.py +++ b/modin/conftest.py @@ -44,13 +44,13 @@ def _saving_make_api_url(token, _make_api_url=modin.utils._make_api_url): import modin.config # noqa: E402 from modin.config import IsExperimental, TestRayClient # noqa: E402 -from modin.core.storage_formats import ( +from modin.core.storage_formats import ( # noqa: E402 PandasQueryCompiler, BaseQueryCompiler, -) # noqa: E402 -from modin.core.execution.python.implementations.pandas_on_python.io import ( +) +from modin.core.execution.python.implementations.pandas_on_python.io import ( # noqa: E402 PandasOnPythonIO, -) # noqa: E402 +) from modin.core.execution.dispatching.factories import factories # noqa: E402 from modin.utils import get_current_backend # noqa: E402 from modin.pandas.test.utils import ( # noqa: E402 diff --git a/modin/core/dataframe/algebra/d2p/__init__.py b/modin/core/dataframe/algebra/d2p/__init__.py index 8575590ad20..2d1e3b9c730 100644 --- a/modin/core/dataframe/algebra/d2p/__init__.py +++ b/modin/core/dataframe/algebra/d2p/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Function module provides templates for a query compiler default-to-pandas methods.""" +"""d2p module provides templates for a query compiler default-to-pandas methods.""" from modin.core.dataframe.algebra.d2p.dataframe_default import DataFrameDefault from modin.core.dataframe.algebra.d2p.datetime_default import DateTimeDefault diff --git a/modin/core/dataframe/algebra/operators/__init__.py b/modin/core/dataframe/algebra/operators/__init__.py index ccc3b6402cd..ef4182b49ef 100644 --- a/modin/core/dataframe/algebra/operators/__init__.py +++ b/modin/core/dataframe/algebra/operators/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Function module provides template for a query compiler methods for a set of common operations.""" +"""Operators module provides template for a query compiler methods for a set of common operations.""" from modin.core.dataframe.algebra.operators.function import Function from modin.core.dataframe.algebra.operators.mapfunction import MapFunction diff --git a/modin/core/execution/ray/implementations/__init__.py b/modin/core/execution/ray/implementations/__init__.py index e69de29bb2d..899e8960ab4 100644 --- a/modin/core/execution/ray/implementations/__init__.py +++ b/modin/core/execution/ray/implementations/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Modin's functionality related to Ray execution engine for specific storage format implementations.""" diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py index e69de29bb2d..cae6413e559 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py @@ -0,0 +1,12 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. diff --git a/modin/experimental/core/execution/ray/__init__.py b/modin/experimental/core/execution/ray/__init__.py index cae6413e559..87d8f6733e2 100644 --- a/modin/experimental/core/execution/ray/__init__.py +++ b/modin/experimental/core/execution/ray/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Experimental Modin's functionality related to Ray execution engine.""" diff --git a/modin/experimental/core/execution/ray/implementations/__init__.py b/modin/experimental/core/execution/ray/implementations/__init__.py new file mode 100644 index 00000000000..0cb7585dc5c --- /dev/null +++ b/modin/experimental/core/execution/ray/implementations/__init__.py @@ -0,0 +1,14 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +"""Experimental Modin's functionality related to Ray execution engine for specific storage format implementations.""" diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py index e69de29bb2d..cae6413e559 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py @@ -0,0 +1,12 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py index 8595d91ed5a..d8d0288ed2b 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py @@ -1,3 +1,16 @@ +# Licensed to Modin Development Team under one or more contributor license agreements. +# See the NOTICE file distributed with this work for additional information regarding +# copyright ownership. The Modin Development Team licenses this file to you under the +# Apache License, Version 2.0 (the "License"); you may not use this file except in +# compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under +# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific language +# governing permissions and limitations under the License. + from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.io import ( PyarrowOnRayIO, ) From 46b69210656fa28859e107147ff6be10eb9f1ee8 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 13:31:36 +0300 Subject: [PATCH 05/18] REFACTOR-#2957: Fix test_api and test_headers Signed-off-by: Igoshev, Yaroslav --- .../pandas_on_dask/io/__init__.py | 2 +- .../dispatching/factories/__init__.py | 2 +- .../pandas_on_python/io/__init__.py | 2 +- .../core/execution/ray/generic/io/__init__.py | 2 +- .../cudf_on_ray/io/__init__.py | 4 +-- .../pandas_on_ray/io/__init__.py | 2 +- modin/core/io/__init__.py | 26 +++++++------- modin/experimental/pandas/__init__.py | 2 +- .../experimental/pandas/{io_exp.py => io.py} | 0 modin/pandas/base.py | 8 +++-- modin/pandas/general.py | 2 +- modin/pandas/groupby.py | 2 +- modin/pandas/io.py | 36 +++++++++---------- modin/pandas/utils.py | 6 ++-- modin/test/test_backends_api.py | 6 +++- modin/test/test_partition_api.py | 2 +- 16 files changed, 56 insertions(+), 48 deletions(-) rename modin/experimental/pandas/{io_exp.py => io.py} (100%) diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py index c5aed1a1599..b7c764d1257 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py @@ -13,7 +13,7 @@ """Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" -from modin.core.execution.dask.implementations.pandas_on_dask.io import PandasOnDaskIO +from .io import PandasOnDaskIO __all__ = [ "PandasOnDaskIO", diff --git a/modin/core/execution/dispatching/factories/__init__.py b/modin/core/execution/dispatching/factories/__init__.py index bdcad80fae7..7641afaac67 100644 --- a/modin/core/execution/dispatching/factories/__init__.py +++ b/modin/core/execution/dispatching/factories/__init__.py @@ -13,7 +13,7 @@ """Factories responsible for dispatching to specific execution.""" -from modin.core.execution.dispatching import factories +from . import factories def _get_remote_engines(): diff --git a/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py index c2ecb0cff2e..df7a1a0a817 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py +++ b/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py @@ -13,7 +13,7 @@ """Functionality implementations with pandas backend and Python engine.""" -from modin.core.execution.python.implementations.pandas_on_python.io import ( +from .io import ( PandasOnPythonIO, ) diff --git a/modin/core/execution/ray/generic/io/__init__.py b/modin/core/execution/ray/generic/io/__init__.py index a67ce9089ca..18f8a6091c3 100644 --- a/modin/core/execution/ray/generic/io/__init__.py +++ b/modin/core/execution/ray/generic/io/__init__.py @@ -11,6 +11,6 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -from modin.core.execution.ray.generic.io import RayIO +from .io import RayIO __all__ = ["RayIO"] diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py index 6937ac9606e..879cd787815 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py @@ -13,10 +13,10 @@ """Package holds implementations of key IO classes using bunch of cuDF and Ray.""" -from modin.core.execution.ray.implementations.cudf_on_ray.io.text.csv_dispatcher import ( +from .text.csv_dispatcher import ( cuDFCSVDispatcher, ) -from modin.core.execution.ray.implementations.cudf_on_ray.io import cuDFOnRayIO +from .io import cuDFOnRayIO __all__ = [ diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py index 5788f2c389f..dbbbe337695 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -11,6 +11,6 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -from modin.core.execution.ray.implementations.pandas_on_ray.io import PandasOnRayIO +from .io import PandasOnRayIO __all__ = ["PandasOnRayIO"] diff --git a/modin/core/io/__init__.py b/modin/core/io/__init__.py index 945c100ddc9..7c84474602d 100644 --- a/modin/core/io/__init__.py +++ b/modin/core/io/__init__.py @@ -13,19 +13,19 @@ """IO functions implementations.""" -from modin.core.io import BaseIO -from modin.core.io.text.csv_dispatcher import CSVDispatcher -from modin.core.io.text.csv_glob_dispatcher import CSVGlobDispatcher -from modin.core.io.text.fwf_dispatcher import FWFDispatcher -from modin.core.io.text.json_dispatcher import JSONDispatcher -from modin.core.io.text.excel_dispatcher import ExcelDispatcher -from modin.core.io.file_dispatcher import FileDispatcher -from modin.core.io.text.text_file_dispatcher import TextFileDispatcher -from modin.core.io.column_stores.parquet_dispatcher import ParquetDispatcher -from modin.core.io.column_stores.hdf_dispatcher import HDFDispatcher -from modin.core.io.column_stores.feather_dispatcher import FeatherDispatcher -from modin.core.io.sql.sql_dispatcher import SQLDispatcher -from modin.core.io.pickle.pickle_dispatcher import PickleExperimentalDispatcher +from .io import BaseIO +from .text.csv_dispatcher import CSVDispatcher +from .text.csv_glob_dispatcher import CSVGlobDispatcher +from .text.fwf_dispatcher import FWFDispatcher +from .text.json_dispatcher import JSONDispatcher +from .text.excel_dispatcher import ExcelDispatcher +from .file_dispatcher import FileDispatcher +from .text.text_file_dispatcher import TextFileDispatcher +from .column_stores.parquet_dispatcher import ParquetDispatcher +from .column_stores.hdf_dispatcher import HDFDispatcher +from .column_stores.feather_dispatcher import FeatherDispatcher +from .sql.sql_dispatcher import SQLDispatcher +from .pickle.pickle_dispatcher import PickleExperimentalDispatcher __all__ = [ "BaseIO", diff --git a/modin/experimental/pandas/__init__.py b/modin/experimental/pandas/__init__.py index 251a7dc153b..2632e1cb785 100644 --- a/modin/experimental/pandas/__init__.py +++ b/modin/experimental/pandas/__init__.py @@ -40,7 +40,7 @@ # in the user code from .numpy_wrap import _CAUGHT_NUMPY # noqa F401 from modin.pandas import * # noqa F401, F403 -from .io_exp import ( # noqa F401 +from .io import ( # noqa F401 read_sql, read_csv_glob, read_pickle_distributed, diff --git a/modin/experimental/pandas/io_exp.py b/modin/experimental/pandas/io.py similarity index 100% rename from modin/experimental/pandas/io_exp.py rename to modin/experimental/pandas/io.py diff --git a/modin/pandas/base.py b/modin/pandas/base.py index 261c301df82..6c7f7c712d3 100644 --- a/modin/pandas/base.py +++ b/modin/pandas/base.py @@ -2528,7 +2528,9 @@ def to_csv( } new_query_compiler = self._query_compiler - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import ( + FactoryDispatcher, + ) return FactoryDispatcher.to_csv(new_query_compiler, **kwargs) @@ -2772,7 +2774,9 @@ def to_sql( # so pandas._to_sql will not write the index to the database as well index = False - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import ( + FactoryDispatcher, + ) FactoryDispatcher.to_sql( new_query_compiler, diff --git a/modin/pandas/general.py b/modin/pandas/general.py index 48a4c03f9fa..892b21b364d 100644 --- a/modin/pandas/general.py +++ b/modin/pandas/general.py @@ -25,7 +25,7 @@ from .dataframe import DataFrame from .series import Series from modin.utils import to_pandas -from modin.backends.base.query_compiler import BaseQueryCompiler +from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler from modin.utils import _inherit_docstrings diff --git a/modin/pandas/groupby.py b/modin/pandas/groupby.py index 4581394cf2a..cb280a98c55 100644 --- a/modin/pandas/groupby.py +++ b/modin/pandas/groupby.py @@ -30,7 +30,7 @@ hashable, wrap_into_list, ) -from modin.backends.base.query_compiler import BaseQueryCompiler +from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler from modin.config import IsExperimental from .series import Series from .utils import is_label diff --git a/modin/pandas/io.py b/modin/pandas/io.py index e3dd182b0ac..f2ab15f482d 100644 --- a/modin/pandas/io.py +++ b/modin/pandas/io.py @@ -53,7 +53,7 @@ def _read(**kwargs): ------- modin.pandas.DataFrame """ - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) pd_obj = FactoryDispatcher.read_csv(**kwargs) @@ -205,7 +205,7 @@ def read_parquet( use_nullable_dtypes: bool = False, **kwargs, ): - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame( @@ -242,7 +242,7 @@ def read_json( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_json(**kwargs)) @@ -267,7 +267,7 @@ def read_gbq( _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) kwargs.update(kwargs.pop("kwargs", {})) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_gbq(**kwargs)) @@ -293,7 +293,7 @@ def read_html( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_html(**kwargs)) @@ -304,7 +304,7 @@ def read_clipboard(sep=r"\s+", **kwargs): # pragma: no cover _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) kwargs.update(kwargs.pop("kwargs", {})) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_clipboard(**kwargs)) @@ -341,7 +341,7 @@ def read_excel( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) intermediate = FactoryDispatcher.read_excel(**kwargs) @@ -371,7 +371,7 @@ def read_hdf( _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) kwargs.update(kwargs.pop("kwargs", {})) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_hdf(**kwargs)) @@ -386,7 +386,7 @@ def read_feather( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_feather(**kwargs)) @@ -409,7 +409,7 @@ def read_stata( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_stata(**kwargs)) @@ -426,7 +426,7 @@ def read_sas( ): # pragma: no cover _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_sas(**kwargs)) @@ -440,7 +440,7 @@ def read_pickle( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_pickle(**kwargs)) @@ -459,7 +459,7 @@ def read_sql( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) if kwargs.get("chunksize") is not None: @@ -479,7 +479,7 @@ def read_fwf( infer_nrows=100, **kwds, ): - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) @@ -508,7 +508,7 @@ def read_sql_table( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_sql_table(**kwargs)) @@ -527,7 +527,7 @@ def read_sql_query( ): _, _, _, kwargs = inspect.getargvalues(inspect.currentframe()) - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame(query_compiler=FactoryDispatcher.read_sql_query(**kwargs)) @@ -539,7 +539,7 @@ def read_spss( usecols: Union[Sequence[str], type(None)] = None, convert_categoricals: bool = True, ): - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) return DataFrame( @@ -555,7 +555,7 @@ def to_pickle( protocol: int = pickle.HIGHEST_PROTOCOL, storage_options: StorageOptions = None, ): - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher Engine.subscribe(_update_engine) if isinstance(obj, DataFrame): diff --git a/modin/pandas/utils.py b/modin/pandas/utils.py index de6026d8462..7557271f8a4 100644 --- a/modin/pandas/utils.py +++ b/modin/pandas/utils.py @@ -37,7 +37,7 @@ def from_non_pandas(df, index, columns, dtype): modin.pandas.DataFrame Converted DataFrame. """ - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher new_qc = FactoryDispatcher.from_non_pandas(df, index, columns, dtype) if new_qc is not None: @@ -61,7 +61,7 @@ def from_pandas(df): modin.pandas.DataFrame A new Modin DataFrame object. """ - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher from .dataframe import DataFrame return DataFrame(query_compiler=FactoryDispatcher.from_pandas(df)) @@ -81,7 +81,7 @@ def from_arrow(at): DataFrame A new Modin DataFrame object. """ - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher from .dataframe import DataFrame return DataFrame(query_compiler=FactoryDispatcher.from_arrow(at)) diff --git a/modin/test/test_backends_api.py b/modin/test/test_backends_api.py index 18a4e7f4ceb..86be61aeab4 100644 --- a/modin/test/test_backends_api.py +++ b/modin/test/test_backends_api.py @@ -11,7 +11,11 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -from modin.backends import BaseQueryCompiler, PandasQueryCompiler, PyarrowQueryCompiler +from modin.core.storage_formats import ( + BaseQueryCompiler, + PandasQueryCompiler, + PyarrowQueryCompiler, +) import pytest diff --git a/modin/test/test_partition_api.py b/modin/test/test_partition_api.py index 019058c10ea..d875c4fcc46 100644 --- a/modin/test/test_partition_api.py +++ b/modin/test/test_partition_api.py @@ -20,7 +20,7 @@ from modin.config import Engine, NPartitions from modin.pandas.test.utils import df_equals from modin.pandas.indexing import compute_sliced_len -from modin.data_management.factories.dispatcher import FactoryDispatcher +from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher PartitionClass = ( FactoryDispatcher.get_factory().io_cls.frame_cls._partition_mgr_cls._partition_class From 18b0c9e4efd1679f8f75b9e51f7392c421c6fd19 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 15:14:59 +0300 Subject: [PATCH 06/18] REFACTOR-#2957: Fix paths in GH actions and doc check Signed-off-by: Igoshev, Yaroslav --- .github/workflows/ci.yml | 57 +++++++++---------- .github/workflows/push.yml | 8 +-- modin/core/dataframe/pandas/__init__.py | 2 +- .../dataframe/pandas/dataframe/__init__.py | 2 +- .../dataframe/pandas/partitioning/__init__.py | 2 +- modin/core/execution/dask/__init__.py | 2 +- .../dask/implementations/__init__.py | 2 +- .../pandas_on_dask/__init__.py | 2 +- .../pandas_on_dask/dataframe/__init__.py | 2 +- .../pandas_on_dask/io/__init__.py | 2 +- .../pandas_on_dask/partitioning/__init__.py | 2 +- modin/core/execution/python/__init__.py | 2 +- .../python/implementations/__init__.py | 2 +- .../pandas_on_python/__init__.py | 2 +- .../pandas_on_python/dataframe/__init__.py | 2 +- .../pandas_on_python/io/__init__.py | 2 +- .../pandas_on_python/partitioning/__init__.py | 2 +- .../core/execution/ray/generic/io/__init__.py | 2 + .../execution/ray/implementations/__init__.py | 2 +- .../implementations/cudf_on_ray/__init__.py | 2 +- .../cudf_on_ray/dataframe/__init__.py | 2 +- .../cudf_on_ray/io/__init__.py | 2 +- .../cudf_on_ray/partitioning/__init__.py | 2 +- .../implementations/pandas_on_ray/__init__.py | 2 + .../pandas_on_ray/dataframe/__init__.py | 2 + .../pandas_on_ray/io/__init__.py | 2 + .../pandas_on_ray/partitioning/__init__.py | 2 + .../execution/ray/implementations/__init__.py | 2 +- .../omnisci_on_ray/__init__.py | 2 + .../omnisci_on_ray/io/__init__.py | 2 + .../implementations/omnisci_on_ray/io/io.py | 12 ++-- .../omnisci_on_ray/partitioning/__init__.py | 2 + .../implementations/pandas_on_ray/__init__.py | 2 +- .../pandas_on_ray/io/__init__.py | 2 +- .../pyarrow_on_ray/__init__.py | 2 +- .../pyarrow_on_ray/dataframe/__init__.py | 2 +- .../pyarrow_on_ray/io/__init__.py | 2 + .../pyarrow_on_ray/partitioning/__init__.py | 2 + .../base/test_internals.py | 0 .../cudf/test_gpu_managers.py | 0 .../cudf/test_internals.py | 0 .../pandas/test_internals.py | 0 42 files changed, 85 insertions(+), 62 deletions(-) rename modin/test/{backends => storage_formats}/base/test_internals.py (100%) rename modin/test/{backends => storage_formats}/cudf/test_gpu_managers.py (100%) rename modin/test/{backends => storage_formats}/cudf/test_internals.py (100%) rename modin/test/{backends => storage_formats}/pandas/test_internals.py (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0c12d4b562..60e3207bc1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,8 +55,8 @@ jobs: modin/pandas/series_utils.py modin/pandas/general.py \ modin/pandas/plotting.py modin/pandas/utils.py \ modin/pandas/iterator.py modin/pandas/indexing.py \ - - run: python scripts/doc_checker.py modin/engines/base/frame - - run: python scripts/doc_checker.py modin/engines/dask + - run: python scripts/doc_checker.py modin/core/dataframe + - run: python scripts/doc_checker.py modin/core/execution/dask - run: | python scripts/doc_checker.py \ modin/pandas/accessor.py modin/pandas/general.py \ @@ -65,18 +65,18 @@ jobs: modin/pandas/series_utils.py modin/pandas/utils.py \ modin/pandas/base.py \ modin/pandas/io.py modin/engines/base/io/io.py \ - modin/engines/base/frame asv_bench/benchmarks/utils \ + asv_bench/benchmarks/utils \ asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \ asv_bench/benchmarks/scalability/__init__.py \ - modin/engines/base/io/column_stores \ - modin/engines/base/io/sql \ - modin/engines/base/io/text \ - modin/engines/base/io/__init__.py \ - modin/engines/base/io/file_dispatcher.py \ - modin/experimental/engines/pandas_on_ray \ - modin/experimental/engines/pyarrow_on_ray \ + modin/core/io/column_stores \ + modin/core/io/sql \ + modin/core/io/text \ + modin/core/io/__init__.py \ + modin/core/io/file_dispatcher.py \ + modin/experimental/core/execution/ray/implementations/pandas_on_ray \ + modin/experimental/core/execution/ray/implementations/pyarrow_on_ray \ modin/pandas/series.py \ - modin/engines/python \ + modin/core/execution/python \ modin/pandas/dataframe.py \ modin/config/__init__.py \ modin/config/__main__.py \ @@ -89,24 +89,23 @@ jobs: python scripts/doc_checker.py modin/experimental/xgboost/__init__.py \ modin/experimental/xgboost/utils.py modin/experimental/xgboost/xgboost.py \ modin/experimental/xgboost/xgboost_ray.py - - run: python scripts/doc_checker.py modin/engines/ray + - run: python scripts/doc_checker.py modin/core/execution/ray - run: | - python scripts/doc_checker.py modin/data_management/functions \ - modin/data_management/factories/factories.py \ - modin/data_management/factories/dispatcher.py \ - modin/data_management/utils.py + python scripts/doc_checker.py modin/core/execution/dispatching/factories/factories.py \ + modin/core/execution/dispatching/factories/dispatcher.py \ - run: python scripts/doc_checker.py scripts/doc_checker.py - run: | - python scripts/doc_checker.py modin/experimental/pandas/io_exp.py \ + python scripts/doc_checker.py modin/experimental/pandas/io.py \ modin/experimental/pandas/numpy_wrap.py modin/experimental/pandas/__init__.py - - run: python scripts/doc_checker.py modin/backends/base - - run: python scripts/doc_checker.py modin/backends/pyarrow - - run: python scripts/doc_checker.py modin/backends/pandas + - run: python scripts/doc_checker.py modin/core/storage_formats/base + - run: python scripts/doc_checker.py modin/core/storage_formats/pyarrow + - run: python scripts/doc_checker.py modin/core/storage_formats/pandas - run: | python scripts/doc_checker.py \ - modin/experimental/engines/omnisci_on_ray/frame \ - modin/experimental/engines/omnisci_on_ray/io.py - - run: python scripts/doc_checker.py modin/experimental/backends/omnisci + modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe \ + modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io \ + modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning \ + - run: python scripts/doc_checker.py modin/experimental/core/storage_formats/omnisci lint-flake8: name: lint (flake8) @@ -239,11 +238,11 @@ jobs: conda info conda list - name: Internals tests - run: python -m pytest modin/data_management/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py + run: python -m pytest modin/core/execution/dispatching/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py - run: python -m pytest modin/config/test - run: python -m pytest modin/test/test_envvar_catcher.py - - run: python -m pytest modin/test/backends/base/test_internals.py - - run: python -m pytest modin/test/backends/pandas/test_internals.py + - run: python -m pytest modin/test/storage_formats/base/test_internals.py + - run: python -m pytest modin/test/storage_formats/pandas/test_internals.py - run: python -m pytest modin/test/test_envvar_npartitions.py - run: python -m pytest -n 2 modin/test/test_partition_api.py - run: python -m pytest modin/test/test_utils.py @@ -280,7 +279,7 @@ jobs: - name: Install HDF5 run: sudo apt update && sudo apt install -y libhdf5-dev - run: pytest modin/experimental/xgboost/test/test_default.py --backend=${{ matrix.backend }} - - run: python -m pytest -n 2 modin/test/backends/base/test_internals.py --backend=${{ matrix.backend }} + - run: python -m pytest -n 2 modin/test/storage_formats/base/test_internals.py --backend=${{ matrix.backend }} - run: pytest -n 2 modin/pandas/test/dataframe/test_binary.py --backend=${{ matrix.backend }} - run: pytest -n 2 modin/pandas/test/dataframe/test_default.py --backend=${{ matrix.backend }} - run: pytest -n 2 modin/pandas/test/dataframe/test_indexing.py --backend=${{ matrix.backend }} @@ -338,7 +337,7 @@ jobs: - name: Install HDF5 run: sudo apt update && sudo apt install -y libhdf5-dev - run: MODIN_BENCHMARK_MODE=True pytest modin/pandas/test/internals/test_benchmark_mode.py - - run: pytest modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py + - run: pytest modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py - run: pytest modin/pandas/test/test_io.py::TestCsv - run: | curl -o codecov https://codecov.io/bash @@ -684,4 +683,4 @@ jobs: run: | conda info conda list - - run: python -m pytest modin/spreadsheet/test/test_general.py + - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0858e583065..69de23560b4 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -23,10 +23,10 @@ jobs: conda info conda list - name: Internals tests - run: python -m pytest modin/data_management/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py + run: python -m pytest modin/core/execution/dispatching/factories/test/test_dispatcher.py modin/experimental/cloud/test/test_cloud.py - run: python -m pytest modin/config/test - run: python -m pytest modin/test/test_envvar_catcher.py - - run: python -m pytest modin/test/backends/pandas/test_internals.py + - run: python -m pytest modin/test/storage_formats/pandas/test_internals.py - run: python -m pytest modin/test/test_envvar_npartitions.py - run: python -m pytest modin/test/test_partition_api.py @@ -115,7 +115,7 @@ jobs: conda list - name: Install HDF5 run: sudo apt update && sudo apt install -y libhdf5-dev - - run: pytest modin/experimental/engines/omnisci_on_ray/test/test_dataframe.py + - run: pytest modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py - run: pytest modin/pandas/test/test_io.py::TestCsv - run: | curl -o codecov https://codecov.io/bash @@ -305,4 +305,4 @@ jobs: run: | conda info conda list - - run: python -m pytest modin/spreadsheet/test/test_general.py + - run: python -m pytest modin/experimental/spreadsheet/test/test_general.py diff --git a/modin/core/dataframe/pandas/__init__.py b/modin/core/dataframe/pandas/__init__.py index 5a2add6dc81..91d5a7d7ab4 100644 --- a/modin/core/dataframe/pandas/__init__.py +++ b/modin/core/dataframe/pandas/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe classes designed for pandas storage format.""" +"""Base Modin Dataframe classes optimized for pandas storage format.""" diff --git a/modin/core/dataframe/pandas/dataframe/__init__.py b/modin/core/dataframe/pandas/dataframe/__init__.py index 556229b52a4..2887bc4785c 100644 --- a/modin/core/dataframe/pandas/dataframe/__init__.py +++ b/modin/core/dataframe/pandas/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe class designed for pandas storage format.""" +"""Base Modin Dataframe class optimized for pandas storage format.""" diff --git a/modin/core/dataframe/pandas/partitioning/__init__.py b/modin/core/dataframe/pandas/partitioning/__init__.py index f0d5bc50a65..0a938aa8dbe 100644 --- a/modin/core/dataframe/pandas/partitioning/__init__.py +++ b/modin/core/dataframe/pandas/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe classes related to its partitioning designed for pandas storage format.""" +"""Base Modin Dataframe classes related to its partitioning and optimized for pandas storage format.""" diff --git a/modin/core/execution/dask/__init__.py b/modin/core/execution/dask/__init__.py index ac0f9b75e15..3c0d8677d20 100644 --- a/modin/core/execution/dask/__init__.py +++ b/modin/core/execution/dask/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" +"""Modin's functionality related to Dask execution engine.""" diff --git a/modin/core/execution/dask/implementations/__init__.py b/modin/core/execution/dask/implementations/__init__.py index ac0f9b75e15..aa8d4b73a02 100644 --- a/modin/core/execution/dask/implementations/__init__.py +++ b/modin/core/execution/dask/implementations/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" +"""Modin's functionality related to Dask execution engine and optimized for specific storage formats.""" diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/__init__.py index 0590385ced3..aeca2f99dba 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/__init__.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``PandasOnDask`` backend.""" +"""Modin's functionality related to Dask execution engine and optimized for pandas storage format.""" diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/__init__.py index 0590385ced3..d95cfcf9a3f 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/__init__.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``PandasOnDask`` backend.""" +"""Base Modin Dataframe class optimized for pandas on Dask execution.""" diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py index b7c764d1257..a81ef2742ee 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" +"""Base IO classes optimized for pandas on Dask execution.""" from .io import PandasOnDaskIO diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py index ac0f9b75e15..d67b93366b4 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``Dask`` as an execution engine.""" +"""Base Modin Dataframe classes related to its partitioning and optimized for pandas on Dask execution.""" diff --git a/modin/core/execution/python/__init__.py b/modin/core/execution/python/__init__.py index d7466faa6f9..4fcbddbb7e7 100644 --- a/modin/core/execution/python/__init__.py +++ b/modin/core/execution/python/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``Python`` as an execution engine.""" +"""Modin's functionality related to Python execution engine.""" diff --git a/modin/core/execution/python/implementations/__init__.py b/modin/core/execution/python/implementations/__init__.py index d7466faa6f9..62b3a07e0ab 100644 --- a/modin/core/execution/python/implementations/__init__.py +++ b/modin/core/execution/python/implementations/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Key base classes to making efficient scale of data with ``Python`` as an execution engine.""" +"""Modin's functionality related to Python execution engine and optimized for specific storage formats.""" diff --git a/modin/core/execution/python/implementations/pandas_on_python/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/__init__.py index 3eefbecb86c..5ce4d52d58f 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/__init__.py +++ b/modin/core/execution/python/implementations/pandas_on_python/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Frame functionality implementations with pandas backend and Python engine.""" +"""Modin's functionality related to Python execution engine and optimized for pandas storage format.""" diff --git a/modin/core/execution/python/implementations/pandas_on_python/dataframe/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/dataframe/__init__.py index b6600935d72..e47c3dc17da 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/dataframe/__init__.py +++ b/modin/core/execution/python/implementations/pandas_on_python/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Functionality implementations with pandas backend and Python engine.""" +"""Base Modin Dataframe class optimized for pandas on Python execution.""" diff --git a/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py index df7a1a0a817..f1edc3ff3db 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py +++ b/modin/core/execution/python/implementations/pandas_on_python/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Functionality implementations with pandas backend and Python engine.""" +"""Base IO classes optimized for pandas on Python execution.""" from .io import ( PandasOnPythonIO, diff --git a/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py index b6600935d72..e2fa73aba67 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Functionality implementations with pandas backend and Python engine.""" +"""Base Modin Dataframe classes related to its partitioning and optimized for pandas on Python execution.""" diff --git a/modin/core/execution/ray/generic/io/__init__.py b/modin/core/execution/ray/generic/io/__init__.py index 18f8a6091c3..0cf383b538f 100644 --- a/modin/core/execution/ray/generic/io/__init__.py +++ b/modin/core/execution/ray/generic/io/__init__.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. +"""Generic IO functionality for Ray execution engine.""" + from .io import RayIO __all__ = ["RayIO"] diff --git a/modin/core/execution/ray/implementations/__init__.py b/modin/core/execution/ray/implementations/__init__.py index 899e8960ab4..d9c425b53c3 100644 --- a/modin/core/execution/ray/implementations/__init__.py +++ b/modin/core/execution/ray/implementations/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Modin's functionality related to Ray execution engine for specific storage format implementations.""" +"""Modin's functionality related to Ray execution engine and optimized for specific storage formats.""" diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/__init__.py index 2294193115a..c6ef522f448 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/__init__.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Package holds implementations of key classes using bunch of cuDF and Ray.""" +"""Modin's functionality related to Ray execution engine and optimized for cuDF storage format.""" diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/__init__.py index 174c5c4b6ce..44cda89c828 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/__init__.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Package holds implementations of key classes to making efficient scale of data using cuDF on Ray.""" +"""Base Modin Dataframe class optimized for cuDF on Ray execution.""" diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py index 879cd787815..11a88fd6bf8 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Package holds implementations of key IO classes using bunch of cuDF and Ray.""" +"""Base IO classes optimized for cuDF on Ray execution.""" from .text.csv_dispatcher import ( cuDFCSVDispatcher, diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py index 174c5c4b6ce..e4f29ad4389 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Package holds implementations of key classes to making efficient scale of data using cuDF on Ray.""" +"""Base Modin Dataframe classes related to its partitioning and optimized for cuDF on Ray execution.""" diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/__init__.py index cae6413e559..d66d85a82f8 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/__init__.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Modin's functionality related to Ray execution engine and optimized for pandas storage format.""" diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/__init__.py index cae6413e559..712ab3cb30a 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/__init__.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Base Modin Dataframe class optimized for pandas on Ray execution.""" diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py index dbbbe337695..fb1700bdb38 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. +"""Base IO classes optimized for pandas on Ray execution.""" + from .io import PandasOnRayIO __all__ = ["PandasOnRayIO"] diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py index cae6413e559..0b36dc4b84c 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Base Modin Dataframe classes related to its partitioning and optimized for pandas on Ray execution.""" diff --git a/modin/experimental/core/execution/ray/implementations/__init__.py b/modin/experimental/core/execution/ray/implementations/__init__.py index 0cb7585dc5c..77b5a7f5ac3 100644 --- a/modin/experimental/core/execution/ray/implementations/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to Ray execution engine for specific storage format implementations.""" +"""Experimental Modin's functionality related to Ray execution engine and optimized for specific storage formats.""" diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py index cae6413e559..0c0d0408d76 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Experimental Modin's functionality related to Ray execution engine and optimized for omnisci storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py index a98d57f5845..850daae3e4d 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. +"""Base IO classes optimized for omnisci on Ray execution.""" + from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.io import ( OmnisciOnRayIO, ) diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/io.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/io.py index 40b6e907c61..ddf4d8e10a4 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/io.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/io.py @@ -23,11 +23,15 @@ import inspect import os -from modin.experimental.backends.omnisci.query_compiler import DFAlgQueryCompiler -from modin.engines.ray.generic.io import RayIO -from modin.experimental.engines.omnisci_on_ray.frame.data import OmnisciOnRayFrame +from modin.experimental.core.storage_formats.omnisci.query_compiler import ( + DFAlgQueryCompiler, +) +from modin.core.execution.ray.generic.io import RayIO +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.dataframe import ( + OmnisciOnRayFrame, +) from modin.error_message import ErrorMessage -from modin.engines.base.io.text.text_file_dispatcher import TextFileDispatcher +from modin.core.io.text.text_file_dispatcher import TextFileDispatcher from pyarrow.csv import read_csv, ParseOptions, ConvertOptions, ReadOptions import pyarrow as pa diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py index cae6413e559..1e057de03fd 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Base Modin Dataframe classes related to its repartitioning and optimized for omnisci on Ray execution.""" diff --git a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py index e77100a581b..1ea107042ff 100644 --- a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental functionality implementations with pandas backend and Ray engine.""" +"""Experimental Modin's functionality related to Ray execution engine and optimized for pandas storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py index 428d5330180..17e63ec0892 100644 --- a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental functionality implementations with pandas backend and Ray engine.""" +"""Base IO classes optimized for pandas on Ray execution.""" from modin.experimental.core.execution.ray.implementations.pandas_on_ray.io import ( ExperimentalPandasOnRayIO, diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py index c1bc5eafd8e..03058192850 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental functionality implementations with PyArrow backend and Ray engine.""" +"""Experimental Modin's functionality related to Ray execution engine and optimized for pyarrow storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py index 7a8076d80e1..cbb3b6dde5b 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental frame functionality implementations with PyArrow backend and Ray engine.""" +"""Base Modin Dataframe class optimized for pyarrow on Ray execution.""" diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py index d8d0288ed2b..cd8d0f465c8 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. +"""Base IO classes optimized for pyarrow on Ray execution.""" + from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.io import ( PyarrowOnRayIO, ) diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py index cae6413e559..12c9bb634db 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Modin's functionality related to Ray execution engine and optimized for pyarrow storage format.""" diff --git a/modin/test/backends/base/test_internals.py b/modin/test/storage_formats/base/test_internals.py similarity index 100% rename from modin/test/backends/base/test_internals.py rename to modin/test/storage_formats/base/test_internals.py diff --git a/modin/test/backends/cudf/test_gpu_managers.py b/modin/test/storage_formats/cudf/test_gpu_managers.py similarity index 100% rename from modin/test/backends/cudf/test_gpu_managers.py rename to modin/test/storage_formats/cudf/test_gpu_managers.py diff --git a/modin/test/backends/cudf/test_internals.py b/modin/test/storage_formats/cudf/test_internals.py similarity index 100% rename from modin/test/backends/cudf/test_internals.py rename to modin/test/storage_formats/cudf/test_internals.py diff --git a/modin/test/backends/pandas/test_internals.py b/modin/test/storage_formats/pandas/test_internals.py similarity index 100% rename from modin/test/backends/pandas/test_internals.py rename to modin/test/storage_formats/pandas/test_internals.py From 9773dd058416f708c076e1683352f60c4750f109 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 15:34:39 +0300 Subject: [PATCH 07/18] REFACTOR-#2957: Fix doc check Signed-off-by: Igoshev, Yaroslav --- .github/workflows/ci.yml | 3 ++- modin/core/dataframe/algebra/d2p/__init__.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60e3207bc1d..30357781217 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,10 +64,11 @@ jobs: modin/pandas/iterator.py modin/pandas/plotting.py \ modin/pandas/series_utils.py modin/pandas/utils.py \ modin/pandas/base.py \ - modin/pandas/io.py modin/engines/base/io/io.py \ + modin/pandas/io.py \ asv_bench/benchmarks/utils \ asv_bench/benchmarks/__init__.py asv_bench/benchmarks/io/__init__.py \ asv_bench/benchmarks/scalability/__init__.py \ + modin/core/io/io.py \ modin/core/io/column_stores \ modin/core/io/sql \ modin/core/io/text \ diff --git a/modin/core/dataframe/algebra/d2p/__init__.py b/modin/core/dataframe/algebra/d2p/__init__.py index 2d1e3b9c730..a9b7a0f21e5 100644 --- a/modin/core/dataframe/algebra/d2p/__init__.py +++ b/modin/core/dataframe/algebra/d2p/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""d2p module provides templates for a query compiler default-to-pandas methods.""" +"""Module d2p provides templates for a query compiler default-to-pandas methods.""" from modin.core.dataframe.algebra.d2p.dataframe_default import DataFrameDefault from modin.core.dataframe.algebra.d2p.datetime_default import DateTimeDefault From 862e27d8c7061f76358ef9cdad202a43a32d8f03 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 16:19:25 +0300 Subject: [PATCH 08/18] REFACTOR-#2957: Fix paths Signed-off-by: Igoshev, Yaroslav --- .../execution/dispatching/factories/test/test_dispatcher.py | 4 ++-- modin/core/storage_formats/pyarrow/query_compiler.py | 2 +- .../execution/ray/implementations/omnisci_on_ray/__init__.py | 2 +- .../ray/implementations/omnisci_on_ray/io/__init__.py | 4 ++-- .../implementations/omnisci_on_ray/partitioning/__init__.py | 2 +- .../ray/implementations/pandas_on_ray/io/__init__.py | 2 +- .../ray/implementations/pyarrow_on_ray/io/__init__.py | 4 +--- modin/experimental/pandas/io.py | 4 ++-- modin/experimental/pandas/numpy_wrap.py | 2 +- 9 files changed, 12 insertions(+), 14 deletions(-) diff --git a/modin/core/execution/dispatching/factories/test/test_dispatcher.py b/modin/core/execution/dispatching/factories/test/test_dispatcher.py index a1762859d24..6c7b94aee08 100644 --- a/modin/core/execution/dispatching/factories/test/test_dispatcher.py +++ b/modin/core/execution/dispatching/factories/test/test_dispatcher.py @@ -16,11 +16,11 @@ from modin.config import Engine, Backend from modin import set_backends -from modin.data_management.factories.dispatcher import ( +from modin.core.execution.dispatching.factories.dispatcher import ( FactoryDispatcher, FactoryNotFoundError, ) -from modin.data_management.factories import factories +from modin.core.execution.dispatching.factories import factories import modin.pandas as pd diff --git a/modin/core/storage_formats/pyarrow/query_compiler.py b/modin/core/storage_formats/pyarrow/query_compiler.py index ca205a25864..b865d599be7 100644 --- a/modin/core/storage_formats/pyarrow/query_compiler.py +++ b/modin/core/storage_formats/pyarrow/query_compiler.py @@ -262,7 +262,7 @@ def query_builder(arrow_table, **kwargs): # it is no longer needed to wrap function to apply. func = self._prepare_method(query_builder, **kwargs) # FIXME: `PandasQueryCompiler._map_across_full_axis` was removed in #721. - # This method call should be replaced to its equivalent from `data_management.function` + # This method call should be replaced to its equivalent from `operators.function` new_data = self._map_across_full_axis(1, func) # Query removes rows, so we need to update the index new_index = self._compute_index(0, new_data, False) diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py index 0c0d0408d76..f816f3a2414 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to Ray execution engine and optimized for omnisci storage format.""" +"""Experimental Modin's functionality related to Ray execution engine and optimized for OmniSci storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py index 850daae3e4d..887ce6a8df7 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/io/__init__.py @@ -11,9 +11,9 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base IO classes optimized for omnisci on Ray execution.""" +"""Base IO classes optimized for OmniSci on Ray execution.""" -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.io import ( +from .io import ( OmnisciOnRayIO, ) diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py index 1e057de03fd..978e19150ff 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe classes related to its repartitioning and optimized for omnisci on Ray execution.""" +"""Base Modin Dataframe classes related to its repartitioning and optimized for OmniSci on Ray execution.""" diff --git a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py index 17e63ec0892..bf09f9b5d7c 100644 --- a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -13,7 +13,7 @@ """Base IO classes optimized for pandas on Ray execution.""" -from modin.experimental.core.execution.ray.implementations.pandas_on_ray.io import ( +from .io import ( ExperimentalPandasOnRayIO, ) diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py index cd8d0f465c8..3e82c2e4ec9 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py @@ -13,8 +13,6 @@ """Base IO classes optimized for pyarrow on Ray execution.""" -from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.io import ( - PyarrowOnRayIO, -) +from .io import PyarrowOnRayIO __all__ = ["PyarrowOnRayIO"] diff --git a/modin/experimental/pandas/io.py b/modin/experimental/pandas/io.py index 9004c68f2f9..94843ca5977 100644 --- a/modin/experimental/pandas/io.py +++ b/modin/experimental/pandas/io.py @@ -24,7 +24,7 @@ from . import DataFrame from modin.config import IsExperimental, Engine -from modin.data_management.factories.dispatcher import FactoryDispatcher +from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher from ...pandas import _update_engine @@ -303,7 +303,7 @@ def to_pickle_distributed( this argument with a non-fsspec URL. See the fsspec and backend storage implementation docs for the set of allowed keys and values. """ - from modin.data_management.factories.dispatcher import FactoryDispatcher + from modin.core.execution.dispatching.factories.dispatcher import FactoryDispatcher obj = self Engine.subscribe(_update_engine) diff --git a/modin/experimental/pandas/numpy_wrap.py b/modin/experimental/pandas/numpy_wrap.py index bdb2608b444..f56cc422100 100644 --- a/modin/experimental/pandas/numpy_wrap.py +++ b/modin/experimental/pandas/numpy_wrap.py @@ -29,7 +29,7 @@ import types import copyreg from modin.config import Engine - from modin.data_management.factories import REMOTE_ENGINES + from modin.core.execution.dispatching.factories import REMOTE_ENGINES import modin import pandas import os From e4baf7112672267e27bdd05902cdd624431a8b79 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 16:48:47 +0300 Subject: [PATCH 09/18] REFACTOR-#2957: Fix spelling and move spreadsheet to exp Signed-off-by: Igoshev, Yaroslav --- .github/workflows/ci.yml | 1 + .github/workflows/push.yml | 1 + .../ray/implementations/omnisci_on_ray/test/test_dataframe.py | 2 +- .../execution/ray/implementations/pyarrow_on_ray/__init__.py | 2 +- .../ray/implementations/pyarrow_on_ray/dataframe/__init__.py | 2 +- .../execution/ray/implementations/pyarrow_on_ray/io/__init__.py | 2 +- .../ray/implementations/pyarrow_on_ray/partitioning/__init__.py | 2 +- modin/experimental/core/storage_formats/omnisci/__init__.py | 2 +- 8 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30357781217..0137ea6c7bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -667,6 +667,7 @@ jobs: python-version: [ "3.7", "3.8" ] engine: ["ray", "dask"] env: + MODIN_EXPERIMENTAL: "True" MODIN_ENGINE: ${{matrix.engine}} name: test-spreadsheet (engine ${{matrix.engine}}, python ${{matrix.python-version}}) steps: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 69de23560b4..5dd68122ab0 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -288,6 +288,7 @@ jobs: python-version: [ "3.7", "3.8" ] engine: ["ray", "dask"] env: + MODIN_EXPERIMENTAL: "True" MODIN_ENGINE: ${{matrix.engine}} name: test-spreadsheet (engine ${{matrix.engine}}, python ${{matrix.python-version}}) steps: diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py index 7ad011c029f..d352010f581 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py @@ -37,7 +37,7 @@ eval_io, ) -from modin.experimental.engines.omnisci_on_ray.frame.partition_manager import ( +from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition_manager import ( OmnisciOnRayFramePartitionManager, ) diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py index 03058192850..ba844fb5909 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to Ray execution engine and optimized for pyarrow storage format.""" +"""Experimental Modin's functionality related to Ray execution engine and optimized for PyArrow storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py index cbb3b6dde5b..e1308335d30 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe class optimized for pyarrow on Ray execution.""" +"""Base Modin Dataframe class optimized for PyArrow on Ray execution.""" diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py index 3e82c2e4ec9..ca2ceae597a 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base IO classes optimized for pyarrow on Ray execution.""" +"""Base IO classes optimized for PyArrow on Ray execution.""" from .io import PyarrowOnRayIO diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py index 12c9bb634db..038365a651a 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Modin's functionality related to Ray execution engine and optimized for pyarrow storage format.""" +"""Modin's functionality related to Ray execution engine and optimized for PyArrow storage format.""" diff --git a/modin/experimental/core/storage_formats/omnisci/__init__.py b/modin/experimental/core/storage_formats/omnisci/__init__.py index 7cb97f9f5db..b768baebee1 100644 --- a/modin/experimental/core/storage_formats/omnisci/__init__.py +++ b/modin/experimental/core/storage_formats/omnisci/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""The module represents the query compiler level for the omnisci backend.""" +"""The module represents the query compiler level for the OmniSci backend.""" From 0b867dd1c34567b50ce7ed04ea4a40b35c9eb17e Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 17:51:56 +0300 Subject: [PATCH 10/18] REFACTOR-#2957: Apply additional changes Signed-off-by: Igoshev, Yaroslav --- modin/core/dataframe/algebra/d2p/__init__.py | 22 +++++++++---------- .../core/dataframe/algebra/d2p/any_default.py | 2 +- .../dataframe/algebra/d2p/binary_default.py | 2 +- .../core/dataframe/algebra/d2p/cat_default.py | 2 +- .../algebra/d2p/dataframe_default.py | 2 +- .../dataframe/algebra/d2p/datetime_default.py | 2 +- .../dataframe/algebra/d2p/groupby_default.py | 2 +- .../dataframe/algebra/d2p/resample_default.py | 2 +- .../dataframe/algebra/d2p/rolling_default.py | 2 +- .../dataframe/algebra/d2p/series_default.py | 2 +- .../core/dataframe/algebra/d2p/str_default.py | 2 +- .../dataframe/algebra/operators/__init__.py | 14 ++++++------ .../algebra/operators/binary_function.py | 2 +- .../algebra/operators/foldfunction.py | 2 +- .../algebra/operators/groupby_function.py | 2 +- .../algebra/operators/mapfunction.py | 2 +- .../algebra/operators/mapreducefunction.py | 2 +- .../algebra/operators/reductionfunction.py | 2 +- .../pandas_on_dask/dataframe/dataframe.py | 2 +- .../partitioning/axis_partition.py | 4 +--- .../partitioning/partition_manager.py | 6 ++--- .../pandas_on_python/dataframe/dataframe.py | 2 +- .../partitioning/axis_partition.py | 4 +--- .../partitioning/partition_manager.py | 6 ++--- .../cudf_on_ray/dataframe/dataframe.py | 4 ++-- .../partitioning/axis_partition.py | 4 +--- .../partitioning/partition_manager.py | 6 ++--- .../pandas_on_ray/dataframe/dataframe.py | 2 +- .../partitioning/axis_partition.py | 4 +--- .../partitioning/partition_manager.py | 8 +++---- modin/core/io/text/excel_dispatcher.py | 2 +- modin/core/storage_formats/__init__.py | 6 ++--- modin/core/storage_formats/base/__init__.py | 2 +- modin/core/storage_formats/cudf/__init__.py | 2 +- modin/core/storage_formats/pandas/__init__.py | 2 +- .../storage_formats/pandas/query_compiler.py | 6 ++--- .../dataframe/calcite_algebra.py | 4 +--- .../dataframe/calcite_builder.py | 6 ++--- .../dataframe/calcite_serializer.py | 4 ++-- .../omnisci_on_ray/dataframe/dataframe.py | 2 +- .../omnisci_on_ray/dataframe/df_algebra.py | 4 +--- .../partitioning/axis_partition.py | 4 +--- .../partitioning/partition_manager.py | 8 +++---- .../omnisci_on_ray/test/test_dataframe.py | 2 +- .../pyarrow_on_ray/dataframe/dataframe.py | 2 +- .../partitioning/axis_partition.py | 4 +--- .../partitioning/partition_manager.py | 4 ++-- .../core/storage_formats/omnisci/__init__.py | 4 ++++ modin/pandas/utils.py | 2 +- 49 files changed, 85 insertions(+), 105 deletions(-) diff --git a/modin/core/dataframe/algebra/d2p/__init__.py b/modin/core/dataframe/algebra/d2p/__init__.py index a9b7a0f21e5..9d7b977bac4 100644 --- a/modin/core/dataframe/algebra/d2p/__init__.py +++ b/modin/core/dataframe/algebra/d2p/__init__.py @@ -13,17 +13,17 @@ """Module d2p provides templates for a query compiler default-to-pandas methods.""" -from modin.core.dataframe.algebra.d2p.dataframe_default import DataFrameDefault -from modin.core.dataframe.algebra.d2p.datetime_default import DateTimeDefault -from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault -from modin.core.dataframe.algebra.d2p.str_default import StrDefault -from modin.core.dataframe.algebra.d2p.binary_default import BinaryDefault -from modin.core.dataframe.algebra.d2p.any_default import AnyDefault -from modin.core.dataframe.algebra.d2p.resample_default import ResampleDefault -from modin.core.dataframe.algebra.d2p.rolling_default import RollingDefault -from modin.core.dataframe.algebra.d2p.default import DefaultMethod -from modin.core.dataframe.algebra.d2p.cat_default import CatDefault -from modin.core.dataframe.algebra.d2p.groupby_default import GroupByDefault +from .dataframe_default import DataFrameDefault +from .datetime_default import DateTimeDefault +from .series_default import SeriesDefault +from .str_default import StrDefault +from .binary_default import BinaryDefault +from .any_default import AnyDefault +from .resample_default import ResampleDefault +from .rolling_default import RollingDefault +from .default import DefaultMethod +from .cat_default import CatDefault +from .groupby_default import GroupByDefault __all__ = [ "DataFrameDefault", diff --git a/modin/core/dataframe/algebra/d2p/any_default.py b/modin/core/dataframe/algebra/d2p/any_default.py index 73a2b7eb399..80ce5b07f1f 100644 --- a/modin/core/dataframe/algebra/d2p/any_default.py +++ b/modin/core/dataframe/algebra/d2p/any_default.py @@ -15,7 +15,7 @@ """Module houses default functions builder class.""" -from modin.core.dataframe.algebra.d2p.default import DefaultMethod +from .default import DefaultMethod class ObjTypeDeterminer: diff --git a/modin/core/dataframe/algebra/d2p/binary_default.py b/modin/core/dataframe/algebra/d2p/binary_default.py index a9b2a2e8ee6..e337f7df558 100644 --- a/modin/core/dataframe/algebra/d2p/binary_default.py +++ b/modin/core/dataframe/algebra/d2p/binary_default.py @@ -13,7 +13,7 @@ """Module houses default binary functions builder class.""" -from modin.core.dataframe.algebra.d2p.any_default import AnyDefault +from .any_default import AnyDefault import pandas from pandas.core.dtypes.common import is_list_like diff --git a/modin/core/dataframe/algebra/d2p/cat_default.py b/modin/core/dataframe/algebra/d2p/cat_default.py index e5c9366d7dc..172fc22502b 100644 --- a/modin/core/dataframe/algebra/d2p/cat_default.py +++ b/modin/core/dataframe/algebra/d2p/cat_default.py @@ -13,7 +13,7 @@ """Module houses default applied-on-category functions builder class.""" -from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault +from .series_default import SeriesDefault class CatDefault(SeriesDefault): diff --git a/modin/core/dataframe/algebra/d2p/dataframe_default.py b/modin/core/dataframe/algebra/d2p/dataframe_default.py index 05bc87ee85d..95746d4b580 100644 --- a/modin/core/dataframe/algebra/d2p/dataframe_default.py +++ b/modin/core/dataframe/algebra/d2p/dataframe_default.py @@ -15,7 +15,7 @@ # FIXME: This whole module is duplicating the logic of `default.py` and should be removed. -from modin.core.dataframe.algebra.d2p.default import DefaultMethod +from .default import DefaultMethod from modin.utils import _inherit_docstrings import pandas diff --git a/modin/core/dataframe/algebra/d2p/datetime_default.py b/modin/core/dataframe/algebra/d2p/datetime_default.py index 17879d1cc1e..c81a655012e 100644 --- a/modin/core/dataframe/algebra/d2p/datetime_default.py +++ b/modin/core/dataframe/algebra/d2p/datetime_default.py @@ -13,7 +13,7 @@ """Module houses default applied-on-datetime functions builder class.""" -from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault +from .series_default import SeriesDefault class DateTimeDefault(SeriesDefault): diff --git a/modin/core/dataframe/algebra/d2p/groupby_default.py b/modin/core/dataframe/algebra/d2p/groupby_default.py index 6d56532eb7e..4999283e18f 100644 --- a/modin/core/dataframe/algebra/d2p/groupby_default.py +++ b/modin/core/dataframe/algebra/d2p/groupby_default.py @@ -13,7 +13,7 @@ """Module houses default GroupBy functions builder class.""" -from modin.core.dataframe.algebra.d2p.default import DefaultMethod +from .default import DefaultMethod import pandas diff --git a/modin/core/dataframe/algebra/d2p/resample_default.py b/modin/core/dataframe/algebra/d2p/resample_default.py index f455e02fec5..b047a08d840 100644 --- a/modin/core/dataframe/algebra/d2p/resample_default.py +++ b/modin/core/dataframe/algebra/d2p/resample_default.py @@ -13,7 +13,7 @@ """Module houses default Resamle functions builder class.""" -from modin.core.dataframe.algebra.d2p.default import DefaultMethod +from .default import DefaultMethod # FIXME: there is no sence of keeping `Resampler` and `ResampleDefault` logic in a different diff --git a/modin/core/dataframe/algebra/d2p/rolling_default.py b/modin/core/dataframe/algebra/d2p/rolling_default.py index 6d84a810d20..4c3debadc6c 100644 --- a/modin/core/dataframe/algebra/d2p/rolling_default.py +++ b/modin/core/dataframe/algebra/d2p/rolling_default.py @@ -13,7 +13,7 @@ """Module houses default Rolling functions builder class.""" -from modin.core.dataframe.algebra.d2p.default import DefaultMethod +from .default import DefaultMethod # FIXME: there is no sence of keeping `Rolling` and `RollingDefault` logic in a different diff --git a/modin/core/dataframe/algebra/d2p/series_default.py b/modin/core/dataframe/algebra/d2p/series_default.py index 449521ad27a..16514967951 100644 --- a/modin/core/dataframe/algebra/d2p/series_default.py +++ b/modin/core/dataframe/algebra/d2p/series_default.py @@ -13,7 +13,7 @@ """Module houses default Series functions builder class.""" -from modin.core.dataframe.algebra.d2p.any_default import AnyDefault +from .any_default import AnyDefault class SeriesDefault(AnyDefault): diff --git a/modin/core/dataframe/algebra/d2p/str_default.py b/modin/core/dataframe/algebra/d2p/str_default.py index 6e9a32c004d..74e744f8d76 100644 --- a/modin/core/dataframe/algebra/d2p/str_default.py +++ b/modin/core/dataframe/algebra/d2p/str_default.py @@ -13,7 +13,7 @@ """Module houses default applied-on-str functions builder class.""" -from modin.core.dataframe.algebra.d2p.series_default import SeriesDefault +from .series_default import SeriesDefault class StrDefault(SeriesDefault): diff --git a/modin/core/dataframe/algebra/operators/__init__.py b/modin/core/dataframe/algebra/operators/__init__.py index ef4182b49ef..1526d63f594 100644 --- a/modin/core/dataframe/algebra/operators/__init__.py +++ b/modin/core/dataframe/algebra/operators/__init__.py @@ -13,13 +13,13 @@ """Operators module provides template for a query compiler methods for a set of common operations.""" -from modin.core.dataframe.algebra.operators.function import Function -from modin.core.dataframe.algebra.operators.mapfunction import MapFunction -from modin.core.dataframe.algebra.operators.mapreducefunction import MapReduceFunction -from modin.core.dataframe.algebra.operators.reductionfunction import ReductionFunction -from modin.core.dataframe.algebra.operators.foldfunction import FoldFunction -from modin.core.dataframe.algebra.operators.binary_function import BinaryFunction -from modin.core.dataframe.algebra.operators.groupby_function import ( +from .function import Function +from .mapfunction import MapFunction +from .mapreducefunction import MapReduceFunction +from .reductionfunction import ReductionFunction +from .foldfunction import FoldFunction +from .binary_function import BinaryFunction +from .groupby_function import ( GroupbyReduceFunction, groupby_reduce_functions, ) diff --git a/modin/core/dataframe/algebra/operators/binary_function.py b/modin/core/dataframe/algebra/operators/binary_function.py index cc6bbd6d94b..975f714b33d 100644 --- a/modin/core/dataframe/algebra/operators/binary_function.py +++ b/modin/core/dataframe/algebra/operators/binary_function.py @@ -16,7 +16,7 @@ import numpy as np import pandas -from modin.core.dataframe.algebra.operators.function import Function +from .function import Function class BinaryFunction(Function): diff --git a/modin/core/dataframe/algebra/operators/foldfunction.py b/modin/core/dataframe/algebra/operators/foldfunction.py index 58f4f422214..077a7a6b46b 100644 --- a/modin/core/dataframe/algebra/operators/foldfunction.py +++ b/modin/core/dataframe/algebra/operators/foldfunction.py @@ -13,7 +13,7 @@ """Module houses Fold functions builder class.""" -from modin.core.dataframe.algebra.operators.function import Function +from .function import Function class FoldFunction(Function): diff --git a/modin/core/dataframe/algebra/operators/groupby_function.py b/modin/core/dataframe/algebra/operators/groupby_function.py index 5794cfb5962..c4046cc0058 100644 --- a/modin/core/dataframe/algebra/operators/groupby_function.py +++ b/modin/core/dataframe/algebra/operators/groupby_function.py @@ -15,7 +15,7 @@ import pandas -from modin.core.dataframe.algebra.operators.mapreducefunction import MapReduceFunction +from .mapreducefunction import MapReduceFunction from modin.utils import try_cast_to_pandas, hashable diff --git a/modin/core/dataframe/algebra/operators/mapfunction.py b/modin/core/dataframe/algebra/operators/mapfunction.py index 576c0b1a9eb..134b05aea0a 100644 --- a/modin/core/dataframe/algebra/operators/mapfunction.py +++ b/modin/core/dataframe/algebra/operators/mapfunction.py @@ -13,7 +13,7 @@ """Module houses Map functions builder class.""" -from modin.core.dataframe.algebra.operators.function import Function +from .function import Function class MapFunction(Function): diff --git a/modin/core/dataframe/algebra/operators/mapreducefunction.py b/modin/core/dataframe/algebra/operators/mapreducefunction.py index 9d95bf7c5d1..16364d5fdb2 100644 --- a/modin/core/dataframe/algebra/operators/mapreducefunction.py +++ b/modin/core/dataframe/algebra/operators/mapreducefunction.py @@ -13,7 +13,7 @@ """Module houses MapReduce functions builder class.""" -from modin.core.dataframe.algebra.operators.function import Function +from .function import Function class MapReduceFunction(Function): diff --git a/modin/core/dataframe/algebra/operators/reductionfunction.py b/modin/core/dataframe/algebra/operators/reductionfunction.py index e30780f3865..3843caeb4db 100644 --- a/modin/core/dataframe/algebra/operators/reductionfunction.py +++ b/modin/core/dataframe/algebra/operators/reductionfunction.py @@ -13,7 +13,7 @@ """Module houses Reduction functions builder class.""" -from modin.core.dataframe.algebra.operators.function import Function +from .function import Function class ReductionFunction(Function): diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py index be4385331c0..368ac6e8427 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/dataframe/dataframe.py @@ -14,7 +14,7 @@ """Module houses class that implements ``PandasFrame``.""" from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame -from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition_manager import ( +from ..partitioning.partition_manager import ( PandasOnDaskFramePartitionManager, ) diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py index 7c77c3601af..06131c07268 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/axis_partition.py @@ -16,9 +16,7 @@ from modin.core.dataframe.pandas.partitioning.axis_partition import ( PandasFrameAxisPartition, ) -from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition import ( - PandasOnDaskFramePartition, -) +from .partition import PandasOnDaskFramePartition from distributed.client import default_client from distributed import Future diff --git a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py index 8031d30970c..bcc6e7e1182 100644 --- a/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py +++ b/modin/core/execution/dask/implementations/pandas_on_dask/partitioning/partition_manager.py @@ -18,13 +18,11 @@ from modin.core.dataframe.pandas.partitioning.partition_manager import ( PandasFramePartitionManager, ) -from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.axis_partition import ( +from .axis_partition import ( PandasOnDaskFrameColumnPartition, PandasOnDaskFrameRowPartition, ) -from modin.core.execution.dask.implementations.pandas_on_dask.partitioning.partition import ( - PandasOnDaskFramePartition, -) +from .partition import PandasOnDaskFramePartition from modin.error_message import ErrorMessage import pandas diff --git a/modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py b/modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py index 79e859c9c98..b727ee60ebe 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py +++ b/modin/core/execution/python/implementations/pandas_on_python/dataframe/dataframe.py @@ -18,7 +18,7 @@ """ from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame -from modin.core.execution.python.implementations.pandas_on_python.partitioning.partition_manager import ( +from ..partitioning.partition_manager import ( PandasOnPythonFramePartitionManager, ) diff --git a/modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py index 5d52b21cd26..07660dc5358 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/axis_partition.py @@ -18,9 +18,7 @@ from modin.core.dataframe.pandas.partitioning.axis_partition import ( PandasFrameAxisPartition, ) -from modin.core.execution.python.implementations.pandas_on_python.partitioning.partition import ( - PandasOnPythonFramePartition, -) +from .partition import PandasOnPythonFramePartition class PandasOnPythonFrameAxisPartition(PandasFrameAxisPartition): diff --git a/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py index 4ba7aaa2072..56c1d00c548 100644 --- a/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py +++ b/modin/core/execution/python/implementations/pandas_on_python/partitioning/partition_manager.py @@ -16,13 +16,11 @@ from modin.core.dataframe.pandas.partitioning.partition_manager import ( PandasFramePartitionManager, ) -from modin.core.execution.python.implementations.pandas_on_python.partitioning.axis_partition import ( +from .axis_partition import ( PandasOnPythonFrameColumnPartition, PandasOnPythonFrameRowPartition, ) -from modin.core.execution.python.implementations.pandas_on_python.partitioning.partition import ( - PandasOnPythonFramePartition, -) +from .partition import PandasOnPythonFramePartition class PandasOnPythonFramePartitionManager(PandasFramePartitionManager): diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py index 15f7bf8bbac..e73cab7f754 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/dataframe/dataframe.py @@ -16,10 +16,10 @@ import numpy as np import ray -from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( +from ..partitioning.partition import ( cuDFOnRayFramePartition, ) -from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition_manager import ( +from ..partitioning.partition_manager import ( cuDFOnRayFramePartitionManager, ) diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py index d1a831d6460..b359ee4508a 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/axis_partition.py @@ -16,9 +16,7 @@ import cudf import ray -from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( - cuDFOnRayFramePartition, -) +from .partition import cuDFOnRayFramePartition class cuDFOnRayFrameAxisPartition(object): diff --git a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py index 67ab790aa59..a44043c03d8 100644 --- a/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py +++ b/modin/core/execution/ray/implementations/cudf_on_ray/partitioning/partition_manager.py @@ -16,13 +16,11 @@ import numpy as np import ray -from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.axis_partition import ( +from .axis_partition import ( cuDFOnRayFrameColumnPartition, cuDFOnRayFrameRowPartition, ) -from modin.core.execution.ray.implementations.cudf_on_ray.partitioning.partition import ( - cuDFOnRayFramePartition, -) +from .partition import cuDFOnRayFramePartition from modin.core.storage_formats.pandas.utils import split_result_of_axis_func_pandas from modin.config import GpuCount diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py index 30c288e69e9..d9ce1a09e61 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/dataframe/dataframe.py @@ -15,7 +15,7 @@ import pandas -from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition_manager import ( +from ..partitioning.partition_manager import ( PandasOnRayFramePartitionManager, ) from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py index 2fdcbbe1c79..8ebbbf1386c 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/axis_partition.py @@ -18,9 +18,7 @@ from modin.core.dataframe.pandas.partitioning.axis_partition import ( PandasFrameAxisPartition, ) -from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( - PandasOnRayFramePartition, -) +from .partition import PandasOnRayFramePartition import ray from ray.services import get_node_ip_address diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py index 4c8d85493f1..12f4125c334 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py @@ -21,16 +21,14 @@ from modin.core.execution.ray.generic.partitioning.partition_manager import ( GenericRayFramePartitionManager, ) -from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.axis_partition import ( +from .axis_partition import ( PandasOnRayFrameColumnPartition, PandasOnRayFrameRowPartition, ) -from modin.core.execution.ray.implementations.pandas_on_ray.partitioning.partition import ( +from .partition import ( PandasOnRayFramePartition, ) -from modin.core.execution.ray.implementations.pandas_on_ray.modin_aqp import ( - call_progress_bar, -) +from ..modin_aqp import call_progress_bar from modin.error_message import ErrorMessage import pandas diff --git a/modin/core/io/text/excel_dispatcher.py b/modin/core/io/text/excel_dispatcher.py index 72a6e6caa92..5334e4c4b30 100644 --- a/modin/core/io/text/excel_dispatcher.py +++ b/modin/core/io/text/excel_dispatcher.py @@ -66,7 +66,7 @@ def _read(cls, io, **kwargs): from openpyxl.worksheet.worksheet import Worksheet from openpyxl.worksheet._reader import WorksheetReader from openpyxl.reader.excel import ExcelReader - from modin.backends.pandas.parsers import PandasExcelParser + from modin.core.storage_formats.pandas.parsers import PandasExcelParser sheet_name = kwargs.get("sheet_name", 0) if sheet_name is None or isinstance(sheet_name, list): diff --git a/modin/core/storage_formats/__init__.py b/modin/core/storage_formats/__init__.py index 13698a88065..0703304ba14 100644 --- a/modin/core/storage_formats/__init__.py +++ b/modin/core/storage_formats/__init__.py @@ -13,12 +13,12 @@ """Modin's functionality related to storage formats supported.""" -from modin.core.storage_formats.base import BaseQueryCompiler -from modin.core.storage_formats.pandas import PandasQueryCompiler +from .base import BaseQueryCompiler +from .pandas import PandasQueryCompiler __all__ = ["BaseQueryCompiler", "PandasQueryCompiler"] try: - from modin.core.storage_formats.pyarrow import PyarrowQueryCompiler # noqa: F401 + from .pyarrow import PyarrowQueryCompiler # noqa: F401 except ImportError: pass else: diff --git a/modin/core/storage_formats/base/__init__.py b/modin/core/storage_formats/base/__init__.py index 1a4c7a3cdbf..8d0a4f40fa3 100644 --- a/modin/core/storage_formats/base/__init__.py +++ b/modin/core/storage_formats/base/__init__.py @@ -13,6 +13,6 @@ """The module represents the base query compiler that defines the common query compiler API.""" -from modin.core.storage_formats.base.query_compiler import BaseQueryCompiler +from .query_compiler import BaseQueryCompiler __all__ = ["BaseQueryCompiler"] diff --git a/modin/core/storage_formats/cudf/__init__.py b/modin/core/storage_formats/cudf/__init__.py index 4175d9c9119..1f8ca89b70c 100644 --- a/modin/core/storage_formats/cudf/__init__.py +++ b/modin/core/storage_formats/cudf/__init__.py @@ -13,6 +13,6 @@ """The module represents the query compiler level for the cuDF backend.""" -from modin.core.storage_formats.cudf.query_compiler import cuDFQueryCompiler +from .query_compiler import cuDFQueryCompiler __all__ = ["cuDFQueryCompiler"] diff --git a/modin/core/storage_formats/pandas/__init__.py b/modin/core/storage_formats/pandas/__init__.py index 06c82683346..0a285af52dd 100644 --- a/modin/core/storage_formats/pandas/__init__.py +++ b/modin/core/storage_formats/pandas/__init__.py @@ -13,6 +13,6 @@ """The module represents the query compiler level for the pandas backend.""" -from modin.core.storage_formats.pandas.query_compiler import PandasQueryCompiler +from .query_compiler import PandasQueryCompiler __all__ = ["PandasQueryCompiler"] diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py index b2cee6e6c27..1676bae1d48 100644 --- a/modin/core/storage_formats/pandas/query_compiler.py +++ b/modin/core/storage_formats/pandas/query_compiler.py @@ -2549,8 +2549,8 @@ def _groupby_dict_reduce( 0 is for index, when 1 is for columns. agg_func : dict(label) -> str Dictionary that maps row/column labels to the function names. - **Note:** specified functions have to be supported by ``modin.data_management.functions.GroupbyReduceFunction``. - Supported functions are listed in the ``modin.data_management.functions.GroupbyReduceFunction.groupby_reduce_functions`` + **Note:** specified functions have to be supported by ``modin.core.dataframe.algebra.operators.GroupbyReduceFunction``. + Supported functions are listed in the ``modin.core.dataframe.algebra.operators.GroupbyReduceFunction.groupby_reduce_functions`` dictionary. agg_args : list Serves the compatibility purpose. Does not affect the result. @@ -2562,7 +2562,7 @@ def _groupby_dict_reduce( If `by` is a QueryCompiler indicates whether or not by-data came from the `self`. **kwargs : dict - Additional parameters to pass to the ``modin.data_management.functions.GroupbyReduceFunction.register``. + Additional parameters to pass to the ``modin.core.dataframe.algebra.operators.GroupbyReduceFunction.register``. Returns ------- diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py index 62f9e446a98..bf4570307db 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_algebra.py @@ -19,9 +19,7 @@ """ import abc -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( - BaseExpr, -) +from .expr import BaseExpr class CalciteInputRefExpr(BaseExpr): diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py index 9fc834f068f..43cf83f6619 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_builder.py @@ -13,14 +13,14 @@ """Module provides ``CalciteBuilder`` class.""" -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( +from .expr import ( InputRefExpr, LiteralExpr, AggregateExpr, build_if_then_else, build_row_idx_filter_expr, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_algebra import ( +from .calcite_algebra import ( CalciteBaseNode, CalciteInputRefExpr, CalciteInputIdxExpr, @@ -33,7 +33,7 @@ CalciteJoinNode, CalciteUnionNode, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.df_algebra import ( +from .df_algebra import ( FrameNode, MaskNode, GroupbyAggNode, diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py index 745d4270a86..b73e67932d6 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/calcite_serializer.py @@ -13,13 +13,13 @@ """Module provides ``CalciteSerializer`` class.""" -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( +from .expr import ( BaseExpr, LiteralExpr, OpExpr, AggregateExpr, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_algebra import ( +from .calcite_algebra import ( CalciteBaseNode, CalciteInputRefExpr, CalciteInputIdxExpr, diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py index 014524ec8c4..1621eefae0b 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/dataframe.py @@ -17,7 +17,7 @@ from modin.experimental.core.storage_formats.omnisci.query_compiler import ( DFAlgQueryCompiler, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition_manager import ( +from ..partitioning.partition_manager import ( OmnisciOnRayFramePartitionManager, ) diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py index f5b4c2e4683..63440b65655 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/dataframe/df_algebra.py @@ -14,9 +14,7 @@ """Module provides classes for lazy DataFrame algebra operations.""" import abc -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.expr import ( - InputRefExpr, -) +from .expr import InputRefExpr from collections import OrderedDict diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py index e61ecc88eaf..7b4247ba68f 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/axis_partition.py @@ -20,9 +20,7 @@ from modin.core.dataframe.pandas.partitioning.axis_partition import ( PandasFrameAxisPartition, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition import ( - OmnisciOnRayFramePartition, -) +from .partition import OmnisciOnRayFramePartition import ray diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py index 2394638a3be..54b102aae25 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/partitioning/partition_manager.py @@ -23,16 +23,16 @@ OmnisciOnRayFrameColumnPartition, OmnisciOnRayFrameRowPartition, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.partitioning.partition import ( +from ..partitioning.partition import ( OmnisciOnRayFramePartition, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.omnisci_worker import ( +from ..dataframe.omnisci_worker import ( OmnisciServer, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_builder import ( +from ..dataframe.calcite_builder import ( CalciteBuilder, ) -from modin.experimental.core.execution.ray.implementations.omnisci_on_ray.dataframe.calcite_serializer import ( +from ..dataframe.calcite_serializer import ( CalciteSerializer, ) from modin.config import DoUseCalcite diff --git a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py index d352010f581..462253b848f 100644 --- a/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py +++ b/modin/experimental/core/execution/ray/implementations/omnisci_on_ray/test/test_dataframe.py @@ -125,7 +125,7 @@ def run_modin( @pytest.mark.usefixtures("TestReadCSVFixture") class TestCSV: - root = os.path.abspath(__file__ + "/.." * 6) # root of modin repo + root = os.path.abspath(__file__ + "/.." * 9) # root of modin repo boston_housing_names = [ "index", diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py index aa726f6d29e..4b576bd6563 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/dataframe.py @@ -20,7 +20,7 @@ import pandas from pandas.core.dtypes.cast import find_common_type -from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition_manager import ( +from ..partitioning.partition_manager import ( PyarrowOnRayFramePartitionManager, ) from modin.core.dataframe.pandas.dataframe.dataframe import PandasFrame diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py index 89c4a231ee6..82cc65cb67e 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/axis_partition.py @@ -16,9 +16,7 @@ from modin.core.dataframe.pandas.partitioning.axis_partition import ( BaseFrameAxisPartition, ) -from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition import ( - PyarrowOnRayFramePartition, -) +from .partition import PyarrowOnRayFramePartition import ray import pyarrow diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py index 8153b9874f4..241cc4309da 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/partition_manager.py @@ -16,11 +16,11 @@ from modin.core.execution.ray.generic.partitioning.partition_manager import ( GenericRayFramePartitionManager, ) -from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.axis_partition import ( +from .axis_partition import ( PyarrowOnRayFrameColumnPartition, PyarrowOnRayFrameRowPartition, ) -from modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.partitioning.partition import ( +from .partition import ( PyarrowOnRayFramePartition, ) diff --git a/modin/experimental/core/storage_formats/omnisci/__init__.py b/modin/experimental/core/storage_formats/omnisci/__init__.py index b768baebee1..cf76d4923f2 100644 --- a/modin/experimental/core/storage_formats/omnisci/__init__.py +++ b/modin/experimental/core/storage_formats/omnisci/__init__.py @@ -12,3 +12,7 @@ # governing permissions and limitations under the License. """The module represents the query compiler level for the OmniSci backend.""" + +from .query_compiler import DFAlgQueryCompiler + +__all__ = ["DFAlgQueryCompiler"] diff --git a/modin/pandas/utils.py b/modin/pandas/utils.py index 7557271f8a4..381ff5e94d2 100644 --- a/modin/pandas/utils.py +++ b/modin/pandas/utils.py @@ -150,7 +150,7 @@ def is_label(obj, label, axis=0): Parameters ---------- - obj : modin.pandas.DataFrame, modin.pandas.Series or modin.backends.base.BaseQueryCompiler + obj : modin.pandas.DataFrame, modin.pandas.Series or modin.core.storage_formats.base.BaseQueryCompiler Object to check. label : object Label name to check. From 6a9d6d09813c8d2dcb722eb5d892cdc03e4e4dae Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Wed, 8 Sep 2021 18:19:35 +0300 Subject: [PATCH 11/18] REFACTOR-#2957: Fix spreadsheet tests by adding exp pandas on dask factory Signed-off-by: Igoshev, Yaroslav --- modin/core/execution/dispatching/factories/factories.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modin/core/execution/dispatching/factories/factories.py b/modin/core/execution/dispatching/factories/factories.py index ef087248b36..dc5cdad916a 100644 --- a/modin/core/execution/dispatching/factories/factories.py +++ b/modin/core/execution/dispatching/factories/factories.py @@ -521,6 +521,11 @@ def _to_pickle_distributed(cls, *args, **kwargs): return cls.io_cls.to_pickle_distributed(*args, **kwargs) +@doc(_doc_factory_class, backend_name="experimental PandasOnDask") +class ExperimentalPandasOnDaskFactory(ExperimentalBaseFactory, PandasOnDaskFactory): + pass + + @doc(_doc_factory_class, backend_name="experimental PandasOnPython") class ExperimentalPandasOnPythonFactory(ExperimentalBaseFactory, PandasOnPythonFactory): pass From 193e872defa8c69bebdb9cb0779691d7fd9386eb Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Thu, 9 Sep 2021 21:42:10 +0300 Subject: [PATCH 12/18] REFACTOR-#2957: d2p -> default2pandas Signed-off-by: Igoshev, Yaroslav --- .../core/dataframe/algebra/{d2p => default2pandas}/__init__.py | 2 +- .../dataframe/algebra/{d2p => default2pandas}/any_default.py | 0 .../dataframe/algebra/{d2p => default2pandas}/binary_default.py | 0 .../dataframe/algebra/{d2p => default2pandas}/cat_default.py | 0 .../algebra/{d2p => default2pandas}/dataframe_default.py | 0 .../algebra/{d2p => default2pandas}/datetime_default.py | 0 modin/core/dataframe/algebra/{d2p => default2pandas}/default.py | 0 .../algebra/{d2p => default2pandas}/groupby_default.py | 0 .../algebra/{d2p => default2pandas}/resample_default.py | 0 .../algebra/{d2p => default2pandas}/rolling_default.py | 0 .../dataframe/algebra/{d2p => default2pandas}/series_default.py | 0 .../dataframe/algebra/{d2p => default2pandas}/str_default.py | 0 modin/core/storage_formats/base/query_compiler.py | 2 +- 13 files changed, 2 insertions(+), 2 deletions(-) rename modin/core/dataframe/algebra/{d2p => default2pandas}/__init__.py (93%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/any_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/binary_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/cat_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/dataframe_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/datetime_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/groupby_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/resample_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/rolling_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/series_default.py (100%) rename modin/core/dataframe/algebra/{d2p => default2pandas}/str_default.py (100%) diff --git a/modin/core/dataframe/algebra/d2p/__init__.py b/modin/core/dataframe/algebra/default2pandas/__init__.py similarity index 93% rename from modin/core/dataframe/algebra/d2p/__init__.py rename to modin/core/dataframe/algebra/default2pandas/__init__.py index 9d7b977bac4..babece603e9 100644 --- a/modin/core/dataframe/algebra/d2p/__init__.py +++ b/modin/core/dataframe/algebra/default2pandas/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Module d2p provides templates for a query compiler default-to-pandas methods.""" +"""Module default2pandas provides templates for a query compiler default-to-pandas methods.""" from .dataframe_default import DataFrameDefault from .datetime_default import DateTimeDefault diff --git a/modin/core/dataframe/algebra/d2p/any_default.py b/modin/core/dataframe/algebra/default2pandas/any_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/any_default.py rename to modin/core/dataframe/algebra/default2pandas/any_default.py diff --git a/modin/core/dataframe/algebra/d2p/binary_default.py b/modin/core/dataframe/algebra/default2pandas/binary_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/binary_default.py rename to modin/core/dataframe/algebra/default2pandas/binary_default.py diff --git a/modin/core/dataframe/algebra/d2p/cat_default.py b/modin/core/dataframe/algebra/default2pandas/cat_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/cat_default.py rename to modin/core/dataframe/algebra/default2pandas/cat_default.py diff --git a/modin/core/dataframe/algebra/d2p/dataframe_default.py b/modin/core/dataframe/algebra/default2pandas/dataframe_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/dataframe_default.py rename to modin/core/dataframe/algebra/default2pandas/dataframe_default.py diff --git a/modin/core/dataframe/algebra/d2p/datetime_default.py b/modin/core/dataframe/algebra/default2pandas/datetime_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/datetime_default.py rename to modin/core/dataframe/algebra/default2pandas/datetime_default.py diff --git a/modin/core/dataframe/algebra/d2p/default.py b/modin/core/dataframe/algebra/default2pandas/default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/default.py rename to modin/core/dataframe/algebra/default2pandas/default.py diff --git a/modin/core/dataframe/algebra/d2p/groupby_default.py b/modin/core/dataframe/algebra/default2pandas/groupby_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/groupby_default.py rename to modin/core/dataframe/algebra/default2pandas/groupby_default.py diff --git a/modin/core/dataframe/algebra/d2p/resample_default.py b/modin/core/dataframe/algebra/default2pandas/resample_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/resample_default.py rename to modin/core/dataframe/algebra/default2pandas/resample_default.py diff --git a/modin/core/dataframe/algebra/d2p/rolling_default.py b/modin/core/dataframe/algebra/default2pandas/rolling_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/rolling_default.py rename to modin/core/dataframe/algebra/default2pandas/rolling_default.py diff --git a/modin/core/dataframe/algebra/d2p/series_default.py b/modin/core/dataframe/algebra/default2pandas/series_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/series_default.py rename to modin/core/dataframe/algebra/default2pandas/series_default.py diff --git a/modin/core/dataframe/algebra/d2p/str_default.py b/modin/core/dataframe/algebra/default2pandas/str_default.py similarity index 100% rename from modin/core/dataframe/algebra/d2p/str_default.py rename to modin/core/dataframe/algebra/default2pandas/str_default.py diff --git a/modin/core/storage_formats/base/query_compiler.py b/modin/core/storage_formats/base/query_compiler.py index 44672547549..1700256efb0 100644 --- a/modin/core/storage_formats/base/query_compiler.py +++ b/modin/core/storage_formats/base/query_compiler.py @@ -19,7 +19,7 @@ import abc -from modin.core.dataframe.algebra.d2p import ( +from modin.core.dataframe.algebra.default2pandas import ( DataFrameDefault, SeriesDefault, DateTimeDefault, From d235de57fcc3ccfb26df68ecb9f3d701eabf3706 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Tue, 28 Sep 2021 09:22:29 -0500 Subject: [PATCH 13/18] REFACTOR-#2957: Address comments Signed-off-by: Devin Petersohn --- .../algebra/default2pandas/__init__.py | 20 +++++++++---------- .../default2pandas/{any_default.py => any.py} | 0 .../{binary_default.py => binary.py} | 2 +- .../default2pandas/{cat_default.py => cat.py} | 2 +- .../{dataframe_default.py => dataframe.py} | 0 .../{datetime_default.py => datetime.py} | 2 +- .../{groupby_default.py => groupby.py} | 0 .../{resample_default.py => resample.py} | 0 .../{rolling_default.py => rolling.py} | 0 .../{series_default.py => series.py} | 2 +- .../default2pandas/{str_default.py => str.py} | 2 +- 11 files changed, 15 insertions(+), 15 deletions(-) rename modin/core/dataframe/algebra/default2pandas/{any_default.py => any.py} (100%) rename modin/core/dataframe/algebra/default2pandas/{binary_default.py => binary.py} (98%) rename modin/core/dataframe/algebra/default2pandas/{cat_default.py => cat.py} (96%) rename modin/core/dataframe/algebra/default2pandas/{dataframe_default.py => dataframe.py} (100%) rename modin/core/dataframe/algebra/default2pandas/{datetime_default.py => datetime.py} (96%) rename modin/core/dataframe/algebra/default2pandas/{groupby_default.py => groupby.py} (100%) rename modin/core/dataframe/algebra/default2pandas/{resample_default.py => resample.py} (100%) rename modin/core/dataframe/algebra/default2pandas/{rolling_default.py => rolling.py} (100%) rename modin/core/dataframe/algebra/default2pandas/{series_default.py => series.py} (97%) rename modin/core/dataframe/algebra/default2pandas/{str_default.py => str.py} (96%) diff --git a/modin/core/dataframe/algebra/default2pandas/__init__.py b/modin/core/dataframe/algebra/default2pandas/__init__.py index babece603e9..4fe0bb90df1 100644 --- a/modin/core/dataframe/algebra/default2pandas/__init__.py +++ b/modin/core/dataframe/algebra/default2pandas/__init__.py @@ -13,17 +13,17 @@ """Module default2pandas provides templates for a query compiler default-to-pandas methods.""" -from .dataframe_default import DataFrameDefault -from .datetime_default import DateTimeDefault -from .series_default import SeriesDefault -from .str_default import StrDefault -from .binary_default import BinaryDefault -from .any_default import AnyDefault -from .resample_default import ResampleDefault -from .rolling_default import RollingDefault +from .dataframe import DataFrameDefault +from .datetime import DateTimeDefault +from .series import SeriesDefault +from .str import StrDefault +from .binary import BinaryDefault +from .any import AnyDefault +from .resample import ResampleDefault +from .rolling import RollingDefault from .default import DefaultMethod -from .cat_default import CatDefault -from .groupby_default import GroupByDefault +from .cat import CatDefault +from .groupby import GroupByDefault __all__ = [ "DataFrameDefault", diff --git a/modin/core/dataframe/algebra/default2pandas/any_default.py b/modin/core/dataframe/algebra/default2pandas/any.py similarity index 100% rename from modin/core/dataframe/algebra/default2pandas/any_default.py rename to modin/core/dataframe/algebra/default2pandas/any.py diff --git a/modin/core/dataframe/algebra/default2pandas/binary_default.py b/modin/core/dataframe/algebra/default2pandas/binary.py similarity index 98% rename from modin/core/dataframe/algebra/default2pandas/binary_default.py rename to modin/core/dataframe/algebra/default2pandas/binary.py index e337f7df558..fe2a91057fb 100644 --- a/modin/core/dataframe/algebra/default2pandas/binary_default.py +++ b/modin/core/dataframe/algebra/default2pandas/binary.py @@ -13,7 +13,7 @@ """Module houses default binary functions builder class.""" -from .any_default import AnyDefault +from .any import AnyDefault import pandas from pandas.core.dtypes.common import is_list_like diff --git a/modin/core/dataframe/algebra/default2pandas/cat_default.py b/modin/core/dataframe/algebra/default2pandas/cat.py similarity index 96% rename from modin/core/dataframe/algebra/default2pandas/cat_default.py rename to modin/core/dataframe/algebra/default2pandas/cat.py index 172fc22502b..146b48a5cb2 100644 --- a/modin/core/dataframe/algebra/default2pandas/cat_default.py +++ b/modin/core/dataframe/algebra/default2pandas/cat.py @@ -13,7 +13,7 @@ """Module houses default applied-on-category functions builder class.""" -from .series_default import SeriesDefault +from .series import SeriesDefault class CatDefault(SeriesDefault): diff --git a/modin/core/dataframe/algebra/default2pandas/dataframe_default.py b/modin/core/dataframe/algebra/default2pandas/dataframe.py similarity index 100% rename from modin/core/dataframe/algebra/default2pandas/dataframe_default.py rename to modin/core/dataframe/algebra/default2pandas/dataframe.py diff --git a/modin/core/dataframe/algebra/default2pandas/datetime_default.py b/modin/core/dataframe/algebra/default2pandas/datetime.py similarity index 96% rename from modin/core/dataframe/algebra/default2pandas/datetime_default.py rename to modin/core/dataframe/algebra/default2pandas/datetime.py index c81a655012e..f29750eba8c 100644 --- a/modin/core/dataframe/algebra/default2pandas/datetime_default.py +++ b/modin/core/dataframe/algebra/default2pandas/datetime.py @@ -13,7 +13,7 @@ """Module houses default applied-on-datetime functions builder class.""" -from .series_default import SeriesDefault +from .series import SeriesDefault class DateTimeDefault(SeriesDefault): diff --git a/modin/core/dataframe/algebra/default2pandas/groupby_default.py b/modin/core/dataframe/algebra/default2pandas/groupby.py similarity index 100% rename from modin/core/dataframe/algebra/default2pandas/groupby_default.py rename to modin/core/dataframe/algebra/default2pandas/groupby.py diff --git a/modin/core/dataframe/algebra/default2pandas/resample_default.py b/modin/core/dataframe/algebra/default2pandas/resample.py similarity index 100% rename from modin/core/dataframe/algebra/default2pandas/resample_default.py rename to modin/core/dataframe/algebra/default2pandas/resample.py diff --git a/modin/core/dataframe/algebra/default2pandas/rolling_default.py b/modin/core/dataframe/algebra/default2pandas/rolling.py similarity index 100% rename from modin/core/dataframe/algebra/default2pandas/rolling_default.py rename to modin/core/dataframe/algebra/default2pandas/rolling.py diff --git a/modin/core/dataframe/algebra/default2pandas/series_default.py b/modin/core/dataframe/algebra/default2pandas/series.py similarity index 97% rename from modin/core/dataframe/algebra/default2pandas/series_default.py rename to modin/core/dataframe/algebra/default2pandas/series.py index 16514967951..615398a29c0 100644 --- a/modin/core/dataframe/algebra/default2pandas/series_default.py +++ b/modin/core/dataframe/algebra/default2pandas/series.py @@ -13,7 +13,7 @@ """Module houses default Series functions builder class.""" -from .any_default import AnyDefault +from .any import AnyDefault class SeriesDefault(AnyDefault): diff --git a/modin/core/dataframe/algebra/default2pandas/str_default.py b/modin/core/dataframe/algebra/default2pandas/str.py similarity index 96% rename from modin/core/dataframe/algebra/default2pandas/str_default.py rename to modin/core/dataframe/algebra/default2pandas/str.py index 74e744f8d76..002b7744fdd 100644 --- a/modin/core/dataframe/algebra/default2pandas/str_default.py +++ b/modin/core/dataframe/algebra/default2pandas/str.py @@ -13,7 +13,7 @@ """Module houses default applied-on-str functions builder class.""" -from .series_default import SeriesDefault +from .series import SeriesDefault class StrDefault(SeriesDefault): From 50a2c4730da0fb0cb615018df9033485cf220303 Mon Sep 17 00:00:00 2001 From: Devin Petersohn Date: Thu, 30 Sep 2021 10:46:15 -0500 Subject: [PATCH 14/18] REFACTOR-#2957: Address comments Signed-off-by: Devin Petersohn --- .../experimental/core/execution/ray/implementations/__init__.py | 2 +- .../execution/ray/implementations/pandas_on_ray/__init__.py | 2 +- .../execution/ray/implementations/pandas_on_ray/io/__init__.py | 2 +- .../execution/ray/implementations/pyarrow_on_ray/__init__.py | 2 +- .../ray/implementations/pyarrow_on_ray/dataframe/__init__.py | 2 +- .../execution/ray/implementations/pyarrow_on_ray/io/__init__.py | 2 +- .../ray/implementations/pyarrow_on_ray/partitioning/__init__.py | 2 +- modin/experimental/core/storage_formats/__init__.py | 2 +- modin/experimental/core/storage_formats/omnisci/__init__.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modin/experimental/core/execution/ray/implementations/__init__.py b/modin/experimental/core/execution/ray/implementations/__init__.py index 77b5a7f5ac3..b19ba5f315f 100644 --- a/modin/experimental/core/execution/ray/implementations/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to Ray execution engine and optimized for specific storage formats.""" +"""Experimental functionality related to Ray execution engine and optimized for specific storage formats.""" diff --git a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py index 1ea107042ff..bccf5e24948 100644 --- a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to Ray execution engine and optimized for pandas storage format.""" +"""Experimental functionality related to Ray execution engine and optimized for pandas storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py index bf09f9b5d7c..b16964ad09e 100644 --- a/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pandas_on_ray/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base IO classes optimized for pandas on Ray execution.""" +"""Experimental Base IO classes optimized for pandas on Ray execution.""" from .io import ( ExperimentalPandasOnRayIO, diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py index ba844fb5909..16805cc615c 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to Ray execution engine and optimized for PyArrow storage format.""" +"""Experimental functionality related to Ray execution engine and optimized for PyArrow storage format.""" diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py index e1308335d30..ec39f99a387 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/dataframe/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe class optimized for PyArrow on Ray execution.""" +"""Experimental Base Modin Dataframe class optimized for PyArrow on Ray execution.""" diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py index ca2ceae597a..da3f745ec7a 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/io/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base IO classes optimized for PyArrow on Ray execution.""" +"""Experimental Base IO classes optimized for PyArrow on Ray execution.""" from .io import PyarrowOnRayIO diff --git a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py index 038365a651a..16805cc615c 100644 --- a/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py +++ b/modin/experimental/core/execution/ray/implementations/pyarrow_on_ray/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Modin's functionality related to Ray execution engine and optimized for PyArrow storage format.""" +"""Experimental functionality related to Ray execution engine and optimized for PyArrow storage format.""" diff --git a/modin/experimental/core/storage_formats/__init__.py b/modin/experimental/core/storage_formats/__init__.py index d7956bac537..fc05b797ac7 100644 --- a/modin/experimental/core/storage_formats/__init__.py +++ b/modin/experimental/core/storage_formats/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Experimental Modin's functionality related to storage formats supported.""" +"""Experimental functionality related to storage formats supported.""" diff --git a/modin/experimental/core/storage_formats/omnisci/__init__.py b/modin/experimental/core/storage_formats/omnisci/__init__.py index cf76d4923f2..ab9c1275ac2 100644 --- a/modin/experimental/core/storage_formats/omnisci/__init__.py +++ b/modin/experimental/core/storage_formats/omnisci/__init__.py @@ -11,7 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""The module represents the query compiler level for the OmniSci backend.""" +"""Experimental query compiler for the OmniSci backend.""" from .query_compiler import DFAlgQueryCompiler From 7a3969153a2f77ae4c5c0876d40c18cd7fae062c Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Mon, 11 Oct 2021 12:23:14 +0300 Subject: [PATCH 15/18] Apply comments --- .../pandas_on_ray => generic}/modin_aqp.py | 0 .../partitioning/partition_manager.py | 2 +- .../omnisci_on_native/__init__.py | 2 + .../{dataframe => }/calcite_algebra.py | 0 .../{dataframe => }/calcite_builder.py | 0 .../{dataframe => }/calcite_serializer.py | 0 .../omnisci_on_native/dataframe/dataframe.py | 4 +- .../omnisci_on_native/dataframe/partition.py | 143 ------------------ .../{dataframe => }/df_algebra.py | 0 .../omnisci_on_native/{dataframe => }/expr.py | 0 .../{dataframe => }/omnisci_worker.py | 0 .../partitioning/__init__.py | 2 +- .../partitioning/partition_manager.py | 6 +- .../omnisci_on_native/test/test_dataframe.py | 3 +- project_structure.txt | 59 -------- 15 files changed, 11 insertions(+), 210 deletions(-) rename modin/core/execution/ray/{implementations/pandas_on_ray => generic}/modin_aqp.py (100%) rename modin/experimental/core/execution/native/implementations/omnisci_on_native/{dataframe => }/calcite_algebra.py (100%) rename modin/experimental/core/execution/native/implementations/omnisci_on_native/{dataframe => }/calcite_builder.py (100%) rename modin/experimental/core/execution/native/implementations/omnisci_on_native/{dataframe => }/calcite_serializer.py (100%) delete mode 100644 modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/partition.py rename modin/experimental/core/execution/native/implementations/omnisci_on_native/{dataframe => }/df_algebra.py (100%) rename modin/experimental/core/execution/native/implementations/omnisci_on_native/{dataframe => }/expr.py (100%) rename modin/experimental/core/execution/native/implementations/omnisci_on_native/{dataframe => }/omnisci_worker.py (100%) delete mode 100644 project_structure.txt diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/modin_aqp.py b/modin/core/execution/ray/generic/modin_aqp.py similarity index 100% rename from modin/core/execution/ray/implementations/pandas_on_ray/modin_aqp.py rename to modin/core/execution/ray/generic/modin_aqp.py diff --git a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py index 12f4125c334..bab0e5e9655 100644 --- a/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py +++ b/modin/core/execution/ray/implementations/pandas_on_ray/partitioning/partition_manager.py @@ -28,7 +28,7 @@ from .partition import ( PandasOnRayFramePartition, ) -from ..modin_aqp import call_progress_bar +from modin.core.execution.ray.generic.modin_aqp import call_progress_bar from modin.error_message import ErrorMessage import pandas diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/__init__.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/__init__.py index cae6413e559..bce2babe696 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/__init__.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/__init__.py @@ -10,3 +10,5 @@ # the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. + +"""Experimental Modin's functionality related to Native execution engine and optimized for OmniSci storage format.""" diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/calcite_algebra.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_algebra.py similarity index 100% rename from modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/calcite_algebra.py rename to modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_algebra.py diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/calcite_builder.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_builder.py similarity index 100% rename from modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/calcite_builder.py rename to modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_builder.py diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/calcite_serializer.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_serializer.py similarity index 100% rename from modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/calcite_serializer.py rename to modin/experimental/core/execution/native/implementations/omnisci_on_native/calcite_serializer.py diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py index 9a656e2f139..23d27f23bb8 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/dataframe.py @@ -27,7 +27,7 @@ from modin.pandas.indexing import is_range_like import pandas as pd -from .df_algebra import ( +from ..df_algebra import ( MaskNode, FrameNode, GroupbyAggNode, @@ -39,7 +39,7 @@ translate_exprs_to_base, replace_frame_in_exprs, ) -from .expr import ( +from ..expr import ( AggregateExpr, InputRefExpr, LiteralExpr, diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/partition.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/partition.py deleted file mode 100644 index 773c41cc5b3..00000000000 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/partition.py +++ /dev/null @@ -1,143 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. - -"""Module provides a partition class for ``OmnisciOnNativeFrame`` frame.""" - -import pandas - -from modin.core.dataframe.pandas.partitioning.partition import PandasFramePartition -import pyarrow - - -class OmnisciOnNativeFramePartition(PandasFramePartition): - """ - A partition of ``OmnisciOnNativeFrame`` frame. - - Class holds either a ``pandas.DataFrame`` or ``pyarrow.Table``. - - Parameters - ---------- - frame_id : str, optional - A corresponding OmniSci table name or None. - pandas_df : pandas.DataFrame, optional - Partition data in pandas format. - arrow_table : pyarrow.Table, optional - Partition data in Arrow format. - length : int, optional - Length of the partition. - width : int, optional - Width of the partition. - - Attributes - ---------- - frame_id : str - A corresponding OmniSci table name if partition was imported - into OmniSci. Otherwise None. - pandas_df : pandas.DataFrame, optional - Partition data in pandas format. - arrow_table : pyarrow.Table - Partition data in Arrow format. None for partitions holding - `pandas.DataFrame`. - _length_cache : int - Length of the partition. - _width_cache : int - Width of the partition. - """ - - def __init__( - self, frame_id=None, pandas_df=None, arrow_table=None, length=None, width=None - ): - self.pandas_df = pandas_df - self.frame_id = frame_id - self.arrow_table = arrow_table - self._length_cache = length - self._width_cache = width - - def to_pandas(self): - """ - Transform to pandas format. - - Returns - ------- - pandas.DataFrame - """ - obj = self.get() - if isinstance(obj, (pandas.DataFrame, pandas.Series)): - return obj - assert isinstance(obj, pyarrow.Table) - return obj.to_pandas() - - def get(self): - """ - Get partition data. - - Returns - ------- - pandas.DataFrame or pyarrow.Table - """ - if self.arrow_table is not None: - return self.arrow_table - return self.pandas_df - - @classmethod - def put(cls, obj): - """ - Create partition from ``pandas.DataFrame`` or ``pandas.Series``. - - Parameters - ---------- - obj : pandas.Series or pandas.DataFrame - Source frame. - - Returns - ------- - OmnisciOnNativeFramePartition - The new partition. - """ - return OmnisciOnNativeFramePartition( - pandas_df=obj, length=len(obj.index), width=len(obj.columns) - ) - - def wait(self): - """ - Wait until the partition data is ready for use. - - Returns - ------- - pandas.DataFrame - The partition that is ready to be used. - """ - if self.arrow_table is not None: - return self.arrow_table - return self.pandas_df - - @classmethod - def put_arrow(cls, obj): - """ - Create partition from ``pyarrow.Table``. - - Parameters - ---------- - obj : pyarrow.Table - Source table. - - Returns - ------- - OmnisciOnNativeFramePartition - The new partition. - """ - return OmnisciOnNativeFramePartition( - arrow_table=obj, - length=len(obj), - width=len(obj.columns), - ) diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/df_algebra.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/df_algebra.py similarity index 100% rename from modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/df_algebra.py rename to modin/experimental/core/execution/native/implementations/omnisci_on_native/df_algebra.py diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/expr.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py similarity index 100% rename from modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/expr.py rename to modin/experimental/core/execution/native/implementations/omnisci_on_native/expr.py diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/omnisci_worker.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/omnisci_worker.py similarity index 100% rename from modin/experimental/core/execution/native/implementations/omnisci_on_native/dataframe/omnisci_worker.py rename to modin/experimental/core/execution/native/implementations/omnisci_on_native/omnisci_worker.py diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/__init__.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/__init__.py index 2b98cd3e0e4..cad252770ab 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/__init__.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/__init__.py @@ -11,4 +11,4 @@ # ANY KIND, either express or implied. See the License for the specific language # governing permissions and limitations under the License. -"""Base Modin Dataframe classes related to its repartitioning and optimized for OmniSci on Native execution.""" +"""Base Modin Dataframe classes related to its partitioning and optimized for OmniSci on Native execution.""" diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition_manager.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition_manager.py index 99ba11a4e6a..8713e1a959d 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition_manager.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/partitioning/partition_manager.py @@ -22,13 +22,13 @@ from ..partitioning.partition import ( OmnisciOnNativeFramePartition, ) -from ..dataframe.omnisci_worker import ( +from ..omnisci_worker import ( OmnisciServer, ) -from ..dataframe.calcite_builder import ( +from ..calcite_builder import ( CalciteBuilder, ) -from ..dataframe.calcite_serializer import ( +from ..calcite_serializer import ( CalciteSerializer, ) from modin.config import DoUseCalcite diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py index ba4393ced98..deea9561b1c 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py @@ -27,6 +27,7 @@ Engine.put("native") Backend.put("omnisci") +import modin import modin.pandas as pd from modin.pandas.test.utils import ( df_equals, @@ -48,7 +49,7 @@ @pytest.mark.usefixtures("TestReadCSVFixture") class TestCSV: - root = os.path.abspath(__file__ + "/.." * 9) # root of modin repo + root = os.path.dirname(os.path.abspath(modin.__file__)) # root of modin repo boston_housing_names = [ "index", diff --git a/project_structure.txt b/project_structure.txt deleted file mode 100644 index 04f6df01678..00000000000 --- a/project_structure.txt +++ /dev/null @@ -1,59 +0,0 @@ -.modin/ - # left `core` folder only since it mostly has changes - .core/ - .dataframe/ - .algebra/ - ./d2p - . # the same as it is now in data_management/functions/default_methods - .operators/ - .map.py - .reduce.py - .tree_reduce.py - .explode.py - .groupby.py - .etc. - .base/ # will contain base (abstract) classes - .pandas/ # contains PandasDataframe classes - .io/ - .column_stores/ - . # the same as it is now - .sql/ - . # the same as it is now - .text/ - . # the same as it is now - .file_dispatcher.py - .io.py - .storage_formats/ - .base/ - .query_compiler.py # BaseQueryCompiler - .pandas/ - .base/ # or default (DefaultPandasQueryCompiler) - .query_compiler.py # DefaultPandasQueryCompiler - .kernels.py - .query_compiler.py # PandasQueryCompiler - .parsers.py - .utils.py # data_management/utils moved here - .execution/ - .ray/ - .common/ - .task_wrapper.py # RayTask - .utils.py # init_ray - .generic/ - .dataframe/ - .dataframe.py # GenericRayDataframe - .partition_management/ - .axis_partition.py # GenericRayDataframeAxisPartition - .partition_manager.py # GenericRayDataframeManager - .partition.py # GenericRayDataframePartition - .io/ - .io.py # GenericRayIO - .implementations/ - .pandas_on_ray/ - . # the same as it is now - .dask/ - . # the same as it is for ray - .python/ - . # the same as it is for ray - .dispatching/ - .factories/ - . # the same as it is now \ No newline at end of file From fd0a6e1ef643ecd728305f1d665cda52bb5d6502 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Mon, 11 Oct 2021 16:32:31 +0300 Subject: [PATCH 16/18] Fix circular import --- modin/core/dataframe/algebra/__init__.py | 22 ++++++++++++ .../{operators => }/binary_function.py | 0 .../algebra/default2pandas/default.py | 2 +- .../algebra/{operators => }/foldfunction.py | 0 .../algebra/{operators => }/function.py | 0 .../{operators => }/groupby_function.py | 2 +- .../algebra/{operators => }/mapfunction.py | 0 .../{operators => }/mapreducefunction.py | 0 .../dataframe/algebra/operators/__init__.py | 36 ------------------- .../{operators => }/reductionfunction.py | 0 .../storage_formats/pandas/query_compiler.py | 8 ++--- 11 files changed, 28 insertions(+), 42 deletions(-) rename modin/core/dataframe/algebra/{operators => }/binary_function.py (100%) rename modin/core/dataframe/algebra/{operators => }/foldfunction.py (100%) rename modin/core/dataframe/algebra/{operators => }/function.py (100%) rename modin/core/dataframe/algebra/{operators => }/groupby_function.py (99%) rename modin/core/dataframe/algebra/{operators => }/mapfunction.py (100%) rename modin/core/dataframe/algebra/{operators => }/mapreducefunction.py (100%) delete mode 100644 modin/core/dataframe/algebra/operators/__init__.py rename modin/core/dataframe/algebra/{operators => }/reductionfunction.py (100%) diff --git a/modin/core/dataframe/algebra/__init__.py b/modin/core/dataframe/algebra/__init__.py index d315ac1b758..637ed31650e 100644 --- a/modin/core/dataframe/algebra/__init__.py +++ b/modin/core/dataframe/algebra/__init__.py @@ -12,3 +12,25 @@ # governing permissions and limitations under the License. """Modin Dataframe algebra (core operators).""" + +from .function import Function +from .mapfunction import MapFunction +from .mapreducefunction import MapReduceFunction +from .reductionfunction import ReductionFunction +from .foldfunction import FoldFunction +from .binary_function import BinaryFunction +from .groupby_function import ( + GroupbyReduceFunction, + groupby_reduce_functions, +) + +__all__ = [ + "Function", + "MapFunction", + "MapReduceFunction", + "ReductionFunction", + "FoldFunction", + "BinaryFunction", + "GroupbyReduceFunction", + "groupby_reduce_functions", +] diff --git a/modin/core/dataframe/algebra/operators/binary_function.py b/modin/core/dataframe/algebra/binary_function.py similarity index 100% rename from modin/core/dataframe/algebra/operators/binary_function.py rename to modin/core/dataframe/algebra/binary_function.py diff --git a/modin/core/dataframe/algebra/default2pandas/default.py b/modin/core/dataframe/algebra/default2pandas/default.py index 88b60153f59..c6629455b03 100644 --- a/modin/core/dataframe/algebra/default2pandas/default.py +++ b/modin/core/dataframe/algebra/default2pandas/default.py @@ -13,7 +13,7 @@ """Module houses default functions builder class.""" -from modin.core.dataframe.algebra.operators.function import Function +from modin.core.dataframe.algebra.function import Function from modin.utils import try_cast_to_pandas from pandas.core.dtypes.common import is_list_like diff --git a/modin/core/dataframe/algebra/operators/foldfunction.py b/modin/core/dataframe/algebra/foldfunction.py similarity index 100% rename from modin/core/dataframe/algebra/operators/foldfunction.py rename to modin/core/dataframe/algebra/foldfunction.py diff --git a/modin/core/dataframe/algebra/operators/function.py b/modin/core/dataframe/algebra/function.py similarity index 100% rename from modin/core/dataframe/algebra/operators/function.py rename to modin/core/dataframe/algebra/function.py diff --git a/modin/core/dataframe/algebra/operators/groupby_function.py b/modin/core/dataframe/algebra/groupby_function.py similarity index 99% rename from modin/core/dataframe/algebra/operators/groupby_function.py rename to modin/core/dataframe/algebra/groupby_function.py index 3294c17d93e..c5d2f4955fc 100644 --- a/modin/core/dataframe/algebra/operators/groupby_function.py +++ b/modin/core/dataframe/algebra/groupby_function.py @@ -16,7 +16,7 @@ import pandas from .mapreducefunction import MapReduceFunction -from .default_methods.groupby_default import GroupBy +from .default2pandas.groupby import GroupBy from modin.utils import try_cast_to_pandas, hashable diff --git a/modin/core/dataframe/algebra/operators/mapfunction.py b/modin/core/dataframe/algebra/mapfunction.py similarity index 100% rename from modin/core/dataframe/algebra/operators/mapfunction.py rename to modin/core/dataframe/algebra/mapfunction.py diff --git a/modin/core/dataframe/algebra/operators/mapreducefunction.py b/modin/core/dataframe/algebra/mapreducefunction.py similarity index 100% rename from modin/core/dataframe/algebra/operators/mapreducefunction.py rename to modin/core/dataframe/algebra/mapreducefunction.py diff --git a/modin/core/dataframe/algebra/operators/__init__.py b/modin/core/dataframe/algebra/operators/__init__.py deleted file mode 100644 index 1526d63f594..00000000000 --- a/modin/core/dataframe/algebra/operators/__init__.py +++ /dev/null @@ -1,36 +0,0 @@ -# Licensed to Modin Development Team under one or more contributor license agreements. -# See the NOTICE file distributed with this work for additional information regarding -# copyright ownership. The Modin Development Team licenses this file to you under the -# Apache License, Version 2.0 (the "License"); you may not use this file except in -# compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under -# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF -# ANY KIND, either express or implied. See the License for the specific language -# governing permissions and limitations under the License. - -"""Operators module provides template for a query compiler methods for a set of common operations.""" - -from .function import Function -from .mapfunction import MapFunction -from .mapreducefunction import MapReduceFunction -from .reductionfunction import ReductionFunction -from .foldfunction import FoldFunction -from .binary_function import BinaryFunction -from .groupby_function import ( - GroupbyReduceFunction, - groupby_reduce_functions, -) - -__all__ = [ - "Function", - "MapFunction", - "MapReduceFunction", - "ReductionFunction", - "FoldFunction", - "BinaryFunction", - "GroupbyReduceFunction", - "groupby_reduce_functions", -] diff --git a/modin/core/dataframe/algebra/operators/reductionfunction.py b/modin/core/dataframe/algebra/reductionfunction.py similarity index 100% rename from modin/core/dataframe/algebra/operators/reductionfunction.py rename to modin/core/dataframe/algebra/reductionfunction.py diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py index 09aabba3eb4..5a91206b906 100644 --- a/modin/core/storage_formats/pandas/query_compiler.py +++ b/modin/core/storage_formats/pandas/query_compiler.py @@ -42,7 +42,7 @@ hashable, _inherit_docstrings, ) -from modin.core.dataframe.algebra.operators import ( +from modin.core.dataframe.algebra import ( FoldFunction, MapFunction, MapReduceFunction, @@ -2543,8 +2543,8 @@ def _groupby_dict_reduce( 0 is for index, when 1 is for columns. agg_func : dict(label) -> str Dictionary that maps row/column labels to the function names. - **Note:** specified functions have to be supported by ``modin.core.dataframe.algebra.operators.GroupbyReduceFunction``. - Supported functions are listed in the ``modin.core.dataframe.algebra.operators.GroupbyReduceFunction.groupby_reduce_functions`` + **Note:** specified functions have to be supported by ``modin.core.dataframe.algebra.GroupbyReduceFunction``. + Supported functions are listed in the ``modin.core.dataframe.algebra.GroupbyReduceFunction.groupby_reduce_functions`` dictionary. agg_args : list Serves the compatibility purpose. Does not affect the result. @@ -2556,7 +2556,7 @@ def _groupby_dict_reduce( If `by` is a QueryCompiler indicates whether or not by-data came from the `self`. **kwargs : dict - Additional parameters to pass to the ``modin.core.dataframe.algebra.operators.GroupbyReduceFunction.register``. + Additional parameters to pass to the ``modin.core.dataframe.algebra.GroupbyReduceFunction.register``. Returns ------- From 7596c88c12486f8f6dd68510ae489ce0bb28566d Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Mon, 11 Oct 2021 16:52:45 +0300 Subject: [PATCH 17/18] Fix omnisci tests --- .../omnisci_on_native/test/test_dataframe.py | 11 ++++++++--- .../implementations/omnisci_on_native/test/utils.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py index deea9561b1c..fa6bc241435 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/test_dataframe.py @@ -27,7 +27,6 @@ Engine.put("native") Backend.put("omnisci") -import modin import modin.pandas as pd from modin.pandas.test.utils import ( df_equals, @@ -44,12 +43,18 @@ from modin.experimental.core.execution.native.implementations.omnisci_on_native.partitioning.partition_manager import ( OmnisciOnNativeFramePartitionManager, ) -from modin.experimental.engines.omnisci_on_native.frame.df_algebra import FrameNode +from modin.experimental.core.execution.native.implementations.omnisci_on_native.df_algebra import ( + FrameNode, +) @pytest.mark.usefixtures("TestReadCSVFixture") class TestCSV: - root = os.path.dirname(os.path.abspath(modin.__file__)) # root of modin repo + from modin import __file__ as modin_root + + root = os.path.dirname( + os.path.dirname(os.path.abspath(modin_root)) + ".." + ) # root of modin repo boston_housing_names = [ "index", diff --git a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/utils.py b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/utils.py index 99f00a2ede4..5075326dc5f 100644 --- a/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/utils.py +++ b/modin/experimental/core/execution/native/implementations/omnisci_on_native/test/utils.py @@ -24,7 +24,7 @@ eval_io as general_eval_io, ) -from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import ( +from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import ( OmnisciServer, ) From dfe2d8f42939e01e3651298fe51a95ed072d0534 Mon Sep 17 00:00:00 2001 From: "Igoshev, Yaroslav" Date: Mon, 11 Oct 2021 19:06:35 +0300 Subject: [PATCH 18/18] Fix paths --- asv_bench/benchmarks/utils/common.py | 2 +- examples/docker/modin-omnisci/census-omnisci.py | 2 +- examples/docker/modin-omnisci/nyc-taxi-omnisci.py | 2 +- examples/docker/modin-omnisci/plasticc-omnisci.py | 2 +- modin/core/storage_formats/pyarrow/query_compiler.py | 3 ++- .../core/storage_formats/omnisci/query_compiler.py | 2 +- modin/pandas/test/test_io.py | 2 +- 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/asv_bench/benchmarks/utils/common.py b/asv_bench/benchmarks/utils/common.py index f9be819b61d..bd643e58032 100644 --- a/asv_bench/benchmarks/utils/common.py +++ b/asv_bench/benchmarks/utils/common.py @@ -458,7 +458,7 @@ def trigger_import(*dfs): """ assert ASV_USE_BACKEND == "omnisci" - from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import ( + from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import ( OmnisciServer, ) diff --git a/examples/docker/modin-omnisci/census-omnisci.py b/examples/docker/modin-omnisci/census-omnisci.py index fcd3df1765f..49aa0b4aa03 100644 --- a/examples/docker/modin-omnisci/census-omnisci.py +++ b/examples/docker/modin-omnisci/census-omnisci.py @@ -14,7 +14,7 @@ import sys import time import modin.pandas as pd -from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer +from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import OmnisciServer from sklearn import config_context import sklearnex diff --git a/examples/docker/modin-omnisci/nyc-taxi-omnisci.py b/examples/docker/modin-omnisci/nyc-taxi-omnisci.py index decaa02fd5e..f566c1dbb8e 100644 --- a/examples/docker/modin-omnisci/nyc-taxi-omnisci.py +++ b/examples/docker/modin-omnisci/nyc-taxi-omnisci.py @@ -14,7 +14,7 @@ import sys import time import modin.pandas as pd -from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer +from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import OmnisciServer def read(filename): diff --git a/examples/docker/modin-omnisci/plasticc-omnisci.py b/examples/docker/modin-omnisci/plasticc-omnisci.py index 622c9c2bad2..bf418ca02cb 100644 --- a/examples/docker/modin-omnisci/plasticc-omnisci.py +++ b/examples/docker/modin-omnisci/plasticc-omnisci.py @@ -16,7 +16,7 @@ from collections import OrderedDict from functools import partial import modin.pandas as pd -from modin.experimental.engines.omnisci_on_native.frame.omnisci_worker import OmnisciServer +from modin.experimental.core.execution.native.implementations.omnisci_on_native.omnisci_worker import OmnisciServer import numpy as np import xgboost as xgb diff --git a/modin/core/storage_formats/pyarrow/query_compiler.py b/modin/core/storage_formats/pyarrow/query_compiler.py index b865d599be7..9b9fb2e6458 100644 --- a/modin/core/storage_formats/pyarrow/query_compiler.py +++ b/modin/core/storage_formats/pyarrow/query_compiler.py @@ -49,7 +49,8 @@ class PyarrowQueryCompiler(PandasQueryCompiler): Query compiler for the PyArrow backend. This class translates common query compiler API into the DataFrame Algebra - queries, that is supposed to be executed by :py:class:`~modin.experimental.engines.pyarrow_on_ray.frame.data.PyarrowOnRayFrame`. + queries, that is supposed to be executed by + :py:class:`~modin.experimental.core.execution.ray.implementations.pyarrow_on_ray.dataframe.dataframe.PyarrowOnRayFrame`. Parameters ---------- diff --git a/modin/experimental/core/storage_formats/omnisci/query_compiler.py b/modin/experimental/core/storage_formats/omnisci/query_compiler.py index f94831d9cd8..a3878ba8e79 100644 --- a/modin/experimental/core/storage_formats/omnisci/query_compiler.py +++ b/modin/experimental/core/storage_formats/omnisci/query_compiler.py @@ -149,7 +149,7 @@ class DFAlgQueryCompiler(BaseQueryCompiler): Query compiler for the OmniSci backend. This class doesn't perform much processing and mostly forwards calls to - :py:class:`~modin.experimental.engines.omnisci_on_native.frame.data.OmnisciOnNativeFrame` + :py:class:`~modin.experimental.core.execution.native.implementations.omnisci_on_native.dataframe.dataframe.OmnisciOnNativeFrame` for lazy execution trees build. Parameters diff --git a/modin/pandas/test/test_io.py b/modin/pandas/test/test_io.py index e0832b3256b..2740422f0f1 100644 --- a/modin/pandas/test/test_io.py +++ b/modin/pandas/test/test_io.py @@ -46,7 +46,7 @@ ) if Backend.get() == "Omnisci": - from modin.experimental.engines.omnisci_on_native.test.utils import ( + from modin.experimental.core.execution.native.implementations.omnisci_on_native.test.utils import ( eval_io, align_datetime_dtypes, )