FIX-#2052: fix spawning of remote cluster

ci/teamcity/Dockerfile.teamcity-ci

@@ -7,7 +7,7 @@ ADD git-rev /modin/git-rev
 
 WORKDIR /modin
 
-RUN conda env create -f environment.yml
+RUN conda env create -f ${ENVIRONMENT}
 
 # Make RUN commands use the new environment:
 SHELL ["conda", "run", "-n", "modin", "/bin/bash", "-c"]

environment.yml

@@ -12,7 +12,7 @@ dependencies:
   - pathlib
   - scipy
   - pip
-  - s3fs
+  - s3fs>=0.4.2
   - feather-format
   - lxml
   - openpyxl
@@ -30,4 +30,4 @@ dependencies:
   - pygithub==1.53
   - pip:
     - ray==0.8.7
-    - rpyc
+    - rpyc==4.1.5

modin/experimental/cloud/ray-autoscaler.yml

@@ -124,6 +124,9 @@ setup_commands:
         conda install python==3.7.6
 
         pip install modin "ray==0.8.7" cloudpickle
+        # ray now executes "ray stop" which expects "ray" to be in $PATH
+        # so place a symlink to current "ray" binary to /usr/local/bin
+        sudo ln -s `which ray` /usr/local/bin/ray
         echo 'export MODIN_RAY_CLUSTER=True' >> ~/.bashrc
 
     # Consider uncommenting these if you also want to run apt-get commands during setup

modin/experimental/cloud/rayscale.py

@@ -156,6 +156,10 @@ def __do_spawn(self):
                 restart_only=False,
                 yes=True,
                 override_cluster_name=None,
+                no_config_cache=False,
+                log_old_style=False,
+                log_color="auto",
+                verbose=1,
             )
             # need to re-load the config, as create_or_update_cluster() modifies it
             with open(self.config_file) as inp:
@@ -176,6 +180,9 @@ def __do_destroy(self):
                 workers_only=False,
                 override_cluster_name=None,
                 keep_min_workers=0,
+                log_old_style=False,
+                log_color="auto",
+                verbose=1,
             )
             self.ready = False
             self.config = None

requirements.txt

@@ -10,7 +10,7 @@ Jinja2
 pathlib
 tables
 scipy
-s3fs
+s3fs>=0.4.2
 pytest
 coverage<5.0
 pytest-cov
@@ -24,4 +24,4 @@ sqlalchemy
 msgpack
 pandas_gbq
 cloudpickle
-rpyc
+rpyc==4.1.5

requirements/env_windows.yml

@@ -23,7 +23,7 @@ dependencies:
     - pathlib
     - tables
     - scipy
-    - s3fs
+    - s3fs>=0.4.2
     - pytest
     - coverage<5.0
     - pytest-cov

requirements/windows_test_requires.txt

@@ -9,7 +9,7 @@ Jinja2
 pathlib
 tables
 scipy
-s3fs
+s3fs>=0.4.2
 pytest
 coverage<5.0
 pytest-cov

setup.py

@@ -39,7 +39,7 @@ def is_pure(self):
 
 dask_deps = ["dask>=2.12.0", "distributed>=2.12.0"]
 ray_deps = ["ray==0.8.7", "pyarrow<0.17"]
-remote_deps = ["rpyc", "cloudpickle", "boto3"]
+remote_deps = ["rpyc==4.1.5", "cloudpickle", "boto3==1.4.8"]
 
 all_deps = dask_deps + ray_deps + remote_deps