Add github workflow to upload builds to S3

and some minor fix for integration tests, still WIP
facebookresearch · Apr 13, 2020 · ee7f500 · ee7f500
1 parent 9bc8ada
commit ee7f500
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 14 deletions.
diff --git a/.github/workflows/builduploads3.yml b/.github/workflows/builduploads3.yml
@@ -0,0 +1,36 @@
+# This workflow uploads the latest wheel of hydra-core & ray-launcher to S3 bucket.
+
+name: Upload lastest hydra core to S3
+
+on:
+   push:
+    branches:
+      - ray-launcher-v2
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v1
+      with:
+        python-version: '3.7'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --quiet --no-cache-dir awscli=='1.18.14'
+        python -m pip install wheel
+    - name: Build and publish
+      env:
+        AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        AWS_REGION: 'us-west-2'
+      run: |
+        # build upload latest hydra-core & ray-launcher
+        python setup.py sdist bdist_wheel
+        python plugins/hydra_ray_launcher/setup.py sdist bdist_wheel
+        aws s3 cp dist/ ${AWS_S3_BUCKET} --recursive --exclude "*" --include "*.whl"
diff --git a/hydra/core/hydra_config.py b/hydra/core/hydra_config.py
@@ -14,6 +14,7 @@ def __init__(self) -> None:
     def set_config(self, cfg: DictConfig) -> None:
         assert cfg is not None
         OmegaConf.set_readonly(cfg.hydra, True)
+        print(f"TYPE: {OmegaConf.get_type(cfg, 'hydra')} ")
         assert OmegaConf.get_type(cfg, "hydra") == HydraConf
         self.cfg = cfg  # type: ignore
 

diff --git a/plugins/hydra_ray_launcher/setup.py b/plugins/hydra_ray_launcher/setup.py
@@ -20,6 +20,6 @@
             "Operating System :: MacOS",
             "Operating System :: POSIX :: Linux",
         ],
-        install_requires=["boto3==1.4.8", "hydra-core==1.0.*", "ray"],
+        install_requires=["boto3>=1.12.34", "hydra-core==1.0.*", "ray"],
         include_package_data=True,
     )
diff --git a/plugins/hydra_ray_launcher/tests/conftest.py b/plugins/hydra_ray_launcher/tests/conftest.py
@@ -0,0 +1,3 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
+# Add Hydra standard test fixtures
+from hydra.test_utils._conftest import *  # noqa F401
diff --git a/plugins/hydra_ray_launcher/tests/test_ray_launcher.py b/plugins/hydra_ray_launcher/tests/test_ray_launcher.py
@@ -9,14 +9,7 @@
     IntegrationTestSuite,
     LauncherTestSuite,
 )
-
-# This has to be included here for the LauncherTestSuite to work.
-# noinspection PyUnresolvedReferences
-from hydra.test_utils.test_utils import (  # noqa: F401
-    TSweepRunner,
-    chdir_plugin_root,
-    sweep_runner,
-)
+from hydra.test_utils.test_utils import TSweepRunner, chdir_plugin_root
 from hydra_plugins.hydra_ray_launcher.ray_launcher import RayLauncher  # type: ignore
 
 chdir_plugin_root()
@@ -53,11 +46,6 @@ class TestRayLauncher(LauncherTestSuite):
                     {"hydra/hydra_logging": "hydra_debug"},
                     {"hydra/job_logging": "disabled"},
                 ],
-                "hydra": {
-                    "launcher": {
-                        "cls": "hydra_plugins.hydra_ray_launcher.ray_launcher.RayLauncher",
-                    }
-                },
             },
             ["-m"],
         )