From c86bcadc99bef1627d901cf940363e0a5a32ae09 Mon Sep 17 00:00:00 2001 From: Elliot Tower Date: Wed, 4 Oct 2023 11:25:07 -0400 Subject: [PATCH 1/5] Re-add ray tutorial tests --- .github/workflows/linux-tutorials-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tutorials-test.yml b/.github/workflows/linux-tutorials-test.yml index 7732a1bdc..f5d671ac7 100644 --- a/.github/workflows/linux-tutorials-test.yml +++ b/.github/workflows/linux-tutorials-test.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11'] - tutorial: [Tianshou, CustomEnvironment, CleanRL, SB3/kaz, SB3/waterworld, SB3/connect_four, SB3/test, AgileRL] # TODO: add back Ray once next release after 2.6.2 + tutorial: [Tianshou, CustomEnvironment, CleanRL, Ray, SB3/kaz, SB3/waterworld, SB3/connect_four, SB3/test, AgileRL] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 01bafa2e77dfd2d7b33f73222b93c3e296e55c6a Mon Sep 17 00:00:00 2001 From: Elliot Tower Date: Wed, 4 Oct 2023 11:26:30 -0400 Subject: [PATCH 2/5] Update requirements.txt --- tutorials/Ray/requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tutorials/Ray/requirements.txt b/tutorials/Ray/requirements.txt index 3cd41dae8..df0623c4b 100644 --- a/tutorials/Ray/requirements.txt +++ b/tutorials/Ray/requirements.txt @@ -1,7 +1,6 @@ PettingZoo[classic,butterfly]>=1.24.0 Pillow>=9.4.0 -# note: currently requires nightly release, see https://docs.ray.io/en/latest/ray-overview/installation.html#daily-releases-nightlies -ray[rllib]>2.6.3 +ray[rllib]>=2.7.0 SuperSuit>=3.9.0 torch>=1.13.1 tensorflow-probability>=0.19.0 From 81ecd0aaf67984301de3d69b296577b5a185a3b0 Mon Sep 17 00:00:00 2001 From: Elliot Tower Date: Wed, 4 Oct 2023 12:57:27 -0400 Subject: [PATCH 3/5] Add checks for checkpoint path arg and exit(0) so the CI passes --- tutorials/Ray/render_rllib_leduc_holdem.py | 17 +++++++++++------ tutorials/Ray/render_rllib_pistonball.py | 4 ++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/tutorials/Ray/render_rllib_leduc_holdem.py b/tutorials/Ray/render_rllib_leduc_holdem.py index b514872ff..af458c785 100644 --- a/tutorials/Ray/render_rllib_leduc_holdem.py +++ b/tutorials/Ray/render_rllib_leduc_holdem.py @@ -7,12 +7,12 @@ import os import numpy as np -import ray -from ray.rllib.algorithms.algorithm import Algorithm -from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv -from ray.rllib.models import ModelCatalog -from ray.tune.registry import register_env -from rllib_leduc_holdem import TorchMaskedActions +# import ray +# from ray.rllib.algorithms.algorithm import Algorithm +# from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv +# from ray.rllib.models import ModelCatalog +# from ray.tune.registry import register_env +# from rllib_leduc_holdem import TorchMaskedActions from pettingzoo.classic import leduc_holdem_v4 @@ -28,6 +28,11 @@ args = parser.parse_args() + +if args.checkpoint_path is None: + print("The following arguments are required: --checkpoint-path") + exit(0) + checkpoint_path = os.path.expanduser(args.checkpoint_path) diff --git a/tutorials/Ray/render_rllib_pistonball.py b/tutorials/Ray/render_rllib_pistonball.py index a15edd3ea..c3db1bb11 100644 --- a/tutorials/Ray/render_rllib_pistonball.py +++ b/tutorials/Ray/render_rllib_pistonball.py @@ -29,6 +29,10 @@ args = parser.parse_args() +if args.checkpoint_path is None: + print("The following arguments are required: --checkpoint-path") + exit(0) + checkpoint_path = os.path.expanduser(args.checkpoint_path) ModelCatalog.register_custom_model("CNNModelV2", CNNModelV2) From 19740dad04e47155faedf97aeb670e3f6ef8d9b2 Mon Sep 17 00:00:00 2001 From: elliottower Date: Wed, 4 Oct 2023 14:13:59 -0400 Subject: [PATCH 4/5] Add CNNModelV2 to render file because imports do not work (not in module) --- tutorials/Ray/render_rllib_leduc_holdem.py | 12 ++++----- tutorials/Ray/render_rllib_pistonball.py | 31 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/tutorials/Ray/render_rllib_leduc_holdem.py b/tutorials/Ray/render_rllib_leduc_holdem.py index af458c785..ac1a7921c 100644 --- a/tutorials/Ray/render_rllib_leduc_holdem.py +++ b/tutorials/Ray/render_rllib_leduc_holdem.py @@ -7,12 +7,12 @@ import os import numpy as np -# import ray -# from ray.rllib.algorithms.algorithm import Algorithm -# from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv -# from ray.rllib.models import ModelCatalog -# from ray.tune.registry import register_env -# from rllib_leduc_holdem import TorchMaskedActions +import ray +from ray.rllib.algorithms.algorithm import Algorithm +from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv +from ray.rllib.models import ModelCatalog +from ray.tune.registry import register_env +from rllib_leduc_holdem import TorchMaskedActions from pettingzoo.classic import leduc_holdem_v4 diff --git a/tutorials/Ray/render_rllib_pistonball.py b/tutorials/Ray/render_rllib_pistonball.py index c3db1bb11..4e29ec3ba 100644 --- a/tutorials/Ray/render_rllib_pistonball.py +++ b/tutorials/Ray/render_rllib_pistonball.py @@ -12,11 +12,40 @@ from ray.rllib.algorithms.ppo import PPO from ray.rllib.env.wrappers.pettingzoo_env import PettingZooEnv from ray.rllib.models import ModelCatalog +from ray.rllib.models.torch.torch_modelv2 import TorchModelV2 from ray.tune.registry import register_env -from tutorials.Ray.rllib_pistonball import CNNModelV2 +from torch import nn from pettingzoo.butterfly import pistonball_v6 + +class CNNModelV2(TorchModelV2, nn.Module): + def __init__(self, obs_space, act_space, num_outputs, *args, **kwargs): + TorchModelV2.__init__(self, obs_space, act_space, num_outputs, *args, **kwargs) + nn.Module.__init__(self) + self.model = nn.Sequential( + nn.Conv2d(3, 32, [8, 8], stride=(4, 4)), + nn.ReLU(), + nn.Conv2d(32, 64, [4, 4], stride=(2, 2)), + nn.ReLU(), + nn.Conv2d(64, 64, [3, 3], stride=(1, 1)), + nn.ReLU(), + nn.Flatten(), + (nn.Linear(3136, 512)), + nn.ReLU(), + ) + self.policy_fn = nn.Linear(512, num_outputs) + self.value_fn = nn.Linear(512, 1) + + def forward(self, input_dict, state, seq_lens): + model_out = self.model(input_dict["obs"].permute(0, 3, 1, 2)) + self._value_out = self.value_fn(model_out) + return self.policy_fn(model_out), state + + def value_function(self): + return self._value_out.flatten() + + os.environ["SDL_VIDEODRIVER"] = "dummy" parser = argparse.ArgumentParser( From a1f9d517053bb9895779324344ac27388e6be2a5 Mon Sep 17 00:00:00 2001 From: elliottower Date: Wed, 4 Oct 2023 15:09:21 -0400 Subject: [PATCH 5/5] Remove Ray CI as it takes a long time and shouldn't run in CI --- .github/workflows/linux-tutorials-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux-tutorials-test.yml b/.github/workflows/linux-tutorials-test.yml index f5d671ac7..4de3f2d42 100644 --- a/.github/workflows/linux-tutorials-test.yml +++ b/.github/workflows/linux-tutorials-test.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: python-version: ['3.8', '3.9', '3.10', '3.11'] - tutorial: [Tianshou, CustomEnvironment, CleanRL, Ray, SB3/kaz, SB3/waterworld, SB3/connect_four, SB3/test, AgileRL] + tutorial: [Tianshou, CustomEnvironment, CleanRL, SB3/kaz, SB3/waterworld, SB3/connect_four, SB3/test, AgileRL] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }}