HITL - Add object visbility handling and hide "self" in viewports. (#…

…1959)

* Add object visiblity and viewport visbility layers.

* Hide self in viewports.

* Don't render viewports if networking is disabled.

examples/hitl/rearrange_v2/rearrange_v2.py

@@ -121,6 +121,7 @@ def __init__(
         client_helper: ClientHelper,
     ):
         self.app_service = app_service
+        self.world = world
         self.user_index = user_index
         self.gui_agent_controller = gui_agent_controller
         self.server_sps_tracker = server_sps_tracker
@@ -163,6 +164,26 @@ def reset(self):
         self.camera_helper.update(self._get_camera_lookat_pos(), dt=0)
         self.ui.reset()
 
+        # If networking is enabled...
+        if self.app_service.client_message_manager:
+            # Assign user agent objects to their own layer.
+            agent_index = self.gui_agent_controller._agent_idx
+            agent_object_ids = self.world.get_agent_object_ids(agent_index)
+            for agent_object_id in agent_object_ids:
+                self.app_service.client_message_manager.set_object_visibility_layer(
+                    object_id=agent_object_id,
+                    layer_id=agent_index,
+                    destination_mask=Mask.from_index(self.user_index),
+                )
+
+            # Show all layers except "user_index" in the default viewport.
+            # This hides the user's own agent in the first person view.
+            self.app_service.client_message_manager.set_viewport_properties(
+                viewport_id=-1,
+                visible_layer_ids=Mask.all_except_index(agent_index),
+                destination_mask=Mask.from_index(self.user_index),
+            )
+
     def update(self, dt: float):
         if self.gui_input.get_key_down(GuiInput.KeyNS.H):
             self.show_gui_text = not self.show_gui_text
@@ -193,14 +214,31 @@ def draw_pip_viewport(self, pip_user_data: UserData):
         """
         Draw a picture-in-picture viewport showing another agent's perspective.
         """
+        # If networking is disabled, skip.
+        if not self.app_service.client_message_manager:
+            return
+
         # Lazy init:
         if not self.pip_initialized:
             self.pip_initialized = True
 
+            # Assign pip agent objects to their own layer.
+            pip_agent_index = pip_user_data.gui_agent_controller._agent_idx
+            agent_object_ids = self.world.get_agent_object_ids(pip_agent_index)
+            for agent_object_id in agent_object_ids:
+                self.app_service.client_message_manager.set_object_visibility_layer(
+                    object_id=agent_object_id,
+                    layer_id=pip_agent_index,
+                    destination_mask=Mask.from_index(self.user_index),
+                )
+
             # Define picture-in-picture (PIP) viewport.
+            # Show all layers except "pip_user_index".
+            # This hides the other agent in the picture-in-picture viewport.
             self.app_service.client_message_manager.set_viewport_properties(
                 viewport_id=PIP_VIEWPORT_ID,
                 viewport_rect_xywh=[0.8, 0.02, 0.18, 0.18],
+                visible_layer_ids=Mask.all_except_index(pip_agent_index),
                 destination_mask=Mask.from_index(self.user_index),
             )
 

habitat-hitl/habitat_hitl/core/client_message_manager.py

@@ -13,6 +13,7 @@
 from habitat_hitl.core.user_mask import Mask, Users
 
 DEFAULT_NORMAL: Final[List[float]] = [0.0, 1.0, 0.0]
+DEFAULT_VIEWPORT_SIZE: Final[List[float]] = [0.0, 0.0, 1.0, 1.0]
 
 
 # TODO: Move to another file.
@@ -242,10 +243,29 @@ def set_server_keyframe_id(
             message = self._messages[user_index]
             message["serverKeyframeId"] = keyframe_id
 
+    def set_object_visibility_layer(
+        self,
+        object_id: int,
+        layer_id: int = -1,
+        destination_mask: Mask = Mask.ALL,
+    ):
+        r"""
+        Set the visibility layer of the instance associated with specified habitat-sim objectId.
+        The layer_id '-1' is the default layer and is visible to all viewports.
+        There are 8 additional layers for controlling visibility (0 to 7).
+        """
+        assert layer_id >= -1
+        assert layer_id < 8
+        for user_index in self._users.indices(destination_mask):
+            message = self._messages[user_index]
+            object_properties = _obtain_object_properties(message, object_id)
+            object_properties["layer"] = layer_id
+
     def set_viewport_properties(
         self,
         viewport_id: int,
-        viewport_rect_xywh: List[float],
+        viewport_rect_xywh: List[float] = DEFAULT_VIEWPORT_SIZE,
+        visible_layer_ids: Mask = Mask.ALL,
         destination_mask: Mask = Mask.ALL,
     ):
         r"""
@@ -255,12 +275,18 @@ def set_viewport_properties(
         viewport_id: Unique identifier of the viewport.
         viewport_rect_xywh: Viewport rect (x position, y position, width, height).
                             In window normalized coordinates, i.e. all values in range [0,1] relative to window size.
+        visible_layer_ids: Visibility layers. Only objects assigned to these layers will be visible to this viewport.
         """
+        layers = Users(8)  # Maximum of 8 layers.
         for user_index in self._users.indices(destination_mask):
             message = self._messages[user_index]
             viewport_properties = _obtain_viewport_properties(
                 message, viewport_id
             )
+            # TODO: Use mask int instead of array
+            viewport_properties["layers"] = []
+            for layer in layers.indices(visible_layer_ids):
+                viewport_properties["layers"].append(layer)
             viewport_properties["rect"] = viewport_rect_xywh
 
     def show_viewport(
@@ -347,6 +373,17 @@ def _create_transform_dict(transform: mn.Matrix4) -> Dict[str, List[float]]:
     }
 
 
+def _obtain_object_properties(
+    message: Message, object_id: int
+) -> Dict[str, Any]:
+    """Get or create the properties dict of an object_id."""
+    if "objects" not in message:
+        message["objects"] = {}
+    if object_id not in message["objects"]:
+        message["objects"][object_id] = {}
+    return message["objects"][object_id]
+
+
 def _obtain_viewport_properties(
     message: Message, viewport_id: int
 ) -> Dict[str, Any]: