Merge branch 'main' into erick/stretch_robot_restructured_arm_updated…

…-merge

.gitignore

@@ -6,6 +6,10 @@ unity/[Bb]uilds/
 unity/tmp/
 unity/[Ll]ogs/
 unity/Assets/AssetStoreTools*
+# fake asset used for testing
+unity/Assets/Resources/FakeResourceAssetManagerMaterial.*
+# ResourceAssetCatalog - generated during the build process
+unity/Assets/Resources/ResourceAssetCatalog.json*
 unity/.vs/
 unity/.idea/
 unity/Assets/[Pp]rivate/
@@ -122,4 +126,5 @@ unity/Assets/Editor/Instant Screenshot.meta
 unity/Assets/Editor/Instant Screenshot/Documentation - Instant screenshot.pdf
 unity/Assets/Editor/Instant Screenshot/Documentation - Instant screenshot.pdf.meta
 
-*.log
+*.log
+tmp/

README.md

@@ -290,4 +290,3 @@ AI2 is a non-profit institute with the mission to contribute to humanity through
 <a href="//prior.allenai.org">
 <p align="center"><img width="100%" src="https://raw.githubusercontent.com/allenai/ai2thor/main/doc/static/ai2-prior.svg" /></p>
 </a>
-

ai2thor/build.py

@@ -58,6 +58,8 @@ def __init__(self):
         self.server_types = ["FIFO", "WSGI"]
         self.url = None
         self.unity_proc = None
+        external_system_platforms = dict(Linux=Linux64, Darwin=OSXIntel64)
+        self.platform = external_system_platforms[platform.system()]
 
     def download(self):
         pass

ai2thor/controller.py

@@ -657,7 +657,9 @@ def key_sort_func(scene_name):
         # with CloudRendering the command-line height/width aren't respected, so
         # we compare here with what the desired height/width are and
         # update the resolution if they are different
-        if (
+        # if Python is running against the Unity Editor then
+        # ChangeResolution won't have an affect, so it gets skipped
+        if (self.server.unity_proc is not None) and (
             target_width != self.last_event.screen_width
             or target_height != self.last_event.screen_height
         ):
@@ -670,6 +672,11 @@ def key_sort_func(scene_name):
             self.width = target_width
             self.height = target_height
 
+        # the command line -quality parameter is not respected with the CloudRendering
+        # engine, so the quality is manually changed after launch
+        if self._build.platform == ai2thor.platform.CloudRendering:
+            self.step(action="ChangeQuality", quality=self.quality)
+
         # updates the initialization parameters
         self.initialization_parameters.update(init_params)
 
@@ -1073,44 +1080,48 @@ def _branch_commits(self, branch):
         import requests
 
         payload = []
-        cache_payload, cache_mtime = self._get_cache_commit_history(branch)
-        # we need to limit how often we hit api.github.com since 
-        # there is a rate limit of 60 per hour per IP
-        if cache_payload and (time.time() - cache_mtime) < 300:
-            payload = cache_payload
-        else:
-            try:
-                res = requests.get(
-                    "https://api.github.com/repos/allenai/ai2thor/commits?sha=%s" % branch
-                )
-                if res.status_code == 404:
-                    raise ValueError("Invalid branch name: %s" % branch)
-                elif res.status_code == 403:
+        makedirs(self.commits_cache_dir) # must make directory for lock to succeed
+        # must lock to handle case when multiple procs are started
+        # and all fetch from api.github.com
+        with LockEx(self._cache_commit_filename(branch)):
+            cache_payload, cache_mtime = self._get_cache_commit_history(branch)
+            # we need to limit how often we hit api.github.com since 
+            # there is a rate limit of 60 per hour per IP
+            if cache_payload and (time.time() - cache_mtime) < 300:
+                payload = cache_payload
+            else:
+                try:
+                    res = requests.get(
+                        "https://api.github.com/repos/allenai/ai2thor/commits?sha=%s" % branch
+                    )
+                    if res.status_code == 404:
+                        raise ValueError("Invalid branch name: %s" % branch)
+                    elif res.status_code == 403:
+                        payload, _ = self._get_cache_commit_history(branch)
+                        if payload:
+                            warnings.warn(
+                                "Error retrieving commits: %s - using cached commit history for %s"
+                                % (res.text, branch)
+                            )
+                        else:
+                            res.raise_for_status()
+                    elif res.status_code == 200:
+                        payload = res.json()
+                        self._cache_commit_history(branch, payload)
+                    else:
+                        res.raise_for_status()
+                except requests.exceptions.ConnectionError as e:
                     payload, _ = self._get_cache_commit_history(branch)
                     if payload:
                         warnings.warn(
-                            "Error retrieving commits: %s - using cached commit history for %s"
-                            % (res.text, branch)
+                            "Unable to connect to github.com: %s - using cached commit history for %s"
+                            % (e, branch)
                         )
                     else:
-                        res.raise_for_status()
-                elif res.status_code == 200:
-                    payload = res.json()
-                    self._cache_commit_history(branch, payload)
-                else:
-                    res.raise_for_status()
-            except requests.exceptions.ConnectionError as e:
-                payload, _ = self._get_cache_commit_history(branch)
-                if payload:
-                    warnings.warn(
-                        "Unable to connect to github.com: %s - using cached commit history for %s"
-                        % (e, branch)
-                    )
-                else:
-                    raise Exception(
-                        "Unable to get commit history for branch %s and no cached history exists: %s"
-                        % (branch, e)
-                    )
+                        raise Exception(
+                            "Unable to get commit history for branch %s and no cached history exists: %s"
+                            % (branch, e)
+                        )
 
         return [c["sha"] for c in payload]
 
@@ -1295,7 +1306,7 @@ def start(
         self.last_event = self.server.receive()
 
         # we should be able to get rid of this since we check the resolution in .reset()
-        if height < 300 or width < 300:
+        if self.server.unity_proc is not None and (height < 300 or width < 300):
             self.last_event = self.step("ChangeResolution", x=width, y=height)
 
         return self.last_event

ai2thor/robot_controller.py

@@ -135,14 +135,14 @@ def step(self, action=None, **action_args):
                     self.depth_format,
                     camera_near_plane=self.camera_near_plane,
                     camera_far_plane=self.camera_far_plane,
-                    depth_width=agent_metadata['depthWidth'],
-                    depth_height=agent_metadata['depthHeight'],
+                    depth_width=agent_metadata.get('depthWidth', agent_metadata['screenWidth']),
+                    depth_height=agent_metadata.get('depthHeight', agent_metadata['screenHeight']),
                     flip_y=False,
                     dtype=np.float64,
                 ),
             }
             for key in image_mapping.keys():
-                if key == 'image_depth' and agent_metadata['depthWidth'] != agent_metadata['screenWidth']:
+                if key == 'image_depth' and 'depthWidth' in agent_metadata and agent_metadata['depthWidth'] != agent_metadata['screenWidth']:
                     warnings.warn("Depth and RGB images are not the same resolutions")
 
                 if key in payload and len(payload[key]) > i:

ai2thor/server.py

@@ -154,6 +154,8 @@ def __init__(self, metadata):
 
         self.process_colors()
         self.process_visible_bounds2D()
+        self.third_party_instance_masks = []
+        self.third_party_class_masks = []
         self.third_party_camera_frames = []
         self.third_party_semantic_segmentation_frames = []
         self.third_party_instance_segmentation_frames = []
@@ -223,12 +225,50 @@ def objects_by_type(self, object_type):
             obj for obj in self.metadata["objects"] if obj["objectType"] == object_type
         ]
 
+
+    def process_thirdparty_color_ids(self, instance_segmentation_frame):
+        instance_masks = {}
+        class_masks = {}
+
+        unique_ids, unique_inverse = unique_rows(
+            instance_segmentation_frame.reshape(-1, 3), return_inverse=True
+        )
+        unique_inverse = unique_inverse.reshape(
+            instance_segmentation_frame.shape[:2]
+        )
+        unique_masks = (
+            np.tile(unique_inverse[np.newaxis, :, :], (len(unique_ids), 1, 1))
+            == np.arange(len(unique_ids))[:, np.newaxis, np.newaxis]
+        )
+
+        for color in unique_ids:
+            color_name = self.color_to_object_id.get(
+                tuple(int(cc) for cc in color), "background"
+            )
+            cls = color_name
+            simObj = False
+            if "|" in cls:
+                cls = cls.split("|")[0]
+                simObj = True
+
+            color_ind = np.argmin(np.sum(np.abs(unique_ids - color), axis=1))
+
+            if simObj:
+                instance_masks[color_name] = unique_masks[color_ind, ...]
+
+            if cls not in class_masks:
+                class_masks[cls] = unique_masks[color_ind, ...]
+            else:
+                class_masks[cls] = np.logical_or(
+                    class_masks[cls], unique_masks[color_ind, ...]
+                )
+
+        return class_masks, instance_masks
+
     def process_colors_ids(self):
         if self.instance_segmentation_frame is None:
             return
 
-        MIN_DETECTION_LEN = 0
-
         self.instance_detections2D = {}
         unique_ids, unique_inverse = unique_rows(
             self.instance_segmentation_frame.reshape(-1, 3), return_inverse=True
@@ -240,9 +280,8 @@ def process_colors_ids(self):
             np.tile(unique_inverse[np.newaxis, :, :], (len(unique_ids), 1, 1))
             == np.arange(len(unique_ids))[:, np.newaxis, np.newaxis]
         )
-        # for unique_color_ind, unique_color in enumerate(unique_ids):
-        for color_bounds in self.metadata["colorBounds"]:
-            color = np.array(color_bounds["color"])
+
+        for color_ind, color in enumerate(unique_ids):
             color_name = self.color_to_object_id.get(
                 tuple(int(cc) for cc in color), "background"
             )
@@ -252,29 +291,29 @@ def process_colors_ids(self):
                 cls = cls.split("|")[0]
                 simObj = True
 
-            bb = np.array(color_bounds["bounds"])
-            bb[[1, 3]] = self.metadata["screenHeight"] - bb[[3, 1]]
-            if not (
-                (bb[2] - bb[0]) < MIN_DETECTION_LEN
-                or (bb[3] - bb[1]) < MIN_DETECTION_LEN
-            ):
-                if cls not in self.class_detections2D:
-                    self.class_detections2D[cls] = []
+            if cls not in self.class_detections2D:
+                self.class_detections2D[cls] = []
+            mask = unique_masks[color_ind, ...]
+            bb = self.mask_bounding_box(mask)
+            self.class_detections2D[cls].append(bb)
+
+            if simObj:
+                self.instance_detections2D[color_name] = bb
+                self.instance_masks[color_name] = mask
 
-                self.class_detections2D[cls].append(bb)
+            if cls not in self.class_masks:
+                self.class_masks[cls] = mask
+            else:
+                self.class_masks[cls] = np.logical_or(self.class_masks[cls], mask)
 
-                color_ind = np.argmin(np.sum(np.abs(unique_ids - color), axis=1))
+    def mask_bounding_box(self, mask):
+        rows = np.any(mask, axis=1)
+        cols = np.any(mask, axis=0)
+        rmin, rmax = np.where(rows)[0][[0, -1]]
+        cmin, cmax = np.where(cols)[0][[0, -1]]
 
-                if simObj:
-                    self.instance_detections2D[color_name] = bb
-                    self.instance_masks[color_name] = unique_masks[color_ind, ...]
+        return cmin, rmin, cmax, rmax
 
-                if cls not in self.class_masks:
-                    self.class_masks[cls] = unique_masks[color_ind, ...]
-                else:
-                    self.class_masks[cls] = np.logical_or(
-                        self.class_masks[cls], unique_masks[color_ind, ...]
-                    )
 
     def _image_depth(self, image_depth_data, **kwargs):
         image_depth = read_buffer_image(
@@ -395,9 +434,11 @@ def add_image_ids(self, image_ids_data):
         self.process_colors_ids()
 
     def add_third_party_image_ids(self, image_ids_data):
-        self.third_party_instance_segmentation_frames.append(
-            read_buffer_image(image_ids_data, self.screen_width, self.screen_height)
-        )
+        instance_segmentation_frame  = read_buffer_image(image_ids_data, self.screen_width, self.screen_height)
+        self.third_party_instance_segmentation_frames.append(instance_segmentation_frame)
+        class_masks, instance_masks = self.process_thirdparty_color_ids(instance_segmentation_frame)
+        self.third_party_instance_masks.append(instance_masks)
+        self.third_party_class_masks.append(class_masks)
 
     def add_image_classes(self, image_classes_data):
         self.semantic_segmentation_frame = read_buffer_image(

ai2thor/tests/data/arm-metadata-schema.json

@@ -603,9 +603,6 @@
     "colors": {
       "type": "null"
     },
-    "colorBounds": {
-      "type": "null"
-    },
     "flatSurfacesOnGrid": {
       "type": "array"
     },
@@ -708,7 +705,6 @@
     "cameraPosition",
     "collided",
     "collidedObjects",
-    "colorBounds",
     "colors",
     "currentTime",
     "distances",

ai2thor/tests/data/metadata-schema.json

@@ -467,9 +467,6 @@
     "colors": {
       "type": "null"
     },
-    "colorBounds": {
-      "type": "null"
-    },
     "flatSurfacesOnGrid": {
       "type": "array"
     },
@@ -571,7 +568,6 @@
     "cameraPosition",
     "collided",
     "collidedObjects",
-    "colorBounds",
     "colors",
     "currentTime",
     "distances",