Merge release/v1.1.0 to develop

app/packages/app/src/components/Nav.tsx

@@ -3,7 +3,7 @@ import {
   GitHubLink,
   Header,
   IconButton,
-  SlackLink,
+  DiscordLink,
   iconContainer,
 } from "@fiftyone/components";
 import { ViewBar } from "@fiftyone/core";
@@ -111,7 +111,7 @@ const Nav: React.FC<
           >
             {mode === "dark" ? <LightMode color="inherit" /> : <DarkMode />}
           </IconButton>
-          <SlackLink />
+          <DiscordLink />
           <GitHubLink />
           <DocsLink />
         </div>

app/packages/app/src/components/Setup.tsx

@@ -2,7 +2,7 @@ import {
   DocsLink,
   GitHubLink,
   Header,
-  SlackLink,
+  DiscordLink,
   iconContainer,
   useTheme,
 } from "@fiftyone/components";
@@ -190,7 +190,7 @@ const Setup = () => {
     <div data-cy="setup-page">
       <Header title={"FiftyOne"}>
         <div className={iconContainer} style={{ flex: 1 }}>
-          <SlackLink />
+          <DiscordLink />
           <GitHubLink />
           <DocsLink />
         </div>

app/packages/components/src/components/Icons/Icons.tsx

@@ -8,18 +8,19 @@ import { iconLink } from "./Icons.module.css";
 
 export { iconContainer } from "./Icons.module.css";
 
-export const SlackLink = () => (
+export const DiscordLink = () => (
   <ExternalLink
     className={iconLink}
-    href={"https://slack.voxel51.com"}
-    title={"Slack"}
+    href={"https://community.voxel51.com/"}
+    title={"Discord"}
   >
     <SvgIcon
       sx={{
         fill: (theme) => theme.palette.text.secondary,
       }}
+      viewBox="0 0 127.14 96.36"
     >
-      <path d="M6 15a2 2 0 0 1-2 2a2 2 0 0 1-2-2a2 2 0 0 1 2-2h2v2m1 0a2 2 0 0 1 2-2a2 2 0 0 1 2 2v5a2 2 0 0 1-2 2a2 2 0 0 1-2-2v-5m2-8a2 2 0 0 1-2-2a2 2 0 0 1 2-2a2 2 0 0 1 2 2v2H9m0 1a2 2 0 0 1 2 2a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2a2 2 0 0 1 2-2h5m8 2a2 2 0 0 1 2-2a2 2 0 0 1 2 2a2 2 0 0 1-2 2h-2v-2m-1 0a2 2 0 0 1-2 2a2 2 0 0 1-2-2V5a2 2 0 0 1 2-2a2 2 0 0 1 2 2v5m-2 8a2 2 0 0 1 2 2a2 2 0 0 1-2 2a2 2 0 0 1-2-2v-2h2m0-1a2 2 0 0 1-2-2a2 2 0 0 1 2-2h5a2 2 0 0 1 2 2a2 2 0 0 1-2 2h-5z" />
+      <path d="M107.7,8.07A105.15,105.15,0,0,0,81.47,0a72.06,72.06,0,0,0-3.36,6.83A97.68,97.68,0,0,0,49,6.83,72.37,72.37,0,0,0,45.64,0,105.89,105.89,0,0,0,19.39,8.09C2.79,32.65-1.71,56.6.54,80.21h0A105.73,105.73,0,0,0,32.71,96.36,77.7,77.7,0,0,0,39.6,85.25a68.42,68.42,0,0,1-10.85-5.18c.91-.66,1.8-1.34,2.66-2a75.57,75.57,0,0,0,64.32,0c.87.71,1.76,1.39,2.66,2a68.68,68.68,0,0,1-10.87,5.19,77,77,0,0,0,6.89,11.1A105.25,105.25,0,0,0,126.6,80.22h0C129.24,52.84,122.09,29.11,107.7,8.07ZM42.45,65.69C36.18,65.69,31,60,31,53s5-12.74,11.43-12.74S54,46,53.89,53,48.84,65.69,42.45,65.69Zm42.24,0C78.41,65.69,73.25,60,73.25,53s5-12.74,11.44-12.74S96.23,46,96.12,53,91.08,65.69,84.69,65.69Z" />
     </SvgIcon>
   </ExternalLink>
 );

app/packages/components/src/components/Icons/index.ts

@@ -12,5 +12,5 @@ export {
   KeyboardArrowUp,
   LookerArrowLeftIcon,
   LookerArrowRightIcon,
-  SlackLink,
+  DiscordLink,
 } from "./Icons";

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/Evaluation.tsx

@@ -1155,12 +1155,28 @@ export default function Evaluation(props: EvaluationProps) {
                         colorscale: confusionMatrixConfig.log
                           ? confusionMatrix?.colorscale || "viridis"
                           : "viridis",
+                        hovertemplate:
+                          [
+                            "<b>count: %{z:d}</b>",
+                            `${
+                              evaluation?.info?.config?.gt_field || "truth"
+                            }: %{y}`,
+                            `${
+                              evaluation?.info?.config?.pred_field ||
+                              "predicted"
+                            }: %{x}`,
+                          ].join(" <br>") + "<extra></extra>",
                       },
                     ]}
                     onClick={({ points }) => {
                       const firstPoint = points[0];
                       loadView("matrix", { x: firstPoint.x, y: firstPoint.y });
                     }}
+                    layout={{
+                      yaxis: {
+                        autorange: "reversed",
+                      },
+                    }}
                   />
                 </Stack>
                 {compareKey && (
@@ -1183,8 +1199,24 @@ export default function Evaluation(props: EvaluationProps) {
                           colorscale: confusionMatrixConfig.log
                             ? compareConfusionMatrix?.colorscale || "viridis"
                             : "viridis",
+                          hovertemplate:
+                            [
+                              "<b>count: %{z:d}</b>",
+                              `${
+                                evaluation?.info?.config?.gt_field || "truth"
+                              }: %{y}`,
+                              `${
+                                evaluation?.info?.config?.pred_field ||
+                                "predicted"
+                              }: %{x}`,
+                            ].join(" <br>") + "<extra></extra>",
                         },
                       ]}
+                      layout={{
+                        yaxis: {
+                          autorange: "reversed",
+                        },
+                      }}
                     />
                   </Stack>
                 )}
@@ -1532,8 +1564,8 @@ function getMatrix(matrices, config) {
   if (!matrices) return;
   const { sortBy = "az", limit } = config;
   const parsedLimit = typeof limit === "number" ? limit : undefined;
-  const classes = matrices[`${sortBy}_classes`].slice(parsedLimit);
-  const matrix = matrices[`${sortBy}_matrix`].slice(parsedLimit);
+  const classes = matrices[`${sortBy}_classes`].slice(0, parsedLimit);
+  const matrix = matrices[`${sortBy}_matrix`].slice(0, parsedLimit);
   const colorscale = matrices[`${sortBy}_colorscale`];
   return { labels: classes, matrix, colorscale };
 }

app/packages/core/src/plugins/SchemaIO/components/NativeModelEvaluationView/EvaluationPlot.tsx

@@ -1,4 +1,5 @@
 import { useTheme } from "@fiftyone/components";
+import { merge } from "lodash";
 import React, { useMemo } from "react";
 import Plot, { PlotParams } from "react-plotly.js";
 
@@ -31,7 +32,6 @@ export default function EvaluationPlot(props: EvaluationPlotProps) {
         color: theme.text.secondary,
         gridcolor: theme.primary.softBorder,
         automargin: true, // Enable automatic margin adjustment
-        scaleanchor: "x",
       },
       autosize: true,
       margin: { t: 20, l: 50, b: 50, r: 20, pad: 0 },
@@ -45,6 +45,11 @@ export default function EvaluationPlot(props: EvaluationPlotProps) {
       },
     };
   }, [theme]);
+
+  const mergedLayout = useMemo(() => {
+    return merge({}, layoutDefaults, layout);
+  }, [layoutDefaults, layout]);
+
   const configDefaults: PlotConfig = useMemo(() => {
     return {
       displaylogo: false,
@@ -66,7 +71,7 @@ export default function EvaluationPlot(props: EvaluationPlotProps) {
   return (
     <Plot
       config={configDefaults}
-      layout={{ ...layoutDefaults, ...layout }}
+      layout={mergedLayout}
       style={{ height: "100%", width: "100%", zIndex: 1, ...style }}
       data={data}
       {...otherProps}

app/packages/looker-3d/src/labels/index.tsx

@@ -135,7 +135,11 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
     const newPolylineOverlays = [];
 
     for (const overlay of rawOverlays) {
-      if (overlay._cls === "Detection") {
+      if (
+        overlay._cls === "Detection" &&
+        overlay.dimensions &&
+        overlay.location
+      ) {
         newCuboidOverlays.push(
           <Cuboid
             key={`cuboid-${overlay.id ?? overlay._id}-${overlay.sampleId}`}

app/packages/looker/src/overlays/detection.ts

@@ -88,7 +88,7 @@ export default class DetectionOverlay<
     this.label.mask && this.drawMask(ctx, state);
     !state.config.thumbnail && this.drawLabelText(ctx, state);
 
-    if (this.is3D) {
+    if (this.is3D && this.label.dimensions && this.label.location) {
       this.fillRectFor3d(ctx, state, this.getColor(state));
     } else {
       this.strokeRect(ctx, state, this.getColor(state));

app/packages/looker/src/worker/threed-label-processor.ts

@@ -47,6 +47,10 @@ const getInferredParamsForUndefinedProjection = (
 
     if (cls === DETECTIONS) {
       for (const detection of label.detections as DetectionLabel[]) {
+        if (!detection.location || !detection.dimensions) {
+          continue;
+        }
+
         const [x, y] = detection.location;
         const [lx, ly] = detection.dimensions;
 
@@ -55,7 +59,7 @@ const getInferredParamsForUndefinedProjection = (
         minY = Math.min(minY, y - ly / 2);
         maxY = Math.max(maxY, y + ly / 2);
       }
-    } else if (cls === "Detection") {
+    } else if (cls === "Detection" && label.location && label.dimensions) {
       const [x, y] = label.location as DetectionLabel["location"];
       const [lx, ly] = label.dimensions as DetectionLabel["dimensions"];
 

docs/source/_templates/layout.html

@@ -54,7 +54,7 @@
         <div class="nav__item nav__dropdown__menu has-arrows">
           <div class="arrow-up--light-primary arrow-left20"></div>
           <ul>
-            <li><a href="{{link_voxel51_slack}}">Community Slack</a></li>
+            <li><a href="{{link_voxel51_discord}}">Community Discord</a></li>
             <li><a href="{{link_voxel51_blog}}">Blog</a></li>
             <li><a href="{{link_events}}">Events</a></li>
           </ul>
@@ -140,7 +140,7 @@
         <a href="{{link_voxel51_blog}}">Blog</a>
         <a href="{{link_docs_fiftyone}}">Docs</a>
         <a href="{{link_voxel51_github}}">GitHub</a>
-        <a href="{{link_voxel51_slack}}">Slack</a>
+        <a href="{{link_voxel51_discord}}">Discord</a>
         <a href="{{link_ourstory}}">About Us</a>
         <a href="{{link_events}}">Events</a>
         <a href="{{link_voxel51_jobs}}">Careers</a>

docs/source/brain.rst

@@ -74,6 +74,13 @@ workflow:
   examples to train on in your data and for visualizing common modes of the
   data.
 
+* :ref:`Leaky Splits <brain-image-leaky-splits>`:
+  Often when sourcing data en masse, duplicates and near duplicates can slip
+  through the cracks. The FiftyOne Brain offers a *leaky-splits analysis* that
+  can be used to find potential leaks between dataset splits. These splits can
+  be misleading when evaluating a model, giving an overly optimistic measure
+  for the quality of training. 
+
 .. note::
 
     Check out the :ref:`tutorials page <tutorials>` for detailed examples
@@ -1759,6 +1766,147 @@ samples being less representative and closer samples being more representative.
    :alt: representativeness
    :align: center
 
+
+.. _brain-image-leaky-splits:
+
+Leaky Splits
+____________
+
+Despite our best efforts, duplicates and other forms of non-IID samples
+show up in our data. When these samples end up in different splits, this
+can have consequences when evaluating a model. It can often be easy to
+overestimate model capability due to this issue. The FiftyOne Brain offers a way
+to identify such cases in dataset splits.
+
+The leaks of a |Dataset| or |DatasetView| can be computed directly without the need
+for the predictions of a pre-trained model via the
+:meth:`compute_leaky_splits() <fiftyone.brain.compute_leaky_splits>`
+method:. The splits of a dataset can be defined in three ways. Through tags, by
+tagging samples with their corresponding split. Through a field, by giving each
+split a unique value in that field. And finally through views, by having views
+corresponding to each split.
+
+.. code-block:: python
+    :linenos:
+
+    import fiftyone as fo
+    import fiftyone.brain as fob
+
+    dataset = fo.load_dataset(...)
+
+    # splits via tags
+    split_tags = ['train', 'test']
+    index, leaks = fob.compute_leaky_splits(dataset, split_tags=split_tags)
+
+    # splits via field
+    split_field = ['split'] # holds split values e.g. 'train' or 'test'
+    index, leaks = fob.compute_leaky_splits(dataset, split_field=split_field)
+
+    # splits via views
+    split_views = {
+        'train' : some_view
+        'test' : some_other_view
+    }
+    index, leaks = fob.compute_leaky_splits(dataset, split_views=split_views)
+
+Here is a sample snippet to run this on the `COCO <https://cocodataset.org/#home>`_ dataset.
+Try it for yourself and see what you may find.
+
+.. code-block:: python
+    :linenos:
+
+    import fiftyone as fo
+    import fiftyone.zoo as foz
+    import fiftyone.utils.random as four
+    from fiftyone.brain import compute_leaky_splits
+
+    # load coco
+    dataset = foz.load_zoo_dataset("coco-2017", split="test")
+
+    # set up splits via tags
+    dataset.untag_samples(dataset.distinct("tags"))
+    four.random_split(dataset, {"train": 0.7, "test": 0.3})
+
+    # compute leaks
+    index, leaks = compute_leaky_splits(dataset, split_tags=['train', 'test'])
+
+Once you have these leaks, it is wise to look through them. You may gain some insight
+into the source of the leaks.
+
+.. code-block:: python
+    :linenos:
+
+    session = fo.launch_app(leaks)
+
+Before evaluating your model on your test set, consider getting a version of it
+with the leaks removed. This can be easily done with the built in method
+:meth:`no_leaks_view() <fiftyone.brain.internal.core.leaky_splits.LeakySplitsIndex.no_leaks_view>`.
+
+.. code-block:: python
+    :linenos:
+
+    # if you already have it
+    test_set = some_view
+
+    # can also be found with the variable `split_views` from the index
+    # make sure to put in the right string based on the field/tag/key in view dict
+    # passed when building the index
+    test_set = index.split_views['test']
+
+    test_set_no_leaks = index.no_leaks_view(test_set) # return a view with leaks removed
+    session.view = test_set_no_leaks
+
+    # do evaluations on test_set_no_leaks rather than test_set
+
+Performance on the clean test set will can be closer to the performance of the
+model in the wild. If you found some leaks in your dataset, consider comparing
+performance on the base test set against the clean test set.
+
+**Input**: A |Dataset| or |DatasetView|, and a definition of splits through one
+of tags, a field, or views.
+
+**Output**: An index that will allow you to look through your leaks and
+provides some useful actions once they are discovered such as automatically
+cleaning the dataset with
+:meth:`no_leaks_view() <fiftyone.brain.internal.core.leaky_splits.LeakySplitsIndex.no_leaks_view>`
+or tagging them for the future with
+:meth:`tag_leaks() <fiftyone.brain.internal.core.leaky_splits.LeakySplitsIndex.tag_leaks>`.
+Besides this, a view with all leaks is returned. Visualization of this view
+can give you an insight into the source of the leaks in your dataset.
+
+**What to expect**: Leakiness find leaks by embedding samples with a powerful
+model and finding very close samples in different splits in this space. Large,
+powerful models that were *not* trained on a dataset can provide insight into
+visual and semantic similarity between images, without creating further leaks
+in the process.
+
+**Similarity**: At its core, the leaky-splits module is a wrapper for the brain's
+:class:`SimilarityIndex <fiftyone.brain.similarity.SimilarityIndex>`. Any similarity
+backend, (see :ref:`similarity backends <brain-similarity-backends>`) that implements
+the :class:`DuplicatesMixin <fiftyone.brain.similarity.DuplicatesMixin>` can be used
+to compute leaky splits. You can either pass an existing similarity index by passing
+its brain key to the argument `similarity_brain_key`, or have the method create one on
+the fly for you. If there is a specific configuration for `Similarity` you would like
+to use, pass it in the argument `similarity_config_dict`.
+
+**Models and Embeddings**: If you opt for the method to create a `SimilarityIndex`
+for you, you can still bring you own model by passing it in the `model` argument.
+Alternatively, compute embeddings and pass the field that they reside on. We will
+handle the rest.
+
+**Thresholds**: The leaky-splits module uses a threshold to decide what samples
+are 'too close' and mark them as potential leaks. This threshold can be changed
+either by passing a value to the `threshold` argument of the `compute_leaky_splits()`
+method, or by using the
+:meth:`set_threshold() <fiftyone.brain.internal.core.leaky_splits.SimilarityIndex.set_threshold>`
+method. The best value for your use-case may vary depending on your dataset, as well
+as the embeddings used. A threshold that's too big will have a lot of false positives,
+a threshold that's too small will have a lot of false negatives.
+
+.. image:: /images/brain/brain-leaky-splits.png
+   :alt: leaky-splits
+   :align: center
+
 .. _brain-managing-runs:
 
 Managing brain runs

docs/source/conf.py

@@ -210,6 +210,7 @@
     "link_voxel51_facebook": "https://www.facebook.com/voxel51/",
     "link_voxel51_github": "https://github.com/voxel51/",
     "link_voxel51_linkedin": "https://www.linkedin.com/company/voxel51/",
+    "link_voxel51_discord": "https://community.voxel51.com",
     "link_voxel51_slack": "https://slack.voxel51.com",
     "link_voxel51_twitter": "https://twitter.com/voxel51",
     "link_voxel51_blog": "https://voxel51.com/blog/",