Prepare Lite for Core Migration (#783)

.github/workflows/lite-tests-and-coverage.yml

@@ -36,7 +36,7 @@ jobs:
       - name: run object detection tests and report coverage
         run: |
           pip install -e ".[test]"
-          COVERAGE_FILE=.coverage.detection python -m coverage run --omit "tests/*"  -m pytest -v tests/detection/
+          COVERAGE_FILE=.coverage.detection python -m coverage run --omit "tests/*"  -m pytest -v tests/object_detection/
           python -m coverage combine
           python -m coverage report -m
           python -m coverage json
@@ -50,7 +50,7 @@ jobs:
       - name: run semantic segmentation tests and report coverage
         run: |
           pip install -e ".[test]"
-          COVERAGE_FILE=.coverage.segmentation python -m coverage run --omit "tests/*"  -m pytest -v tests/segmentation/
+          COVERAGE_FILE=.coverage.segmentation python -m coverage run --omit "tests/*"  -m pytest -v tests/semantic_segmentation/
           python -m coverage combine
           python -m coverage report -m
           python -m coverage json
@@ -61,3 +61,17 @@ jobs:
             exit 1
           fi
         working-directory: ./lite
+      # - name: run nlp generation tests and report coverage
+      #   run: |
+      #     pip install -e ".[test]"
+      #     COVERAGE_FILE=.coverage.generation python -m coverage run --omit "tests/*"  -m pytest -v tests/text_generation
+      #     python -m coverage combine
+      #     python -m coverage report -m
+      #     python -m coverage json
+      #     export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
+      #     echo "total=$TOTAL" >> $GITHUB_ENV
+      #     if (( $TOTAL < 90 )); then
+      #       echo "Coverage is below 90%"
+      #       exit 1
+      #     fi
+      #   working-directory: ./lite

lite/README.md

@@ -1 +1,139 @@
-# valor-lite: Compute classification, object detection, and segmentation metrics locally.
+# valor-lite: Fast, local machine learning evaluation.
+
+valor-lite is a lightweight, numpy-based library designed for fast and seamless evaluation of machine learning models. It is optimized for environments where quick, responsive evaluations are essential, whether as part of a larger service or embedded within user-facing tools.
+
+valor-lite is maintained by Striveworks, a cutting-edge MLOps company based in Austin, Texas. If you'd like to learn more or have questions, we invite you to connect with us on [Slack](https://striveworks-public.slack.com/join/shared_invite/zt-1a0jx768y-2J1fffN~b4fXYM8GecvOhA#/shared-invite/email) or explore our [GitHub repository](https://github.com/striveworks/valor).
+
+For additional details, be sure to check out our user [documentation](https://striveworks.github.io/valor/). We're excited to support you in making the most of Valor!
+
+## Usage
+
+### Classification
+
+```python
+from valor_lite.classification import DataLoader, Classification, MetricType
+
+classifications = [
+    Classification(
+        uid="uid0",
+        groundtruth="dog",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.75, 0.2, 0.05],
+    ),
+    Classification(
+        uid="uid1",
+        groundtruth="cat",
+        predictions=["dog", "cat", "bird"],
+        scores=[0.41, 0.39, 0.1],
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(classifications)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': [0.5],
+    'parameters': {
+        'score_thresholds': [0.0],
+        'hardmax': True,
+        'label': 'dog'
+    }
+}
+```
+
+### Object Detection
+
+```python
+from valor_lite.object_detection import DataLoader, Detection, BoundingBox, MetricType
+
+detections = [
+    Detection(
+        uid="uid0",
+        groundtruths=[
+            BoundingBox(
+                xmin=0, xmax=10,
+                ymin=0, ymax=10,
+                labels=["dog"]
+            ),
+            BoundingBox(
+                xmin=20, xmax=30,
+                ymin=20, ymax=30,
+                labels=["cat"]
+            ),
+        ],
+        predictions=[
+            BoundingBox(
+                xmin=1, xmax=11,
+                ymin=1, ymax=11,
+                labels=["dog", "cat", "bird"],
+                scores=[0.85, 0.1, 0.05]
+            ),
+            BoundingBox(
+                xmin=21, xmax=31,
+                ymin=21, ymax=31,
+                labels=["dog", "cat", "bird"],
+                scores=[0.34, 0.33, 0.33]
+            ),
+        ],
+    ),
+]
+
+loader = DataLoader()
+loader.add_bounding_boxes(detections)
+evaluator = loader.finalize()
+
+metrics = evaluator.evaluate()
+
+assert metrics[MetricType.Precision][0].to_dict() == {
+    'type': 'Precision',
+    'value': 0.5,
+    'parameters': {
+        'iou_threshold': 0.5,
+        'score_threshold': 0.5,
+        'label': 'dog'
+    }
+}
+```
+
+### Semantic Segmentation
+
+```python
+import numpy as np
+from valor_lite.semantic_segmentation import DataLoader, Segmentation, Bitmask, MetricType
+
+segmentations = [
+    Segmentation(
+        uid="uid0",
+        groundtruths=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ],
+        predictions=[
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="sky",
+            ),
+            Bitmask(
+                mask=np.random.randint(2, size=(10,10), dtype=np.bool_),
+                label="ground",
+            )
+        ]
+    ),
+]
+
+loader = DataLoader()
+loader.add_data(segmentations)
+evaluator = loader.finalize()
+
+print(metrics[MetricType.Precision][0])
+```

lite/benchmarks/benchmark_objdet.py

@@ -8,7 +8,7 @@
 
 import requests
 from tqdm import tqdm
-from valor_lite.detection import DataLoader, MetricType
+from valor_lite.object_detection import DataLoader, MetricType
 
 
 class AnnotationType(str, Enum):

lite/examples/object-detection.ipynb

@@ -44,7 +44,7 @@
         "import pandas as pd\n",
         "import matplotlib.pyplot as plt\n",
         "\n",
-        "from valor_lite.detection import DataLoader, MetricType"
+        "from valor_lite.object_detection import DataLoader, MetricType"
       ]
     },
     {

lite/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "valor-lite"
 dynamic = ["version"]
-description = "Compute valor metrics directly in your client."
+description = "Compute valor metrics locally."
 readme = "README.md"
 requires-python = ">=3.10"
 license = { file = "LICENSE" }

lite/tests/detection/conftest.py → lite/tests/object_detection/conftest.py

@@ -1,7 +1,12 @@
 import numpy as np
 import pytest
 from shapely.geometry import Polygon as ShapelyPolygon
-from valor_lite.detection import Bitmask, BoundingBox, Detection, Polygon
+from valor_lite.object_detection import (
+    Bitmask,
+    BoundingBox,
+    Detection,
+    Polygon,
+)
 
 
 @pytest.fixture

lite/tests/detection/test_average_precision.py → lite/tests/object_detection/test_average_precision.py

@@ -1,5 +1,5 @@
 import numpy as np
-from valor_lite.detection import (
+from valor_lite.object_detection import (
     DataLoader,
     Detection,
     MetricType,

lite/tests/detection/test_average_recall.py → lite/tests/object_detection/test_average_recall.py

@@ -1,5 +1,5 @@
 import numpy as np
-from valor_lite.detection import (
+from valor_lite.object_detection import (
     DataLoader,
     Detection,
     MetricType,

lite/tests/detection/test_confusion_matrix.py → lite/tests/object_detection/test_confusion_matrix.py

@@ -1,6 +1,11 @@
 import numpy as np
-from valor_lite.detection import DataLoader, Detection, Evaluator, MetricType
-from valor_lite.detection.computation import compute_confusion_matrix
+from valor_lite.object_detection import (
+    DataLoader,
+    Detection,
+    Evaluator,
+    MetricType,
+)
+from valor_lite.object_detection.computation import compute_confusion_matrix
 
 
 def test_confusion_matrix_no_data():

lite/tests/detection/test_counts.py → lite/tests/object_detection/test_counts.py

@@ -1,4 +1,4 @@
-from valor_lite.detection import DataLoader, Detection, MetricType
+from valor_lite.object_detection import DataLoader, Detection, MetricType
 
 
 def test_counts_metrics_first_class(

lite/tests/detection/test_dataloader.py → lite/tests/object_detection/test_dataloader.py

@@ -3,7 +3,7 @@
 import numpy as np
 import pytest
 from shapely.geometry import Polygon as ShapelyPolygon
-from valor_lite.detection import (
+from valor_lite.object_detection import (
     Bitmask,
     BoundingBox,
     DataLoader,

lite/tests/detection/test_evaluator.py → lite/tests/object_detection/test_evaluator.py

@@ -1,4 +1,4 @@
-from valor_lite.detection import DataLoader, Detection, MetricType
+from valor_lite.object_detection import DataLoader, Detection, MetricType
 
 
 def test_metadata_using_torch_metrics_example(

lite/tests/detection/test_filtering.py → lite/tests/object_detection/test_filtering.py

@@ -2,7 +2,12 @@
 
 import numpy as np
 import pytest
-from valor_lite.detection import BoundingBox, DataLoader, Detection, MetricType
+from valor_lite.object_detection import (
+    BoundingBox,
+    DataLoader,
+    Detection,
+    MetricType,
+)
 
 
 @pytest.fixture

lite/tests/detection/test_iou.py → lite/tests/object_detection/test_iou.py

@@ -1,6 +1,6 @@
 import numpy as np
 from shapely.geometry import Polygon as ShapelyPolygon
-from valor_lite.detection.computation import (
+from valor_lite.object_detection.computation import (
     compute_bbox_iou,
     compute_bitmask_iou,
     compute_polygon_iou,

lite/tests/detection/test_pr_curve.py → lite/tests/object_detection/test_pr_curve.py

@@ -1,5 +1,5 @@
 import numpy as np
-from valor_lite.detection import (
+from valor_lite.object_detection import (
     DataLoader,
     Detection,
     MetricType,

lite/tests/detection/test_precision.py → lite/tests/object_detection/test_precision.py

@@ -1,4 +1,4 @@
-from valor_lite.detection import DataLoader, Detection, MetricType
+from valor_lite.object_detection import DataLoader, Detection, MetricType
 
 
 def test_precision_metrics_first_class(

lite/tests/detection/test_recall.py → lite/tests/object_detection/test_recall.py

@@ -1,4 +1,4 @@
-from valor_lite.detection import DataLoader, Detection, MetricType
+from valor_lite.object_detection import DataLoader, Detection, MetricType
 
 
 def test_recall_metrics_first_class(

lite/tests/detection/test_schemas.py → lite/tests/object_detection/test_schemas.py

@@ -1,7 +1,12 @@
 import numpy as np
 import pytest
 from shapely.geometry import Polygon as ShapelyPolygon
-from valor_lite.detection import Bitmask, BoundingBox, Detection, Polygon
+from valor_lite.object_detection import (
+    Bitmask,
+    BoundingBox,
+    Detection,
+    Polygon,
+)
 
 
 def test_BoundingBox():

lite/tests/detection/test_stability.py → lite/tests/object_detection/test_stability.py

@@ -1,6 +1,6 @@
 from random import choice, uniform
 
-from valor_lite.detection import BoundingBox, DataLoader, Detection
+from valor_lite.object_detection import BoundingBox, DataLoader, Detection
 
 
 def _generate_random_detections(

lite/tests/segmentation/conftest.py → lite/tests/semantic_segmentation/conftest.py

@@ -1,6 +1,6 @@
 import numpy as np
 import pytest
-from valor_lite.segmentation import Bitmask, Segmentation
+from valor_lite.semantic_segmentation import Bitmask, Segmentation
 
 
 def _generate_boolean_mask(

lite/tests/segmentation/test_accuracy.py → lite/tests/semantic_segmentation/test_accuracy.py

@@ -1,4 +1,4 @@
-from valor_lite.segmentation import (
+from valor_lite.semantic_segmentation import (
     Accuracy,
     DataLoader,
     MetricType,

lite/tests/segmentation/test_annotation.py → lite/tests/semantic_segmentation/test_annotation.py

@@ -1,6 +1,6 @@
 import numpy as np
 import pytest
-from valor_lite.segmentation import Bitmask, Segmentation
+from valor_lite.semantic_segmentation import Bitmask, Segmentation
 
 
 def test_bitmask():

lite/tests/segmentation/test_confusion_matrix.py → lite/tests/semantic_segmentation/test_confusion_matrix.py

@@ -1,4 +1,8 @@
-from valor_lite.segmentation import DataLoader, MetricType, Segmentation
+from valor_lite.semantic_segmentation import (
+    DataLoader,
+    MetricType,
+    Segmentation,
+)
 
 
 def test_confusion_matrix_basic_segmentations(

lite/tests/segmentation/test_dataloader.py → lite/tests/semantic_segmentation/test_dataloader.py

@@ -1,5 +1,5 @@
 import pytest
-from valor_lite.segmentation import DataLoader
+from valor_lite.semantic_segmentation import DataLoader
 
 
 def test_no_data():

lite/tests/segmentation/test_evaluator.py → lite/tests/semantic_segmentation/test_evaluator.py

@@ -1,4 +1,4 @@
-from valor_lite.segmentation import DataLoader, Segmentation
+from valor_lite.semantic_segmentation import DataLoader, Segmentation
 
 
 def test_metadata_using_large_random_segmentations(

lite/tests/segmentation/test_f1.py → lite/tests/semantic_segmentation/test_f1.py

@@ -1,4 +1,9 @@
-from valor_lite.segmentation import F1, DataLoader, MetricType, Segmentation
+from valor_lite.semantic_segmentation import (
+    F1,
+    DataLoader,
+    MetricType,
+    Segmentation,
+)
 
 
 def test_f1_basic_segmentations(basic_segmentations: list[Segmentation]):

lite/tests/segmentation/test_filtering.py → lite/tests/semantic_segmentation/test_filtering.py

@@ -1,5 +1,5 @@
 import numpy as np
-from valor_lite.segmentation import DataLoader, Segmentation
+from valor_lite.semantic_segmentation import DataLoader, Segmentation
 
 
 def test_filtering(segmentations_from_boxes: list[Segmentation]):

lite/tests/segmentation/test_iou.py → lite/tests/semantic_segmentation/test_iou.py

@@ -1,4 +1,4 @@
-from valor_lite.segmentation import (
+from valor_lite.semantic_segmentation import (
     DataLoader,
     IoU,
     MetricType,