Migrate Text Generation Metrics to valor-lite (#802)

Co-authored-by: b.nativi <[email protected]> Co-authored-by: Charles Zaloom <[email protected]>
Striveworks · Oct 31, 2024 · ce8d261 · ce8d261
1 parent 18fbfd0
commit ce8d261
Show file tree

Hide file tree

Showing 47 changed files with 7,548 additions and 108 deletions.
diff --git a/.github/workflows/core-benchmark-evaluations.yml b/.github/workflows/core-benchmark-evaluations.yml
diff --git a/.github/workflows/core-tests-and-coverage.yml b/.github/workflows/core-tests-and-coverage.yml
diff --git a/.github/workflows/lite-tests-and-coverage.yml b/.github/workflows/lite-tests-and-coverage.yml
@@ -22,56 +22,56 @@ jobs:
       - name: run classification tests and report coverage
         run: |
           pip install -e ".[test]"
-          COVERAGE_FILE=.coverage.classification python -m coverage run --omit "tests/*"  -m pytest -v tests/classification/
+          COVERAGE_FILE=.coverage.classification python -m coverage run --include "valor_lite/*"  -m pytest -v tests/classification/
           python -m coverage combine
           python -m coverage report -m
           python -m coverage json
           export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
           echo "total=$TOTAL" >> $GITHUB_ENV
-          if (( $TOTAL < 90 )); then
-            echo "Coverage is below 90%"
+          if (( $TOTAL < 99 )); then
+            echo "Coverage is below 99%"
             exit 1
           fi
         working-directory: ./lite
       - name: run object detection tests and report coverage
         run: |
           pip install -e ".[test]"
-          COVERAGE_FILE=.coverage.detection python -m coverage run --omit "tests/*"  -m pytest -v tests/object_detection/
+          COVERAGE_FILE=.coverage.detection python -m coverage run --include "valor_lite/*"  -m pytest -v tests/object_detection/
           python -m coverage combine
           python -m coverage report -m
           python -m coverage json
           export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
           echo "total=$TOTAL" >> $GITHUB_ENV
-          if (( $TOTAL < 90 )); then
-            echo "Coverage is below 90%"
+          if (( $TOTAL < 99 )); then
+            echo "Coverage is below 99%"
             exit 1
           fi
         working-directory: ./lite
       - name: run semantic segmentation tests and report coverage
         run: |
           pip install -e ".[test]"
-          COVERAGE_FILE=.coverage.segmentation python -m coverage run --omit "tests/*"  -m pytest -v tests/semantic_segmentation/
+          COVERAGE_FILE=.coverage.segmentation python -m coverage run --include "valor_lite/*"  -m pytest -v tests/semantic_segmentation/
           python -m coverage combine
           python -m coverage report -m
           python -m coverage json
           export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
           echo "total=$TOTAL" >> $GITHUB_ENV
-          if (( $TOTAL < 90 )); then
-            echo "Coverage is below 90%"
+          if (( $TOTAL < 99 )); then
+            echo "Coverage is below 99%"
+            exit 1
+          fi
+        working-directory: ./lite
+      - name: run text generation tests and report coverage
+        run: |
+          pip install -e ".[test,openai,mistral]"
+          COVERAGE_FILE=.coverage.text_generation python -m coverage run --include "valor_lite/*"  -m pytest -v tests/text_generation/
+          python -m coverage combine
+          python -m coverage report -m
+          python -m coverage json
+          export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
+          echo "total=$TOTAL" >> $GITHUB_ENV
+          if (( $TOTAL < 99 )); then
+            echo "Coverage is below 99%"
             exit 1
           fi
         working-directory: ./lite
-      # - name: run nlp generation tests and report coverage
-      #   run: |
-      #     pip install -e ".[test]"
-      #     COVERAGE_FILE=.coverage.generation python -m coverage run --omit "tests/*"  -m pytest -v tests/text_generation
-      #     python -m coverage combine
-      #     python -m coverage report -m
-      #     python -m coverage json
-      #     export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
-      #     echo "total=$TOTAL" >> $GITHUB_ENV
-      #     if (( $TOTAL < 90 )); then
-      #       echo "Coverage is below 90%"
-      #       exit 1
-      #     fi
-      #   working-directory: ./lite
diff --git a/Makefile b/Makefile
@@ -31,6 +31,9 @@ core-tests:
 	pytest ./core/tests/unit-tests
 	pytest ./core/tests/functional-tests
 
+lite-tests:
+	pytest ./lite/tests/text_generation
+
 start-server:
 	POSTGRES_PASSWORD=password POSTGRES_HOST=localhost POSTGRES_DB=valor uvicorn valor_api.main:app --host 0.0.0.0