Fix coverage command (#33)

Signed-off-by: Sun, Xuehao <[email protected]>
Co-authored-by: chensuyue <[email protected]>

.github/workflows/model_test_cpu.yml

@@ -34,7 +34,7 @@ jobs:
       matrix:
         include:
           - modelName: "opt-125m"
-            datasets: "piqa"
+            datasets: "lambada_openai"
             device: "cpu"
             tasks: "text-generation"
       fail-fast: true

.github/workflows/model_test_hpu.yml

@@ -34,7 +34,7 @@ jobs:
       matrix:
         include:
           - modelName: "opt-125m"
-            datasets: "piqa"
+            datasets: "lambada_openai"
             device: "hpu"
             tasks: "text-generation"
       fail-fast: true

.github/workflows/scripts/install_evals.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+echo -e "\n Install GenAIEval ... "
+cd /GenAIEval
+python -m pip install --no-cache-dir -r requirements.txt
+python setup.py bdist_wheel
+pip install dist/opea_eval*.whl
+
+pip list

.github/workflows/scripts/unittest/calc_coverage.sh

@@ -3,11 +3,11 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-source ../../change_color
+source ../change_color
 LOG_DIR=$1
 coverage_compare="${LOG_DIR}/coverage_compare.html"
-coverage_log_pr="${LOG_DIR}/UnitTestPR-test/coverage_pr"
-coverage_log_base="${LOG_DIR}/UnitTestbaseline/coverage_base"
+coverage_log_pr="${LOG_DIR}/UnitTestPR-test/pr"
+coverage_log_base="${LOG_DIR}/UnitTestbaseline/base"
 
 function get_coverage_data() {
     # Input argument

.github/workflows/scripts/unittest/unittest.sh

@@ -27,8 +27,8 @@ function pytest() {
     ut_log_name="${LOG_DIR}/unit_test_$1.log"
     export GLOG_minloglevel=2
 
-    genaieval_path=$(python -c 'import GenAIEval; import os; print(os.path.dirname(GenAIEval.__file__))')
-    find . -name "test*.py" | sed 's,\.\/,coverage run --source='"${genaieval_path}"' --append ,g' | sed 's/$/ --verbose/' >run.sh
+    genaieval_path=$(python3 -c 'import evals; print(evals.__path__[0])')
+    find . -name "test*.py" | sed "s,\.\/,coverage run --source=\"${genaieval_path}\" --append ,g" | sed 's/$/ --verbose/' >run.sh
     coverage erase
 
     # run UT

.github/workflows/unittest.yml

@@ -13,6 +13,7 @@ on:
       - GenAIEval/**
       - setup.py
       - tests/**
+      - .github/workflows/scripts/unittest/**
   workflow_dispatch:
 
 # If there is a new commit, the previous jobs will be canceled
@@ -61,7 +62,7 @@ jobs:
 
         - name: Install Dependencies
           run: |
-            docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /GenAIEval && pip install -r requirements.txt && python setup.py install"
+            docker exec ${{ env.CONTAINER_NAME }} bash -c "bash /GenAIEval/.github/workflows/scripts/install_evals.sh"
 
         - name: Run UT
           run: |

evals/evaluation/lm_evaluation_harness/lm_eval/models/huggingface.py

@@ -88,7 +88,7 @@ def __init__(
         dtype: Optional[Union[str, torch.dtype]] = "auto",
         batch_size: Optional[Union[int, str]] = 1,
         max_batch_size: Optional[int] = 64,
-        trust_remote_code: Optional[bool] = False,
+        trust_remote_code: Optional[bool] = True,
         use_fast_tokenizer: Optional[bool] = True,
         add_bos_token: Optional[bool] = False,
         prefix_token_id: Optional[int] = None,

tests/test_lm_eval.py

@@ -21,13 +21,14 @@ def test_lm_eval(self):
             model="hf",
             user_model=user_model,
             tokenizer=tokenizer,
-            tasks="piqa",
+            tasks="lambada_openai",
             device="cpu",
             batch_size=1,
             limit=5,
+            trust_remote_code=True,
         )
         results = evaluate(args)
-        self.assertEqual(results["results"]["piqa"]["acc,none"], 0.6)
+        self.assertEqual(results["results"]["lambada_openai"]["acc,none"], 0.6)
 
 
 if __name__ == "__main__":