Merge pull request #3 from sjmonson/develop

Rename to llm-eval-test

README.md

@@ -1,4 +1,4 @@
-# perf-llm-eval
+# llm-eval-test
 
 A wrapper around [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness) and [Unitxt](https://github.com/IBM/unitxt) designed for evaluation of a local inference endpoint.
 
@@ -25,16 +25,16 @@ python -m venv venv
 source venv/bin/activate
 
 # Install the package
-pip install git+https://github.com/sjmonson/perf-llm-eval.git
+pip install git+https://github.com/sjmonson/llm-eval-test.git
 
 # View run options
-perf-llm-eval run --help
+llm-eval-test run --help
 ```
 
 ## Usage
 
 ```
-usage: perf-llm-eval run [-h] [--catalog_path PATH] [--tasks_path PATH] [-v | -q] -H ENDPOINT -m MODEL -t TASKS -d PATH [-b INT] [-o OUTPUT]
+usage: llm-eval-test run [-h] [--catalog_path PATH] [--tasks_path PATH] [-v | -q] -H ENDPOINT -m MODEL -t TASKS -d PATH [-b INT] [-o OUTPUT]
 
 Run tasks
 
@@ -74,5 +74,5 @@ huggingface-cli download $DATASET --repo-type dataset --local-dir $DATASETS_DIR/
 # Run the benchmark
 ENDPOINT=http://127.0.0.1:8000/v1/completions # An OpenAI API-compatable completions endpoint
 MODEL_NAME=meta-llama/Llama-3.1-8B # Name of the model hosted on the inference server
-perf-llm-eval run --endpoint $ENDPOINT --model $MODEL_NAME --datasets $DATASETS_DIR --tasks mmlu_pro
+llm-eval-test run --endpoint $ENDPOINT --model $MODEL_NAME --datasets $DATASETS_DIR --tasks mmlu_pro
 ```

perf_llm_eval/__main__.py → llm_eval_test/__main__.py

@@ -4,9 +4,9 @@
 import logging
 import tempfile
 
-from perf_llm_eval.parser import setup_parser
+from llm_eval_test.parser import setup_parser
 
-logger = logging.getLogger("perf-llm-eval")
+logger = logging.getLogger("llm-eval-test")
 
 
 def eval_cli():
@@ -35,7 +35,7 @@ def eval_cli():
     os.environ["UNITXT_ARTIFACTORIES"] = args.catalog_path
 
     # Late import to avoid slow cli
-    from perf_llm_eval.lm_eval_wrapper import LMEvalWrapper
+    from llm_eval_test.lm_eval_wrapper import LMEvalWrapper
 
     if args.command == 'list':
         LMEvalWrapper.list_tasks(args.tasks_path)

perf_llm_eval/lm_eval_wrapper.py → llm_eval_test/lm_eval_wrapper.py

@@ -8,7 +8,7 @@
 from lm_eval.tasks import TaskManager  # type: ignore
 from lm_eval.utils import handle_non_serializable, make_table
 
-logger = logging.getLogger("perf-llm-eval")
+logger = logging.getLogger("llm-eval-test")
 
 class LMEvalWrapper(object):
     @staticmethod

pyproject.toml

@@ -1,5 +1,5 @@
 [tool.poetry]
-name = "perf-llm-eval"
+name = "llm-eval-test"
 version = "0.1.0"
 description = ""
 authors = ["Samuel Monson <[email protected]>"]
@@ -12,7 +12,7 @@ unitxt = "^1.13.1"
 
 
 [tool.poetry.scripts]
-perf-llm-eval = "perf_llm_eval.__main__:eval_cli"
+llm-eval-test = "llm_eval_test.__main__:eval_cli"
 
 [build-system]
 requires = ["poetry-core"]