From 1249c4fe72bad455f34717f17fc26647f6f6fafc Mon Sep 17 00:00:00 2001 From: Sharan Shirodkar <91109427+sharanshirodkar7@users.noreply.github.com> Date: Tue, 17 Sep 2024 22:32:16 -0400 Subject: [PATCH] Prediction Guard LVM component (#676) * added files for PG lvm component Signed-off-by: sharanshirodkar7 * Fix pre-commit issues: end-of-file, requirements.txt, trailing whitespace, imports, and formatting Signed-off-by: sharanshirodkar7 * added package Signed-off-by: sharanshirodkar7 --------- Signed-off-by: sharanshirodkar7 --- .../docker/compose/lvms-compose-cd.yaml | 4 ++ comps/lvms/predictionguard/Dockerfile | 18 +++++ comps/lvms/predictionguard/README.md | 54 +++++++++++++++ comps/lvms/predictionguard/__init__.py | 2 + comps/lvms/predictionguard/lvm.py | 60 ++++++++++++++++ comps/lvms/predictionguard/requirements.txt | 14 ++++ tests/lvms/test_lvms_predictionguard.sh | 68 +++++++++++++++++++ 7 files changed, 220 insertions(+) create mode 100644 comps/lvms/predictionguard/Dockerfile create mode 100644 comps/lvms/predictionguard/README.md create mode 100644 comps/lvms/predictionguard/__init__.py create mode 100644 comps/lvms/predictionguard/lvm.py create mode 100644 comps/lvms/predictionguard/requirements.txt create mode 100644 tests/lvms/test_lvms_predictionguard.sh diff --git a/.github/workflows/docker/compose/lvms-compose-cd.yaml b/.github/workflows/docker/compose/lvms-compose-cd.yaml index a1790cf93..781c49cc8 100644 --- a/.github/workflows/docker/compose/lvms-compose-cd.yaml +++ b/.github/workflows/docker/compose/lvms-compose-cd.yaml @@ -17,3 +17,7 @@ services: build: dockerfile: comps/lvms/llava/dependency/Dockerfile.intel_hpu image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest} + lvm-predictionguard: + build: + dockerfile: comps/lvms/predictionguard/Dockerfile + image: ${REGISTRY:-opea}/lvm-predictionguard:${TAG:-latest} diff --git a/comps/lvms/predictionguard/Dockerfile b/comps/lvms/predictionguard/Dockerfile new file mode 100644 index 000000000..65fac9974 --- /dev/null +++ b/comps/lvms/predictionguard/Dockerfile @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Prediction Guard, Inc. +# SPDX-License-Identified: Apache-2.0 + +FROM python:3.11-slim + +# Set environment variables +ENV LANG=en_US.UTF-8 + +COPY comps /home/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/comps/lvms/predictionguard/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home + +WORKDIR /home/comps/lvms/predictionguard + +ENTRYPOINT ["python", "lvm.py"] diff --git a/comps/lvms/predictionguard/README.md b/comps/lvms/predictionguard/README.md new file mode 100644 index 000000000..50ec1a207 --- /dev/null +++ b/comps/lvms/predictionguard/README.md @@ -0,0 +1,54 @@ +# LVM Prediction Guard Microservice + +[Prediction Guard](https://docs.predictionguard.com) allows you to utilize hosted open access LLMs, LVMs, and embedding functionality with seamlessly integrated safeguards. In addition to providing a scalable access to open models, Prediction Guard allows you to configure factual consistency checks, toxicity filters, PII filters, and prompt injection blocking. Join the [Prediction Guard Discord channel](https://discord.gg/TFHgnhAFKd) and request an API key to get started. + +Visual Question and Answering is one of the multimodal tasks empowered by LVMs (Large Visual Models). This microservice supports visual Q&A by using a LLaVA model available via the Prediction Guard API. It accepts two inputs: a prompt and an image. It outputs the answer to the prompt about the image. + +# 🚀1. Start Microservice with Python + +## 1.1 Install Requirements + +```bash +pip install -r requirements.txt +``` + +## 1.2 Start LVM Service + +```bash +python lvm.py +``` + +# 🚀2. Start Microservice with Docker (Option 2) + +## 2.1 Setup Environment Variables + +Setup the following environment variables first + +```bash +export PREDICTIONGUARD_API_KEY=${your_predictionguard_api_key} +``` + +## 2.1 Build Docker Images + +```bash +cd ../../.. +docker build -t opea/lvm-predictionguard:latest -f comps/lvms/predictionguard/Dockerfile . +``` + +## 2.2 Start Service + +```bash +docker run -d --name="lvm-predictionguard" -p 9399:9399 -e PREDICTIONGUARD_API_KEY=$PREDICTIONGUARD_API_KEY opea/lvm-predictionguard:latest +``` + +# 🚀3. Consume LVM Service + +```bash +curl -X POST http://localhost:9399/v1/lvm \ + -H 'Content-Type: application/json' \ + -d '{ + "image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", + "prompt": "What is this?", + "max_new_tokens": 30 + }' +``` diff --git a/comps/lvms/predictionguard/__init__.py b/comps/lvms/predictionguard/__init__.py new file mode 100644 index 000000000..d7ab0a7f4 --- /dev/null +++ b/comps/lvms/predictionguard/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Prediction Guard, Inc. +# SPDX-License-Identified: Apache-2.0 diff --git a/comps/lvms/predictionguard/lvm.py b/comps/lvms/predictionguard/lvm.py new file mode 100644 index 000000000..1a452c457 --- /dev/null +++ b/comps/lvms/predictionguard/lvm.py @@ -0,0 +1,60 @@ +# Copyright (C) 2024 Prediction Guard, Inc. +# SPDX-License-Identified: Apache-2.0 + +import time + +from docarray import BaseDoc +from predictionguard import PredictionGuard + +from comps import ServiceType, TextDoc, opea_microservices, register_microservice, register_statistics, statistics_dict + + +class LVMDoc(BaseDoc): + image: str + prompt: str + max_new_tokens: int = 100 + top_k: int = 50 + top_p: float = 0.99 + temperature: float = 1.0 + + +@register_microservice( + name="opea_service@lvm_predictionguard", + service_type=ServiceType.LVM, + endpoint="/v1/lvm", + host="0.0.0.0", + port=9399, + input_datatype=LVMDoc, + output_datatype=TextDoc, +) +@register_statistics(names=["opea_service@lvm_predictionguard"]) +async def lvm(request: LVMDoc) -> TextDoc: + start = time.time() + + # make a request to the Prediction Guard API using the LlaVa model + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": request.prompt}, + {"type": "image_url", "image_url": {"url": request.image}}, + ], + }, + ] + result = client.chat.completions.create( + model="llava-1.5-7b-hf", + messages=messages, + max_tokens=request.max_new_tokens, + top_k=request.top_k, + top_p=request.top_p, + temperature=request.temperature, + ) + + statistics_dict["opea_service@lvm_predictionguard"].append_latency(time.time() - start, None) + + return TextDoc(text=result["choices"][0]["message"]["content"]) + + +if __name__ == "__main__": + client = PredictionGuard() + opea_microservices["opea_service@lvm_predictionguard"].start() diff --git a/comps/lvms/predictionguard/requirements.txt b/comps/lvms/predictionguard/requirements.txt new file mode 100644 index 000000000..4179c4587 --- /dev/null +++ b/comps/lvms/predictionguard/requirements.txt @@ -0,0 +1,14 @@ +aiohttp +docarray +fastapi +httpx +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +Pillow +predictionguard +prometheus-fastapi-instrumentator +pyyaml +requests +shortuuid +uvicorn diff --git a/tests/lvms/test_lvms_predictionguard.sh b/tests/lvms/test_lvms_predictionguard.sh new file mode 100644 index 000000000..77d7a7779 --- /dev/null +++ b/tests/lvms/test_lvms_predictionguard.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (C) 2024 Prediction Guard, Inc. +# SPDX-License-Identifier: Apache-2.0 + +set -x # Print commands and their arguments as they are executed + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') # Adjust to a more reliable command +if [ -z "$ip_address" ]; then + ip_address="localhost" # Default to localhost if IP address is empty +fi + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build --no-cache -t opea/lvm-pg:comps -f comps/lvms/predictionguard/Dockerfile . + if [ $? -ne 0 ]; then + echo "opea/lvm-pg build failed" + exit 1 + else + echo "opea/lvm-pg built successfully" + fi +} + +function start_service() { + lvm_service_port=9399 + unset http_proxy + docker run -d --name=test-comps-lvm-pg-server \ + -e http_proxy= -e https_proxy= \ + -e PREDICTIONGUARD_API_KEY=${PREDICTIONGUARD_API_KEY} \ + -p 9399:9399 --ipc=host opea/lvm-pg:comps + sleep 60 # Sleep for 1 minute to allow the service to start +} + +function validate_microservice() { + lvm_service_port=9399 + result=$(http_proxy="" curl http://${ip_address}:${lvm_service_port}/v1/lvm \ + -X POST \ + -d '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt": "Describe the image.", "max_new_tokens": 100}' \ + -H 'Content-Type: application/json') + + if [[ $result == *"text"* ]]; then + echo "Service response is correct." + else + echo "Result wrong. Received was $result" + docker logs test-comps-lvm-pg-server + exit 1 + fi +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-lvm-pg-*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune +} + +main