Prediction Guard LVM component

.github/workflows/docker/compose/lvms-compose-cd.yaml

@@ -17,3 +17,7 @@ services:
     build:
       dockerfile: comps/lvms/llava/dependency/Dockerfile.intel_hpu
     image: ${REGISTRY:-opea}/llava-hpu:${TAG:-latest}
+  lvm-predictionguard:
+    build:
+      dockerfile: comps/lvms/predictionguard/Dockerfile
+    image: ${REGISTRY:-opea}/lvm-predictionguard:${TAG:-latest}

comps/lvms/predictionguard/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright (C) 2024 Prediction Guard, Inc.
+# SPDX-License-Identified: Apache-2.0
+
+FROM python:3.11-slim
+
+# Set environment variables
+ENV LANG=en_US.UTF-8
+
+COPY comps /home/comps
+
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir -r /home/comps/lvms/predictionguard/requirements.txt
+
+ENV PYTHONPATH=$PYTHONPATH:/home
+
+WORKDIR /home/comps/lvms/predictionguard
+
+ENTRYPOINT ["python", "lvm.py"]

comps/lvms/predictionguard/README.md

@@ -0,0 +1,54 @@
+# LVM Prediction Guard Microservice
+
+[Prediction Guard](https://docs.predictionguard.com) allows you to utilize hosted open access LLMs, LVMs, and embedding functionality with seamlessly integrated safeguards. In addition to providing a scalable access to open models, Prediction Guard allows you to configure factual consistency checks, toxicity filters, PII filters, and prompt injection blocking. Join the [Prediction Guard Discord channel](https://discord.gg/TFHgnhAFKd) and request an API key to get started.
+
+Visual Question and Answering is one of the multimodal tasks empowered by LVMs (Large Visual Models). This microservice supports visual Q&A by using a LLaVA model available via the Prediction Guard API. It accepts two inputs: a prompt and an image. It outputs the answer to the prompt about the image.
+
+# 🚀1. Start Microservice with Python
+
+## 1.1 Install Requirements
+
+```bash
+pip install -r requirements.txt
+```
+
+## 1.2 Start LVM Service
+
+```bash
+python lvm.py
+```
+
+# 🚀2. Start Microservice with Docker (Option 2)
+
+## 2.1 Setup Environment Variables
+
+Setup the following environment variables first
+
+```bash
+export PREDICTIONGUARD_API_KEY=${your_predictionguard_api_key}
+```
+
+## 2.1 Build Docker Images
+
+```bash
+cd ../../..
+docker build -t opea/lvm-predictionguard:latest -f comps/lvms/predictionguard/Dockerfile .
+```
+
+## 2.2 Start Service
+
+```bash
+docker run -d --name="lvm-predictionguard" -p 9399:9399 -e PREDICTIONGUARD_API_KEY=$PREDICTIONGUARD_API_KEY opea/lvm-predictionguard:latest
+```
+
+# 🚀3. Consume LVM Service
+
+```bash
+curl -X POST http://localhost:9399/v1/lvm \
+    -H 'Content-Type: application/json' \
+    -d '{
+      "image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC",
+      "prompt": "What is this?",
+      "max_new_tokens": 30
+    }'
+```

comps/lvms/predictionguard/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (C) 2024 Prediction Guard, Inc.
+# SPDX-License-Identified: Apache-2.0

comps/lvms/predictionguard/lvm.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2024 Prediction Guard, Inc.
+# SPDX-License-Identified: Apache-2.0
+
+import time
+
+from docarray import BaseDoc
+from predictionguard import PredictionGuard
+
+from comps import ServiceType, TextDoc, opea_microservices, register_microservice, register_statistics, statistics_dict
+
+
+class LVMDoc(BaseDoc):
+    image: str
+    prompt: str
+    max_new_tokens: int = 100
+    top_k: int = 50
+    top_p: float = 0.99
+    temperature: float = 1.0
+
+
+@register_microservice(
+    name="opea_service@lvm_predictionguard",
+    service_type=ServiceType.LVM,
+    endpoint="/v1/lvm",
+    host="0.0.0.0",
+    port=9399,
+    input_datatype=LVMDoc,
+    output_datatype=TextDoc,
+)
+@register_statistics(names=["opea_service@lvm_predictionguard"])
+async def lvm(request: LVMDoc) -> TextDoc:
+    start = time.time()
+
+    # make a request to the Prediction Guard API using the LlaVa model
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": request.prompt},
+                {"type": "image_url", "image_url": {"url": request.image}},
+            ],
+        },
+    ]
+    result = client.chat.completions.create(
+        model="llava-1.5-7b-hf",
+        messages=messages,
+        max_tokens=request.max_new_tokens,
+        top_k=request.top_k,
+        top_p=request.top_p,
+        temperature=request.temperature,
+    )
+
+    statistics_dict["opea_service@lvm_predictionguard"].append_latency(time.time() - start, None)
+
+    return TextDoc(text=result["choices"][0]["message"]["content"])
+
+
+if __name__ == "__main__":
+    client = PredictionGuard()
+    opea_microservices["opea_service@lvm_predictionguard"].start()

comps/lvms/predictionguard/requirements.txt

@@ -0,0 +1,14 @@
+aiohttp
+docarray
+fastapi
+httpx
+opentelemetry-api
+opentelemetry-exporter-otlp
+opentelemetry-sdk
+Pillow
+predictionguard
+prometheus-fastapi-instrumentator
+pyyaml
+requests
+shortuuid
+uvicorn

tests/lvms/test_lvms_predictionguard.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Copyright (C) 2024 Prediction Guard, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+set -x  # Print commands and their arguments as they are executed
+
+WORKPATH=$(dirname "$PWD")
+ip_address=$(hostname -I | awk '{print $1}')  # Adjust to a more reliable command
+if [ -z "$ip_address" ]; then
+    ip_address="localhost"  # Default to localhost if IP address is empty
+fi
+
+function build_docker_images() {
+    cd $WORKPATH
+    echo $(pwd)
+    docker build --no-cache -t opea/lvm-pg:comps -f comps/lvms/predictionguard/Dockerfile .
+    if [ $? -ne 0 ]; then
+        echo "opea/lvm-pg build failed"
+        exit 1
+    else
+        echo "opea/lvm-pg built successfully"
+    fi
+}
+
+function start_service() {
+    lvm_service_port=9399
+    unset http_proxy
+    docker run -d --name=test-comps-lvm-pg-server \
+        -e http_proxy= -e https_proxy= \
+        -e PREDICTIONGUARD_API_KEY=${PREDICTIONGUARD_API_KEY} \
+        -p 9399:9399 --ipc=host opea/lvm-pg:comps
+    sleep 60  # Sleep for 1 minute to allow the service to start
+}
+
+function validate_microservice() {
+    lvm_service_port=9399
+    result=$(http_proxy="" curl http://${ip_address}:${lvm_service_port}/v1/lvm \
+        -X POST \
+        -d '{"image": "iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8/5+hnoEIwDiqkL4KAcT9GO0U4BxoAAAAAElFTkSuQmCC", "prompt": "Describe the image.", "max_new_tokens": 100}' \
+        -H 'Content-Type: application/json')
+
+    if [[ $result == *"text"* ]]; then
+        echo "Service response is correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-lvm-pg-server
+        exit 1
+    fi
+}
+
+function stop_docker() {
+    cid=$(docker ps -aq --filter "name=test-comps-lvm-pg-*")
+    if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
+}
+
+function main() {
+    stop_docker
+
+    build_docker_images
+    start_service
+
+    validate_microservice
+
+    stop_docker
+    echo y | docker system prune
+}
+
+main