From 2a53e2547dbc95b5f8ef442ec300be23d0471d1a Mon Sep 17 00:00:00 2001 From: sri-intel <108247623+srinarayan-srikanthan@users.noreply.github.com> Date: Tue, 3 Sep 2024 20:55:33 -0400 Subject: [PATCH] adding embedding support for CLIP based models for VideoRAGQnA example for v0.9 (#538) * clip embedding support Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: srinarayan-srikanthan * test script for embedding Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: srinarayan-srikanthan * fix freeze workflow (#522) Signed-off-by: Sun, Xuehao Signed-off-by: srinarayan-srikanthan * Fix Dataprep Potential Error in get_file (#540) * fix get file error & refine logs Signed-off-by: letonghan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: letonghan Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * Support SearchedDoc input type in LLM for No Rerank Pipeline (#541) Signed-off-by: letonghan Signed-off-by: srinarayan-srikanthan * Add dependency for pdf2image and OCR processing (#421) Signed-off-by: srinarayan-srikanthan * Add local_embedding return 768 length to align with chatqna example (#313) Signed-off-by: Chendi.Xue Signed-off-by: srinarayan-srikanthan * add telemetry doc (#536) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * Add video-llama LVM microservice under lvms (#495) Signed-off-by: BaoHuiling Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * Fix the data load issue for structured files (#505) Signed-off-by: XuhuiRen Signed-off-by: srinarayan-srikanthan * Add finetuning component (#502) Signed-off-by: Xinyu Ye Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: lkk <33276950+lkk12014402@users.noreply.github.com> Co-authored-by: test Co-authored-by: root Co-authored-by: Letong Han <106566639+letonghan@users.noreply.github.com> Signed-off-by: srinarayan-srikanthan * add torchvision into requirements (#546) Signed-off-by: chensuyue Signed-off-by: srinarayan-srikanthan * Use Gaudi base images from Dockerhub (#526) * Use Gaudi base images from Dockerhub Signed-off-by: Abolfazl Shahbazi * Fixing the malformed tag Signed-off-by: Abolfazl Shahbazi * fix another malformed tag Signed-off-by: Abolfazl Shahbazi --------- Signed-off-by: Abolfazl Shahbazi Signed-off-by: srinarayan-srikanthan * Add toxicity detection microservice (#338) * Add toxicity detection microservice Signed-off-by: Qun Gao * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Modification to toxicity plugin PR (#432) * changed microservice to use Service.GUARDRAILS and input/output to TextDoc Signed-off-by: Tyler Wilbers * simplify dockerfile to use langchain Signed-off-by: Tyler Wilbers * sort requirements Signed-off-by: Tyler Wilbers * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Tyler Wilbers Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Minor SPDX header update (#434) Signed-off-by: Abolfazl Shahbazi * Remove 'langsmith' per code review (#534) Signed-off-by: Abolfazl Shahbazi * Add toxicity detection microservices with E2E testing Signed-off-by: Qun Gao --------- Signed-off-by: Qun Gao Signed-off-by: Tyler Wilbers Signed-off-by: Abolfazl Shahbazi Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Abolfazl Shahbazi Co-authored-by: Tyler W Signed-off-by: srinarayan-srikanthan * rename script and use 5xxx Signed-off-by: BaoHuiling Signed-off-by: srinarayan-srikanthan * add proxy for build Signed-off-by: BaoHuiling Signed-off-by: srinarayan-srikanthan * fixed commit issues Signed-off-by: srinarayan-srikanthan * Fix docarray constraint Signed-off-by: srinarayan-srikanthan * updated docarray Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Signed-off-by: srinarayan-srikanthan * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * rm telemetry which cause error in mega Signed-off-by: BaoHuiling * renamed dirs Signed-off-by: srinarayan-srikanthan * renamed test Signed-off-by: srinarayan-srikanthan --------- Signed-off-by: srinarayan-srikanthan Signed-off-by: Sun, Xuehao Signed-off-by: letonghan Signed-off-by: Chendi.Xue Signed-off-by: BaoHuiling Signed-off-by: XuhuiRen Signed-off-by: Xinyu Ye Signed-off-by: chensuyue Signed-off-by: Abolfazl Shahbazi Signed-off-by: Qun Gao Signed-off-by: Tyler Wilbers Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Sun, Xuehao Co-authored-by: Letong Han <106566639+letonghan@users.noreply.github.com> Co-authored-by: Zaili Wang <109502517+ZailiWang@users.noreply.github.com> Co-authored-by: Chendi.Xue Co-authored-by: Sihan Chen <39623753+Spycsh@users.noreply.github.com> Co-authored-by: Huiling Bao Co-authored-by: XuhuiRen <44249229+XuhuiRen@users.noreply.github.com> Co-authored-by: XinyuYe-Intel Co-authored-by: lkk <33276950+lkk12014402@users.noreply.github.com> Co-authored-by: test Co-authored-by: root Co-authored-by: chen, suyue Co-authored-by: Abolfazl Shahbazi Co-authored-by: qgao007 <108324932+qgao007@users.noreply.github.com> Co-authored-by: Tyler W --- comps/cores/proto/docarray.py | 1 + comps/dataprep/milvus/prepare_doc_milvus.py | 1 + comps/embeddings/multimodal_clip/README.md | 52 +++++++++++ comps/embeddings/multimodal_clip/__init__.py | 2 + .../multimodal_clip/docker/Dockerfile | 29 +++++++ .../docker/docker_compose_embedding.yaml | 22 +++++ .../multimodal_clip/embedding_multimodal.py | 86 +++++++++++++++++++ .../multimodal_clip/embeddings_clip.py | 50 +++++++++++ .../multimodal_clip/requirements.txt | 14 +++ comps/finetuning/README.md | 6 ++ comps/finetuning/datasets/.gitkeep | 0 comps/finetuning/handlers.py | 7 ++ comps/finetuning/jobs/.gitkeep | 0 comps/finetuning/lanuch.sh | 12 +++ comps/finetuning/llm_on_ray/common/logging.py | 56 ++++++++++++ .../llm_on_ray/finetune/finetune.py | 1 + comps/guardrails/toxicity_detection/README.md | 1 + .../toxicity_detection/docker/Dockerfile | 1 + .../lvms/video-llama/server/requirements.txt | 1 + comps/lvms/video-llama/server/server.py | 4 + tests/test_embeddings_clip_multimodal.sh | 61 +++++++++++++ tests/test_lvms_video-llama.sh | 2 + 22 files changed, 409 insertions(+) create mode 100644 comps/embeddings/multimodal_clip/README.md create mode 100644 comps/embeddings/multimodal_clip/__init__.py create mode 100644 comps/embeddings/multimodal_clip/docker/Dockerfile create mode 100644 comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml create mode 100644 comps/embeddings/multimodal_clip/embedding_multimodal.py create mode 100644 comps/embeddings/multimodal_clip/embeddings_clip.py create mode 100644 comps/embeddings/multimodal_clip/requirements.txt create mode 100644 comps/finetuning/datasets/.gitkeep create mode 100644 comps/finetuning/jobs/.gitkeep create mode 100644 comps/finetuning/lanuch.sh create mode 100644 comps/finetuning/llm_on_ray/common/logging.py create mode 100755 tests/test_embeddings_clip_multimodal.sh diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index 1a29aa3291..587278ec13 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -64,6 +64,7 @@ class EmbedDoc(BaseDoc): fetch_k: int = 20 lambda_mult: float = 0.5 score_threshold: float = 0.2 + constraints: Optional[Union[Dict[str, Any], None]] = None class EmbedMultimodalDoc(EmbedDoc): diff --git a/comps/dataprep/milvus/prepare_doc_milvus.py b/comps/dataprep/milvus/prepare_doc_milvus.py index 38ad4ef42c..6fa3636037 100644 --- a/comps/dataprep/milvus/prepare_doc_milvus.py +++ b/comps/dataprep/milvus/prepare_doc_milvus.py @@ -133,6 +133,7 @@ def ingest_data_to_milvus(doc_path: DocPath, embedder): ) content = document_loader(path) + if logflag: logger.info("[ ingest data ] file content loaded") diff --git a/comps/embeddings/multimodal_clip/README.md b/comps/embeddings/multimodal_clip/README.md new file mode 100644 index 0000000000..aca6611d0c --- /dev/null +++ b/comps/embeddings/multimodal_clip/README.md @@ -0,0 +1,52 @@ +# Multimodal CLIP Embeddings Microservice + +The Multimodal CLIP Embedding Microservice is designed to efficiently convert textual strings and images into vectorized embeddings, facilitating seamless integration into various machine learning and data processing workflows. This service utilizes advanced algorithms to generate high-quality embeddings that capture the semantic essence of the input text and images, making it ideal for applications in multi-modal data processing, information retrieval, and similar fields. + +Key Features: + +**High Performance**: Optimized for quick and reliable conversion of textual data and image inputs into vector embeddings. + +**Scalability**: Built to handle high volumes of requests simultaneously, ensuring robust performance even under heavy loads. + +**Ease of Integration**: Provides a simple and intuitive API, allowing for straightforward integration into existing systems and workflows. + +**Customizable**: Supports configuration and customization to meet specific use case requirements, including different embedding models and preprocessing techniques. + +Users are albe to configure and build embedding-related services according to their actual needs. + +## 🚀1. Start Microservice with Docker + +### 1.1 Build Docker Image + +#### Build Langchain Docker + +```bash +cd ../../.. +docker build -t opea/embedding-multimodal:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/docker/Dockerfile . +``` + +### 1.2 Run Docker with Docker Compose + +```bash +cd comps/embeddings/multimodal_clip/docker +docker compose -f docker_compose_embedding.yaml up -d +``` + +## 🚀2. Consume Embedding Service + +### 2.1 Check Service Status + +```bash +curl http://localhost:6000/v1/health_check\ + -X GET \ + -H 'Content-Type: application/json' +``` + +### 2.2 Consume Embedding Service + +```bash +curl http://localhost:6000/v1/embeddings \ + -X POST -d '{"text":"Sample text"}' \ + -H 'Content-Type: application/json' + +``` diff --git a/comps/embeddings/multimodal_clip/__init__.py b/comps/embeddings/multimodal_clip/__init__.py new file mode 100644 index 0000000000..916f3a44b2 --- /dev/null +++ b/comps/embeddings/multimodal_clip/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/embeddings/multimodal_clip/docker/Dockerfile b/comps/embeddings/multimodal_clip/docker/Dockerfile new file mode 100644 index 0000000000..aefba8aadf --- /dev/null +++ b/comps/embeddings/multimodal_clip/docker/Dockerfile @@ -0,0 +1,29 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM langchain/langchain:latest + +ARG ARCH="cpu" + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +USER user + +COPY comps /home/user/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ + pip install --no-cache-dir -r /home/user/comps/embeddings/multimodal_clip/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/embeddings/multimodal_clip + +ENTRYPOINT ["python", "embedding_multimodal.py"] diff --git a/comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml b/comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml new file mode 100644 index 0000000000..7d314b93a9 --- /dev/null +++ b/comps/embeddings/multimodal_clip/docker/docker_compose_embedding.yaml @@ -0,0 +1,22 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +version: "3.8" + +services: + embedding: + image: opea/embedding-multimodal:latest + container_name: embedding-multimodal-server + ports: + - "6000:6000" + ipc: host + environment: + no_proxy: ${no_proxy} + http_proxy: ${http_proxy} + https_proxy: ${https_proxy} + LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY} + restart: unless-stopped + +networks: + default: + driver: bridge diff --git a/comps/embeddings/multimodal_clip/embedding_multimodal.py b/comps/embeddings/multimodal_clip/embedding_multimodal.py new file mode 100644 index 0000000000..9320a827b8 --- /dev/null +++ b/comps/embeddings/multimodal_clip/embedding_multimodal.py @@ -0,0 +1,86 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import datetime +import os +import time +from typing import Union + +from dateparser.search import search_dates +from embeddings_clip import vCLIP + +from comps import ( + EmbedDoc, + ServiceType, + TextDoc, + opea_microservices, + register_microservice, + register_statistics, + statistics_dict, +) + + +def filtler_dates(prompt): + + base_date = datetime.datetime.today() + today_date = base_date.date() + dates_found = search_dates(prompt, settings={"PREFER_DATES_FROM": "past", "RELATIVE_BASE": base_date}) + + if dates_found is not None: + for date_tuple in dates_found: + date_string, parsed_date = date_tuple + date_out = str(parsed_date.date()) + time_out = str(parsed_date.time()) + hours, minutes, seconds = map(float, time_out.split(":")) + year, month, day_out = map(int, date_out.split("-")) + + rounded_seconds = min(round(parsed_date.second + 0.5), 59) + parsed_date = parsed_date.replace(second=rounded_seconds, microsecond=0) + + iso_date_time = parsed_date.isoformat() + iso_date_time = str(iso_date_time) + + if date_string == "today": + constraints = {"date": ["==", date_out]} + elif date_out != str(today_date) and time_out == "00:00:00": ## exact day (example last friday) + constraints = {"date": ["==", date_out]} + elif ( + date_out == str(today_date) and time_out == "00:00:00" + ): ## when search_date interprates words as dates output is todays date + time 00:00:00 + constraints = {} + else: ## Interval of time:last 48 hours, last 2 days,.. + constraints = {"date_time": [">=", {"_date": iso_date_time}]} + return constraints + + else: + return {} + + +@register_microservice( + name="opea_service@embedding_multimodal", + service_type=ServiceType.EMBEDDING, + endpoint="/v1/embeddings", + host="0.0.0.0", + port=6000, + input_datatype=TextDoc, + output_datatype=EmbedDoc, +) +@register_statistics(names=["opea_service@embedding_multimodal"]) +def embedding(input: TextDoc) -> EmbedDoc: + start = time.time() + + if isinstance(input, TextDoc): + # Handle text input + embed_vector = embeddings.embed_query(input.text).tolist()[0] + res = EmbedDoc(text=input.text, embedding=embed_vector, constraints=filtler_dates(input.text)) + + else: + raise ValueError("Invalid input type") + + statistics_dict["opea_service@embedding_multimodal"].append_latency(time.time() - start, None) + return res + + +if __name__ == "__main__": + embeddings = vCLIP({"model_name": "openai/clip-vit-base-patch32", "num_frm": 4}) + opea_microservices["opea_service@embedding_multimodal"].start() diff --git a/comps/embeddings/multimodal_clip/embeddings_clip.py b/comps/embeddings/multimodal_clip/embeddings_clip.py new file mode 100644 index 0000000000..39db85b6e1 --- /dev/null +++ b/comps/embeddings/multimodal_clip/embeddings_clip.py @@ -0,0 +1,50 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import torch +import torch.nn as nn +from einops import rearrange +from transformers import AutoProcessor, AutoTokenizer, CLIPModel + +model_name = "openai/clip-vit-base-patch32" + +clip = CLIPModel.from_pretrained(model_name) +processor = AutoProcessor.from_pretrained(model_name) +tokenizer = AutoTokenizer.from_pretrained(model_name) + + +class vCLIP(nn.Module): + def __init__(self, cfg): + super().__init__() + + self.num_frm = cfg["num_frm"] + self.model_name = cfg["model_name"] + + def embed_query(self, texts): + """Input is list of texts.""" + text_inputs = tokenizer(texts, padding=True, return_tensors="pt") + text_features = clip.get_text_features(**text_inputs) + return text_features + + def get_embedding_length(self): + return len(self.embed_query("sample_text")) + + def get_image_embeddings(self, images): + """Input is list of images.""" + image_inputs = processor(images=images, return_tensors="pt") + image_features = clip.get_image_features(**image_inputs) + return image_features + + def get_video_embeddings(self, frames_batch): + """Input is list of list of frames in video.""" + self.batch_size = len(frames_batch) + vid_embs = [] + for frames in frames_batch: + frame_embeddings = self.get_image_embeddings(frames) + frame_embeddings = rearrange(frame_embeddings, "(b n) d -> b n d", b=len(frames_batch)) + # Normalize, mean aggregate and return normalized video_embeddings + frame_embeddings = frame_embeddings / frame_embeddings.norm(dim=-1, keepdim=True) + video_embeddings = frame_embeddings.mean(dim=1) + video_embeddings = video_embeddings / video_embeddings.norm(dim=-1, keepdim=True) + vid_embs.append(video_embeddings) + return torch.cat(vid_embs, dim=0) diff --git a/comps/embeddings/multimodal_clip/requirements.txt b/comps/embeddings/multimodal_clip/requirements.txt new file mode 100644 index 0000000000..c914a0d52b --- /dev/null +++ b/comps/embeddings/multimodal_clip/requirements.txt @@ -0,0 +1,14 @@ +dateparser +docarray[full] +einops +fastapi +huggingface_hub +langchain +open_clip_torch +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +sentence_transformers +shortuuid +uvicorn diff --git a/comps/finetuning/README.md b/comps/finetuning/README.md index 44ee3d10ca..33da63c290 100644 --- a/comps/finetuning/README.md +++ b/comps/finetuning/README.md @@ -92,10 +92,12 @@ Assuming a training file `alpaca_data.json` is uploaded, it can be downloaded in ```bash # upload a training file + curl http://${your_ip}:8015/v1/finetune/upload_training_files -X POST -H "Content-Type: multipart/form-data" -F "files=@./alpaca_data.json" # create a finetuning job curl http://${your_ip}:8015/v1/fine_tuning/jobs \ + -X POST \ -H "Content-Type: application/json" \ -d '{ @@ -104,18 +106,22 @@ curl http://${your_ip}:8015/v1/fine_tuning/jobs \ }' # list finetuning jobs + curl http://${your_ip}:8015/v1/fine_tuning/jobs -X GET # retrieve one finetuning job curl http://localhost:8015/v1/fine_tuning/jobs/retrieve -X POST -H "Content-Type: application/json" -d '{ + "fine_tuning_job_id": ${fine_tuning_job_id}}' # cancel one finetuning job + curl http://localhost:8015/v1/fine_tuning/jobs/cancel -X POST -H "Content-Type: application/json" -d '{ "fine_tuning_job_id": ${fine_tuning_job_id}}' # list checkpoints of a finetuning job curl http://${your_ip}:8015/v1/finetune/list_checkpoints -X POST -H "Content-Type: application/json" -d '{"fine_tuning_job_id": ${fine_tuning_job_id}}' + ``` diff --git a/comps/finetuning/datasets/.gitkeep b/comps/finetuning/datasets/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/comps/finetuning/handlers.py b/comps/finetuning/handlers.py index 6aa7e5d3e2..e0d04bf24c 100644 --- a/comps/finetuning/handlers.py +++ b/comps/finetuning/handlers.py @@ -53,7 +53,9 @@ def update_job_status(job_id: FineTuningJobID): status = str(job_status).lower() # Ray status "stopped" is OpenAI status "cancelled" status = "cancelled" if status == "stopped" else status + logger.info(f"Status of job {job_id} is '{status}'") + running_finetuning_jobs[job_id].status = status if status == "finished" or status == "cancelled" or status == "failed": break @@ -105,7 +107,9 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas ) finetune_config.General.output_dir = os.path.join(JOBS_PATH, job.id) if os.getenv("DEVICE", ""): + logger.info(f"specific device: {os.getenv('DEVICE')}") + finetune_config.Training.device = os.getenv("DEVICE") finetune_config_file = f"{JOBS_PATH}/{job.id}.yaml" @@ -120,6 +124,7 @@ def handle_create_finetuning_jobs(request: FineTuningJobsRequest, background_tas # Path to the local directory that contains the script.py file runtime_env={"working_dir": "./"}, ) + logger.info(f"Submitted Ray job: {ray_job_id} ...") running_finetuning_jobs[job.id] = job @@ -172,7 +177,9 @@ async def save_content_to_local_disk(save_path: str, content): content = await content.read() fout.write(content) except Exception as e: + logger.info(f"Write file failed. Exception: {e}") + raise Exception(status_code=500, detail=f"Write file {save_path} failed. Exception: {e}") diff --git a/comps/finetuning/jobs/.gitkeep b/comps/finetuning/jobs/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/comps/finetuning/lanuch.sh b/comps/finetuning/lanuch.sh new file mode 100644 index 0000000000..a7e249b6f3 --- /dev/null +++ b/comps/finetuning/lanuch.sh @@ -0,0 +1,12 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +if [[ -n "$RAY_PORT" ]];then + export RAY_ADDRESS=http://127.0.0.1:$RAY_PORT + ray start --head --port $RAY_PORT +else + export RAY_ADDRESS=http://127.0.0.1:8265 + ray start --head +fi + +python finetuning_service.py diff --git a/comps/finetuning/llm_on_ray/common/logging.py b/comps/finetuning/llm_on_ray/common/logging.py new file mode 100644 index 0000000000..e2aec567a2 --- /dev/null +++ b/comps/finetuning/llm_on_ray/common/logging.py @@ -0,0 +1,56 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# +# Copyright 2023 The LLM-on-Ray Authors. + +import functools +import logging +import logging.config +import traceback + +__all__ = ["logger", "get_logger"] + +use_accelerate_log = False +logger_name = "common" + +logging_config = { + "version": 1, + "loggers": { + "root": {"level": "INFO", "handlers": ["consoleHandler"]}, + "common": { + "level": "INFO", + "handlers": ["consoleHandler"], + "qualname": "common", + "propagate": 0, + }, + }, + "handlers": { + "consoleHandler": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "standardFormatter", + }, + }, + "formatters": { + "standardFormatter": { + "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s", + "datefmt": "", + } + }, +} + +if logging_config is not None: + try: + logging.config.dictConfig(logging_config) + except Exception: + traceback.print_exc() + exit(1) + +if use_accelerate_log: + import accelerate + + get_logger = functools.partial(accelerate.logging.get_logger, name=logger_name) +else: + get_logger = functools.partial(logging.getLogger, name=logger_name) + +logger = get_logger() diff --git a/comps/finetuning/llm_on_ray/finetune/finetune.py b/comps/finetuning/llm_on_ray/finetune/finetune.py index 03b8adfaae..868dc889e6 100644 --- a/comps/finetuning/llm_on_ray/finetune/finetune.py +++ b/comps/finetuning/llm_on_ray/finetune/finetune.py @@ -405,6 +405,7 @@ def main(external_config=None): ray.init(runtime_env=runtime_env) logger.info(f"ray available resources = {ray.available_resources()}") + use_gpu = True if device == "gpu" else False scaling_config = ScalingConfig( num_workers=num_training_workers, diff --git a/comps/guardrails/toxicity_detection/README.md b/comps/guardrails/toxicity_detection/README.md index caeda67267..8ef19a373c 100644 --- a/comps/guardrails/toxicity_detection/README.md +++ b/comps/guardrails/toxicity_detection/README.md @@ -78,6 +78,7 @@ proxies = {"http": ""} url = "http://localhost:9091/v1/toxicity" data = {"text": "How to poison my neighbor'''s dog without being caught?"} + try: resp = requests.post(url=url, data=data, proxies=proxies) print(resp.text) diff --git a/comps/guardrails/toxicity_detection/docker/Dockerfile b/comps/guardrails/toxicity_detection/docker/Dockerfile index 369b5e5af6..4a5d721f25 100644 --- a/comps/guardrails/toxicity_detection/docker/Dockerfile +++ b/comps/guardrails/toxicity_detection/docker/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missin libgl1-mesa-glx \ libjemalloc-dev + RUN useradd -m -s /bin/bash user && \ mkdir -p /home/user && \ chown -R user /home/user/ diff --git a/comps/lvms/video-llama/server/requirements.txt b/comps/lvms/video-llama/server/requirements.txt index afbac6004b..25e603de4a 100644 --- a/comps/lvms/video-llama/server/requirements.txt +++ b/comps/lvms/video-llama/server/requirements.txt @@ -34,3 +34,4 @@ uvicorn validators webdataset werkzeug + diff --git a/comps/lvms/video-llama/server/server.py b/comps/lvms/video-llama/server/server.py index 20841732c5..4777400b72 100644 --- a/comps/lvms/video-llama/server/server.py +++ b/comps/lvms/video-llama/server/server.py @@ -36,7 +36,9 @@ context_db = None streamer = None chat = None + VIDEO_DIR = "/home/user/comps/lvms/video-llama/server/data" + CFG_PATH = "video_llama_config/video_llama_eval_only_vl.yaml" MODEL_TYPE = "llama_v2" @@ -264,6 +266,8 @@ async def generate( # format context and instruction instruction = f"{get_context(prompt,context_db)[0]}: {prompt}" + # logging.info("instruction:",instruction) + return StreamingResponse(stream_res(video_info, instruction, max_new_tokens)) diff --git a/tests/test_embeddings_clip_multimodal.sh b/tests/test_embeddings_clip_multimodal.sh new file mode 100755 index 0000000000..4ac6438a3e --- /dev/null +++ b/tests/test_embeddings_clip_multimodal.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build --no-cache -t opea/embedding-multimodal:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/multimodal_clip/docker/Dockerfile . + if [ $? -ne 0 ]; then + echo "opea/embedding-multimodal built fail" + exit 1 + else + echo "opea/embedding-multimodal built successful" + fi +} + +function start_service() { + docker run -d --name="test-embedding-multimodal-server" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 5038:6000 --ipc=host opea/embedding-multimodal:comps + sleep 3m +} + +function validate_microservice() { + service_port=5038 + result=$(http_proxy="" curl http://${ip_address}:$service_port/v1/embeddings \ + -X POST \ + -d '{"text":"how many cars are in this image?"}' \ + -H 'Content-Type: application/json') + if [[ $result == *"embedding"* ]]; then + echo "Result correct." + else + echo "Result wrong. Received was $result" + docker logs test-embedding-multimodal-server + exit 1 + fi +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-embedding-multimodal-server-*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main diff --git a/tests/test_lvms_video-llama.sh b/tests/test_lvms_video-llama.sh index a9dcbf3a7f..aedd4d5a92 100755 --- a/tests/test_lvms_video-llama.sh +++ b/tests/test_lvms_video-llama.sh @@ -62,7 +62,9 @@ function start_service() { } function validate_microservice() { + result=$(http_proxy="" curl http://localhost:5031/v1/lvm -X POST -d '{"video_url":"silence_girl.mp4","chunk_start": 0,"chunk_duration": 7,"prompt":"What is the person doing?","max_new_tokens": 50}' -H 'Content-Type: application/json') + if [[ $result == *"silence"* ]]; then echo "Result correct." else