From a9d1c47290bcfc76af02db2fd78662b611a76e4b Mon Sep 17 00:00:00 2001 From: Xinyu Ye Date: Mon, 12 Aug 2024 15:25:38 +0800 Subject: [PATCH 01/12] added image2video microservice. Signed-off-by: Xinyu Ye --- comps/image2video/Dockerfile | 18 +++++++ comps/image2video/README.md | 67 ++++++++++++++++++++++++++ comps/image2video/__init__.py | 2 + comps/image2video/image2video.py | 43 +++++++++++++++++ comps/image2video/requirements.txt | 11 +++++ comps/image2video/svd/Dockerfile | 22 +++++++++ comps/image2video/svd/requirements.txt | 7 +++ comps/image2video/svd/svd_server.py | 64 ++++++++++++++++++++++++ 8 files changed, 234 insertions(+) create mode 100644 comps/image2video/Dockerfile create mode 100644 comps/image2video/README.md create mode 100644 comps/image2video/__init__.py create mode 100644 comps/image2video/image2video.py create mode 100644 comps/image2video/requirements.txt create mode 100644 comps/image2video/svd/Dockerfile create mode 100644 comps/image2video/svd/requirements.txt create mode 100644 comps/image2video/svd/svd_server.py diff --git a/comps/image2video/Dockerfile b/comps/image2video/Dockerfile new file mode 100644 index 000000000..9adef002c --- /dev/null +++ b/comps/image2video/Dockerfile @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM python:3.11-slim + +# Set environment variables +ENV LANG=en_US.UTF-8 + +COPY comps /home/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/comps/image2video/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home + +WORKDIR /home/comps/image2video + +ENTRYPOINT ["python", "image2video.py"] \ No newline at end of file diff --git a/comps/image2video/README.md b/comps/image2video/README.md new file mode 100644 index 000000000..ab652a852 --- /dev/null +++ b/comps/image2video/README.md @@ -0,0 +1,67 @@ +# Image-to-Video Microservice + +Image-to-Video is a task that generate video in conditioning the provided image(s). This microservice supports image-to-video task by using Stable Video Diffusion (SVD) model. + +# 🚀1. Start Microservice with Python (Option 1) + +## 1.1 Install Requirements + +```bash +pip install -r requirements.txt +pip install -r svd/requirements.txt +``` + +## 1.2 Start SVD Service + +```bash +# Start SVD service +cd svd/ +python svd_server.py +``` + +## 1.3 Start Image-to-Video Microservice + +```bash +cd .. +# Start the OPEA Microservice +python image2video.py +``` + +# 🚀2. Start Microservice with Docker (Option 2) + +## 2.1 Build Images + +### 2.1.1 SVD Server Image + +```bash +cd ../.. +docker build -t opea/svd:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/image2video/svd/Dockerfile . +``` + +### 2.1.2 Image-to-Video Service Image + +```bash +docker build -t opea/image2video:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/image2video/Dockerfile . +``` + +## 2.2 Start SVD and Image-to-Video Service + +### 2.2.1 Start SVD server + +```bash +docker run --ipc=host -p 9368:9368 -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/svd:latest +``` + +### 2.2.2 Start Image-to-Video service + +```bash +ip_address=$(hostname -I | awk '{print $1}') + +docker run -p 9369:9369 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e SVD_ENDPOINT=http://$ip_address:9368 opea/image2video:latest +``` + +### 2.2.3 Test + +```bash +http_proxy="" curl http://localhost:9369/v1/image2video -XPOST -d '{"images_path":[{"image_path":"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/svd/rocket.png"}]}' -H 'Content-Type: application/json' +``` diff --git a/comps/image2video/__init__.py b/comps/image2video/__init__.py new file mode 100644 index 000000000..916f3a44b --- /dev/null +++ b/comps/image2video/__init__.py @@ -0,0 +1,2 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/comps/image2video/image2video.py b/comps/image2video/image2video.py new file mode 100644 index 000000000..40726b368 --- /dev/null +++ b/comps/image2video/image2video.py @@ -0,0 +1,43 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + + +import json +import os +import requests +import time +from comps import ( + ImagesPath, + VideoPath, + ServiceType, + opea_microservices, + register_microservice, + register_statistics, + statistics_dict, +) + + +@register_microservice( + name="opea_service@image2video", + service_type=ServiceType.IMAGE2VIDEO, + endpoint="/v1/image2video", + host="0.0.0.0", + port=9369, + input_datatype=ImagesPath, + output_datatype=VideoPath, +) +@register_statistics(names=["opea_service@image2video"]) +async def image2video(input: ImagesPath): + start = time.time() + images_path = [img.image_path for img in input.images_path] + inputs = {"images_path": images_path} + video_path = requests.post(url=f"{svd_endpoint}/generate", data=json.dumps(inputs), proxies={"http": None}).json()["video_path"] + + statistics_dict["opea_service@image2video"].append_latency(time.time() - start, None) + return VideoPath(video_path=video_path) + + +if __name__ == "__main__": + svd_endpoint = os.getenv("SVD_ENDPOINT", "http://localhost:9368") + print("Image2video server started.") + opea_microservices["opea_service@image2video"].start() diff --git a/comps/image2video/requirements.txt b/comps/image2video/requirements.txt new file mode 100644 index 000000000..ba4704876 --- /dev/null +++ b/comps/image2video/requirements.txt @@ -0,0 +1,11 @@ +datasets +docarray[full] +fastapi +opentelemetry-api +opentelemetry-exporter-otlp +opentelemetry-sdk +prometheus-fastapi-instrumentator +pydantic==2.7.2 +pydub +shortuuid +uvicorn \ No newline at end of file diff --git a/comps/image2video/svd/Dockerfile b/comps/image2video/svd/Dockerfile new file mode 100644 index 000000000..72d0b7b9d --- /dev/null +++ b/comps/image2video/svd/Dockerfile @@ -0,0 +1,22 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +FROM python:3.11-slim + +# Set environment variables +ENV LANG=en_US.UTF-8 + +ARG ARCH="cpu" + +COPY comps /home/comps + +RUN apt-get update && apt-get install python3-opencv -y && \ + pip install --no-cache-dir --upgrade pip && \ + if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ + pip install --no-cache-dir -r /home/comps/image2video/svd/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home + +WORKDIR /home/comps/image2video/svd + +ENTRYPOINT ["python", "svd_server.py"] \ No newline at end of file diff --git a/comps/image2video/svd/requirements.txt b/comps/image2video/svd/requirements.txt new file mode 100644 index 000000000..2e1030c75 --- /dev/null +++ b/comps/image2video/svd/requirements.txt @@ -0,0 +1,7 @@ +accelerate +diffusers +fastapi +opencv-python +torch +transformers +uvicorn \ No newline at end of file diff --git a/comps/image2video/svd/svd_server.py b/comps/image2video/svd/svd_server.py new file mode 100644 index 000000000..53bfd13c1 --- /dev/null +++ b/comps/image2video/svd/svd_server.py @@ -0,0 +1,64 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""Stand-alone Stable Video Diffusion FastAPI Server.""" + +import argparse +import os +import time + +import torch +import uvicorn +from diffusers import StableVideoDiffusionPipeline +from diffusers.utils import load_image, export_to_video +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response + + +app = FastAPI() + + +@app.get("/health") +async def health() -> Response: + """Health check.""" + return Response(status_code=200) + + +@app.post("/generate") +async def generate(request: Request) -> Response: + print("SVD generation begin.") + request_dict = await request.json() + images_path = request_dict.pop("images_path") + + start = time.time() + images = [load_image(img) for img in images_path] + images = [image.resize((1024, 576)) for image in images] + + generator = torch.manual_seed(args.seed) + frames = pipe(images, decode_chunk_size=8, generator=generator).frames[0] + video_path = os.path.join(os.getcwd(), args.video_path) + export_to_video(frames, video_path, fps=7) + end = time.time() + print(f"SVD video output in {video_path}, time = {end-start}s") + return JSONResponse({"video_path": video_path}) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="0.0.0.0") + parser.add_argument("--port", type=int, default=9368) + parser.add_argument("--model_name_or_path", type=str, default="stabilityai/stable-video-diffusion-img2vid-xt") + parser.add_argument("--video_path", type=str, default="generated.mp4") + parser.add_argument("--seed", type=int, default=42) + + args = parser.parse_args() + pipe = StableVideoDiffusionPipeline.from_pretrained( + args.model_name_or_path + ) + print("Stable Video Diffusion model initialized.") + + uvicorn.run( + app, + host=args.host, + port=args.port, + log_level="debug", + ) From 93264ecce487654dc1c5b68cd377100226c2d175 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 07:30:35 +0000 Subject: [PATCH 02/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/image2video/README.md | 2 +- comps/image2video/image2video.py | 10 +++++++--- comps/image2video/requirements.txt | 2 +- comps/image2video/svd/requirements.txt | 2 +- comps/image2video/svd/svd_server.py | 7 ++----- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/comps/image2video/README.md b/comps/image2video/README.md index ab652a852..c37d6b040 100644 --- a/comps/image2video/README.md +++ b/comps/image2video/README.md @@ -11,7 +11,7 @@ pip install -r requirements.txt pip install -r svd/requirements.txt ``` -## 1.2 Start SVD Service +## 1.2 Start SVD Service ```bash # Start SVD service diff --git a/comps/image2video/image2video.py b/comps/image2video/image2video.py index 40726b368..31e4adf4f 100644 --- a/comps/image2video/image2video.py +++ b/comps/image2video/image2video.py @@ -4,12 +4,14 @@ import json import os -import requests import time + +import requests + from comps import ( ImagesPath, - VideoPath, ServiceType, + VideoPath, opea_microservices, register_microservice, register_statistics, @@ -31,7 +33,9 @@ async def image2video(input: ImagesPath): start = time.time() images_path = [img.image_path for img in input.images_path] inputs = {"images_path": images_path} - video_path = requests.post(url=f"{svd_endpoint}/generate", data=json.dumps(inputs), proxies={"http": None}).json()["video_path"] + video_path = requests.post(url=f"{svd_endpoint}/generate", data=json.dumps(inputs), proxies={"http": None}).json()[ + "video_path" + ] statistics_dict["opea_service@image2video"].append_latency(time.time() - start, None) return VideoPath(video_path=video_path) diff --git a/comps/image2video/requirements.txt b/comps/image2video/requirements.txt index ba4704876..069279834 100644 --- a/comps/image2video/requirements.txt +++ b/comps/image2video/requirements.txt @@ -8,4 +8,4 @@ prometheus-fastapi-instrumentator pydantic==2.7.2 pydub shortuuid -uvicorn \ No newline at end of file +uvicorn diff --git a/comps/image2video/svd/requirements.txt b/comps/image2video/svd/requirements.txt index 2e1030c75..120fd7721 100644 --- a/comps/image2video/svd/requirements.txt +++ b/comps/image2video/svd/requirements.txt @@ -4,4 +4,4 @@ fastapi opencv-python torch transformers -uvicorn \ No newline at end of file +uvicorn diff --git a/comps/image2video/svd/svd_server.py b/comps/image2video/svd/svd_server.py index 53bfd13c1..af177b6cb 100644 --- a/comps/image2video/svd/svd_server.py +++ b/comps/image2video/svd/svd_server.py @@ -9,11 +9,10 @@ import torch import uvicorn from diffusers import StableVideoDiffusionPipeline -from diffusers.utils import load_image, export_to_video +from diffusers.utils import export_to_video, load_image from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, Response - app = FastAPI() @@ -51,9 +50,7 @@ async def generate(request: Request) -> Response: parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() - pipe = StableVideoDiffusionPipeline.from_pretrained( - args.model_name_or_path - ) + pipe = StableVideoDiffusionPipeline.from_pretrained(args.model_name_or_path) print("Stable Video Diffusion model initialized.") uvicorn.run( From 72974e198a7db9ddd26e455a8117c29e9efc6257 Mon Sep 17 00:00:00 2001 From: Xinyu Ye Date: Tue, 13 Aug 2024 13:16:02 +0800 Subject: [PATCH 03/12] addition changes. Signed-off-by: Xinyu Ye --- comps/__init__.py | 3 +++ comps/cores/mega/constants.py | 2 ++ comps/cores/proto/docarray.py | 12 ++++++++++++ 3 files changed, 17 insertions(+) diff --git a/comps/__init__.py b/comps/__init__.py index 9baecbbbd..e537f4081 100644 --- a/comps/__init__.py +++ b/comps/__init__.py @@ -19,6 +19,9 @@ RAGASScores, GraphDoc, LVMDoc, + ImagePath, + ImagesPath, + VideoPath, ) # Constants diff --git a/comps/cores/mega/constants.py b/comps/cores/mega/constants.py index 05eab5284..c0fc9e5d5 100644 --- a/comps/cores/mega/constants.py +++ b/comps/cores/mega/constants.py @@ -29,6 +29,8 @@ class ServiceType(Enum): LVM = 12 KNOWLEDGE_GRAPH = 13 WEB_RETRIEVER = 14 + IMAGE2VIDEO = 15 + TEXT2IMAGE = 16 class MegaServiceEndpoint(Enum): diff --git a/comps/cores/proto/docarray.py b/comps/cores/proto/docarray.py index a5034aa28..c21fe727d 100644 --- a/comps/cores/proto/docarray.py +++ b/comps/cores/proto/docarray.py @@ -135,3 +135,15 @@ class LVMDoc(BaseDoc): image: str prompt: str max_new_tokens: conint(ge=0, le=1024) = 512 + + +class ImagePath(BaseDoc): + image_path: str + + +class ImagesPath(BaseDoc): + images_path: DocList[ImagePath] + + +class VideoPath(BaseDoc): + video_path: str \ No newline at end of file From b9b9833f3a0c829bb86fa896ac5d57a687cb82b6 Mon Sep 17 00:00:00 2001 From: Xinyu Ye Date: Tue, 13 Aug 2024 16:48:48 +0800 Subject: [PATCH 04/12] minor changes Signed-off-by: Xinyu Ye --- comps/image2video/README.md | 3 +-- comps/image2video/svd/svd_server.py | 6 ------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/comps/image2video/README.md b/comps/image2video/README.md index c37d6b040..797ec78cf 100644 --- a/comps/image2video/README.md +++ b/comps/image2video/README.md @@ -1,6 +1,6 @@ # Image-to-Video Microservice -Image-to-Video is a task that generate video in conditioning the provided image(s). This microservice supports image-to-video task by using Stable Video Diffusion (SVD) model. +Image-to-Video is a task that generate video conditioning on the provided image(s). This microservice supports image-to-video task by using Stable Video Diffusion (SVD) model. # 🚀1. Start Microservice with Python (Option 1) @@ -56,7 +56,6 @@ docker run --ipc=host -p 9368:9368 -e http_proxy=$http_proxy -e https_proxy=$htt ```bash ip_address=$(hostname -I | awk '{print $1}') - docker run -p 9369:9369 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e SVD_ENDPOINT=http://$ip_address:9368 opea/image2video:latest ``` diff --git a/comps/image2video/svd/svd_server.py b/comps/image2video/svd/svd_server.py index af177b6cb..d6d94d19e 100644 --- a/comps/image2video/svd/svd_server.py +++ b/comps/image2video/svd/svd_server.py @@ -16,12 +16,6 @@ app = FastAPI() -@app.get("/health") -async def health() -> Response: - """Health check.""" - return Response(status_code=200) - - @app.post("/generate") async def generate(request: Request) -> Response: print("SVD generation begin.") From 209aea7b43bec728556db062f64a74e358106d02 Mon Sep 17 00:00:00 2001 From: "Ye, Xinyu" Date: Wed, 21 Aug 2024 23:06:51 -0400 Subject: [PATCH 05/12] added ut test Signed-off-by: Ye, Xinyu --- tests/test_image2video.sh | 68 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/test_image2video.sh diff --git a/tests/test_image2video.sh b/tests/test_image2video.sh new file mode 100644 index 000000000..73229f310 --- /dev/null +++ b/tests/test_image2video.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -x + +WORKPATH=$(dirname "$PWD") +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + echo $(pwd) + docker build --no-cache -t opea/svd:latest -f comps/image2video/svd/Dockerfile . + if [ $? -ne 0 ]; then + echo "opea/svd built fail" + exit 1 + else + echo "opea/svd built successful" + fi + docker build --no-cache -t opea/image2video:latest -f comps/image2video/Dockerfile . + if [ $? -ne 0 ]; then + echo "opea/image2video built fail" + exit 1 + else + echo "opea/image2video built successful" + fi +} + +function start_service() { + unset http_proxy + docker run -d --name="test-comps-image2video-svd" -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 9368:9368 --ipc=host opea/svd:latest + docker run -d --name="test-comps-image2video" -e SVD_ENDPOINT=http://$ip_address:9368 -e http_proxy=$http_proxy -e https_proxy=$https_proxy -p 9369:9369 --ipc=host opea/image2video:latest + sleep 3m +} + +function validate_microservice() { + result=$(http_proxy="" curl http://localhost:9369/v1/image2video -XPOST -d '{"images_path":[{"image_path":"https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/svd/rocket.png"}]}' -H 'Content-Type: application/json') + if [[ $result == *"generated.mp4"* ]]; then + echo "Result correct." + else + echo "Result wrong." + docker logs test-comps-tts-speecht5 + docker logs test-comps-tts + exit 1 + fi + +} + +function stop_docker() { + cid=$(docker ps -aq --filter "name=test-comps-image2video*") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi +} + +function main() { + + stop_docker + + build_docker_images + start_service + + validate_microservice + + stop_docker + echo y | docker system prune + +} + +main From 40186684921b5f13ce312dcde75c995b7618a4e4 Mon Sep 17 00:00:00 2001 From: "Ye, Xinyu" Date: Tue, 17 Sep 2024 22:43:08 -0400 Subject: [PATCH 06/12] unified path. Signed-off-by: Ye, Xinyu --- comps/image2video/{svd => dependency}/Dockerfile | 0 comps/image2video/{svd => dependency}/requirements.txt | 0 comps/image2video/{svd => dependency}/svd_server.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename comps/image2video/{svd => dependency}/Dockerfile (100%) rename comps/image2video/{svd => dependency}/requirements.txt (100%) rename comps/image2video/{svd => dependency}/svd_server.py (100%) diff --git a/comps/image2video/svd/Dockerfile b/comps/image2video/dependency/Dockerfile similarity index 100% rename from comps/image2video/svd/Dockerfile rename to comps/image2video/dependency/Dockerfile diff --git a/comps/image2video/svd/requirements.txt b/comps/image2video/dependency/requirements.txt similarity index 100% rename from comps/image2video/svd/requirements.txt rename to comps/image2video/dependency/requirements.txt diff --git a/comps/image2video/svd/svd_server.py b/comps/image2video/dependency/svd_server.py similarity index 100% rename from comps/image2video/svd/svd_server.py rename to comps/image2video/dependency/svd_server.py From 105b2d87670b6a48945b07132bc8f120fbc38819 Mon Sep 17 00:00:00 2001 From: "Ye, Xinyu" Date: Tue, 17 Sep 2024 22:44:16 -0400 Subject: [PATCH 07/12] added gaudi support for svd. Signed-off-by: Ye, Xinyu --- comps/image2video/README.md | 23 +++++++++++++-- .../dependency/Dockerfile.intel_hpu | 28 +++++++++++++++++++ comps/image2video/dependency/svd_server.py | 20 ++++++++++++- 3 files changed, 67 insertions(+), 4 deletions(-) create mode 100644 comps/image2video/dependency/Dockerfile.intel_hpu diff --git a/comps/image2video/README.md b/comps/image2video/README.md index 797ec78cf..a8ec7926c 100644 --- a/comps/image2video/README.md +++ b/comps/image2video/README.md @@ -8,14 +8,14 @@ Image-to-Video is a task that generate video conditioning on the provided image( ```bash pip install -r requirements.txt -pip install -r svd/requirements.txt +pip install -r dependency/requirements.txt ``` ## 1.2 Start SVD Service ```bash # Start SVD service -cd svd/ +cd dependency/ python svd_server.py ``` @@ -33,9 +33,18 @@ python image2video.py ### 2.1.1 SVD Server Image +Build SVD server image on Xeon with below command: + +```bash +cd ../.. +docker build -t opea/svd:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/image2video/dependency/Dockerfile . +``` + +Build SVD server image on Gaudi with below command: + ```bash cd ../.. -docker build -t opea/svd:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/image2video/svd/Dockerfile . +docker build -t opea/svd-gaudi:latest --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/image2video/dependency/Dockerfile.intel_hpu . ``` ### 2.1.2 Image-to-Video Service Image @@ -48,10 +57,18 @@ docker build -t opea/image2video:latest --build-arg https_proxy=$https_proxy --b ### 2.2.1 Start SVD server +Start SVD server on Xeon with below command: + ```bash docker run --ipc=host -p 9368:9368 -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/svd:latest ``` +Start SVD server on Gaudi with below command: + +```bash +docker run -p 9368:9368 --runtime=habana -e HABANA_VISIBLE_DEVICES=all -e OMPI_MCA_btl_vader_single_copy_mechanism=none --cap-add=sys_nice --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy opea/svd-gaudi:latest +``` + ### 2.2.2 Start Image-to-Video service ```bash diff --git a/comps/image2video/dependency/Dockerfile.intel_hpu b/comps/image2video/dependency/Dockerfile.intel_hpu new file mode 100644 index 000000000..e8c5e72d8 --- /dev/null +++ b/comps/image2video/dependency/Dockerfile.intel_hpu @@ -0,0 +1,28 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# HABANA environment +# FROM vault.habana.ai/gaudi-docker/1.16.1/ubuntu22.04/habanalabs/pytorch-installer-2.2.2:latest as hpu +FROM opea/habanalabs:1.16.1-pytorch-installer-2.2.2 as hpu +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +RUN rm -rf /etc/ssh/ssh_host* +USER user +# Set environment variables +ENV LANG=en_US.UTF-8 +ENV PYTHONPATH=/home/user:/usr/lib/habanalabs/:/optimum-habana + +COPY comps /home/user/comps + +# Install requirements and optimum habana +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/user/comps/image2video/dependency/requirements.txt && \ + pip install --no-cache-dir optimum[habana] + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/image2video/dependency + +ENTRYPOINT ["python", "svd_server.py", "--device", "hpu"] \ No newline at end of file diff --git a/comps/image2video/dependency/svd_server.py b/comps/image2video/dependency/svd_server.py index d6d94d19e..734be59f1 100644 --- a/comps/image2video/dependency/svd_server.py +++ b/comps/image2video/dependency/svd_server.py @@ -41,10 +41,28 @@ async def generate(request: Request) -> Response: parser.add_argument("--port", type=int, default=9368) parser.add_argument("--model_name_or_path", type=str, default="stabilityai/stable-video-diffusion-img2vid-xt") parser.add_argument("--video_path", type=str, default="generated.mp4") + parser.add_argument("--use_hpu_graphs", default=False, action="store_true") + parser.add_argument("--device", type=str, default="cpu") parser.add_argument("--seed", type=int, default=42) args = parser.parse_args() - pipe = StableVideoDiffusionPipeline.from_pretrained(args.model_name_or_path) + if args.device == "hpu": + from optimum.habana.diffusers import GaudiEulerDiscreteScheduler, GaudiStableVideoDiffusionPipeline + scheduler = GaudiEulerDiscreteScheduler.from_pretrained(args.model_name_or_path, subfolder="scheduler") + kwargs = { + "scheduler": scheduler, + "use_habana": True, + "use_hpu_graphs": args.use_hpu_graphs, + "gaudi_config": "Habana/stable-diffusion", + } + pipeline = GaudiStableVideoDiffusionPipeline.from_pretrained( + args.model_name_or_path, + **kwargs, + ) + elif args.device == "cpu": + pipe = StableVideoDiffusionPipeline.from_pretrained(args.model_name_or_path) + else: + raise NotImplementedError(f"Only support cpu and hpu device now, device {args.device} not supported.") print("Stable Video Diffusion model initialized.") uvicorn.run( From 5ed0acf973da75992de145f6c70e7ef8cfe7e083 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 02:44:54 +0000 Subject: [PATCH 08/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/image2video/dependency/svd_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/comps/image2video/dependency/svd_server.py b/comps/image2video/dependency/svd_server.py index 734be59f1..1b699dabd 100644 --- a/comps/image2video/dependency/svd_server.py +++ b/comps/image2video/dependency/svd_server.py @@ -48,6 +48,7 @@ async def generate(request: Request) -> Response: args = parser.parse_args() if args.device == "hpu": from optimum.habana.diffusers import GaudiEulerDiscreteScheduler, GaudiStableVideoDiffusionPipeline + scheduler = GaudiEulerDiscreteScheduler.from_pretrained(args.model_name_or_path, subfolder="scheduler") kwargs = { "scheduler": scheduler, From bdee37ef0c425499e33467f00e2c70027b48fab0 Mon Sep 17 00:00:00 2001 From: "Ye, Xinyu" Date: Tue, 17 Sep 2024 22:56:41 -0400 Subject: [PATCH 09/12] fix bug Signed-off-by: Ye, Xinyu --- comps/image2video/dependency/svd_server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/comps/image2video/dependency/svd_server.py b/comps/image2video/dependency/svd_server.py index 1b699dabd..443d5f139 100644 --- a/comps/image2video/dependency/svd_server.py +++ b/comps/image2video/dependency/svd_server.py @@ -26,7 +26,7 @@ async def generate(request: Request) -> Response: images = [load_image(img) for img in images_path] images = [image.resize((1024, 576)) for image in images] - generator = torch.manual_seed(args.seed) + generator = torch.manual_seed(args.seed) if args.device == "cpu" else None frames = pipe(images, decode_chunk_size=8, generator=generator).frames[0] video_path = os.path.join(os.getcwd(), args.video_path) export_to_video(frames, video_path, fps=7) @@ -48,7 +48,8 @@ async def generate(request: Request) -> Response: args = parser.parse_args() if args.device == "hpu": from optimum.habana.diffusers import GaudiEulerDiscreteScheduler, GaudiStableVideoDiffusionPipeline - + from optimum.habana.utils import set_seed + set_seed(args.seed) scheduler = GaudiEulerDiscreteScheduler.from_pretrained(args.model_name_or_path, subfolder="scheduler") kwargs = { "scheduler": scheduler, @@ -56,7 +57,7 @@ async def generate(request: Request) -> Response: "use_hpu_graphs": args.use_hpu_graphs, "gaudi_config": "Habana/stable-diffusion", } - pipeline = GaudiStableVideoDiffusionPipeline.from_pretrained( + pipe = GaudiStableVideoDiffusionPipeline.from_pretrained( args.model_name_or_path, **kwargs, ) From 5800be4b8528f0a5abcae99fb4042a43e73f2d21 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 18 Sep 2024 02:57:14 +0000 Subject: [PATCH 10/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- comps/image2video/dependency/svd_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/comps/image2video/dependency/svd_server.py b/comps/image2video/dependency/svd_server.py index 443d5f139..db1194c9c 100644 --- a/comps/image2video/dependency/svd_server.py +++ b/comps/image2video/dependency/svd_server.py @@ -49,6 +49,7 @@ async def generate(request: Request) -> Response: if args.device == "hpu": from optimum.habana.diffusers import GaudiEulerDiscreteScheduler, GaudiStableVideoDiffusionPipeline from optimum.habana.utils import set_seed + set_seed(args.seed) scheduler = GaudiEulerDiscreteScheduler.from_pretrained(args.model_name_or_path, subfolder="scheduler") kwargs = { From f81d9e0585d3bb6bb09bfb0d5819abfc2c2e6b2a Mon Sep 17 00:00:00 2001 From: "Ye, Xinyu" Date: Fri, 20 Sep 2024 03:20:48 -0400 Subject: [PATCH 11/12] fix ut Signed-off-by: Ye, Xinyu --- comps/image2video/dependency/Dockerfile | 2 +- comps/image2video/dependency/Dockerfile.intel_hpu | 6 ++++-- tests/test_image2video.sh | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/comps/image2video/dependency/Dockerfile b/comps/image2video/dependency/Dockerfile index 72d0b7b9d..4290f67c0 100644 --- a/comps/image2video/dependency/Dockerfile +++ b/comps/image2video/dependency/Dockerfile @@ -13,7 +13,7 @@ COPY comps /home/comps RUN apt-get update && apt-get install python3-opencv -y && \ pip install --no-cache-dir --upgrade pip && \ if [ ${ARCH} = "cpu" ]; then pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu; fi && \ - pip install --no-cache-dir -r /home/comps/image2video/svd/requirements.txt + pip install --no-cache-dir -r /home/comps/image2video/dependency/requirements.txt ENV PYTHONPATH=$PYTHONPATH:/home diff --git a/comps/image2video/dependency/Dockerfile.intel_hpu b/comps/image2video/dependency/Dockerfile.intel_hpu index e8c5e72d8..050a8c391 100644 --- a/comps/image2video/dependency/Dockerfile.intel_hpu +++ b/comps/image2video/dependency/Dockerfile.intel_hpu @@ -8,14 +8,16 @@ RUN useradd -m -s /bin/bash user && \ mkdir -p /home/user && \ chown -R user /home/user/ +COPY comps /home/user/comps + +RUN chown -R user /home/user/comps/image2video + RUN rm -rf /etc/ssh/ssh_host* USER user # Set environment variables ENV LANG=en_US.UTF-8 ENV PYTHONPATH=/home/user:/usr/lib/habanalabs/:/optimum-habana -COPY comps /home/user/comps - # Install requirements and optimum habana RUN pip install --no-cache-dir --upgrade pip && \ pip install --no-cache-dir -r /home/user/comps/image2video/dependency/requirements.txt && \ diff --git a/tests/test_image2video.sh b/tests/test_image2video.sh index 73229f310..efb3cb2fb 100644 --- a/tests/test_image2video.sh +++ b/tests/test_image2video.sh @@ -10,7 +10,7 @@ ip_address=$(hostname -I | awk '{print $1}') function build_docker_images() { cd $WORKPATH echo $(pwd) - docker build --no-cache -t opea/svd:latest -f comps/image2video/svd/Dockerfile . + docker build --no-cache -t opea/svd:latest -f comps/image2video/dependency/Dockerfile . if [ $? -ne 0 ]; then echo "opea/svd built fail" exit 1 From abc4beafa5bf7a2fcb2a5dc2d888aa19f0323e71 Mon Sep 17 00:00:00 2001 From: "Ye, Xinyu" Date: Fri, 20 Sep 2024 03:29:28 -0400 Subject: [PATCH 12/12] add docker image release file Signed-off-by: Ye, Xinyu --- .../docker/compose/image2video-compose-cd.yaml | 18 ++++++++++++++++++ comps/image2video/dependency/Dockerfile | 2 +- tests/test_image2video.sh | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/docker/compose/image2video-compose-cd.yaml diff --git a/.github/workflows/docker/compose/image2video-compose-cd.yaml b/.github/workflows/docker/compose/image2video-compose-cd.yaml new file mode 100644 index 000000000..a28c45d21 --- /dev/null +++ b/.github/workflows/docker/compose/image2video-compose-cd.yaml @@ -0,0 +1,18 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +# this file should be run in the root of the repo +# images used by GenAIExamples: image2video,svd,svd-gaudi +services: + image2video: + build: + dockerfile: comps/image2video/Dockerfile + image: ${REGISTRY:-opea}/image2video:${TAG:-latest} + svd: + build: + dockerfile: comps/image2video/dependency/Dockerfile + image: ${REGISTRY:-opea}/svd:${TAG:-latest} + svd-gaudi: + build: + dockerfile: comps/image2video/dependency/Dockerfile.intel_hpu + image: ${REGISTRY:-opea}/svd-gaudi:${TAG:-latest} diff --git a/comps/image2video/dependency/Dockerfile b/comps/image2video/dependency/Dockerfile index 4290f67c0..1ae4dfb75 100644 --- a/comps/image2video/dependency/Dockerfile +++ b/comps/image2video/dependency/Dockerfile @@ -17,6 +17,6 @@ RUN apt-get update && apt-get install python3-opencv -y && \ ENV PYTHONPATH=$PYTHONPATH:/home -WORKDIR /home/comps/image2video/svd +WORKDIR /home/comps/image2video/dependency ENTRYPOINT ["python", "svd_server.py"] \ No newline at end of file diff --git a/tests/test_image2video.sh b/tests/test_image2video.sh index efb3cb2fb..185dfe9c0 100644 --- a/tests/test_image2video.sh +++ b/tests/test_image2video.sh @@ -39,8 +39,8 @@ function validate_microservice() { echo "Result correct." else echo "Result wrong." - docker logs test-comps-tts-speecht5 - docker logs test-comps-tts + docker logs test-comps-image2video-svd + docker logs test-comps-image2video exit 1 fi