From 659a241d8cd00d3cea2053ae682f08a035bab051 Mon Sep 17 00:00:00 2001 From: Abolfazl Shahbazi Date: Sun, 30 Jun 2024 16:49:07 -0700 Subject: [PATCH 1/5] optional vllm microservice container build Signed-off-by: Abolfazl Shahbazi --- .../text-generation/vllm/docker/Dockerfile.microservice | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/comps/llms/text-generation/vllm/docker/Dockerfile.microservice b/comps/llms/text-generation/vllm/docker/Dockerfile.microservice index 765b8b33e..a37b3ea25 100644 --- a/comps/llms/text-generation/vllm/docker/Dockerfile.microservice +++ b/comps/llms/text-generation/vllm/docker/Dockerfile.microservice @@ -1,9 +1,10 @@ - # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 FROM langchain/langchain:latest +ARG ARCH="cpu" + RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ libjemalloc-dev \ @@ -18,7 +19,11 @@ USER user COPY comps /home/user/comps RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir -r /home/user/comps/llms/text-generation/vllm/requirements.txt + if [ ${ARCH} = "cpu" ]; then \ + pip install --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu -r /home/user/comps/llms/text-generation/vllm/requirements.txt; \ + else \ + pip install --no-cache-dir -r /home/user/comps/llms/text-generation/vllm/requirements.txt; \ + fi ENV PYTHONPATH=$PYTHONPATH:/home/user From af0253f448cc0762a787b3d8b6ae0d3df3b8e0af Mon Sep 17 00:00:00 2001 From: Abolfazl Shahbazi Date: Sun, 30 Jun 2024 16:57:40 -0700 Subject: [PATCH 2/5] Minor inline comment for the 'ARCH' build-arg Signed-off-by: Abolfazl Shahbazi --- comps/llms/text-generation/vllm/docker/Dockerfile.microservice | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comps/llms/text-generation/vllm/docker/Dockerfile.microservice b/comps/llms/text-generation/vllm/docker/Dockerfile.microservice index a37b3ea25..af5a0cc19 100644 --- a/comps/llms/text-generation/vllm/docker/Dockerfile.microservice +++ b/comps/llms/text-generation/vllm/docker/Dockerfile.microservice @@ -3,7 +3,7 @@ FROM langchain/langchain:latest -ARG ARCH="cpu" +ARG ARCH="cpu" # Set this to "cpu" or "gpu" RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ libgl1-mesa-glx \ From 58a966f3fdbff2341742cac119065566720380e4 Mon Sep 17 00:00:00 2001 From: Abolfazl Shahbazi Date: Wed, 3 Jul 2024 16:29:20 -0700 Subject: [PATCH 3/5] Introduce 'entrypoint.sh' for some Containers Signed-off-by: Abolfazl Shahbazi --- .../text-generation/vllm/docker/Dockerfile.microservice | 2 +- comps/llms/text-generation/vllm/entrypoint.sh | 8 ++++++++ comps/llms/text-generation/vllm/requirements-runtime.txt | 1 + comps/llms/text-generation/vllm/requirements.txt | 1 - 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 comps/llms/text-generation/vllm/entrypoint.sh create mode 100644 comps/llms/text-generation/vllm/requirements-runtime.txt diff --git a/comps/llms/text-generation/vllm/docker/Dockerfile.microservice b/comps/llms/text-generation/vllm/docker/Dockerfile.microservice index af5a0cc19..ccd977273 100644 --- a/comps/llms/text-generation/vllm/docker/Dockerfile.microservice +++ b/comps/llms/text-generation/vllm/docker/Dockerfile.microservice @@ -29,4 +29,4 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user WORKDIR /home/user/comps/llms/text-generation/vllm -ENTRYPOINT ["python", "llm.py"] +ENTRYPOINT ["bash", "entrypoint.sh"] diff --git a/comps/llms/text-generation/vllm/entrypoint.sh b/comps/llms/text-generation/vllm/entrypoint.sh new file mode 100644 index 000000000..d60eddd36 --- /dev/null +++ b/comps/llms/text-generation/vllm/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +pip --no-cache-dir install -r requirements-runtime.txt + +python llm.py diff --git a/comps/llms/text-generation/vllm/requirements-runtime.txt b/comps/llms/text-generation/vllm/requirements-runtime.txt new file mode 100644 index 000000000..225adde27 --- /dev/null +++ b/comps/llms/text-generation/vllm/requirements-runtime.txt @@ -0,0 +1 @@ +langserve diff --git a/comps/llms/text-generation/vllm/requirements.txt b/comps/llms/text-generation/vllm/requirements.txt index 79cdf238d..e1568305c 100644 --- a/comps/llms/text-generation/vllm/requirements.txt +++ b/comps/llms/text-generation/vllm/requirements.txt @@ -2,7 +2,6 @@ docarray[full] fastapi huggingface_hub langchain==0.1.16 -langserve opentelemetry-api opentelemetry-exporter-otlp opentelemetry-sdk From db57d3b16837d652af6f9d9f9045ec4c30f8c187 Mon Sep 17 00:00:00 2001 From: Abolfazl Shahbazi Date: Wed, 3 Jul 2024 22:19:47 -0700 Subject: [PATCH 4/5] Fixing remaning Dockerfiles that require entrypoint script Signed-off-by: Abolfazl Shahbazi --- comps/llms/summarization/tgi/Dockerfile | 2 +- comps/llms/summarization/tgi/entrypoint.sh | 8 ++++++++ comps/llms/summarization/tgi/requirements-runtime.txt | 1 + comps/llms/summarization/tgi/requirements.txt | 1 - comps/llms/text-generation/tgi/Dockerfile | 2 +- comps/llms/text-generation/tgi/entrypoint.sh | 8 ++++++++ comps/llms/text-generation/tgi/requirements-runtime.txt | 1 + comps/llms/text-generation/tgi/requirements.txt | 1 - 8 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 comps/llms/summarization/tgi/entrypoint.sh create mode 100644 comps/llms/summarization/tgi/requirements-runtime.txt create mode 100644 comps/llms/text-generation/tgi/entrypoint.sh create mode 100644 comps/llms/text-generation/tgi/requirements-runtime.txt diff --git a/comps/llms/summarization/tgi/Dockerfile b/comps/llms/summarization/tgi/Dockerfile index ccbf204d6..c1e1fdcca 100644 --- a/comps/llms/summarization/tgi/Dockerfile +++ b/comps/llms/summarization/tgi/Dockerfile @@ -24,4 +24,4 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user WORKDIR /home/user/comps/llms/summarization/tgi -ENTRYPOINT ["python", "llm.py"] +ENTRYPOINT ["bash", "entrypoint.sh"] diff --git a/comps/llms/summarization/tgi/entrypoint.sh b/comps/llms/summarization/tgi/entrypoint.sh new file mode 100644 index 000000000..d60eddd36 --- /dev/null +++ b/comps/llms/summarization/tgi/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +pip --no-cache-dir install -r requirements-runtime.txt + +python llm.py diff --git a/comps/llms/summarization/tgi/requirements-runtime.txt b/comps/llms/summarization/tgi/requirements-runtime.txt new file mode 100644 index 000000000..225adde27 --- /dev/null +++ b/comps/llms/summarization/tgi/requirements-runtime.txt @@ -0,0 +1 @@ +langserve diff --git a/comps/llms/summarization/tgi/requirements.txt b/comps/llms/summarization/tgi/requirements.txt index f770cf5d7..92c652351 100644 --- a/comps/llms/summarization/tgi/requirements.txt +++ b/comps/llms/summarization/tgi/requirements.txt @@ -2,7 +2,6 @@ docarray[full] fastapi huggingface_hub langchain==0.1.16 -langserve langsmith opentelemetry-api opentelemetry-exporter-otlp diff --git a/comps/llms/text-generation/tgi/Dockerfile b/comps/llms/text-generation/tgi/Dockerfile index b4e200f8c..545af59df 100644 --- a/comps/llms/text-generation/tgi/Dockerfile +++ b/comps/llms/text-generation/tgi/Dockerfile @@ -24,4 +24,4 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user WORKDIR /home/user/comps/llms/text-generation/tgi -ENTRYPOINT ["python", "llm.py"] +ENTRYPOINT ["bash", "entrypoint.sh"] diff --git a/comps/llms/text-generation/tgi/entrypoint.sh b/comps/llms/text-generation/tgi/entrypoint.sh new file mode 100644 index 000000000..d60eddd36 --- /dev/null +++ b/comps/llms/text-generation/tgi/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +pip --no-cache-dir install -r requirements-runtime.txt + +python llm.py diff --git a/comps/llms/text-generation/tgi/requirements-runtime.txt b/comps/llms/text-generation/tgi/requirements-runtime.txt new file mode 100644 index 000000000..225adde27 --- /dev/null +++ b/comps/llms/text-generation/tgi/requirements-runtime.txt @@ -0,0 +1 @@ +langserve diff --git a/comps/llms/text-generation/tgi/requirements.txt b/comps/llms/text-generation/tgi/requirements.txt index f770cf5d7..92c652351 100644 --- a/comps/llms/text-generation/tgi/requirements.txt +++ b/comps/llms/text-generation/tgi/requirements.txt @@ -2,7 +2,6 @@ docarray[full] fastapi huggingface_hub langchain==0.1.16 -langserve langsmith opentelemetry-api opentelemetry-exporter-otlp From ba85b9e5ddd186e1e305a51703959c1bf0d69a58 Mon Sep 17 00:00:00 2001 From: Abolfazl Shahbazi Date: Wed, 3 Jul 2024 23:11:12 -0700 Subject: [PATCH 5/5] Final batch of Dockerfiles requiring entrypoint script Signed-off-by: Abolfazl Shahbazi --- comps/llms/text-generation/ollama/Dockerfile | 2 +- .../llms/text-generation/ollama/entrypoint.sh | 8 ++++ .../ollama/requirements-runtime.txt | 1 + .../text-generation/ollama/requirements.txt | 1 - .../ray_serve/docker/Dockerfile.microservice | 2 +- .../text-generation/ray_serve/entrypoint.sh | 8 ++++ .../ray_serve/requirements-runtime.txt | 1 + .../ray_serve/requirements.txt | 1 - comps/ragas/tgi/Dockerfile | 46 +++++++++---------- comps/ragas/tgi/entrypoint.sh | 8 ++++ comps/ragas/tgi/requirements-runtime.txt | 1 + comps/ragas/tgi/requirements.txt | 1 - 12 files changed, 52 insertions(+), 28 deletions(-) create mode 100644 comps/llms/text-generation/ollama/entrypoint.sh create mode 100644 comps/llms/text-generation/ollama/requirements-runtime.txt create mode 100644 comps/llms/text-generation/ray_serve/entrypoint.sh create mode 100644 comps/llms/text-generation/ray_serve/requirements-runtime.txt create mode 100644 comps/ragas/tgi/entrypoint.sh create mode 100644 comps/ragas/tgi/requirements-runtime.txt diff --git a/comps/llms/text-generation/ollama/Dockerfile b/comps/llms/text-generation/ollama/Dockerfile index 4655ce688..876ca1eef 100644 --- a/comps/llms/text-generation/ollama/Dockerfile +++ b/comps/llms/text-generation/ollama/Dockerfile @@ -25,4 +25,4 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user WORKDIR /home/user/comps/llms/text-generation/ollama -ENTRYPOINT ["python", "llm.py"] +ENTRYPOINT ["bash", "entrypoint.sh"] diff --git a/comps/llms/text-generation/ollama/entrypoint.sh b/comps/llms/text-generation/ollama/entrypoint.sh new file mode 100644 index 000000000..d60eddd36 --- /dev/null +++ b/comps/llms/text-generation/ollama/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +pip --no-cache-dir install -r requirements-runtime.txt + +python llm.py diff --git a/comps/llms/text-generation/ollama/requirements-runtime.txt b/comps/llms/text-generation/ollama/requirements-runtime.txt new file mode 100644 index 000000000..225adde27 --- /dev/null +++ b/comps/llms/text-generation/ollama/requirements-runtime.txt @@ -0,0 +1 @@ +langserve diff --git a/comps/llms/text-generation/ollama/requirements.txt b/comps/llms/text-generation/ollama/requirements.txt index f770cf5d7..92c652351 100644 --- a/comps/llms/text-generation/ollama/requirements.txt +++ b/comps/llms/text-generation/ollama/requirements.txt @@ -2,7 +2,6 @@ docarray[full] fastapi huggingface_hub langchain==0.1.16 -langserve langsmith opentelemetry-api opentelemetry-exporter-otlp diff --git a/comps/llms/text-generation/ray_serve/docker/Dockerfile.microservice b/comps/llms/text-generation/ray_serve/docker/Dockerfile.microservice index d01acd136..39a50ac8a 100644 --- a/comps/llms/text-generation/ray_serve/docker/Dockerfile.microservice +++ b/comps/llms/text-generation/ray_serve/docker/Dockerfile.microservice @@ -34,4 +34,4 @@ ENV PYTHONPATH=$PYTHONPATH:/home/user WORKDIR /home/user/comps/llms/text-generation/ray_serve -ENTRYPOINT ["python", "llm.py"] \ No newline at end of file +ENTRYPOINT ["bash", "entrypoint.sh"] diff --git a/comps/llms/text-generation/ray_serve/entrypoint.sh b/comps/llms/text-generation/ray_serve/entrypoint.sh new file mode 100644 index 000000000..d60eddd36 --- /dev/null +++ b/comps/llms/text-generation/ray_serve/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +pip --no-cache-dir install -r requirements-runtime.txt + +python llm.py diff --git a/comps/llms/text-generation/ray_serve/requirements-runtime.txt b/comps/llms/text-generation/ray_serve/requirements-runtime.txt new file mode 100644 index 000000000..225adde27 --- /dev/null +++ b/comps/llms/text-generation/ray_serve/requirements-runtime.txt @@ -0,0 +1 @@ +langserve diff --git a/comps/llms/text-generation/ray_serve/requirements.txt b/comps/llms/text-generation/ray_serve/requirements.txt index 1378ad140..2f8b2ff4e 100644 --- a/comps/llms/text-generation/ray_serve/requirements.txt +++ b/comps/llms/text-generation/ray_serve/requirements.txt @@ -3,7 +3,6 @@ fastapi huggingface_hub langchain==0.1.16 langchain_openai -langserve langsmith openai opentelemetry-api diff --git a/comps/ragas/tgi/Dockerfile b/comps/ragas/tgi/Dockerfile index f38bc7849..55d4229a0 100644 --- a/comps/ragas/tgi/Dockerfile +++ b/comps/ragas/tgi/Dockerfile @@ -1,26 +1,26 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -FROM langchain/langchain:latest - -RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ - libgl1-mesa-glx \ - libjemalloc-dev \ - vim - -RUN useradd -m -s /bin/bash user && \ - mkdir -p /home/user && \ - chown -R user /home/user/ - -USER user - -COPY comps /home/user/comps - -RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir -r /home/user/comps/ragas/tgi/requirements.txt - -ENV PYTHONPATH=$PYTHONPATH:/home/user - -WORKDIR /home/user/comps/ragas/tgi/ - -ENTRYPOINT ["python", "llm.py"] +FROM langchain/langchain:latest + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +USER user + +COPY comps /home/user/comps + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r /home/user/comps/ragas/tgi/requirements.txt + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +WORKDIR /home/user/comps/ragas/tgi/ + +ENTRYPOINT ["bash", "entrypoint.sh"] diff --git a/comps/ragas/tgi/entrypoint.sh b/comps/ragas/tgi/entrypoint.sh new file mode 100644 index 000000000..d60eddd36 --- /dev/null +++ b/comps/ragas/tgi/entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +pip --no-cache-dir install -r requirements-runtime.txt + +python llm.py diff --git a/comps/ragas/tgi/requirements-runtime.txt b/comps/ragas/tgi/requirements-runtime.txt new file mode 100644 index 000000000..225adde27 --- /dev/null +++ b/comps/ragas/tgi/requirements-runtime.txt @@ -0,0 +1 @@ +langserve diff --git a/comps/ragas/tgi/requirements.txt b/comps/ragas/tgi/requirements.txt index 2805cad49..29cad9670 100644 --- a/comps/ragas/tgi/requirements.txt +++ b/comps/ragas/tgi/requirements.txt @@ -3,7 +3,6 @@ docarray[full] fastapi huggingface_hub langchain==0.1.16 -langserve langsmith opentelemetry-api opentelemetry-exporter-otlp