Migrate AutoGPT agent to poetry

Co-authored-by: rickythefox <[email protected]>
Significant-Gravitas · Sep 14, 2023 · 859c5f6 · 859c5f6
1 parent 0ee7209
commit 859c5f6
Show file tree

Hide file tree

Showing 10 changed files with 121 additions and 115 deletions.
diff --git a/.github/workflows/autogpt-ci.yml b/.github/workflows/autogpt-ci.yml
@@ -57,13 +57,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
-      - name: Install dependencies
+      - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Lint with flake8
         run: flake8
@@ -156,13 +156,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Run pytest with coverage
         run: |

diff --git a/.github/workflows/benchmark-ci.yml b/.github/workflows/benchmark-ci.yml
@@ -162,8 +162,8 @@ jobs:
           elif [ "$AGENT_NAME" == "Auto-GPT" ]; then
             python -m venv venv
             source venv/bin/activate
-            pip install -r requirements.txt
-            pip uninstall agbenchmark -y
+            curl -sSL https://install.python-poetry.org | python3 -
+            poetry install --without benchmark
           elif [ "$AGENT_NAME" == "mini-agi" ]; then
             python -m venv venv
             source venv/bin/activate

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -39,13 +39,13 @@ jobs:
       - name: Set up Python dependency cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}-${{ steps.get_date.outputs.date }}
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-${{ hashFiles('autogpts/autogpt/pyproject.toml') }}-${{ steps.get_date.outputs.date }}
 
       - name: Install Python dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry install
 
       - name: Run pytest with coverage
         run: |

diff --git a/autogpts/autogpt/.devcontainer/devcontainer.json b/autogpts/autogpt/.devcontainer/devcontainer.json
@@ -46,11 +46,11 @@
   // "forwardPorts": [],
 
   // Use 'postCreateCommand' to run commands after the container is created.
-  // "postCreateCommand": "pip3 install --user -r requirements.txt",
+  // "postCreateCommand": "poetry install",
 
   // Set `remoteUser` to `root` to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
   "remoteUser": "vscode",
 
   // Add the freshly containerized repo to the list of safe repositories
-  "postCreateCommand": "git config --global --add safe.directory /workspace/Auto-GPT && pip3 install --user -r requirements.txt"
-}
+  "postCreateCommand": "git config --global --add safe.directory /workspace/Auto-GPT && poetry install"
+}
diff --git a/autogpts/autogpt/Dockerfile b/autogpts/autogpt/Dockerfile
@@ -17,25 +17,28 @@ RUN apt-get update && apt-get install -y \
 # Set environment variables
 ENV PIP_NO_CACHE_DIR=yes \
     PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1
+    PYTHONDONTWRITEBYTECODE=1 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_CREATE=0 \
+    POETRY_NO_INTERACTION=1
 
-# Install the required python packages globally
-ENV PATH="$PATH:/root/.local/bin"
-COPY requirements.txt .
+# Install and configure Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="$POETRY_HOME/bin:$PATH"
+RUN poetry config installer.max-workers 10
 
 # Set the entrypoint
-ENTRYPOINT ["python", "-m", "autogpt", "--install-plugin-deps"]
+CMD ["poetry", "run", "--install-plugin-deps"]
 
 # dev build -> include everything
 FROM autogpt-base as autogpt-dev
-RUN pip install --no-cache-dir -r requirements.txt
+RUN poetry install
 WORKDIR /app
 ONBUILD COPY . ./
 
 # release build -> include bare minimum
 FROM autogpt-base as autogpt-release
-RUN sed -i '/Items below this point will not be included in the Docker Image/,$d' requirements.txt && \
-	pip install --no-cache-dir -r requirements.txt
+RUN poetry install --without dev,benchmark
 WORKDIR /app
 ONBUILD COPY autogpt/ ./autogpt
 ONBUILD COPY scripts/ ./scripts

diff --git a/autogpts/autogpt/pyproject.toml b/autogpts/autogpt/pyproject.toml
@@ -1,25 +1,98 @@
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[project]
+[tool.poetry]
 name = "agpt"
 version = "0.4.7"
 authors = [
-  { name="Torantulino", email="[email protected]" },
+  "Significant Gravitas <[email protected]>",
 ]
 readme = "README.md"
-requires-python = ">=3.10"
+description = "An open-source attempt to make GPT-4 autonomous"
+homepage = "https://github.com/Significant-Gravitas/Auto-GPT/tree/master/autogpts/autogpt"
 classifiers = [
     "Programming Language :: Python :: 3",
     "License :: OSI Approved :: MIT License",
     "Operating System :: OS Independent",
 ]
-description = "An open-source attempt to make GPT-4 autonomous"
 
-[project.urls]
-"Homepage" = "https://github.com/Significant-Gravitas/Auto-GPT"
-"Bug Tracker" = "https://github.com/Significant-Gravitas/Auto-GPT"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+beautifulsoup4 = "^4.12.2"
+colorama = "^0.4.6"
+distro = "^1.8.0"
+openai = "^0.27.8"
+playsound = "^1.2.2"
+python-dotenv = "^1.0.0"
+pyyaml = "^6.0"
+PyPDF2 = "*"
+python-docx = "*"
+markdown = "*"
+pylatexenc = "*"
+readability-lxml = "^0.8.1"
+requests = "*"
+tiktoken = "^0.3.3"
+gTTS = "^2.3.1"
+docker = "*"
+duckduckgo-search = "^3.0.2"
+google-api-python-client = "*"
+pinecone-client = "^2.2.1"
+redis = "*"
+orjson = "^3.8.10"
+ftfy = "^6.1.1"
+Pillow = "*"
+selenium = "^4.11.2"
+webdriver-manager = "*"
+jsonschema = "*"
+click = "*"
+charset-normalizer = "^3.1.0"
+spacy = "^3.0.0"
+en-core-web-sm = {url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.5.0/en_core_web_sm-3.5.0-py3-none-any.whl"}
+prompt_toolkit = "^3.0.38"
+pydantic = "*"
+inflection = "*"
+agent-protocol = "^0.1.1"
+
+# web server
+fastapi = "*"
+uvicorn = "*"
+
+# OpenAI and Generic plugins import
+openapi-python-client = "^0.13.4"
+
+[tool.poetry.group.dev.dependencies]
+coverage = "*"
+flake8 = "*"
+numpy = "*"
+pre-commit = "*"
+black = "*"
+isort = "*"
+gitpython = "^3.1.32"
+auto-gpt-plugin-template = {git = "https://github.com/Significant-Gravitas/Auto-GPT-Plugin-Template", rev = "0.1.0"}
+mypy = "*"
+types-Markdown = "*"
+types-beautifulsoup4 = "*"
+types-colorama = "*"
+types-Pillow = "*"
+
+# Testing dependencies
+pytest = "*"
+asynctest = "*"
+pytest-asyncio = "*"
+pytest-benchmark = "*"
+pytest-cov = "*"
+pytest-integration = "*"
+pytest-mock = "*"
+vcrpy = {git = "https://github.com/Significant-Gravitas/vcrpy.git", rev = "master"}
+pytest-recording = "*"
+pytest-xdist = "*"
+
+[tool.poetry.group.benchmark.dependencies]
+agbenchmark = "0.0.9"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
 
 [tool.black]
 line-length = 88
@@ -56,6 +129,7 @@ skip = '''
 
 '''
 
+
 [tool.pytest.ini_options]
 markers = [
     "requires_openai_api_key",

diff --git a/autogpts/autogpt/requirements.txt b/autogpts/autogpt/requirements.txt
diff --git a/autogpts/autogpt/run.bat b/autogpts/autogpt/run.bat
@@ -15,10 +15,10 @@ pause
 exit /B 1
 
 :Found
-%PYTHON_CMD% scripts/check_requirements.py requirements.txt
+%PYTHON_CMD% scripts/check_requirements.py pyproject.toml
 if errorlevel 1 (
     echo Installing missing packages...
-    %PYTHON_CMD% -m pip install -r requirements.txt
+    %PYTHON_CMD% -m poetry install --without dev
 )
 %PYTHON_CMD% -m autogpt %*
-pause
+pause
diff --git a/autogpts/autogpt/run.sh b/autogpts/autogpt/run.sh
@@ -16,11 +16,11 @@ function find_python_command() {
 PYTHON_CMD=$(find_python_command)
 
 if $PYTHON_CMD -c "import sys; sys.exit(sys.version_info < (3, 10))"; then
-    $PYTHON_CMD scripts/check_requirements.py requirements.txt
+    $PYTHON_CMD scripts/check_requirements.py pyproject.toml
     if [ $? -eq 1 ]
     then
         echo Installing missing packages...
-        $PYTHON_CMD -m pip install -r requirements.txt
+        $PYTHON_CMD -m poetry install --without dev
     fi
     $PYTHON_CMD -m autogpt "$@"
     read -p "Press any key to continue..."

diff --git a/docs/content/configuration/memory.md b/docs/content/configuration/memory.md
@@ -153,14 +153,14 @@ data objects. To set up a Weaviate database, check out their [Quickstart Tutoria
 
 Although still experimental, [Embedded Weaviate](https://weaviate.io/developers/weaviate/installation/embedded)
 is supported which allows the Auto-GPT process itself to start a Weaviate instance.
-To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `pip install "weaviate-client>=3.15.4"`.
+To enable it, set `USE_WEAVIATE_EMBEDDED` to `True` and make sure you `poetry add weaviate-client@^3.15.4`.
 
 #### Install the Weaviate client
 
 Install the Weaviate client before usage.
 
 ```shell
-$ pip install weaviate-client
+$ poetry add weaviate-client
 ```
 
 #### Setting up environment variables