Merge branch 'main' into topology_fixes

.dockerignore

@@ -0,0 +1,16 @@
+bin
+dist
+docs
+evals
+tests
+tools
+assets
+notebooks
+cognee-frontend
+
+.dlt
+.venv
+.github
+
+.idea
+.vscode

.env.template

@@ -15,4 +15,6 @@ AWS_ACCESS_KEY_ID =
 AWS_SECRET_ACCESS_KEY =
 QDRANT_API_KEY =
 QDRANT_API_URL =
-DB_TYPE = sqlite
+DB_TYPE = sqlite
+GRAPHISTRY_USERNAME = xxx
+GRAPHISTRY_PASSWORD = xxx

.github/workflows/cd.yaml

@@ -1,4 +1,4 @@
-name: publish dev |  Docker image
+name: publish dev | Docker image
 
 on:
   push:

.github/workflows/cd_prd.yaml

@@ -1,4 +1,4 @@
-name: publish prd |  Docker image
+name: publish prd | Docker image
 
 on:
   push:

.github/workflows/ci.yaml

@@ -1,4 +1,4 @@
-name: build test |  Docker image
+name: build test | Docker image
 
 on: pull_request
 

.github/workflows/docker_compose.yml

@@ -0,0 +1,44 @@
+name: test | docker compose
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  docker-compose-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Build Docker images
+      run: |
+        docker-compose -f docker-compose.yml build
+
+    - name: Run Docker Compose
+      run: |
+        docker-compose -f docker-compose.yml up -d
+
+    - name: Wait for services to be ready
+      run: |
+        # Add any necessary health checks or wait commands
+        sleep 30
+
+    # - name: Run tests
+    #   run: |
+    #     docker-compose -f docker-compose.yml run --rm <test-service> <test-command>
+    #   # Replace <test-service> with the name of the service running the tests
+    #   # Replace <test-command> with the actual command to run your tests
+
+    - name: Shut down Docker Compose
+      if: always()
+      run: |
+        docker-compose -f docker-compose.yml down

.github/workflows/test_neo4j.yml

@@ -1,4 +1,4 @@
-name: test neo4j
+name: test | neo4j
 
 on:
   pull_request:
@@ -18,7 +18,7 @@ jobs:
     name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
-  run_common:
+  run_neo4j_integration_test:
     name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'

Dockerfile

@@ -1,52 +1,32 @@
-FROM python:3.11
+FROM python:3.11-slim
 
 # Set build argument
 ARG DEBUG
-ARG API_ENABLED
 
 # Set environment variable based on the build argument
 ENV DEBUG=${DEBUG}
-ENV API_ENABLED=${API_ENABLED}
 ENV PIP_NO_CACHE_DIR=true
 ENV PATH="${PATH}:/root/.poetry/bin"
 
-RUN pip install poetry
-
 WORKDIR /app
 COPY pyproject.toml poetry.lock /app/
 
-# Install the dependencies
-RUN poetry config virtualenvs.create false && \
-    poetry install --no-root --no-dev
-
-RUN apt-get update -q && \
-    apt-get install -y -q \
-        gcc \
-        python3-dev \
-        curl \
-        zip \
-        jq \
-#        libgl1-mesa-glx \
-        netcat-traditional && \
-    pip install poetry && \
-    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
-    unzip -qq awscliv2.zip && \
-    ./aws/install && \
-    apt-get clean && \
-    rm -rf \
-        awscliv2.zip \
-        /var/lib/apt/lists/* \
-        /tmp/* \
-        /var/tmp/*
+RUN pip install poetry
 
-WORKDIR /app
+# Create virtualenv
+RUN poetry config virtualenvs.create false
+
+# Install the dependencies
+RUN poetry install --no-root --no-dev
+
 # Set the PYTHONPATH environment variable to include the /app directory
 ENV PYTHONPATH=/app
 
 COPY cognee/ /app/cognee
 
-
 COPY entrypoint.sh /app/entrypoint.sh
 RUN chmod +x /app/entrypoint.sh
 
+RUN sed -i 's/\r$//' /app/entrypoint.sh
+
 ENTRYPOINT ["/app/entrypoint.sh"]

Dockerfile-old

@@ -0,0 +1,49 @@
+FROM python:3.11
+
+# Set build argument
+ARG DEBUG
+
+# Set environment variable based on the build argument
+ENV DEBUG=${DEBUG}
+ENV PIP_NO_CACHE_DIR=true
+ENV PATH="${PATH}:/root/.poetry/bin"
+
+RUN pip install poetry
+
+WORKDIR /app
+COPY pyproject.toml poetry.lock /app/
+
+# Install the dependencies
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root --no-dev
+
+RUN apt-get update -q && \
+    apt-get install -y -q \
+        gcc \
+        python3-dev \
+        curl \
+        zip \
+        jq \
+        netcat-traditional && \
+    pip install poetry && \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip -qq awscliv2.zip && \
+    ./aws/install && \
+    apt-get clean && \
+    rm -rf \
+        awscliv2.zip \
+        /var/lib/apt/lists/* \
+        /tmp/* \
+        /var/tmp/*
+
+WORKDIR /app
+# Set the PYTHONPATH environment variable to include the /app directory
+ENV PYTHONPATH=/app
+
+COPY cognee/ /app/cognee
+
+
+COPY entrypoint.sh /app/entrypoint.sh
+RUN chmod +x /app/entrypoint.sh
+
+ENTRYPOINT ["/app/entrypoint.sh"]

README.md

@@ -74,7 +74,7 @@ or
 import cognee
 cognee.config.llm_api_key = "YOUR_OPENAI_API_KEY"
 ```
-If you are using Networkx, create an account on Graphistry to vizualize results:
+If you are using Networkx, create an account on Graphistry to visualize results:
 ```
    
    cognee.config.set_graphistry_username = "YOUR_USERNAME"
@@ -97,11 +97,11 @@ import cognee
 text = """Natural language processing (NLP) is an interdisciplinary
        subfield of computer science and information retrieval"""
 
-cognee.add([text], "example_dataset") # Add a new piece of information
+await cognee.add([text], "example_dataset") # Add a new piece of information
 
-cognee.cognify() # Use LLMs and cognee to create knowledge
+await cognee.cognify() # Use LLMs and cognee to create knowledge
 
-search_results = cognee.search("SIMILARITY", {'query': 'Tell me about NLP'}) # Query cognee for the knowledge
+await search_results = cognee.search("SIMILARITY", {'query': 'Tell me about NLP'}) # Query cognee for the knowledge
 
 print(search_results)
 

cognee-frontend/.dockerignore

@@ -0,0 +1,2 @@
+.next
+node_modules

cognee-frontend/Dockerfile

@@ -1,20 +1,20 @@
 # Use an official Node.js runtime as a parent image
-FROM node:18.17.0
+FROM node:18-alpine
 
 # Set the working directory to /app
 WORKDIR /app
 
 # Copy package.json and package-lock.json to the working directory
-COPY package*.json ./
+COPY package.json package-lock.json ./
 
 # Install any needed packages specified in package.json
-RUN npm install
+RUN npm ci
 
 # Copy the rest of the application code to the working directory
-COPY . .
-
-# Make port 3000 available to the world outside this container
-EXPOSE 3000
+COPY src ./src
+COPY public ./public
+COPY next.config.mjs .
+COPY tsconfig.json .
 
 # Build the app and run it
-CMD ["sh", "-c", "npm run dev"]
+CMD npm run dev

cognee/modules/data/extraction/knowledge_graph/extract_knowledge_graph.py

@@ -1,24 +1,4 @@
-import asyncio
-import logging
-from cognee.root_dir import get_absolute_path
-from .extract_knowledge_graph_module import ExtractKnowledgeGraph
 from .extract_content_graph import extract_content_graph
 
-logger = logging.getLogger("extract_knowledge_graph(text: str)")
-
 async def extract_knowledge_graph(text: str, cognitive_layer, graph_model):
-    try:
-        compiled_extract_knowledge_graph = ExtractKnowledgeGraph()
-        compiled_extract_knowledge_graph.load(get_absolute_path("./programs/extract_knowledge_graph/extract_knowledge_graph.json"))
-
-        event_loop = asyncio.get_event_loop()
-
-        def sync_extract_knowledge_graph():
-            return compiled_extract_knowledge_graph(context = text, question = "")
-
-        return (await event_loop.run_in_executor(None, sync_extract_knowledge_graph)).graph
-        # return compiled_extract_knowledge_graph(text, question = "").graph
-    except Exception as error:
-        # TODO: Log error to Sentry
-
-        return await extract_content_graph(text, cognitive_layer, graph_model)
+    return await extract_content_graph(text, cognitive_layer, graph_model)

cognee/modules/search/vector/search_similarity.py

@@ -1,8 +1,7 @@
 from cognee.infrastructure.databases.graph.get_graph_client import get_graph_client
-from cognee.infrastructure.databases.graph.config import get_graph_config
 from cognee.infrastructure.databases.vector import get_vector_engine
 
-async def search_similarity(query: str, graph):
+async def search_similarity(query: str, graph): # graph must be there in order to be compatible with generic call
     # graph_config = get_graph_config()
     #
     # graph_db_type = graph_config.graph_engine

cognee/tests/test_qdrant.py

@@ -73,4 +73,4 @@ async def main():
 
 if __name__ == "__main__":
     import asyncio
-    asyncio.run(main())
+    asyncio.run(main())

docker-compose-vanilla.yml

@@ -0,0 +1,41 @@
+services:
+  cognee:
+    container_name: cognee
+    networks:
+      - cognee-network
+    build:
+      context: .
+      dockerfile: Dockerfile
+    volumes:
+      - .:/app
+      - /app/cognee-frontend/ # Ignore frontend code
+    environment:
+      - HOST=0.0.0.0
+      - ENVIRONMENT=local
+      - PYTHONPATH=.
+    ports:
+      - 8000:8000
+      # - 5678:5678 # Debugging
+    deploy:
+      resources:
+        limits:
+          cpus: 4.0
+          memory: 8GB
+
+  frontend:
+    container_name: frontend
+    build:
+      context: ./cognee-frontend
+      dockerfile: Dockerfile
+    volumes:
+      - ./cognee-frontend/src:/app/src
+      - ./cognee-frontend/public:/app/public
+    ports:
+      - 3000:3000
+      # - 9229:9229 # Debugging
+    networks:
+      - cognee-network
+
+networks:
+  cognee-network:
+    name: cognee-network