Merge branch 'dev' into feat/COG-946-abstract-eval-dataset

topoteretes · Jan 9, 2025 · ddf1bb8 · ddf1bb8
2 parents a67512d + 3fb9b05
commit ddf1bb8
Show file tree

Hide file tree

Showing 349 changed files with 6,450 additions and 4,568 deletions.
diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml
@@ -17,7 +17,7 @@ jobs:
 
   publish_docker_to_ecr:
     name: Publish Cognee Docker image
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       id-token: write
       contents: read

diff --git a/.github/workflows/cd_prd.yaml b/.github/workflows/cd_prd.yaml
@@ -17,7 +17,7 @@ jobs:
 
   publish_docker_to_ecr:
     name: Publish Docker PromethAI image
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     permissions:
       id-token: write
       contents: read

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -9,7 +9,7 @@ jobs:
 
   build_docker:
     name: Build Cognee Backend Docker App Image
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out Cognee code
         uses: actions/checkout@v3

diff --git a/.github/workflows/community_greetings.yml b/.github/workflows/community_greetings.yml
@@ -4,7 +4,7 @@ on: [pull_request, issues]
 
 jobs:
   greeting:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - uses: actions/first-interaction@v1
       with:

diff --git a/.github/workflows/docker_compose.yml b/.github/workflows/docker_compose.yml
@@ -12,7 +12,7 @@ on:
 
 jobs:
   docker-compose-test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Checkout repository

diff --git a/.github/workflows/dockerhub.yml b/.github/workflows/dockerhub.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   docker-build-and-push:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Checkout repository
@@ -32,7 +32,7 @@ jobs:
       run: |
         IMAGE_NAME=cognee/cognee
         TAG_VERSION="${BRANCH_NAME}-${COMMIT_SHA}"
-        
+
         echo "Building image: ${IMAGE_NAME}:${TAG_VERSION}"
         docker buildx build \
           --platform linux/amd64,linux/arm64 \

diff --git a/.github/workflows/profiling.yaml b/.github/workflows/profiling.yaml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   profiler:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     # Checkout the code from the repository with full history
@@ -47,7 +47,7 @@ jobs:
         python-version: '3.10'
 
     - name: Install Poetry
-      uses: snok/install-poetry@v1.3.2
+      uses: snok/install-poetry@v1.4.1
       with:
         virtualenvs-create: true
         virtualenvs-in-project: true
@@ -57,6 +57,8 @@ jobs:
       run: |
         poetry install --no-interaction --all-extras
         poetry run pip install pyinstrument
+        poetry run pip install parso
+        poetry run pip install jedi
 
 
     # Set environment variables for SHAs
@@ -80,50 +82,50 @@ jobs:
         poetry run pyinstrument --renderer json -o base_results.json cognee/api/v1/cognify/code_graph_pipeline.py
 
     # Run profiler on head branch
-    - name: Run profiler on head branch
-      env:
-        HEAD_SHA: ${{ env.HEAD_SHA }}
-      run: |
-        echo "Profiling the head branch for code_graph_pipeline.py"
-        echo "Checking out head SHA: $HEAD_SHA"
-        git checkout $HEAD_SHA
-        echo "This is the working directory: $PWD"
-        # Ensure the script is executable
-        chmod +x cognee/api/v1/cognify/code_graph_pipeline.py
-        # Run Scalene
-        poetry run pyinstrument --renderer json -o head_results.json cognee/api/v1/cognify/code_graph_pipeline.py
-
-    # Compare profiling results
-    - name: Compare profiling results
-      run: |
-        python -c '
-        import json
-        try:
-            with open("base_results.json") as f:
-                base = json.load(f)
-            with open("head_results.json") as f:
-                head = json.load(f)
-            cpu_diff = head.get("total_cpu_samples_python", 0) - base.get("total_cpu_samples_python", 0)
-            memory_diff = head.get("malloc_samples", 0) - base.get("malloc_samples", 0)
-            results = [
-                f"CPU Usage Difference: {cpu_diff}",
-                f"Memory Usage Difference: {memory_diff} bytes"
-            ]
-            with open("profiling_diff.txt", "w") as f:
-                f.write("\\n".join(results) + "\\n")
-            print("\\n".join(results))  # Print results to terminal
-        except Exception as e:
-            error_message = f"Error comparing profiling results: {e}"
-            with open("profiling_diff.txt", "w") as f:
-                f.write(error_message + "\\n")
-            print(error_message)  # Print error to terminal
-            '
-
-    - name: Upload profiling diff artifact
-      uses: actions/upload-artifact@v3
-      with:
-        name: profiling-diff
-        path: profiling_diff.txt
+#    - name: Run profiler on head branch
+#      env:
+#        HEAD_SHA: ${{ env.HEAD_SHA }}
+#      run: |
+#        echo "Profiling the head branch for code_graph_pipeline.py"
+#        echo "Checking out head SHA: $HEAD_SHA"
+#        git checkout $HEAD_SHA
+#        echo "This is the working directory: $PWD"
+#        # Ensure the script is executable
+#        chmod +x cognee/api/v1/cognify/code_graph_pipeline.py
+#        # Run Scalene
+#        poetry run pyinstrument --renderer json -o head_results.json cognee/api/v1/cognify/code_graph_pipeline.py
+#
+#    # Compare profiling results
+#    - name: Compare profiling results
+#      run: |
+#        python -c '
+#        import json
+#        try:
+#            with open("base_results.json") as f:
+#                base = json.load(f)
+#            with open("head_results.json") as f:
+#                head = json.load(f)
+#            cpu_diff = head.get("total_cpu_samples_python", 0) - base.get("total_cpu_samples_python", 0)
+#            memory_diff = head.get("malloc_samples", 0) - base.get("malloc_samples", 0)
+#            results = [
+#                f"CPU Usage Difference: {cpu_diff}",
+#                f"Memory Usage Difference: {memory_diff} bytes"
+#            ]
+#            with open("profiling_diff.txt", "w") as f:
+#                f.write("\\n".join(results) + "\\n")
+#            print("\\n".join(results))  # Print results to terminal
+#        except Exception as e:
+#            error_message = f"Error comparing profiling results: {e}"
+#            with open("profiling_diff.txt", "w") as f:
+#                f.write(error_message + "\\n")
+#            print(error_message)  # Print error to terminal
+#            '
+#
+#    - name: Upload profiling diff artifact
+#      uses: actions/upload-artifact@v3
+#      with:
+#        name: profiling-diff
+#        path: profiling_diff.txt
 
     # Post results to the pull request
 #    - name: Post profiling results to PR

diff --git a/.github/workflows/py_lint.yml b/.github/workflows/py_lint.yml
@@ -16,8 +16,8 @@ jobs:
       fail-fast: true
       matrix:
         os:
-          - ubuntu-latest
-        python-version: ["3.8.x", "3.9.x", "3.10.x", "3.11.x"]
+          - ubuntu-22.04
+        python-version: ["3.10.x", "3.11.x"]
 
     defaults:
       run:

diff --git a/.github/workflows/release_discord_action.yml b/.github/workflows/release_discord_action.yml
@@ -6,7 +6,7 @@ on:
 
 jobs:
   github-releases-to-discord:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/reusable_notebook.yml b/.github/workflows/reusable_notebook.yml
@@ -22,7 +22,7 @@ jobs:
 
   run_notebook_test:
     name: test
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
@@ -36,7 +36,7 @@ jobs:
           python-version: '3.11.x'
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
@@ -58,4 +58,4 @@ jobs:
           --to notebook \
           --execute ${{ inputs.notebook-location }} \
           --output executed_notebook.ipynb \
-          --ExecutePreprocessor.timeout=1200
+          --ExecutePreprocessor.timeout=1200
diff --git a/.github/workflows/reusable_python_example.yml b/.github/workflows/reusable_python_example.yml
@@ -22,7 +22,7 @@ jobs:
 
   run_notebook_test:
     name: test
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
@@ -33,10 +33,10 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11.x'
+          python-version: '3.12.x'
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true
@@ -49,7 +49,8 @@ jobs:
       - name: Execute Python Example
         env:
           ENV: 'dev'
+          PYTHONFAULTHANDLER: 1
           LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           GRAPHISTRY_USERNAME: ${{ secrets.GRAPHISTRY_USERNAME }}
           GRAPHISTRY_PASSWORD: ${{ secrets.GRAPHISTRY_PASSWORD }}
-        run: poetry run python ${{ inputs.example-location }}
+        run: poetry run python ${{ inputs.example-location }}
diff --git a/.github/workflows/ruff_format.yaml b/.github/workflows/ruff_format.yaml
@@ -3,7 +3,7 @@ on: [ pull_request ]
 
 jobs:
   ruff:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/ruff-action@v2

diff --git a/.github/workflows/ruff_lint.yaml b/.github/workflows/ruff_lint.yaml
@@ -3,7 +3,7 @@ on: [ pull_request ]
 
 jobs:
   ruff:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: astral-sh/ruff-action@v2
diff --git a/.github/workflows/test_deduplication.yml b/.github/workflows/test_deduplication.yml
@@ -16,8 +16,7 @@ env:
 jobs:
   run_deduplication_test:
     name: test
-    runs-on: ubuntu-latest
-    if:  ${{ github.event.label.name == 'run-checks' }}
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
@@ -46,7 +45,7 @@ jobs:
           python-version: '3.11.x'
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true

diff --git a/.github/workflows/test_milvus.yml b/.github/workflows/test_milvus.yml
@@ -17,8 +17,7 @@ jobs:
 
   run_milvus:
     name: test
-    runs-on: ubuntu-latest
-    if:  ${{ github.event.label.name == 'run-checks' }}
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
     defaults:
@@ -36,7 +35,7 @@ jobs:
 
       - name: Install Poetry
         # https://github.com/snok/install-poetry#running-on-windows
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true

diff --git a/.github/workflows/test_neo4j.yml b/.github/workflows/test_neo4j.yml
@@ -15,8 +15,7 @@ env:
 jobs:
   run_neo4j_integration_test:
     name: test
-    if:  ${{ github.event.label.name == 'run-checks' }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     defaults:
       run:
@@ -32,7 +31,7 @@ jobs:
           python-version: '3.11.x'
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true

diff --git a/.github/workflows/test_pgvector.yml b/.github/workflows/test_pgvector.yml
@@ -17,8 +17,7 @@ jobs:
 
   run_pgvector_integration_test:
     name: test
-    runs-on: ubuntu-latest
-    if:  ${{ github.event.label.name == 'run-checks' }}
+    runs-on: ubuntu-22.04
     defaults:
       run:
         shell: bash
@@ -47,7 +46,7 @@ jobs:
           python-version: '3.11.x'
 
       - name: Install Poetry
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true

diff --git a/.github/workflows/test_python_3_10.yml b/.github/workflows/test_python_3_10.yml
@@ -14,11 +14,9 @@ env:
   ENV: 'dev'
 
 jobs:
-
   run_common:
     name: test
-    runs-on: ubuntu-latest
-    if:  ${{ github.event.label.name == 'run-checks' }}
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
     defaults:
@@ -36,7 +34,7 @@ jobs:
 
       - name: Install Poetry
         # https://github.com/snok/install-poetry#running-on-windows
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true

diff --git a/.github/workflows/test_python_3_11.yml b/.github/workflows/test_python_3_11.yml
@@ -17,8 +17,7 @@ jobs:
 
   run_common:
     name: test
-    runs-on: ubuntu-latest
-    if:  ${{ github.event.label.name == 'run-checks' }}
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
     defaults:
@@ -36,7 +35,7 @@ jobs:
 
       - name: Install Poetry
         # https://github.com/snok/install-poetry#running-on-windows
-        uses: snok/install-poetry@v1.3.2
+        uses: snok/install-poetry@v1.4.1
         with:
           virtualenvs-create: true
           virtualenvs-in-project: true