Merge pull request #123 from topoteretes/COG-206

added boilerplate for user management
topoteretes · Aug 6, 2024 · 80e4f29 · 80e4f29
2 parents 26bca01 + e3c3f35
commit 80e4f29
Show file tree

Hide file tree

Showing 115 changed files with 2,840 additions and 1,428 deletions.
diff --git a/.github/workflows/docker_compose.yml b/.github/workflows/docker_compose.yml
@@ -17,28 +17,22 @@ jobs:
       uses: actions/checkout@v2
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
 
     - name: Build Docker images
       run: |
-        docker-compose -f docker-compose.yml build
+        docker compose -f docker-compose.yml build
 
     - name: Run Docker Compose
       run: |
-        docker-compose -f docker-compose.yml up -d
+        docker compose -f docker-compose.yml up -d
 
     - name: Wait for services to be ready
       run: |
         # Add any necessary health checks or wait commands
         sleep 30
 
-    # - name: Run tests
-    #   run: |
-    #     docker-compose -f docker-compose.yml run --rm <test-service> <test-command>
-    #   # Replace <test-service> with the name of the service running the tests
-    #   # Replace <test-command> with the actual command to run your tests
-
     - name: Shut down Docker Compose
       if: always()
       run: |
-        docker-compose -f docker-compose.yml down
+        docker compose -f docker-compose.yml down
diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml
@@ -18,14 +18,38 @@ jobs:
     name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
+  setup_docker:
+    name: Set up Docker Buildx
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # os: ["ubuntu-latest", "macos-latest"]
+        os: ["ubuntu-latest"]
+    steps:
+      # - name: Install Docker
+      #   run: |
+      #     HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask docker
+      #     sudo /Applications/Docker.app/Contents/MacOS/Docker --unattended --install-privileged-components
+      #     open -a /Applications/Docker.app --args --unattended --accept-license
+      #     echo "We are waiting for Docker to be up and running. It can take over 2 minutes..."
+      #     while ! /Applications/Docker.app/Contents/Resources/bin/docker info &>/dev/null; do sleep 1; done
+      #   if: runner.os == 'macOS'
+
+      # - name: Set up QEMU
+      #   uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
   run_common:
     name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-latest", "macos-latest"] #, "windows-latest"
+        os: ["ubuntu-latest"] #, "windows-latest", "macos-latest"
         python-version: ["3.11.x"]
         # Test all python versions on ubuntu only
         include:
@@ -40,6 +64,18 @@ jobs:
       run:
         shell: bash
 
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+        volumes:
+          - postgres_data:/var/lib/postgresql/data
+        ports:
+          - 5432:5432
+
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -79,6 +115,12 @@ jobs:
           mkdir .cognee_system
           echo $(pwd)/.cognee_system
 
+      - name: Wait for PostgreSQL to be ready
+        run: |
+          echo "Waiting for PostgreSQL to be ready..."
+          until pg_isready -h localhost -p 5432 -U cognee; do
+            sleep 1
+          done
 
       - name: Run tests
         run: poetry run pytest tests/
@@ -88,6 +130,15 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ENV: 'dev'
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          DESTINATION__POSTGRES__CREDENTIALS__HOST: 127.0.0.1
+          DESTINATION__POSTGRES__CREDENTIALS__USERNAME: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__PASSWORD: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__DATABASE: cognee_db
         run: poetry run python ./cognee/tests/test_library.py
 
       - name: Clean up disk space

diff --git a/.github/workflows/test_neo4j.yml b/.github/workflows/test_neo4j.yml
@@ -18,16 +18,35 @@ jobs:
     name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
+  setup_docker:
+    name: Set up Docker Buildx
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
   run_neo4j_integration_test:
     name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
 
     defaults:
       run:
         shell: bash
 
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+        volumes:
+          - postgres_data:/var/lib/postgresql/data
+        ports:
+          - 5432:5432
+
     steps:
       - name: Check out
         uses: actions/checkout@v2
@@ -52,6 +71,13 @@ jobs:
           mkdir .cognee_system
           echo $(pwd)/.cognee_system
 
+      - name: Wait for PostgreSQL to be ready
+        run: |
+          echo "Waiting for PostgreSQL to be ready..."
+          until pg_isready -h localhost -p 5432 -U cognee; do
+            sleep 1
+          done
+
       - name: Run default Neo4j
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
@@ -60,4 +86,13 @@ jobs:
           GRAPH_DATABASE_PASSWORD: ${{ secrets.NEO4J_API_KEY }}
           GRAPH_DATABASE_USERNAME: "neo4j"
           ENV: 'dev'
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          DESTINATION__POSTGRES__CREDENTIALS__HOST: 127.0.0.1
+          DESTINATION__POSTGRES__CREDENTIALS__USERNAME: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__PASSWORD: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__DATABASE: cognee_db
         run: poetry run python ./cognee/tests/test_neo4j.py
diff --git a/.github/workflows/test_qdrant.yml b/.github/workflows/test_qdrant.yml
@@ -18,16 +18,35 @@ jobs:
     name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
+  setup_docker:
+    name: Set up Docker Buildx
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
   run_qdrant_integration_test:
     name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
 
     defaults:
       run:
         shell: bash
 
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+        volumes:
+          - postgres_data:/var/lib/postgresql/data
+        ports:
+          - 5432:5432
+
     steps:
       - name: Check out
         uses: actions/checkout@v2
@@ -52,11 +71,27 @@ jobs:
           mkdir .cognee_system
           echo $(pwd)/.cognee_system
 
+      - name: Wait for PostgreSQL to be ready
+        run: |
+          echo "Waiting for PostgreSQL to be ready..."
+          until pg_isready -h localhost -p 5432 -U cognee; do
+            sleep 1
+          done
+
       - name: Run default Qdrant
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           VECTOR_DB_URL: ${{ secrets.QDRANT_API_URL }}
           VECTOR_DB_KEY: ${{ secrets.QDRANT_API_KEY }}
           ENV: 'dev'
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          DESTINATION__POSTGRES__CREDENTIALS__HOST: 127.0.0.1
+          DESTINATION__POSTGRES__CREDENTIALS__USERNAME: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__PASSWORD: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__DATABASE: cognee_db
         run: poetry run python ./cognee/tests/test_qdrant.py
diff --git a/.github/workflows/test_weaviate.yml b/.github/workflows/test_weaviate.yml
@@ -18,16 +18,35 @@ jobs:
     name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
+  setup_docker:
+    name: Set up Docker Buildx
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
   run_weaviate_integration_test:
     name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
 
     defaults:
       run:
         shell: bash
 
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+        volumes:
+          - postgres_data:/var/lib/postgresql/data
+        ports:
+          - 5432:5432
+
     steps:
       - name: Check out
         uses: actions/checkout@v2
@@ -52,11 +71,27 @@ jobs:
           mkdir .cognee_system
           echo $(pwd)/.cognee_system
 
+      - name: Wait for PostgreSQL to be ready
+        run: |
+          echo "Waiting for PostgreSQL to be ready..."
+          until pg_isready -h localhost -p 5432 -U cognee; do
+            sleep 1
+          done
+
       - name: Run default Weaviate
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           VECTOR_DB_URL: ${{ secrets.WEAVIATE_API_URL }}
           VECTOR_DB_KEY: ${{ secrets.WEAVIATE_API_KEY }}
           ENV: 'dev'
+          POSTGRES_USER: cognee
+          POSTGRES_PASSWORD: cognee
+          POSTGRES_DB: cognee_db
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          DESTINATION__POSTGRES__CREDENTIALS__HOST: 127.0.0.1
+          DESTINATION__POSTGRES__CREDENTIALS__USERNAME: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__PASSWORD: cognee
+          DESTINATION__POSTGRES__CREDENTIALS__DATABASE: cognee_db
         run: poetry run python ./cognee/tests/test_weaviate.py
diff --git a/.pylintrc b/.pylintrc
@@ -433,8 +433,9 @@ disable=raw-checker-failed,
         use-implicit-booleaness-not-comparison-to-zero,
         missing-module-docstring,
         missing-function-docstring,
-        missing-class-docstring
-
+        missing-class-docstring,
+        relative-beyond-top-level
+
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

diff --git a/cognee-frontend/src/app/page.tsx b/cognee-frontend/src/app/page.tsx
@@ -48,15 +48,15 @@ export default function Home() {
       });
   }, [showNotification])
 
-  const onDatasetCognify = useCallback((dataset: { id: string }) => {
-    showNotification(`Cognification started for dataset "${dataset.id}".`, 5000);
+  const onDatasetCognify = useCallback((dataset: { id: string, name: string }) => {
+    showNotification(`Cognification started for dataset "${dataset.name}".`, 5000);
 
     return cognifyDataset(dataset)
       .then(() => {
-        showNotification(`Dataset "${dataset.id}" cognified.`, 5000);
+        showNotification(`Dataset "${dataset.name}" cognified.`, 5000);
       })
       .catch(() => {
-        showNotification(`Dataset "${dataset.id}" cognification failed. Please try again.`, 5000);
+        showNotification(`Dataset "${dataset.name}" cognification failed. Please try again.`, 5000);
       });
   }, [showNotification]);
 

diff --git a/cognee-frontend/src/modules/datasets/cognifyDataset.ts b/cognee-frontend/src/modules/datasets/cognifyDataset.ts
@@ -1,5 +1,5 @@
-export default function cognifyDataset(dataset: { id: string }) {
-  return fetch('http://0.0.0.0:8000/cognify', {
+export default function cognifyDataset(dataset: { id: string, name: string }) {
+  return fetch('http://127.0.0.1:8000/cognify', {
     method: 'POST',
     headers: {
       'Content-Type': 'application/json',

diff --git a/cognee-frontend/src/modules/datasets/deleteDataset.ts b/cognee-frontend/src/modules/datasets/deleteDataset.ts
@@ -1,5 +1,5 @@
 export default function deleteDataset(dataset: { id: string }) {
-  return fetch(`http://0.0.0.0:8000/datasets/${dataset.id}`, {
+  return fetch(`http://127.0.0.1:8000/datasets/${dataset.id}`, {
     method: 'DELETE',
   })
 }
diff --git a/cognee-frontend/src/modules/datasets/getDatasetData.ts b/cognee-frontend/src/modules/datasets/getDatasetData.ts
@@ -1,4 +1,4 @@
 export default function getDatasetData(dataset: { id: string }) {
-  return fetch(`http://0.0.0.0:8000/datasets/${dataset.id}/data`)
+  return fetch(`http://127.0.0.1:8000/datasets/${dataset.id}/data`)
       .then((response) => response.json());
 }
diff --git a/cognee-frontend/src/modules/exploration/getExplorationGraphUrl.ts b/cognee-frontend/src/modules/exploration/getExplorationGraphUrl.ts
@@ -1,5 +1,5 @@
 export default function getExplorationGraphUrl(dataset: { id: string }) {
-  return fetch(`http://0.0.0.0:8000/datasets/${dataset.id}/graph`)
+  return fetch(`http://127.0.0.1:8000/datasets/${dataset.id}/graph`)
       .then(async (response) => {
         if (response.status !== 200) {
           throw new Error((await response.text()).replaceAll("\"", ""));

diff --git a/cognee-frontend/src/modules/ingestion/DataView/DataView.module.css b/cognee-frontend/src/modules/ingestion/DataView/DataView.module.css
@@ -12,6 +12,7 @@
 }
 
 .dataTable {
+  color: white;
   border-collapse: collapse;
 }
 .dataTable td, .dataTable th {