feat(qdrant): safe to clone with internal arc

bit more explicit for tarpaulin

tune tarpaulin

add debugging to persist

set correct grpc port default

more cleanup

disable coverage for now

.github/workflows/coverage.yml

@@ -1,38 +1,38 @@
-name: Coverage
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  test:
-    name: coverage
-    runs-on: ubuntu-latest
-    services:
-      qdrant:
-        image: qdrant/qdrant:v1.9.7
-        ports:
-          - 6334:6334
-    env:
-      RUST_LOG: DEBUG
-      RUST_BACKTRACE: 1
-      QDRANT_URL: http://qdrant:6334
-    container:
-      image: xd009642/tarpaulin:develop-nightly
-      options: --security-opt seccomp=unconfined
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Generate code coverage
-        run: |
-          cargo +nightly tarpaulin --verbose --all-features --workspace --timeout 120 --out xml
-
-      - name: Coveralls
-        uses: coverallsapp/github-action@v2
+# name: Coverage
+#
+# on:
+#   pull_request:
+#   push:
+#     branches:
+#       - master
+#
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+#   cancel-in-progress: true
+#
+# jobs:
+#   test:
+#     name: coverage
+#     runs-on: ubuntu-latest
+#     services:
+#       qdrant:
+#         image: qdrant/qdrant:v1.9.7
+#         ports:
+#           - 6334:6334
+#     env:
+#       RUST_LOG: swiftide=debug
+#       RUST_BACKTRACE: 1
+#       QDRANT_URL: http://qdrant:6334
+#     container:
+#       image: xd009642/tarpaulin:develop-nightly
+#       options: --security-opt seccomp=unconfined
+#     steps:
+#       - name: Checkout repository
+#         uses: actions/checkout@v4
+#
+#       - name: Generate code coverage
+#         run: |
+#           cargo tarpaulin --verbose --all-features -p swiftide --timeout 120 --out xml
+#
+#       - name: Coveralls
+#         uses: coverallsapp/github-action@v2

README.md

@@ -26,8 +26,11 @@
 *** for contributors-url, forks-url, etc. This is an optional, concise syntax you may use.
 *** https://www.markdownguide.org/basic-syntax/#reference-style-links
 -->
+
 ![CI](https://img.shields.io/github/actions/workflow/status/bosun-ai/swiftide/test.yml?style=flat-square)
-![Coverage Status](https://img.shields.io/coverallsCoverage/github/bosun-ai/swiftide?style=flat-square)
+
+<!-- ![Coverage Status](https://img.shields.io/coverallsCoverage/github/bosun-ai/swiftide?style=flat-square) -->
+
 [![Crate Badge]][Crate]
 [![Docs Badge]][API Docs]
 [![Contributors][contributors-shield]][contributors-url]

swiftide/Cargo.toml

@@ -32,7 +32,7 @@ pin-project-lite = "0.2"
 
 # Integrations
 async-openai = { version = "0.23.2", optional = true }
-qdrant-client = { version = "1.9.0", optional = true }
+qdrant-client = { version = "1.10.1", optional = true }
 redis = { version = "0.25.4", features = [
   "aio",
   "tokio-comp",

swiftide/src/integrations/qdrant/mod.rs

@@ -5,17 +5,22 @@
 mod ingestion_node;
 mod persist;
 
+use std::sync::Arc;
+
 use anyhow::{Context as _, Result};
 use derive_builder::Builder;
 use qdrant_client::qdrant::{CreateCollectionBuilder, Distance, VectorParamsBuilder};
 
 const DEFAULT_COLLECTION_NAME: &str = "swiftide";
+const DEFAULT_QDRANT_URL: &str = "http://localhost:6334";
 
 /// A struct representing a Qdrant client with configuration options.
 ///
 /// This struct is used to interact with the Qdrant vector database, providing methods to create and manage
 /// vector collections, store data, and ensure proper indexing for efficient searches.
-#[derive(Builder)]
+///
+/// Can be cloned with relative low cost as the client is shared.
+#[derive(Builder, Clone)]
 #[builder(
     pattern = "owned",
     setter(strip_option),
@@ -25,8 +30,9 @@ pub struct Qdrant {
     /// The Qdrant client used to interact with the Qdrant vector database.
     ///
     /// By default the client will be build from QDRANT_URL and option QDRANT_API_KEY.
+    /// It will fall back to `http://localhost:6334` if QDRANT_URL is not set.
     #[builder(setter(into), default = "self.default_client()?")]
-    client: qdrant_client::Qdrant,
+    client: Arc<qdrant_client::Qdrant>,
     /// The name of the collection to be used in Qdrant. Defaults to "swiftide".
     #[builder(default = "DEFAULT_COLLECTION_NAME.to_string()")]
     #[builder(setter(into))]
@@ -90,13 +96,15 @@ impl Qdrant {
 }
 
 impl QdrantBuilder {
-    fn default_client(&self) -> Result<qdrant_client::Qdrant> {
-        qdrant_client::Qdrant::from_url(
-            &std::env::var("QDRANT_URL").unwrap_or("http://localhost:6333".to_string()),
+    fn default_client(&self) -> Result<Arc<qdrant_client::Qdrant>> {
+        let client = qdrant_client::Qdrant::from_url(
+            &std::env::var("QDRANT_URL").unwrap_or(DEFAULT_QDRANT_URL.to_string()),
         )
         .api_key(std::env::var("QDRANT_API_KEY"))
         .build()
-        .context("Could not build default qdrant client")
+        .context("Could not build default qdrant client")?;
+
+        Ok(Arc::new(client))
     }
 }
 

swiftide/src/integrations/qdrant/persist.rs

@@ -55,6 +55,9 @@ impl Persist for Qdrant {
     #[tracing::instrument(skip_all, err, name = "storage.qdrant.store")]
     async fn store(&self, node: crate::ingestion::IngestionNode) -> Result<IngestionNode> {
         let point = node.clone().try_into()?;
+
+        tracing::debug!(?node, ?point, "Storing node");
+
         self.client
             .upsert_points(UpsertPointsBuilder::new(
                 self.collection_name.to_string(),
@@ -90,6 +93,8 @@ impl Persist for Qdrant {
 
         let points = points.unwrap();
 
+        tracing::debug!("Storing batch of {} nodes", points.len());
+
         let result = self
             .client
             .upsert_points(UpsertPointsBuilder::new(

swiftide/tests/ingestion_pipeline.rs

@@ -2,6 +2,7 @@
 //! The tests validate the functionality of the pipeline, ensuring it processes data correctly
 //! from a temporary file, simulates API responses, and stores data accurately in the Qdrant vector database.
 
+use qdrant_client::qdrant::{SearchPointsBuilder, Value};
 use serde_json::json;
 use swiftide::{ingestion::IngestionPipeline, loaders::FileLoader, *};
 use temp_dir::TempDir;
@@ -66,7 +67,7 @@ async fn test_ingestion_pipeline() {
           "data": [
             {
               "object": "embedding",
-            "embedding": vec![0; 1536],
+              "embedding": vec![0; 1536],
               "index": 0
             }
           ],
@@ -174,20 +175,18 @@ async fn test_ingestion_pipeline() {
 
     result.expect("Ingestion pipeline failed");
 
-    use qdrant_client::prelude::*;
-    let qdrant_client = QdrantClient::from_url(&qdrant_url).build().unwrap();
-    let search_result = qdrant_client
-        .search_points(&SearchPoints {
-            collection_name: "swiftide-test".to_string(),
-            vector: vec![0_f32; 1536],
-            limit: 10,
-            with_payload: Some(true.into()),
-            ..Default::default()
-        })
-        .await
+    let qdrant_client = qdrant_client::Qdrant::from_url(&qdrant_url)
+        .build()
         .unwrap();
 
-    let first = search_result.result.first().unwrap();
+    let search_request =
+        SearchPointsBuilder::new("swiftide-test", vec![0_f32; 1536], 10).with_payload(true);
+
+    let search_response = qdrant_client.search_points(search_request).await.unwrap();
+
+    dbg!(&search_response);
+
+    let first = search_response.result.first().unwrap();
 
     assert!(first
         .payload