diff --git a/Cargo.lock b/Cargo.lock index fe11cfae..79d804eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2328,6 +2328,7 @@ dependencies = [ "indoc", "itertools 0.13.0", "mockall", + "num_cpus", "qdrant-client", "redis", "serde", diff --git a/swiftide/Cargo.toml b/swiftide/Cargo.toml index 4772db8e..066827c0 100644 --- a/swiftide/Cargo.toml +++ b/swiftide/Cargo.toml @@ -25,6 +25,7 @@ tokio = { version = "1.38.0", features = ["full"] } tracing = { version = "0.1.40", features = ["log"] } strum = "0.26.2" strum_macros = "0.26.4" +num_cpus = "1.16.0" # Integrations async-openai = { version = "0.23.2", optional = true } diff --git a/swiftide/src/ingestion/ingestion_pipeline.rs b/swiftide/src/ingestion/ingestion_pipeline.rs index 5e8c88f5..acc76572 100644 --- a/swiftide/src/ingestion/ingestion_pipeline.rs +++ b/swiftide/src/ingestion/ingestion_pipeline.rs @@ -12,14 +12,23 @@ pub struct IngestionPipeline { concurrency: usize, } +impl Default for IngestionPipeline { + fn default() -> Self { + Self { + stream: Box::pin(futures_util::stream::empty()), + storage: None, + concurrency: num_cpus::get(), + } + } +} + // A lazy pipeline for ingesting files, adding metadata, chunking, transforming, embedding and then storing them. impl IngestionPipeline { pub fn from_loader(loader: impl Loader + 'static) -> Self { let stream = loader.into_stream(); Self { stream: stream.boxed(), - storage: None, - concurrency: 10, + ..Default::default() } }