diff --git a/README.md b/README.md index 31e80d8b..5320a253 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,8 @@ indexing::Pipeline::from_loader(FileLoader::new(".").with_extensions(&["rs"])) .await?; ``` +_You can find more examples in [/examples](https://github.com/bosun-ai/swiftide/tree/master/examples)_ +

(back to top)

## Features diff --git a/swiftide/src/indexing/indexing_stream.rs b/swiftide/src/indexing/indexing_stream.rs index 6c83db9a..69431dc0 100644 --- a/swiftide/src/indexing/indexing_stream.rs +++ b/swiftide/src/indexing/indexing_stream.rs @@ -1,6 +1,7 @@ #![allow(clippy::from_over_into)] #![cfg(not(tarpaulin_include))] -//! This module defines the `IndexingStream` type, which is used for handling asynchronous streams of `Node` items in the indexing pipeline. + +//! This module defines the `IndexingStream` type, which is used internally by a pipeline for handling asynchronous streams of `Node` items in the indexing pipeline. use anyhow::Result; use futures_util::stream::{self, Stream}; @@ -73,7 +74,6 @@ impl IndexingStream { } } - // NOTE: Can we really guarantee that the iterator will outlive the stream? pub fn iter(iter: I) -> Self where I: IntoIterator> + Send + 'static, diff --git a/swiftide/src/indexing/pipeline.rs b/swiftide/src/indexing/pipeline.rs index b37e2d16..ea5b6201 100644 --- a/swiftide/src/indexing/pipeline.rs +++ b/swiftide/src/indexing/pipeline.rs @@ -36,7 +36,7 @@ impl Default for Pipeline { } impl Pipeline { - /// Creates an `Pipeline` from a given loader. + /// Creates a `Pipeline` from a given loader. /// /// # Arguments /// @@ -53,7 +53,7 @@ impl Pipeline { } } - /// Creates an `Pipeline` from a given stream. + /// Creates a `Pipeline` from a given stream. /// /// # Arguments /// @@ -69,7 +69,8 @@ impl Pipeline { } } - /// Sets the concurrency level for the pipeline. + /// Sets the concurrency level for the pipeline. By default the concurrency is set to the + /// number of cpus. /// /// # Arguments /// @@ -84,7 +85,8 @@ impl Pipeline { self } - /// Sets the embed mode for the pipeline. + /// Sets the embed mode for the pipeline. The embed mode controls what (combination) fields of a [`Node`] + /// be embedded with a vector when transforming with [`crate::transformers::Embed`] /// /// See also [`super::node::EmbedMode`]. /// diff --git a/swiftide/src/ingestion/ingestion_stream.rs b/swiftide/src/ingestion/ingestion_stream.rs deleted file mode 100644 index c67ffdfd..00000000 --- a/swiftide/src/ingestion/ingestion_stream.rs +++ /dev/null @@ -1,85 +0,0 @@ -#![allow(clippy::from_over_into)] -//! This module defines the `IngestionStream` type, which is used for handling asynchronous streams of `IngestionNode` items in the ingestion pipeline. - -use anyhow::Result; -use futures_util::stream::{self, Stream}; -use pin_project_lite::pin_project; -use std::pin::Pin; -use tokio::sync::mpsc::Receiver; - -use super::IngestionNode; - -pub use futures_util::{StreamExt, TryStreamExt}; - -// We need to inform the compiler that `inner` is pinned as well -pin_project! { - /// An asynchronous stream of `IngestionNode` items. - /// - /// Wraps an internal stream of `Result` items. - /// - /// Streams, iterators and vectors of `Result` can be converted into an `IngestionStream`. - pub struct IngestionStream { - #[pin] - pub(crate) inner: Pin> + Send>>, - } -} - -impl Stream for IngestionStream { - type Item = Result; - - fn poll_next( - self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - let this = self.project(); - this.inner.poll_next(cx) - } -} - -impl Into for Vec> { - fn into(self) -> IngestionStream { - IngestionStream::iter(self) - } -} - -impl Into for Result> { - fn into(self) -> IngestionStream { - match self { - Ok(nodes) => IngestionStream::iter(nodes.into_iter().map(Ok)), - Err(err) => IngestionStream::iter(vec![Err(err)]), - } - } -} - -impl Into for Pin> + Send>> { - fn into(self) -> IngestionStream { - IngestionStream { inner: self } - } -} - -impl Into for Receiver> { - fn into(self) -> IngestionStream { - IngestionStream { - inner: tokio_stream::wrappers::ReceiverStream::new(self).boxed(), - } - } -} - -impl IngestionStream { - pub fn empty() -> Self { - IngestionStream { - inner: stream::empty().boxed(), - } - } - - // NOTE: Can we really guarantee that the iterator will outlive the stream? - pub fn iter(iter: I) -> Self - where - I: IntoIterator> + Send + 'static, - ::IntoIter: Send, - { - IngestionStream { - inner: stream::iter(iter).boxed(), - } - } -} diff --git a/swiftide/src/prompt.rs b/swiftide/src/prompt.rs index 441125a6..7d0b2ac4 100644 --- a/swiftide/src/prompt.rs +++ b/swiftide/src/prompt.rs @@ -71,7 +71,7 @@ pub enum PromptTemplate { } impl<'tmpl> PromptTemplate { - /// Creates a reference to a template stored in the repository + /// Creates a reference to a template already stored in the repository pub fn from_compiled_template_name(name: impl Into) -> PromptTemplate { PromptTemplate::CompiledTemplate(name.into()) } diff --git a/swiftide/src/transformers/mod.rs b/swiftide/src/transformers/mod.rs index b5038d8e..993b0544 100644 --- a/swiftide/src/transformers/mod.rs +++ b/swiftide/src/transformers/mod.rs @@ -1,7 +1,10 @@ //! Various transformers for chunking, embedding and transforming data //! -//! These transformers are generic over their implementation. In most cases you will also need -//! to enable integrations. +//! These transformers are generic over their implementation and many require an +//! [`crate::integrations`] to be configured. +//! +//! Transformers that prompt have a default prompt configured. Prompts can be customized +//! and tailored, supporting Jinja style templating based on [`terra`]. See [`crate::prompt::Prompt`] and [`crate::prompt::PromptTemplate`] #[cfg(feature = "tree-sitter")] pub mod chunk_code;