docs(swiftide): documentation improvements and cleanup (#176)

- **chore: remove ingestion stream**
- **Documentation and grammar**

README.md

@@ -108,6 +108,8 @@ indexing::Pipeline::from_loader(FileLoader::new(".").with_extensions(&["rs"]))
         .await?;
 ```
 
+_You can find more examples in [/examples](https://github.com/bosun-ai/swiftide/tree/master/examples)_
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 ## Features

swiftide/src/indexing/indexing_stream.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::from_over_into)]
 #![cfg(not(tarpaulin_include))]
-//! This module defines the `IndexingStream` type, which is used for handling asynchronous streams of `Node` items in the indexing pipeline.
+
+//! This module defines the `IndexingStream` type, which is used internally by a pipeline  for handling asynchronous streams of `Node` items in the indexing pipeline.
 
 use anyhow::Result;
 use futures_util::stream::{self, Stream};
@@ -73,7 +74,6 @@ impl IndexingStream {
         }
     }
 
-    // NOTE: Can we really guarantee that the iterator will outlive the stream?
     pub fn iter<I>(iter: I) -> Self
     where
         I: IntoIterator<Item = Result<Node>> + Send + 'static,

swiftide/src/indexing/pipeline.rs

@@ -36,7 +36,7 @@ impl Default for Pipeline {
 }
 
 impl Pipeline {
-    /// Creates an `Pipeline` from a given loader.
+    /// Creates a `Pipeline` from a given loader.
     ///
     /// # Arguments
     ///
@@ -53,7 +53,7 @@ impl Pipeline {
         }
     }
 
-    /// Creates an `Pipeline` from a given stream.
+    /// Creates a `Pipeline` from a given stream.
     ///
     /// # Arguments
     ///
@@ -69,7 +69,8 @@ impl Pipeline {
         }
     }
 
-    /// Sets the concurrency level for the pipeline.
+    /// Sets the concurrency level for the pipeline. By default the concurrency is set to the
+    /// number of cpus.
     ///
     /// # Arguments
     ///
@@ -84,7 +85,8 @@ impl Pipeline {
         self
     }
 
-    /// Sets the embed mode for the pipeline.
+    /// Sets the embed mode for the pipeline. The embed mode controls what (combination) fields of a [`Node`]
+    /// be embedded with a vector when transforming with [`crate::transformers::Embed`]
     ///
     /// See also [`super::node::EmbedMode`].
     ///

swiftide/src/prompt.rs

@@ -72,7 +72,7 @@ pub enum PromptTemplate {
 }
 
 impl<'tmpl> PromptTemplate {
-    /// Creates a reference to a template stored in the repository
+    /// Creates a reference to a template already stored in the repository
     pub fn from_compiled_template_name(name: impl Into<String>) -> PromptTemplate {
         PromptTemplate::CompiledTemplate(name.into())
     }

swiftide/src/transformers/mod.rs

@@ -1,7 +1,10 @@
 //! Various transformers for chunking, embedding and transforming data
 //!
-//! These transformers are generic over their implementation. In most cases you will also need
-//! to enable integrations.
+//! These transformers are generic over their implementation and many require an
+//! [`crate::integrations`] to be configured.
+//!
+//! Transformers that prompt have a default prompt configured. Prompts can be customized
+//! and tailored, supporting Jinja style templating based on [`terra`]. See [`crate::prompt::Prompt`] and [`crate::prompt::PromptTemplate`]
 
 #[cfg(feature = "tree-sitter")]
 pub mod chunk_code;