feat(integrations,transformers): add transformer for converting html …

…to markdown

swiftide/Cargo.toml

@@ -47,6 +47,7 @@ tree-sitter-typescript = { version = "0.21.1", optional = true }
 tree-sitter-javascript = { version = "0.21.3", optional = true }
 fastembed = { version = "3.6.1", optional = true }
 spider = { version = "1.98", optional = true }
+htmd = { version = "0.1.3", optional = true }
 
 [features]
 default = []
@@ -63,7 +64,7 @@ tree-sitter = [
 ]
 openai = ["dep:async-openai"]
 fastembed = ["dep:fastembed"]
-scraping = ["dep:spider"]
+scraping = ["dep:spider", "dep:htmd"]
 
 [dev-dependencies]
 test-log = "0.2.16"

swiftide/src/integrations/scraping/html_to_markdown_transformer.rs

@@ -0,0 +1,75 @@
+use anyhow::Result;
+use async_trait::async_trait;
+use derive_builder::Builder;
+use htmd::HtmlToMarkdown;
+
+use crate::{ingestion::IngestionNode, Transformer};
+
+#[derive(Builder)]
+#[builder(pattern = "owned")]
+/// Transforms HTML content into markdown.
+///
+/// Useful for converting scraping results into markdown.
+pub struct HtmlToMarkdownTransformer {
+    /// The `HtmlToMarkdown` instance used to convert HTML to markdown.
+    ///
+    /// Sets a sane default, but can be customized.
+    htmd: HtmlToMarkdown,
+    #[builder(default)]
+    concurrency: Option<usize>,
+}
+
+impl Default for HtmlToMarkdownTransformer {
+    fn default() -> Self {
+        Self {
+            htmd: HtmlToMarkdown::builder()
+                .skip_tags(vec!["script", "style"])
+                .build(),
+            concurrency: None,
+        }
+    }
+}
+
+impl HtmlToMarkdownTransformer {
+    #[allow(dead_code)]
+    pub fn builder() -> HtmlToMarkdownTransformerBuilder {
+        HtmlToMarkdownTransformerBuilder::default()
+    }
+}
+
+impl std::fmt::Debug for HtmlToMarkdownTransformer {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("HtmlToMarkdownTransformer").finish()
+    }
+}
+
+#[async_trait]
+impl Transformer for HtmlToMarkdownTransformer {
+    /// Converts the HTML content in the `IngestionNode` to markdown.
+    ///
+    /// Will Err the node if the conversion fails.
+    async fn transform_node(&self, node: IngestionNode) -> Result<IngestionNode> {
+        let chunk = self.htmd.convert(&node.chunk);
+        Ok(IngestionNode {
+            chunk: chunk?,
+            ..node
+        })
+    }
+
+    fn concurrency(&self) -> Option<usize> {
+        self.concurrency
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[tokio::test]
+    async fn test_html_to_markdown() {
+        let node = IngestionNode::new("<h1>Hello, World!</h1>");
+        let transformer = HtmlToMarkdownTransformer::default();
+        let transformed = transformer.transform_node(node).await.unwrap();
+        assert_eq!(transformed.chunk, "# Hello, World!");
+    }
+}

swiftide/src/integrations/scraping/loader.rs

@@ -1,8 +1,4 @@
-use std::{borrow::BorrowMut, cell::RefCell};
-
-use anyhow::{Context as _, Result};
 use derive_builder::Builder;
-use futures_util::stream;
 use spider::website::Website;
 use tokio::sync::RwLock;
 
@@ -23,6 +19,7 @@ pub struct ScrapingLoader {
 
 impl ScrapingLoader {
     // Constructs a scrapingloader from a `spider::Website` configuration
+    #![allow(dead_code)]
     pub fn from_spider(spider_website: Website) -> Self {
         Self {
             spider_website: RwLock::new(spider_website),

swiftide/src/integrations/scraping/mod.rs

@@ -1 +1,2 @@
+mod html_to_markdown_transformer;
 mod loader;