chore(ci): enable continous benchmarking and improve benchmarks (#98)

.github/workflows/bench.yml

@@ -0,0 +1,33 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain update nightly && rustup default nightly
+      - name: Run benchmark
+        run: cd examples/rust && cargo +nightly bench | tee output.txt
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Rust Benchmark
+          tool: "cargo"
+          output-file-path: examples/rust/output.txt
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: "200%"
+          comment-on-alert: true
+          fail-on-alert: true
+          alert-comment-cc-users: "@timonv"

benchmarks/Cargo.toml

@@ -19,6 +19,6 @@ path = "fileloader.rs"
 harness = false
 
 [[bench]]
-name = "local_pipeline"
+name = "ingest-readme-local"
 path = "local_pipeline.rs"
 harness = false

benchmarks/fileloader.rs

@@ -5,7 +5,7 @@ use swiftide::{ingestion::StreamExt, traits::Loader};
 
 async fn run_fileloader(num_files: usize) -> Result<usize> {
     let mut total_nodes = 0;
-    let mut stream = swiftide::loaders::FileLoader::new(".")
+    let mut stream = swiftide::loaders::FileLoader::new("./benchmarks/fileloader.rs")
         .with_extensions(&["rs"])
         .into_stream()
         .take(num_files);

benchmarks/local_pipeline.rs

@@ -2,22 +2,17 @@ use anyhow::Result;
 use criterion::{criterion_group, criterion_main, Criterion};
 use swiftide::{
     ingestion::IngestionPipeline,
-    integrations::{fastembed::FastEmbed, qdrant::Qdrant},
+    integrations::fastembed::FastEmbed,
     loaders::FileLoader,
-    transformers::Embed,
+    persist::MemoryStorage,
+    transformers::{ChunkMarkdown, Embed},
 };
 
 async fn run_pipeline() -> Result<()> {
-    let qdrant_url = "http://localhost:6334";
-    IngestionPipeline::from_loader(FileLoader::new(".").with_extensions(&["rs"]))
+    IngestionPipeline::from_loader(FileLoader::new("README.md").with_extensions(&["md"]))
+        .then_chunk(ChunkMarkdown::with_chunk_range(20..256))
         .then_in_batch(10, Embed::new(FastEmbed::builder().batch_size(10).build()?))
-        .then_store_with(
-            Qdrant::try_from_url(qdrant_url)?
-                .batch_size(50)
-                .vector_size(384)
-                .collection_name("swiftide-examples-fastembed".to_string())
-                .build()?,
-        )
+        .then_store_with(MemoryStorage::default())
         .run()
         .await
 }