From a866d38b2cbaf7154ace957472c75b99f1c9c070 Mon Sep 17 00:00:00 2001
From: Timon Vonk <timonv@gmail.com>
Date: Sun, 27 Oct 2024 16:43:30 +0100
Subject: [PATCH] feat(integrations): Support in process hugging face models
 via mistralrs (#386)

---
 .github/workflows/coverage.yml                |   2 +-
 .github/workflows/test.yml                    |   6 +-
 Cargo.lock                                    | 959 +++++++++++++++++-
 Cargo.toml                                    |   1 +
 README.md                                     |   4 +-
 deny.toml                                     |  21 +-
 examples/Cargo.toml                           |   5 +
 examples/mistralrs.rs                         |  34 +
 swiftide-integrations/Cargo.toml              |   5 +-
 swiftide-integrations/src/lib.rs              |   2 +
 swiftide-integrations/src/mistralrs/mod.rs    |   9 +
 .../src/mistralrs/simple_prompt.rs            | 110 ++
 swiftide-test-utils/Cargo.toml                |   4 +-
 swiftide/Cargo.toml                           |   8 +-
 swiftide/tests/lancedb.rs                     |  66 +-
 15 files changed, 1141 insertions(+), 95 deletions(-)
 create mode 100644 examples/mistralrs.rs
 create mode 100644 swiftide-integrations/src/mistralrs/mod.rs
 create mode 100644 swiftide-integrations/src/mistralrs/simple_prompt.rs

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
index 4de9da59..4b0e93e3 100644
--- a/.github/workflows/coverage.yml
+++ b/.github/workflows/coverage.yml
@@ -2,6 +2,7 @@ name: Coverage
 
 on:
   pull_request:
+  workflow_dispatch:
   push:
     branches:
       - master
@@ -33,6 +34,5 @@ jobs:
       - name: Generate code coverage
         run: |
           cargo +nightly llvm-cov --tests --all-features --lcov --output-path lcov.info
-
       - name: Coveralls
         uses: coverallsapp/github-action@v2
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index bac7cdf0..2a5c4fcb 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -3,6 +3,7 @@ name: CI
 on:
   pull_request:
   merge_group:
+  workflow_dispatch:
   push:
     branches:
       - master
@@ -25,12 +26,10 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: rustfmt
-      - uses: r7kamura/rust-problem-matchers@v1
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: r7kamura/rust-problem-matchers@v1
       - name: Check typos
         uses: crate-ci/typos@master
       - name: "Rustfmt"
@@ -50,6 +49,5 @@ jobs:
         uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: r7kamura/rust-problem-matchers@v1
       - name: "Test"
-        run: cargo test --all-features --tests
+        run: cargo test --all-features --tests -j 2
diff --git a/Cargo.lock b/Cargo.lock
index 702128b1..03128b1f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,16 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "Inflector"
+version = "0.11.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
+dependencies = [
+ "lazy_static",
+ "regex",
+]
+
 [[package]]
 name = "addr2line"
 version = "0.24.2"
@@ -46,6 +56,12 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "akin"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1763692fc1416554cf051efc56a3de5595eca47299d731cc5c2b583adf8b4d2f"
+
 [[package]]
 name = "aligned-vec"
 version = "0.5.0"
@@ -154,6 +170,9 @@ name = "arbitrary"
 version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
+dependencies = [
+ "derive_arbitrary",
+]
 
 [[package]]
 name = "arc-swap"
@@ -406,6 +425,12 @@ dependencies = [
  "regex-syntax 0.8.5",
 ]
 
+[[package]]
+name = "as-any"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b8a30a44e99a1c83ccb2a6298c563c888952a1c9134953db26876528f84c93a"
+
 [[package]]
 name = "assert-json-diff"
 version = "2.0.2"
@@ -861,7 +886,7 @@ dependencies = [
  "percent-encoding",
  "pin-project-lite",
  "tracing",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -1423,6 +1448,21 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "buildstructor"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3907aac66c65520545ae3cb3c195306e20d5ed5c90bfbb992e061cf12a104d0"
+dependencies = [
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "str_inflector",
+ "syn 2.0.82",
+ "thiserror",
+ "try_match",
+]
+
 [[package]]
 name = "built"
 version = "0.7.5"
@@ -1446,6 +1486,20 @@ name = "bytemuck"
 version = "1.18.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
+dependencies = [
+ "bytemuck_derive",
+]
+
+[[package]]
+name = "bytemuck_derive"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
 
 [[package]]
 name = "byteorder"
@@ -1496,6 +1550,40 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "candle-core"
+version = "0.7.2"
+source = "git+https://github.com/EricLBuehler/candle.git?rev=20a57c4#20a57c4bcf300e4bc6c1e48f7f3702668ae8cb80"
+dependencies = [
+ "byteorder",
+ "gemm",
+ "half",
+ "memmap2",
+ "num-traits",
+ "num_cpus",
+ "rand",
+ "rand_distr",
+ "rayon",
+ "safetensors",
+ "thiserror",
+ "yoke",
+ "zip",
+]
+
+[[package]]
+name = "candle-nn"
+version = "0.7.2"
+source = "git+https://github.com/EricLBuehler/candle.git?rev=20a57c4#20a57c4bcf300e4bc6c1e48f7f3702668ae8cb80"
+dependencies = [
+ "candle-core",
+ "half",
+ "num-traits",
+ "rayon",
+ "safetensors",
+ "serde",
+ "thiserror",
+]
+
 [[package]]
 name = "cargo-platform"
 version = "0.1.8"
@@ -1582,6 +1670,20 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
+[[package]]
+name = "cfgrammar"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6026d8cd82ada8bbcfe337805dd1eb6afdc9e80fa4d57e977b3a36315e0c5525"
+dependencies = [
+ "indexmap 2.6.0",
+ "lazy_static",
+ "num-traits",
+ "regex",
+ "serde",
+ "vob",
+]
+
 [[package]]
 name = "chardetng"
 version = "0.1.17"
@@ -1664,6 +1766,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
 dependencies = [
  "clap_builder",
+ "clap_derive",
 ]
 
 [[package]]
@@ -1672,8 +1775,22 @@ version = "4.5.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
 dependencies = [
+ "anstream",
  "anstyle",
  "clap_lex",
+ "strsim 0.11.1",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.5.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
+dependencies = [
+ "heck 0.5.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
 ]
 
 [[package]]
@@ -1935,6 +2052,31 @@ version = "0.8.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
 
+[[package]]
+name = "crossterm"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
+dependencies = [
+ "bitflags 1.3.2",
+ "crossterm_winapi",
+ "libc",
+ "mio 0.8.11",
+ "parking_lot 0.12.3",
+ "signal-hook",
+ "signal-hook-mio",
+ "winapi",
+]
+
+[[package]]
+name = "crossterm_winapi"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "crunchy"
 version = "0.2.2"
@@ -1995,14 +2137,38 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "darling"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbffa8f8e38810422f320ca457a93cf1cd0056dc9c06c556b867558e0d471463"
+dependencies = [
+ "darling_core 0.11.0",
+ "darling_macro 0.11.0",
+]
+
 [[package]]
 name = "darling"
 version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
 dependencies = [
- "darling_core",
- "darling_macro",
+ "darling_core 0.20.10",
+ "darling_macro 0.20.10",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06e172685d94b7b83800e3256a63261537b9d6129e10f21c8e13ddf9dba8c64d"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim 0.10.0",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -2015,17 +2181,28 @@ dependencies = [
  "ident_case",
  "proc-macro2",
  "quote",
- "strsim",
+ "strsim 0.11.1",
  "syn 2.0.82",
 ]
 
+[[package]]
+name = "darling_macro"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0618ac802792cebd1918ac6042a6ea1eeab92db34b35656afaa577929820788"
+dependencies = [
+ "darling_core 0.11.0",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "darling_macro"
 version = "0.20.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
 dependencies = [
- "darling_core",
+ "darling_core 0.20.10",
  "quote",
  "syn 2.0.82",
 ]
@@ -2109,7 +2286,7 @@ dependencies = [
  "tempfile",
  "tokio",
  "url",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -2216,7 +2393,7 @@ dependencies = [
  "rand",
  "regex",
  "unicode-segmentation",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -2438,6 +2615,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "defmac"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aafbece59594ed57696a1a69e8bb3ca1683fbc9cdb41d5c02726070b2cd8f19d"
+
 [[package]]
 name = "deranged"
 version = "0.3.11"
@@ -2448,6 +2631,28 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "derive-new"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
+[[package]]
+name = "derive_arbitrary"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
 [[package]]
 name = "derive_builder"
 version = "0.20.2"
@@ -2463,7 +2668,7 @@ version = "0.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
 dependencies = [
- "darling",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
  "syn 2.0.82",
@@ -2533,6 +2738,17 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "displaydoc"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
 [[package]]
 name = "doc-comment"
 version = "0.3.3"
@@ -2592,6 +2808,16 @@ version = "1.0.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125"
 
+[[package]]
+name = "dyn-stack"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"
+dependencies = [
+ "bytemuck",
+ "reborrow",
+]
+
 [[package]]
 name = "educe"
 version = "0.6.0"
@@ -2615,6 +2841,9 @@ name = "either"
 version = "1.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "encode_unicode"
@@ -2631,6 +2860,12 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "endian-type"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
+
 [[package]]
 name = "enum-as-inner"
 version = "0.6.1"
@@ -2714,6 +2949,9 @@ name = "esaxx-rs"
 version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
+dependencies = [
+ "cc",
+]
 
 [[package]]
 name = "etcetera"
@@ -3424,6 +3662,134 @@ dependencies = [
  "byteorder",
 ]
 
+[[package]]
+name = "galil-seiferas"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "794ac25cfda3fa11d2b07ff8c65889c6c03411646df54e59e606878d899e1d5a"
+dependencies = [
+ "defmac",
+ "unchecked-index",
+]
+
+[[package]]
+name = "gemm"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"
+dependencies = [
+ "dyn-stack",
+ "gemm-c32",
+ "gemm-c64",
+ "gemm-common",
+ "gemm-f16",
+ "gemm-f32",
+ "gemm-f64",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-c32"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-c64"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-common"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"
+dependencies = [
+ "bytemuck",
+ "dyn-stack",
+ "half",
+ "num-complex",
+ "num-traits",
+ "once_cell",
+ "paste",
+ "pulp",
+ "raw-cpuid",
+ "rayon",
+ "seq-macro",
+ "sysctl",
+]
+
+[[package]]
+name = "gemm-f16"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "gemm-f32",
+ "half",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "rayon",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-f32"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
+[[package]]
+name = "gemm-f64"
+version = "0.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"
+dependencies = [
+ "dyn-stack",
+ "gemm-common",
+ "num-complex",
+ "num-traits",
+ "paste",
+ "raw-cpuid",
+ "seq-macro",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.7"
@@ -3549,9 +3915,12 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
 dependencies = [
+ "bytemuck",
  "cfg-if",
  "crunchy",
  "num-traits",
+ "rand",
+ "rand_distr",
 ]
 
 [[package]]
@@ -4135,6 +4504,7 @@ dependencies = [
  "instant",
  "number_prefix",
  "portable-atomic",
+ "rayon",
  "unicode-width",
 ]
 
@@ -4373,7 +4743,7 @@ dependencies = [
  "tokio",
  "tracing",
  "url",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -4583,7 +4953,7 @@ dependencies = [
  "tempfile",
  "tokio",
  "tracing",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -4690,7 +5060,7 @@ dependencies = [
  "tokio",
  "tracing",
  "url",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -4945,6 +5315,19 @@ dependencies = [
  "imgref",
 ]
 
+[[package]]
+name = "lrtable"
+version = "0.13.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d42d2752cb50a171efadda0cb6fa97432e8bf05accfff3eed320b87e80a2f69e"
+dependencies = [
+ "cfgrammar",
+ "fnv",
+ "num-traits",
+ "sparsevec",
+ "vob",
+]
+
 [[package]]
 name = "lru"
 version = "0.12.5"
@@ -5102,6 +5485,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f"
 dependencies = [
  "libc",
+ "stable_deref_trait",
 ]
 
 [[package]]
@@ -5120,6 +5504,26 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "minijinja"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9ca8daf4b0b4029777f1bc6e1aedd1aec7b74c276a43bc6f620a8e1a1c0a90e"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "minijinja-contrib"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ffd46ee854be23604a20efd6c9655374fefbe4d44b949dc0f907305d92873a"
+dependencies = [
+ "minijinja",
+ "serde",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -5145,6 +5549,18 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "mio"
+version = "0.8.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c"
+dependencies = [
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "mio"
 version = "1.0.2"
@@ -5157,6 +5573,116 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "mistralrs"
+version = "0.3.1"
+source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=9dfbab1#9dfbab1a1f3f260f130680f6c4f02f8656ea28ed"
+dependencies = [
+ "anyhow",
+ "candle-core",
+ "either",
+ "futures",
+ "image",
+ "indexmap 2.6.0",
+ "mistralrs-core",
+ "rand",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "tokio",
+]
+
+[[package]]
+name = "mistralrs-core"
+version = "0.3.1"
+source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=9dfbab1#9dfbab1a1f3f260f130680f6c4f02f8656ea28ed"
+dependencies = [
+ "akin",
+ "anyhow",
+ "as-any",
+ "async-trait",
+ "base64 0.22.1",
+ "buildstructor",
+ "bytemuck",
+ "bytemuck_derive",
+ "candle-core",
+ "candle-nn",
+ "cfgrammar",
+ "chrono",
+ "clap",
+ "csv",
+ "derive-new",
+ "derive_more",
+ "dirs",
+ "either",
+ "futures",
+ "galil-seiferas",
+ "half",
+ "hf-hub",
+ "image",
+ "indexmap 2.6.0",
+ "indicatif",
+ "itertools 0.13.0",
+ "lrtable",
+ "minijinja",
+ "minijinja-contrib",
+ "mistralrs-quant",
+ "mistralrs-vision",
+ "once_cell",
+ "radix_trie",
+ "rand",
+ "rand_isaac",
+ "rayon",
+ "regex",
+ "regex-automata 0.4.8",
+ "reqwest",
+ "rustc-hash 2.0.0",
+ "safetensors",
+ "schemars",
+ "serde",
+ "serde_json",
+ "serde_plain",
+ "serde_yaml",
+ "strum",
+ "sysinfo",
+ "thiserror",
+ "tokenizers",
+ "tokio",
+ "tokio-rayon",
+ "toml",
+ "tqdm",
+ "tracing",
+ "tracing-subscriber",
+ "uuid 1.11.0",
+ "variantly",
+ "vob",
+]
+
+[[package]]
+name = "mistralrs-quant"
+version = "0.3.1"
+source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=9dfbab1#9dfbab1a1f3f260f130680f6c4f02f8656ea28ed"
+dependencies = [
+ "byteorder",
+ "candle-core",
+ "candle-nn",
+ "half",
+ "lazy_static",
+ "paste",
+ "rayon",
+ "serde",
+ "tracing",
+]
+
+[[package]]
+name = "mistralrs-vision"
+version = "0.3.1"
+source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=9dfbab1#9dfbab1a1f3f260f130680f6c4f02f8656ea28ed"
+dependencies = [
+ "candle-core",
+ "image",
+]
+
 [[package]]
 name = "mock_instant"
 version = "0.3.2"
@@ -5214,7 +5740,7 @@ dependencies = [
  "tagptr",
  "thiserror",
  "triomphe",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -5288,6 +5814,15 @@ version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
 
+[[package]]
+name = "nibble_vec"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "nix"
 version = "0.29.0"
@@ -5316,6 +5851,15 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8"
 
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.46.0"
@@ -5356,6 +5900,7 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
 dependencies = [
+ "bytemuck",
  "num-traits",
 ]
 
@@ -5427,6 +5972,27 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "num_enum"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179"
+dependencies = [
+ "num_enum_derive",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
 [[package]]
 name = "number_prefix"
 version = "0.4.0"
@@ -5628,9 +6194,19 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
 name = "ownedbytes"
 version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
+checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558"
+dependencies = [
+ "stable_deref_trait",
+]
+
+[[package]]
+name = "packedvec"
+version = "1.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bde3c690ec20e4a2b4fb46f0289a451181eb50011a1e2acc8d85e2fde9062a45"
 dependencies = [
- "stable_deref_trait",
+ "num-traits",
+ "serde",
 ]
 
 [[package]]
@@ -6111,6 +6687,15 @@ dependencies = [
  "syn 2.0.82",
 ]
 
+[[package]]
+name = "proc-macro-crate"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b"
+dependencies = [
+ "toml_edit",
+]
+
 [[package]]
 name = "proc-macro-hack"
 version = "0.5.20+deprecated"
@@ -6268,6 +6853,18 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "pulp"
+version = "0.18.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"
+dependencies = [
+ "bytemuck",
+ "libm",
+ "num-complex",
+ "reborrow",
+]
+
 [[package]]
 name = "qdrant-client"
 version = "1.12.1"
@@ -6395,6 +6992,16 @@ version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
 
+[[package]]
+name = "radix_trie"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
+dependencies = [
+ "endian-type",
+ "nibble_vec",
+]
+
 [[package]]
 name = "rand"
 version = "0.8.5"
@@ -6435,6 +7042,15 @@ dependencies = [
  "rand",
 ]
 
+[[package]]
+name = "rand_isaac"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fac4373cd91b4f55722c553fb0f286edbb81ef3ff6eec7b99d1898a4110a0b28"
+dependencies = [
+ "rand_core",
+]
+
 [[package]]
 name = "rangemap"
 version = "1.5.1"
@@ -6536,6 +7152,12 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "reborrow"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
+
 [[package]]
 name = "redb"
 version = "2.1.4"
@@ -6675,6 +7297,7 @@ dependencies = [
  "cookie",
  "cookie_store",
  "encoding_rs",
+ "futures-channel",
  "futures-core",
  "futures-util",
  "h2 0.4.6",
@@ -6962,6 +7585,16 @@ version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
 
+[[package]]
+name = "safetensors"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -6989,6 +7622,30 @@ dependencies = [
  "parking_lot 0.12.3",
 ]
 
+[[package]]
+name = "schemars"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92"
+dependencies = [
+ "dyn-clone",
+ "schemars_derive",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "schemars_derive"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn 2.0.82",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -7098,6 +7755,17 @@ dependencies = [
  "syn 2.0.82",
 ]
 
+[[package]]
+name = "serde_derive_internals"
+version = "0.29.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
 [[package]]
 name = "serde_json"
 version = "1.0.132"
@@ -7110,6 +7778,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_plain"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_repr"
 version = "0.1.19"
@@ -7166,7 +7843,7 @@ version = "3.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d"
 dependencies = [
- "darling",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
  "syn 2.0.82",
@@ -7235,6 +7912,27 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
+[[package]]
+name = "signal-hook"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801"
+dependencies = [
+ "libc",
+ "signal-hook-registry",
+]
+
+[[package]]
+name = "signal-hook-mio"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd"
+dependencies = [
+ "libc",
+ "mio 0.8.11",
+ "signal-hook",
+]
+
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.2"
@@ -7375,6 +8073,18 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "sparsevec"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35df5d2e580b29f3f7ec5b4ed49b0ab3acf7f3624122b3e823cafb9630f293b8"
+dependencies = [
+ "num-traits",
+ "packedvec",
+ "serde",
+ "vob",
+]
+
 [[package]]
 name = "spider"
 version = "2.10.7"
@@ -7476,6 +8186,16 @@ version = "0.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978"
 
+[[package]]
+name = "str_inflector"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0b848d5a7695b33ad1be00f84a3c079fe85c9278a325ff9159e6c99cef4ef7"
+dependencies = [
+ "lazy_static",
+ "regex",
+]
+
 [[package]]
 name = "string-interner"
 version = "0.17.0"
@@ -7519,6 +8239,12 @@ version = "0.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c3c3ee6129eec20fed59acf2e9cfb3ffd20d0bbe39fe334c22af0edc56dfe752"
 
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
 [[package]]
 name = "strsim"
 version = "0.11.1"
@@ -7581,11 +8307,9 @@ name = "swiftide"
 version = "0.13.4"
 dependencies = [
  "anyhow",
- "arrow-array",
  "async-openai",
  "document-features",
  "insta",
- "lancedb",
  "mockall",
  "qdrant-client",
  "serde",
@@ -7626,7 +8350,7 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tracing",
- "uuid",
+ "uuid 1.11.0",
 ]
 
 [[package]]
@@ -7695,6 +8419,7 @@ dependencies = [
  "insta",
  "itertools 0.13.0",
  "lancedb",
+ "mistralrs",
  "mockall",
  "ollama-rs",
  "parquet",
@@ -7731,7 +8456,7 @@ dependencies = [
 name = "swiftide-macros"
 version = "0.13.4"
 dependencies = [
- "darling",
+ "darling 0.20.10",
  "proc-macro2",
  "quote",
  "syn 2.0.82",
@@ -7821,6 +8546,46 @@ dependencies = [
  "futures-core",
 ]
 
+[[package]]
+name = "synstructure"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
+[[package]]
+name = "sysctl"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
+dependencies = [
+ "bitflags 2.6.0",
+ "byteorder",
+ "enum-as-inner",
+ "libc",
+ "thiserror",
+ "walkdir",
+]
+
+[[package]]
+name = "sysinfo"
+version = "0.30.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3"
+dependencies = [
+ "cfg-if",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "windows",
+]
+
 [[package]]
 name = "system-configuration"
 version = "0.6.1"
@@ -7908,7 +8673,7 @@ dependencies = [
  "tempfile",
  "thiserror",
  "time",
- "uuid",
+ "uuid 1.11.0",
  "winapi",
 ]
 
@@ -8307,6 +9072,7 @@ dependencies = [
  "derive_builder",
  "esaxx-rs",
  "getrandom",
+ "indicatif",
  "itertools 0.12.1",
  "lazy_static",
  "log",
@@ -8337,7 +9103,7 @@ dependencies = [
  "backtrace",
  "bytes",
  "libc",
- "mio",
+ "mio 1.0.2",
  "parking_lot 0.12.3",
  "pin-project-lite",
  "signal-hook-registry",
@@ -8367,6 +9133,16 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-rayon"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cf33a76e0b1dd03b778f83244137bd59887abf25c0e87bc3e7071105f457693"
+dependencies = [
+ "rayon",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-retry2"
 version = "0.5.6"
@@ -8554,6 +9330,17 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
+[[package]]
+name = "tqdm"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa2d2932240205a99b65f15d9861992c95fbb8c9fb280b3a1f17a92db6dc611f"
+dependencies = [
+ "anyhow",
+ "crossterm",
+ "once_cell",
+]
+
 [[package]]
 name = "tracing"
 version = "0.1.40"
@@ -8706,6 +9493,26 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
+[[package]]
+name = "try_match"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b065c869a3f832418e279aa4c1d7088f9d5d323bde15a60a08e20c2cd4549082"
+dependencies = [
+ "try_match_inner",
+]
+
+[[package]]
+name = "try_match_inner"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9c81686f7ab4065ccac3df7a910c4249f8c0f3fb70421d6ddec19b9311f63f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+]
+
 [[package]]
 name = "twox-hash"
 version = "1.6.3"
@@ -8740,6 +9547,12 @@ version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
 
+[[package]]
+name = "unchecked-index"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c"
+
 [[package]]
 name = "unic-char-property"
 version = "0.9.0"
@@ -8914,6 +9727,15 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "getrandom",
+]
+
 [[package]]
 name = "uuid"
 version = "1.11.0"
@@ -8948,6 +9770,20 @@ version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101"
 
+[[package]]
+name = "variantly"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72a332341ba79a179d9e9b33c0d72fbf3dc2c80e1be79416401a08d2b820ef56"
+dependencies = [
+ "Inflector",
+ "darling 0.11.0",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "uuid 0.8.2",
+]
+
 [[package]]
 name = "vcpkg"
 version = "0.2.15"
@@ -8966,6 +9802,17 @@ version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
 
+[[package]]
+name = "vob"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c058f4c41e71a043c67744cb76dcc1ae63ece328c1732a72489ccccc2dec23e6"
+dependencies = [
+ "num-traits",
+ "rustc_version",
+ "serde",
+]
+
 [[package]]
 name = "vsimd"
 version = "0.8.0"
@@ -9145,6 +9992,16 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
+dependencies = [
+ "windows-core",
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-core"
 version = "0.52.0"
@@ -9431,6 +10288,30 @@ version = "0.13.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
 
+[[package]]
+name = "yoke"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
+dependencies = [
+ "serde",
+ "stable_deref_trait",
+ "yoke-derive",
+ "zerofrom",
+]
+
+[[package]]
+name = "yoke-derive"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+ "synstructure",
+]
+
 [[package]]
 name = "zerocopy"
 version = "0.7.35"
@@ -9452,12 +10333,48 @@ dependencies = [
  "syn 2.0.82",
 ]
 
+[[package]]
+name = "zerofrom"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
+dependencies = [
+ "zerofrom-derive",
+]
+
+[[package]]
+name = "zerofrom-derive"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.82",
+ "synstructure",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
 
+[[package]]
+name = "zip"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164"
+dependencies = [
+ "arbitrary",
+ "crc32fast",
+ "crossbeam-utils",
+ "displaydoc",
+ "indexmap 2.6.0",
+ "num_enum",
+ "thiserror",
+]
+
 [[package]]
 name = "zstd"
 version = "0.13.2"
diff --git a/Cargo.toml b/Cargo.toml
index 6988f83e..f6c3a2b4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,6 @@
 [workspace]
 members = ["swiftide", "swiftide-*", "examples", "benchmarks"]
+default-members = ["swiftide", "swiftide-*"]
 
 resolver = "2"
 
diff --git a/README.md b/README.md
index 69cbf69c..8df7e5ca 100644
--- a/README.md
+++ b/README.md
@@ -174,7 +174,7 @@ Our goal is to create a fast, extendable platform for Retrieval Augmented Genera
 - Splitting and merging pipelines
 - Jinja-like templating for prompts
 - Store into multiple backends
-- Integrations with OpenAI, Groq, Redis, Qdrant, Ollama, FastEmbed-rs, Fluvio, LanceDB, and Treesitter
+- Integrations with OpenAI, Groq, Redis, Qdrant, Ollama, FastEmbed-rs, Fluvio, LanceDB, Mistral-rs, and Treesitter
 - Evaluate pipelines with RAGAS
 - Sparse vector support for hybrid search
 - `tracing` supported for logging and tracing, see /examples and the `tracing` crate for more information.
@@ -183,7 +183,7 @@ Our goal is to create a fast, extendable platform for Retrieval Augmented Genera
 
 | **Feature**                                  | **Details**                                                                                                                                                          |
 | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Supported Large Language Model providers** | OpenAI (and Azure) - All models and embeddings <br> AWS Bedrock - Anthropic and Titan <br> Groq - All models <br> Ollama - All models                                |
+| **Supported Large Language Model providers** | OpenAI (and Azure) - All models and embeddings <br> AWS Bedrock - Anthropic and Titan <br> Groq - All models <br> Ollama - All models <br> Mistral-rs - Text models  |
 | **Loading data**                             | Files <br> Scraping <br> Fluvio <br> Parquet <br> Other pipelines and streams                                                                                        |
 | **Transformers and metadata generation**     | Generate Question and answerers for both text and code (Hyde) <br> Summaries, titles and queries via an LLM <br> Extract definitions and references with tree-sitter |
 | **Splitting and chunking**                   | Markdown <br> Text (text_splitter) <br> Code (with tree-sitter)                                                                                                      |
diff --git a/deny.toml b/deny.toml
index 4ed3d640..310d35d5 100644
--- a/deny.toml
+++ b/deny.toml
@@ -19,7 +19,26 @@ allow = [
   "Zlib",
   "0BSD",
 ]
-exceptions = [{ allow = ["OpenSSL"], crate = "ring" }]
+exceptions = [
+  { allow = [
+    "OpenSSL",
+  ], crate = "ring" },
+  { allow = [
+    "Unicode-3.0",
+  ], crate = "zerofrom" },
+  { allow = [
+    "Unicode-3.0",
+  ], crate = "zerofrom-derive" },
+  { allow = [
+    "Unicode-3.0",
+  ], crate = "yoke" },
+  { allow = [
+    "Unicode-3.0",
+  ], crate = "yoke-derive" },
+  { allow = [
+    "Zlib",
+  ], crate = "foldhash" },
+]
 
 [advisories]
 version = 2
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index b1046027..6bfc6acb 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -21,6 +21,7 @@ swiftide = { path = "../swiftide/", features = [
   "ollama",
   "fluvio",
   "lancedb",
+  "mistralrs",
 ] }
 tracing-subscriber = "0.3"
 serde_json = { workspace = true }
@@ -91,3 +92,7 @@ path = "fluvio.rs"
 [[example]]
 name = "lancedb"
 path = "lancedb.rs"
+
+[[example]]
+name = "mistralrs"
+path = "mistralrs.rs"
diff --git a/examples/mistralrs.rs b/examples/mistralrs.rs
new file mode 100644
index 00000000..d42752c2
--- /dev/null
+++ b/examples/mistralrs.rs
@@ -0,0 +1,34 @@
+use swiftide::{
+    indexing::{
+        self,
+        loaders::FileLoader,
+        persist::MemoryStorage,
+        transformers::{ChunkMarkdown, Embed, MetadataQAText},
+    },
+    integrations,
+};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    tracing_subscriber::fmt::init();
+
+    let embedding_model = integrations::fastembed::FastEmbed::try_default()?;
+
+    let prompt_model = integrations::mistralrs::Mistralrs::builder()
+        .model_name("microsoft/Phi-3.5-mini-instruct")
+        .build()?;
+
+    let memory_storage = MemoryStorage::default();
+
+    indexing::Pipeline::from_loader(FileLoader::new("README.md"))
+        .then_chunk(ChunkMarkdown::from_chunk_range(10..2048))
+        .then(MetadataQAText::new(prompt_model))
+        .then_in_batch(Embed::new(embedding_model))
+        .then_store_with(memory_storage.clone())
+        .run()
+        .await?;
+
+    println!("{:#?}", memory_storage.get_all_values().await);
+
+    Ok(())
+}
diff --git a/swiftide-integrations/Cargo.toml b/swiftide-integrations/Cargo.toml
index e786fffa..bc2cfeaa 100644
--- a/swiftide-integrations/Cargo.toml
+++ b/swiftide-integrations/Cargo.toml
@@ -77,13 +77,14 @@ parquet = { workspace = true, optional = true, features = [
 arrow = { workspace = true, optional = true }
 redb = { workspace = true, optional = true }
 
+mistralrs = { git = "https://github.com/EricLBuehler/mistral.rs.git", rev = "9dfbab1", optional = true }
+
 [dev-dependencies]
 swiftide-core = { path = "../swiftide-core", features = ["test-utils"] }
 swiftide-test-utils = { path = "../swiftide-test-utils", features = [
   "test-utils",
 ] }
 temp-dir = { workspace = true }
-
 arrow = { workspace = true, features = ["test_utils"] }
 
 # Used for hacking fluv to play nice
@@ -137,6 +138,8 @@ fluvio = ["dep:fluvio"]
 parquet = ["dep:arrow-array", "dep:parquet", "dep:arrow"]
 # Redb as an embeddable node cache
 redb = ["dep:redb"]
+# Mistral.rs
+mistralrs = ["dep:mistralrs"]
 
 [lints]
 workspace = true
diff --git a/swiftide-integrations/src/lib.rs b/swiftide-integrations/src/lib.rs
index d1e38a08..9845297e 100644
--- a/swiftide-integrations/src/lib.rs
+++ b/swiftide-integrations/src/lib.rs
@@ -10,6 +10,8 @@ pub mod fluvio;
 pub mod groq;
 #[cfg(feature = "lancedb")]
 pub mod lancedb;
+#[cfg(feature = "mistralrs")]
+pub mod mistralrs;
 #[cfg(feature = "ollama")]
 pub mod ollama;
 #[cfg(feature = "openai")]
diff --git a/swiftide-integrations/src/mistralrs/mod.rs b/swiftide-integrations/src/mistralrs/mod.rs
new file mode 100644
index 00000000..2cca9742
--- /dev/null
+++ b/swiftide-integrations/src/mistralrs/mod.rs
@@ -0,0 +1,9 @@
+//! Use mistral.rs right inside your pipeline
+//!
+//! Mistral.rs supports a variety of hugging face models.
+//!
+//! See [the mistral.rs github page](https://github.com/EricLBuehler/mistral.rs/) for more
+//! information.
+mod simple_prompt;
+
+pub use simple_prompt::{Mistralrs, MistralrsBuilder};
diff --git a/swiftide-integrations/src/mistralrs/simple_prompt.rs b/swiftide-integrations/src/mistralrs/simple_prompt.rs
new file mode 100644
index 00000000..837515a2
--- /dev/null
+++ b/swiftide-integrations/src/mistralrs/simple_prompt.rs
@@ -0,0 +1,110 @@
+use anyhow::Result;
+use std::{
+    fmt::{Debug, Formatter},
+    sync::Arc,
+};
+use swiftide_core::{prompt::Prompt, SimplePrompt};
+
+use async_trait::async_trait;
+use derive_builder::Builder;
+#[allow(unused_imports)]
+pub use mistralrs::{IsqType, PagedAttentionMetaBuilder, TextModelBuilder};
+use mistralrs::{RequestBuilder, TextMessageRole};
+
+/// Prompt popular hugging face text models provided by [`mistralrs`].
+///
+/// You can either provide a model name, or use mistral builder. Using the mistral builder allows
+/// models beyond text.
+///
+/// See [the mistral.rs github page](https://github.com/EricLBuehler/mistral.rs/) for more
+/// information and a list of supported models.
+///
+/// When building the model, when using the default implementation, it will block on the current
+/// runtime to build the model. Ensure that you only do this once.
+///
+///
+/// # Example
+///
+/// ```ignore
+/// let model = MistralTextModel::builder()
+///     .model_name("microsoft/Phi-3.5-mini-instruct")
+///     .build()?;
+///
+/// // ... later in the pipeline
+/// pipeline.then(MetadataQACode::new(model))
+/// ```
+#[derive(Builder, Clone)]
+// #[builder(pattern = "owned")]
+#[builder(setter(into), build_fn(error = "anyhow::Error"))]
+pub struct Mistralrs {
+    /// Internal model used. Can be overwritten via the builder.
+    ///
+    /// See [`::mistralrs::TextModelBuilder`].
+    #[builder(default = "self.default_from_model_name()?")]
+    model: Arc<::mistralrs::v0_4_api::Model>,
+
+    /// Optional model name to build a default model
+    #[builder(default, setter(strip_option))]
+    #[allow(dead_code)]
+    model_name: Option<String>,
+}
+
+impl Mistralrs {
+    pub fn builder() -> MistralrsBuilder {
+        MistralrsBuilder::default()
+    }
+}
+
+impl MistralrsBuilder {
+    fn default_from_model_name(&self) -> Result<Arc<::mistralrs::v0_4_api::Model>> {
+        tokio::task::block_in_place(move || {
+            tokio::runtime::Handle::current()
+                .block_on(async { self.default_from_model_name_async().await })
+        })
+    }
+
+    async fn default_from_model_name_async(&self) -> Result<Arc<::mistralrs::v0_4_api::Model>> {
+        let model_name = self
+            .model_name
+            .clone()
+            .flatten()
+            .ok_or(anyhow::anyhow!("Missing model name"))?;
+
+        tracing::warn!("Setting up model {model_name}");
+
+        let model = Arc::new(
+            TextModelBuilder::new(&model_name)
+                .with_paged_attn(|| PagedAttentionMetaBuilder::default().build())?
+                .build()
+                .await?,
+        );
+
+        tracing::warn!("Set up model {model_name}");
+
+        Ok(model)
+    }
+}
+
+impl Debug for Mistralrs {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("MistralPrompt").finish()
+    }
+}
+
+#[async_trait]
+impl SimplePrompt for Mistralrs {
+    #[tracing::instrument(skip_all)]
+    async fn prompt(&self, prompt: Prompt) -> Result<String> {
+        let request =
+            RequestBuilder::new().add_message(TextMessageRole::User, prompt.render().await?);
+
+        tracing::info!("Sending request to mistral prompt");
+
+        let mut response = self.model.send_chat_request(request).await?;
+        response.choices[0]
+            .message
+            .content
+            .take()
+            .ok_or_else(|| anyhow::anyhow!("No content in response"))
+    }
+}
diff --git a/swiftide-test-utils/Cargo.toml b/swiftide-test-utils/Cargo.toml
index 5df13155..2815a324 100644
--- a/swiftide-test-utils/Cargo.toml
+++ b/swiftide-test-utils/Cargo.toml
@@ -13,7 +13,9 @@ homepage.workspace = true
 
 [dependencies]
 swiftide-core = { path = "../swiftide-core", features = ["test-utils"] }
-swiftide-integrations = { path = "../swiftide-integrations", all-features = true }
+swiftide-integrations = { path = "../swiftide-integrations", features = [
+  "openai",
+] }
 
 async-openai = { workspace = true }
 qdrant-client = { workspace = true, default-features = false, features = [
diff --git a/swiftide/Cargo.toml b/swiftide/Cargo.toml
index 6a38f209..ae868498 100644
--- a/swiftide/Cargo.toml
+++ b/swiftide/Cargo.toml
@@ -80,6 +80,12 @@ parquet = ["swiftide-integrations/parquet"]
 
 ## Redb embeddable nodecache
 redb = ["swiftide-integrations/redb"]
+# In process models with mistral.rs
+#
+# Mistral provides feature flags for metal, cuda, etc.
+# If those are needed, you will need to include mistralrs
+# yourself and add the features.
+mistralrs = ["swiftide-integrations/mistralrs"]
 
 #! ### Other features
 
@@ -106,8 +112,6 @@ insta = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 tokio = { workspace = true }
-lancedb = { workspace = true }
-arrow-array = { workspace = true }
 
 [lints]
 workspace = true
diff --git a/swiftide/tests/lancedb.rs b/swiftide/tests/lancedb.rs
index 5464b5fd..5202c5d0 100644
--- a/swiftide/tests/lancedb.rs
+++ b/swiftide/tests/lancedb.rs
@@ -1,16 +1,9 @@
-use arrow_array::{cast::AsArray, Array, RecordBatch, StringArray};
-use lancedb::query::ExecutableQuery;
 use swiftide::indexing;
-use swiftide::query::{self, states, Query, TransformationEvent};
-use swiftide::{
-    indexing::{
-        transformers::{
-            metadata_qa_code::NAME as METADATA_QA_CODE_NAME, ChunkCode, MetadataQACode,
-        },
-        EmbeddedField,
-    },
-    query::TryStreamExt as _,
+use swiftide::indexing::{
+    transformers::{metadata_qa_code::NAME as METADATA_QA_CODE_NAME, ChunkCode, MetadataQACode},
+    EmbeddedField,
 };
+use swiftide::query::{self, states, Query, TransformationEvent};
 use swiftide_indexing::{loaders, transformers, Pipeline};
 use swiftide_integrations::{fastembed::FastEmbed, lancedb::LanceDB};
 use swiftide_query::{answers, query_transformers, response_transformers};
@@ -95,55 +88,4 @@ async fn test_lancedb() {
         documents.first().unwrap(),
         "fn main() { println!(\"Hello, World!\"); }"
     );
-
-    // Manually assert everything was stored as expected
-    let conn = lancedb.get_connection().await.unwrap();
-    let table = conn.open_table("swiftide_test").execute().await.unwrap();
-
-    let result: RecordBatch = table
-        .query()
-        .execute()
-        .await
-        .unwrap()
-        .try_collect::<Vec<_>>()
-        .await
-        .unwrap()
-        .first()
-        .unwrap()
-        .clone();
-
-    assert_eq!(result.num_rows(), 1);
-    assert_eq!(result.num_columns(), 5);
-    dbg!(result.columns());
-    assert!(result.column_by_name("id").is_some());
-    assert_eq!(
-        result
-            .column_by_name("chunk")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<StringArray>() // as_string() doesn't work, wtf
-            .unwrap()
-            .value(0),
-        code
-    );
-    assert_eq!(
-        result
-            .column_by_name("questions_and_answers__code_")
-            .unwrap()
-            .as_any()
-            .downcast_ref::<StringArray>() // as_string() doesn't work, wtf
-            .unwrap()
-            .value(0),
-        "\n\nHello there, how may I assist you today?"
-    );
-
-    assert_eq!(
-        result
-            .column_by_name("vector_combined")
-            .unwrap()
-            .as_fixed_size_list()
-            .value(0)
-            .len(),
-        384
-    );
 }