-
Notifications
You must be signed in to change notification settings - Fork 28
/
Cargo.toml
55 lines (52 loc) · 2.62 KB
/
Cargo.toml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
[package]
name = "candle-vllm"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
axum = { version = "0.7.4", features = ["tokio"] }
utoipa = { version = "4.2", features = ["axum_extras"] }
tower-http = { version = "0.5.1", features = ["cors"]}
flume = "0.10.14"
#actix-web = "4.8.0"
anyhow = "1.0.75"
rand = "0.8.5"
rayon="1.10.0"
hyper = { version = "0.14", features = ["full"] }
candle-core = { git = "https://github.com/huggingface/candle.git", version = "0.6.0" }
candle-examples = { git = "https://github.com/huggingface/candle.git", version = "0.6.0" }
#candle-lora = { git = "https://github.com/EricLBuehler/candle-lora.git", version = "0.2.0" }
#candle-lora-macro = { git = "https://github.com/EricLBuehler/candle-lora.git", version = "0.2.0" }
#candle-lora-transformers = { git = "https://github.com/EricLBuehler/candle-lora.git", version = "0.2.0" }
candle-nn = { git = "https://github.com/huggingface/candle.git", version = "0.6.0" }
dyn-fmt = "0.4.0"
serde = { version = "1.0.190", features = ["serde_derive"] }
tokenizers = "0.19.1"
uuid = { version = "1.5.0", features = ["v4"] }
candle-transformers = { git = "https://github.com/huggingface/candle.git", version = "0.6.0" }
hf-hub = "0.3.2"
serde_json = "1.0.108"
derive_more = "0.99.17"
accelerate-src = { version = "0.3.2", optional = true }
intel-mkl-src = { version = "0.8.1", features = ["mkl-static-lp64-iomp"], optional = true }
cudarc = { version = "0.9.14", features = ["f16"], optional = true }
half = { version = "2.3.1", features = ["num-traits", "use-intrinsics", "rand_distr"] }
candle-flash-attn = { git = "https://github.com/huggingface/candle.git", version = "0.6.0", optional = true }
clap = { version = "4.4.7", features = ["derive"] }
#candle-sampling = { git = "https://github.com/EricLBuehler/candle-sampling.git", version = "0.2.0" }
futures = "0.3.29"
tokio = { version = "1.38.0", features = ["sync"] }
env_logger = "0.10.1"
tracing = "0.1.40"
range-checked = { git = "https://github.com/EricLBuehler/range-checked.git", version = "0.1.0" }
either = { version = "1.13.0", features = ["serde"] }
dirs = "5.0.1"
kernels = {path = "./kernels", version="0.1.0"}
[features]
default = ["cuda"]
accelerate = ["dep:accelerate-src", "candle-core/accelerate", "candle-nn/accelerate", "candle-transformers/accelerate"]
cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"]
cudnn = ["candle-core/cudnn"]
flash-attn = ["cuda", "candle-transformers/flash-attn"]
mkl = ["dep:intel-mkl-src", "candle-core/mkl", "candle-nn/mkl", "candle-transformers/mkl"]
nccl = ["cuda", "cudarc/nccl"]