diff --git a/README.md b/README.md index 2c228618..86bb438f 100644 --- a/README.md +++ b/README.md @@ -64,20 +64,22 @@ Ember, GTE and E5. TEI implements many features such as: #### Text Embeddings Text Embeddings Inference currently supports Nomic, BERT, CamemBERT, XLM-RoBERTa models with absolute positions, JinaBERT -model with Alibi positions and Mistral, Alibabe GTE models with Rope positions. +model with Alibi positions and Mistral, Alibaba GTE and Qwen2 models with Rope positions. Below are some examples of the currently supported models: -| MTEB Rank | Model Size | Model Type | Model ID | -|-----------|----------------|-------------|--------------------------------------------------------------------------------------------------| -| 1 | 7B (Very Slow) | Mistral | [Salesforce/SFR-Embedding-2_R](https://hf.co/Salesforce/SFR-Embedding-2_R) | -| 15 | 0.4B | Alibaba GTE | [Alibaba-NLP/gte-large-en-v1.5](Alibaba-NLP/gte-large-en-v1.5) | -| 20 | 0.3B | Bert | [WhereIsAI/UAE-Large-V1](https://hf.co/WhereIsAI/UAE-Large-V1) | -| 24 | 0.5B | XLM-RoBERTa | [intfloat/multilingual-e5-large-instruct](https://hf.co/intfloat/multilingual-e5-large-instruct) | -| N/A | 0.1B | NomicBert | [nomic-ai/nomic-embed-text-v1](https://hf.co/nomic-ai/nomic-embed-text-v1) | -| N/A | 0.1B | NomicBert | [nomic-ai/nomic-embed-text-v1.5](https://hf.co/nomic-ai/nomic-embed-text-v1.5) | -| N/A | 0.1B | JinaBERT | [jinaai/jina-embeddings-v2-base-en](https://hf.co/jinaai/jina-embeddings-v2-base-en) | -| N/A | 0.1B | JinaBERT | [jinaai/jina-embeddings-v2-base-code](https://hf.co/jinaai/jina-embeddings-v2-base-code) | +| MTEB Rank | Model Size | Model Type | Model ID | +|-----------|---------------------|-------------|--------------------------------------------------------------------------------------------------| +| 1 | 7B (Very Expensive) | Mistral | [Salesforce/SFR-Embedding-2_R](https://hf.co/Salesforce/SFR-Embedding-2_R) | +| 2 | 7B (Very Expensive) | Qwen2 | [Alibaba-NLP/gte-Qwen2-7B-instruct](https://hf.co/Alibaba-NLP/gte-Qwen2-7B-instruct) | +| 9 | 1.5B (Expensive) | Qwen2 | [Alibaba-NLP/gte-Qwen2-1.5B-instruct](https://hf.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct) | +| 15 | 0.4B | Alibaba GTE | [Alibaba-NLP/gte-large-en-v1.5](Alibaba-NLP/gte-large-en-v1.5) | +| 20 | 0.3B | Bert | [WhereIsAI/UAE-Large-V1](https://hf.co/WhereIsAI/UAE-Large-V1) | +| 24 | 0.5B | XLM-RoBERTa | [intfloat/multilingual-e5-large-instruct](https://hf.co/intfloat/multilingual-e5-large-instruct) | +| N/A | 0.1B | NomicBert | [nomic-ai/nomic-embed-text-v1](https://hf.co/nomic-ai/nomic-embed-text-v1) | +| N/A | 0.1B | NomicBert | [nomic-ai/nomic-embed-text-v1.5](https://hf.co/nomic-ai/nomic-embed-text-v1.5) | +| N/A | 0.1B | JinaBERT | [jinaai/jina-embeddings-v2-base-en](https://hf.co/jinaai/jina-embeddings-v2-base-en) | +| N/A | 0.1B | JinaBERT | [jinaai/jina-embeddings-v2-base-code](https://hf.co/jinaai/jina-embeddings-v2-base-code) | To explore the list of best performing text embeddings models, visit the diff --git a/backends/candle/src/lib.rs b/backends/candle/src/lib.rs index 5204536b..acd93750 100644 --- a/backends/candle/src/lib.rs +++ b/backends/candle/src/lib.rs @@ -12,12 +12,12 @@ use crate::compute_cap::{ }; use crate::models::{ BertConfig, BertModel, DistilBertConfig, DistilBertModel, GTEConfig, JinaBertModel, - JinaCodeBertModel, MistralConfig, Model, NomicBertModel, NomicConfig, + JinaCodeBertModel, MistralConfig, Model, NomicBertModel, NomicConfig, Qwen2Config, }; #[cfg(feature = "cuda")] use crate::models::{ FlashBertModel, FlashDistilBertModel, FlashGTEModel, FlashJinaBertModel, - FlashJinaCodeBertModel, FlashMistralModel, FlashNomicBertModel, + FlashJinaCodeBertModel, FlashMistralModel, FlashNomicBertModel, FlashQwen2Model, }; use anyhow::Context; use candle::{DType, Device}; @@ -59,6 +59,7 @@ enum Config { Mistral(MistralConfig), #[serde(rename = "new")] Gte(GTEConfig), + Qwen2(Qwen2Config), } pub struct CandleBackend { @@ -221,6 +222,10 @@ impl CandleBackend { "GTE is only supported on Cuda devices in fp16 with flash attention enabled" .to_string(), )), + (Config::Qwen2(_), Device::Cpu | Device::Metal(_)) => Err(BackendError::Start( + "Qwen2 is only supported on Cuda devices in fp16 with flash attention enabled" + .to_string(), + )), #[cfg(feature = "cuda")] (Config::Bert(config), Device::Cuda(_)) => { if cfg!(any(feature = "flash-attn", feature = "flash-attn-v1")) @@ -342,14 +347,25 @@ impl CandleBackend { #[cfg(feature = "cuda")] (Config::Gte(config), Device::Cuda(_)) => { if dtype != DType::F16 - || !cfg!(feature = "flash-attn") - || get_runtime_compute_cap().unwrap() < 80 + || !cfg!(any(feature = "flash-attn", feature = "flash-attn-v1")) { - return Err(BackendError::Start("GTE is only supported on Cuda devices in fp16 with flash attention v2 enabled".to_string())); + return Err(BackendError::Start("GTE is only supported on Cuda devices in fp16 with flash attention enabled".to_string())); } tracing::info!("Starting FlashGTE model on {:?}", device); Ok(Box::new(FlashGTEModel::load(vb, &config, model_type).s()?)) } + #[cfg(feature = "cuda")] + (Config::Qwen2(config), Device::Cuda(_)) => { + if dtype != DType::F16 + || !cfg!(any(feature = "flash-attn", feature = "flash-attn-v1")) + { + return Err(BackendError::Start("Qwen2 is only supported on Cuda devices in fp16 with flash attention v2 enabled".to_string())); + } + tracing::info!("Starting FlashQwen2 model on {:?}", device); + Ok(Box::new( + FlashQwen2Model::load(vb, &config, model_type).s()?, + )) + } }; Ok(Self { diff --git a/backends/candle/src/models/flash_bert.rs b/backends/candle/src/models/flash_bert.rs index 8f20f027..8951dcc1 100644 --- a/backends/candle/src/models/flash_bert.rs +++ b/backends/candle/src/models/flash_bert.rs @@ -400,7 +400,10 @@ impl FlashBertModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_distilbert.rs b/backends/candle/src/models/flash_distilbert.rs index f8dd294b..7f060601 100644 --- a/backends/candle/src/models/flash_distilbert.rs +++ b/backends/candle/src/models/flash_distilbert.rs @@ -266,7 +266,10 @@ impl FlashDistilBertModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_gte.rs b/backends/candle/src/models/flash_gte.rs index 62bf65bd..f3aec220 100644 --- a/backends/candle/src/models/flash_gte.rs +++ b/backends/candle/src/models/flash_gte.rs @@ -352,7 +352,10 @@ impl FlashGTEModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_jina.rs b/backends/candle/src/models/flash_jina.rs index 0bab863f..83c5b0a4 100644 --- a/backends/candle/src/models/flash_jina.rs +++ b/backends/candle/src/models/flash_jina.rs @@ -327,7 +327,10 @@ impl FlashJinaBertModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_jina_code.rs b/backends/candle/src/models/flash_jina_code.rs index b20438fe..56ab1976 100644 --- a/backends/candle/src/models/flash_jina_code.rs +++ b/backends/candle/src/models/flash_jina_code.rs @@ -380,7 +380,10 @@ impl FlashJinaCodeBertModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_mistral.rs b/backends/candle/src/models/flash_mistral.rs index 53e5d3c4..eb94c913 100644 --- a/backends/candle/src/models/flash_mistral.rs +++ b/backends/candle/src/models/flash_mistral.rs @@ -334,7 +334,10 @@ impl FlashMistralModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_nomic.rs b/backends/candle/src/models/flash_nomic.rs index 8ad1ab89..7f558b9e 100644 --- a/backends/candle/src/models/flash_nomic.rs +++ b/backends/candle/src/models/flash_nomic.rs @@ -311,7 +311,10 @@ impl FlashNomicBertModel { // Get token indices form cu_seqlens let mut indices = match self.pool { Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, - Pool::LastToken => cu_seqlens.narrow(0, 1, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } _ => unreachable!(), }; diff --git a/backends/candle/src/models/flash_qwen2.rs b/backends/candle/src/models/flash_qwen2.rs new file mode 100644 index 00000000..4dba7868 --- /dev/null +++ b/backends/candle/src/models/flash_qwen2.rs @@ -0,0 +1,455 @@ +use crate::flash_attn::flash_attn_varlen; +use crate::layers::{HiddenAct, Linear, RMSNorm}; +use crate::models::{Model, Qwen2Config}; +use candle::{DType, Device, IndexOp, Result, Tensor}; +use candle_nn::{Embedding, Module, VarBuilder}; +use text_embeddings_backend_core::{Batch, ModelType, Pool}; + +struct Qwen2Attention { + qkv_linear: Linear, + o_proj: Linear, + + num_attention_heads: usize, + num_key_value_heads: usize, + attention_head_size: usize, + + softmax_scale: f32, + + span: tracing::Span, +} + +impl Qwen2Attention { + pub fn load(vb: VarBuilder, config: &Qwen2Config) -> Result { + if config.use_sliding_window { + candle::bail!("Sliding window is not supported"); + } + + let num_attention_heads = config.num_attention_heads; + let attention_head_size = config.hidden_size / config.num_attention_heads; + let num_key_value_heads = config.num_key_value_heads; + let hidden_size = config.hidden_size; + + let query_weight = vb.pp("q_proj").get((hidden_size, hidden_size), "weight")?; + let query_bias = vb.pp("q_proj").get(hidden_size, "bias")?; + + let key_weight = vb.pp("k_proj").get( + (num_key_value_heads * attention_head_size, hidden_size), + "weight", + )?; + let key_bias = vb + .pp("k_proj") + .get(num_key_value_heads * attention_head_size, "bias")?; + + let value_weight = vb.pp("v_proj").get( + (num_key_value_heads * attention_head_size, hidden_size), + "weight", + )?; + let value_bias = vb + .pp("v_proj") + .get(num_key_value_heads * attention_head_size, "bias")?; + + let qkv_weight = Tensor::cat(&[&query_weight, &key_weight, &value_weight], 0)?; + let qkv_bias = Tensor::cat(&[&query_bias, &key_bias, &value_bias], 0)?; + let qkv_linear = Linear::new(qkv_weight, Some(qkv_bias), None); + + let o_proj_weight = vb.pp("o_proj").get((hidden_size, hidden_size), "weight")?; + + let o_proj = Linear::new(o_proj_weight, None, None); + + let softmax_scale = (1. / (attention_head_size as f64).sqrt()) as f32; + + Ok(Self { + qkv_linear, + o_proj, + num_attention_heads, + num_key_value_heads, + attention_head_size, + softmax_scale, + span: tracing::span!(tracing::Level::TRACE, "attention"), + }) + } + + pub fn forward( + &self, + hidden_states: &Tensor, + cu_seqlens: &Tensor, + cos: &Tensor, + sin: &Tensor, + max_s: usize, + ) -> Result { + let _enter = self.span.enter(); + + let qkv = self.qkv_linear.forward(hidden_states)?; + + // Reshape to [tokens, heads, head_size] + let mut new_qkv_shape = qkv.dims().to_vec(); + new_qkv_shape.pop(); + new_qkv_shape.push(self.num_attention_heads + 2 * self.num_key_value_heads); + new_qkv_shape.push(self.attention_head_size); + + let qkv = qkv.reshape(new_qkv_shape)?; + + // Split qkv tensor + let q = qkv.narrow(1, 0, self.num_attention_heads)?; + let k = qkv.narrow(1, self.num_attention_heads, self.num_key_value_heads)?; + let v = qkv.narrow( + 1, + self.num_attention_heads + self.num_key_value_heads, + self.num_key_value_heads, + )?; + + candle_rotary::apply_rotary_inplace(&q, &k, &cos, &sin, true)?; + + let attention = flash_attn_varlen( + &q, + &k, + &v, + None, + cu_seqlens, + cu_seqlens, + max_s, + max_s, + self.softmax_scale, + false, + None, + )?; + let attention = attention.flatten_from(candle::D::Minus2)?; + + self.o_proj.forward(&attention) + } +} + +struct Qwen2MLP { + gate_up_proj: Linear, + down_proj: Linear, + + act: HiddenAct, + intermediate_size: usize, + + span: tracing::Span, +} + +impl Qwen2MLP { + pub fn load(vb: VarBuilder, config: &Qwen2Config) -> Result { + let intermediate_size = config.intermediate_size; + + let gate_proj_weight = vb + .pp("gate_proj") + .get((intermediate_size, config.hidden_size), "weight")?; + + let up_proj_weight = vb + .pp("up_proj") + .get((intermediate_size, config.hidden_size), "weight")?; + + let gate_up_proj_weight = Tensor::cat(&[&gate_proj_weight, &up_proj_weight], 0)?; + let gate_up_proj = Linear::new(gate_up_proj_weight, None, None); + + let down_proj_weight = vb + .pp("down_proj") + .get((config.hidden_size, intermediate_size), "weight")?; + let down_proj = Linear::new(down_proj_weight, None, None); + + Ok(Self { + gate_up_proj, + down_proj, + intermediate_size, + act: config.hidden_act.clone(), + span: tracing::span!(tracing::Level::TRACE, "mlp"), + }) + } + + pub fn forward(&self, hidden_states: &Tensor) -> Result { + let _enter = self.span.enter(); + + let gate_up_states = self.gate_up_proj.forward(hidden_states)?; + let gate_states = gate_up_states.narrow(1, 0, self.intermediate_size)?; + let up_states = gate_up_states.narrow(1, self.intermediate_size, self.intermediate_size)?; + + let gate_states = match self.act { + HiddenAct::Gelu => gate_states.gelu(), + HiddenAct::Relu => gate_states.relu(), + HiddenAct::Swiglu => gate_states.silu(), + }?; + let r = self.down_proj.forward(&(gate_states * up_states)?); + r + } +} + +struct Qwen2Layer { + attention: Qwen2Attention, + mlp: Qwen2MLP, + input_layer_norm: RMSNorm, + post_attention_layer_norm: RMSNorm, + + span: tracing::Span, +} + +impl Qwen2Layer { + pub fn load(vb: VarBuilder, config: &Qwen2Config) -> Result { + let attention = Qwen2Attention::load(vb.pp("self_attn"), config)?; + let mlp = Qwen2MLP::load(vb.pp("mlp"), config)?; + + let input_layer_norm = RMSNorm::load( + vb.pp("input_layernorm"), + config.hidden_size, + config.rms_norm_eps, + )?; + let post_attention_layer_norm = RMSNorm::load( + vb.pp("post_attention_layernorm"), + config.hidden_size, + config.rms_norm_eps, + )?; + + Ok(Self { + attention, + mlp, + input_layer_norm, + post_attention_layer_norm, + span: tracing::span!(tracing::Level::TRACE, "layer"), + }) + } + + pub fn forward( + &self, + hidden_states: &Tensor, + residual: Option<&Tensor>, + cu_seqlens: &Tensor, + cos: &Tensor, + sin: &Tensor, + max_s: usize, + ) -> Result<(Tensor, Tensor)> { + let _enter = self.span.enter(); + + let (normed_hidden_states, res) = self.input_layer_norm.forward(hidden_states, residual)?; + let attn_output = + self.attention + .forward(&normed_hidden_states, cu_seqlens, cos, sin, max_s)?; + let (normed_attn_res_output, attn_res) = self + .post_attention_layer_norm + .forward(&attn_output, Some(&res))?; + let mlp_output = self.mlp.forward(&normed_attn_res_output)?; + + Ok((mlp_output, attn_res)) + } +} + +pub struct FlashQwen2Model { + embeddings: Embedding, + layers: Vec, + norm: RMSNorm, + cos_cache: Tensor, + sin_cache: Tensor, + pool: Pool, + pub device: Device, + + span: tracing::Span, +} + +impl FlashQwen2Model { + pub fn load(vb: VarBuilder, config: &Qwen2Config, model_type: ModelType) -> Result { + match vb.device() { + Device::Cuda(_) => {} + _ => candle::bail!("FlashQwen2 requires Cuda"), + } + + if vb.dtype() != DType::F16 { + candle::bail!("FlashQwen2 requires DType::F16") + } + + let pool = match model_type { + ModelType::Classifier => { + candle::bail!("`classifier` model type is not supported for Qwen2") + } + ModelType::Embedding(pool) => pool, + }; + + let vb = vb.pp("model"); + + let embeddings = Embedding::new( + vb.pp("embed_tokens") + .get((config.vocab_size, config.hidden_size), "weight")?, + config.hidden_size, + ); + + let layers = (0..config.num_hidden_layers) + .map(|index| Qwen2Layer::load(vb.pp(format!("layers.{index}")), config)) + .collect::>>()?; + + let norm = RMSNorm::load(vb.pp("norm"), config.hidden_size, config.rms_norm_eps)?; + + let inv_freqs = candle_rotary::inv_freqs( + layers[0].attention.attention_head_size, + config.rope_theta, + vb.device(), + )?; + let (cos_cache, sin_cache) = + candle_rotary::cos_sin(config.max_position_embeddings, &inv_freqs, vb.dtype())?; + + Ok(Self { + embeddings, + layers, + norm, + cos_cache, + sin_cache, + pool, + device: vb.device().clone(), + span: tracing::span!(tracing::Level::TRACE, "model"), + }) + } + + pub fn forward(&self, batch: Batch) -> Result<(Option, Option)> { + let _enter = self.span.enter(); + + let batch_size = batch.cumulative_seq_lengths.len() - 1; + let shape = batch.input_ids.len(); + + // Create Cuda tensors + let input_ids = Tensor::from_vec(batch.input_ids, shape, &self.device)?; + let position_ids = Tensor::from_vec(batch.position_ids, shape, &self.device)?; + let cu_seqlens = Tensor::from_vec( + batch.cumulative_seq_lengths.clone(), + batch_size + 1, + &self.device, + )?; + + let mut hidden_states = self.embeddings.forward(&input_ids)?; + + let cos = self.cos_cache.index_select(&position_ids, 0)?; + let sin = self.sin_cache.index_select(&position_ids, 0)?; + + let mut residual = None; + for layer in &self.layers { + let (h, r) = layer.forward( + &hidden_states, + residual.as_ref(), + &cu_seqlens, + &cos, + &sin, + batch.max_length as usize, + )?; + hidden_states = h; + residual = Some(r); + } + + let (outputs, _) = self.norm.forward(&hidden_states, residual.as_ref())?; + + let has_pooling_requests = !batch.pooled_indices.is_empty(); + let has_raw_requests = !batch.raw_indices.is_empty(); + + let pooled_embeddings = if has_pooling_requests { + match self.pool { + // CLS and LastToken pooling + Pool::Cls | Pool::LastToken => { + if batch_size > 1 { + // Get token indices form cu_seqlens + let mut indices = match self.pool { + Pool::Cls => cu_seqlens.narrow(0, 0, batch_size)?, + Pool::LastToken => { + let end = cu_seqlens.narrow(0, 1, batch_size)?; + (&end - &end.ones_like()?)? + } + _ => unreachable!(), + }; + + // If raw_indices is empty, we don't need to do anything with + // the pooled_indices + if has_raw_requests { + // We need the pooled indices to select the correct cls indices + let pooled_indices = Tensor::from_vec( + batch.pooled_indices.clone(), + batch.pooled_indices.len(), + &self.device, + )?; + + // Only select indices that requires pooling + indices = indices.index_select(&pooled_indices, 0)? + } + + // Select tokens + Some(outputs.index_select(&indices, 0)?) + } else { + Some( + match self.pool { + Pool::Cls => outputs.i(0)?, + Pool::LastToken => { + outputs.i(batch.cumulative_seq_lengths[1] as usize - 1)? + } + _ => unreachable!(), + } + .unsqueeze(0)?, + ) + } + } + // Mean pooling + Pool::Mean => { + if batch_size > 1 { + // for each request that requires pooling + let results: Result> = batch + .pooled_indices + .into_iter() + .map(|i| { + let i = i as usize; + let start = batch.cumulative_seq_lengths[i]; + let len = batch.cumulative_seq_lengths[i + 1] - start; + + // Mean + let embeddings = outputs.narrow(0, start as usize, len as usize)?; + embeddings.sum_keepdim(0)? / (len as f64) + }) + .collect(); + + // Concatenate all results + Some(Tensor::cat(&results?, 0)?) + } else { + Some((outputs.sum_keepdim(0)? / (batch.max_length as f64))?) + } + } + Pool::Splade => { + unreachable!(); + } + } + } else { + None + }; + + let raw_embeddings = if has_raw_requests { + if batch_size > 1 && has_pooling_requests { + // Create indexing vector for the embeddings + let mut final_indices: Vec = Vec::with_capacity(shape); + for i in batch.raw_indices.into_iter() { + let i = i as usize; + // Get start/end token index of this specific member of the batch + let start = batch.cumulative_seq_lengths[i]; + let end = batch.cumulative_seq_lengths[i + 1]; + + for j in start..end { + // Add indices for the tokens of this specific member of the batch + final_indices.push(j); + } + } + + let final_indices_length = final_indices.len(); + let final_indices = + Tensor::from_vec(final_indices, final_indices_length, &self.device)?; + + // Select the tokens with final indices + Some(outputs.index_select(&final_indices, 0)?) + } else { + Some(outputs) + } + } else { + None + }; + + Ok((pooled_embeddings, raw_embeddings)) + } +} + +impl Model for FlashQwen2Model { + fn is_padded(&self) -> bool { + false + } + fn embed(&self, batch: Batch) -> Result<(Option, Option)> { + self.forward(batch) + } +} diff --git a/backends/candle/src/models/mod.rs b/backends/candle/src/models/mod.rs index 9f616a2a..b1e9f937 100644 --- a/backends/candle/src/models/mod.rs +++ b/backends/candle/src/models/mod.rs @@ -30,7 +30,11 @@ mod flash_distilbert; mod flash_gte; #[cfg(feature = "cuda")] mod flash_mistral; + +#[cfg(feature = "cuda")] +mod flash_qwen2; mod gte; +mod qwen2; pub use bert::{BertConfig, BertModel, PositionEmbeddingType}; use candle::{Result, Tensor}; @@ -41,6 +45,7 @@ pub use jina::JinaBertModel; pub use jina_code::JinaCodeBertModel; pub use mistral::MistralConfig; pub use nomic::{NomicBertModel, NomicConfig}; +pub use qwen2::Qwen2Config; use text_embeddings_backend_core::Batch; #[cfg(feature = "cuda")] @@ -64,6 +69,9 @@ pub use flash_mistral::FlashMistralModel; #[cfg(feature = "cuda")] pub use flash_gte::FlashGTEModel; +#[cfg(feature = "cuda")] +pub use flash_qwen2::FlashQwen2Model; + pub(crate) trait Model { fn is_padded(&self) -> bool; diff --git a/backends/candle/src/models/qwen2.rs b/backends/candle/src/models/qwen2.rs new file mode 100644 index 00000000..8d2d1f86 --- /dev/null +++ b/backends/candle/src/models/qwen2.rs @@ -0,0 +1,18 @@ +use crate::layers::HiddenAct; +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct Qwen2Config { + pub vocab_size: usize, + pub hidden_size: usize, + pub intermediate_size: usize, + pub num_hidden_layers: usize, + pub num_attention_heads: usize, + pub num_key_value_heads: usize, + pub hidden_act: HiddenAct, + pub max_position_embeddings: usize, + pub rms_norm_eps: f32, + pub rope_theta: f32, + pub sliding_window: usize, + pub use_sliding_window: bool, +} diff --git a/backends/candle/tests/common.rs b/backends/candle/tests/common.rs index a3d74d16..0c069a47 100644 --- a/backends/candle/tests/common.rs +++ b/backends/candle/tests/common.rs @@ -107,7 +107,11 @@ pub fn download_artifacts( model_id: &'static str, revision: Option<&'static str>, ) -> Result { - let builder = ApiBuilder::new().with_progress(false); + let mut builder = ApiBuilder::new().with_progress(false); + + if let Some(cache_dir) = std::env::var_os("HUGGINGFACE_HUB_CACHE") { + builder = builder.with_cache_dir(cache_dir.into()); + } let api = builder.build().unwrap(); let api_repo = if let Some(revision) = revision { diff --git a/backends/candle/tests/snapshots/test_flash_qwen2__qwen2_batch.snap b/backends/candle/tests/snapshots/test_flash_qwen2__qwen2_batch.snap new file mode 100644 index 00000000..7a252f73 --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_qwen2__qwen2_batch.snap @@ -0,0 +1,4613 @@ +--- +source: backends/candle/tests/test_flash_qwen2.rs +assertion_line: 61 +expression: embeddings_batch +--- +- - -4.9882813 + - 6.7734375 + - -1.6269531 + - -3.21875 + - 7.078125 + - -0.5107422 + - -2.75 + - -1.5371094 + - 4.9257813 + - 0.9790039 + - 6.03125 + - -4.0195313 + - -4.9648438 + - 0.9160156 + - 0.63916016 + - -0.90771484 + - 5.375 + - -0.3293457 + - 2.8535156 + - 2.7773438 + - 0.25170898 + - -4.671875 + - 1.7490234 + - -1.7080078 + - 2.1933594 + - -0.7470703 + - -5.6445313 + - 0.31762695 + - 6.4375 + - 1.9873047 + - -5.265625 + - 0.6816406 + - 0.89404297 + - 12.1640625 + - -3.3867188 + - -5.015625 + - -2.890625 + - 1.3457031 + - -0.47094727 + - -1.1201172 + - 1.2382813 + - -0.6855469 + - -1.5957031 + - -0.39916992 + - 10.4921875 + - 5.3867188 + - -7.9570313 + - -5.6914063 + - 2.9589844 + - -2.2753906 + - 4.1210938 + - -0.3178711 + - -1.15625 + - 5.828125 + - 3.4746094 + - -0.51953125 + - 1.9931641 + - 1.15625 + - -3.9140625 + - -0.30517578 + - 4.390625 + - 0.96191406 + - -5.0078125 + - 5.8242188 + - 1.9091797 + - 2.2949219 + - -3.2460938 + - -3.9570313 + - 5.3789063 + - 6.4648438 + - 6.9375 + - 7.6210938 + - 0.76464844 + - -0.41430664 + - -3.7480469 + - 3.1015625 + - -0.22961426 + - -3.3007813 + - -5.2890625 + - -8.6953125 + - 2.3320313 + - -0.9013672 + - -5.3789063 + - -10.9453125 + - -0.20263672 + - -3.359375 + - -6.1367188 + - 2.6777344 + - -2.7753906 + - 2.0957031 + - -2.3105469 + - -2.7382813 + - -17.140625 + - 5.2070313 + - 3.4707031 + - -2.0371094 + - 0.82421875 + - -3.5566406 + - 2.1445313 + - -4.8164063 + - -1.2802734 + - 0.42407227 + - 0.98095703 + - 1.3271484 + - -1.9667969 + - 0.36328125 + - -3.90625 + - 5.0859375 + - -1.7578125 + - -1.8056641 + - -5.6210938 + - -2.875 + - -2.9453125 + - -1.5351563 + - 3.1445313 + - 3.8066406 + - 0.78759766 + - 2.8574219 + - 3.6015625 + - 10.6171875 + - -0.24499512 + - -0.52734375 + - -0.61816406 + - -2.9453125 + - -1.0263672 + - 3.359375 + - 6.6640625 + - -5.4179688 + - 0.9326172 + - 5.09375 + - -3.796875 + - 2.3691406 + - -3.9375 + - 6.0117188 + - 1.1669922 + - 0.07067871 + - -0.63916016 + - -7.7304688 + - -0.11804199 + - -5.1992188 + - -0.9707031 + - 2.8730469 + - 0.36157227 + - 4.0898438 + - -3.59375 + - -0.14904785 + - 6.3164063 + - 1.0546875 + - 12.78125 + - 5.265625 + - 2.4238281 + - -0.40429688 + - 2.5800781 + - -2.7871094 + - 0.19958496 + - -0.4580078 + - -4.5585938 + - -7.4570313 + - 0.7397461 + - -2.4921875 + - 2.3066406 + - 6.7929688 + - 5.4023438 + - -9.0859375 + - -0.38134766 + - 5.0117188 + - 3.9003906 + - 4.3515625 + - -3.7050781 + - 5.2226563 + - -4.4726563 + - -3.5234375 + - -3.5878906 + - 3.2519531 + - -4.1484375 + - 0.6220703 + - 0.796875 + - -0.8642578 + - -4.4023438 + - 7.171875 + - -2.1464844 + - -5.0898438 + - -3.7910156 + - 9.359375 + - 1.7636719 + - 2.4023438 + - -3.3632813 + - -0.3083496 + - 1.5644531 + - 8.6015625 + - 1.6787109 + - 3.1347656 + - 0.828125 + - -1.140625 + - 7.734375 + - 1.7255859 + - 0.40942383 + - -6.578125 + - 4.8867188 + - 8.0625 + - 3.3789063 + - 4.3359375 + - 1.0917969 + - -7.2265625 + - 11.5078125 + - 7.2929688 + - -0.17590332 + - -12.3046875 + - -1.6904297 + - -1.0439453 + - -6.5117188 + - -1.4707031 + - 3.1171875 + - -6.1835938 + - -0.48608398 + - 0.20043945 + - -2.1621094 + - -14.5859375 + - -2.4316406 + - -7.3242188 + - 2.8027344 + - -19.578125 + - 1.2392578 + - -0.20324707 + - -1.3583984 + - -3.8515625 + - 2.2460938 + - -0.011695862 + - -3.9257813 + - 5.6132813 + - 7.421875 + - -2.6035156 + - 1.0546875 + - -3.15625 + - -0.39282227 + - -1.7597656 + - -10.796875 + - -8.1875 + - -0.53271484 + - 1.5058594 + - 1.9404297 + - -0.18969727 + - 0.14782715 + - 1.4873047 + - -1.9814453 + - 2.9199219 + - -5.5859375 + - 0.45458984 + - -0.087768555 + - -1.2929688 + - -6.359375 + - -3.5957031 + - -3.4257813 + - -1.75 + - 6.2070313 + - 5.6796875 + - 2.3300781 + - 2.0546875 + - -0.85839844 + - 6.0039063 + - 6.4101563 + - 1.9902344 + - 0.55322266 + - -1.5361328 + - 0.30200195 + - 6.3828125 + - 4.6679688 + - 3.1835938 + - -10.171875 + - -1.8203125 + - -4.0351563 + - -5.3164063 + - 0.97802734 + - -3.1679688 + - 8.0703125 + - 0.1373291 + - 1.2431641 + - -0.47998047 + - 3.6601563 + - -6.7265625 + - -1.1728516 + - -1.5908203 + - 0.23620605 + - 5.5898438 + - 4.5234375 + - -5.7226563 + - -4.40625 + - 5.0234375 + - 1.5634766 + - 0.6376953 + - 4.3320313 + - -2.2246094 + - -1.6318359 + - -0.1784668 + - -7 + - -1.6015625 + - 0.10180664 + - 3.1777344 + - 4.5351563 + - 1.7783203 + - 9.765625 + - -0.8173828 + - 1.7431641 + - 0.5605469 + - -1.4628906 + - 5.53125 + - -5.7578125 + - -0.7885742 + - -4.2226563 + - -6.1171875 + - -0.27783203 + - 3.5566406 + - -1.7841797 + - 0.12915039 + - -1.7714844 + - -5.046875 + - -0.14331055 + - 4.3710938 + - -3.421875 + - -1.6601563 + - -2.8886719 + - -4.21875 + - 7.5625 + - -6.5195313 + - -4.1289063 + - 4.3203125 + - 2.7089844 + - -0.018585205 + - 2.265625 + - 2.2285156 + - -2.5683594 + - -5.6679688 + - -4.3085938 + - 0.05795288 + - 5.4296875 + - -1.515625 + - -1.5429688 + - -3.8125 + - 4.3242188 + - -2.0351563 + - 6.0898438 + - -8.421875 + - 1.3779297 + - 5.9804688 + - -2.3671875 + - -1.8222656 + - 0.30078125 + - 3.3515625 + - 5.6367188 + - 0.91796875 + - -4.109375 + - 0.05508423 + - 2.5761719 + - -5.8984375 + - -0.72998047 + - 3.875 + - -0.97998047 + - 9.25 + - 0.640625 + - 5.125 + - -7.7734375 + - -0.4169922 + - 5.8203125 + - 0.023406982 + - -4.9765625 + - 3.3613281 + - 2.6699219 + - 1.2724609 + - 1.9111328 + - -0.86621094 + - 1.2138672 + - -0.8588867 + - 0.053955078 + - 2.6777344 + - 1.3916016 + - -1.3056641 + - 9.7890625 + - -4.0390625 + - 8.625 + - 4.1835938 + - 1.5400391 + - -7.9414063 + - 2.4550781 + - -0.32177734 + - -1.7421875 + - -2.6972656 + - -1.7070313 + - -6.2539063 + - 1.2714844 + - 5.6132813 + - -1.7275391 + - 2.8261719 + - 7.0585938 + - -6.4453125 + - 1.7128906 + - 2.1816406 + - 2.4511719 + - -6.5351563 + - -1.2958984 + - 8.484375 + - 0.90234375 + - -5.25 + - -0.9995117 + - -1.2177734 + - 2.0703125 + - 0.79785156 + - -2.8261719 + - 1.6298828 + - -4.7148438 + - 9.40625 + - -0.09173584 + - 3.0546875 + - 0.1574707 + - 2.953125 + - 3.7773438 + - -3.0742188 + - -8.9375 + - 9.046875 + - 6.8203125 + - -1.7255859 + - 0.2626953 + - -5.984375 + - 2.5605469 + - -1.8837891 + - 2.0429688 + - 1.4042969 + - 1.9941406 + - -1.1591797 + - 2.6152344 + - 4.703125 + - -5.7460938 + - 0.14221191 + - -4.7539063 + - 4.578125 + - 5.7109375 + - -13.2734375 + - 1.0605469 + - 0.42138672 + - -3.5136719 + - 9.265625 + - 12.375 + - -0.6269531 + - -0.1352539 + - 5.5 + - 2.8027344 + - -0.9560547 + - 1.4589844 + - 3.3085938 + - 0.44677734 + - -0.15759277 + - 1.1865234 + - -6.4414063 + - 0.11468506 + - 2.5605469 + - -6.6953125 + - 0.7290039 + - -0.9814453 + - -0.4482422 + - 3.6191406 + - -11.109375 + - -2.4199219 + - -0.37817383 + - -4.4023438 + - 0.5107422 + - -1.9541016 + - -1.9423828 + - 4.6523438 + - 0.53466797 + - -1.1777344 + - -2.8515625 + - 1.3115234 + - 3.4199219 + - 2.6503906 + - -5.1796875 + - 0.5048828 + - -3.7246094 + - 0.83984375 + - -2.8574219 + - 0.23120117 + - 24.703125 + - 0.38232422 + - -3.6171875 + - -1.5263672 + - -1.0390625 + - 4.734375 + - -5.2539063 + - -0.95703125 + - 4.921875 + - -5.5898438 + - -3.3417969 + - -1.3525391 + - -0.8027344 + - 2.8535156 + - 7.7734375 + - 0.6699219 + - 4.7070313 + - -3.3847656 + - -3.4589844 + - -1.9072266 + - 0.2685547 + - 3.4707031 + - -3.1367188 + - -6.328125 + - 2.5371094 + - -1.0742188 + - -6.953125 + - -3.6738281 + - -7.65625 + - 5.046875 + - 2.3125 + - 4.5820313 + - 1.3212891 + - 13.640625 + - -3.5839844 + - 2.8710938 + - -5.4140625 + - -0.63378906 + - -0.013664246 + - 1.6123047 + - 1.6660156 + - 0.81933594 + - 2.390625 + - 0.98779297 + - -0.5605469 + - 6.3632813 + - 1.7509766 + - -9.1328125 + - 4.7109375 + - -7.0703125 + - 1.8564453 + - 7.3007813 + - -1.6103516 + - 0.048553467 + - 0.42163086 + - 2.2148438 + - 5.765625 + - 0.3413086 + - 0.67822266 + - 6.0585938 + - 1.4267578 + - -0.82128906 + - 1.7109375 + - -3.1972656 + - -1.5332031 + - 4.0234375 + - -2.8535156 + - 3.7539063 + - -3.4824219 + - 8.328125 + - 1.3730469 + - -2.9941406 + - -8.578125 + - 5.8046875 + - -2.8417969 + - -5.7773438 + - -1.1640625 + - -0.9033203 + - 4.0039063 + - -0.3083496 + - -3.1113281 + - -1.1943359 + - 7.1796875 + - -3.7792969 + - 7.3203125 + - 2.7539063 + - -1.5244141 + - -1.8544922 + - 2.2890625 + - 1.4609375 + - -9.1171875 + - -3.3105469 + - 4.484375 + - 3.4863281 + - 8.3984375 + - -1.7675781 + - -2.0234375 + - 8.4453125 + - 3.3222656 + - 6.2148438 + - 3.4746094 + - 4.8085938 + - -2.5410156 + - -0.58251953 + - -0.090148926 + - -2.890625 + - -0.17236328 + - -5.3554688 + - -5.8007813 + - -2.40625 + - 2.78125 + - -1.1171875 + - 6.5273438 + - 4.3867188 + - -2.9414063 + - 0.3076172 + - 6.0117188 + - -0.32128906 + - -2.9785156 + - -1.1113281 + - 3.5566406 + - 7.1875 + - 3.7851563 + - -2.5820313 + - 0.12878418 + - 7.1367188 + - 9.9921875 + - -0.06323242 + - 0.703125 + - 0.70166016 + - -0.7714844 + - -2.2695313 + - 0.033081055 + - 0.029953003 + - 6.6523438 + - 2.6367188 + - -3.8769531 + - -4.7773438 + - -5.1367188 + - 4.796875 + - -2.0136719 + - 3.34375 + - -3.4160156 + - -5.078125 + - 0.74902344 + - 2.3574219 + - -1.8564453 + - 1.9130859 + - -3.640625 + - -3.2773438 + - 1.1289063 + - 1.7148438 + - -1.8642578 + - 4.4296875 + - 0.4741211 + - 1.3945313 + - -5.5742188 + - -4.109375 + - -0.8774414 + - -7.5585938 + - -3.9628906 + - 7.3789063 + - -0.8852539 + - -0.62402344 + - -2.0722656 + - -3.5429688 + - 5.9609375 + - 8.21875 + - 3.890625 + - -3.3476563 + - -1.4951172 + - 2.9648438 + - 0.18395996 + - -3.9511719 + - -4.3398438 + - -2.3964844 + - -1.3242188 + - -0.8041992 + - -2.7441406 + - -1.4892578 + - 7.1210938 + - -6.9375 + - -2.296875 + - -2.4550781 + - 6.3242188 + - 1.0410156 + - -4.59375 + - -0.5957031 + - -0.8564453 + - 7.0195313 + - 1.7265625 + - 4.515625 + - 2.6054688 + - 5.5703125 + - -3.0195313 + - -3.1523438 + - 2.2929688 + - -4.4414063 + - 6.46875 + - 4.9375 + - -1.9814453 + - 4.4453125 + - 1.5048828 + - 4.5585938 + - -5.0117188 + - 1.3818359 + - 5.90625 + - -6.7695313 + - -1.9111328 + - -0.071777344 + - 0.9746094 + - -0.8261719 + - -1.3583984 + - 2.1132813 + - -3.2734375 + - 4.6132813 + - 3.1328125 + - 0.60839844 + - 3.640625 + - -4.6992188 + - -2.4726563 + - 1.2744141 + - 2.1425781 + - -0.7441406 + - 0.024169922 + - -2.4355469 + - -3.28125 + - 0.57373047 + - -7.5195313 + - -6.2070313 + - 4.7226563 + - 6.1914063 + - 8.3515625 + - 12.2421875 + - -2.53125 + - -1.2783203 + - -2.5019531 + - 0.6977539 + - 2.1933594 + - -1.5917969 + - 1.9853516 + - 0.14929199 + - -3.9550781 + - -1.8896484 + - 8.78125 + - 0.7001953 + - 7.5625 + - -4.9570313 + - -6.9882813 + - 5.9453125 + - -1.8261719 + - 5.9882813 + - 3.2597656 + - 0.9448242 + - -8.3203125 + - -2.0175781 + - -2.5527344 + - 1.1171875 + - -3.6699219 + - 3.6796875 + - 3.5019531 + - 2.9042969 + - -4.9257813 + - 5.4414063 + - -1.0283203 + - -1.8310547 + - -4.5351563 + - 6.1914063 + - -3.5488281 + - 1.0585938 + - -4.3867188 + - 0.074645996 + - 7.7734375 + - 6.8945313 + - -5.1367188 + - -0.5361328 + - 3.4316406 + - -2.5214844 + - 1.7695313 + - 0.6152344 + - -1.7314453 + - -2.2070313 + - -2.2402344 + - -4.0742188 + - -0.6567383 + - -2.8925781 + - 1.6826172 + - -4.6640625 + - -4.1601563 + - 3.5527344 + - 3.8261719 + - -0.7451172 + - -0.43188477 + - 1.5214844 + - 0.14746094 + - 1.9335938 + - 6.421875 + - -0.24707031 + - 7.6679688 + - -5.765625 + - 7.8671875 + - 0.8486328 + - 2.9960938 + - 1.3359375 + - 3.6855469 + - -3.7617188 + - 3.7382813 + - -6.0234375 + - -6.40625 + - 1.3740234 + - 1.9501953 + - 3.1679688 + - -0.5288086 + - -1.0214844 + - -4.5351563 + - 9.4375 + - -5.8984375 + - 8 + - 3.6210938 + - -15.5859375 + - -2.7734375 + - 1.0527344 + - 3.0488281 + - -7.7148438 + - -4.7148438 + - 1.1132813 + - -0.6948242 + - -5.7929688 + - 3.9042969 + - 10.0625 + - 0.10675049 + - -4.1992188 + - 1.2373047 + - 1.3027344 + - -0.26367188 + - 0.115478516 + - 21.40625 + - 1.1445313 + - -1.7099609 + - -0.13208008 + - 2.6171875 + - -4.9335938 + - 6.8164063 + - 2.7988281 + - -2.1015625 + - -5.8007813 + - -4.8515625 + - 0.9658203 + - 1.8544922 + - 0.90283203 + - -1.2851563 + - 2.1660156 + - -0.25219727 + - 0.020111084 + - -5.4804688 + - -2.8789063 + - 3.1835938 + - -4.2070313 + - -2.7792969 + - -1.0927734 + - 0.67626953 + - -1.7822266 + - 0.8491211 + - -9.9140625 + - -1.6845703 + - -1.7011719 + - 1.8603516 + - 5.21875 + - -0.21923828 + - -0.1303711 + - -3.2890625 + - 3.203125 + - 0.4099121 + - 1.0439453 + - 1.7529297 + - 6.2148438 + - 0.9067383 + - 2.5820313 + - 1.8457031 + - -0.9951172 + - -0.27026367 + - -2.7050781 + - -5.828125 + - -1.5742188 + - 5.6796875 + - -1.8222656 + - 5.390625 + - 4.8125 + - -2.6464844 + - -1.1611328 + - 0.83154297 + - -1.1591797 + - -3.9667969 + - 6.53125 + - 1.890625 + - -3.125 + - 3.7128906 + - -0.37451172 + - -10.171875 + - 0.27197266 + - 4.375 + - 2.171875 + - 4.0820313 + - 0.96191406 + - 6.5742188 + - 1.1435547 + - 3.4101563 + - 14.1484375 + - 2.0058594 + - -6.46875 + - -5.5351563 + - -2.3808594 + - 5.1484375 + - -2.9238281 + - -3.0195313 + - 2.390625 + - -1.4082031 + - 2.9199219 + - -1.7109375 + - 2.4238281 + - 5.2421875 + - -1.5576172 + - 3.0644531 + - 9.1953125 + - -6.7304688 + - 0.80126953 + - -0.8017578 + - -1.3261719 + - 0.9580078 + - 3.1933594 + - 3.2617188 + - -10.828125 + - -5.5234375 + - 4.6445313 + - -2.640625 + - -4.7734375 + - 0.61279297 + - 4.0820313 + - -0.9604492 + - -5.203125 + - 0.64453125 + - -0.8691406 + - -0.31176758 + - -2.9414063 + - -3.9609375 + - -6.171875 + - 0.4567871 + - -0.5551758 + - -4.7070313 + - 4.1835938 + - -2.7480469 + - 2.9570313 + - 0.69677734 + - 3.5410156 + - 3.2050781 + - -1.5 + - -3.6210938 + - -0.8125 + - 0.7993164 + - 0.234375 + - -1.6123047 + - 6.1835938 + - 4.0117188 + - 2.1601563 + - -4.8125 + - 0.29003906 + - 6.5546875 + - -0.56884766 + - -3.1816406 + - 4.7617188 + - -4.2578125 + - -0.0072288513 + - 4.6445313 + - -4.40625 + - -0.7939453 + - -5.5820313 + - -8.15625 + - -6.9921875 + - -4.0664063 + - -0.8754883 + - 0.9082031 + - -7.7304688 + - -1.1416016 + - -3.3574219 + - -7.6796875 + - -1.3916016 + - -3.9472656 + - 5.5429688 + - -2.8222656 + - -5.3554688 + - 0.6171875 + - -0.82177734 + - -0.1071167 + - 2.5546875 + - -4.1914063 + - 0.03567505 + - -1.4912109 + - -18.234375 + - -3.4882813 + - -0.61621094 + - -2.9511719 + - -2.3105469 + - -2.7207031 + - -4.4453125 + - 0.22436523 + - 3.4179688 + - 2.2519531 + - 3.9589844 + - 0.53515625 + - 0.16992188 + - -9.9765625 + - -1.6142578 + - -1.0683594 + - 2.5488281 + - 5.4765625 + - 3.3320313 + - 1.9384766 + - -5.3203125 + - -0.64501953 + - 1.9863281 + - -2.2285156 + - 0.6845703 + - -1.9570313 + - -0.99902344 + - -7.109375 + - 0.2980957 + - -3.0253906 + - 0.4033203 + - -1.8222656 + - 0.4440918 + - 2.0332031 + - -3.109375 + - -4.3320313 + - -9.6640625 + - -3.9921875 + - -0.23498535 + - -8.546875 + - -1.2666016 + - 7.6328125 + - 1.7714844 + - -3.4804688 + - -12.171875 + - -3.0878906 + - -7.7578125 + - -2.0957031 + - 0.08728027 + - 6.3476563 + - 2.7285156 + - -1.2490234 + - -2.7011719 + - 1.4521484 + - 2.7929688 + - 1.5078125 + - -1.7939453 + - -3.4882813 + - -7.3828125 + - -8.0234375 + - -1.2158203 + - -2.7832031 + - 1.6845703 + - -6.6835938 + - -5.9882813 + - -5.0429688 + - -0.43139648 + - -2.5878906 + - -4.0820313 + - -4.4101563 + - -0.0793457 + - -1.3710938 + - 0.28515625 + - -5.1914063 + - -4.4804688 + - -0.12731934 + - 2.2890625 + - -0.1171875 + - -2.6132813 + - -2.9160156 + - -5.3867188 + - 15.71875 + - 4.9179688 + - 2.6601563 + - 0.9135742 + - -7.9453125 + - -11.1875 + - 4.671875 + - 0.26464844 + - -0.44628906 + - -4.6914063 + - 8.9296875 + - -4.0117188 + - -0.059295654 + - 0.6850586 + - 2.0800781 + - -4.9609375 + - 0.091918945 + - 1.5673828 + - 0.88671875 + - -0.11450195 + - -4.34375 + - -0.3581543 + - 0.34960938 + - 3.2792969 + - 0.31469727 + - -1.2236328 + - 3.0585938 + - -2.1601563 + - -8.734375 + - 6.3984375 + - 0.15588379 + - -3.0253906 + - -3.7089844 + - 4.4804688 + - -3.390625 + - -4.046875 + - 0.32348633 + - 1.7861328 + - 5.8554688 + - 4.234375 + - -0.5810547 + - -1.4169922 + - 1.4697266 + - -1.1591797 + - 2.3320313 + - -3.3945313 + - -5.0351563 + - -4.8476563 + - -0.95751953 + - -1.7558594 + - -0.6669922 + - 0.06011963 + - -2.2792969 + - -0.08380127 + - -3.7167969 + - 7.8867188 + - 4.2304688 + - -2.8574219 + - -3.3203125 + - 5.625 + - -2.6347656 + - 2.0019531 + - -0.5317383 + - 1.3701172 + - -0.08496094 + - -3.8242188 + - -3.0117188 + - 4.203125 + - 4.7265625 + - -4.328125 + - 11.1875 + - -1.6337891 + - -0.8666992 + - -5.9023438 + - -5.859375 + - -0.9355469 + - -8.4375 + - -5.46875 + - 1.1240234 + - -2.1289063 + - -7.890625 + - 5.0898438 + - -5.2382813 + - -0.027267456 + - 2.1738281 + - 3.9980469 + - 6.4375 + - 4.203125 + - 5.4414063 + - -2.7070313 + - 0.10546875 + - -2.6132813 + - 5.5234375 + - -1.9365234 + - 3.7636719 + - -2.2285156 + - 1.6455078 + - 2.8125 + - -0.21313477 + - 5.1953125 + - 7.0351563 + - -2.828125 + - 6.2382813 + - -3.6425781 + - 5.7382813 + - -0.30249023 + - -3.5917969 + - -1.5507813 + - 6.0976563 + - -0.92333984 + - 2.9238281 + - -8.75 + - -0.2763672 + - 0.54296875 + - 3.0019531 + - -2.3320313 + - 1.6035156 + - 7.4257813 + - -5.984375 + - -0.48168945 + - 0.1427002 + - 0.8515625 + - -6.4023438 + - -1.2822266 + - -2.8828125 + - -1.2177734 + - 2.5761719 + - -3.8085938 + - 3.5957031 + - 11.265625 + - -9.984375 + - 3.9042969 + - 7.6015625 + - -2.4746094 + - 1.7685547 + - -0.30004883 + - -4.1484375 + - 0.3190918 + - -4.4609375 + - -0.3984375 + - 3.4667969 + - -1.7177734 + - 1.8867188 + - -8.171875 + - 3.046875 + - 6.234375 + - 0.22839355 + - 8.5234375 + - -3.0292969 + - -1.9580078 + - 4.546875 + - 3.2421875 + - 0.69628906 + - -2.8554688 + - 1.2177734 + - -6.3398438 + - 1.2490234 + - 1.2724609 + - -2.9121094 + - 1.3271484 + - -2.3066406 + - 0.703125 + - -6.4804688 + - -4.1015625 + - -2.4296875 + - -0.3515625 + - 1.6445313 + - -6.1914063 + - 4.171875 + - -13.109375 + - -1.1240234 + - 6.53125 + - 3.2871094 + - 4.0664063 + - 2.9746094 + - 2.9785156 + - 7.7773438 + - -2.7148438 + - -3.8300781 + - 6.3632813 + - 3.1503906 + - -8.09375 + - -1.3330078 + - -4.8632813 + - -2.21875 + - -0.6171875 + - -1.8818359 + - 4.5820313 + - -3.9160156 + - -3.078125 + - -1.4169922 + - 20.34375 + - -1.7783203 + - 2.125 + - -2.96875 + - 0.42333984 + - -4.7617188 + - -0.68115234 + - 1.4033203 + - 2.0527344 + - 3.875 + - 1.7890625 + - 0.61572266 + - -2.4238281 + - -0.953125 + - 5.6679688 + - 0.8208008 + - 0.022460938 + - -1.7705078 + - 6.6289063 + - 0.48901367 + - -6.4804688 + - -3.3085938 + - -12.15625 + - -0.35791016 + - -0.8105469 + - -2.0058594 + - 5.0039063 + - -3.0957031 + - 0.19445801 + - 10.140625 + - 4.4453125 + - -3.2050781 + - 1.7548828 + - -8.9921875 + - 1.5078125 + - 6.7304688 + - 9.0390625 + - -2.6171875 + - -5.4492188 + - 4.4492188 + - -0.63623047 + - -5.9296875 + - 0.28222656 + - 4.015625 + - 7.4023438 + - 4.328125 + - -6.84375 + - -2.6796875 + - -4.1992188 + - -2.0078125 + - 1.015625 + - -9.4296875 + - -6.2109375 + - -4.7890625 + - -10.484375 + - -0.08459473 + - 1.7314453 + - -5.8632813 + - 6.8164063 + - -2.2597656 + - -2.8125 + - 5.4101563 + - -2.2597656 + - 7.6367188 + - 0.67333984 + - 0.20532227 + - -12.8359375 + - -0.19580078 + - -1.0927734 + - 4.1835938 + - -2.2226563 + - 3.0800781 + - 3.5605469 + - -7.2460938 + - 0.02708435 + - 1.2529297 + - -9.328125 + - -8.2734375 + - -2.2617188 + - -0.78125 + - 6.7929688 + - -1.9189453 + - -11.796875 + - -0.23071289 + - 3.5976563 + - -2.78125 + - 0.56689453 + - -5.1992188 + - -2.3203125 + - -3.0351563 + - -4.1679688 + - 3.7460938 + - -2.4394531 + - -2.0507813 + - -3.3808594 + - -2.3320313 + - 3.6464844 + - 9.140625 + - -5.2070313 + - 3.0605469 + - 0.37329102 + - -1.1757813 + - -5.2304688 + - 2.1074219 + - -10.1484375 + - 1.8974609 + - 0.28686523 + - 0.6074219 + - 3.9355469 + - 5.15625 + - 0.68310547 + - -10.734375 + - -2.1621094 + - 1.3164063 + - -1.2197266 + - 2.8183594 + - -1.2998047 + - -0.5415039 + - 1.3759766 + - 12.25 + - -1.0390625 + - 4.4960938 + - 7.28125 + - 1.3095703 + - 1.9541016 + - 3.953125 + - 1.5693359 + - 1.5996094 + - -2.046875 + - 1.7978516 + - 4.3515625 + - -7.734375 + - 2.1425781 + - 0.71191406 + - -4.109375 + - 0.81689453 + - 12.375 + - -7.5390625 + - -10.0078125 + - 1.4414063 + - -0.7026367 + - 0.74365234 + - 9.2265625 + - -7.2148438 + - 1.0693359 + - 4.2070313 + - 2.0507813 + - -4.5859375 + - -6.4101563 + - -4.6992188 + - -3.4394531 + - -1.2207031 + - 17.21875 + - 2.9785156 + - 1.2050781 + - 5.796875 + - -4.9648438 + - 2.5605469 + - -2.625 + - -1.5097656 + - -8.375 + - -0.81152344 + - 3.8359375 + - -2.8007813 + - -0.53759766 + - 2.09375 + - 10.5 + - 6.3164063 + - 4.921875 + - 4.0585938 + - -6.2226563 + - -1.1142578 + - 3.1269531 + - -5.7695313 + - 6.15625 + - 2.7421875 + - -0.1463623 + - -6.1523438 + - 0.4934082 + - -5.0664063 + - -4.90625 + - 2.6738281 + - 4.734375 + - 4.1171875 + - -3.7304688 + - 1.7646484 + - 1.0849609 + - -1.8232422 + - -4.1835938 + - 1.6884766 + - 7.5703125 + - 5.0039063 + - 2.1699219 + - -5.6992188 + - -0.73828125 + - 1.7001953 + - -3.1308594 + - -3.1015625 + - -9.46875 + - -3.6074219 + - 6.9882813 + - 1.6279297 + - -2.3378906 + - -6.125 + - 2.8574219 + - 2.4628906 + - -1.3818359 + - 2.9199219 + - 0.46801758 + - 2.4082031 + - -1.7470703 + - -0.8823242 + - -0.16455078 + - 1.5869141 + - 0.17614746 + - -3.265625 + - -0.16381836 + - 5.3359375 + - 5.5703125 + - -5.15625 + - -2.5078125 + - 0.9633789 + - 12.1640625 + - -1.7744141 + - -1.1025391 + - 0.73779297 + - 8.3828125 + - 4.140625 + - -6.90625 + - 1.8007813 + - 6.4179688 + - -4.1328125 + - 0.4506836 + - 2.5058594 + - 6.0742188 + - -1.140625 + - 1.0488281 + - 5.3554688 + - 3.8164063 + - -2.2597656 + - 0.92041016 + - 4.9570313 + - 1.9746094 + - 9.4453125 + - 0.60791016 + - -2.4003906 + - -1.9726563 + - 2.2070313 + - -3.171875 + - -10.796875 + - 0.55810547 + - -6.234375 + - 4.9765625 + - 1.2148438 + - 0.44091797 + - 3.8847656 + - -2.8613281 + - -2.1171875 + - 2.5742188 + - -0.7753906 + - -1.8837891 + - -8.0859375 + - 3.7070313 + - 0.09869385 + - -4.3515625 + - -2.8476563 + - 4.8476563 + - 1.9140625 + - -2.6347656 + - 1.4101563 + - 0.88916016 + - 2.3554688 + - -9.1171875 + - 1.078125 + - 1.3027344 + - -2.2402344 + - 4.8671875 + - -3.1484375 + - -6.3945313 + - -1.2197266 + - 4.7617188 + - 0.3034668 + - -0.44091797 + - 5.7773438 + - -4.1445313 + - -4.4648438 + - -7.8125 + - -3.3515625 +- - -2.6601563 + - 6.1328125 + - 1.4423828 + - -3.2871094 + - 5.0429688 + - -2.6582031 + - -0.17053223 + - -0.06591797 + - 0.16113281 + - 1.5869141 + - 6.71875 + - -4.5 + - -3.9179688 + - -1.1357422 + - -1.4628906 + - -0.0803833 + - 1.6572266 + - -1.1748047 + - -0.22253418 + - 7.5351563 + - -1.2246094 + - -7.7929688 + - -5.9960938 + - -2.1679688 + - -0.036895752 + - -1.4746094 + - -7.046875 + - 1.9560547 + - 5.9023438 + - 3.2734375 + - -5.4648438 + - 2.125 + - 7.3632813 + - 13.5234375 + - -7.1484375 + - -4.1132813 + - -0.8017578 + - -1.9736328 + - -2.0019531 + - -1.5009766 + - 2.8671875 + - 1.2753906 + - -0.5551758 + - -0.10852051 + - 5.8007813 + - 5.4648438 + - -10.828125 + - -5.4492188 + - 0.025390625 + - 2.4375 + - 2.6425781 + - -5.3632813 + - -3.2480469 + - 2.4453125 + - 7.5117188 + - -0.40014648 + - 2.6855469 + - -0.59228516 + - -1.859375 + - -0.8886719 + - 2.8535156 + - -2.9765625 + - -0.14123535 + - 8.5078125 + - 2.8730469 + - 5.7265625 + - -2.1171875 + - -2.2460938 + - 4.6679688 + - 3.4609375 + - 6.203125 + - 1.3486328 + - 4.484375 + - -0.121276855 + - -0.7529297 + - 5.7148438 + - 2.4394531 + - -2.9511719 + - -8.984375 + - -10.390625 + - 1.5205078 + - -0.20666504 + - -7.734375 + - -11.8203125 + - -1.140625 + - -2.7675781 + - -7.3945313 + - 3.6523438 + - -0.55810547 + - -1.9707031 + - 2.1757813 + - -3.0585938 + - -16.484375 + - 4.6289063 + - 6.96875 + - -2.9804688 + - 0.44555664 + - -2.6210938 + - -0.57958984 + - 1.2226563 + - -2.09375 + - 0.953125 + - 2.0136719 + - -0.25390625 + - -1.7011719 + - -1.0419922 + - -15.3515625 + - 3.6621094 + - -0.7661133 + - -5.6953125 + - -7.734375 + - 1.2558594 + - 0.4765625 + - -3.7578125 + - 1.9130859 + - 5.0546875 + - 1.4794922 + - -0.13244629 + - 0.08148193 + - 8.5625 + - 2.1894531 + - -0.39794922 + - 5.1171875 + - -3.6523438 + - -1.8945313 + - 2.9882813 + - 7.1328125 + - -4.328125 + - -5.4726563 + - 4.15625 + - -7.0351563 + - -4.8085938 + - -2.2929688 + - 4.5234375 + - 0.86328125 + - 2.4472656 + - -3.2773438 + - -1.8867188 + - 0.9248047 + - -4.6054688 + - -0.74609375 + - 3.5742188 + - 2.0410156 + - 3.7363281 + - -0.87109375 + - -1.2783203 + - 5.6523438 + - 1.3505859 + - 10.5078125 + - 0.54345703 + - 5.8164063 + - -2.8085938 + - 3.9511719 + - -1.7529297 + - -3.4785156 + - 1.4970703 + - -1.0410156 + - -9.0078125 + - -4.734375 + - -7.4257813 + - 2.2207031 + - 3.0976563 + - 8.34375 + - -5.6835938 + - 2.2109375 + - 4.7890625 + - 13.2890625 + - 5.0898438 + - -4.6132813 + - 4.0429688 + - -1.1464844 + - 1.4775391 + - -3.4023438 + - -2.6445313 + - -2.5273438 + - 2.6835938 + - -0.74658203 + - 3.1464844 + - -9.5234375 + - 10.8984375 + - 1.9990234 + - -6.5859375 + - -6.25 + - 5.6171875 + - -1.9248047 + - 0.8442383 + - -3.28125 + - 1.25 + - 7.7851563 + - 9.0078125 + - 0.5917969 + - 4.765625 + - 0.015930176 + - -2.9296875 + - 4.34375 + - -1.0742188 + - 2.5703125 + - -5.8125 + - 5.7226563 + - 4.546875 + - 1.765625 + - 0.17333984 + - 1.3652344 + - -3.5605469 + - 8.6328125 + - 5.0820313 + - 1.6074219 + - -4.0429688 + - 2.5878906 + - 0.7651367 + - -7.4609375 + - -1.2744141 + - 0.19689941 + - -8.96875 + - -0.3930664 + - -5.4804688 + - -1.7539063 + - -14.65625 + - -1.9550781 + - -6.84375 + - 2.7832031 + - -14.296875 + - -1.4101563 + - 4.2226563 + - 2.2714844 + - -3.6992188 + - -4.2070313 + - 2.03125 + - -2.4453125 + - 1.1230469 + - 7.2539063 + - -0.9663086 + - 0.8432617 + - -2.0117188 + - 0.10241699 + - -1.1904297 + - -4.78125 + - -5.0546875 + - -4.4179688 + - 1.3056641 + - -1.1845703 + - 0.1685791 + - -2.78125 + - -1.0908203 + - -3.7988281 + - 4.4648438 + - 0.07293701 + - 0.1451416 + - 0.16137695 + - 2.9746094 + - -7.1953125 + - -2.6445313 + - -5.6523438 + - -0.88623047 + - 6.4453125 + - 5.453125 + - 1.2021484 + - 3.4453125 + - 0.020889282 + - 6.859375 + - 8.0390625 + - 2.3554688 + - -1.4697266 + - -1.296875 + - 0.10217285 + - 1.1445313 + - -2.1113281 + - 3.9980469 + - -8.8984375 + - -2.8730469 + - 2.0644531 + - -5.3085938 + - 0.5878906 + - -4.7578125 + - 7.9726563 + - -4.5742188 + - 0.18920898 + - -0.13659668 + - 8 + - -7.7929688 + - -0.23095703 + - 0.21203613 + - -0.8671875 + - 7.9101563 + - 7.3359375 + - -10.8359375 + - -1.6132813 + - 7.8710938 + - 7.5234375 + - 5.28125 + - 4.5820313 + - 2.8164063 + - -4.5039063 + - 4.3046875 + - -9.515625 + - -3.0351563 + - 3.8886719 + - 5.3515625 + - 4.4179688 + - 3.125 + - 5.5234375 + - -3.890625 + - 4.8984375 + - -1.1513672 + - 0.91308594 + - 4.7070313 + - -3.3652344 + - 2.4472656 + - -3.03125 + - -4.4101563 + - 2.0136719 + - 1.2832031 + - -5.2734375 + - 2.21875 + - -4.2460938 + - -3.84375 + - 1.0859375 + - 6.65625 + - -3.1855469 + - -1.8798828 + - 0.041168213 + - -3.6855469 + - 0.77490234 + - -2.3007813 + - -4.4257813 + - 2.609375 + - 1.3867188 + - -0.044128418 + - -0.7211914 + - 5.3945313 + - -6.078125 + - -6.4179688 + - -1.9951172 + - 0.359375 + - 7.6796875 + - -0.8066406 + - -1.2041016 + - 3.0761719 + - 0.62646484 + - 5.0273438 + - 3.2441406 + - -5.671875 + - -0.9892578 + - 5.7148438 + - -4.0507813 + - -7.6328125 + - 1.5166016 + - 1.1181641 + - 5.8984375 + - 3.6933594 + - -2.1738281 + - 0.1184082 + - 1.4492188 + - -5.7851563 + - -6.3710938 + - 2.8007813 + - -0.59033203 + - 6.6953125 + - 0.048034668 + - 2.8378906 + - -6.5039063 + - 3.1464844 + - 0.66308594 + - -3.4394531 + - -3.3339844 + - 2.4492188 + - 3.1132813 + - 2.1269531 + - 3.0214844 + - -3.2460938 + - -7.984375 + - -4.1640625 + - -0.45507813 + - -1.5615234 + - 4.6757813 + - 2.546875 + - 5.5820313 + - 0.18457031 + - 4.7148438 + - -2.34375 + - -1.5302734 + - -10.890625 + - 2.1230469 + - 0.78808594 + - -3.1542969 + - -0.5385742 + - -3.1464844 + - -2.4003906 + - -5.1367188 + - -0.85253906 + - 4.953125 + - -0.8486328 + - 6.25 + - -8.46875 + - -0.049713135 + - 5.7382813 + - -0.13757324 + - -10.8984375 + - -1.4873047 + - 6.1523438 + - 5.0664063 + - -2.1640625 + - -3.0039063 + - -0.36743164 + - 2.296875 + - -0.54345703 + - 0.31982422 + - 1.4697266 + - -6.3789063 + - 12.203125 + - 0.6308594 + - 4.2539063 + - 2.4960938 + - 1.5224609 + - 4.9609375 + - -1.6894531 + - -10.15625 + - 3.1660156 + - 2.7207031 + - -0.026367188 + - -2.4804688 + - -6.0820313 + - 2.40625 + - -3.4570313 + - 0.58496094 + - 7.1601563 + - -2.171875 + - 2.6113281 + - 5.234375 + - -1.2363281 + - -5.375 + - -1.2138672 + - -5 + - 6.25 + - 4.0234375 + - -12.5546875 + - 1.2861328 + - 1.6513672 + - -0.002565384 + - 11.203125 + - 4.2734375 + - 5.6171875 + - 2.9765625 + - -0.39501953 + - 1.9853516 + - -5.15625 + - -0.9848633 + - 8.125 + - -0.72314453 + - -4.59375 + - -0.8491211 + - -6.765625 + - -0.7060547 + - 2.4765625 + - 0.0541687 + - -0.6845703 + - 1.9365234 + - 4.9257813 + - -2.25 + - -9.71875 + - -2.3769531 + - 2.9121094 + - -3.6035156 + - 1.8974609 + - 2.1054688 + - -1.0136719 + - 1.0263672 + - -1.2988281 + - -2.7597656 + - -1.0136719 + - 1.8779297 + - 1.5869141 + - 1.0380859 + - -9.1328125 + - -2.4023438 + - -1.7392578 + - -1.6982422 + - -7.0585938 + - 1.7822266 + - 39.8125 + - -5.09375 + - -3.1484375 + - 0.76953125 + - -1.5371094 + - 3.6054688 + - -2.9160156 + - 0.24438477 + - -0.3605957 + - -1.9970703 + - -0.61083984 + - 1.6347656 + - -1.6269531 + - 2.7558594 + - 7.7382813 + - -2.5117188 + - 0.86572266 + - 1.4677734 + - -8.40625 + - 0.6088867 + - 0.55126953 + - 2.0175781 + - -3.5507813 + - -2.453125 + - 9.2421875 + - 0.9399414 + - -10.3359375 + - -0.5683594 + - -3.953125 + - 6.671875 + - 2.6230469 + - 3.90625 + - -0.3112793 + - 13.515625 + - -6 + - 0.34204102 + - -7.2851563 + - -3.1035156 + - 0.515625 + - -5.78125 + - 0.85546875 + - -2.7324219 + - 1.2373047 + - -0.025985718 + - -0.18713379 + - 9.3515625 + - -1.5898438 + - -8.6015625 + - 8.421875 + - -1.6669922 + - 5.375 + - 5.0273438 + - -0.95947266 + - 5.3242188 + - -0.91064453 + - 2.5449219 + - 1.6660156 + - -1.7597656 + - 0.26757813 + - 1.7070313 + - 2.2539063 + - 2.4140625 + - -5.3046875 + - -2.9082031 + - -3.8769531 + - 5.734375 + - -0.52246094 + - 1.4697266 + - -5.3632813 + - 4.3945313 + - 3.7207031 + - -1.7265625 + - -4.9726563 + - 4.9296875 + - -0.62353516 + - -10.2890625 + - -4.0429688 + - 0.35913086 + - 5.1171875 + - -2.7167969 + - -2.7109375 + - 1.6943359 + - 2.5195313 + - -0.8520508 + - 2.1855469 + - 0.50097656 + - 1.7402344 + - 1.2587891 + - 3.2460938 + - 4.0195313 + - -8.9140625 + - -2.4785156 + - 2.0273438 + - 4.8867188 + - 6.4257813 + - 0.8745117 + - -3.9453125 + - 3.3671875 + - 4.0429688 + - 8.625 + - 1.9970703 + - 5.921875 + - -2.6054688 + - 1.6337891 + - -4.0703125 + - -1.4013672 + - 0.79003906 + - -0.86083984 + - -4.6132813 + - 0.5859375 + - 4.0820313 + - -2 + - 4.5820313 + - 2.5175781 + - -1.0537109 + - 0.2220459 + - 8.8125 + - -0.56396484 + - -2.4121094 + - 0.51904297 + - 3.5546875 + - 5.4960938 + - 4 + - -9.0390625 + - 2.2109375 + - 6.921875 + - 12.1328125 + - -2.1503906 + - 3.421875 + - 2.0566406 + - -6.1484375 + - -3.5410156 + - 4.6601563 + - -2.4863281 + - 3.4824219 + - 0.7192383 + - -0.5786133 + - -0.4128418 + - -6.5625 + - 3.3574219 + - -2.8007813 + - 6.2773438 + - -5.625 + - -5.828125 + - -0.121276855 + - 1.8955078 + - 0.29077148 + - 1.4560547 + - 4.578125 + - -4.8515625 + - -2.484375 + - 0.29882813 + - -0.75439453 + - 9.1171875 + - 2.1835938 + - -0.734375 + - -5.28125 + - -2.9121094 + - -2.6679688 + - -4.0742188 + - -0.091796875 + - 3.265625 + - -1.7441406 + - -0.107666016 + - -0.97021484 + - -3.2792969 + - 5.3242188 + - 11.7890625 + - -1.390625 + - -1.453125 + - 1.4892578 + - 3.6269531 + - 2.9707031 + - -5.0039063 + - -0.8676758 + - -3.015625 + - -0.79785156 + - 2.9472656 + - -0.68359375 + - -3.8457031 + - 8.53125 + - -4.4492188 + - 1.7265625 + - -2.8515625 + - 10.6953125 + - 0.6298828 + - -6.3632813 + - 3.7890625 + - 0.02583313 + - 5.625 + - 1.1943359 + - 4.1015625 + - -0.79345703 + - 6.5039063 + - -1.7978516 + - -0.92822266 + - -0.37963867 + - -5.75 + - 4.8164063 + - 0.5751953 + - -2.0644531 + - 6.7578125 + - 3.3007813 + - 2.6074219 + - -1.8779297 + - 1.2832031 + - 3.7246094 + - -5.46875 + - -1.5478516 + - -1.1992188 + - 0.7739258 + - 0.105163574 + - -1.8105469 + - -0.64941406 + - -3.3828125 + - 6.1445313 + - 1.7167969 + - 3.1875 + - -4.1328125 + - -4.625 + - -4.4921875 + - 3.7929688 + - -0.50683594 + - 2.8789063 + - 1.0751953 + - -4.9453125 + - -7.296875 + - -1.0917969 + - -4.7734375 + - -1.640625 + - 1.9980469 + - 8.75 + - 8.7265625 + - 14.8125 + - 2.234375 + - -0.7573242 + - 0.8901367 + - 1.3173828 + - 2.3144531 + - -2.1035156 + - 1.5166016 + - 0.86035156 + - -3.2167969 + - -4.6796875 + - 12.4453125 + - 3.2558594 + - 13.234375 + - -2.7363281 + - -1.7490234 + - 8.5703125 + - -1.1708984 + - 2.7617188 + - 2.6484375 + - -0.42285156 + - 3.1640625 + - -1.3808594 + - -0.55908203 + - 2.0273438 + - -4.2578125 + - 1.7412109 + - -1.9677734 + - 2.8867188 + - -9.578125 + - 5.5546875 + - -0.62109375 + - -7.4648438 + - -1.7392578 + - 5.5351563 + - -3.0996094 + - 0.29907227 + - -1.9863281 + - -3.7441406 + - 2.5019531 + - 6.0117188 + - -3.2832031 + - 1.7216797 + - 2.40625 + - -5.9960938 + - 2.1640625 + - 0.33935547 + - -2.8359375 + - -1.9365234 + - -7.03125 + - -3.9179688 + - 0.18847656 + - -2.8261719 + - -1.8232422 + - -5.953125 + - -4.7617188 + - 3.109375 + - -1.1181641 + - -1.3964844 + - 0.25854492 + - -0.2355957 + - 8.96875 + - 6.265625 + - 5.125 + - 4.9453125 + - 7.1210938 + - 0.37353516 + - 2.5292969 + - -0.16174316 + - -1.0419922 + - -1.9121094 + - 0.56933594 + - -4.5117188 + - 0.48046875 + - -3.2109375 + - -4.7773438 + - 2.4765625 + - 3.4785156 + - 1.6933594 + - 2.1542969 + - 0.36767578 + - -5.1015625 + - 5.4570313 + - -8.4609375 + - 8.9921875 + - 0.23986816 + - -23.265625 + - -1.5517578 + - 0.9589844 + - 1.53125 + - -19.390625 + - -0.5551758 + - -2.2207031 + - 0.55029297 + - -1.5292969 + - 0.6479492 + - 7.5273438 + - -0.5708008 + - -3.3007813 + - 1.0556641 + - 1.1162109 + - 0.17236328 + - -1.8681641 + - 19.296875 + - -0.9951172 + - 0.34594727 + - 1.5986328 + - 1.4570313 + - -2.7304688 + - 6.4101563 + - 5.2773438 + - -0.9682617 + - -3.5585938 + - -5.8320313 + - 0.5102539 + - 1.859375 + - -0.7089844 + - 1.1777344 + - 2.2617188 + - 2.7636719 + - 3.7714844 + - -4.0390625 + - -5.6171875 + - 5.8828125 + - -3.6054688 + - -3.2324219 + - -2.2675781 + - -0.6738281 + - 0.40112305 + - -0.09869385 + - -3.1074219 + - -0.46240234 + - -0.48095703 + - 2.7304688 + - 1.6044922 + - -2.2265625 + - -2.7949219 + - -3.9296875 + - 2.4296875 + - -2.0625 + - 0.37548828 + - -0.33764648 + - 5.0234375 + - 0.34765625 + - 2.265625 + - 1.3535156 + - -3.1386719 + - -1.3925781 + - -4.6796875 + - -6.4101563 + - -6.4921875 + - 7.6367188 + - -2.6816406 + - 4.1367188 + - 5.0078125 + - -2.6152344 + - 0.42163086 + - -0.6689453 + - 3.59375 + - -2.7167969 + - 4.7382813 + - -2.2949219 + - -4.7421875 + - 1.5917969 + - -0.45043945 + - -9.171875 + - 0.073791504 + - 2.8574219 + - 2.5 + - 0.6435547 + - -0.9448242 + - 5.3476563 + - -1.9765625 + - 4.0625 + - 1.6523438 + - -5.359375 + - -12.7578125 + - -1.3681641 + - -5.8828125 + - 1.2304688 + - -4.3789063 + - -4.265625 + - 10.28125 + - -5.4296875 + - 3.1484375 + - -6.71875 + - 4.5078125 + - 7.765625 + - -1.0263672 + - 0.31323242 + - 6.59375 + - -6.703125 + - 1.7832031 + - 1.7119141 + - -2.9882813 + - -0.1149292 + - 3.5136719 + - 5.9101563 + - -11.8828125 + - -3.2910156 + - 1.7861328 + - -3.1464844 + - -1.7050781 + - -2.28125 + - -0.76953125 + - -1.6191406 + - -2.7109375 + - -1.8359375 + - -9.4609375 + - 0.1262207 + - -4.0664063 + - -2.3066406 + - -7.0703125 + - 3.1738281 + - -3.9589844 + - -6.2265625 + - -1.5527344 + - -13.1875 + - 7.2421875 + - -0.0066375732 + - 1.4453125 + - 2.6679688 + - 2.8925781 + - 0.24084473 + - -0.66503906 + - -0.71240234 + - -1.6630859 + - -1.5644531 + - 3.6171875 + - 3.7050781 + - 2.4921875 + - -2.59375 + - -3.7949219 + - 2.2636719 + - -2.6210938 + - -2.5761719 + - 1.2060547 + - 0.30908203 + - -1.0556641 + - 1.5283203 + - 1.6806641 + - -2.3945313 + - -0.33447266 + - -8.3984375 + - -5.7578125 + - -2.9589844 + - -3.5253906 + - -2.6210938 + - -6.2109375 + - -2.1367188 + - -0.83740234 + - -11.296875 + - -0.96875 + - -1.9716797 + - 4.7734375 + - -5.1757813 + - -3.9824219 + - -0.67041016 + - 1.6728516 + - 4.3046875 + - 0.90527344 + - -7.2421875 + - 0.036132813 + - -3.5195313 + - -13.8671875 + - 3.3222656 + - -2.0019531 + - 0.5576172 + - 0.29882813 + - -0.18347168 + - -0.3491211 + - 1.0224609 + - 4.578125 + - 4.4726563 + - 2.5214844 + - 3 + - 3.1679688 + - -5.1835938 + - -1.859375 + - -2.3632813 + - 2.1582031 + - -0.28344727 + - 0.05682373 + - 0.5776367 + - -3.5917969 + - 0.93847656 + - 0.8540039 + - -2.6035156 + - 1.2148438 + - -3.453125 + - -3.9765625 + - 0.35839844 + - 0.89404297 + - -7.96875 + - -6.3007813 + - -1.3164063 + - -3.7949219 + - 0.9560547 + - -0.8100586 + - -4.4414063 + - -9.0390625 + - -8.4453125 + - -1.5175781 + - -6.59375 + - -5.3632813 + - 4.1328125 + - 0.10809326 + - -4.375 + - -11.578125 + - -2.9238281 + - -9.4140625 + - -2.1308594 + - 1.7373047 + - 0.40307617 + - 3.1074219 + - 5.625 + - 0.9135742 + - 1.1972656 + - 1.7744141 + - 0.93603516 + - 0.76123047 + - -2.5605469 + - -2.4824219 + - -5.28125 + - -4.9960938 + - -2.4609375 + - -2.3085938 + - -3.6230469 + - -3.3359375 + - 0.33666992 + - 1.5625 + - -2.6152344 + - -3.1367188 + - 2.0566406 + - -0.2607422 + - -0.42089844 + - 1.6015625 + - -5.046875 + - -4.625 + - 2.0507813 + - 4.96875 + - -5.7070313 + - -0.081604004 + - -5.9609375 + - -7.3867188 + - 8.0234375 + - 11.2578125 + - 4.7890625 + - 1.5742188 + - -1.8105469 + - -0.61865234 + - -2.8242188 + - 1.4355469 + - 0.78125 + - -4.1445313 + - 11.5625 + - -5.6328125 + - -0.08343506 + - 0.6743164 + - -0.26049805 + - -5.1679688 + - 0.0869751 + - -0.36010742 + - 0.25952148 + - -1.2255859 + - 0.41088867 + - 1.3222656 + - -2.6699219 + - -0.19616699 + - 2.7460938 + - -1.5703125 + - -0.5654297 + - -4.265625 + - -2.7421875 + - 5.0195313 + - -3.421875 + - -1.4833984 + - -3.5117188 + - 4.5585938 + - 2.3671875 + - -1.7558594 + - 4.8867188 + - -0.26049805 + - 3.6953125 + - 0.7705078 + - -0.19335938 + - 1.4179688 + - 6.0234375 + - 2.3105469 + - 8.2734375 + - 0.4169922 + - -4.8203125 + - -2.0410156 + - 2.0371094 + - 0.4267578 + - 0.9321289 + - 1.3779297 + - -3.2714844 + - -0.6591797 + - -1.5068359 + - 6.21875 + - 0.39135742 + - 3.2871094 + - -1.5976563 + - 4.46875 + - -1.5048828 + - 1.9814453 + - -0.7651367 + - -1.7587891 + - 3.5175781 + - -3.4199219 + - -3.2207031 + - 7.3789063 + - 1.9667969 + - -1.9277344 + - 6.0234375 + - -1.46875 + - -4.9140625 + - -5.7890625 + - -3.7246094 + - 1.0273438 + - -8.3515625 + - -3.6269531 + - 0.2890625 + - -4.8007813 + - -6.0742188 + - 1.8652344 + - -5.859375 + - -0.1303711 + - 2.6875 + - 9.4921875 + - 3.5761719 + - 1.6201172 + - 7.3710938 + - -2.7460938 + - 1.2705078 + - 0.5307617 + - 7.4375 + - 4.203125 + - -1.6572266 + - -0.66015625 + - 1.6787109 + - 1.0703125 + - -0.025161743 + - 3.4375 + - 4.5429688 + - -4.7851563 + - 3.6210938 + - 0.124572754 + - 14.265625 + - 0.038726807 + - -2.5 + - 0.9296875 + - 0.9111328 + - -0.020980835 + - -3.5566406 + - -7.8984375 + - -0.67578125 + - 0.74316406 + - 2.9394531 + - -4.2695313 + - 4.6679688 + - 3.2148438 + - 0.8149414 + - -2.0859375 + - 1.2001953 + - 6.875 + - -7.359375 + - 0.28637695 + - -0.44140625 + - -2.5585938 + - 6.03125 + - -6.6132813 + - 5.2851563 + - 12.390625 + - -6.203125 + - 2.6601563 + - 2.3554688 + - -3.4375 + - 2.3964844 + - 0.4946289 + - -0.06878662 + - -1.6601563 + - -2.6933594 + - 5.8359375 + - 4.1796875 + - -1.9238281 + - 0.15405273 + - -8.7421875 + - 4.4335938 + - 5.6015625 + - -0.14611816 + - 7.9335938 + - -2.0878906 + - -0.06939697 + - 2.4472656 + - 5.0078125 + - -5.7578125 + - -5.1601563 + - -1.3964844 + - -8.4921875 + - 2.9550781 + - 0.82470703 + - -0.56152344 + - 0.011161804 + - -6.1171875 + - -1.2744141 + - -7.0390625 + - -1.6132813 + - -3.421875 + - 1.4443359 + - 0.36621094 + - 0.10784912 + - 3.0273438 + - -7.2265625 + - -2.1015625 + - 5.8632813 + - -3.7988281 + - 3.5605469 + - 3.7402344 + - 4.9179688 + - 4.3671875 + - -2 + - -1.7324219 + - 4.6171875 + - 7.6757813 + - -1.4951172 + - 0.140625 + - -8.0546875 + - -1.8964844 + - 0.2668457 + - 2.8417969 + - -4.84375 + - 4.1210938 + - 2.515625 + - 2.9414063 + - 28.0625 + - -6.4609375 + - -1.2070313 + - -0.26635742 + - 4.1210938 + - -4.0195313 + - -0.9873047 + - 1.6308594 + - 0.43603516 + - 2.2363281 + - 2.2539063 + - 1.3857422 + - 4.2265625 + - -2.1757813 + - 5.3164063 + - 3.7636719 + - -3.6074219 + - -4.8515625 + - 7.4492188 + - -2.0019531 + - -2.921875 + - -5.3632813 + - -10.65625 + - -1.1523438 + - -2.0136719 + - -3.4921875 + - 3.7089844 + - -1.5585938 + - -1.4443359 + - 5.3828125 + - 3.7558594 + - -2.5097656 + - 1.2724609 + - -10.34375 + - -0.81884766 + - 7.7890625 + - 3.5351563 + - -0.9116211 + - -2.2734375 + - 4.171875 + - -1.5917969 + - -4.5976563 + - -1.7949219 + - 4.1992188 + - -1.2099609 + - 3.4199219 + - -5.9101563 + - -1.9345703 + - -10.2578125 + - -0.40527344 + - -2.3398438 + - -3.6699219 + - -5.4179688 + - -4.9648438 + - -5.6367188 + - -1.8330078 + - 1.6630859 + - -6.359375 + - 6.2851563 + - 1.6992188 + - -1.0722656 + - 5.765625 + - -0.43920898 + - 12.328125 + - 0.0869751 + - 3.8710938 + - -9.390625 + - -0.45825195 + - -1.1894531 + - 3.4160156 + - -3.6367188 + - -0.08227539 + - 3.3320313 + - -5.8828125 + - -5.2382813 + - 3.6289063 + - -9.7421875 + - -15.734375 + - 1.84375 + - -3.28125 + - 6.5703125 + - -1.015625 + - -6.390625 + - -0.7758789 + - -0.5004883 + - 3.4296875 + - -0.6660156 + - 1.7900391 + - -2.53125 + - -3.1582031 + - 2.1074219 + - 0.82373047 + - -4.15625 + - 0.3564453 + - -5.0625 + - 0.46826172 + - -0.20117188 + - 9.4296875 + - -1.3701172 + - 6.21875 + - -1.1826172 + - -0.50927734 + - 0.97509766 + - 1.5351563 + - -9.1015625 + - 1.2363281 + - 1.1484375 + - 4.2460938 + - 4.6484375 + - 5.0078125 + - -1.4257813 + - -8.8046875 + - -2.5878906 + - 3.6601563 + - 0.5317383 + - 0.8198242 + - -2.7597656 + - -0.5253906 + - 2.9589844 + - 10.453125 + - -3.3242188 + - 1.2519531 + - 7.5820313 + - -1.4306641 + - 0.2076416 + - 9.6015625 + - 1.8046875 + - 3.9414063 + - 3.5351563 + - -3.0703125 + - 0.03564453 + - -3.2148438 + - 1.125 + - -1.9433594 + - -4.84375 + - 1.5664063 + - 9.125 + - -3.109375 + - -5.609375 + - 1.5107422 + - 1.7890625 + - 1.6806641 + - 9.0703125 + - -5.46875 + - 0.28930664 + - 1.1523438 + - 2.3300781 + - -2.265625 + - -4.3164063 + - 1.2324219 + - 0.8979492 + - -0.7207031 + - 14.6171875 + - 6.0703125 + - 1.9316406 + - 2.6289063 + - -1.1337891 + - 0.21948242 + - -2.4433594 + - -0.31347656 + - -4.25 + - -3.2949219 + - 5.4609375 + - -3.3808594 + - 1.8662109 + - 1.7060547 + - 5.5273438 + - -0.16638184 + - 2.9453125 + - 1.7265625 + - -11.4921875 + - -0.5644531 + - 4.7539063 + - -4.1992188 + - 1.3359375 + - 3.3535156 + - 0.17980957 + - -2.5195313 + - -0.31640625 + - -2.515625 + - -5.7851563 + - 1.5869141 + - 3.09375 + - 4.25 + - -0.20983887 + - -1.2236328 + - 1.1054688 + - -1.4736328 + - -2.6875 + - 3.3945313 + - 1.2041016 + - 4.8710938 + - 0.4609375 + - -3.3300781 + - 0.5957031 + - 4.734375 + - -1.6015625 + - -2.8535156 + - -6.3476563 + - -3.4667969 + - -0.015586853 + - 2.0605469 + - -7.609375 + - -1.2441406 + - 1.4921875 + - 3.8867188 + - -2.015625 + - 3.8066406 + - 2.0976563 + - 3.3398438 + - -3.6445313 + - 0.39282227 + - 0.13439941 + - 2.8496094 + - 2.1542969 + - -0.36157227 + - 3.0332031 + - 4.4140625 + - 4.8359375 + - -1.9169922 + - -1.2685547 + - -0.16906738 + - 2.2167969 + - -1.4179688 + - -1.7714844 + - 5.78125 + - 8.3359375 + - -3.6367188 + - -6.6015625 + - 2.7207031 + - 4.8085938 + - -1.6699219 + - -0.8544922 + - 2.5 + - 6.7265625 + - -1.78125 + - -2.8378906 + - 2.7578125 + - 3.6367188 + - 0.48828125 + - -2.4121094 + - 4.46875 + - 0.028198242 + - 0.9223633 + - 0.29736328 + - -3.4199219 + - 0.06951904 + - 2.6640625 + - -0.0012645721 + - -1.8964844 + - -0.4909668 + - -2.859375 + - 5.4804688 + - 1.0214844 + - 1.234375 + - 1.5986328 + - -3.7890625 + - -2.46875 + - 2.8691406 + - -2.2089844 + - -1.4101563 + - -8.25 + - -1.3613281 + - 2.7675781 + - -6.8632813 + - -2.1367188 + - 1.6796875 + - 2.0703125 + - -4.4179688 + - 2.9082031 + - -0.06329346 + - 2.3125 + - -7.7890625 + - -3.7441406 + - -0.3395996 + - -0.15661621 + - 8.2109375 + - -2.4023438 + - -5.0976563 + - -2.3925781 + - -1.0058594 + - 3.0703125 + - -0.8886719 + - 5.75 + - -5.0195313 + - -4.2851563 + - 2.0683594 + - -5.28125 +- - -4.9882813 + - 6.7734375 + - -1.6269531 + - -3.21875 + - 7.078125 + - -0.5107422 + - -2.75 + - -1.5371094 + - 4.9257813 + - 0.9790039 + - 6.03125 + - -4.0195313 + - -4.9648438 + - 0.9160156 + - 0.63916016 + - -0.90771484 + - 5.375 + - -0.3293457 + - 2.8535156 + - 2.7773438 + - 0.25170898 + - -4.671875 + - 1.7490234 + - -1.7080078 + - 2.1933594 + - -0.7470703 + - -5.6445313 + - 0.31762695 + - 6.4375 + - 1.9873047 + - -5.265625 + - 0.6816406 + - 0.89404297 + - 12.1640625 + - -3.3867188 + - -5.015625 + - -2.890625 + - 1.3457031 + - -0.47094727 + - -1.1201172 + - 1.2382813 + - -0.6855469 + - -1.5957031 + - -0.39916992 + - 10.4921875 + - 5.3867188 + - -7.9570313 + - -5.6914063 + - 2.9589844 + - -2.2753906 + - 4.1210938 + - -0.3178711 + - -1.15625 + - 5.828125 + - 3.4746094 + - -0.51953125 + - 1.9931641 + - 1.15625 + - -3.9140625 + - -0.30517578 + - 4.390625 + - 0.96191406 + - -5.0078125 + - 5.8242188 + - 1.9091797 + - 2.2949219 + - -3.2460938 + - -3.9570313 + - 5.3789063 + - 6.4648438 + - 6.9375 + - 7.6210938 + - 0.76464844 + - -0.41430664 + - -3.7480469 + - 3.1015625 + - -0.22961426 + - -3.3007813 + - -5.2890625 + - -8.6953125 + - 2.3320313 + - -0.9013672 + - -5.3789063 + - -10.9453125 + - -0.20263672 + - -3.359375 + - -6.1367188 + - 2.6777344 + - -2.7753906 + - 2.0957031 + - -2.3105469 + - -2.7382813 + - -17.140625 + - 5.2070313 + - 3.4707031 + - -2.0371094 + - 0.82421875 + - -3.5566406 + - 2.1445313 + - -4.8164063 + - -1.2802734 + - 0.42407227 + - 0.98095703 + - 1.3271484 + - -1.9667969 + - 0.36328125 + - -3.90625 + - 5.0859375 + - -1.7578125 + - -1.8056641 + - -5.6210938 + - -2.875 + - -2.9453125 + - -1.5351563 + - 3.1445313 + - 3.8066406 + - 0.78759766 + - 2.8574219 + - 3.6015625 + - 10.6171875 + - -0.24499512 + - -0.52734375 + - -0.61816406 + - -2.9453125 + - -1.0263672 + - 3.359375 + - 6.6640625 + - -5.4179688 + - 0.9326172 + - 5.09375 + - -3.796875 + - 2.3691406 + - -3.9375 + - 6.0117188 + - 1.1669922 + - 0.07067871 + - -0.63916016 + - -7.7304688 + - -0.11804199 + - -5.1992188 + - -0.9707031 + - 2.8730469 + - 0.36157227 + - 4.0898438 + - -3.59375 + - -0.14904785 + - 6.3164063 + - 1.0546875 + - 12.78125 + - 5.265625 + - 2.4238281 + - -0.40429688 + - 2.5800781 + - -2.7871094 + - 0.19958496 + - -0.4580078 + - -4.5585938 + - -7.4570313 + - 0.7397461 + - -2.4921875 + - 2.3066406 + - 6.7929688 + - 5.4023438 + - -9.0859375 + - -0.38134766 + - 5.0117188 + - 3.9003906 + - 4.3515625 + - -3.7050781 + - 5.2226563 + - -4.4726563 + - -3.5234375 + - -3.5878906 + - 3.2519531 + - -4.1484375 + - 0.6220703 + - 0.796875 + - -0.8642578 + - -4.4023438 + - 7.171875 + - -2.1464844 + - -5.0898438 + - -3.7910156 + - 9.359375 + - 1.7636719 + - 2.4023438 + - -3.3632813 + - -0.3083496 + - 1.5644531 + - 8.6015625 + - 1.6787109 + - 3.1347656 + - 0.828125 + - -1.140625 + - 7.734375 + - 1.7255859 + - 0.40942383 + - -6.578125 + - 4.8867188 + - 8.0625 + - 3.3789063 + - 4.3359375 + - 1.0917969 + - -7.2265625 + - 11.5078125 + - 7.2929688 + - -0.17590332 + - -12.3046875 + - -1.6904297 + - -1.0439453 + - -6.5117188 + - -1.4707031 + - 3.1171875 + - -6.1835938 + - -0.48608398 + - 0.20043945 + - -2.1621094 + - -14.5859375 + - -2.4316406 + - -7.3242188 + - 2.8027344 + - -19.578125 + - 1.2392578 + - -0.20324707 + - -1.3583984 + - -3.8515625 + - 2.2460938 + - -0.011695862 + - -3.9257813 + - 5.6132813 + - 7.421875 + - -2.6035156 + - 1.0546875 + - -3.15625 + - -0.39282227 + - -1.7597656 + - -10.796875 + - -8.1875 + - -0.53271484 + - 1.5058594 + - 1.9404297 + - -0.18969727 + - 0.14782715 + - 1.4873047 + - -1.9814453 + - 2.9199219 + - -5.5859375 + - 0.45458984 + - -0.087768555 + - -1.2929688 + - -6.359375 + - -3.5957031 + - -3.4257813 + - -1.75 + - 6.2070313 + - 5.6796875 + - 2.3300781 + - 2.0546875 + - -0.85839844 + - 6.0039063 + - 6.4101563 + - 1.9902344 + - 0.55322266 + - -1.5361328 + - 0.30200195 + - 6.3828125 + - 4.6679688 + - 3.1835938 + - -10.171875 + - -1.8203125 + - -4.0351563 + - -5.3164063 + - 0.97802734 + - -3.1679688 + - 8.0703125 + - 0.1373291 + - 1.2431641 + - -0.47998047 + - 3.6601563 + - -6.7265625 + - -1.1728516 + - -1.5908203 + - 0.23620605 + - 5.5898438 + - 4.5234375 + - -5.7226563 + - -4.40625 + - 5.0234375 + - 1.5634766 + - 0.6376953 + - 4.3320313 + - -2.2246094 + - -1.6318359 + - -0.1784668 + - -7 + - -1.6015625 + - 0.10180664 + - 3.1777344 + - 4.5351563 + - 1.7783203 + - 9.765625 + - -0.8173828 + - 1.7431641 + - 0.5605469 + - -1.4628906 + - 5.53125 + - -5.7578125 + - -0.7885742 + - -4.2226563 + - -6.1171875 + - -0.27783203 + - 3.5566406 + - -1.7841797 + - 0.12915039 + - -1.7714844 + - -5.046875 + - -0.14331055 + - 4.3710938 + - -3.421875 + - -1.6601563 + - -2.8886719 + - -4.21875 + - 7.5625 + - -6.5195313 + - -4.1289063 + - 4.3203125 + - 2.7089844 + - -0.018585205 + - 2.265625 + - 2.2285156 + - -2.5683594 + - -5.6679688 + - -4.3085938 + - 0.05795288 + - 5.4296875 + - -1.515625 + - -1.5429688 + - -3.8125 + - 4.3242188 + - -2.0351563 + - 6.0898438 + - -8.421875 + - 1.3779297 + - 5.9804688 + - -2.3671875 + - -1.8222656 + - 0.30078125 + - 3.3515625 + - 5.6367188 + - 0.91796875 + - -4.109375 + - 0.05508423 + - 2.5761719 + - -5.8984375 + - -0.72998047 + - 3.875 + - -0.97998047 + - 9.25 + - 0.640625 + - 5.125 + - -7.7734375 + - -0.4169922 + - 5.8203125 + - 0.023406982 + - -4.9765625 + - 3.3613281 + - 2.6699219 + - 1.2724609 + - 1.9111328 + - -0.86621094 + - 1.2138672 + - -0.8588867 + - 0.053955078 + - 2.6777344 + - 1.3916016 + - -1.3056641 + - 9.7890625 + - -4.0390625 + - 8.625 + - 4.1835938 + - 1.5400391 + - -7.9414063 + - 2.4550781 + - -0.32177734 + - -1.7421875 + - -2.6972656 + - -1.7070313 + - -6.2539063 + - 1.2714844 + - 5.6132813 + - -1.7275391 + - 2.8261719 + - 7.0585938 + - -6.4453125 + - 1.7128906 + - 2.1816406 + - 2.4511719 + - -6.5351563 + - -1.2958984 + - 8.484375 + - 0.90234375 + - -5.25 + - -0.9995117 + - -1.2177734 + - 2.0703125 + - 0.79785156 + - -2.8261719 + - 1.6298828 + - -4.7148438 + - 9.40625 + - -0.09173584 + - 3.0546875 + - 0.1574707 + - 2.953125 + - 3.7773438 + - -3.0742188 + - -8.9375 + - 9.046875 + - 6.8203125 + - -1.7255859 + - 0.2626953 + - -5.984375 + - 2.5605469 + - -1.8837891 + - 2.0429688 + - 1.4042969 + - 1.9941406 + - -1.1591797 + - 2.6152344 + - 4.703125 + - -5.7460938 + - 0.14221191 + - -4.7539063 + - 4.578125 + - 5.7109375 + - -13.2734375 + - 1.0605469 + - 0.42138672 + - -3.5136719 + - 9.265625 + - 12.375 + - -0.6269531 + - -0.1352539 + - 5.5 + - 2.8027344 + - -0.9560547 + - 1.4589844 + - 3.3085938 + - 0.44677734 + - -0.15759277 + - 1.1865234 + - -6.4414063 + - 0.11468506 + - 2.5605469 + - -6.6953125 + - 0.7290039 + - -0.9814453 + - -0.4482422 + - 3.6191406 + - -11.109375 + - -2.4199219 + - -0.37817383 + - -4.4023438 + - 0.5107422 + - -1.9541016 + - -1.9423828 + - 4.6523438 + - 0.53466797 + - -1.1777344 + - -2.8515625 + - 1.3115234 + - 3.4199219 + - 2.6503906 + - -5.1796875 + - 0.5048828 + - -3.7246094 + - 0.83984375 + - -2.8574219 + - 0.23120117 + - 24.703125 + - 0.38232422 + - -3.6171875 + - -1.5263672 + - -1.0390625 + - 4.734375 + - -5.2539063 + - -0.95703125 + - 4.921875 + - -5.5898438 + - -3.3417969 + - -1.3525391 + - -0.8027344 + - 2.8535156 + - 7.7734375 + - 0.6699219 + - 4.7070313 + - -3.3847656 + - -3.4589844 + - -1.9072266 + - 0.2685547 + - 3.4707031 + - -3.1367188 + - -6.328125 + - 2.5371094 + - -1.0742188 + - -6.953125 + - -3.6738281 + - -7.65625 + - 5.046875 + - 2.3125 + - 4.5820313 + - 1.3212891 + - 13.640625 + - -3.5839844 + - 2.8710938 + - -5.4140625 + - -0.63378906 + - -0.013664246 + - 1.6123047 + - 1.6660156 + - 0.81933594 + - 2.390625 + - 0.98779297 + - -0.5605469 + - 6.3632813 + - 1.7509766 + - -9.1328125 + - 4.7109375 + - -7.0703125 + - 1.8564453 + - 7.3007813 + - -1.6103516 + - 0.048553467 + - 0.42163086 + - 2.2148438 + - 5.765625 + - 0.3413086 + - 0.67822266 + - 6.0585938 + - 1.4267578 + - -0.82128906 + - 1.7109375 + - -3.1972656 + - -1.5332031 + - 4.0234375 + - -2.8535156 + - 3.7539063 + - -3.4824219 + - 8.328125 + - 1.3730469 + - -2.9941406 + - -8.578125 + - 5.8046875 + - -2.8417969 + - -5.7773438 + - -1.1640625 + - -0.9033203 + - 4.0039063 + - -0.3083496 + - -3.1113281 + - -1.1943359 + - 7.1796875 + - -3.7792969 + - 7.3203125 + - 2.7539063 + - -1.5244141 + - -1.8544922 + - 2.2890625 + - 1.4609375 + - -9.1171875 + - -3.3105469 + - 4.484375 + - 3.4863281 + - 8.3984375 + - -1.7675781 + - -2.0234375 + - 8.4453125 + - 3.3222656 + - 6.2148438 + - 3.4746094 + - 4.8085938 + - -2.5410156 + - -0.58251953 + - -0.090148926 + - -2.890625 + - -0.17236328 + - -5.3554688 + - -5.8007813 + - -2.40625 + - 2.78125 + - -1.1171875 + - 6.5273438 + - 4.3867188 + - -2.9414063 + - 0.3076172 + - 6.0117188 + - -0.32128906 + - -2.9785156 + - -1.1113281 + - 3.5566406 + - 7.1875 + - 3.7851563 + - -2.5820313 + - 0.12878418 + - 7.1367188 + - 9.9921875 + - -0.06323242 + - 0.703125 + - 0.70166016 + - -0.7714844 + - -2.2695313 + - 0.033081055 + - 0.029953003 + - 6.6523438 + - 2.6367188 + - -3.8769531 + - -4.7773438 + - -5.1367188 + - 4.796875 + - -2.0136719 + - 3.34375 + - -3.4160156 + - -5.078125 + - 0.74902344 + - 2.3574219 + - -1.8564453 + - 1.9130859 + - -3.640625 + - -3.2773438 + - 1.1289063 + - 1.7148438 + - -1.8642578 + - 4.4296875 + - 0.4741211 + - 1.3945313 + - -5.5742188 + - -4.109375 + - -0.8774414 + - -7.5585938 + - -3.9628906 + - 7.3789063 + - -0.8852539 + - -0.62402344 + - -2.0722656 + - -3.5429688 + - 5.9609375 + - 8.21875 + - 3.890625 + - -3.3476563 + - -1.4951172 + - 2.9648438 + - 0.18395996 + - -3.9511719 + - -4.3398438 + - -2.3964844 + - -1.3242188 + - -0.8041992 + - -2.7441406 + - -1.4892578 + - 7.1210938 + - -6.9375 + - -2.296875 + - -2.4550781 + - 6.3242188 + - 1.0410156 + - -4.59375 + - -0.5957031 + - -0.8564453 + - 7.0195313 + - 1.7265625 + - 4.515625 + - 2.6054688 + - 5.5703125 + - -3.0195313 + - -3.1523438 + - 2.2929688 + - -4.4414063 + - 6.46875 + - 4.9375 + - -1.9814453 + - 4.4453125 + - 1.5048828 + - 4.5585938 + - -5.0117188 + - 1.3818359 + - 5.90625 + - -6.7695313 + - -1.9111328 + - -0.071777344 + - 0.9746094 + - -0.8261719 + - -1.3583984 + - 2.1132813 + - -3.2734375 + - 4.6132813 + - 3.1328125 + - 0.60839844 + - 3.640625 + - -4.6992188 + - -2.4726563 + - 1.2744141 + - 2.1425781 + - -0.7441406 + - 0.024169922 + - -2.4355469 + - -3.28125 + - 0.57373047 + - -7.5195313 + - -6.2070313 + - 4.7226563 + - 6.1914063 + - 8.3515625 + - 12.2421875 + - -2.53125 + - -1.2783203 + - -2.5019531 + - 0.6977539 + - 2.1933594 + - -1.5917969 + - 1.9853516 + - 0.14929199 + - -3.9550781 + - -1.8896484 + - 8.78125 + - 0.7001953 + - 7.5625 + - -4.9570313 + - -6.9882813 + - 5.9453125 + - -1.8261719 + - 5.9882813 + - 3.2597656 + - 0.9448242 + - -8.3203125 + - -2.0175781 + - -2.5527344 + - 1.1171875 + - -3.6699219 + - 3.6796875 + - 3.5019531 + - 2.9042969 + - -4.9257813 + - 5.4414063 + - -1.0283203 + - -1.8310547 + - -4.5351563 + - 6.1914063 + - -3.5488281 + - 1.0585938 + - -4.3867188 + - 0.074645996 + - 7.7734375 + - 6.8945313 + - -5.1367188 + - -0.5361328 + - 3.4316406 + - -2.5214844 + - 1.7695313 + - 0.6152344 + - -1.7314453 + - -2.2070313 + - -2.2402344 + - -4.0742188 + - -0.6567383 + - -2.8925781 + - 1.6826172 + - -4.6640625 + - -4.1601563 + - 3.5527344 + - 3.8261719 + - -0.7451172 + - -0.43188477 + - 1.5214844 + - 0.14746094 + - 1.9335938 + - 6.421875 + - -0.24707031 + - 7.6679688 + - -5.765625 + - 7.8671875 + - 0.8486328 + - 2.9960938 + - 1.3359375 + - 3.6855469 + - -3.7617188 + - 3.7382813 + - -6.0234375 + - -6.40625 + - 1.3740234 + - 1.9501953 + - 3.1679688 + - -0.5288086 + - -1.0214844 + - -4.5351563 + - 9.4375 + - -5.8984375 + - 8 + - 3.6210938 + - -15.5859375 + - -2.7734375 + - 1.0527344 + - 3.0488281 + - -7.7148438 + - -4.7148438 + - 1.1132813 + - -0.6948242 + - -5.7929688 + - 3.9042969 + - 10.0625 + - 0.10675049 + - -4.1992188 + - 1.2373047 + - 1.3027344 + - -0.26367188 + - 0.115478516 + - 21.40625 + - 1.1445313 + - -1.7099609 + - -0.13208008 + - 2.6171875 + - -4.9335938 + - 6.8164063 + - 2.7988281 + - -2.1015625 + - -5.8007813 + - -4.8515625 + - 0.9658203 + - 1.8544922 + - 0.90283203 + - -1.2851563 + - 2.1660156 + - -0.25219727 + - 0.020111084 + - -5.4804688 + - -2.8789063 + - 3.1835938 + - -4.2070313 + - -2.7792969 + - -1.0927734 + - 0.67626953 + - -1.7822266 + - 0.8491211 + - -9.9140625 + - -1.6845703 + - -1.7011719 + - 1.8603516 + - 5.21875 + - -0.21923828 + - -0.1303711 + - -3.2890625 + - 3.203125 + - 0.4099121 + - 1.0439453 + - 1.7529297 + - 6.2148438 + - 0.9067383 + - 2.5820313 + - 1.8457031 + - -0.9951172 + - -0.27026367 + - -2.7050781 + - -5.828125 + - -1.5742188 + - 5.6796875 + - -1.8222656 + - 5.390625 + - 4.8125 + - -2.6464844 + - -1.1611328 + - 0.83154297 + - -1.1591797 + - -3.9667969 + - 6.53125 + - 1.890625 + - -3.125 + - 3.7128906 + - -0.37451172 + - -10.171875 + - 0.27197266 + - 4.375 + - 2.171875 + - 4.0820313 + - 0.96191406 + - 6.5742188 + - 1.1435547 + - 3.4101563 + - 14.1484375 + - 2.0058594 + - -6.46875 + - -5.5351563 + - -2.3808594 + - 5.1484375 + - -2.9238281 + - -3.0195313 + - 2.390625 + - -1.4082031 + - 2.9199219 + - -1.7109375 + - 2.4238281 + - 5.2421875 + - -1.5576172 + - 3.0644531 + - 9.1953125 + - -6.7304688 + - 0.80126953 + - -0.8017578 + - -1.3261719 + - 0.9580078 + - 3.1933594 + - 3.2617188 + - -10.828125 + - -5.5234375 + - 4.6445313 + - -2.640625 + - -4.7734375 + - 0.61279297 + - 4.0820313 + - -0.9604492 + - -5.203125 + - 0.64453125 + - -0.8691406 + - -0.31176758 + - -2.9414063 + - -3.9609375 + - -6.171875 + - 0.4567871 + - -0.5551758 + - -4.7070313 + - 4.1835938 + - -2.7480469 + - 2.9570313 + - 0.69677734 + - 3.5410156 + - 3.2050781 + - -1.5 + - -3.6210938 + - -0.8125 + - 0.7993164 + - 0.234375 + - -1.6123047 + - 6.1835938 + - 4.0117188 + - 2.1601563 + - -4.8125 + - 0.29003906 + - 6.5546875 + - -0.56884766 + - -3.1816406 + - 4.7617188 + - -4.2578125 + - -0.0072288513 + - 4.6445313 + - -4.40625 + - -0.7939453 + - -5.5820313 + - -8.15625 + - -6.9921875 + - -4.0664063 + - -0.8754883 + - 0.9082031 + - -7.7304688 + - -1.1416016 + - -3.3574219 + - -7.6796875 + - -1.3916016 + - -3.9472656 + - 5.5429688 + - -2.8222656 + - -5.3554688 + - 0.6171875 + - -0.82177734 + - -0.1071167 + - 2.5546875 + - -4.1914063 + - 0.03567505 + - -1.4912109 + - -18.234375 + - -3.4882813 + - -0.61621094 + - -2.9511719 + - -2.3105469 + - -2.7207031 + - -4.4453125 + - 0.22436523 + - 3.4179688 + - 2.2519531 + - 3.9589844 + - 0.53515625 + - 0.16992188 + - -9.9765625 + - -1.6142578 + - -1.0683594 + - 2.5488281 + - 5.4765625 + - 3.3320313 + - 1.9384766 + - -5.3203125 + - -0.64501953 + - 1.9863281 + - -2.2285156 + - 0.6845703 + - -1.9570313 + - -0.99902344 + - -7.109375 + - 0.2980957 + - -3.0253906 + - 0.4033203 + - -1.8222656 + - 0.4440918 + - 2.0332031 + - -3.109375 + - -4.3320313 + - -9.6640625 + - -3.9921875 + - -0.23498535 + - -8.546875 + - -1.2666016 + - 7.6328125 + - 1.7714844 + - -3.4804688 + - -12.171875 + - -3.0878906 + - -7.7578125 + - -2.0957031 + - 0.08728027 + - 6.3476563 + - 2.7285156 + - -1.2490234 + - -2.7011719 + - 1.4521484 + - 2.7929688 + - 1.5078125 + - -1.7939453 + - -3.4882813 + - -7.3828125 + - -8.0234375 + - -1.2158203 + - -2.7832031 + - 1.6845703 + - -6.6835938 + - -5.9882813 + - -5.0429688 + - -0.43139648 + - -2.5878906 + - -4.0820313 + - -4.4101563 + - -0.0793457 + - -1.3710938 + - 0.28515625 + - -5.1914063 + - -4.4804688 + - -0.12731934 + - 2.2890625 + - -0.1171875 + - -2.6132813 + - -2.9160156 + - -5.3867188 + - 15.71875 + - 4.9179688 + - 2.6601563 + - 0.9135742 + - -7.9453125 + - -11.1875 + - 4.671875 + - 0.26464844 + - -0.44628906 + - -4.6914063 + - 8.9296875 + - -4.0117188 + - -0.059295654 + - 0.6850586 + - 2.0800781 + - -4.9609375 + - 0.091918945 + - 1.5673828 + - 0.88671875 + - -0.11450195 + - -4.34375 + - -0.3581543 + - 0.34960938 + - 3.2792969 + - 0.31469727 + - -1.2236328 + - 3.0585938 + - -2.1601563 + - -8.734375 + - 6.3984375 + - 0.15588379 + - -3.0253906 + - -3.7089844 + - 4.4804688 + - -3.390625 + - -4.046875 + - 0.32348633 + - 1.7861328 + - 5.8554688 + - 4.234375 + - -0.5810547 + - -1.4169922 + - 1.4697266 + - -1.1591797 + - 2.3320313 + - -3.3945313 + - -5.0351563 + - -4.8476563 + - -0.95751953 + - -1.7558594 + - -0.6669922 + - 0.06011963 + - -2.2792969 + - -0.08380127 + - -3.7167969 + - 7.8867188 + - 4.2304688 + - -2.8574219 + - -3.3203125 + - 5.625 + - -2.6347656 + - 2.0019531 + - -0.5317383 + - 1.3701172 + - -0.08496094 + - -3.8242188 + - -3.0117188 + - 4.203125 + - 4.7265625 + - -4.328125 + - 11.1875 + - -1.6337891 + - -0.8666992 + - -5.9023438 + - -5.859375 + - -0.9355469 + - -8.4375 + - -5.46875 + - 1.1240234 + - -2.1289063 + - -7.890625 + - 5.0898438 + - -5.2382813 + - -0.027267456 + - 2.1738281 + - 3.9980469 + - 6.4375 + - 4.203125 + - 5.4414063 + - -2.7070313 + - 0.10546875 + - -2.6132813 + - 5.5234375 + - -1.9365234 + - 3.7636719 + - -2.2285156 + - 1.6455078 + - 2.8125 + - -0.21313477 + - 5.1953125 + - 7.0351563 + - -2.828125 + - 6.2382813 + - -3.6425781 + - 5.7382813 + - -0.30249023 + - -3.5917969 + - -1.5507813 + - 6.0976563 + - -0.92333984 + - 2.9238281 + - -8.75 + - -0.2763672 + - 0.54296875 + - 3.0019531 + - -2.3320313 + - 1.6035156 + - 7.4257813 + - -5.984375 + - -0.48168945 + - 0.1427002 + - 0.8515625 + - -6.4023438 + - -1.2822266 + - -2.8828125 + - -1.2177734 + - 2.5761719 + - -3.8085938 + - 3.5957031 + - 11.265625 + - -9.984375 + - 3.9042969 + - 7.6015625 + - -2.4746094 + - 1.7685547 + - -0.30004883 + - -4.1484375 + - 0.3190918 + - -4.4609375 + - -0.3984375 + - 3.4667969 + - -1.7177734 + - 1.8867188 + - -8.171875 + - 3.046875 + - 6.234375 + - 0.22839355 + - 8.5234375 + - -3.0292969 + - -1.9580078 + - 4.546875 + - 3.2421875 + - 0.69628906 + - -2.8554688 + - 1.2177734 + - -6.3398438 + - 1.2490234 + - 1.2724609 + - -2.9121094 + - 1.3271484 + - -2.3066406 + - 0.703125 + - -6.4804688 + - -4.1015625 + - -2.4296875 + - -0.3515625 + - 1.6445313 + - -6.1914063 + - 4.171875 + - -13.109375 + - -1.1240234 + - 6.53125 + - 3.2871094 + - 4.0664063 + - 2.9746094 + - 2.9785156 + - 7.7773438 + - -2.7148438 + - -3.8300781 + - 6.3632813 + - 3.1503906 + - -8.09375 + - -1.3330078 + - -4.8632813 + - -2.21875 + - -0.6171875 + - -1.8818359 + - 4.5820313 + - -3.9160156 + - -3.078125 + - -1.4169922 + - 20.34375 + - -1.7783203 + - 2.125 + - -2.96875 + - 0.42333984 + - -4.7617188 + - -0.68115234 + - 1.4033203 + - 2.0527344 + - 3.875 + - 1.7890625 + - 0.61572266 + - -2.4238281 + - -0.953125 + - 5.6679688 + - 0.8208008 + - 0.022460938 + - -1.7705078 + - 6.6289063 + - 0.48901367 + - -6.4804688 + - -3.3085938 + - -12.15625 + - -0.35791016 + - -0.8105469 + - -2.0058594 + - 5.0039063 + - -3.0957031 + - 0.19445801 + - 10.140625 + - 4.4453125 + - -3.2050781 + - 1.7548828 + - -8.9921875 + - 1.5078125 + - 6.7304688 + - 9.0390625 + - -2.6171875 + - -5.4492188 + - 4.4492188 + - -0.63623047 + - -5.9296875 + - 0.28222656 + - 4.015625 + - 7.4023438 + - 4.328125 + - -6.84375 + - -2.6796875 + - -4.1992188 + - -2.0078125 + - 1.015625 + - -9.4296875 + - -6.2109375 + - -4.7890625 + - -10.484375 + - -0.08459473 + - 1.7314453 + - -5.8632813 + - 6.8164063 + - -2.2597656 + - -2.8125 + - 5.4101563 + - -2.2597656 + - 7.6367188 + - 0.67333984 + - 0.20532227 + - -12.8359375 + - -0.19580078 + - -1.0927734 + - 4.1835938 + - -2.2226563 + - 3.0800781 + - 3.5605469 + - -7.2460938 + - 0.02708435 + - 1.2529297 + - -9.328125 + - -8.2734375 + - -2.2617188 + - -0.78125 + - 6.7929688 + - -1.9189453 + - -11.796875 + - -0.23071289 + - 3.5976563 + - -2.78125 + - 0.56689453 + - -5.1992188 + - -2.3203125 + - -3.0351563 + - -4.1679688 + - 3.7460938 + - -2.4394531 + - -2.0507813 + - -3.3808594 + - -2.3320313 + - 3.6464844 + - 9.140625 + - -5.2070313 + - 3.0605469 + - 0.37329102 + - -1.1757813 + - -5.2304688 + - 2.1074219 + - -10.1484375 + - 1.8974609 + - 0.28686523 + - 0.6074219 + - 3.9355469 + - 5.15625 + - 0.68310547 + - -10.734375 + - -2.1621094 + - 1.3164063 + - -1.2197266 + - 2.8183594 + - -1.2998047 + - -0.5415039 + - 1.3759766 + - 12.25 + - -1.0390625 + - 4.4960938 + - 7.28125 + - 1.3095703 + - 1.9541016 + - 3.953125 + - 1.5693359 + - 1.5996094 + - -2.046875 + - 1.7978516 + - 4.3515625 + - -7.734375 + - 2.1425781 + - 0.71191406 + - -4.109375 + - 0.81689453 + - 12.375 + - -7.5390625 + - -10.0078125 + - 1.4414063 + - -0.7026367 + - 0.74365234 + - 9.2265625 + - -7.2148438 + - 1.0693359 + - 4.2070313 + - 2.0507813 + - -4.5859375 + - -6.4101563 + - -4.6992188 + - -3.4394531 + - -1.2207031 + - 17.21875 + - 2.9785156 + - 1.2050781 + - 5.796875 + - -4.9648438 + - 2.5605469 + - -2.625 + - -1.5097656 + - -8.375 + - -0.81152344 + - 3.8359375 + - -2.8007813 + - -0.53759766 + - 2.09375 + - 10.5 + - 6.3164063 + - 4.921875 + - 4.0585938 + - -6.2226563 + - -1.1142578 + - 3.1269531 + - -5.7695313 + - 6.15625 + - 2.7421875 + - -0.1463623 + - -6.1523438 + - 0.4934082 + - -5.0664063 + - -4.90625 + - 2.6738281 + - 4.734375 + - 4.1171875 + - -3.7304688 + - 1.7646484 + - 1.0849609 + - -1.8232422 + - -4.1835938 + - 1.6884766 + - 7.5703125 + - 5.0039063 + - 2.1699219 + - -5.6992188 + - -0.73828125 + - 1.7001953 + - -3.1308594 + - -3.1015625 + - -9.46875 + - -3.6074219 + - 6.9882813 + - 1.6279297 + - -2.3378906 + - -6.125 + - 2.8574219 + - 2.4628906 + - -1.3818359 + - 2.9199219 + - 0.46801758 + - 2.4082031 + - -1.7470703 + - -0.8823242 + - -0.16455078 + - 1.5869141 + - 0.17614746 + - -3.265625 + - -0.16381836 + - 5.3359375 + - 5.5703125 + - -5.15625 + - -2.5078125 + - 0.9633789 + - 12.1640625 + - -1.7744141 + - -1.1025391 + - 0.73779297 + - 8.3828125 + - 4.140625 + - -6.90625 + - 1.8007813 + - 6.4179688 + - -4.1328125 + - 0.4506836 + - 2.5058594 + - 6.0742188 + - -1.140625 + - 1.0488281 + - 5.3554688 + - 3.8164063 + - -2.2597656 + - 0.92041016 + - 4.9570313 + - 1.9746094 + - 9.4453125 + - 0.60791016 + - -2.4003906 + - -1.9726563 + - 2.2070313 + - -3.171875 + - -10.796875 + - 0.55810547 + - -6.234375 + - 4.9765625 + - 1.2148438 + - 0.44091797 + - 3.8847656 + - -2.8613281 + - -2.1171875 + - 2.5742188 + - -0.7753906 + - -1.8837891 + - -8.0859375 + - 3.7070313 + - 0.09869385 + - -4.3515625 + - -2.8476563 + - 4.8476563 + - 1.9140625 + - -2.6347656 + - 1.4101563 + - 0.88916016 + - 2.3554688 + - -9.1171875 + - 1.078125 + - 1.3027344 + - -2.2402344 + - 4.8671875 + - -3.1484375 + - -6.3945313 + - -1.2197266 + - 4.7617188 + - 0.3034668 + - -0.44091797 + - 5.7773438 + - -4.1445313 + - -4.4648438 + - -7.8125 + - -3.3515625 diff --git a/backends/candle/tests/snapshots/test_flash_qwen2__qwen2_single.snap b/backends/candle/tests/snapshots/test_flash_qwen2__qwen2_single.snap new file mode 100644 index 00000000..362e1ebf --- /dev/null +++ b/backends/candle/tests/snapshots/test_flash_qwen2__qwen2_single.snap @@ -0,0 +1,1541 @@ +--- +source: backends/candle/tests/test_flash_qwen2.rs +assertion_line: 72 +expression: embeddings_single +--- +- - -4.9960938 + - 6.8007813 + - -1.6386719 + - -3.2109375 + - 7.0507813 + - -0.5097656 + - -2.75 + - -1.5380859 + - 4.9414063 + - 0.96533203 + - 6.015625 + - -3.9980469 + - -4.96875 + - 0.9296875 + - 0.66064453 + - -0.93359375 + - 5.3671875 + - -0.3383789 + - 2.8457031 + - 2.7714844 + - 0.24035645 + - -4.671875 + - 1.7734375 + - -1.7138672 + - 2.2050781 + - -0.7290039 + - -5.6445313 + - 0.32226563 + - 6.4179688 + - 1.9970703 + - -5.2617188 + - 0.6928711 + - 0.91845703 + - 12.171875 + - -3.4003906 + - -5.015625 + - -2.8925781 + - 1.3515625 + - -0.47045898 + - -1.1005859 + - 1.25 + - -0.69433594 + - -1.6308594 + - -0.39135742 + - 10.484375 + - 5.359375 + - -7.9609375 + - -5.6796875 + - 2.953125 + - -2.2792969 + - 4.109375 + - -0.31152344 + - -1.1337891 + - 5.828125 + - 3.4648438 + - -0.5341797 + - 1.9941406 + - 1.1660156 + - -3.9179688 + - -0.30688477 + - 4.3945313 + - 0.94091797 + - -4.984375 + - 5.8085938 + - 1.9296875 + - 2.3066406 + - -3.2460938 + - -3.9511719 + - 5.3828125 + - 6.453125 + - 6.921875 + - 7.625 + - 0.7944336 + - -0.40039063 + - -3.7519531 + - 3.1035156 + - -0.24145508 + - -3.2636719 + - -5.3085938 + - -8.671875 + - 2.328125 + - -0.89697266 + - -5.3554688 + - -10.9140625 + - -0.19873047 + - -3.3789063 + - -6.1328125 + - 2.6660156 + - -2.7675781 + - 2.109375 + - -2.3144531 + - -2.7578125 + - -17.109375 + - 5.1796875 + - 3.4550781 + - -2.0351563 + - 0.81103516 + - -3.5664063 + - 2.1542969 + - -4.78125 + - -1.2695313 + - 0.4140625 + - 0.9916992 + - 1.3408203 + - -1.9746094 + - 0.35302734 + - -3.9082031 + - 5.0859375 + - -1.7539063 + - -1.7861328 + - -5.6210938 + - -2.8828125 + - -2.9453125 + - -1.5458984 + - 3.1503906 + - 3.8007813 + - 0.78271484 + - 2.8496094 + - 3.6035156 + - 10.6171875 + - -0.2565918 + - -0.5229492 + - -0.6142578 + - -2.9453125 + - -1.0302734 + - 3.359375 + - 6.671875 + - -5.4335938 + - 0.94677734 + - 5.0898438 + - -3.7832031 + - 2.3710938 + - -3.9414063 + - 6.0117188 + - 1.1777344 + - 0.06689453 + - -0.63183594 + - -7.75 + - -0.12237549 + - -5.203125 + - -0.9794922 + - 2.8847656 + - 0.37353516 + - 4.09375 + - -3.5800781 + - -0.14013672 + - 6.3320313 + - 1.0683594 + - 12.765625 + - 5.2617188 + - 2.4140625 + - -0.37573242 + - 2.5839844 + - -2.8105469 + - 0.21374512 + - -0.4416504 + - -4.5703125 + - -7.4765625 + - 0.74072266 + - -2.4765625 + - 2.3164063 + - 6.7929688 + - 5.3828125 + - -9.109375 + - -0.38305664 + - 5.0234375 + - 3.9101563 + - 4.3320313 + - -3.7070313 + - 5.2226563 + - -4.4882813 + - -3.5117188 + - -3.5878906 + - 3.2539063 + - -4.1757813 + - 0.62841797 + - 0.7871094 + - -0.86376953 + - -4.3945313 + - 7.140625 + - -2.171875 + - -5.0742188 + - -3.7890625 + - 9.34375 + - 1.7626953 + - 2.3925781 + - -3.3476563 + - -0.3005371 + - 1.5576172 + - 8.59375 + - 1.6884766 + - 3.1152344 + - 0.7973633 + - -1.1503906 + - 7.7421875 + - 1.7197266 + - 0.39697266 + - -6.578125 + - 4.8945313 + - 8.0703125 + - 3.3984375 + - 4.3398438 + - 1.1044922 + - -7.2421875 + - 11.53125 + - 7.3007813 + - -0.16821289 + - -12.28125 + - -1.6777344 + - -1.0253906 + - -6.4960938 + - -1.4628906 + - 3.0859375 + - -6.1796875 + - -0.49194336 + - 0.19616699 + - -2.1660156 + - -14.546875 + - -2.4335938 + - -7.296875 + - 2.8144531 + - -19.578125 + - 1.2509766 + - -0.21643066 + - -1.3544922 + - -3.8457031 + - 2.234375 + - -0.04156494 + - -3.9394531 + - 5.625 + - 7.4335938 + - -2.6210938 + - 1.0830078 + - -3.1738281 + - -0.39697266 + - -1.7597656 + - -10.8359375 + - -8.2109375 + - -0.5527344 + - 1.5097656 + - 1.9316406 + - -0.17895508 + - 0.14587402 + - 1.4873047 + - -1.9521484 + - 2.890625 + - -5.6171875 + - 0.4423828 + - -0.07745361 + - -1.2978516 + - -6.3554688 + - -3.5976563 + - -3.4277344 + - -1.7441406 + - 6.2148438 + - 5.6914063 + - 2.3378906 + - 2.0449219 + - -0.8623047 + - 6.0078125 + - 6.3945313 + - 1.9785156 + - 0.55566406 + - -1.5371094 + - 0.3017578 + - 6.3867188 + - 4.6679688 + - 3.1757813 + - -10.1640625 + - -1.8203125 + - -4.0546875 + - -5.3359375 + - 0.98339844 + - -3.1601563 + - 8.0859375 + - 0.15698242 + - 1.2539063 + - -0.48583984 + - 3.6367188 + - -6.7226563 + - -1.171875 + - -1.5888672 + - 0.23486328 + - 5.59375 + - 4.53125 + - -5.7070313 + - -4.390625 + - 5.0351563 + - 1.5537109 + - 0.6123047 + - 4.3476563 + - -2.2285156 + - -1.6386719 + - -0.1932373 + - -6.9960938 + - -1.6064453 + - 0.08795166 + - 3.1738281 + - 4.5390625 + - 1.7880859 + - 9.7578125 + - -0.82666016 + - 1.7480469 + - 0.56933594 + - -1.4628906 + - 5.5195313 + - -5.7578125 + - -0.81396484 + - -4.2070313 + - -6.0976563 + - -0.29736328 + - 3.5664063 + - -1.7910156 + - 0.1303711 + - -1.7832031 + - -5.0507813 + - -0.1394043 + - 4.3945313 + - -3.4082031 + - -1.6884766 + - -2.8925781 + - -4.1992188 + - 7.5625 + - -6.5195313 + - -4.125 + - 4.3125 + - 2.6972656 + - -0.018585205 + - 2.2734375 + - 2.2402344 + - -2.5351563 + - -5.6796875 + - -4.3085938 + - 0.068481445 + - 5.4335938 + - -1.5068359 + - -1.53125 + - -3.8144531 + - 4.3203125 + - -2.0097656 + - 6.0859375 + - -8.421875 + - 1.3945313 + - 6.0117188 + - -2.3554688 + - -1.8203125 + - 0.296875 + - 3.3613281 + - 5.609375 + - 0.91064453 + - -4.1640625 + - 0.05505371 + - 2.5898438 + - -5.8867188 + - -0.72802734 + - 3.8847656 + - -0.9814453 + - 9.2421875 + - 0.6357422 + - 5.1289063 + - -7.7773438 + - -0.42138672 + - 5.8476563 + - 0.025741577 + - -4.9804688 + - 3.3417969 + - 2.6757813 + - 1.2695313 + - 1.9111328 + - -0.86083984 + - 1.203125 + - -0.8442383 + - 0.06298828 + - 2.6914063 + - 1.3818359 + - -1.3125 + - 9.796875 + - -4.046875 + - 8.6171875 + - 4.1953125 + - 1.5576172 + - -7.9257813 + - 2.4628906 + - -0.3244629 + - -1.7304688 + - -2.7363281 + - -1.6865234 + - -6.2890625 + - 1.2685547 + - 5.6054688 + - -1.7246094 + - 2.8554688 + - 7.0625 + - -6.4296875 + - 1.7138672 + - 2.1933594 + - 2.4492188 + - -6.515625 + - -1.2949219 + - 8.4609375 + - 0.89941406 + - -5.21875 + - -0.97509766 + - -1.2324219 + - 2.0644531 + - 0.8183594 + - -2.8574219 + - 1.6621094 + - -4.703125 + - 9.421875 + - -0.103027344 + - 3.0566406 + - 0.13635254 + - 2.9472656 + - 3.7675781 + - -3.0820313 + - -8.9140625 + - 9.078125 + - 6.8242188 + - -1.7558594 + - 0.25634766 + - -5.9960938 + - 2.5488281 + - -1.8525391 + - 2.0527344 + - 1.4238281 + - 1.9941406 + - -1.1503906 + - 2.5820313 + - 4.703125 + - -5.7421875 + - 0.16186523 + - -4.7421875 + - 4.5625 + - 5.7265625 + - -13.25 + - 1.0585938 + - 0.4350586 + - -3.4902344 + - 9.2734375 + - 12.359375 + - -0.6357422 + - -0.124572754 + - 5.5039063 + - 2.7949219 + - -0.94921875 + - 1.4667969 + - 3.3203125 + - 0.44360352 + - -0.15441895 + - 1.2050781 + - -6.453125 + - 0.10571289 + - 2.578125 + - -6.7109375 + - 0.74609375 + - -0.9873047 + - -0.4951172 + - 3.640625 + - -11.140625 + - -2.4179688 + - -0.40185547 + - -4.40625 + - 0.5078125 + - -1.9570313 + - -1.9472656 + - 4.6640625 + - 0.51660156 + - -1.1796875 + - -2.8574219 + - 1.3056641 + - 3.4199219 + - 2.6523438 + - -5.171875 + - 0.5048828 + - -3.7070313 + - 0.86279297 + - -2.8691406 + - 0.23120117 + - 24.640625 + - 0.3701172 + - -3.6191406 + - -1.5244141 + - -1.0380859 + - 4.71875 + - -5.2773438 + - -0.9580078 + - 4.9179688 + - -5.6054688 + - -3.3203125 + - -1.3261719 + - -0.8261719 + - 2.8613281 + - 7.7460938 + - 0.6645508 + - 4.7226563 + - -3.3789063 + - -3.46875 + - -1.9121094 + - 0.27294922 + - 3.453125 + - -3.1210938 + - -6.3476563 + - 2.5351563 + - -1.0654297 + - -6.9414063 + - -3.6738281 + - -7.6484375 + - 5.046875 + - 2.3300781 + - 4.5820313 + - 1.3105469 + - 13.625 + - -3.5546875 + - 2.8828125 + - -5.3984375 + - -0.63183594 + - -0.016036987 + - 1.6474609 + - 1.6591797 + - 0.8183594 + - 2.4023438 + - 0.9926758 + - -0.55322266 + - 6.375 + - 1.7548828 + - -9.140625 + - 4.703125 + - -7.0976563 + - 1.8603516 + - 7.2851563 + - -1.6132813 + - 0.02911377 + - 0.43164063 + - 2.2402344 + - 5.765625 + - 0.35839844 + - 0.6904297 + - 6.0546875 + - 1.4228516 + - -0.8232422 + - 1.7285156 + - -3.2128906 + - -1.5332031 + - 4.0507813 + - -2.8613281 + - 3.7402344 + - -3.4824219 + - 8.3125 + - 1.390625 + - -2.9746094 + - -8.5703125 + - 5.8164063 + - -2.8300781 + - -5.7617188 + - -1.1474609 + - -0.92578125 + - 4.0117188 + - -0.29614258 + - -3.1054688 + - -1.203125 + - 7.1953125 + - -3.7753906 + - 7.3203125 + - 2.7734375 + - -1.5205078 + - -1.8730469 + - 2.3007813 + - 1.4492188 + - -9.1171875 + - -3.3105469 + - 4.484375 + - 3.5097656 + - 8.40625 + - -1.7578125 + - -2.0429688 + - 8.46875 + - 3.2988281 + - 6.203125 + - 3.4882813 + - 4.8085938 + - -2.5253906 + - -0.5834961 + - -0.06707764 + - -2.8964844 + - -0.16894531 + - -5.3710938 + - -5.8203125 + - -2.390625 + - 2.7773438 + - -1.1132813 + - 6.5390625 + - 4.375 + - -2.921875 + - 0.2998047 + - 6.0078125 + - -0.33642578 + - -2.984375 + - -1.1054688 + - 3.5332031 + - 7.2070313 + - 3.78125 + - -2.5390625 + - 0.12194824 + - 7.1328125 + - 9.9921875 + - -0.084228516 + - 0.6791992 + - 0.66796875 + - -0.76123047 + - -2.2597656 + - 0.029327393 + - 0.017471313 + - 6.6523438 + - 2.6425781 + - -3.8828125 + - -4.7773438 + - -5.15625 + - 4.7851563 + - -2.0273438 + - 3.3554688 + - -3.4199219 + - -5.0625 + - 0.74365234 + - 2.3789063 + - -1.8720703 + - 1.9023438 + - -3.6445313 + - -3.2441406 + - 1.1259766 + - 1.7216797 + - -1.8857422 + - 4.453125 + - 0.49365234 + - 1.4111328 + - -5.5742188 + - -4.1054688 + - -0.8588867 + - -7.6054688 + - -3.9433594 + - 7.3984375 + - -0.8857422 + - -0.62890625 + - -2.0683594 + - -3.5390625 + - 5.9726563 + - 8.2265625 + - 3.9296875 + - -3.3457031 + - -1.5029297 + - 2.9707031 + - 0.16552734 + - -3.9511719 + - -4.3515625 + - -2.3828125 + - -1.3417969 + - -0.8066406 + - -2.7402344 + - -1.4658203 + - 7.125 + - -6.96875 + - -2.3066406 + - -2.4609375 + - 6.3242188 + - 1.0380859 + - -4.59375 + - -0.5942383 + - -0.8486328 + - 7.0351563 + - 1.7070313 + - 4.53125 + - 2.625 + - 5.5390625 + - -3.0136719 + - -3.1464844 + - 2.2832031 + - -4.4453125 + - 6.4765625 + - 4.9648438 + - -1.9853516 + - 4.4453125 + - 1.4960938 + - 4.5625 + - -5.03125 + - 1.3789063 + - 5.8984375 + - -6.7695313 + - -1.9003906 + - -0.059814453 + - 1.0039063 + - -0.8183594 + - -1.3632813 + - 2.109375 + - -3.2890625 + - 4.6015625 + - 3.1347656 + - 0.61572266 + - 3.6699219 + - -4.6914063 + - -2.4804688 + - 1.2744141 + - 2.1523438 + - -0.7558594 + - 0.023544312 + - -2.4082031 + - -3.2578125 + - 0.59716797 + - -7.4960938 + - -6.2070313 + - 4.7460938 + - 6.1914063 + - 8.359375 + - 12.2109375 + - -2.5332031 + - -1.2861328 + - -2.484375 + - 0.7158203 + - 2.1933594 + - -1.5917969 + - 1.9853516 + - 0.1508789 + - -3.9667969 + - -1.875 + - 8.7890625 + - 0.7089844 + - 7.5859375 + - -4.9414063 + - -6.9726563 + - 5.9179688 + - -1.8388672 + - 5.984375 + - 3.2382813 + - 0.93408203 + - -8.3359375 + - -2.0605469 + - -2.5722656 + - 1.1279297 + - -3.6679688 + - 3.6679688 + - 3.4804688 + - 2.9121094 + - -4.90625 + - 5.4453125 + - -1.0234375 + - -1.8085938 + - -4.53125 + - 6.1992188 + - -3.578125 + - 1.0693359 + - -4.3828125 + - 0.09295654 + - 7.7890625 + - 6.9023438 + - -5.1328125 + - -0.515625 + - 3.4316406 + - -2.5097656 + - 1.78125 + - 0.6191406 + - -1.7460938 + - -2.2148438 + - -2.2421875 + - -4.0742188 + - -0.65527344 + - -2.90625 + - 1.6816406 + - -4.6679688 + - -4.140625 + - 3.5644531 + - 3.8457031 + - -0.7373047 + - -0.44580078 + - 1.5253906 + - 0.107666016 + - 1.9316406 + - 6.4101563 + - -0.25610352 + - 7.6679688 + - -5.7929688 + - 7.8945313 + - 0.8520508 + - 3.0117188 + - 1.3261719 + - 3.6914063 + - -3.7578125 + - 3.7519531 + - -6.0507813 + - -6.3945313 + - 1.3876953 + - 1.9541016 + - 3.1757813 + - -0.5571289 + - -1.0400391 + - -4.5351563 + - 9.4453125 + - -5.8710938 + - 8.0078125 + - 3.6621094 + - -15.5859375 + - -2.7792969 + - 1.046875 + - 3.0683594 + - -7.7148438 + - -4.7421875 + - 1.1230469 + - -0.6894531 + - -5.7890625 + - 3.9199219 + - 10.0625 + - 0.101257324 + - -4.1796875 + - 1.2480469 + - 1.3203125 + - -0.27148438 + - 0.115478516 + - 21.40625 + - 1.1445313 + - -1.7050781 + - -0.12695313 + - 2.609375 + - -4.953125 + - 6.8007813 + - 2.8164063 + - -2.109375 + - -5.8085938 + - -4.8476563 + - 0.9741211 + - 1.8603516 + - 0.9091797 + - -1.2900391 + - 2.1367188 + - -0.2775879 + - 0.025131226 + - -5.4648438 + - -2.8417969 + - 3.1738281 + - -4.1875 + - -2.7988281 + - -1.0859375 + - 0.6738281 + - -1.7871094 + - 0.84228516 + - -9.953125 + - -1.6787109 + - -1.6992188 + - 1.8554688 + - 5.2421875 + - -0.24304199 + - -0.12646484 + - -3.2910156 + - 3.203125 + - 0.40673828 + - 1.0351563 + - 1.765625 + - 6.2382813 + - 0.9277344 + - 2.5976563 + - 1.8193359 + - -0.9863281 + - -0.26635742 + - -2.6953125 + - -5.8359375 + - -1.5664063 + - 5.71875 + - -1.8359375 + - 5.3867188 + - 4.8164063 + - -2.6757813 + - -1.171875 + - 0.83496094 + - -1.1396484 + - -3.9433594 + - 6.5351563 + - 1.8955078 + - -3.1074219 + - 3.7167969 + - -0.3630371 + - -10.1875 + - 0.25561523 + - 4.359375 + - 2.1953125 + - 4.0820313 + - 0.9638672 + - 6.5546875 + - 1.1445313 + - 3.4003906 + - 14.1640625 + - 2.0019531 + - -6.4414063 + - -5.53125 + - -2.390625 + - 5.1523438 + - -2.9179688 + - -3.0097656 + - 2.3632813 + - -1.4111328 + - 2.921875 + - -1.7275391 + - 2.4277344 + - 5.2421875 + - -1.5703125 + - 3.0390625 + - 9.203125 + - -6.7304688 + - 0.8095703 + - -0.8105469 + - -1.3173828 + - 0.94873047 + - 3.2070313 + - 3.265625 + - -10.828125 + - -5.5390625 + - 4.640625 + - -2.6425781 + - -4.7734375 + - 0.64453125 + - 4.0859375 + - -0.96777344 + - -5.1992188 + - 0.6503906 + - -0.8745117 + - -0.30981445 + - -2.9609375 + - -3.9453125 + - -6.1523438 + - 0.44311523 + - -0.5488281 + - -4.6992188 + - 4.1601563 + - -2.7402344 + - 2.9355469 + - 0.7158203 + - 3.5253906 + - 3.2011719 + - -1.5136719 + - -3.609375 + - -0.81103516 + - 0.79589844 + - 0.25219727 + - -1.6132813 + - 6.1953125 + - 3.9785156 + - 2.1484375 + - -4.8085938 + - 0.31274414 + - 6.5585938 + - -0.56152344 + - -3.203125 + - 4.7578125 + - -4.2890625 + - 0.0036144257 + - 4.6523438 + - -4.3789063 + - -0.7910156 + - -5.5859375 + - -8.171875 + - -6.9921875 + - -4.0625 + - -0.8754883 + - 0.9135742 + - -7.7265625 + - -1.1416016 + - -3.3457031 + - -7.6601563 + - -1.4003906 + - -3.9570313 + - 5.5585938 + - -2.8339844 + - -5.34375 + - 0.62597656 + - -0.8334961 + - -0.10827637 + - 2.5410156 + - -4.1640625 + - 0.035461426 + - -1.4824219 + - -18.234375 + - -3.5136719 + - -0.61621094 + - -2.9492188 + - -2.3125 + - -2.7324219 + - -4.4375 + - 0.2253418 + - 3.4140625 + - 2.2773438 + - 3.96875 + - 0.52001953 + - 0.16186523 + - -9.984375 + - -1.5878906 + - -1.0664063 + - 2.5273438 + - 5.4726563 + - 3.3398438 + - 1.9677734 + - -5.3046875 + - -0.65478516 + - 1.9716797 + - -2.21875 + - 0.68408203 + - -1.9677734 + - -0.9736328 + - -7.125 + - 0.2919922 + - -3.0234375 + - 0.421875 + - -1.8300781 + - 0.45654297 + - 2.0429688 + - -3.1308594 + - -4.3320313 + - -9.6875 + - -4.0117188 + - -0.2512207 + - -8.546875 + - -1.2548828 + - 7.6445313 + - 1.7714844 + - -3.4707031 + - -12.15625 + - -3.0859375 + - -7.7617188 + - -2.0898438 + - 0.09484863 + - 6.34375 + - 2.7226563 + - -1.2441406 + - -2.7050781 + - 1.4414063 + - 2.7851563 + - 1.5 + - -1.7880859 + - -3.4667969 + - -7.3945313 + - -8.015625 + - -1.2070313 + - -2.78125 + - 1.703125 + - -6.6367188 + - -5.9804688 + - -5.0585938 + - -0.41381836 + - -2.5898438 + - -4.078125 + - -4.4101563 + - -0.11029053 + - -1.3505859 + - 0.2614746 + - -5.1835938 + - -4.4726563 + - -0.12854004 + - 2.2910156 + - -0.09680176 + - -2.6269531 + - -2.9023438 + - -5.421875 + - 15.734375 + - 4.890625 + - 2.6542969 + - 0.8989258 + - -7.96875 + - -11.1640625 + - 4.6835938 + - 0.24902344 + - -0.44335938 + - -4.6757813 + - 8.921875 + - -3.9941406 + - -0.05895996 + - 0.6982422 + - 2.0800781 + - -4.9375 + - 0.091918945 + - 1.5839844 + - 0.8911133 + - -0.09967041 + - -4.3867188 + - -0.35253906 + - 0.36157227 + - 3.2910156 + - 0.29663086 + - -1.2138672 + - 3.0644531 + - -2.1875 + - -8.7578125 + - 6.421875 + - 0.1496582 + - -3.0273438 + - -3.7128906 + - 4.4882813 + - -3.3886719 + - -4.0507813 + - 0.31323242 + - 1.7900391 + - 5.8515625 + - 4.21875 + - -0.57714844 + - -1.3984375 + - 1.4736328 + - -1.1748047 + - 2.3183594 + - -3.3847656 + - -5.015625 + - -4.84375 + - -0.9609375 + - -1.7646484 + - -0.703125 + - 0.055725098 + - -2.2929688 + - -0.08001709 + - -3.7246094 + - 7.8984375 + - 4.2265625 + - -2.875 + - -3.3457031 + - 5.640625 + - -2.6347656 + - 2.0097656 + - -0.5444336 + - 1.3837891 + - -0.0869751 + - -3.8085938 + - -3.0058594 + - 4.2109375 + - 4.7578125 + - -4.3320313 + - 11.1640625 + - -1.6669922 + - -0.8486328 + - -5.8945313 + - -5.8632813 + - -0.9379883 + - -8.4453125 + - -5.4570313 + - 1.1132813 + - -2.1230469 + - -7.9101563 + - 5.0976563 + - -5.2382813 + - -0.027252197 + - 2.1542969 + - 3.9746094 + - 6.4414063 + - 4.1953125 + - 5.4414063 + - -2.6855469 + - 0.093933105 + - -2.6210938 + - 5.515625 + - -1.9550781 + - 3.7851563 + - -2.2285156 + - 1.6621094 + - 2.8261719 + - -0.21313477 + - 5.1914063 + - 7.0585938 + - -2.84375 + - 6.21875 + - -3.6640625 + - 5.7851563 + - -0.29492188 + - -3.5859375 + - -1.5400391 + - 6.1171875 + - -0.92285156 + - 2.9316406 + - -8.7265625 + - -0.25317383 + - 0.546875 + - 2.9980469 + - -2.3164063 + - 1.6005859 + - 7.4179688 + - -6.0273438 + - -0.48217773 + - 0.13757324 + - 0.84765625 + - -6.4023438 + - -1.2714844 + - -2.8984375 + - -1.1894531 + - 2.5839844 + - -3.8261719 + - 3.5957031 + - 11.265625 + - -9.9765625 + - 3.890625 + - 7.6054688 + - -2.4746094 + - 1.7714844 + - -0.30078125 + - -4.1523438 + - 0.3125 + - -4.4492188 + - -0.3779297 + - 3.453125 + - -1.7099609 + - 1.8554688 + - -8.171875 + - 3.0625 + - 6.2382813 + - 0.23913574 + - 8.5234375 + - -3.0488281 + - -1.9902344 + - 4.5351563 + - 3.2519531 + - 0.7084961 + - -2.8632813 + - 1.2373047 + - -6.3320313 + - 1.2314453 + - 1.2734375 + - -2.9257813 + - 1.3251953 + - -2.2890625 + - 0.7158203 + - -6.4804688 + - -4.09375 + - -2.4257813 + - -0.36157227 + - 1.6533203 + - -6.1914063 + - 4.1601563 + - -13.125 + - -1.1171875 + - 6.5039063 + - 3.3125 + - 4.0625 + - 2.9902344 + - 2.9628906 + - 7.8007813 + - -2.71875 + - -3.8261719 + - 6.375 + - 3.1621094 + - -8.0859375 + - -1.3447266 + - -4.8710938 + - -2.1953125 + - -0.62402344 + - -1.9199219 + - 4.578125 + - -3.9277344 + - -3.0625 + - -1.4238281 + - 20.28125 + - -1.7685547 + - 2.109375 + - -2.984375 + - 0.41845703 + - -4.765625 + - -0.69091797 + - 1.3945313 + - 2.0566406 + - 3.9042969 + - 1.7929688 + - 0.6171875 + - -2.4375 + - -0.94677734 + - 5.6601563 + - 0.81689453 + - 0.041137695 + - -1.7695313 + - 6.6289063 + - 0.49926758 + - -6.5 + - -3.3007813 + - -12.1796875 + - -0.38208008 + - -0.8300781 + - -1.9794922 + - 5.0234375 + - -3.1015625 + - 0.17346191 + - 10.1328125 + - 4.4179688 + - -3.2011719 + - 1.7480469 + - -8.9765625 + - 1.4941406 + - 6.7226563 + - 9.0390625 + - -2.6210938 + - -5.4492188 + - 4.4453125 + - -0.6225586 + - -5.9101563 + - 0.28149414 + - 4.0273438 + - 7.40625 + - 4.3203125 + - -6.875 + - -2.65625 + - -4.1875 + - -1.9970703 + - 1.03125 + - -9.4453125 + - -6.1835938 + - -4.7851563 + - -10.5 + - -0.062408447 + - 1.7246094 + - -5.8710938 + - 6.8046875 + - -2.2578125 + - -2.8085938 + - 5.453125 + - -2.2617188 + - 7.625 + - 0.67285156 + - 0.17578125 + - -12.84375 + - -0.18786621 + - -1.0966797 + - 4.1523438 + - -2.2402344 + - 3.0917969 + - 3.5488281 + - -7.2695313 + - 0.022140503 + - 1.2539063 + - -9.3203125 + - -8.265625 + - -2.2792969 + - -0.7631836 + - 6.8046875 + - -1.9013672 + - -11.8046875 + - -0.24047852 + - 3.5859375 + - -2.7890625 + - 0.5673828 + - -5.1953125 + - -2.3183594 + - -3.0273438 + - -4.1875 + - 3.7460938 + - -2.4316406 + - -2.0429688 + - -3.3789063 + - -2.3417969 + - 3.6582031 + - 9.1484375 + - -5.2226563 + - 3.0546875 + - 0.36157227 + - -1.1728516 + - -5.2539063 + - 2.1308594 + - -10.15625 + - 1.9169922 + - 0.2722168 + - 0.60839844 + - 3.9472656 + - 5.1679688 + - 0.6791992 + - -10.6953125 + - -2.1757813 + - 1.3056641 + - -1.2558594 + - 2.8378906 + - -1.3095703 + - -0.5136719 + - 1.3476563 + - 12.265625 + - -1.0419922 + - 4.5078125 + - 7.2890625 + - 1.2998047 + - 1.9511719 + - 3.9648438 + - 1.5673828 + - 1.5996094 + - -2.0605469 + - 1.8046875 + - 4.3632813 + - -7.7304688 + - 2.1425781 + - 0.7246094 + - -4.0976563 + - 0.8198242 + - 12.375 + - -7.5390625 + - -10 + - 1.4423828 + - -0.7089844 + - 0.7211914 + - 9.21875 + - -7.203125 + - 1.078125 + - 4.21875 + - 2.0488281 + - -4.5898438 + - -6.4414063 + - -4.6914063 + - -3.4394531 + - -1.21875 + - 17.265625 + - 2.96875 + - 1.1923828 + - 5.8125 + - -4.9609375 + - 2.5761719 + - -2.640625 + - -1.4892578 + - -8.359375 + - -0.8305664 + - 3.8242188 + - -2.7851563 + - -0.5551758 + - 2.0800781 + - 10.5078125 + - 6.3085938 + - 4.9375 + - 4.046875 + - -6.2148438 + - -1.1171875 + - 3.1464844 + - -5.7695313 + - 6.1328125 + - 2.7324219 + - -0.15002441 + - -6.1601563 + - 0.4855957 + - -5.0703125 + - -4.9296875 + - 2.6796875 + - 4.75 + - 4.1015625 + - -3.7539063 + - 1.7714844 + - 1.0996094 + - -1.8242188 + - -4.1914063 + - 1.6787109 + - 7.546875 + - 4.984375 + - 2.171875 + - -5.7109375 + - -0.77441406 + - 1.6904297 + - -3.1386719 + - -3.1113281 + - -9.4765625 + - -3.6054688 + - 7.015625 + - 1.6132813 + - -2.328125 + - -6.1367188 + - 2.8710938 + - 2.4628906 + - -1.3759766 + - 2.9160156 + - 0.4428711 + - 2.3945313 + - -1.7441406 + - -0.88671875 + - -0.1694336 + - 1.5869141 + - 0.16357422 + - -3.2832031 + - -0.15612793 + - 5.3320313 + - 5.5351563 + - -5.1640625 + - -2.4960938 + - 0.9658203 + - 12.1640625 + - -1.7626953 + - -1.1142578 + - 0.7270508 + - 8.3671875 + - 4.140625 + - -6.8867188 + - 1.8105469 + - 6.4296875 + - -4.1367188 + - 0.45458984 + - 2.5019531 + - 6.0742188 + - -1.1699219 + - 1.0537109 + - 5.3554688 + - 3.8125 + - -2.2871094 + - 0.9238281 + - 4.9296875 + - 1.9853516 + - 9.484375 + - 0.5942383 + - -2.4199219 + - -1.9736328 + - 2.2128906 + - -3.1914063 + - -10.7734375 + - 0.5703125 + - -6.2734375 + - 4.9804688 + - 1.1855469 + - 0.4416504 + - 3.8925781 + - -2.8496094 + - -2.1054688 + - 2.5566406 + - -0.7841797 + - -1.8798828 + - -8.125 + - 3.7285156 + - 0.09362793 + - -4.3515625 + - -2.8378906 + - 4.8515625 + - 1.8964844 + - -2.6269531 + - 1.4208984 + - 0.90185547 + - 2.3457031 + - -9.125 + - 1.0859375 + - 1.3027344 + - -2.2382813 + - 4.875 + - -3.1582031 + - -6.3789063 + - -1.2128906 + - 4.7773438 + - 0.27392578 + - -0.43188477 + - 5.8125 + - -4.1367188 + - -4.4570313 + - -7.8046875 + - -3.3515625 diff --git a/backends/candle/tests/test_flash_qwen2.rs b/backends/candle/tests/test_flash_qwen2.rs new file mode 100644 index 00000000..38e45553 --- /dev/null +++ b/backends/candle/tests/test_flash_qwen2.rs @@ -0,0 +1,77 @@ +#![allow(dead_code, unused_imports)] + +mod common; + +use crate::common::{sort_embeddings, SnapshotEmbeddings}; +use anyhow::Result; +use common::{batch, cosine_matcher, download_artifacts, load_tokenizer}; +use text_embeddings_backend_candle::CandleBackend; +use text_embeddings_backend_core::{Backend, ModelType, Pool}; +use tokenizers::processors::sequence::Sequence; +use tokenizers::processors::template::TemplateProcessing; +use tokenizers::{PostProcessorWrapper, Tokenizer}; + +#[test] +#[serial_test::serial] +#[cfg(all(feature = "cuda", feature = "flash-attn"))] +fn test_flash_qwen2() -> Result<()> { + let model_root = download_artifacts("Alibaba-NLP/gte-Qwen2-1.5B-instruct", None)?; + let mut tokenizer = load_tokenizer(&model_root)?; + // Qwen2 updates the post processor manually instead of into the tokenizer.json... + // https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct/blob/main/tokenization_qwen.py#L246 + let template = TemplateProcessing::builder() + .try_single("$A:0 <|endoftext|>:0") + .unwrap() + .try_pair("$A:0 <|endoftext|>:0 $B:1 <|endoftext|>:1") + .unwrap() + .special_tokens(vec![("<|endoftext|>", 151643)]) + .build() + .unwrap(); + match tokenizer.get_post_processor() { + None => tokenizer.with_post_processor(template), + Some(post_processor) => { + let post_processor = Sequence::new(vec![ + post_processor.clone(), + PostProcessorWrapper::Template(template), + ]); + tokenizer.with_post_processor(post_processor) + } + }; + + let backend = CandleBackend::new( + model_root, + "float16".to_string(), + ModelType::Embedding(Pool::LastToken), + )?; + + let input_batch = batch( + vec![ + tokenizer.encode("What is Deep Learning?", true).unwrap(), + tokenizer.encode("Deep Learning is...", true).unwrap(), + tokenizer.encode("What is Deep Learning?", true).unwrap(), + ], + [0, 1, 2].to_vec(), + vec![], + ); + + let matcher = cosine_matcher(); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_batch)?); + let embeddings_batch = SnapshotEmbeddings::from(pooled_embeddings); + insta::assert_yaml_snapshot!("qwen2_batch", embeddings_batch, &matcher); + + let input_single = batch( + vec![tokenizer.encode("What is Deep Learning?", true).unwrap()], + [0].to_vec(), + vec![], + ); + + let (pooled_embeddings, _) = sort_embeddings(backend.embed(input_single)?); + let embeddings_single = SnapshotEmbeddings::from(pooled_embeddings); + + insta::assert_yaml_snapshot!("qwen2_single", embeddings_single, &matcher); + assert_eq!(embeddings_batch[0], embeddings_single[0]); + assert_eq!(embeddings_batch[2], embeddings_single[0]); + + Ok(()) +} diff --git a/docs/source/en/supported_models.md b/docs/source/en/supported_models.md index f638639f..1a0f997b 100644 --- a/docs/source/en/supported_models.md +++ b/docs/source/en/supported_models.md @@ -21,7 +21,7 @@ We are continually expanding our support for other model types and plan to inclu ## Supported embeddings models Text Embeddings Inference currently supports Nomic, BERT, CamemBERT, XLM-RoBERTa models with absolute positions, JinaBERT -model with Alibi positions and Mistral, Alibabe GTE models with Rope positions. +model with Alibi positions and Mistral, Alibaba GTE and Qwen2 models with Rope positions. Below are some examples of the currently supported models: diff --git a/router/src/lib.rs b/router/src/lib.rs index f5fd102c..57dcecb6 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -35,7 +35,9 @@ use text_embeddings_core::infer::Infer; use text_embeddings_core::queue::Queue; use text_embeddings_core::tokenization::Tokenization; use text_embeddings_core::TextEmbeddingsError; -use tokenizers::Tokenizer; +use tokenizers::processors::sequence::Sequence; +use tokenizers::processors::template::TemplateProcessing; +use tokenizers::{PostProcessorWrapper, Tokenizer}; use tracing::Span; pub use logging::init_logging; @@ -144,6 +146,28 @@ pub async fn run( "tokenizer.json not found. text-embeddings-inference only supports fast tokenizers", ); tokenizer.with_padding(None); + // Qwen2 updates the post processor manually instead of into the tokenizer.json... + // https://huggingface.co/Alibaba-NLP/gte-Qwen2-1.5B-instruct/blob/main/tokenization_qwen.py#L246 + if config.model_type == "qwen2" { + let template = TemplateProcessing::builder() + .try_single("$A:0 <|endoftext|>:0") + .unwrap() + .try_pair("$A:0 <|endoftext|>:0 $B:1 <|endoftext|>:1") + .unwrap() + .special_tokens(vec![("<|endoftext|>", 151643)]) + .build() + .unwrap(); + match tokenizer.get_post_processor() { + None => tokenizer.with_post_processor(template), + Some(post_processor) => { + let post_processor = Sequence::new(vec![ + post_processor.clone(), + PostProcessorWrapper::Template(template), + ]); + tokenizer.with_post_processor(post_processor) + } + }; + } // Position IDs offset. Used for Roberta and camembert. let position_offset = if &config.model_type == "xlm-roberta"