error[E0597]: XXX does not live long enough

[aytekinar]

Hi,

Together with my colleague, @aykut-bozkurt, we have been trying out ort with Hugging Face's tokenizers library. We wanted to run intfloat/e5-small-v2 with the following dependencies:

# Cargo.toml

[package]
name = "embedding-playground"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
ndarray = "0.15.0"
ort = { version = "1.15.0", features = ["cuda"] }
tokenizers = { version = "0.14.0", default-features = false, features = ["esaxx_fast", "onig"] }

and the following source file:

// src/main.rs

use std::path::PathBuf;

use ndarray;
use ort;
use tokenizers;

fn main() {
    let tokenizer_path = PathBuf::from("models")
        .join(format!("tokenizer-e5-{}-v2", "small"))
        .join("tokenizer.json");
    let tokenizer =
        tokenizers::tokenizer::Tokenizer::from_file(tokenizer_path.to_str().unwrap()).unwrap();

    let mut encodings = tokenizer.encode_batch(vec![
      "query: how much protein should a female eat",
      "query: summit define",
      "passage: As a general guideline, the CDC's average requirement of protein for women ages 19 to 70 is 46 grams per day. But, as you can see from this chart, you'll need to increase that if you're expecting or training for a marathon. Check out the chart below to see how much protein you should be eating each day.",
      "passage: Definition of summit for English Language Learners. : 1  the highest point of a mountain : the top of a mountain. : 2  the highest level. : 3  a meeting or series of meetings between the leaders of two or more governments."
    ], true).unwrap();

    let model_path = PathBuf::from("models").join(format!("model-e5-{}-v2.onnx", "small"));
    let ort_env = ort::Environment::builder()
        .with_name("ortenv")
        .build()
        .unwrap();
    let session = ort::SessionBuilder::new(&ort_env.into_arc())
        .unwrap()
        .with_model_from_file(model_path)
        .unwrap();

    // Get input and output names
    println!("input_name: {:?}", session.inputs);
    println!("output_name: {:?}", session.outputs);

    let (input_ids, attention_mask, token_type_ids) = encodings.iter_mut().fold(
        (Vec::new(), Vec::new(), Vec::new()),
        |(mut input_ids, mut attention_mask, mut token_type_ids), encoding| {
            encoding.pad(
                512,
                u32::MAX,
                u32::MAX,
                "[PAD]",
                tokenizers::PaddingDirection::Right,
            );
            encoding.truncate(512, 0, tokenizers::TruncationDirection::Right);
            input_ids.extend(encoding.get_ids().iter().map(|item| *item as i64));
            attention_mask.extend(
                encoding
                    .get_attention_mask()
                    .iter()
                    .map(|item| *item as i64),
            );
            token_type_ids.extend(encoding.get_type_ids().iter().map(|item| *item as i64));
            (input_ids, attention_mask, token_type_ids)
        },
    );

    // Run inference
    let batch_size = encodings.len();
    let sequence_length = 512;

    let input_ids = ndarray::CowArray::from(&input_ids)
        .into_shape((batch_size, sequence_length))
        .unwrap()
        .into_dyn();
    let input_ids = ort::Value::from_array(session.allocator(), &input_ids).unwrap();

    print!("input_ids: {:?}", input_ids);

    let attention_mask = ndarray::CowArray::from(&attention_mask)
        .into_shape((batch_size, sequence_length))
        .unwrap()
        .into_dyn();
    let attention_mask = ort::Value::from_array(session.allocator(), &attention_mask).unwrap();

    print!("attention_mask: {:?}", attention_mask);

    let token_type_ids = ndarray::CowArray::from(&token_type_ids)
        .into_shape((batch_size, sequence_length))
        .unwrap()
        .into_dyn();
    let token_type_ids = ort::Value::from_array(session.allocator(), &token_type_ids).unwrap();

    print!("token_type_ids: {:?}", token_type_ids);

    let output = session
        .run(vec![input_ids, attention_mask, token_type_ids])
        .unwrap();
    print!("output: {:?}", output)
}

When we try to compile the code, we get the following error:

$ cargo run --release
   Compiling embedding-playground v0.1.0 (/home/aytekinar/Projects/work/embedding-playground)
error[E0597]: `attention_mask` does not live long enough
  --> src/main.rs:74:70
   |
70 |     let attention_mask = ndarray::CowArray::from(&attention_mask)
   |         -------------- binding `attention_mask` declared here
...
74 |     let attention_mask = ort::Value::from_array(session.allocator(), &attention_mask).unwrap();
   |                                                                      ^^^^^^^^^^^^^^^ borrowed value does not live long enough
...
90 | }
   | -
   | |
   | `attention_mask` dropped here while still borrowed
   | borrow might be used here, when `input_ids` is dropped and runs the `Drop` code for type `Value`
   |
   = note: values in a scope are dropped in the opposite order they are defined

error[E0597]: `token_type_ids` does not live long enough
  --> src/main.rs:82:70
   |
78 |     let token_type_ids = ndarray::CowArray::from(&token_type_ids)
   |         -------------- binding `token_type_ids` declared here
...
82 |     let token_type_ids = ort::Value::from_array(session.allocator(), &token_type_ids).unwrap();
   |                                                                      ^^^^^^^^^^^^^^^ borrowed value does not live long enough
...
90 | }
   | -
   | |
   | `token_type_ids` dropped here while still borrowed
   | borrow might be used here, when `input_ids` is dropped and runs the `Drop` code for type `Value`
   |
   = note: values in a scope are dropped in the opposite order they are defined

For more information about this error, try `rustc --explain E0597`.
error: could not compile `embedding-playground` (bin "embedding-playground") due to 2 previous errors

First, are we using the libraries as they are supposed to be used? We could not find an example that uses multiple inputs (from the tokenizers library). Is this a limitation on ort's or ndarray's side? Otherwise, what would you recommend?

[decahedron1]

Indeed this is a limitation on ort's part as of 1.15. This should be fixed with #78 and #89.

[aytekinar]

Thank you for your fast response! Looking forward to the patch...

[decahedron1]

I believe 9a631f1 should be stable enough to use as a Git dependency until the full release of v2.0.

[dependencies]
ort = { git = "https://github.com/pykeio/ort.git", rev = "9a631f1", features = [ "cuda" ]  }

A few changes are required:

• change all instances of session.allocator() to None as it is not required in this case
• use ort::inputs! instead of vec! to build the inputs array

	let input_ids = ndarray::CowArray::from(&input_ids)
		.into_shape((batch_size, sequence_length))
		.unwrap()
		.into_dyn();
	let input_ids = ort::Value::from_array(None, &input_ids).unwrap();

	print!("input_ids: {:?}", input_ids);

	let attention_mask = ndarray::CowArray::from(&attention_mask)
		.into_shape((batch_size, sequence_length))
		.unwrap()
		.into_dyn();
	let attention_mask = ort::Value::from_array(None, &attention_mask).unwrap();

	print!("attention_mask: {:?}", attention_mask);

	let token_type_ids = ndarray::CowArray::from(&token_type_ids)
		.into_shape((batch_size, sequence_length))
		.unwrap()
		.into_dyn();
	let token_type_ids = ort::Value::from_array(None, &token_type_ids).unwrap();

	print!("token_type_ids: {:?}", token_type_ids);

	let outputs = session.run(ort::inputs![input_ids, attention_mask, token_type_ids].unwrap()).unwrap();
	print!("output: {:?}", outputs[0])

[aytekinar]

Cool... THANKS!

I have the following lines in the end:

    // same as above

    let embeddings = session
        .run(ort::inputs![input_ids, attention_mask, token_type_ids].unwrap())
        .unwrap();
    let embeddings = embeddings[0].extract_tensor::<f32>().unwrap();
    let embeddings = embeddings.view();

    for embedding in embeddings.axis_iter(ndarray::Axis(0)) {
        println!("embedding: {:?}", embedding);
    }

This works, and I get some numbers, which are most likely correct.

How can I get matrix multiplication, for instance? That would be great to get the matrix multiplication embeddings * embeddings.t() to check the embeddings (w.r.t their relative distances to each other).

[decahedron1]

How can I get matrix multiplication, for instance?

See ndarray docs, it would look something like a.dot(b.t()). Note that ort only does the model inference, what you do with the inputs & outputs is done with ndarray.