Skip to content

Commit

Permalink
get config out of loop
Browse files Browse the repository at this point in the history
  • Loading branch information
minhthuc2502 committed Mar 1, 2024
1 parent 76ebbfc commit 4b78fb6
Showing 1 changed file with 12 additions and 6 deletions.
18 changes: 12 additions & 6 deletions src/models/model.cc
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,18 @@ namespace ctranslate2 {
const auto num_variables = consume<uint32_t>(model_file);
model->_variable_index.reserve(num_variables);

// check config for tensor parallel
bool multi_query_attention = false;
if (tensor_parallel)
{

if (model->config.contains("multi_query_attention"))
multi_query_attention = model->config["multi_query_attention"];
else
spdlog::warn("Running model in mode tensor parallel but missing multi_query_attention option in"
" the config.json could lead to error! Try using the latest version of converters");
}

for (uint32_t i = 0; i < num_variables; ++i) {
auto name = consume<std::string>(model_file);
const size_t rank = consume<uint8_t>(model_file);
Expand All @@ -621,12 +633,6 @@ namespace ctranslate2 {
StorageView variable(std::move(shape), dtype);
consume<char>(model_file, num_bytes, static_cast<char*>(variable.buffer()));
if (tensor_parallel) {
bool multi_query_attention = false;
if (model->config.contains("multi_query_attention"))
multi_query_attention = model->config["multi_query_attention"];
else
spdlog::warn("Running model in mode tensor parallel but missing multi_query_attention option in"
" the config.json could lead to error! Try using the latest version of converters");
int outer_dim = 0;
int inner_dim = 1;
static dim_t model_dim = 0;
Expand Down

0 comments on commit 4b78fb6

Please sign in to comment.