Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: move logic to file manager utils #1405

Merged
merged 1 commit into from
Oct 3, 2024
Merged

chore: move logic to file manager utils #1405

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions engine/commands/chat_completion_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ void ChatCompletionCmd::Exec(const std::string& host, int port,
return;
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto mc = yaml_handler.GetModelConfig();
Exec(host, port, model_handle, mc, std::move(msg));
Expand Down
4 changes: 2 additions & 2 deletions engine/commands/model_get_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ void ModelGetCmd::Exec(const std::string& model_handle) {
return;
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto model_config = yaml_handler.GetModelConfig();

Expand Down
2 changes: 1 addition & 1 deletion engine/commands/model_import_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ void ModelImportCmd::Exec() {
try {
// Use relative path for model_yaml_path. In case of import, we use absolute path for model
auto yaml_rel_path =
fmu::Subtract(fs::path(model_yaml_path), fmu::GetCortexDataPath());
fmu::ToRelativeCortexDataPath(fs::path(model_yaml_path));
cortex::db::ModelEntry model_entry{model_handle_, "local", "imported",
yaml_rel_path.string(), model_handle_};

Expand Down
6 changes: 3 additions & 3 deletions engine/commands/model_list_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,9 @@ void ModelListCmd::Exec() {
try {
count += 1;
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.path_to_model_yaml))
.string());
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.path_to_model_yaml))
.string());
auto model_config = yaml_handler.GetModelConfig();
table.add_row({std::to_string(count), model_entry.model,
model_entry.model_alias, model_config.engine,
Expand Down
5 changes: 2 additions & 3 deletions engine/commands/model_upd_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ void ModelUpdCmd::Exec(
CLI_LOG("Error: " + model_entry.error());
return;
}
auto yaml_fp =
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml));
auto yaml_fp = fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml));
yaml_handler_.ModelConfigFromFile(yaml_fp.string());
model_config_ = yaml_handler_.GetModelConfig();

Expand Down
4 changes: 2 additions & 2 deletions engine/commands/run_cmd.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ void RunCmd::Exec(bool chat_flag) {
return;
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto mc = yaml_handler.GetModelConfig();

Expand Down
2 changes: 1 addition & 1 deletion engine/config/yaml_config.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ void YamlHandler::ReadYamlFile(const std::string& file_path) {
if (yaml_node_["engine"] &&
yaml_node_["engine"].as<std::string>() == "cortex.llamacpp") {
auto abs_path = s.substr(0, s.find_last_of('/')) + "/model.gguf";
auto rel_path = fmu::Subtract(fs::path(abs_path), fmu::GetCortexDataPath());
auto rel_path = fmu::ToRelativeCortexDataPath(fs::path(abs_path));
v.emplace_back(rel_path.string());
} else {
v.emplace_back(s.substr(0, s.find_last_of('/')));
Expand Down
15 changes: 7 additions & 8 deletions engine/controllers/models.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ void Models::ListModel(
// auto model_entry = modellist_handler.GetModelInfo(model_handle);
try {
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.path_to_model_yaml))
.string());
auto model_config = yaml_handler.GetModelConfig();
Json::Value obj = model_config.ToJson();
Expand Down Expand Up @@ -132,8 +132,8 @@ void Models::GetModel(const HttpRequestPtr& req,
return;
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto model_config = yaml_handler.GetModelConfig();

Expand Down Expand Up @@ -187,9 +187,8 @@ void Models::UpdateModel(const HttpRequestPtr& req,
cortex::db::Models model_list_utils;
auto model_entry = model_list_utils.GetModelInfo(model_id);
config::YamlHandler yaml_handler;
auto yaml_fp =
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml));
auto yaml_fp = fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml));
yaml_handler.ModelConfigFromFile(yaml_fp.string());
config::ModelConfig model_config = yaml_handler.GetModelConfig();
model_config.FromJson(json_body);
Expand Down Expand Up @@ -243,7 +242,7 @@ void Models::ImportModel(
try {
// Use relative path for model_yaml_path. In case of import, we use absolute path for model
auto yaml_rel_path =
fmu::Subtract(fs::path(model_yaml_path), fmu::GetCortexDataPath());
fmu::ToRelativeCortexDataPath(fs::path(model_yaml_path));
cortex::db::ModelEntry model_entry{modelHandle, "local", "imported",
yaml_rel_path.string(), modelHandle};

Expand Down
37 changes: 17 additions & 20 deletions engine/services/model_service.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ void ParseGguf(const DownloadItem& ggufDownloadItem,
model_config.id =
ggufDownloadItem.localPath.parent_path().filename().string();
// use relative path for files
auto file_rel_path = fmu::Subtract(fs::path(ggufDownloadItem.localPath),
fmu::GetCortexDataPath());
auto file_rel_path =
fmu::ToRelativeCortexDataPath(fs::path(ggufDownloadItem.localPath));
model_config.files = {file_rel_path.string()};
model_config.model = ggufDownloadItem.id;
yaml_handler.UpdateModelConfig(model_config);
Expand All @@ -44,8 +44,7 @@ void ParseGguf(const DownloadItem& ggufDownloadItem,
auto branch = url_obj.pathParams[3];
CTL_INF("Adding model to modellist with branch: " << branch);

auto rel = file_manager_utils::Subtract(
yaml_name, file_manager_utils::GetCortexDataPath());
auto rel = file_manager_utils::ToRelativeCortexDataPath(yaml_name);
CTL_INF("path_to_model_yaml: " << rel.string());

auto author_id = author.has_value() ? author.value() : "cortexso";
Expand Down Expand Up @@ -304,8 +303,8 @@ cpp::result<std::string, std::string> ModelService::DownloadModelFromCortexso(
yaml_handler.UpdateModelConfig(mc);
yaml_handler.WriteYamlFile(model_yml_item->localPath.string());

auto rel = file_manager_utils::Subtract(
model_yml_item->localPath, file_manager_utils::GetCortexDataPath());
auto rel =
file_manager_utils::ToRelativeCortexDataPath(model_yml_item->localPath);
CTL_INF("path_to_model_yaml: " << rel.string());

cortex::db::Models modellist_utils_obj;
Expand Down Expand Up @@ -376,9 +375,8 @@ cpp::result<void, std::string> ModelService::DeleteModel(
CTL_WRN("Error: " + model_entry.error());
return cpp::fail(model_entry.error());
}
auto yaml_fp =
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml));
auto yaml_fp = fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml));
yaml_handler.ModelConfigFromFile(yaml_fp.string());
auto mc = yaml_handler.GetModelConfig();
// Remove yaml file
Expand All @@ -389,12 +387,12 @@ cpp::result<void, std::string> ModelService::DeleteModel(
if (mc.engine == "cortex.llamacpp") {
for (auto& file : mc.files) {
std::filesystem::path gguf_p(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(), fs::path(file)));
fmu::ToAbsoluteCortexDataPath(fs::path(file)));
std::filesystem::remove(gguf_p);
}
} else {
std::filesystem::path f(fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(mc.files[0])));
std::filesystem::path f(
fmu::ToAbsoluteCortexDataPath(fs::path(mc.files[0])));
std::filesystem::remove_all(f);
}
} else {
Expand Down Expand Up @@ -428,8 +426,8 @@ cpp::result<bool, std::string> ModelService::StartModel(
return cpp::fail(model_entry.error());
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto mc = yaml_handler.GetModelConfig();

Expand All @@ -439,8 +437,7 @@ cpp::result<bool, std::string> ModelService::StartModel(
if (mc.files.size() > 0) {
// TODO(sang) support multiple files
json_data["model_path"] =
fmu::GetAbsolutePath(fmu::GetCortexDataPath(), fs::path(mc.files[0]))
.string();
fmu::ToAbsoluteCortexDataPath(fs::path(mc.files[0])).string();
} else {
LOG_WARN << "model_path is empty";
return false;
Expand Down Expand Up @@ -489,8 +486,8 @@ cpp::result<bool, std::string> ModelService::StopModel(
return cpp::fail(model_entry.error());
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto mc = yaml_handler.GetModelConfig();

Expand Down Expand Up @@ -538,8 +535,8 @@ cpp::result<bool, std::string> ModelService::GetModelStatus(
return cpp::fail(model_entry.error());
}
yaml_handler.ModelConfigFromFile(
fmu::GetAbsolutePath(fmu::GetCortexDataPath(),
fs::path(model_entry.value().path_to_model_yaml))
fmu::ToAbsoluteCortexDataPath(
fs::path(model_entry.value().path_to_model_yaml))
.string());
auto mc = yaml_handler.GetModelConfig();

Expand Down
10 changes: 10 additions & 0 deletions engine/utils/file_manager_utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -317,4 +317,14 @@ inline std::filesystem::path Subtract(const std::filesystem::path& path,
}
}

inline std::filesystem::path ToRelativeCortexDataPath(
const std::filesystem::path& path) {
return Subtract(path, GetCortexDataPath());
}

inline std::filesystem::path ToAbsoluteCortexDataPath(
const std::filesystem::path& path) {
return GetAbsolutePath(GetCortexDataPath(), path);
}

} // namespace file_manager_utils
Loading