Skip to content

Commit

Permalink
chore: cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@vansangpfiev @sangjanai
vansangpfiev authored and sangjanai committed Dec 28, 2024
1 parent ca35c1f commit 5da3540
Show file tree
Hide file tree
Showing 7 changed files with 68 additions and 97 deletions.
5 changes: 0 additions & 5 deletions engine/controllers/engines.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,11 +289,6 @@ void Engines::InstallRemoteEngine(
resp->setStatusCode(k400BadRequest);
callback(resp);
} else {
// auto gr = engine_service_->GenerateRemoteModel(engine);
// if (gr.has_error()) {
// CTL_INF("Error: " << gr.error());
// }

Json::Value res;
if (get_models_url.empty()) {
res["warning"] =
Expand Down
39 changes: 17 additions & 22 deletions engine/extensions/remote-engine/remote_engine.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,6 @@ bool is_anthropic(const std::string& model) {
return model.find("claude") != std::string::npos;
}

bool is_openai(const std::string& model) {
return model.find("gpt") != std::string::npos;
}

constexpr const std::array<std::string_view, 5> kAnthropicModels = {
"claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022",
"claude-3-opus-20240229", "claude-3-sonnet-20240229",
Expand Down Expand Up @@ -59,23 +55,20 @@ size_t StreamWriteCallback(char* ptr, size_t size, size_t nmemb,

// Parse the JSON
Json::Value chunk_json;
if (!is_openai(context->model)) {
std::string s = line.substr(6);
try {
auto root = json_helper::ParseJsonString(s);
root["model"] = context->model;
root["id"] = context->id;
root["stream"] = true;
auto result = context->renderer.Render(context->stream_template, root);
CTL_DBG(result);
chunk_json["data"] = "data: " + result + "\n\n";
} catch (const std::exception& e) {
CTL_WRN("JSON parse error: " << e.what());
continue;
}
} else {
chunk_json["data"] = line + "\n\n";
std::string s = line.substr(6);
try {
auto root = json_helper::ParseJsonString(s);
root["model"] = context->model;
root["id"] = context->id;
root["stream"] = true;
auto result = context->renderer.Render(context->stream_template, root);
CTL_DBG(result);
chunk_json["data"] = "data: " + result + "\n\n";
} catch (const std::exception& e) {
CTL_WRN("JSON parse error: " << e.what());
continue;
}

Json::Reader reader;

Json::Value status;
Expand Down Expand Up @@ -181,7 +174,7 @@ static size_t WriteCallback(char* ptr, size_t size, size_t nmemb,
}

RemoteEngine::RemoteEngine(const std::string& engine_name)
: engine_name_(engine_name) {
: engine_name_(engine_name), q_(1 /*n_parallel*/, engine_name) {
curl_global_init(CURL_GLOBAL_ALL);
}

Expand Down Expand Up @@ -552,7 +545,9 @@ void RemoteEngine::HandleChatCompletion(
}

if (is_stream) {
MakeStreamingChatCompletionRequest(*model_config, result, callback);
q_.runTaskInQueue([this, model_config, result, cb = std::move(callback)] {
MakeStreamingChatCompletionRequest(*model_config, result, cb);
});
} else {

auto response = MakeChatCompletionRequest(*model_config, result);
Expand Down
2 changes: 2 additions & 0 deletions engine/extensions/remote-engine/remote_engine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <unordered_map>
#include "cortex-common/remote_enginei.h"
#include "extensions/remote-engine/template_renderer.h"
#include "trantor/utils/ConcurrentTaskQueue.h"
#include "utils/engine_constants.h"
#include "utils/file_logger.h"
// Helper for CURL response
Expand Down Expand Up @@ -52,6 +53,7 @@ class RemoteEngine : public RemoteEngineI {
std::string chat_res_template_;
std::string api_key_header_;
std::string engine_name_;
trantor::ConcurrentTaskQueue q_;

// Helper functions
CurlResponse MakeChatCompletionRequest(const ModelConfig& config,
Expand Down
65 changes: 0 additions & 65 deletions engine/services/engine_service.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1140,71 +1140,6 @@ bool EngineService::IsRemoteEngine(const std::string& engine_name) {
return true;
}

cpp::result<bool, std::string> EngineService::GenerateRemoteModel(
const std::string& engine_name) {
namespace fmu = file_manager_utils;
namespace fs = std::filesystem;
auto exist_engine = GetEngineByNameAndVariant(engine_name);
if (exist_engine.has_error()) {
return cpp::fail("Remote engine '" + engine_name + "' is not installed");
}

if (!IsEngineLoaded(engine_name)) {
engines_[engine_name].engine = new remote_engine::RemoteEngine(engine_name);
CTL_INF("Loaded engine: " << engine_name);
}

auto remote_engine_json = exist_engine.value().ToJson();
auto& e = std::get<RemoteEngineI*>(engines_[engine_name].engine);
auto url = remote_engine_json["metadata"]["get_models_url"].asString();
auto api_key = remote_engine_json["api_key"].asString();
auto api_key_template =
remote_engine_json["metadata"]["api_key_template"].asString();
auto res = e->GetRemoteModels(url, api_key, api_key_template);
if (!res["error"].isNull()) {
return cpp::fail(res["error"].asString());
} else {
for (auto& d : res["data"]) {
auto model_handle = d["id"].asString();
config::RemoteModelConfig model_config;
Json::Value body =
json_helper::ParseJsonString(config::kDefaultRemoteModelConfig);
body["model"] = model_handle;
body["engine"] = engine_name;
// CTL_INF(body.toStyledString());
model_config.LoadFromJson(body);
cortex::db::Models modellist_utils_obj;

std::string model_yaml_path =
(file_manager_utils::GetModelsContainerPath() /
std::filesystem::path("remote") /
std::filesystem::path(model_handle + ".yml"))
.string();
try {
auto yaml_rel_path =
fmu::ToRelativeCortexDataPath(fs::path(model_yaml_path));
cortex::db::ModelEntry model_entry{
model_handle, "", "", yaml_rel_path.string(),
model_handle, "remote", "imported", cortex::db::ModelStatus::Remote,
engine_name};
std::filesystem::create_directories(
std::filesystem::path(model_yaml_path).parent_path());
if (modellist_utils_obj.AddModelEntry(model_entry).value()) {
model_config.SaveToYamlFile(model_yaml_path);
} else {
CTL_INF("Fail to import model, model_id '" + model_handle +
"' already exists!");
}
} catch (const std::exception& e) {
return cpp::fail("Error while adding Remote model with model_id '" +
model_handle + "': " + e.what());
}
}
}

return true;
}

cpp::result<std::vector<std::string>, std::string>
EngineService::GetSupportedEngineNames() {
return file_manager_utils::GetCortexConfig().supportedEngines;
Expand Down
3 changes: 0 additions & 3 deletions engine/services/engine_service.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,9 +152,6 @@ class EngineService : public EngineServiceI {

bool IsRemoteEngine(const std::string& engine_name) override;

cpp::result<bool, std::string> GenerateRemoteModel(
const std::string& engine_name);

private:
bool IsEngineLoaded(const std::string& engine);

Expand Down
3 changes: 3 additions & 0 deletions engine/services/model_service.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1253,5 +1253,8 @@ ModelService::GetModelMetadata(const std::string& model_id) const {

std::shared_ptr<ModelMetadata> ModelService::GetCachedModelMetadata(
const std::string& model_id) const {
if (loaded_model_metadata_map_.find(model_id) ==
loaded_model_metadata_map_.end())
return nullptr;
return loaded_model_metadata_map_.at(model_id);
}
48 changes: 46 additions & 2 deletions engine/test/components/test_remote_engine.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,19 +25,22 @@ TEST_F(RemoteEngineTest, OpenAiToAnthropicRequest) {
{% endfor %}
]
{% endif %}
{% if not loop.is_last %},{% endif %}
{% else if key == "system" or key == "model" or key == "temperature" or key == "store" or key == "max_tokens" or key == "stream" or key == "presence_penalty" or key == "metadata" or key == "frequency_penalty" or key == "tools" or key == "tool_choice" or key == "logprobs" or key == "top_logprobs" or key == "logit_bias" or key == "n" or key == "modalities" or key == "prediction" or key == "response_format" or key == "service_tier" or key == "seed" or key == "stop" or key == "stream_options" or key == "top_p" or key == "parallel_tool_calls" or key == "user" %}
"{{ key }}": {{ tojson(value) }}
{% if not loop.is_last %},{% endif %}
{% endif %}
{% if not loop.is_last %},{% endif %}
{% endfor %} })";
{
std::string message_with_system = R"({
"engine" : "anthropic",
"max_tokens" : 1024,
"messages": [
{"role": "system", "content": "You are a seasoned data scientist at a Fortune 500 company."},
{"role": "user", "content": "Hello, world"}
],
"model": "claude-3-5-sonnet-20241022",
"max_tokens": 1024,
"stream" : true
})";

auto data = json_helper::ParseJsonString(message_with_system);
Expand Down Expand Up @@ -78,4 +81,45 @@ TEST_F(RemoteEngineTest, OpenAiToAnthropicRequest) {
EXPECT_EQ(data["messages"][0]["content"].asString(),
res_json["messages"][0]["content"].asString());
}
}

TEST_F(RemoteEngineTest, OpenAiResponse) {
std::string tpl = R"({
{% set first = true %}
{% for key, value in input_request %}
{% if key == "choices" or key == "created" or key == "model" or key == "service_tier" or key == "system_fingerprint" or key == "stream" or key == "object" or key == "usage" %}
{% if not first %},{% endif %}
"{{ key }}": {{ tojson(value) }}
{% set first = false %}
{% endif %}
{% endfor %}
})";
std::string message = R"(
{
"choices": [
{
"delta": {
"content": " questions"
},
"finish_reason": null,
"index": 0
}
],
"created": 1735372587,
"id": "",
"model": "o1-preview",
"object": "chat.completion.chunk",
"stream": true,
"system_fingerprint": "fp_1ddf0263de"
})";
auto data = json_helper::ParseJsonString(message);

remote_engine::TemplateRenderer rdr;
auto res = rdr.Render(tpl, data);

auto res_json = json_helper::ParseJsonString(res);
EXPECT_EQ(data["model"].asString(), res_json["model"].asString());
EXPECT_EQ(data["created"].asInt(), res_json["created"].asInt());
EXPECT_EQ(data["choices"][0]["delta"]["content"].asString(),
res_json["choices"][0]["delta"]["content"].asString());
}

0 comments on commit 5da3540

Please sign in to comment.