From 891bfcd06918474cfa3aca5c5329ef8248d3a939 Mon Sep 17 00:00:00 2001 From: Waldemar Kozaczuk Date: Sun, 16 Jan 2022 18:59:24 -0500 Subject: [PATCH] httpserver-monitoring-api: stop using kernel internal C++ API When OSv kernel is built to hide most symbols but glibc ones, the OSv applications like httpserver monitoring API can not function corretly as they rely on number of internal C++ API. This patch modifies httpserver monitoring API to stop using kernel internal C++ API. It does so by replacing some of the calls to internal C++ symbols with new module C-style API symbols: for example, sched::with_all_threads() with new osv_get_all_threads(). In other scenarios, we fall back to standard glibc API: for example osv::current_mounts() is replaced with getmntent_r() and related functions. Finally, we link httpserver monitoring app with core/options.cc and thus remove need to have those symbols exposed by the kernel. Signed-off-by: Waldemar Kozaczuk --- modules/httpserver-api/api/fs.cc | 42 ++++++++++---- modules/httpserver-api/api/hardware.cc | 23 +++++--- modules/httpserver-api/api/network.cc | 19 ++++-- modules/httpserver-api/api/os.cc | 67 +++++++++++++--------- modules/httpserver-api/global_server.cc | 34 +++++++---- modules/httpserver-api/global_server.hh | 5 ++ modules/httpserver-api/openssl-init.cc | 15 +++-- modules/httpserver-api/ssl_server.cc | 9 ++- modules/httpserver-monitoring-api/Makefile | 6 +- 9 files changed, 150 insertions(+), 70 deletions(-) diff --git a/modules/httpserver-api/api/fs.cc b/modules/httpserver-api/api/fs.cc index 94eec77b7c..52e58c391c 100644 --- a/modules/httpserver-api/api/fs.cc +++ b/modules/httpserver-api/api/fs.cc @@ -6,12 +6,12 @@ */ #include "fs.hh" -#include "osv/mount.h" #include "json/formatter.hh" #include "autogen/fs.json.hh" #include #include #include +#include namespace httpserver { @@ -23,9 +23,9 @@ using namespace std; using namespace json; using namespace fs_json; -static void fill_dfstat(DFStat& dfstat, const osv::mount_desc& mount, const struct statvfs& st) { - dfstat.filesystem = mount.special; - dfstat.mount = mount.path; +static void fill_dfstat(DFStat& dfstat, mntent* mount, const struct statvfs& st) { + dfstat.filesystem = mount->mnt_fsname; + dfstat.mount = mount->mnt_dir; dfstat.btotal = st.f_blocks; dfstat.bfree = st.f_bfree; dfstat.ftotal = st.f_files; @@ -46,21 +46,31 @@ void init(routes& routes) { getDFStats.set_handler("json", [](const_req req) { - using namespace osv; const std::string onemount = req.param.at("mount"); struct statvfs st; httpserver::json::DFStat dfstat; vector dfstats; - for (mount_desc mount : osv::current_mounts()) { - if ((mount.type == "zfs" || mount.type == "rofs") && (onemount == "" || onemount == mount.path)) { - if (statvfs(mount.path.c_str(),&st) != 0) { + FILE *mounts_fp = setmntent("/proc/mounts", "r"); + if (!mounts_fp) { + throw server_error_exception("failed to get mounts information"); + } + + struct mntent* mount; + mntent mnt; + char strings[4096]; + while ((mount = getmntent_r(mounts_fp, &mnt, strings, sizeof(strings)))) { + std::string fstype(mount->mnt_type); + if ((fstype == "zfs" || fstype == "rofs") && (onemount == "" || onemount == mount->mnt_dir)) { + if (statvfs(mount->mnt_dir,&st) != 0) { + endmntent(mounts_fp); throw not_found_exception("mount does not exist"); } fill_dfstat(dfstat, mount, st); dfstats.push_back(dfstat); } }; + endmntent(mounts_fp); // checking if a specific file system was requested and if we found it if (onemount != "" && dfstats.size() == 0) { @@ -76,14 +86,24 @@ void init(routes& routes) { httpserver::json::DFStat dfstat; vector res; - for (osv::mount_desc mount : osv::current_mounts()) { - if (mount.type == "zfs" || mount.type == "rofs") { - if (statvfs(mount.path.c_str(),&st) == 0) { + FILE *mounts_fp = setmntent("/proc/mounts", "r"); + if (!mounts_fp) { + throw server_error_exception("failed to get mounts information"); + } + + struct mntent* mount; + mntent mnt; + char strings[4096]; + while ((mount = getmntent_r(mounts_fp, &mnt, strings, sizeof(strings)))) { + std::string fstype(mount->mnt_type); + if (fstype == "zfs" || fstype == "rofs") { + if (statvfs(mount->mnt_dir,&st) == 0) { fill_dfstat(dfstat, mount, st); res.push_back(dfstat); } } } + endmntent(mounts_fp); return res; }); diff --git a/modules/httpserver-api/api/hardware.cc b/modules/httpserver-api/api/hardware.cc index f023e394ba..fe069913e0 100644 --- a/modules/httpserver-api/api/hardware.cc +++ b/modules/httpserver-api/api/hardware.cc @@ -10,9 +10,8 @@ #include "autogen/hardware.json.hh" #include "processor.hh" #include "cpuid.hh" -#include -#include -#include +#include +#include namespace httpserver { @@ -30,26 +29,36 @@ extern "C" void httpserver_plugin_register_routes(httpserver::routes* routes) { } #endif +static std::string from_c_string(char *c_str) { + if (c_str) { + std::string str(c_str); + free(c_str); + return str; + } else { + return std::string(); + } +} + void init(routes& routes) { hardware_json_init_path("Hardware management API"); processorFeatures.set_handler([](const_req req) { - return processor::features_str(); + return from_c_string(osv_processor_features()); }); processorCount.set_handler([](const_req req) { - return sched::cpus.size(); + return get_nprocs(); }); firmware_vendor.set_handler([](const_req) { - return osv::firmware_vendor(); + return from_c_string(osv_firmware_vendor()); }); hypervisor_name.set_handler([](const_req) { - return osv::hypervisor_name(); + return from_c_string(osv_hypervisor_name()); }); } diff --git a/modules/httpserver-api/api/network.cc b/modules/httpserver-api/api/network.cc index 7d31faf812..595faf4c92 100644 --- a/modules/httpserver-api/api/network.cc +++ b/modules/httpserver-api/api/network.cc @@ -10,7 +10,7 @@ #include "../libtools/network_interface.hh" #include "exception.hh" #include -#include +#include namespace httpserver { @@ -57,13 +57,16 @@ void init(routes& routes) network_json_init_path("Hardware management API"); network_json::listIfconfig.set_handler([](const_req req) { vector res; - auto time = duration_cast - (osv::clock::uptime::now().time_since_epoch()).count(); + timespec time; + if (clock_gettime(CLOCK_BOOTTIME, &time)) { + return res; + } + auto time_mc = time.tv_sec * 1000000 + time.tv_nsec / 1000; for (unsigned int i = 0; i <= number_of_interfaces(); i++) { auto* ifp = get_interface_by_index(i); if (ifp != nullptr) { - res.push_back(get_interface(get_interface_name(ifp), ifp, time)); + res.push_back(get_interface(get_interface_name(ifp), ifp, time_mc)); } } return res; @@ -76,8 +79,12 @@ void init(routes& routes) if (ifp == nullptr) { throw not_found_exception(string("Interface ") + name + " not found"); } - auto time = duration_cast(osv::clock::uptime::now().time_since_epoch()).count(); - return get_interface(name, ifp, time); + timespec time; + if (clock_gettime(CLOCK_BOOTTIME, &time)) { + throw not_found_exception("Failed to get time"); + } + auto time_mc = time.tv_sec * 1000000 + time.tv_nsec / 1000; + return get_interface(name, ifp, time_mc); }); network_json::getRoute.set_handler([](const_req req) { diff --git a/modules/httpserver-api/api/os.cc b/modules/httpserver-api/api/os.cc index 036f916285..38a662a793 100644 --- a/modules/httpserver-api/api/os.cc +++ b/modules/httpserver-api/api/os.cc @@ -6,23 +6,20 @@ */ #include +#include #include "os.hh" -#include "osv/version.hh" #include "json/formatter.hh" #include "autogen/os.json.hh" #include #include #include #include -#include -#include #include #include +#include #include #include "../java-base/balloon/balloon_api.hh" -extern char debug_buffer[DEBUG_BUFFER_SIZE]; - namespace httpserver { namespace api { @@ -39,6 +36,16 @@ extern "C" void httpserver_plugin_register_routes(httpserver::routes* routes) { } #endif +static std::string from_c_string(char *c_str) { + if (c_str) { + std::string str(c_str); + free(c_str); + return str; + } else { + return std::string(); + } +} + void init(routes& routes) { os_json_init_path("OS core API"); @@ -48,7 +55,7 @@ void init(routes& routes) }); os_version.set_handler([](const_req req) { - return osv::version(); + return from_c_string(osv_version()); }); os_vendor.set_handler([](const_req req) { @@ -103,7 +110,7 @@ void init(routes& routes) #endif os_dmesg.set_handler([](const_req req) { - return debug_buffer; + return osv_debug_buffer(); }); os_get_hostname.set_handler([](const_req req) @@ -122,31 +129,39 @@ void init(routes& routes) #endif os_threads.set_handler([](const_req req) { - using namespace std::chrono; httpserver::json::Threads threads; - threads.time_ms = duration_cast - (osv::clock::wall::now().time_since_epoch()).count(); + timeval timeofday; + if (gettimeofday(&timeofday, nullptr)) { + return threads; + } + threads.time_ms = timeofday.tv_sec * 1000 + timeofday.tv_usec / 1000; httpserver::json::Thread thread; - sched::with_all_threads([&](sched::thread &t) { - thread.id = t.id(); - thread.status = t.get_status(); - auto tcpu = t.tcpu(); - thread.cpu = tcpu ? tcpu->id : -1; - thread.cpu_ms = duration_cast(t.thread_clock()).count(); - thread.switches = t.stat_switches.get(); - thread.migrations = t.stat_migrations.get(); - thread.preemptions = t.stat_preemptions.get(); - thread.name = t.name(); - thread.priority = t.priority(); - thread.stack_size = t.get_stack_info().size; - thread.status = t.get_status(); - threads.list.push(thread); - }); + osv_thread *osv_threads; + size_t threads_num; + if (!osv_get_all_threads(&osv_threads, &threads_num)) { + for (size_t i = 0; i < threads_num; i++) { + auto &t = osv_threads[i]; + thread.id = t.id; + thread.status = t.status; + thread.cpu = t.cpu_id; + thread.cpu_ms = t.cpu_ms; + thread.switches = t.switches; + thread.migrations = t.migrations; + thread.preemptions = t.preemptions; + thread.name = t.name; + free(t.name); + thread.priority = t.priority; + thread.stack_size = t.stack_size; + thread.status = t.status; + threads.list.push(thread); + } + free(osv_threads); + } return threads; }); os_get_cmdline.set_handler([](const_req req) { - return osv::getcmdline(); + return from_c_string(osv_cmdline()); }); #if !defined(MONITORING) diff --git a/modules/httpserver-api/global_server.cc b/modules/httpserver-api/global_server.cc index 5cf1c62eff..649e62cf39 100644 --- a/modules/httpserver-api/global_server.cc +++ b/modules/httpserver-api/global_server.cc @@ -8,15 +8,14 @@ #include "global_server.hh" #include "path_holder.hh" #include -#include #include #include #include #include #include "json/api_docs.hh" -#include #include "transformers.hh" #include +#include #if !defined(MONITORING) #include "yaml-cpp/yaml.h" #else @@ -41,6 +40,13 @@ global_server& global_server::get() return *instance; } +void global_server::termination_handler() { + get().s->close(); + for( auto plugin : get().plugins) { + dlclose(plugin); + } +} + bool global_server::run(std::map>& _config) { if (get().s != nullptr) { @@ -107,12 +113,7 @@ bool global_server::run(std::map>& _config) auto port = get().config["port"][0]; get().s = new http::server::server(get().config, &get()._routes); - osv::this_application::on_termination_request([&] { - get().s->close(); - for( auto plugin : get().plugins) { - dlclose(plugin); - } - }); + osv_current_app_on_termination_request(termination_handler); std::cout << "Rest API server running on port " << port << std::endl; get().s->run(); @@ -121,6 +122,7 @@ bool global_server::run(std::map>& _config) #if !defined(MONITORING) void global_server::setup_file_mappings(const YAML::Node& file_mappings_node) { + auto debug_enabled = osv_debug_enabled(); for (auto node : file_mappings_node) { const YAML::Node path = node["path"]; if (path && node["directory"]) { @@ -130,7 +132,9 @@ void global_server::setup_file_mappings(const YAML::Node& file_mappings_node) { new directory_handler(directory, new content_replace(content_replace_node.as())) : new directory_handler(directory); _routes.add(GET, url(path.as()).remainder("path"), handler); - debug("httpserver: setup directory mapping: [%s] -> [%s]\n", path.as().c_str(), directory.c_str()); + if (debug_enabled) { + std::cout << "httpserver: setup directory mapping: [" << path.as() << "] -> [" << directory << "]" << std::endl; + } } else if (path && node["file"]) { const std::string file = node["file"].as(); @@ -142,7 +146,9 @@ void global_server::setup_file_mappings(const YAML::Node& file_mappings_node) { else { _routes.add(GET, url(path.as()).remainder("path"), handler); } - debug("httpserver: setup file mapping: [%s] -> [%s]\n", path.as().c_str(), file.c_str()); + if (debug_enabled) { + std::cout << "httpserver: setup file mapping: [" << path.as() << "] -> [" << file << "]" << std::endl; + } } } } @@ -165,7 +171,9 @@ void global_server::setup_redirects(const YAML::Node& redirects_node) { return ""; }); _routes.put(GET, path, redirect); - debug("httpserver: setup redirect: [%s] -> [%s]\n", path.c_str(), target_path.c_str()); + if (osv_debug_enabled()) { + std::cout << "httpserver: setup redirect: [" << path << "] -> [" << target_path << "]" << std::endl; + } } } } @@ -242,6 +250,8 @@ void global_server::load_plugin(const std::string& path) } plugins.push_back(plugin); httpserver_plugin_register_routes(&_routes); - debug("httpserver: loaded plugin from path: %s\n",path.c_str()); + if (osv_debug_enabled()) { + std::cout << "httpserver: loaded plugin from path: " << path << std::endl; + } } } diff --git a/modules/httpserver-api/global_server.hh b/modules/httpserver-api/global_server.hh index e70b785429..587860fbf1 100644 --- a/modules/httpserver-api/global_server.hh +++ b/modules/httpserver-api/global_server.hh @@ -29,6 +29,11 @@ public: */ static global_server& get(); + /** + * cleanup routine: shutdown server and close all plugins + */ + static void termination_handler(); + /** * get the route object * @return a reference to the route object diff --git a/modules/httpserver-api/openssl-init.cc b/modules/httpserver-api/openssl-init.cc index 8630d019eb..a1bed2bdbf 100644 --- a/modules/httpserver-api/openssl-init.cc +++ b/modules/httpserver-api/openssl-init.cc @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include @@ -16,9 +16,16 @@ #include "openssl-init.hh" +static void debug_line(const char *line) { + auto debug_enabled = osv_debug_enabled(); + if (debug_enabled) { + std::cout << line << std::endl; + } +} + static void seed_openssl() { - debug("Seeding OpenSSL...\n"); + debug_line("Seeding OpenSSL..."); for (;;) { char buf[32]; @@ -39,10 +46,10 @@ static void seed_openssl() // fedora's openssl-1.0.1h-5.fc20 needs 48 bytes instead // of 32, which is what is required by its upstream version // (tag OpenSSL_1_0_1h). In general, we should make no assumptions. - debug("Still not seeded, retrying\n"); + debug_line("Still not seeded, retrying"); } - debug("OpenSSL seeding done.\n"); + debug_line("OpenSSL seeding done."); } void ensure_openssl_initialized() diff --git a/modules/httpserver-api/ssl_server.cc b/modules/httpserver-api/ssl_server.cc index a4f8ada258..a064bd6b0d 100644 --- a/modules/httpserver-api/ssl_server.cc +++ b/modules/httpserver-api/ssl_server.cc @@ -5,11 +5,12 @@ * BSD license as described in the LICENSE file in the top-level directory. */ -#include +#include #include "transport.hh" #include "ssl_server.hh" #include +#include namespace http { @@ -83,8 +84,10 @@ void ssl_acceptor::do_accept(callback_t callback) [this, socket, callback] (boost::system::error_code ec) { if (ec) { auto remote = socket->lowest_layer().remote_endpoint(); - debug("handshake with " + remote.address().to_string() - + " failed: " + ec.message() + "\n"); + if (osv_debug_enabled()) { + std::cout << "handshake with " << remote.address().to_string() + << " failed: " << ec.message() << std::endl; + } } if (!_tcp_acceptor.is_open()) { diff --git a/modules/httpserver-monitoring-api/Makefile b/modules/httpserver-monitoring-api/Makefile index e26fc06b58..8ac415b7be 100644 --- a/modules/httpserver-monitoring-api/Makefile +++ b/modules/httpserver-monitoring-api/Makefile @@ -45,7 +45,7 @@ all: $(module_out)/lib$(TARGET).so $(src)/scripts/manifest_from_host.sh $(module_out)/lib$(TARGET).so > usr.manifest; \ fi -$(module_out)/lib$(TARGET).so: $(JSON_OBJ_FILES) $(OBJ_FILES) +$(module_out)/lib$(TARGET).so: $(JSON_OBJ_FILES) $(OBJ_FILES) $(module_out)/options.o $(call quiet, $(CXX) $(CXXFLAGS) $(LDFLAGS) -shared $(STATIC_LIBS) -o $@ $^ $(DYN_LIBS), LINK $@) ifneq ($(MAKECMDGOALS),clean) @@ -64,6 +64,10 @@ $(JSON_OBJ_FILES): $(module_out)/autogen/%.o: autogen/%.cc $(call very-quiet, mkdir -p $(module_out)/autogen) $(call quiet, $(CXX) $(CXXFLAGS) -c -MMD -o $@ $<, CXX $@) +$(module_out)/options.o: $(src)/core/options.cc + $(call very-quiet, mkdir -p $(module_out)) + $(call quiet, $(CXX) $(CXXFLAGS) -c -MMD -o $@ $<, CXX $@) + clean: $(call quiet, $(RM) -f $(TARGET), CLEAN) $(call very-quiet, $(RM) -rf $(module_out))