diff --git a/.gitignore b/.gitignore index ba37a7bb55f..1543d19fe19 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ init/fluent-bit.service lib/chunkio/include/chunkio/cio_version.h lib/monkey/monkey.service lib/monkey/include/monkey/mk_core/mk_core_info.h +lib/cprofiles/include/cprofiles/cprof_info.h packaging/.env packaging/packages diff --git a/include/fluent-bit/calyptia/calyptia_constants.h b/include/fluent-bit/calyptia/calyptia_constants.h index f2271b87a67..dc1c2a0216d 100644 --- a/include/fluent-bit/calyptia/calyptia_constants.h +++ b/include/fluent-bit/calyptia/calyptia_constants.h @@ -36,8 +36,10 @@ #define CALYPTIA_ENDPOINT_TRACE "/v1/traces/%s" #define CALYPTIA_ENDPOINT_FLEETS "/v1/fleets" -#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini" +#define CALYPTIA_ENDPOINT_FLEET_CONFIG_INI "/v1/fleets/%s/config?format=ini&config_format=ini" +#define CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML "/v1/fleets/%s/config?format=yaml&config_format=yaml" #define CALYPTIA_ENDPOINT_FLEET_FILES "/v1/fleets/%s/files" +#define CALYPTIA_ENDPOINT_FLEET_BY_NAME "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true" /* Storage */ #define CALYPTIA_SESSION_FILE "session.CALYPTIA" diff --git a/lib/cprofiles/CMakeLists.txt b/lib/cprofiles/CMakeLists.txt index 51e2db560ee..ef1621fb782 100644 --- a/lib/cprofiles/CMakeLists.txt +++ b/lib/cprofiles/CMakeLists.txt @@ -353,7 +353,6 @@ if(CPROF_SYSTEM_MACOS) if (CPACK_GENERATOR MATCHES "productbuild") set(CPACK_SET_DESTDIR "ON") - configure_file(cpack/macos/welcome.txt.cmakein ${CMAKE_CURRENT_SOURCE_DIR}/welcome.txt) configure_file(LICENSE ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt) find_program(CONVERTER textutil) if (NOT CONVERTER) @@ -363,7 +362,6 @@ if(CPROF_SYSTEM_MACOS) execute_process(COMMAND ${CONVERTER} -convert html "${CMAKE_CURRENT_SOURCE_DIR}/README.md" -output "${CMAKE_CURRENT_SOURCE_DIR}/README.html") endif() set(CPACK_PACKAGE_FILE_NAME "${CMETRICS_PKG}") - set(CPACK_RESOURCE_FILE_WELCOME ${CMAKE_CURRENT_SOURCE_DIR}/welcome.txt) set(CPACK_RESOURCE_FILE_LICENSE ${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt) set(CPACK_RESOURCE_FILE_README ${CMAKE_CURRENT_SOURCE_DIR}/README.html) set(CPACK_PRODUCTBUILD_IDENTIFIER "com.calyptia.${CPACK_PACKAGE_NAME}") diff --git a/plugins/custom_calyptia/calyptia.c b/plugins/custom_calyptia/calyptia.c index dba68a25f10..c296b944a29 100644 --- a/plugins/custom_calyptia/calyptia.c +++ b/plugins/custom_calyptia/calyptia.c @@ -232,6 +232,7 @@ int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance * flb_input_set_property(fleet, "host", ctx->cloud_host); flb_input_set_property(fleet, "port", ctx->cloud_port); flb_input_set_property(fleet, "config_dir", ctx->fleet_config_dir); + flb_input_set_property(fleet, "fleet_config_legacy_format", ctx->fleet_config_legacy_format == 1 ? "on" : "off"); /* Set TLS properties */ flb_input_set_property(fleet, "tls", ctx->cloud_tls == 1 ? "on" : "off"); @@ -617,7 +618,7 @@ static int cb_calyptia_init(struct flb_custom_instance *ins, flb_free(ctx); return -1; } - ctx->machine_id_auto_configured = 1; + ctx->machine_id_auto_configured = FLB_TRUE; } /* input collector */ @@ -774,6 +775,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct calyptia, register_retry_on_flush), "Retry agent registration on flush if failed on init." }, + { + FLB_CONFIG_MAP_BOOL, "fleet_config_legacy_format", "true", + 0, FLB_TRUE, offsetof(struct calyptia, fleet_config_legacy_format), + "If set, use legacy (TOML) format for configuration files." + }, /* EOF */ {0} }; diff --git a/plugins/custom_calyptia/calyptia.h b/plugins/custom_calyptia/calyptia.h index 5da14d80ace..0dff625c694 100644 --- a/plugins/custom_calyptia/calyptia.h +++ b/plugins/custom_calyptia/calyptia.h @@ -20,8 +20,6 @@ #ifndef FLB_CALYPTIA_H #define FLB_CALYPTIA_H -#include - struct calyptia { /* config map options */ flb_sds_t api_key; @@ -55,7 +53,8 @@ struct calyptia { flb_sds_t fleet_max_http_buffer_size; flb_sds_t fleet_interval_sec; flb_sds_t fleet_interval_nsec; - bool register_retry_on_flush; /* retry registration on flush if failed */ + int register_retry_on_flush; /* retry registration on flush if failed */ + int fleet_config_legacy_format; /* Fleet config format to use: INI (true) or YAML (false) */ }; int set_fleet_input_properties(struct calyptia *ctx, struct flb_input_instance *fleet); diff --git a/plugins/in_calyptia_fleet/in_calyptia_fleet.c b/plugins/in_calyptia_fleet/in_calyptia_fleet.c index 5a423856c32..143bf90ce31 100644 --- a/plugins/in_calyptia_fleet/in_calyptia_fleet.c +++ b/plugins/in_calyptia_fleet/in_calyptia_fleet.c @@ -42,6 +42,8 @@ #include +#include "in_calyptia_fleet.h" + /* Glob support */ #ifndef _MSC_VER #include @@ -53,70 +55,15 @@ #define PATH_MAX MAX_PATH #endif -#define CALYPTIA_HEADERS_PROJECT "X-Project-Token" -#define CALYPTIA_HEADERS_CTYPE "Content-Type" -#define CALYPTIA_HEADERS_CTYPE_JSON "application/json" - #define DEFAULT_INTERVAL_SEC "15" #define DEFAULT_INTERVAL_NSEC "0" #define DEFAULT_MAX_HTTP_BUFFER_SIZE "10485760" -struct flb_in_calyptia_fleet_config { - /* Time interval check */ - int interval_sec; - int interval_nsec; - - /* maximum http buffer size */ - int max_http_buffer_size; - - /* Grabbed from the cfg_path, used to check if configuration has - * has been updated. - */ - long config_timestamp; - - flb_sds_t api_key; - - flb_sds_t fleet_id; - - /* flag used to mark fleet_id for release when found automatically. */ - int fleet_id_found; - - flb_sds_t fleet_name; - flb_sds_t machine_id; - flb_sds_t config_dir; - flb_sds_t cloud_host; - flb_sds_t cloud_port; - - flb_sds_t fleet_url; - flb_sds_t fleet_files_url; - - struct flb_input_instance *ins; /* plugin instance */ - - /* Networking */ - struct flb_upstream *u; - - int collect_fd; -}; - -struct reload_ctx { - flb_ctx_t *flb; - flb_sds_t cfg_path; -}; - -static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); - -#define new_fleet_config_filename(a) fleet_config_filename((a), "new") -#define cur_fleet_config_filename(a) fleet_config_filename((a), "cur") -#define old_fleet_config_filename(a) fleet_config_filename((a), "old") -#define hdr_fleet_config_filename(a) fleet_config_filename((a), "header") - +static int fleet_cur_chdir(struct flb_in_calyptia_fleet_config *ctx); static int get_calyptia_files(struct flb_in_calyptia_fleet_config *ctx, - const char *url, time_t timestamp); -static int fleet_cur_chdir(struct flb_in_calyptia_fleet_config *ctx); - #ifndef FLB_SYSTEM_WINDOWS static int is_link(const char *path) { @@ -231,12 +178,17 @@ static flb_sds_t generate_base_fleet_directory(struct flb_in_calyptia_fleet_conf } if (*fleet_dir == NULL) { - *fleet_dir = flb_sds_create_size(4096); + *fleet_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (*fleet_dir == NULL) { return NULL; } } + /* Ensure we have a valid value */ + if (ctx->config_dir == NULL) { + ctx->config_dir = FLEET_DEFAULT_CONFIG_DIR; + } + if (ctx->fleet_name != NULL) { return flb_sds_printf(fleet_dir, "%s" PATH_SEPARATOR "%s" PATH_SEPARATOR "%s", ctx->config_dir, ctx->machine_id, ctx->fleet_name); @@ -245,7 +197,7 @@ static flb_sds_t generate_base_fleet_directory(struct flb_in_calyptia_fleet_conf ctx->config_dir, ctx->machine_id, ctx->fleet_id); } -static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname) +flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname) { flb_sds_t cfgname = NULL; flb_sds_t ret; @@ -258,7 +210,12 @@ static flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, return NULL; } - ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname); + if (ctx->fleet_config_legacy_format) { + ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.conf", fname); + } else { + ret = flb_sds_printf(&cfgname, PATH_SEPARATOR "%s.yaml", fname); + } + if (ret == NULL) { flb_sds_destroy(cfgname); return NULL; @@ -366,7 +323,7 @@ static int is_timestamped_fleet_config_path(struct flb_in_calyptia_fleet_config char *end; long val; - if (path == NULL) { + if (path == NULL || ctx == NULL) { return FLB_FALSE; } @@ -384,7 +341,12 @@ static int is_timestamped_fleet_config_path(struct flb_in_calyptia_fleet_config return FLB_FALSE; } - if (strcmp(end, ".conf") == 0) { + if (ctx->fleet_config_legacy_format) { + if (strcmp(end, ".conf") == 0) { + return FLB_TRUE; + } + } + else if (strcmp(end, ".yaml") == 0) { return FLB_TRUE; } @@ -528,7 +490,7 @@ static int test_config_is_valid(struct flb_in_calyptia_fleet_config *ctx, conf = flb_cf_create(); if (conf == NULL) { - flb_plg_debug(ctx->ins, "unable to create conf during validation test: %s", + flb_plg_debug(ctx->ins, "unable to create config during validation test: %s", cfgpath); goto config_init_error; } @@ -536,7 +498,7 @@ static int test_config_is_valid(struct flb_in_calyptia_fleet_config *ctx, conf = flb_cf_create_from_file(conf, cfgpath); if (conf == NULL) { flb_plg_debug(ctx->ins, - "unable to create conf from file during validation test: %s", + "unable to create config from file during validation test: %s", cfgpath); goto cf_create_from_file_error; } @@ -555,7 +517,7 @@ static int parse_config_name_timestamp(struct flb_in_calyptia_fleet_config *ctx, { char *ext = NULL; long timestamp; - char realname[4096] = {0}; + char realname[CALYPTIA_MAX_DIR_SIZE] = {0}; char *fname; ssize_t len; @@ -564,13 +526,17 @@ static int parse_config_name_timestamp(struct flb_in_calyptia_fleet_config *ctx, } switch (is_link(cfgpath)) { + /* Prevent undefined references due to use of readlink */ +#ifndef FLB_SYSTEM_WINDOWS case FLB_TRUE: + len = readlink(cfgpath, realname, sizeof(realname)); if (len > sizeof(realname)) { return FLB_FALSE; } break; +#endif /* FLB_SYSTEM_WINDOWS */ case FLB_FALSE: strncpy(realname, cfgpath, sizeof(realname)-1); break; @@ -957,13 +923,13 @@ static int get_calyptia_fleet_id_by_name(struct flb_in_calyptia_fleet_config *ct return -1; } - url = flb_sds_create_size(4096); + url = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (url == NULL) { flb_sds_destroy(project_id); return -1; } - flb_sds_printf(&url, "/v1/search?project_id=%s&resource=fleet&term=%s&exact=true", + flb_sds_printf(&url, CALYPTIA_ENDPOINT_FLEET_BY_NAME, project_id, ctx->fleet_name); client = fleet_http_do(ctx, url); @@ -1349,7 +1315,12 @@ static int calyptia_config_delete_old(struct flb_in_calyptia_fleet_config *ctx) return -1; } - if (flb_sds_cat_safe(&glob_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + if (ctx->fleet_config_legacy_format) { + if (flb_sds_cat_safe(&glob_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + flb_sds_destroy(glob_files); + return -1; + } + } else if (flb_sds_cat_safe(&glob_files, PATH_SEPARATOR "*.yaml", strlen(PATH_SEPARATOR "*.yaml")) != 0) { flb_sds_destroy(glob_files); return -1; } @@ -1413,7 +1384,13 @@ static flb_sds_t calyptia_config_get_newest(struct flb_in_calyptia_fleet_config return NULL; } - if (flb_sds_cat_safe(&glob_conf_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + if (ctx->fleet_config_legacy_format) { + if (flb_sds_cat_safe(&glob_conf_files, PATH_SEPARATOR "*.conf", strlen(PATH_SEPARATOR "*.conf")) != 0) { + flb_plg_error(ctx->ins, "unable to concatenate fleet glob"); + flb_sds_destroy(glob_conf_files); + return NULL; + } + } else if (flb_sds_cat_safe(&glob_conf_files, PATH_SEPARATOR "*.yaml", strlen(PATH_SEPARATOR "*.yaml")) != 0) { flb_plg_error(ctx->ins, "unable to concatenate fleet glob"); flb_sds_destroy(glob_conf_files); return NULL; @@ -1421,7 +1398,7 @@ static flb_sds_t calyptia_config_get_newest(struct flb_in_calyptia_fleet_config inis = read_glob(glob_conf_files); if (inis == NULL) { - flb_plg_error(ctx->ins, "unable to read fleet directory for conf files: %s", + flb_plg_error(ctx->ins, "unable to read fleet directory for config files: %s", glob_conf_files); flb_sds_destroy(glob_conf_files); return NULL; @@ -1608,7 +1585,7 @@ static int calyptia_config_rollback(struct flb_in_calyptia_fleet_config *ctx, } #endif -static void fleet_config_get_properties(flb_sds_t *buf, struct mk_list *props) +static void fleet_config_get_properties(flb_sds_t *buf, struct mk_list *props, int fleet_config_legacy_format) { struct mk_list *head; struct flb_kv *kv; @@ -1617,7 +1594,11 @@ static void fleet_config_get_properties(flb_sds_t *buf, struct mk_list *props) kv = mk_list_entry(head, struct flb_kv, _head); if (kv->key != NULL && kv->val != NULL) { - flb_sds_printf(buf, " %s ", kv->key); + if (fleet_config_legacy_format) { + flb_sds_printf(buf, " %s ", kv->key); + } else { + flb_sds_printf(buf, " %s: ", kv->key); + } flb_sds_cat_safe(buf, kv->val, strlen(kv->val)); flb_sds_cat_safe(buf, "\n", 1); } @@ -1637,7 +1618,6 @@ static flb_sds_t get_fleet_id_from_header(struct flb_in_calyptia_fleet_config *c cf_hdr = flb_cf_create_from_file(NULL, hdr_fleet_config_filename(ctx)); if (cf_hdr == NULL) { - flb_cf_destroy(cf_hdr); return NULL; } @@ -1673,9 +1653,10 @@ static flb_sds_t get_fleet_id_from_header(struct flb_in_calyptia_fleet_config *c flb_cf_destroy(cf_hdr); return fleet_id; } + + flb_cf_destroy(cf_hdr); } - flb_cf_destroy(cf_hdr); return NULL; } @@ -1687,8 +1668,11 @@ flb_sds_t fleet_config_get(struct flb_in_calyptia_fleet_config *ctx) flb_ctx_t *flb = flb_context_get(); flb_sds_t fleet_id = NULL; + if (!ctx) { + return NULL; + } - buf = flb_sds_create_size(2048); + buf = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (!buf) { return NULL; @@ -1699,14 +1683,23 @@ flb_sds_t fleet_config_get(struct flb_in_calyptia_fleet_config *ctx) if (strcasecmp(c_ins->p->name, "calyptia")) { continue; } - flb_sds_printf(&buf, "[CUSTOM]\n"); - flb_sds_printf(&buf, " name %s\n", c_ins->p->name); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&buf, "[CUSTOM]\n"); + flb_sds_printf(&buf, " name %s\n", c_ins->p->name); + } else { + flb_sds_printf(&buf, "customs:\n"); + flb_sds_printf(&buf, " - name: %s\n", c_ins->p->name); + } - fleet_config_get_properties(&buf, &c_ins->properties); + fleet_config_get_properties(&buf, &c_ins->properties, ctx->fleet_config_legacy_format); if (flb_config_prop_get("fleet_id", &c_ins->properties) == NULL) { if (ctx->fleet_id != NULL) { - flb_sds_printf(&buf, " fleet_id %s\n", ctx->fleet_id); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&buf, " fleet_id %s\n", ctx->fleet_id); + } else { + flb_sds_printf(&buf, " fleet_id: %s\n", ctx->fleet_id); + } } else { fleet_id = get_fleet_id_from_header(ctx); @@ -1716,7 +1709,11 @@ flb_sds_t fleet_config_get(struct flb_in_calyptia_fleet_config *ctx) return NULL; } - flb_sds_printf(&buf, " fleet_id %s\n", fleet_id); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&buf, " fleet_id %s\n", fleet_id); + } else { + flb_sds_printf(&buf, " fleet_id: %s\n", fleet_id); + } flb_sds_destroy(fleet_id); } } @@ -1765,7 +1762,7 @@ static int create_fleet_header(struct flb_in_calyptia_fleet_config *ctx) return rc; } -static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) +int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) { flb_sds_t cfgname; flb_sds_t cfgnewname; @@ -1775,17 +1772,21 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) int ret = -1; if (ctx->fleet_url == NULL) { - ctx->fleet_url = flb_sds_create_size(4096); + ctx->fleet_url = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (ctx->fleet_url == NULL) { return -1; } - flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, ctx->fleet_id); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, ctx->fleet_id); + } else { + flb_sds_printf(&ctx->fleet_url, CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML, ctx->fleet_id); + } } if (ctx->fleet_files_url == NULL) { - ctx->fleet_files_url = flb_sds_create_size(4096); + ctx->fleet_files_url = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (ctx->fleet_files_url == NULL) { return -1; @@ -1797,8 +1798,12 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) create_fleet_header(ctx); hdrname = fleet_config_filename(ctx, "header"); - header = flb_sds_create_size(32); - flb_sds_printf(&header, "@include %s\n\n", hdrname); + header = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); + if (ctx->fleet_config_legacy_format) { + flb_sds_printf(&header, "@include %s\n\n", hdrname); + } else { + flb_sds_printf(&header, "includes: \n - %s\n", hdrname); + } flb_sds_destroy(hdrname); /* create the base file. */ @@ -1807,7 +1812,7 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) /* new file created! */ if (ret == 1) { - get_calyptia_files(ctx, ctx->fleet_files_url, time_last_modified); + get_calyptia_files(ctx, time_last_modified); cfgname = time_fleet_config_filename(ctx, time_last_modified); @@ -1831,6 +1836,8 @@ static int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx) return -1; } #endif + + flb_sds_destroy(cfgname); } return 0; @@ -2119,13 +2126,12 @@ static int create_fleet_files(struct flb_in_calyptia_fleet_config *ctx, } static int get_calyptia_files(struct flb_in_calyptia_fleet_config *ctx, - const char *url, time_t timestamp) { struct flb_http_client *client; int ret = -1; - if (ctx == NULL || url == NULL) { + if (ctx == NULL || ctx->fleet_files_url == NULL) { return -1; } @@ -2194,7 +2200,7 @@ static int in_calyptia_fleet_init(struct flb_input_instance *in, return -1; } - ctx->config_dir = flb_sds_create_size(4096); + ctx->config_dir = flb_sds_create_size(CALYPTIA_MAX_DIR_SIZE); if (ctx->config_dir == NULL) { flb_plg_error(in, "unable to allocate config-dir."); @@ -2355,6 +2361,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, interval_nsec), "Set the collector interval (nanoseconds)" }, + { + FLB_CONFIG_MAP_BOOL, "fleet_config_legacy_format", "true", + 0, FLB_TRUE, offsetof(struct flb_in_calyptia_fleet_config, fleet_config_legacy_format), + "If set, use legacy (TOML) format for configuration files." + }, /* EOF */ {0} }; diff --git a/plugins/in_calyptia_fleet/in_calyptia_fleet.h b/plugins/in_calyptia_fleet/in_calyptia_fleet.h new file mode 100644 index 00000000000..ed97c73f376 --- /dev/null +++ b/plugins/in_calyptia_fleet/in_calyptia_fleet.h @@ -0,0 +1,80 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_IN_CALYPTIA_FLEET_H +#define FLB_IN_CALYPTIA_FLEET_H + +#include +#include + +struct flb_in_calyptia_fleet_config { + /* Time interval check */ + int interval_sec; + int interval_nsec; + + /* maximum http buffer size */ + int max_http_buffer_size; + + /* Grabbed from the cfg_path, used to check if configuration has + * has been updated. + */ + long config_timestamp; + + flb_sds_t api_key; + + flb_sds_t fleet_id; + + /* flag used to mark fleet_id for release when found automatically. */ + int fleet_id_found; + + flb_sds_t fleet_name; + flb_sds_t machine_id; + flb_sds_t config_dir; + flb_sds_t cloud_host; + flb_sds_t cloud_port; + + flb_sds_t fleet_url; + flb_sds_t fleet_files_url; + + /* whether to use legacy INI/TOML or YAML format */ + int fleet_config_legacy_format; + + struct flb_input_instance *ins; /* plugin instance */ + + /* Networking */ + struct flb_upstream *u; + + int collect_fd; +}; + +struct reload_ctx { + flb_ctx_t *flb; + flb_sds_t cfg_path; +}; + +flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); + +#define new_fleet_config_filename(a) fleet_config_filename((a), "new") +#define cur_fleet_config_filename(a) fleet_config_filename((a), "cur") +#define old_fleet_config_filename(a) fleet_config_filename((a), "old") +#define hdr_fleet_config_filename(a) fleet_config_filename((a), "header") + +int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx); + +#endif /* FLB_IN_CALYPTIA_FLEET_H */ diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index 7c5143cbfca..49ed0e0cf5d 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -73,6 +73,7 @@ if (FLB_CUSTOM_CALYPTIA) "custom_calyptia_test.c" "custom_calyptia_registration_retry_test.c" "custom_calyptia_input_test.c" + "in_calyptia_fleet_test.c" ) foreach(TEST_SOURCE ${CALYPTIA_TESTS}) @@ -82,6 +83,7 @@ if (FLB_CUSTOM_CALYPTIA) add_executable(${TEST_TARGET} ${TEST_SOURCE} "../../plugins/custom_calyptia/calyptia.c" + "../../plugins/in_calyptia_fleet/in_calyptia_fleet.c" ) target_link_libraries(${TEST_TARGET} @@ -113,7 +115,7 @@ if(FLB_IN_EBPF) # Function to create eBPF handler tests function(add_ebpf_handler_test name source_file handler_file) set(TEST_TARGET "flb-rt-in_ebpf_${name}_handler") - + # Add executable add_executable(${TEST_TARGET} ${source_file} diff --git a/tests/runtime/custom_calyptia_input_test.c b/tests/runtime/custom_calyptia_input_test.c index 5cf023afa76..7b8be6b2a77 100644 --- a/tests/runtime/custom_calyptia_input_test.c +++ b/tests/runtime/custom_calyptia_input_test.c @@ -67,6 +67,7 @@ static struct test_context *init_test_context() t_ctx->ctx->fleet_max_http_buffer_size = flb_strdup("1024"); t_ctx->ctx->fleet_interval_sec = flb_strdup("60"); t_ctx->ctx->fleet_interval_nsec = flb_strdup("500000000"); + t_ctx->ctx->fleet_config_legacy_format = FLB_TRUE; t_ctx->fleet = flb_input_new(t_ctx->config, "calyptia_fleet", NULL, FLB_FALSE); if (!t_ctx->fleet) { @@ -160,13 +161,13 @@ void test_set_fleet_input_properties() TEST_MSG("max_http_buffer_size expected=%s got=%s", t_ctx->ctx->fleet_max_http_buffer_size, value); TEST_CHECK(value && strcmp(value, t_ctx->ctx->fleet_max_http_buffer_size) == 0); - // /* Check interval_sec */ + /* Check interval_sec */ value = flb_input_get_property("interval_sec", t_ctx->fleet); TEST_CHECK(value != NULL); TEST_MSG("interval_sec expected=%s got=%s", t_ctx->ctx->fleet_interval_sec, value); TEST_CHECK(value && strcmp(value, t_ctx->ctx->fleet_interval_sec) == 0); - // /* Check interval_nsec */ + /* Check interval_nsec */ value = flb_input_get_property("interval_nsec", t_ctx->fleet); TEST_CHECK(value != NULL); TEST_MSG("interval_nsec expected=%s got=%s", t_ctx->ctx->fleet_interval_nsec, value); diff --git a/tests/runtime/in_calyptia_fleet_test.c b/tests/runtime/in_calyptia_fleet_test.c new file mode 100644 index 00000000000..2bbd7bea0f7 --- /dev/null +++ b/tests/runtime/in_calyptia_fleet_test.c @@ -0,0 +1,271 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +#include +#include +#include +#include + +#include +#include + +#include "flb_tests_runtime.h" +#include "../../plugins/in_calyptia_fleet/in_calyptia_fleet.h" + +#define MOCK_SERVER_HOST "127.0.0.1" +#define MOCK_SERVER_PORT "9876" + +flb_sds_t fleet_config_filename(struct flb_in_calyptia_fleet_config *ctx, char *fname); +int get_calyptia_fleet_config(struct flb_in_calyptia_fleet_config *ctx); + +static int tomlRequests = 0; +static int yamlRequests = 0; + +/* Test context structure */ +struct test_context { + struct flb_in_calyptia_fleet_config *ctx; + struct flb_config *config; +}; + +/* Initialize test context */ +static struct test_context *init_test_context() +{ + struct test_context *t_ctx = flb_calloc(1, sizeof(struct test_context)); + if (!t_ctx) { + return NULL; + } + + t_ctx->config = flb_config_init(); + if (!t_ctx->config) { + flb_free(t_ctx); + return NULL; + } + + t_ctx->ctx = flb_calloc(1, sizeof(struct flb_in_calyptia_fleet_config)); + if (!t_ctx->ctx) { + flb_config_exit(t_ctx->config); + flb_free(t_ctx); + return NULL; + } + + /* Initialize plugin instance for logging */ + t_ctx->ctx->ins = flb_calloc(1, sizeof(struct flb_input_instance)); + if (!t_ctx->ctx->ins) { + flb_free(t_ctx->ctx); + flb_config_exit(t_ctx->config); + flb_free(t_ctx); + return NULL; + } + + /* Initialize test values in ctx */ + t_ctx->ctx->api_key = flb_strdup("test_api_key"); + t_ctx->ctx->fleet_id = flb_strdup("test_fleet_id"); + + t_ctx->ctx->fleet_name = flb_strdup("test_fleet"); + t_ctx->ctx->machine_id = flb_strdup("test_machine_id"); + + char mock_url[256] = {0}; + snprintf(mock_url, sizeof(mock_url) - 1, "%s:%s", MOCK_SERVER_HOST, MOCK_SERVER_PORT); + t_ctx->ctx->fleet_files_url = flb_strdup(mock_url); + + t_ctx->ctx->fleet_config_legacy_format = FLB_TRUE; + + return t_ctx; +} + +static void cleanup_test_context(struct test_context *t_ctx) +{ + if (!t_ctx) { + return; + } + + if (t_ctx->ctx) { + if (t_ctx->ctx->api_key) flb_free(t_ctx->ctx->api_key); + if (t_ctx->ctx->fleet_id) flb_free(t_ctx->ctx->fleet_id); + + if (t_ctx->ctx->fleet_name) flb_free(t_ctx->ctx->fleet_name); + if (t_ctx->ctx->machine_id) flb_free(t_ctx->ctx->machine_id); + if (t_ctx->ctx->fleet_files_url) flb_free(t_ctx->ctx->fleet_files_url); + + if (t_ctx->ctx->ins) flb_free(t_ctx->ctx->ins); + flb_free(t_ctx->ctx); + } + + if (t_ctx->config) { + /* Destroy the config which will cleanup any remaining instances */ + flb_config_exit(t_ctx->config); + } + + flb_free(t_ctx); +} + +static void test_in_fleet_format() { + struct test_context *t_ctx = init_test_context(); + TEST_CHECK(t_ctx != NULL); + + /* Ensure we create TOML files by default */ + char expectedValue[CALYPTIA_MAX_DIR_SIZE]; + int ret = sprintf(expectedValue, "%s/%s/%s/test.conf", FLEET_DEFAULT_CONFIG_DIR, t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); + TEST_CHECK(ret > 0); + + flb_sds_t value = fleet_config_filename( t_ctx->ctx, "test" ); + TEST_CHECK(value != NULL); + TEST_MSG("fleet_config_filename expected=%s got=%s", expectedValue, value); + TEST_CHECK(value && strcmp(value, expectedValue) == 0); + flb_sds_destroy(value); + value = NULL; + + /* Ensure we create YAML files if configured to do so */ + t_ctx->ctx->fleet_config_legacy_format = FLB_FALSE; + + ret = sprintf(expectedValue, "%s/%s/%s/test.yaml", FLEET_DEFAULT_CONFIG_DIR, t_ctx->ctx->machine_id, t_ctx->ctx->fleet_name); + TEST_CHECK(ret > 0); + + value = fleet_config_filename( t_ctx->ctx, "test" ); + TEST_CHECK(value != NULL); + TEST_MSG("fleet_config_filename expected=%s got=%s", expectedValue, value); + TEST_CHECK(value && strcmp(value, expectedValue) == 0); + flb_sds_destroy(value); + value = NULL; + + cleanup_test_context(t_ctx); +} + +static void mock_server_fleet_files_toml(mk_request_t *request, void *data) +{ + tomlRequests++; + /* Use a local buffer with correct size */ + char *response = "{\"id\":\"test_fleet_id\"}"; + size_t response_len = strlen(response); + + mk_http_status(request, 200); + mk_http_header(request, "Content-Type", sizeof("Content-Type") - 1, + "application/json", sizeof("application/json") - 1); + mk_http_send(request, response, response_len, NULL); + mk_http_done(request); +} + +static void mock_server_fleet_files_yaml(mk_request_t *request, void *data) +{ + yamlRequests++; + /* Use a local buffer with correct size */ + char *response = "{\"id\":\"test_fleet_id\"}"; + size_t response_len = strlen(response); + + mk_http_status(request, 200); + mk_http_header(request, "Content-Type", sizeof("Content-Type") - 1, + "application/json", sizeof("application/json") - 1); + mk_http_send(request, response, response_len, NULL); + mk_http_done(request); +} + +static void mock_server_registration(mk_request_t *request, void *data) +{ + /* Use a local buffer with correct size */ + const char *response = "{\"id\":\"test_fleet_id\"}"; + size_t response_len = strlen(response); // Ensure size is accurate + + mk_http_status(request, 200); + mk_http_header(request, "Content-Type", sizeof("Content-Type") - 1, + "application/json", sizeof("application/json") - 1); + mk_http_send(request, response, response_len, NULL); // Use response_len + mk_http_done(request); +} + + +static void test_in_fleet_get_calyptia_files() { + char tmp[256] = {0}; + struct test_context *t_ctx = init_test_context(); + TEST_CHECK(t_ctx != NULL); + + /* Init mock server */ + mk_ctx_t *mock_ctx = mk_create(); + TEST_CHECK(mock_ctx != NULL); + + /* Compose listen address */ + snprintf(tmp, sizeof(tmp) - 1, "%s:%s", MOCK_SERVER_HOST, MOCK_SERVER_PORT); + int ret = mk_config_set(mock_ctx, "Listen", tmp, NULL); + TEST_CHECK(ret == 0); + + int vid = mk_vhost_create(mock_ctx, NULL); + TEST_CHECK(vid >= 0); + + sprintf(tmp, CALYPTIA_ENDPOINT_FLEET_CONFIG_INI, t_ctx->ctx->fleet_id); + ret = mk_vhost_handler(mock_ctx, vid, tmp, mock_server_fleet_files_toml, NULL); + TEST_CHECK(ret == 0); + + sprintf(tmp, CALYPTIA_ENDPOINT_FLEET_CONFIG_YAML, t_ctx->ctx->fleet_id); + ret = mk_vhost_handler(mock_ctx, vid, tmp, mock_server_fleet_files_yaml, NULL); + TEST_CHECK(ret == 0); + + ret = mk_vhost_handler(mock_ctx, vid, "/v1/agents", mock_server_registration, NULL); + TEST_CHECK(ret == 0); + + ret = mk_vhost_handler(mock_ctx, vid, "/v1/agents/test_fleet_id", mock_server_registration, NULL); + TEST_CHECK(ret == 0); + + ret = mk_start(mock_ctx); + TEST_CHECK(ret == 0); + + /* Allow the mock server to initialize */ + flb_time_msleep(500); + + tomlRequests = 0; + yamlRequests = 0; + + /* Init Fluent Bit context */ + flb_ctx_t *ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + ret = flb_service_set(ctx, + "Log_Level", "debug", + NULL); + TEST_CHECK(ret == 0); + + /* Create dummy input */ + int in_ffd = flb_input(ctx, (char *)"dummy", NULL); + TEST_CHECK(in_ffd >= 0); + + /* Create custom Calyptia plugin */ + struct flb_custom_instance *calyptia = flb_custom_new(ctx->config, (char *)"calyptia", NULL); + TEST_CHECK(calyptia != NULL); + + /* Set custom plugin properties */ + flb_custom_set_property(calyptia, "api_key", "test-key"); + flb_custom_set_property(calyptia, "log_level", "debug"); + flb_custom_set_property(calyptia, "calyptia_host", MOCK_SERVER_HOST); + flb_custom_set_property(calyptia, "calyptia_port", MOCK_SERVER_PORT); + flb_custom_set_property(calyptia, "calyptia_tls", "off"); + flb_custom_set_property(calyptia, "calyptia_tls.verify", "off"); + + /* Start the engine */ + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + /* TOML */ + ret = get_calyptia_fleet_config(t_ctx->ctx); + TEST_CHECK(ret == 0); + TEST_CHECK(tomlRequests == 1); + TEST_CHECK(yamlRequests == 0); + + /* YAML */ + t_ctx->ctx->fleet_config_legacy_format = FLB_FALSE; + ret = get_calyptia_fleet_config(t_ctx->ctx); + TEST_CHECK(ret == 0); + TEST_CHECK(tomlRequests == 1); + TEST_CHECK(yamlRequests == 1); + + /* Cleanup */ + flb_stop(ctx); + flb_destroy(ctx); + mk_stop(mock_ctx); + mk_destroy(mock_ctx); + + cleanup_test_context(t_ctx); +} + +/* Define test list */ +TEST_LIST = { + {"in_calyptia_fleet_format", test_in_fleet_format}, + {"in_calyptia_fleet_get_files", test_in_fleet_get_calyptia_files}, + {NULL, NULL} +}; \ No newline at end of file