feat: support to rename and delete tag

alibaba · Oct 14, 2024 · f00f1af · f00f1af
1 parent e32d092
commit f00f1af
Show file tree

Hide file tree

Showing 65 changed files with 898 additions and 381 deletions.
diff --git a/core/common/CircularBuffer.h b/core/common/CircularBuffer.h
@@ -16,8 +16,10 @@
 
 #pragma once
 #include <atomic>
+
 #include "MemoryBarrier.h"
 #include "Semaphore.h"
+#include "common/TimeUtil.h"
 
 namespace logtail {
 

diff --git a/core/common/Constants.cpp b/core/common/Constants.cpp
@@ -22,18 +22,6 @@ const std::string OS_NAME = "Linux";
 const std::string OS_NAME = "Windows";
 #endif
 
-const std::string LOG_RESERVED_KEY_SOURCE = "__source__";
-const std::string LOG_RESERVED_KEY_TOPIC = "__topic__";
-const std::string LOG_RESERVED_KEY_USER_DEFINED_ID = "__user_defined_id__";
-const std::string LOG_RESERVED_KEY_MACHINE_UUID = "__machine_uuid__";
-const std::string LOG_RESERVED_KEY_HOSTNAME = "__hostname__";
-const std::string LOG_RESERVED_KEY_PATH = "__path__";
-const std::string LOG_RESERVED_KEY_PACKAGE_ID = "__pack_id__";
-const std::string LOG_RESERVED_KEY_TRUNCATE_INFO = "__truncate_info__";
-// const std::string LOG_RESERVED_KEY_ALIPAY_ZONE = "__alipay_zone__";
-const std::string LOG_RESERVED_KEY_INODE = "__inode__";
-const std::string LOG_RESERVED_KEY_FILE_OFFSET = "__file_offset__";
-
 const char* SLS_EMPTY_STR_FOR_INDEX = "\01";
 
 // profile project

diff --git a/core/common/Constants.h b/core/common/Constants.h
@@ -22,19 +22,6 @@ namespace logtail {
 // OS name, Linux, Windows.
 extern const std::string OS_NAME;
 
-// Resevered key in log.
-extern const std::string LOG_RESERVED_KEY_SOURCE;
-extern const std::string LOG_RESERVED_KEY_TOPIC;
-extern const std::string LOG_RESERVED_KEY_USER_DEFINED_ID;
-extern const std::string LOG_RESERVED_KEY_MACHINE_UUID;
-extern const std::string LOG_RESERVED_KEY_HOSTNAME;
-extern const std::string LOG_RESERVED_KEY_PATH;
-extern const std::string LOG_RESERVED_KEY_PACKAGE_ID;
-extern const std::string LOG_RESERVED_KEY_TRUNCATE_INFO;
-// extern const std::string LOG_RESERVED_KEY_ALIPAY_ZONE;
-extern const std::string LOG_RESERVED_KEY_INODE;
-extern const std::string LOG_RESERVED_KEY_FILE_OFFSET;
-
 extern const char* SLS_EMPTY_STR_FOR_INDEX;
 
 // profile project

diff --git a/core/common/ParamExtractor.h b/core/common/ParamExtractor.h
@@ -285,4 +285,6 @@ bool IsValidList(const Json::Value& config, const std::string& key, std::string&
 
 bool IsValidMap(const Json::Value& config, const std::string& key, std::string& errorMsg);
 
+bool IsKeyExist(const Json::Value& config, const std::string& key);
+
 } // namespace logtail
diff --git a/core/common/TagConstants.h b/core/common/TagConstants.h
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2024 iLogtail Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <string>
+
+namespace logtail {
+
+const std::string LOG_RESERVED_KEY_SOURCE = "__source__";
+const std::string LOG_RESERVED_KEY_TOPIC = "__topic__";
+const std::string LOG_RESERVED_KEY_MACHINE_UUID = "__machine_uuid__";
+const std::string LOG_RESERVED_KEY_PACKAGE_ID = "__pack_id__";
+const std::string LOG_RESERVED_KEY_TRUNCATE_INFO = "__truncate_info__";
+
+const std::string DEFAULT_CONFIG_TAG_KEY_VALUE = "__default__";
+
+enum TagKey {
+    FILE_OFFSET_KEY = 0,
+    FILE_INODE_TAG_KEY = 1,
+    FILE_PATH_TAG_KEY = 2,
+    K8S_NAMESPACE_TAG_KEY = 3,
+    K8S_POD_NAME_TAG_KEY = 4,
+    K8S_POD_UID_TAG_KEY = 5,
+    CONTAINER_NAME_TAG_KEY = 6,
+    CONTAINER_IP_TAG_KEY = 7,
+    CONTAINER_IMAGE_NAME_TAG_KEY = 8,
+    K8S_CONTAINER_NAME_TAG_KEY = 9,
+    K8S_CONTAINER_IMAGE_NAME_TAG_KEY = 10,
+    K8S_CONTAINER_IP_TAG_KEY = 11,
+    HOST_NAME = 12,
+    NUM_VALUES = 13,
+    UNKOWN = 14,
+};
+
+#ifdef __ENTERPRISE__
+const std::string TagDefaultKey[NUM_VALUES] = {
+    "__file_offset__",
+    "__inode__",
+    "__path__",
+    "_namespace_",
+    "_pod_name_",
+    "_pod_uid_",
+    "_container_name_",
+    "_container_ip_",
+    "_image_name_",
+    "_container_name_",
+    "_image_name_",
+    "_container_ip_",
+    "__hostname__",
+};
+const std::string AGENT_TAG_DEFAULT_KEY = "__user_defined_id__";
+#else
+const std::string TagDefaultKey[NUM_VALUES] = {
+    "log.file.offset",
+    "log.file.inode",
+    "log.file.path",
+    "k8s.namespace.name",
+    "k8s.pod.name",
+    "k8s.pod.uid",
+    "container.name",
+    "container.ip",
+    "container.image.name",
+    "k8s.container.name",
+    "k8s.container.image.name",
+    "k8s.container.ip",
+    "host.name",
+};
+const std::string HOST_IP_DEFAULT_KEY = "host.ip";
+#endif
+
+} // namespace logtail
diff --git a/core/file_server/ContainerInfo.cpp b/core/file_server/ContainerInfo.cpp
@@ -21,7 +21,7 @@
 
 namespace logtail {
 
-const std::unordered_set<std::string> containerNameTag = {
+const std::vector<std::string> containerNameTag = {
     "_image_name_",
     "_container_name_",
     "_pod_name_",
@@ -30,6 +30,15 @@ const std::unordered_set<std::string> containerNameTag = {
     "_container_ip_",
 };
 
+const std::vector<TagKey> containerNameTagKey = {
+    TagKey::CONTAINER_IMAGE_NAME_TAG_KEY,
+    TagKey::CONTAINER_NAME_TAG_KEY,
+    TagKey::K8S_POD_NAME_TAG_KEY,
+    TagKey::K8S_NAMESPACE_TAG_KEY,
+    TagKey::K8S_POD_UID_TAG_KEY,
+    TagKey::CONTAINER_IP_TAG_KEY,
+};
+
 bool ContainerInfo::ParseAllByJSONObj(const Json::Value& paramsAll,
                                       std::unordered_map<std::string, ContainerInfo>& containerInfoMap,
                                       std::string& errorMsg) {
@@ -98,7 +107,7 @@ bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& con
                 sls_logs::LogTag tag;
                 tag.set_key(metaDatas[i - 1].asString());
                 tag.set_value(metaDatas[i].asString());
-                containerInfo.mMetadatas.emplace_back(tag);
+                containerInfo.mMetadatas->emplace_back(tag);
             }
         }
     }
@@ -109,14 +118,7 @@ bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& con
                 sls_logs::LogTag tag;
                 tag.set_key(tags[i - 1].asString());
                 tag.set_value(tags[i].asString());
-                // 不是老版本
-                if (!isOldCheckpoint) {
-                    containerInfo.mTags.emplace_back(tag);
-                } else if (containerNameTag.find(tags[i - 1].asString()) != containerNameTag.end()) {
-                    containerInfo.mMetadatas.emplace_back(tag);
-                } else {
-                    containerInfo.mTags.emplace_back(tag);
-                }
+                containerInfo.mTags->emplace_back(tag);
             }
         }
     }
@@ -129,4 +131,13 @@ bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& con
     return true;
 }
 
+TagKey ContainerInfo::GetFileTagKey(const std::string& key) {
+    for (size_t i = 0; i < containerNameTag.size(); ++i) {
+        if (containerNameTag[i] == key) {
+            return containerNameTagKey[i];
+        }
+    }
+    return TagKey::UNKOWN;
+}
+
 } // namespace logtail
diff --git a/core/file_server/ContainerInfo.h b/core/file_server/ContainerInfo.h
@@ -23,7 +23,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "container_manager/ConfigContainerInfoUpdateCmd.h"
+#include "common/TagConstants.h"
 #include "protobuf/sls/sls_logs.pb.h"
 
 namespace logtail {
@@ -44,12 +44,14 @@ struct ContainerInfo {
     std::string mLogPath;
     std::string mUpperDir;
     std::vector<Mount> mMounts; // mounts of this container
-    std::vector<sls_logs::LogTag> mTags; // ContainerNameTag
-    std::vector<sls_logs::LogTag> mMetadatas; // ExternalEnvTag and ExternalK8sLabelTag
+    // Tags is hold by each reader and cost memory, so use shared_ptr to share the same tags.
+    std::shared_ptr<std::vector<sls_logs::LogTag>> mTags; // ContainerNameTag, ExternalEnvTag and ExternalK8sLabelTag.
+    std::shared_ptr<std::vector<sls_logs::LogTag>> mMetadatas;
     Json::Value mJson; // this obj's json, for saving to local file
 
     static bool ParseByJSONObj(const Json::Value&, ContainerInfo&, std::string&);
     static bool ParseAllByJSONObj(const Json::Value&, std::unordered_map<std::string, ContainerInfo>&, std::string&);
+    static TagKey GetFileTagKey(const std::string& key);
 
     bool operator==(const ContainerInfo& rhs) const {
         if (mID != rhs.mID) {
@@ -74,22 +76,22 @@ struct ContainerInfo {
                 return false;
             }
         }
-        if (mMetadatas.size() != rhs.mMetadatas.size()) {
+        if (mMetadatas->size() != rhs.mMetadatas->size()) {
             return false;
         }
-        for (size_t idx = 0; idx < mMetadatas.size(); ++idx) {
-            const auto& lhsTag = mMetadatas[idx];
-            const auto& rhsTag = rhs.mMetadatas[idx];
+        for (size_t idx = 0; idx < mMetadatas->size(); ++idx) {
+            const auto& lhsTag = (*mMetadatas)[idx];
+            const auto& rhsTag = (*rhs.mMetadatas)[idx];
             if (lhsTag.key() != rhsTag.key() || lhsTag.value() != rhsTag.value()) {
                 return false;
             }
         }
-        if (mTags.size() != rhs.mTags.size()) {
+        if (mTags->size() != rhs.mTags->size()) {
             return false;
         }
-        for (size_t idx = 0; idx < mTags.size(); ++idx) {
-            const auto& lhsTag = mTags[idx];
-            const auto& rhsTag = rhs.mTags[idx];
+        for (size_t idx = 0; idx < mTags->size(); ++idx) {
+            const auto& lhsTag = (*mTags)[idx];
+            const auto& rhsTag = (*rhs.mTags)[idx];
             if (lhsTag.key() != rhsTag.key() || lhsTag.value() != rhsTag.value()) {
                 return false;
             }

diff --git a/core/file_server/FileServer.cpp b/core/file_server/FileServer.cpp
@@ -32,8 +32,8 @@ using namespace std;
 namespace logtail {
 
 FileServer::FileServer() {
-    WriteMetrics::GetInstance()->PrepareMetricsRecordRef(mMetricsRecordRef,
-                                                         {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}});
+    WriteMetrics::GetInstance()->PrepareMetricsRecordRef(
+        mMetricsRecordRef, {{METRIC_LABEL_KEY_RUNNER_NAME, METRIC_LABEL_VALUE_RUNNER_NAME_FILE_SERVER}});
 }
 
 // 启动文件服务，包括加载配置、处理检查点、注册事件等
@@ -178,6 +178,29 @@ void FileServer::RemoveMultilineConfig(const string& name) {
     mPipelineNameMultilineConfigsMap.erase(name);
 }
 
+// 获取给定名称的Tag配置
+FileTagConfig FileServer::GetFileTagConfig(const string& name) const {
+    ReadLock lock(mReadWriteLock);
+    auto itr = mPipelineNameFileTagConfigsMap.find(name);
+    if (itr != mPipelineNameFileTagConfigsMap.end()) {
+        return itr->second;
+    }
+    return make_pair(nullptr, nullptr);
+}
+
+// 添加Tag配置
+void FileServer::AddFileTagConfig(const std::string& name, const FileTagOptions* opts, const PipelineContext* ctx) {
+    WriteLock lock(mReadWriteLock);
+    mPipelineNameFileTagConfigsMap[name] = make_pair(opts, ctx);
+}
+
+// 移除给定名称的Tag配置
+void FileServer::RemoveFileTagConfig(const string& name) {
+    WriteLock lock(mReadWriteLock);
+    mPipelineNameFileTagConfigsMap.erase(name);
+}
+
+
 // 保存容器信息
 void FileServer::SaveContainerInfo(const string& pipeline, const shared_ptr<vector<ContainerInfo>>& info) {
     WriteLock lock(mReadWriteLock);

diff --git a/core/file_server/FileServer.h b/core/file_server/FileServer.h
@@ -22,6 +22,7 @@
 
 #include "common/Lock.h"
 #include "file_server/FileDiscoveryOptions.h"
+#include "file_server/FileTagOptions.h"
 #include "file_server/MultilineOptions.h"
 #include "file_server/reader/FileReaderOptions.h"
 #include "monitor/LogtailMetric.h"
@@ -66,6 +67,13 @@ class FileServer {
     void AddMultilineConfig(const std::string& name, const MultilineOptions* opts, const PipelineContext* ctx);
     void RemoveMultilineConfig(const std::string& name);
 
+    FileTagConfig GetFileTagConfig(const std::string& name) const;
+    const std::unordered_map<std::string, FileTagConfig>& GetAllFileTagConfigs() const {
+        return mPipelineNameFileTagConfigsMap;
+    }
+    void AddFileTagConfig(const std::string& name, const FileTagOptions* opts, const PipelineContext* ctx);
+    void RemoveFileTagConfig(const std::string& name);
+
     void SaveContainerInfo(const std::string& pipeline, const std::shared_ptr<std::vector<ContainerInfo>>& info);
     std::shared_ptr<std::vector<ContainerInfo>> GetAndRemoveContainerInfo(const std::string& pipeline);
     void ClearContainerInfo();
@@ -101,6 +109,7 @@ class FileServer {
     std::unordered_map<std::string, FileDiscoveryConfig> mPipelineNameFileDiscoveryConfigsMap;
     std::unordered_map<std::string, FileReaderConfig> mPipelineNameFileReaderConfigsMap;
     std::unordered_map<std::string, MultilineConfig> mPipelineNameMultilineConfigsMap;
+    std::unordered_map<std::string, FileTagConfig> mPipelineNameFileTagConfigsMap;
     std::unordered_map<std::string, std::shared_ptr<std::vector<ContainerInfo>>> mAllContainerInfoMap;
     std::unordered_map<std::string, PluginMetricManagerPtr> mPipelineNamePluginMetricManagersMap;
     // 过渡使用