From ed04b931f703df684e6f3deae875f5e44437f4fd Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 02:36:22 +0000 Subject: [PATCH 01/17] Install libyang in azure pipeline --- .azure-pipelines/build-sairedis-template.yml | 26 ++++++++++- .azure-pipelines/build-swss-template.yml | 7 ++- .azure-pipelines/build-template.yml | 45 +++++++++++++++++++ .../test-docker-sonic-vs-template.yml | 2 +- BUILD | 2 +- azure-pipelines.yml | 6 ++- 6 files changed, 80 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/build-sairedis-template.yml b/.azure-pipelines/build-sairedis-template.yml index 7c6e1173..e9cd5c31 100644 --- a/.azure-pipelines/build-sairedis-template.yml +++ b/.azure-pipelines/build-sairedis-template.yml @@ -20,6 +20,10 @@ parameters: - name: sonic_slave type: string +- name: debian_version + type: string + default: buster + - name: swss_common_artifact_name type: string @@ -82,12 +86,31 @@ jobs: sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf + sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf sudo service redis-server start sudo apt-get install -y rsyslog sudo service rsyslog start displayName: "Install dependencies" + - task: DownloadPipelineArtifact@2 + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib + patterns: | + target/debs/${{ parameters.debian_version }}/libyang-*.deb + target/debs/${{ parameters.debian_version }}/libyang_*.deb + displayName: "Download libyang from common lib" + - script: | + set -ex + sudo dpkg -i $(find ./download -name *.deb) + workingDirectory: $(Build.ArtifactStagingDirectory) + displayName: "Install libyang from common lib" - task: DownloadPipelineArtifact@2 inputs: artifact: ${{ parameters.swss_common_artifact_name }} @@ -136,6 +159,7 @@ jobs: contents: 'syslog-all.tgz' targetFolder: $(Build.ArtifactStagingDirectory) - publish: $(Build.ArtifactStagingDirectory)/ - artifact: ${{ parameters.syslog_artifact_name }} + # publish artifact with retry count, otherwise "artifact already exist" error will block job re-run. + artifact: ${{ parameters.syslog_artifact_name }}_${System.JobAttempt} displayName: "Publish syslog artifacts" condition: always() diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml index d0d25e99..cb2dc47b 100644 --- a/.azure-pipelines/build-swss-template.yml +++ b/.azure-pipelines/build-swss-template.yml @@ -72,9 +72,6 @@ jobs: inputs: artifact: ${{ parameters.swss_common_artifact_name }} path: $(Build.ArtifactStagingDirectory)/download - patterns: | - libswsscommon_1.0.0_*.deb - libswsscommon-dev_1.0.0*.deb displayName: "Download pre-stage built ${{ parameters.swss_common_artifact_name }}" - task: DownloadPipelineArtifact@2 inputs: @@ -103,6 +100,8 @@ jobs: target/debs/${{ parameters.debian_version }}/libnl-genl*.deb target/debs/${{ parameters.debian_version }}/libnl-route*.deb target/debs/${{ parameters.debian_version }}/libnl-nf*.deb + target/debs/${{ parameters.debian_version }}/libyang-*.deb + target/debs/${{ parameters.debian_version }}/libyang_*.deb displayName: "Download common libs" - script: | @@ -110,7 +109,7 @@ jobs: sudo dpkg -i $(find ./download -name *.deb) rm -rf download || true workingDirectory: $(Build.ArtifactStagingDirectory) - displayName: "Install libnl3, sonic swss common, and sairedis" + displayName: "Install libnl3, libyang, sonic swss common, and sairedis" - script: | set -ex rm ../*.deb || true diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index e05faa65..3e5da933 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -67,6 +67,47 @@ jobs: libnl-nf-3-dev \ swig displayName: "Install dependencies" + - task: DownloadPipelineArtifact@2 + # amd64 artifact name does not has arch suffix + condition: eq('${{ parameters.arch }}', 'amd64') + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib + patterns: | + target/debs/${{ parameters.debian_version }}/libyang-*.deb + target/debs/${{ parameters.debian_version }}/libyang_*.deb + displayName: "Download libyang from amd64 common lib" + - task: DownloadPipelineArtifact@2 + condition: ne('${{ parameters.arch }}', 'amd64') + inputs: + source: specific + project: build + pipeline: Azure.sonic-buildimage.common_libs + runVersion: 'latestFromBranch' + runBranch: 'refs/heads/$(BUILD_BRANCH)' + path: $(Build.ArtifactStagingDirectory)/download + artifact: common-lib.${{ parameters.arch }} + patterns: | + target/debs/${{ parameters.debian_version }}/libyang-*.deb + target/debs/${{ parameters.debian_version }}/libyang_*.deb + displayName: "Download libyang from common lib" + - script: | + set -ex + sudo dpkg -i $(find ./download -name *.deb) + workingDirectory: $(Build.ArtifactStagingDirectory) + condition: ne('${{ parameters.debian_version }}', 'bullseye') + displayName: "Install libyang from common lib" + # Azure.sonic-buildimage.common_libs does not build libs for bullseye, so install libyang-dev by apt-get + - script: | + sudo apt-get install -qq -y \ + libyang-dev + condition: eq('${{ parameters.debian_version }}', 'bullseye') + displayName: "Install libyang for bullseye" - script: | set -ex rm ../*.deb || true @@ -93,7 +134,9 @@ jobs: sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf + sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf sudo service redis-server restart + sudo mkdir /usr/local/yang-models sudo dpkg -i libswsscommon_*.deb sudo dpkg -i python-swsscommon_*.deb @@ -103,6 +146,8 @@ jobs: pytest --cov=. --cov-report=xml mv coverage.xml tests/coverage.xml gcovr -r ./ -e ".*/swsscommon_wrap.cpp" --exclude-unreachable-branches --exclude-throw-branches -x --xml-pretty -o coverage.xml + + rm -rf $(Build.ArtifactStagingDirectory)/download displayName: "Run swss common unit tests" - publish: $(System.DefaultWorkingDirectory)/ artifact: ${{ parameters.artifact_name }} diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml index 4748b09f..698bfc77 100644 --- a/.azure-pipelines/test-docker-sonic-vs-template.yml +++ b/.azure-pipelines/test-docker-sonic-vs-template.yml @@ -41,7 +41,7 @@ jobs: ls -l sudo sonic-swss-common/.azure-pipelines/build_and_install_module.sh - sudo apt-get install -y libhiredis0.14 + sudo apt-get install -y libhiredis0.14 libyang0.16 sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb diff --git a/BUILD b/BUILD index 9b4fb653..a66a57a6 100644 --- a/BUILD +++ b/BUILD @@ -15,7 +15,7 @@ cc_library( includes = [ "common", ], - linkopts = ["-lpthread -lhiredis -lnl-genl-3 -lnl-nf-3 -lnl-route-3 -lnl-3 -lzmq -lboost_serialization -luuid"], + linkopts = ["-lpthread -lhiredis -lnl-genl-3 -lnl-nf-3 -lnl-route-3 -lnl-3 -lzmq -lboost_serialization -luuid -lyang"], visibility = ["//visibility:public"], ) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ea8e4464..c0c08c5a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,7 +68,7 @@ stages: sudo apt-get install -y python3-pip sudo pip3 install pytest sudo apt-get install -y python - sudo apt-get install cmake libgtest-dev libgmock-dev + sudo apt-get install cmake libgtest-dev libgmock-dev libyang-dev cd /usr/src/gtest && sudo cmake . && sudo make ARCH=$(dpkg --print-architecture) set -x @@ -95,6 +95,7 @@ stages: artifact_name: sonic-swss-common run_unit_test: true archive_gcov: true + debian_version: ${{ parameters.debian_version }} - stage: BuildArm dependsOn: Build @@ -107,6 +108,7 @@ stages: pool: sonicbld-armhf sonic_slave: sonic-slave-${{ parameters.debian_version }}-armhf artifact_name: sonic-swss-common.armhf + debian_version: ${{ parameters.debian_version }} - template: .azure-pipelines/build-template.yml parameters: @@ -115,6 +117,7 @@ stages: pool: sonicbld-arm64 sonic_slave: sonic-slave-${{ parameters.debian_version }}-arm64 artifact_name: sonic-swss-common.arm64 + debian_version: ${{ parameters.debian_version }} - stage: BuildBullseye dependsOn: Build @@ -139,6 +142,7 @@ stages: swss_common_artifact_name: sonic-swss-common artifact_name: sonic-sairedis syslog_artifact_name: sonic-sairedis.syslog + debian_version: ${{ parameters.debian_version }} - stage: BuildSwss dependsOn: BuildSairedis From 3d769138dba255b446be181d30dd2e930c2edbb5 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 04:19:10 +0000 Subject: [PATCH 02/17] Add profile config provider. --- common/Makefile.am | 1 + common/configdb.cpp | 19 +- common/database_config.json | 5 + common/dbconnector.cpp | 23 + common/dbconnector.h | 1 + common/defaultvalueprovider.cpp | 544 ++++++++++++++++++ common/defaultvalueprovider.h | 118 ++++ common/schema.h | 4 + pyext/swsscommon.i | 2 + tests/Makefile.am | 1 + tests/profileprovider_ut.cpp | 103 ++++ .../database_config.json | 5 + .../database_config0.json | 5 + .../database_config1.json | 5 + 14 files changed, 818 insertions(+), 18 deletions(-) create mode 100755 common/defaultvalueprovider.cpp create mode 100755 common/defaultvalueprovider.h create mode 100755 tests/profileprovider_ut.cpp diff --git a/common/Makefile.am b/common/Makefile.am index b47cfd21..df31d5ae 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -62,6 +62,7 @@ libswsscommon_la_SOURCES = \ linkcache.cpp \ portmap.cpp \ pubsub.cpp \ + profileprovider.cpp \ tokenize.cpp \ exec.cpp \ saiaclschema.cpp \ diff --git a/common/configdb.cpp b/common/configdb.cpp index 2d733f9f..ececb2d3 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -244,24 +244,7 @@ void ConfigDBConnector_Native::mod_config(const map>> ConfigDBConnector_Native::get_config() { auto& client = get_redis_client(m_db_name); - auto const& keys = client.keys("*"); - map>> data; - for (string key: keys) - { - size_t pos = key.find(m_table_name_separator); - if (pos == string::npos) { - continue; - } - string table_name = key.substr(0, pos); - string row = key.substr(pos + 1); - auto const& entry = client.hgetall>(key); - - if (!entry.empty()) - { - data[table_name][row] = entry; - } - } - return data; + return client.getall(); } std::string ConfigDBConnector_Native::getKeySeparator() const diff --git a/common/database_config.json b/common/database_config.json index 67b5cda8..642902f2 100644 --- a/common/database_config.json +++ b/common/database_config.json @@ -91,6 +91,11 @@ "id" : 14, "separator": ":", "instance" : "redis" + }, + "PROFILE_DB" : { + "id" : 15, + "separator": ":", + "instance" : "redis" } }, "VERSION" : "1.0" diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index ac17928a..d5807781 100755 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -905,3 +905,26 @@ void DBConnector::del(const std::vector& keys) pipe.flush(); } + +map>> DBConnector::getall() +{ + const string separator = SonicDBConfig::getSeparator(this); + auto const& keys = this->keys("*"); + map>> data; + for (string key: keys) + { + size_t pos = key.find(separator); + if (pos == string::npos) { + continue; + } + string table_name = key.substr(0, pos); + string row = key.substr(pos + 1); + auto const& entry = this->hgetall>(key); + + if (!entry.empty()) + { + data[table_name][row] = entry; + } + } + return data; +} \ No newline at end of file diff --git a/common/dbconnector.h b/common/dbconnector.h index eb37a6e8..8a70a2cf 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -248,6 +248,7 @@ class DBConnector : public RedisContext bool flushdb(); + std::map>> getall(); private: void setNamespace(const std::string &netns); diff --git a/common/defaultvalueprovider.cpp b/common/defaultvalueprovider.cpp new file mode 100755 index 00000000..5be472ea --- /dev/null +++ b/common/defaultvalueprovider.cpp @@ -0,0 +1,544 @@ +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "defaultvalueprovider.h" +#include "logger.h" +#include "table.h" +#include "json.h" + +#if defined(__arm__) || defined(__aarch64__) +#define WARNINGS_NO_CAST_ALIGN \ + _Pragma ("GCC diagnostic push") \ + _Pragma ("GCC diagnostic ignored \"-Wcast-align\"") +#define WARNINGS_RESET \ + _Pragma ("GCC diagnostic pop") +#else +#define WARNINGS_NO_CAST_ALIGN +#define WARNINGS_RESET +#endif + +using namespace std; +using namespace swss; + +[[noreturn]] void ThrowRunTimeError(string message) +{ + SWSS_LOG_ERROR("DefaultValueProvider: %s", message.c_str()); + throw runtime_error(message); +} + +TableInfoBase::TableInfoBase() +{ + // C++ need this empty ctor +} + +shared_ptr TableInfoBase::GetDefaultValue(const string &key, const string &field) +{ + assert(!key.empty()); + assert(!field.empty()); + + SWSS_LOG_DEBUG("TableInfoBase::GetDefaultValue %s %s\n", key.c_str(), field.c_str()); + FieldDefaultValueMapping *fieldMappingPtr; + if (!FindFieldMappingByKey(key, &fieldMappingPtr)) { + SWSS_LOG_DEBUG("Can't found default value mapping for key %s\n", key.c_str()); + return nullptr; + } + + auto fieldData = fieldMappingPtr->find(field); + if (fieldData == fieldMappingPtr->end()) + { + SWSS_LOG_DEBUG("Can't found default value for field %s\n", field.c_str()); + return nullptr; + } + + SWSS_LOG_DEBUG("Found default value for field %s=%s\n", field.c_str(), fieldData->second.c_str()); + return make_shared(fieldData->second); +} + +// existedValues and targetValues can be same container. +void TableInfoBase::AppendDefaultValues(const string &row, map& existedValues, map& targetValues) +{ + assert(!row.empty()); + + SWSS_LOG_DEBUG("TableInfoBase::AppendDefaultValues %s\n", row.c_str()); + FieldDefaultValueMapping *fieldMappingPtr; + if (!FindFieldMappingByKey(row, &fieldMappingPtr)) { + SWSS_LOG_DEBUG("Can't found default value mapping for row %s\n", row.c_str()); + return; + } + + for (auto &defaultValue : *fieldMappingPtr) + { + auto fieldData = existedValues.find(defaultValue.first); + if (fieldData != existedValues.end()) + { + // ignore when a field already has value in existedValues + continue; + } + + SWSS_LOG_DEBUG("Append default value: %s=%s\n",defaultValue.first.c_str(), defaultValue.second.c_str()); + targetValues.emplace(defaultValue.first, defaultValue.second); + } +} + +TableInfoDict::TableInfoDict(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) +{ + for (auto& fieldMapping : fieldInfoMapping) + { + // KeyInfo.first is key value + string keyValue = fieldMapping.first.first; + m_defaultValueMapping.emplace(keyValue, fieldMapping.second); + + SWSS_LOG_DEBUG("TableInfoDict::TableInfoDict %s\n", keyValue.c_str()); + } +} + +bool TableInfoDict::FindFieldMappingByKey(const string &key, FieldDefaultValueMapping ** foundedMappingPtr) +{ + assert(!key.empty()); + assert(foundedMappingPtr != nullptr); + + SWSS_LOG_DEBUG("TableInfoDict::FindFieldMappingByKey %s\n", key.c_str()); + auto keyResult = m_defaultValueMapping.find(key); + *foundedMappingPtr = keyResult->second.get(); + return keyResult != m_defaultValueMapping.end(); +} + +TableInfoSingleList::TableInfoSingleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) +{ + m_defaultValueMapping = fieldInfoMapping.begin()->second; +} + +bool TableInfoSingleList::FindFieldMappingByKey(const string &key, FieldDefaultValueMapping ** foundedMappingPtr) +{ + assert(!key.empty()); + assert(foundedMappingPtr != nullptr); + + SWSS_LOG_DEBUG("TableInfoSingleList::FindFieldMappingByKey %s\n", key.c_str()); + *foundedMappingPtr = m_defaultValueMapping.get(); + return true; +} + +TableInfoMultipleList::TableInfoMultipleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) +{ + for (auto& fieldMapping : fieldInfoMapping) + { + // KeyInfo.second is field count + int fieldCount = fieldMapping.first.second; + m_defaultValueMapping.emplace(fieldCount, fieldMapping.second); + } +} + +bool TableInfoMultipleList::FindFieldMappingByKey(const string &key, FieldDefaultValueMapping ** foundedMappingPtr) +{ + assert(!key.empty()); + assert(foundedMappingPtr != nullptr); + + SWSS_LOG_DEBUG("TableInfoMultipleList::FindFieldMappingByKey %s\n", key.c_str()); + int fieldCount = (int)count(key.begin(), key.end(), '|') + 1; + auto keyInfo = m_defaultValueMapping.find(fieldCount); + + // when not found, key_info still a valied iterator + *foundedMappingPtr = keyInfo->second.get(); + + // return false when not found + return keyInfo != m_defaultValueMapping.end(); +} + +DefaultValueProvider& DefaultValueProvider::instance() +{ + static DefaultValueProvider instance; +#ifndef UNITTEST + if (instance.m_context == nullptr) + { + instance.Initialize(); + } +#endif + + return instance; +} + +shared_ptr DefaultValueProvider::FindDefaultValueInfo(const string &table) +{ + assert(!table.empty()); + + SWSS_LOG_DEBUG("DefaultValueProvider::FindDefaultValueInfo %s\n", table.c_str()); + auto findResult = m_defaultValueMapping.find(table); + if (findResult == m_defaultValueMapping.end()) + { + SWSS_LOG_DEBUG("Not found default value info for %s\n", table.c_str()); + return nullptr; + } + + return findResult->second; +} + +shared_ptr DefaultValueProvider::getDefaultValue(const string &table, const string &key, const string &field) +{ + assert(!table.empty()); + assert(!key.empty()); + assert(!field.empty()); + + SWSS_LOG_DEBUG("DefaultValueProvider::GetDefaultValue %s %s %s\n", table.c_str(), key.c_str(), field.c_str()); + + auto defaultValueInfo = FindDefaultValueInfo(table); + if (defaultValueInfo == nullptr) + { + SWSS_LOG_DEBUG("Not found default value info for %s\n", table.c_str()); + return nullptr; + } + + return defaultValueInfo->GetDefaultValue(key, field); +} + +void DefaultValueProvider::InternalAppendDefaultValues(const string &table, const string &key, map& existedValues, map& defaultValues) +{ + assert(!table.empty()); + assert(!key.empty()); + + SWSS_LOG_DEBUG("DefaultValueProvider::InternalAppendDefaultValues %s %s\n", table.c_str(), key.c_str()); + + auto defaultValueInfo = FindDefaultValueInfo(table); + if (defaultValueInfo == nullptr) + { + SWSS_LOG_DEBUG("Not found default value info for %s\n", table.c_str()); + return; + } + + defaultValueInfo->AppendDefaultValues(key, existedValues, defaultValues); +} + +void DefaultValueProvider::appendDefaultValues(const string &table, const string &key, vector> &values) +{ + map existedValues; + map defaultValues; + for (auto& fieldValuePair : values) + { + existedValues.emplace(fieldValuePair.first, fieldValuePair.second); + } + + InternalAppendDefaultValues(table, key, existedValues, defaultValues); + + for (auto& fieldValuePair : defaultValues) + { + values.emplace_back(fieldValuePair.first, fieldValuePair.second); + } +} + +map DefaultValueProvider::getDefaultValues(const string &table, const string &key) +{ + map result; + InternalAppendDefaultValues(table, key, result, result); + return result; +} + +DefaultValueProvider::DefaultValueProvider() +{ +} + + +DefaultValueProvider::~DefaultValueProvider() +{ + if (m_context) + { + // set private_destructor to NULL because no any private data + ly_ctx_destroy(m_context, NULL); + } +} + +void DefaultValueProvider::Initialize(char* modulePath) +{ + assert(modulePath != nullptr && strlen(modulePath) != 0); + assert(m_context == nullptr); + + DIR *moduleDir = opendir(modulePath); + if (!moduleDir) + { + ThrowRunTimeError("Open Yang model path " + string(modulePath) + " failed"); + } + + m_context = ly_ctx_new(modulePath, LY_CTX_ALLIMPLEMENTED); + struct dirent *subDir; + while ((subDir = readdir(moduleDir)) != nullptr) + { + if (subDir->d_type != DT_REG) + { + continue; + } + + SWSS_LOG_DEBUG("file name: %s\n", subDir->d_name); + string fileName(subDir->d_name); + int pos = (int)fileName.find(".yang"); + string moduleName = fileName.substr(0, pos); + + LoadModule(moduleName, fileName, m_context); + } + closedir(moduleDir); +} + +void DefaultValueProvider::LoadModule(const string name, const string path, struct ly_ctx *context) +{ + const struct lys_module *module = ly_ctx_load_module( + context, + name.c_str(), + EMPTY_STR); // Use EMPTY_STR to revision to load the latest revision + if (module == nullptr) + { + const char* err = ly_errmsg(context); + SWSS_LOG_ERROR("Load Yang file %s failed: %s.\n", path.c_str(), err); + return; + } + + if (module->data == nullptr) + { + // Not every yang file should contains yang model + SWSS_LOG_WARN("Yang file %s does not contains model %s.\n", path.c_str(), name.c_str()); + return; + } + + struct lys_node *topLevelNode = module->data; + while (topLevelNode) + { + if (topLevelNode->nodetype != LYS_CONTAINER) + { + SWSS_LOG_DEBUG("ignore top level element %s, tyoe %d\n",topLevelNode->name, topLevelNode->nodetype); + // Config DB table schema is defined by top level container + topLevelNode = topLevelNode->next; + continue; + } + + SWSS_LOG_DEBUG("top level container: %s\n",topLevelNode->name); + auto container = topLevelNode->child; + while (container) + { + SWSS_LOG_DEBUG("container name: %s\n",container->name); + + AppendTableInfoToMapping(container); + container = container->next; + } + + topLevelNode = topLevelNode->next; + } +} + +shared_ptr DefaultValueProvider::GetKeyInfo(struct lys_node* tableChildNode) +{ + assert(tableChildNode != nullptr); + SWSS_LOG_DEBUG("DefaultValueProvider::GetKeyInfo %s\n",tableChildNode->name); + + int keyFieldCount = 0; + string keyValue = ""; + if (tableChildNode->nodetype == LYS_LIST) + { + SWSS_LOG_DEBUG("Child list: %s\n",tableChildNode->name); + + // when a top level container contains list, the key defined by the 'keys' field. + struct lys_node_list *listNode = (struct lys_node_list*)tableChildNode; + if (listNode->keys_str == nullptr) + { + SWSS_LOG_ERROR("Ignore empty key string on list: %s\n",tableChildNode->name); + return nullptr; + } + + string key(listNode->keys_str); + keyFieldCount = (int)count(key.begin(), key.end(), ' ') + 1; + } + else if (tableChildNode->nodetype == LYS_CONTAINER) + { + SWSS_LOG_DEBUG("Child container name: %s\n",tableChildNode->name); + + // when a top level container not contains any list, the key is child container name + keyValue = string(tableChildNode->name); + } + else + { + SWSS_LOG_DEBUG("Ignore child element: %s\n",tableChildNode->name); + return nullptr; + } + + return make_shared(keyValue, keyFieldCount); +} + +void DefaultValueProvider::GetDefaultValueInfoForLeaf(struct lys_node* field, shared_ptr fieldMapping) +{ + assert(field != nullptr); + assert(fieldMapping != nullptr); + assert(field->nodetype == LYS_LEAF); + + WARNINGS_NO_CAST_ALIGN + struct lys_node_leaf *leafNode = reinterpret_cast(field); + WARNINGS_RESET + if (leafNode->dflt) + { + SWSS_LOG_DEBUG("field: %s, default: %s\n",leafNode->name, leafNode->dflt); + fieldMapping->emplace(string(field->name), string(leafNode->dflt)); + } +} + +void DefaultValueProvider::GetDefaultValueInfoForChoice(struct lys_node* field, shared_ptr fieldMapping) +{ + assert(field != nullptr); + assert(fieldMapping != nullptr); + assert(field->nodetype == LYS_CHOICE); + + struct lys_node_choice *choiceNode = (struct lys_node_choice *)field; + if (choiceNode->dflt == nullptr) + { + return; + } + + // Get choice default value according to: https://www.rfc-editor.org/rfc/rfc7950.html#section-7.9 + auto dfltChoice = choiceNode->dflt; + auto fieldInChoice = dfltChoice->child; + while (fieldInChoice) + { + if (fieldInChoice->nodetype != LYS_LEAF) + { + SWSS_LOG_ERROR("choice case %s is not a leaf node\n",fieldInChoice->name); + continue; + } + + SWSS_LOG_DEBUG("default choice leaf field: %s\n",fieldInChoice->name); + WARNINGS_NO_CAST_ALIGN + struct lys_node_leaf *dfltLeafNode = reinterpret_cast(fieldInChoice); + WARNINGS_RESET + if (dfltLeafNode->dflt) + { + SWSS_LOG_DEBUG("default choice leaf field: %s, default: %s\n",dfltLeafNode->name, dfltLeafNode->dflt); + fieldMapping->emplace(string(fieldInChoice->name), string(dfltLeafNode->dflt)); + } + + fieldInChoice = fieldInChoice->next; + } +} + +void DefaultValueProvider::GetDefaultValueInfoForLeaflist(struct lys_node* field, shared_ptr fieldMapping) +{ + assert(field != nullptr); + assert(fieldMapping != nullptr); + assert(field->nodetype == LYS_LEAFLIST); + + // Get leaf-list default value according to:https://www.rfc-editor.org/rfc/rfc7950.html#section-7.7 + WARNINGS_NO_CAST_ALIGN + struct lys_node_leaflist *listNode = reinterpret_cast(field); + WARNINGS_RESET + + if (listNode->dflt) + { + const char** dfltValues = listNode->dflt; + //convert list default value to json string + string dfltValueJson = JSon::buildJson(dfltValues); + SWSS_LOG_DEBUG("list field: %s, default: %s\n",field->name, dfltValueJson.c_str()); + fieldMapping->emplace(string(field->name), dfltValueJson); + } +} + +FieldDefaultValueMappingPtr DefaultValueProvider::GetDefaultValueInfo(struct lys_node* tableChildNode) +{ + assert(tableChildNode != nullptr); + SWSS_LOG_DEBUG("DefaultValueProvider::GetDefaultValueInfo %s\n",tableChildNode->name); + + auto field = tableChildNode->child; + auto fieldMapping = make_shared(); + while (field) + { + if (field->nodetype == LYS_LEAF) + { + SWSS_LOG_DEBUG("leaf field: %s\n",field->name); + GetDefaultValueInfoForLeaf(field, fieldMapping); + } + else if (field->nodetype == LYS_CHOICE) + { + SWSS_LOG_DEBUG("choice field: %s\n",field->name); + GetDefaultValueInfoForChoice(field, fieldMapping); + } + else if (field->nodetype == LYS_LEAFLIST) + { + SWSS_LOG_DEBUG("list field: %s\n",field->name); + GetDefaultValueInfoForLeaflist(field, fieldMapping); + } +#ifdef DEBUG + else + { + SWSS_LOG_DEBUG("Field %s with type %d does not support default value\n",field->name, field->nodetype); + } +#endif + + field = field->next; + } + + return fieldMapping; +} + +int DefaultValueProvider::BuildFieldMappingList(struct lys_node* table, KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) +{ + assert(table != nullptr); + + int childListCount = 0; + auto nextChild = table->child; + while (nextChild) + { + // get key from schema + auto keyInfo = GetKeyInfo(nextChild); + if (keyInfo == nullptr) + { + nextChild = nextChild->next; + continue; + } + else if (keyInfo->second != 0) + { + // when key field count not 0, it's a list node. + childListCount++; + } + + // get field name to default value mappings from schema + fieldInfoMapping.emplace(*keyInfo, GetDefaultValueInfo(nextChild)); + + nextChild = nextChild->next; + } + + return childListCount; +} + +// Load default value info from yang model and append to default value mapping +void DefaultValueProvider::AppendTableInfoToMapping(struct lys_node* table) +{ + assert(table != nullptr); + + SWSS_LOG_DEBUG("DefaultValueProvider::AppendTableInfoToMapping table name: %s\n",table->name); + KeyInfoToDefaultValueInfoMapping fieldInfoMapping; + int listCount = BuildFieldMappingList(table, fieldInfoMapping); + + // create container data by list count + shared_ptr tableInfoPtr = nullptr; + switch (listCount) + { + case 0: + { + tableInfoPtr = shared_ptr(new TableInfoDict(fieldInfoMapping)); + } + break; + + case 1: + { + tableInfoPtr = shared_ptr(new TableInfoSingleList(fieldInfoMapping)); + } + break; + + default: + { + tableInfoPtr = shared_ptr(new TableInfoMultipleList(fieldInfoMapping)); + } + break; + } + + m_defaultValueMapping.emplace(string(table->name), tableInfoPtr); +} \ No newline at end of file diff --git a/common/defaultvalueprovider.h b/common/defaultvalueprovider.h new file mode 100755 index 00000000..c29556a4 --- /dev/null +++ b/common/defaultvalueprovider.h @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include "common/table.h" + +#define DEFAULT_YANG_MODULE_PATH "/usr/local/yang-models" +#define EMPTY_STR "" + +struct ly_ctx; + +// Key information +typedef std::pair KeyInfo; + +// Field name to default value mapping +typedef std::map FieldDefaultValueMapping; +typedef std::shared_ptr FieldDefaultValueMappingPtr; + +// Key info to default value info mapping +typedef std::map KeyInfoToDefaultValueInfoMapping; + +namespace swss { + +class TableInfoBase +{ +public: + TableInfoBase(); + + void AppendDefaultValues(const std::string &row, std::map& sourceValues, std::map& targetValues); + + std::shared_ptr GetDefaultValue(const std::string &row, const std::string &field); + +protected: + virtual bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr) = 0; +}; + +class TableInfoDict : public TableInfoBase +{ +public: + TableInfoDict(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping); + +private: + // Mapping: key value -> field -> default + std::map m_defaultValueMapping; + + bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr); +}; + +class TableInfoSingleList : public TableInfoBase +{ +public: + TableInfoSingleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping); + +private: + // Mapping: field -> default + FieldDefaultValueMappingPtr m_defaultValueMapping; + + bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr); +}; + +struct TableInfoMultipleList : public TableInfoBase +{ +public: + TableInfoMultipleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping); + +private: + // Mapping: key field count -> field -> default + std::map m_defaultValueMapping; + + bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr); +}; + +class DefaultValueProvider +{ +public: + static DefaultValueProvider& instance(); + + void appendDefaultValues(const std::string &table, const std::string &key, std::vector> &values); + + std::shared_ptr getDefaultValue(const std::string &table, const std::string &key, const std::string &field); + + std::map getDefaultValues(const std::string &table, const std::string &key); + +#ifndef UNITTEST +private: +#endif + DefaultValueProvider(); + ~DefaultValueProvider(); + + // libyang context + struct ly_ctx *m_context = nullptr; + + // The table name to table default value info mapping + std::map > m_defaultValueMapping; + + void Initialize(char* modulePath = DEFAULT_YANG_MODULE_PATH); + + void LoadModule(const std::string name, const std::string path, struct ly_ctx *context); + + // Load default value info from yang model and append to default value mapping + void AppendTableInfoToMapping(struct lys_node* table); + + std::shared_ptr FindDefaultValueInfo(const std::string &table); + + int BuildFieldMappingList(struct lys_node* table, KeyInfoToDefaultValueInfoMapping& fieldMappingList); + + std::shared_ptr GetKeyInfo(struct lys_node* table_child_node); + FieldDefaultValueMappingPtr GetDefaultValueInfo(struct lys_node* tableChildNode); + + void InternalAppendDefaultValues(const std::string &table, const std::string &key, std::map& existedValues, std::map& defaultValues); + + void GetDefaultValueInfoForChoice(struct lys_node* field, std::shared_ptr fieldMapping); + void GetDefaultValueInfoForLeaf(struct lys_node* field, std::shared_ptr fieldMapping); + void GetDefaultValueInfoForLeaflist(struct lys_node* field, std::shared_ptr fieldMapping); +}; + +} diff --git a/common/schema.h b/common/schema.h index 73fb3b94..86e0f832 100644 --- a/common/schema.h +++ b/common/schema.h @@ -490,6 +490,10 @@ namespace swss { #define STATE_FLOW_COUNTER_CAPABILITY_TABLE_NAME "FLOW_COUNTER_CAPABILITY_TABLE" +/***** PROFILE DATABASE *****/ + +#define PROFILE_DELETE_TABLE "PROFILE_DELETE" + /***** MISC *****/ #define IPV4_NAME "IPv4" diff --git a/pyext/swsscommon.i b/pyext/swsscommon.i index 41f2280c..8be02ebd 100644 --- a/pyext/swsscommon.i +++ b/pyext/swsscommon.i @@ -29,6 +29,7 @@ #include "consumertablebase.h" #include "consumerstatetable.h" #include "producertable.h" +#include "profileprovider.h" #include "consumertable.h" #include "subscriberstatetable.h" #include "notificationconsumer.h" @@ -153,6 +154,7 @@ T castSelectableObj(swss::Selectable *temp) %include "dbconnector.h" %include "sonicv2connector.h" %include "pubsub.h" +%include "profileprovider.h" %include "selectable.h" %include "select.h" %include "rediscommand.h" diff --git a/tests/Makefile.am b/tests/Makefile.am index d53f8899..3af4dc81 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -41,6 +41,7 @@ tests_SOURCES = redis_ut.cpp \ events_common_ut.cpp \ events_service_ut.cpp \ events_ut.cpp \ + profileprovider_ut.cpp \ main.cpp tests_CFLAGS = $(DBGFLAGS) $(AM_CFLAGS) $(CFLAGS_COMMON) $(CFLAGS_GTEST) $(LIBNL_CFLAGS) diff --git a/tests/profileprovider_ut.cpp b/tests/profileprovider_ut.cpp new file mode 100755 index 00000000..f3882454 --- /dev/null +++ b/tests/profileprovider_ut.cpp @@ -0,0 +1,103 @@ +#include "gtest/gtest.h" +#include "common/profileprovider.h" +#include "common/dbconnector.h" +#include "common/configdb.h" + +using namespace std; +using namespace swss; + +string profile_table = "INTERFACE"; +string profile_key = "TEST_INTERFACE"; +string profile_field = "profile"; +string profile_value = "value"; + +void clearDB(const string &dbName) +{ + DBConnector db(dbName, 0, true); + RedisReply r(&db, "FLUSHALL", REDIS_REPLY_STATUS); + r.checkStatusOK(); +} + +void initializeProfileDB() +{ + clearDB("PROFILE_DB"); + clearDB("CONFIG_DB"); + + auto db = ConfigDBConnector_Native(); + db.db_connect("PROFILE_DB"); + map profile_map = { + { profile_field, profile_value } + }; + db.set_entry(profile_table, profile_key, profile_map); + + auto item = db.get_entry(profile_table, profile_key); + EXPECT_EQ(item[profile_field], profile_value); +} + +TEST(DECORATOR, GetConfigs) +{ + initializeProfileDB(); + + DBConnector db("CONFIG_DB", 0, true); + + auto configs = ProfileProvider::instance().getConfigs(&db); + EXPECT_EQ(configs[profile_table][profile_key][profile_field], profile_value); + + auto item = ProfileProvider::instance().getConfigs(profile_table, profile_key, &db); + EXPECT_EQ(item[profile_field], profile_value); + + auto valueptr = ProfileProvider::instance().getConfig(profile_table, profile_key, profile_field, &db); + EXPECT_EQ(*valueptr, profile_value); + + auto keys = ProfileProvider::instance().getKeys(profile_table, &db); + EXPECT_EQ(keys.size(), 1); + EXPECT_EQ(keys[0], profile_key); +} + +TEST(DECORATOR, DeleteAndRevertProfile) +{ + initializeProfileDB(); + + DBConnector db("CONFIG_DB", 0, true); + auto connector = ConfigDBConnector_Native(); + connector.connect(false); + + // test delete + bool result = ProfileProvider::instance().tryDeleteItem(profile_table, profile_key, &db); + EXPECT_EQ(result, true); + + auto deletedItems = connector.get_keys(PROFILE_DELETE_TABLE, false); + EXPECT_EQ(deletedItems.size(), 1); + + auto configs = ProfileProvider::instance().getConfigs(&db); + EXPECT_EQ(configs[profile_table].find(profile_key), configs[profile_table].end()); + + auto item = ProfileProvider::instance().getConfigs(profile_table, profile_key, &db); + EXPECT_EQ(item.size(), 0); + + auto valueptr = ProfileProvider::instance().getConfig(profile_table, profile_key, profile_field, &db); + EXPECT_EQ(valueptr, nullptr); + + auto keys = ProfileProvider::instance().getKeys(profile_table, &db); + EXPECT_EQ(keys.size(), 0); + + // test revert + result = ProfileProvider::instance().tryRevertItem(profile_table, profile_key, &db); + EXPECT_EQ(result, true); + + deletedItems = connector.get_keys(PROFILE_DELETE_TABLE, false); + EXPECT_EQ(deletedItems.size(), 0); + + configs = ProfileProvider::instance().getConfigs(&db); + EXPECT_EQ(configs[profile_table][profile_key][profile_field], profile_value); + + item = ProfileProvider::instance().getConfigs(profile_table, profile_key, &db); + EXPECT_EQ(item[profile_field], profile_value); + + valueptr = ProfileProvider::instance().getConfig(profile_table, profile_key, profile_field, &db); + EXPECT_EQ(*valueptr, profile_value); + + keys = ProfileProvider::instance().getKeys(profile_table, &db); + EXPECT_EQ(keys.size(), 1); + EXPECT_EQ(keys[0], profile_key); +} diff --git a/tests/redis_multi_db_ut_config/database_config.json b/tests/redis_multi_db_ut_config/database_config.json index 141bb65f..2165eec1 100644 --- a/tests/redis_multi_db_ut_config/database_config.json +++ b/tests/redis_multi_db_ut_config/database_config.json @@ -106,6 +106,11 @@ "id" : 15, "separator": ":", "instance" : "redis" + }, + "PROFILE_DB" : { + "id" : 16, + "separator": ":", + "instance" : "redis" } }, "VERSION" : "1.0" diff --git a/tests/redis_multi_db_ut_config/database_config0.json b/tests/redis_multi_db_ut_config/database_config0.json index 3d9287e8..899912d4 100644 --- a/tests/redis_multi_db_ut_config/database_config0.json +++ b/tests/redis_multi_db_ut_config/database_config0.json @@ -86,6 +86,11 @@ "id" : 15, "separator": ":", "instance" : "redis" + }, + "PROFILE_DB" : { + "id" : 16, + "separator": ":", + "instance" : "redis" } }, "VERSION" : "1.0" diff --git a/tests/redis_multi_db_ut_config/database_config1.json b/tests/redis_multi_db_ut_config/database_config1.json index 58dfb7f7..dbaef6ef 100644 --- a/tests/redis_multi_db_ut_config/database_config1.json +++ b/tests/redis_multi_db_ut_config/database_config1.json @@ -86,6 +86,11 @@ "id" : 15, "separator": ":", "instance" : "redis" + }, + "PROFILE_DB" : { + "id" : 16, + "separator": ":", + "instance" : "redis" } }, "VERSION" : "1.0" From 364f353e8f2911545ee37b21052bd2f03b1caf26 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 05:03:15 +0000 Subject: [PATCH 03/17] Fix missing file --- common/defaultvalueprovider.cpp | 544 -------------------------------- common/defaultvalueprovider.h | 118 ------- common/profileprovider.cpp | 216 +++++++++++++ common/profileprovider.h | 61 ++++ 4 files changed, 277 insertions(+), 662 deletions(-) delete mode 100755 common/defaultvalueprovider.cpp delete mode 100755 common/defaultvalueprovider.h create mode 100755 common/profileprovider.cpp create mode 100755 common/profileprovider.h diff --git a/common/defaultvalueprovider.cpp b/common/defaultvalueprovider.cpp deleted file mode 100755 index 5be472ea..00000000 --- a/common/defaultvalueprovider.cpp +++ /dev/null @@ -1,544 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include "defaultvalueprovider.h" -#include "logger.h" -#include "table.h" -#include "json.h" - -#if defined(__arm__) || defined(__aarch64__) -#define WARNINGS_NO_CAST_ALIGN \ - _Pragma ("GCC diagnostic push") \ - _Pragma ("GCC diagnostic ignored \"-Wcast-align\"") -#define WARNINGS_RESET \ - _Pragma ("GCC diagnostic pop") -#else -#define WARNINGS_NO_CAST_ALIGN -#define WARNINGS_RESET -#endif - -using namespace std; -using namespace swss; - -[[noreturn]] void ThrowRunTimeError(string message) -{ - SWSS_LOG_ERROR("DefaultValueProvider: %s", message.c_str()); - throw runtime_error(message); -} - -TableInfoBase::TableInfoBase() -{ - // C++ need this empty ctor -} - -shared_ptr TableInfoBase::GetDefaultValue(const string &key, const string &field) -{ - assert(!key.empty()); - assert(!field.empty()); - - SWSS_LOG_DEBUG("TableInfoBase::GetDefaultValue %s %s\n", key.c_str(), field.c_str()); - FieldDefaultValueMapping *fieldMappingPtr; - if (!FindFieldMappingByKey(key, &fieldMappingPtr)) { - SWSS_LOG_DEBUG("Can't found default value mapping for key %s\n", key.c_str()); - return nullptr; - } - - auto fieldData = fieldMappingPtr->find(field); - if (fieldData == fieldMappingPtr->end()) - { - SWSS_LOG_DEBUG("Can't found default value for field %s\n", field.c_str()); - return nullptr; - } - - SWSS_LOG_DEBUG("Found default value for field %s=%s\n", field.c_str(), fieldData->second.c_str()); - return make_shared(fieldData->second); -} - -// existedValues and targetValues can be same container. -void TableInfoBase::AppendDefaultValues(const string &row, map& existedValues, map& targetValues) -{ - assert(!row.empty()); - - SWSS_LOG_DEBUG("TableInfoBase::AppendDefaultValues %s\n", row.c_str()); - FieldDefaultValueMapping *fieldMappingPtr; - if (!FindFieldMappingByKey(row, &fieldMappingPtr)) { - SWSS_LOG_DEBUG("Can't found default value mapping for row %s\n", row.c_str()); - return; - } - - for (auto &defaultValue : *fieldMappingPtr) - { - auto fieldData = existedValues.find(defaultValue.first); - if (fieldData != existedValues.end()) - { - // ignore when a field already has value in existedValues - continue; - } - - SWSS_LOG_DEBUG("Append default value: %s=%s\n",defaultValue.first.c_str(), defaultValue.second.c_str()); - targetValues.emplace(defaultValue.first, defaultValue.second); - } -} - -TableInfoDict::TableInfoDict(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) -{ - for (auto& fieldMapping : fieldInfoMapping) - { - // KeyInfo.first is key value - string keyValue = fieldMapping.first.first; - m_defaultValueMapping.emplace(keyValue, fieldMapping.second); - - SWSS_LOG_DEBUG("TableInfoDict::TableInfoDict %s\n", keyValue.c_str()); - } -} - -bool TableInfoDict::FindFieldMappingByKey(const string &key, FieldDefaultValueMapping ** foundedMappingPtr) -{ - assert(!key.empty()); - assert(foundedMappingPtr != nullptr); - - SWSS_LOG_DEBUG("TableInfoDict::FindFieldMappingByKey %s\n", key.c_str()); - auto keyResult = m_defaultValueMapping.find(key); - *foundedMappingPtr = keyResult->second.get(); - return keyResult != m_defaultValueMapping.end(); -} - -TableInfoSingleList::TableInfoSingleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) -{ - m_defaultValueMapping = fieldInfoMapping.begin()->second; -} - -bool TableInfoSingleList::FindFieldMappingByKey(const string &key, FieldDefaultValueMapping ** foundedMappingPtr) -{ - assert(!key.empty()); - assert(foundedMappingPtr != nullptr); - - SWSS_LOG_DEBUG("TableInfoSingleList::FindFieldMappingByKey %s\n", key.c_str()); - *foundedMappingPtr = m_defaultValueMapping.get(); - return true; -} - -TableInfoMultipleList::TableInfoMultipleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) -{ - for (auto& fieldMapping : fieldInfoMapping) - { - // KeyInfo.second is field count - int fieldCount = fieldMapping.first.second; - m_defaultValueMapping.emplace(fieldCount, fieldMapping.second); - } -} - -bool TableInfoMultipleList::FindFieldMappingByKey(const string &key, FieldDefaultValueMapping ** foundedMappingPtr) -{ - assert(!key.empty()); - assert(foundedMappingPtr != nullptr); - - SWSS_LOG_DEBUG("TableInfoMultipleList::FindFieldMappingByKey %s\n", key.c_str()); - int fieldCount = (int)count(key.begin(), key.end(), '|') + 1; - auto keyInfo = m_defaultValueMapping.find(fieldCount); - - // when not found, key_info still a valied iterator - *foundedMappingPtr = keyInfo->second.get(); - - // return false when not found - return keyInfo != m_defaultValueMapping.end(); -} - -DefaultValueProvider& DefaultValueProvider::instance() -{ - static DefaultValueProvider instance; -#ifndef UNITTEST - if (instance.m_context == nullptr) - { - instance.Initialize(); - } -#endif - - return instance; -} - -shared_ptr DefaultValueProvider::FindDefaultValueInfo(const string &table) -{ - assert(!table.empty()); - - SWSS_LOG_DEBUG("DefaultValueProvider::FindDefaultValueInfo %s\n", table.c_str()); - auto findResult = m_defaultValueMapping.find(table); - if (findResult == m_defaultValueMapping.end()) - { - SWSS_LOG_DEBUG("Not found default value info for %s\n", table.c_str()); - return nullptr; - } - - return findResult->second; -} - -shared_ptr DefaultValueProvider::getDefaultValue(const string &table, const string &key, const string &field) -{ - assert(!table.empty()); - assert(!key.empty()); - assert(!field.empty()); - - SWSS_LOG_DEBUG("DefaultValueProvider::GetDefaultValue %s %s %s\n", table.c_str(), key.c_str(), field.c_str()); - - auto defaultValueInfo = FindDefaultValueInfo(table); - if (defaultValueInfo == nullptr) - { - SWSS_LOG_DEBUG("Not found default value info for %s\n", table.c_str()); - return nullptr; - } - - return defaultValueInfo->GetDefaultValue(key, field); -} - -void DefaultValueProvider::InternalAppendDefaultValues(const string &table, const string &key, map& existedValues, map& defaultValues) -{ - assert(!table.empty()); - assert(!key.empty()); - - SWSS_LOG_DEBUG("DefaultValueProvider::InternalAppendDefaultValues %s %s\n", table.c_str(), key.c_str()); - - auto defaultValueInfo = FindDefaultValueInfo(table); - if (defaultValueInfo == nullptr) - { - SWSS_LOG_DEBUG("Not found default value info for %s\n", table.c_str()); - return; - } - - defaultValueInfo->AppendDefaultValues(key, existedValues, defaultValues); -} - -void DefaultValueProvider::appendDefaultValues(const string &table, const string &key, vector> &values) -{ - map existedValues; - map defaultValues; - for (auto& fieldValuePair : values) - { - existedValues.emplace(fieldValuePair.first, fieldValuePair.second); - } - - InternalAppendDefaultValues(table, key, existedValues, defaultValues); - - for (auto& fieldValuePair : defaultValues) - { - values.emplace_back(fieldValuePair.first, fieldValuePair.second); - } -} - -map DefaultValueProvider::getDefaultValues(const string &table, const string &key) -{ - map result; - InternalAppendDefaultValues(table, key, result, result); - return result; -} - -DefaultValueProvider::DefaultValueProvider() -{ -} - - -DefaultValueProvider::~DefaultValueProvider() -{ - if (m_context) - { - // set private_destructor to NULL because no any private data - ly_ctx_destroy(m_context, NULL); - } -} - -void DefaultValueProvider::Initialize(char* modulePath) -{ - assert(modulePath != nullptr && strlen(modulePath) != 0); - assert(m_context == nullptr); - - DIR *moduleDir = opendir(modulePath); - if (!moduleDir) - { - ThrowRunTimeError("Open Yang model path " + string(modulePath) + " failed"); - } - - m_context = ly_ctx_new(modulePath, LY_CTX_ALLIMPLEMENTED); - struct dirent *subDir; - while ((subDir = readdir(moduleDir)) != nullptr) - { - if (subDir->d_type != DT_REG) - { - continue; - } - - SWSS_LOG_DEBUG("file name: %s\n", subDir->d_name); - string fileName(subDir->d_name); - int pos = (int)fileName.find(".yang"); - string moduleName = fileName.substr(0, pos); - - LoadModule(moduleName, fileName, m_context); - } - closedir(moduleDir); -} - -void DefaultValueProvider::LoadModule(const string name, const string path, struct ly_ctx *context) -{ - const struct lys_module *module = ly_ctx_load_module( - context, - name.c_str(), - EMPTY_STR); // Use EMPTY_STR to revision to load the latest revision - if (module == nullptr) - { - const char* err = ly_errmsg(context); - SWSS_LOG_ERROR("Load Yang file %s failed: %s.\n", path.c_str(), err); - return; - } - - if (module->data == nullptr) - { - // Not every yang file should contains yang model - SWSS_LOG_WARN("Yang file %s does not contains model %s.\n", path.c_str(), name.c_str()); - return; - } - - struct lys_node *topLevelNode = module->data; - while (topLevelNode) - { - if (topLevelNode->nodetype != LYS_CONTAINER) - { - SWSS_LOG_DEBUG("ignore top level element %s, tyoe %d\n",topLevelNode->name, topLevelNode->nodetype); - // Config DB table schema is defined by top level container - topLevelNode = topLevelNode->next; - continue; - } - - SWSS_LOG_DEBUG("top level container: %s\n",topLevelNode->name); - auto container = topLevelNode->child; - while (container) - { - SWSS_LOG_DEBUG("container name: %s\n",container->name); - - AppendTableInfoToMapping(container); - container = container->next; - } - - topLevelNode = topLevelNode->next; - } -} - -shared_ptr DefaultValueProvider::GetKeyInfo(struct lys_node* tableChildNode) -{ - assert(tableChildNode != nullptr); - SWSS_LOG_DEBUG("DefaultValueProvider::GetKeyInfo %s\n",tableChildNode->name); - - int keyFieldCount = 0; - string keyValue = ""; - if (tableChildNode->nodetype == LYS_LIST) - { - SWSS_LOG_DEBUG("Child list: %s\n",tableChildNode->name); - - // when a top level container contains list, the key defined by the 'keys' field. - struct lys_node_list *listNode = (struct lys_node_list*)tableChildNode; - if (listNode->keys_str == nullptr) - { - SWSS_LOG_ERROR("Ignore empty key string on list: %s\n",tableChildNode->name); - return nullptr; - } - - string key(listNode->keys_str); - keyFieldCount = (int)count(key.begin(), key.end(), ' ') + 1; - } - else if (tableChildNode->nodetype == LYS_CONTAINER) - { - SWSS_LOG_DEBUG("Child container name: %s\n",tableChildNode->name); - - // when a top level container not contains any list, the key is child container name - keyValue = string(tableChildNode->name); - } - else - { - SWSS_LOG_DEBUG("Ignore child element: %s\n",tableChildNode->name); - return nullptr; - } - - return make_shared(keyValue, keyFieldCount); -} - -void DefaultValueProvider::GetDefaultValueInfoForLeaf(struct lys_node* field, shared_ptr fieldMapping) -{ - assert(field != nullptr); - assert(fieldMapping != nullptr); - assert(field->nodetype == LYS_LEAF); - - WARNINGS_NO_CAST_ALIGN - struct lys_node_leaf *leafNode = reinterpret_cast(field); - WARNINGS_RESET - if (leafNode->dflt) - { - SWSS_LOG_DEBUG("field: %s, default: %s\n",leafNode->name, leafNode->dflt); - fieldMapping->emplace(string(field->name), string(leafNode->dflt)); - } -} - -void DefaultValueProvider::GetDefaultValueInfoForChoice(struct lys_node* field, shared_ptr fieldMapping) -{ - assert(field != nullptr); - assert(fieldMapping != nullptr); - assert(field->nodetype == LYS_CHOICE); - - struct lys_node_choice *choiceNode = (struct lys_node_choice *)field; - if (choiceNode->dflt == nullptr) - { - return; - } - - // Get choice default value according to: https://www.rfc-editor.org/rfc/rfc7950.html#section-7.9 - auto dfltChoice = choiceNode->dflt; - auto fieldInChoice = dfltChoice->child; - while (fieldInChoice) - { - if (fieldInChoice->nodetype != LYS_LEAF) - { - SWSS_LOG_ERROR("choice case %s is not a leaf node\n",fieldInChoice->name); - continue; - } - - SWSS_LOG_DEBUG("default choice leaf field: %s\n",fieldInChoice->name); - WARNINGS_NO_CAST_ALIGN - struct lys_node_leaf *dfltLeafNode = reinterpret_cast(fieldInChoice); - WARNINGS_RESET - if (dfltLeafNode->dflt) - { - SWSS_LOG_DEBUG("default choice leaf field: %s, default: %s\n",dfltLeafNode->name, dfltLeafNode->dflt); - fieldMapping->emplace(string(fieldInChoice->name), string(dfltLeafNode->dflt)); - } - - fieldInChoice = fieldInChoice->next; - } -} - -void DefaultValueProvider::GetDefaultValueInfoForLeaflist(struct lys_node* field, shared_ptr fieldMapping) -{ - assert(field != nullptr); - assert(fieldMapping != nullptr); - assert(field->nodetype == LYS_LEAFLIST); - - // Get leaf-list default value according to:https://www.rfc-editor.org/rfc/rfc7950.html#section-7.7 - WARNINGS_NO_CAST_ALIGN - struct lys_node_leaflist *listNode = reinterpret_cast(field); - WARNINGS_RESET - - if (listNode->dflt) - { - const char** dfltValues = listNode->dflt; - //convert list default value to json string - string dfltValueJson = JSon::buildJson(dfltValues); - SWSS_LOG_DEBUG("list field: %s, default: %s\n",field->name, dfltValueJson.c_str()); - fieldMapping->emplace(string(field->name), dfltValueJson); - } -} - -FieldDefaultValueMappingPtr DefaultValueProvider::GetDefaultValueInfo(struct lys_node* tableChildNode) -{ - assert(tableChildNode != nullptr); - SWSS_LOG_DEBUG("DefaultValueProvider::GetDefaultValueInfo %s\n",tableChildNode->name); - - auto field = tableChildNode->child; - auto fieldMapping = make_shared(); - while (field) - { - if (field->nodetype == LYS_LEAF) - { - SWSS_LOG_DEBUG("leaf field: %s\n",field->name); - GetDefaultValueInfoForLeaf(field, fieldMapping); - } - else if (field->nodetype == LYS_CHOICE) - { - SWSS_LOG_DEBUG("choice field: %s\n",field->name); - GetDefaultValueInfoForChoice(field, fieldMapping); - } - else if (field->nodetype == LYS_LEAFLIST) - { - SWSS_LOG_DEBUG("list field: %s\n",field->name); - GetDefaultValueInfoForLeaflist(field, fieldMapping); - } -#ifdef DEBUG - else - { - SWSS_LOG_DEBUG("Field %s with type %d does not support default value\n",field->name, field->nodetype); - } -#endif - - field = field->next; - } - - return fieldMapping; -} - -int DefaultValueProvider::BuildFieldMappingList(struct lys_node* table, KeyInfoToDefaultValueInfoMapping &fieldInfoMapping) -{ - assert(table != nullptr); - - int childListCount = 0; - auto nextChild = table->child; - while (nextChild) - { - // get key from schema - auto keyInfo = GetKeyInfo(nextChild); - if (keyInfo == nullptr) - { - nextChild = nextChild->next; - continue; - } - else if (keyInfo->second != 0) - { - // when key field count not 0, it's a list node. - childListCount++; - } - - // get field name to default value mappings from schema - fieldInfoMapping.emplace(*keyInfo, GetDefaultValueInfo(nextChild)); - - nextChild = nextChild->next; - } - - return childListCount; -} - -// Load default value info from yang model and append to default value mapping -void DefaultValueProvider::AppendTableInfoToMapping(struct lys_node* table) -{ - assert(table != nullptr); - - SWSS_LOG_DEBUG("DefaultValueProvider::AppendTableInfoToMapping table name: %s\n",table->name); - KeyInfoToDefaultValueInfoMapping fieldInfoMapping; - int listCount = BuildFieldMappingList(table, fieldInfoMapping); - - // create container data by list count - shared_ptr tableInfoPtr = nullptr; - switch (listCount) - { - case 0: - { - tableInfoPtr = shared_ptr(new TableInfoDict(fieldInfoMapping)); - } - break; - - case 1: - { - tableInfoPtr = shared_ptr(new TableInfoSingleList(fieldInfoMapping)); - } - break; - - default: - { - tableInfoPtr = shared_ptr(new TableInfoMultipleList(fieldInfoMapping)); - } - break; - } - - m_defaultValueMapping.emplace(string(table->name), tableInfoPtr); -} \ No newline at end of file diff --git a/common/defaultvalueprovider.h b/common/defaultvalueprovider.h deleted file mode 100755 index c29556a4..00000000 --- a/common/defaultvalueprovider.h +++ /dev/null @@ -1,118 +0,0 @@ -#pragma once - -#include -#include -#include -#include "common/table.h" - -#define DEFAULT_YANG_MODULE_PATH "/usr/local/yang-models" -#define EMPTY_STR "" - -struct ly_ctx; - -// Key information -typedef std::pair KeyInfo; - -// Field name to default value mapping -typedef std::map FieldDefaultValueMapping; -typedef std::shared_ptr FieldDefaultValueMappingPtr; - -// Key info to default value info mapping -typedef std::map KeyInfoToDefaultValueInfoMapping; - -namespace swss { - -class TableInfoBase -{ -public: - TableInfoBase(); - - void AppendDefaultValues(const std::string &row, std::map& sourceValues, std::map& targetValues); - - std::shared_ptr GetDefaultValue(const std::string &row, const std::string &field); - -protected: - virtual bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr) = 0; -}; - -class TableInfoDict : public TableInfoBase -{ -public: - TableInfoDict(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping); - -private: - // Mapping: key value -> field -> default - std::map m_defaultValueMapping; - - bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr); -}; - -class TableInfoSingleList : public TableInfoBase -{ -public: - TableInfoSingleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping); - -private: - // Mapping: field -> default - FieldDefaultValueMappingPtr m_defaultValueMapping; - - bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr); -}; - -struct TableInfoMultipleList : public TableInfoBase -{ -public: - TableInfoMultipleList(KeyInfoToDefaultValueInfoMapping &fieldInfoMapping); - -private: - // Mapping: key field count -> field -> default - std::map m_defaultValueMapping; - - bool FindFieldMappingByKey(const std::string &row, FieldDefaultValueMapping ** foundedMappingPtr); -}; - -class DefaultValueProvider -{ -public: - static DefaultValueProvider& instance(); - - void appendDefaultValues(const std::string &table, const std::string &key, std::vector> &values); - - std::shared_ptr getDefaultValue(const std::string &table, const std::string &key, const std::string &field); - - std::map getDefaultValues(const std::string &table, const std::string &key); - -#ifndef UNITTEST -private: -#endif - DefaultValueProvider(); - ~DefaultValueProvider(); - - // libyang context - struct ly_ctx *m_context = nullptr; - - // The table name to table default value info mapping - std::map > m_defaultValueMapping; - - void Initialize(char* modulePath = DEFAULT_YANG_MODULE_PATH); - - void LoadModule(const std::string name, const std::string path, struct ly_ctx *context); - - // Load default value info from yang model and append to default value mapping - void AppendTableInfoToMapping(struct lys_node* table); - - std::shared_ptr FindDefaultValueInfo(const std::string &table); - - int BuildFieldMappingList(struct lys_node* table, KeyInfoToDefaultValueInfoMapping& fieldMappingList); - - std::shared_ptr GetKeyInfo(struct lys_node* table_child_node); - FieldDefaultValueMappingPtr GetDefaultValueInfo(struct lys_node* tableChildNode); - - void InternalAppendDefaultValues(const std::string &table, const std::string &key, std::map& existedValues, std::map& defaultValues); - - void GetDefaultValueInfoForChoice(struct lys_node* field, std::shared_ptr fieldMapping); - void GetDefaultValueInfoForLeaf(struct lys_node* field, std::shared_ptr fieldMapping); - void GetDefaultValueInfoForLeaflist(struct lys_node* field, std::shared_ptr fieldMapping); -}; - -} diff --git a/common/profileprovider.cpp b/common/profileprovider.cpp new file mode 100755 index 00000000..751c86b1 --- /dev/null +++ b/common/profileprovider.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "profileprovider.h" +#include "logger.h" +#include "table.h" +#include "schema.h" + +using namespace std; +using namespace swss; + +ProfileProvider& ProfileProvider::instance() +{ + static ProfileProvider instance; + return instance; +} + +shared_ptr ProfileProvider::getConfig(const string &table, const string &key, const string &field, DBConnector* cfgDbConnector) +{ + assert(!table.empty()); + assert(!key.empty()); + assert(!field.empty()); + + auto staticConfig = getConfigs(table, key, cfgDbConnector); + auto result = staticConfig.find(field); + if (result != staticConfig.end()) + { + return make_shared(result->second); + } + + // Config not found is different with 'empty' config + return nullptr; +} + +bool ProfileProvider::appendConfigs(const string &table, const string &key, vector > &values, DBConnector* cfgDbConnector) +{ + assert(!table.empty()); + assert(!key.empty()); + + SWSS_LOG_DEBUG("DefaultValueProvider::AppendDefaultValues %s %s\n", table.c_str(), key.c_str()); + + auto staticConfig = getConfigs(table, key, cfgDbConnector); + + map existedValues; + for (auto& fieldValuePair : values) + { + existedValues.emplace(fieldValuePair.first, fieldValuePair.second); + } + + bool appendValues = false; + for (auto& fieldValuePair : staticConfig) + { + auto findresult = existedValues.find(fieldValuePair.first); + if (findresult == existedValues.end()) + { + appendValues = true; + values.emplace_back(fieldValuePair.first, fieldValuePair.second); + } + } + + return appendValues; +} + +map ProfileProvider::getConfigs(const string &table, const string &key, DBConnector* cfgDbConnector) +{ + if (itemDeleted(table, key, cfgDbConnector)) + { + SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs item %s %s deleted.\n", table.c_str(), key.c_str()); + map map; + return map; + } + + auto& staticCfgDbConnector = getStaticCfgDBConnector(cfgDbConnector); + auto itemkey = getKeyName(table, key, &staticCfgDbConnector); + return staticCfgDbConnector.hgetall>(itemkey); +} + +map>> ProfileProvider::getConfigs(DBConnector* cfgDbConnector) +{ + auto& staticCfgDbConnector = getStaticCfgDBConnector(cfgDbConnector); + auto configs = staticCfgDbConnector.getall(); + + // If a profile item mark as 'deleted', it's shoud not exist in result. + list> deletedItems; + for(auto const& tableItem: configs) + { + auto table = tableItem.first; + for(auto const& item: tableItem.second) + { + auto key = item.first; + if (itemDeleted(table, key, cfgDbConnector)) + { + SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs item %s %s deleted.\n", table.c_str(), key.c_str()); + deletedItems.push_back(make_pair(table, key)); + } + } + } + + for(auto const& deletedItem: deletedItems) + { + auto table = deletedItem.first; + auto key = deletedItem.second; + SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs remove deleted item %s %s from result.\n", table.c_str(), key.c_str()); + configs[table].erase(key); + } + + return configs; +} + +vector ProfileProvider::getKeys(const string &table, DBConnector* cfgDbConnector) +{ + auto& staticCfgDbConnector = getStaticCfgDBConnector(cfgDbConnector); + auto pattern = getKeyName(table, "*", &staticCfgDbConnector); + auto keys = staticCfgDbConnector.keys(pattern); + + const auto separator = SonicDBConfig::getSeparator(&staticCfgDbConnector); + vector result; + for(auto const& itemKey: keys) + { + size_t pos = itemKey.find(separator); + if (pos == string::npos) + { + SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs can't find separator %s in %s.\n", separator.c_str(), itemKey.c_str()); + continue; + } + + auto row = itemKey.substr(pos + 1); + if (!itemDeleted(table, row, cfgDbConnector)) + { + result.push_back(row); + } + else + { + SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs item %s %s deleted.\n", table.c_str(), row.c_str()); + } + } + + return result; +} + +ProfileProvider::ProfileProvider() +{ +} + +ProfileProvider::~ProfileProvider() +{ +} + +bool ProfileProvider::tryRevertItem(const string &table, const string &key, DBConnector* cfgDbConnector) +{ + if (itemDeleted(table, key, cfgDbConnector)) + { + revertItem(table, key, cfgDbConnector); + return true; + } + + return false; +} + +bool ProfileProvider::tryDeleteItem(const string &table, const string &key, DBConnector* cfgDbConnector) +{ + if (!itemDeleted(table, key, cfgDbConnector)) + { + deleteItem(table, key, cfgDbConnector); + return true; + } + + return false; +} + +bool ProfileProvider::itemDeleted(const string &table, const string &key, DBConnector* cfgDbConnector) +{ + auto deletedkey = getDeletedKeyName(table, key, cfgDbConnector); + return cfgDbConnector->exists(deletedkey) == true; +} + +void ProfileProvider::deleteItem(const string &table, const string &key, DBConnector* cfgDbConnector) +{ + auto deletedkey = getDeletedKeyName(table, key, cfgDbConnector); + // Only need deletedkey to mark the item is deleted. + cfgDbConnector->hset(deletedkey, "", ""); +} + +void ProfileProvider::revertItem(const string &table, const string &key, DBConnector* cfgDbConnector) +{ + auto deletedkey = getDeletedKeyName(table, key,cfgDbConnector); + cfgDbConnector->del(deletedkey); +} + +DBConnector& ProfileProvider::getStaticCfgDBConnector(DBConnector* cfgDbConnector) +{ + auto ns = cfgDbConnector->getNamespace(); + auto result = m_staticCfgDBMap.find(ns); + if (result != m_staticCfgDBMap.end()) + { + return result->second; + } + + // Create new DBConnector instance to PROFILE_DB + const string staticDbName = "PROFILE_DB"; + m_staticCfgDBMap.emplace(piecewise_construct, + forward_as_tuple(ns), + forward_as_tuple(staticDbName, SonicDBConfig::getDbId(staticDbName, ns), true, ns)); + + result = m_staticCfgDBMap.find(ns); + return result->second; +} \ No newline at end of file diff --git a/common/profileprovider.h b/common/profileprovider.h new file mode 100755 index 00000000..46a47f7e --- /dev/null +++ b/common/profileprovider.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include "common/table.h" +#include "common/dbconnector.h" +#include "common/converter.h" + +namespace swss { + +const std::string DELETED_KEY_SEPARATOR = "_"; + +class ProfileProvider +{ +public: + static ProfileProvider& instance(); + + bool appendConfigs(const std::string &table, const std::string &key, std::vector > &values, DBConnector* cfgDbConnector); + + std::shared_ptr getConfig(const std::string &table, const std::string &key, const std::string &field, DBConnector* cfgDbConnector); + + std::map getConfigs(const std::string &table, const std::string &key, DBConnector* cfgDbConnector); + + std::map>> getConfigs(DBConnector* cfgDbConnector); + + std::vector getKeys(const std::string &table, DBConnector* cfgDbConnector); + + bool tryRevertItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector); + + bool tryDeleteItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector); + + std::string getDeletedKeyName(const std::string &table, const std::string &key, DBConnector* dbConnector) + { + auto itemKey = to_upper(table) + DELETED_KEY_SEPARATOR + key; + return getKeyName(PROFILE_DELETE_TABLE, itemKey, dbConnector); + } + +private: + ProfileProvider(); + ~ProfileProvider(); + + std::string getKeyName(const std::string &table, const std::string &key, DBConnector* dbConnector) + { + const auto separator = SonicDBConfig::getSeparator(dbConnector); + // Profile DB follow Config DB: table name is case insensetive. + return to_upper(table) + separator + key; + } + + bool itemDeleted(const std::string &table, const std::string &key, DBConnector* cfgDbConnector); + + void deleteItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector); + + void revertItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector); + + DBConnector& getStaticCfgDBConnector(DBConnector* cfgDbConnector); + + std::map m_staticCfgDBMap; +}; + +} From 269b00e2ea00070fdb427d2cabdb95f6044341ea Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 05:09:50 +0000 Subject: [PATCH 04/17] Fix file mode --- common/dbconnector.cpp | 0 common/profileprovider.cpp | 0 common/profileprovider.h | 0 tests/profileprovider_ut.cpp | 0 4 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 common/dbconnector.cpp mode change 100755 => 100644 common/profileprovider.cpp mode change 100755 => 100644 common/profileprovider.h mode change 100755 => 100644 tests/profileprovider_ut.cpp diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp old mode 100755 new mode 100644 diff --git a/common/profileprovider.cpp b/common/profileprovider.cpp old mode 100755 new mode 100644 diff --git a/common/profileprovider.h b/common/profileprovider.h old mode 100755 new mode 100644 diff --git a/tests/profileprovider_ut.cpp b/tests/profileprovider_ut.cpp old mode 100755 new mode 100644 From 44c9a7527945af53bb09cfc890d1ec22ee43037b Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 05:25:10 +0000 Subject: [PATCH 05/17] Fix build issue --- common/profileprovider.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/common/profileprovider.cpp b/common/profileprovider.cpp index 751c86b1..9481a76e 100644 --- a/common/profileprovider.cpp +++ b/common/profileprovider.cpp @@ -8,8 +8,6 @@ #include #include -#include - #include "profileprovider.h" #include "logger.h" #include "table.h" From 61db251498b0d5bd97274ffae9a5dfd305e63ded Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Fri, 9 Sep 2022 07:50:40 +0000 Subject: [PATCH 06/17] Fix config issue --- .azure-pipelines/build-sairedis-template.yml | 2 +- .azure-pipelines/build-swss-template.yml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/build-sairedis-template.yml b/.azure-pipelines/build-sairedis-template.yml index e9cd5c31..f0b067d2 100644 --- a/.azure-pipelines/build-sairedis-template.yml +++ b/.azure-pipelines/build-sairedis-template.yml @@ -160,6 +160,6 @@ jobs: targetFolder: $(Build.ArtifactStagingDirectory) - publish: $(Build.ArtifactStagingDirectory)/ # publish artifact with retry count, otherwise "artifact already exist" error will block job re-run. - artifact: ${{ parameters.syslog_artifact_name }}_${System.JobAttempt} + artifact: ${{ parameters.syslog_artifact_name }}_$(System.JobAttempt) displayName: "Publish syslog artifacts" condition: always() diff --git a/.azure-pipelines/build-swss-template.yml b/.azure-pipelines/build-swss-template.yml index cb2dc47b..4e655fd9 100644 --- a/.azure-pipelines/build-swss-template.yml +++ b/.azure-pipelines/build-swss-template.yml @@ -72,6 +72,9 @@ jobs: inputs: artifact: ${{ parameters.swss_common_artifact_name }} path: $(Build.ArtifactStagingDirectory)/download + patterns: | + libswsscommon_1.0.0_*.deb + libswsscommon-dev_1.0.0*.deb displayName: "Download pre-stage built ${{ parameters.swss_common_artifact_name }}" - task: DownloadPipelineArtifact@2 inputs: From 4c3c9516f43a43638f290fbf9f60859375785c8b Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Tue, 13 Sep 2022 01:49:03 +0000 Subject: [PATCH 07/17] Remove add new DB from redis config --- .azure-pipelines/build-template.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 3e5da933..4116b097 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -134,7 +134,6 @@ jobs: sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf - sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf sudo service redis-server restart sudo mkdir /usr/local/yang-models From 4a12abf4702f798ab555a519441bbaa89ed968ac Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Tue, 13 Sep 2022 01:56:50 +0000 Subject: [PATCH 08/17] Add new database for UT --- .azure-pipelines/build-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 4116b097..3e5da933 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -134,6 +134,7 @@ jobs: sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf + sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf sudo service redis-server restart sudo mkdir /usr/local/yang-models From 92acd07b6389b5c23cf15e8f14e57868a527f15e Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Tue, 13 Sep 2022 02:11:42 +0000 Subject: [PATCH 09/17] Fix empty file issue --- common/dbconnector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index d5807781..0b627ece 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -191,7 +191,7 @@ void SonicDBConfig::validateNamespace(const string &netns) SWSS_LOG_ENTER(); - // With valid namespace input and database_global.json is not loaded, ask user to initializeGlobalConfig first + // With valid namespace input and database_global.json is not loaded, ask user to initializeGlobalConfig first. if(!netns.empty()) { // If global initialization is not done, ask user to initialize global DB Config first. From 57b9f599b3fab91a5fc29195ee1c4d61e628cd62 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 28 Sep 2022 07:33:56 +0000 Subject: [PATCH 10/17] Improve UT config file --- .azure-pipelines/build-template.yml | 1 - tests/redis_multi_db_ut_config/database_config.json | 10 +++++----- tests/redis_multi_db_ut_config/database_config0.json | 10 +++++----- tests/redis_multi_db_ut_config/database_config1.json | 10 +++++----- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 3e5da933..4116b097 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -134,7 +134,6 @@ jobs: sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf - sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf sudo service redis-server restart sudo mkdir /usr/local/yang-models diff --git a/tests/redis_multi_db_ut_config/database_config.json b/tests/redis_multi_db_ut_config/database_config.json index 2165eec1..de8db035 100644 --- a/tests/redis_multi_db_ut_config/database_config.json +++ b/tests/redis_multi_db_ut_config/database_config.json @@ -92,6 +92,11 @@ "separator": ":", "instance": "redis" }, + "PROFILE_DB" : { + "id" : 12, + "separator": ":", + "instance" : "redis" + }, "STATE_DB2" : { "id" : 13, "separator": "|", @@ -106,11 +111,6 @@ "id" : 15, "separator": ":", "instance" : "redis" - }, - "PROFILE_DB" : { - "id" : 16, - "separator": ":", - "instance" : "redis" } }, "VERSION" : "1.0" diff --git a/tests/redis_multi_db_ut_config/database_config0.json b/tests/redis_multi_db_ut_config/database_config0.json index 899912d4..6be34a50 100644 --- a/tests/redis_multi_db_ut_config/database_config0.json +++ b/tests/redis_multi_db_ut_config/database_config0.json @@ -72,6 +72,11 @@ "separator": ":", "instance": "redis" }, + "PROFILE_DB" : { + "id" : 12, + "separator": ":", + "instance" : "redis" + }, "STATE_DB2" : { "id" : 13, "separator": "|", @@ -86,11 +91,6 @@ "id" : 15, "separator": ":", "instance" : "redis" - }, - "PROFILE_DB" : { - "id" : 16, - "separator": ":", - "instance" : "redis" } }, "VERSION" : "1.0" diff --git a/tests/redis_multi_db_ut_config/database_config1.json b/tests/redis_multi_db_ut_config/database_config1.json index dbaef6ef..42af9904 100644 --- a/tests/redis_multi_db_ut_config/database_config1.json +++ b/tests/redis_multi_db_ut_config/database_config1.json @@ -72,6 +72,11 @@ "separator": ":", "instance": "redis" }, + "PROFILE_DB" : { + "id" : 12, + "separator": ":", + "instance" : "redis" + }, "STATE_DB2" : { "id" : 13, "separator": "|", @@ -86,11 +91,6 @@ "id" : 15, "separator": ":", "instance" : "redis" - }, - "PROFILE_DB" : { - "id" : 16, - "separator": ":", - "instance" : "redis" } }, "VERSION" : "1.0" From 02af2246a2f44c2b9225053d6b3d751545557159 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 28 Sep 2022 08:04:05 +0000 Subject: [PATCH 11/17] Fix build issue --- tests/defaultvalueprovider_ut.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/defaultvalueprovider_ut.cpp b/tests/defaultvalueprovider_ut.cpp index 14c21853..20d6b957 100644 --- a/tests/defaultvalueprovider_ut.cpp +++ b/tests/defaultvalueprovider_ut.cpp @@ -4,8 +4,8 @@ using namespace std; using namespace swss; -string profile_table = "INTERFACE"; -string profile_key = "TEST_INTERFACE"; +static string profile_table = "INTERFACE"; +static string profile_key = "TEST_INTERFACE"; class MockDefaultValueProvider : public DefaultValueProvider { From c558e6bcd1b13d2abf2087141945d78cfdf98f9c Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 28 Sep 2022 08:05:51 +0000 Subject: [PATCH 12/17] Improve UT --- tests/profileprovider_ut.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/profileprovider_ut.cpp b/tests/profileprovider_ut.cpp index f3882454..098ff8cb 100644 --- a/tests/profileprovider_ut.cpp +++ b/tests/profileprovider_ut.cpp @@ -6,10 +6,10 @@ using namespace std; using namespace swss; -string profile_table = "INTERFACE"; -string profile_key = "TEST_INTERFACE"; -string profile_field = "profile"; -string profile_value = "value"; +static string profile_table = "INTERFACE"; +static string profile_key = "TEST_INTERFACE"; +static string profile_field = "profile"; +static string profile_value = "value"; void clearDB(const string &dbName) { From 9b0653dd2aa1ad872a9cb072db39d9ab924de174 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 28 Sep 2022 08:15:32 +0000 Subject: [PATCH 13/17] Refactor getall method --- common/configdb.cpp | 19 +------------------ common/dbconnector.cpp | 23 +++++++++++++++++++++++ common/dbconnector.h | 1 + 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/common/configdb.cpp b/common/configdb.cpp index 2d733f9f..ececb2d3 100644 --- a/common/configdb.cpp +++ b/common/configdb.cpp @@ -244,24 +244,7 @@ void ConfigDBConnector_Native::mod_config(const map>> ConfigDBConnector_Native::get_config() { auto& client = get_redis_client(m_db_name); - auto const& keys = client.keys("*"); - map>> data; - for (string key: keys) - { - size_t pos = key.find(m_table_name_separator); - if (pos == string::npos) { - continue; - } - string table_name = key.substr(0, pos); - string row = key.substr(pos + 1); - auto const& entry = client.hgetall>(key); - - if (!entry.empty()) - { - data[table_name][row] = entry; - } - } - return data; + return client.getall(); } std::string ConfigDBConnector_Native::getKeySeparator() const diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index ac17928a..d5807781 100755 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -905,3 +905,26 @@ void DBConnector::del(const std::vector& keys) pipe.flush(); } + +map>> DBConnector::getall() +{ + const string separator = SonicDBConfig::getSeparator(this); + auto const& keys = this->keys("*"); + map>> data; + for (string key: keys) + { + size_t pos = key.find(separator); + if (pos == string::npos) { + continue; + } + string table_name = key.substr(0, pos); + string row = key.substr(pos + 1); + auto const& entry = this->hgetall>(key); + + if (!entry.empty()) + { + data[table_name][row] = entry; + } + } + return data; +} \ No newline at end of file diff --git a/common/dbconnector.h b/common/dbconnector.h index eb37a6e8..8a70a2cf 100644 --- a/common/dbconnector.h +++ b/common/dbconnector.h @@ -248,6 +248,7 @@ class DBConnector : public RedisContext bool flushdb(); + std::map>> getall(); private: void setNamespace(const std::string &netns); From 35b677484d0d2fc026dbf56e012ab8c22c5f3206 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Wed, 28 Sep 2022 08:45:41 +0000 Subject: [PATCH 14/17] Remove unecessary change --- common/dbconnector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp index 0b627ece..d5807781 100644 --- a/common/dbconnector.cpp +++ b/common/dbconnector.cpp @@ -191,7 +191,7 @@ void SonicDBConfig::validateNamespace(const string &netns) SWSS_LOG_ENTER(); - // With valid namespace input and database_global.json is not loaded, ask user to initializeGlobalConfig first. + // With valid namespace input and database_global.json is not loaded, ask user to initializeGlobalConfig first if(!netns.empty()) { // If global initialization is not done, ask user to initialize global DB Config first. From a2094ce612240770eb50d913a22634df2b26f32f Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Thu, 29 Sep 2022 02:17:07 +0000 Subject: [PATCH 15/17] revert db config change. --- common/database_config.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common/database_config.json b/common/database_config.json index 642902f2..67b5cda8 100644 --- a/common/database_config.json +++ b/common/database_config.json @@ -91,11 +91,6 @@ "id" : 14, "separator": ":", "instance" : "redis" - }, - "PROFILE_DB" : { - "id" : 15, - "separator": ":", - "instance" : "redis" } }, "VERSION" : "1.0" From eef426af7fe986425af3d9f013e8e20aa1dda843 Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Thu, 29 Sep 2022 07:56:48 +0000 Subject: [PATCH 16/17] Fix DB seperator issue --- tests/redis_multi_db_ut_config/database_config.json | 2 +- tests/redis_multi_db_ut_config/database_config0.json | 2 +- tests/redis_multi_db_ut_config/database_config1.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/redis_multi_db_ut_config/database_config.json b/tests/redis_multi_db_ut_config/database_config.json index de8db035..7ce7820a 100644 --- a/tests/redis_multi_db_ut_config/database_config.json +++ b/tests/redis_multi_db_ut_config/database_config.json @@ -94,7 +94,7 @@ }, "PROFILE_DB" : { "id" : 12, - "separator": ":", + "separator": "|", "instance" : "redis" }, "STATE_DB2" : { diff --git a/tests/redis_multi_db_ut_config/database_config0.json b/tests/redis_multi_db_ut_config/database_config0.json index 6be34a50..45606fc3 100644 --- a/tests/redis_multi_db_ut_config/database_config0.json +++ b/tests/redis_multi_db_ut_config/database_config0.json @@ -74,7 +74,7 @@ }, "PROFILE_DB" : { "id" : 12, - "separator": ":", + "separator": "|", "instance" : "redis" }, "STATE_DB2" : { diff --git a/tests/redis_multi_db_ut_config/database_config1.json b/tests/redis_multi_db_ut_config/database_config1.json index 42af9904..03637eb8 100644 --- a/tests/redis_multi_db_ut_config/database_config1.json +++ b/tests/redis_multi_db_ut_config/database_config1.json @@ -74,7 +74,7 @@ }, "PROFILE_DB" : { "id" : 12, - "separator": ":", + "separator": "|", "instance" : "redis" }, "STATE_DB2" : { From c2a377e4c3af47d0a55dcbf81af6c2ed0ae778ae Mon Sep 17 00:00:00 2001 From: liuh-80 Date: Thu, 29 Sep 2022 10:39:25 +0000 Subject: [PATCH 17/17] Fix mode issue --- common/dbconnector.cpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 common/dbconnector.cpp diff --git a/common/dbconnector.cpp b/common/dbconnector.cpp old mode 100644 new mode 100755