From b83b8ca39665667d615330cefb3e7ede0f37a3a9 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 16:03:24 -0700 Subject: [PATCH 01/18] Don't track event "tags" i.e. event types in rulesets Modify rulesets to not keep track of the event types for a given set filter. Instead, using the changes in https://github.com/falcosecurity/libs/pull/74 event types are returned directly by the filter. Within each ruleset, there's a vector that maps from event number to set of filters that are related to that event number. There's also a general set of filters for all event types. run() both indexes into the per-event vector as well as iterate over the all event types set. Also, used shared_ptr instead of direct pointers, which matches the updated interface used by lua_parser. This simplifies the bookkeeping a bit (no more delete when removing rulesets). Given these changes, there's no need for a separate falco_sinsp_ruleset class any longer, so remove it. Signed-off-by: Mark Stemm --- tests/engine/test_rulesets.cpp | 61 +++--- userspace/engine/ruleset.cpp | 326 ++++++++++----------------------- userspace/engine/ruleset.h | 90 +++------ 3 files changed, 151 insertions(+), 326 deletions(-) diff --git a/tests/engine/test_rulesets.cpp b/tests/engine/test_rulesets.cpp index 1a7f1692299..45e93431b86 100644 --- a/tests/engine/test_rulesets.cpp +++ b/tests/engine/test_rulesets.cpp @@ -25,15 +25,14 @@ static uint16_t default_ruleset = 0; static uint16_t non_default_ruleset = 3; static uint16_t other_non_default_ruleset = 2; static std::set tags = {"some_tag", "some_other_tag"}; -static std::set event_tags = {1}; TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("one_rule", exact_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -45,10 +44,10 @@ TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("one_rule", exact_match, enabled, non_default_ruleset); REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1); @@ -64,10 +63,10 @@ TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[ruleset TEST_CASE("Should not enable for exact match different rule name", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("some_other_rule", exact_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); @@ -76,10 +75,10 @@ TEST_CASE("Should not enable for exact match different rule name", "[rulesets]") TEST_CASE("Should enable/disable for exact match w/ substring and default ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("one_rule", substring_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -91,10 +90,10 @@ TEST_CASE("Should enable/disable for exact match w/ substring and default rulese TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("one_", exact_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 0); @@ -103,10 +102,10 @@ TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]") TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("one_", substring_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -118,10 +117,10 @@ TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[ruleset TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("_rule", substring_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -133,10 +132,10 @@ TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[ruleset TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("ne_ru", substring_match, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -148,10 +147,10 @@ TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rule TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable("ne_ru", substring_match, enabled, non_default_ruleset); REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1); @@ -167,11 +166,11 @@ TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rul TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; std::set want_tags = {"some_tag"}; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable_tags(want_tags, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -183,11 +182,11 @@ TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]") TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; std::set want_tags = {"some_tag"}; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable_tags(want_tags, enabled, non_default_ruleset); REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 1); @@ -203,11 +202,11 @@ TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]") TEST_CASE("Should not enable for different tags", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; std::set want_tags = {"some_different_tag"}; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable_tags(want_tags, enabled); REQUIRE(r.num_rules_for_ruleset(non_default_ruleset) == 0); @@ -216,11 +215,11 @@ TEST_CASE("Should not enable for different tags", "[rulesets]") TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]") { falco_ruleset r; - gen_event_filter *filter = new gen_event_filter(); + std::shared_ptr filter(new gen_event_filter()); string rule_name = "one_rule"; std::set want_tags = {"some_tag", "some_different_tag"}; - r.add(rule_name, tags, event_tags, filter); + r.add(rule_name, tags, filter); r.enable_tags(want_tags, enabled); REQUIRE(r.num_rules_for_ruleset(default_ruleset) == 1); @@ -232,15 +231,15 @@ TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]") TEST_CASE("Should enable/disable for incremental adding tags", "[rulesets]") { falco_ruleset r; - gen_event_filter *rule1_filter = new gen_event_filter(); + std::shared_ptr rule1_filter(new gen_event_filter()); string rule1_name = "one_rule"; std::set rule1_tags = {"rule1_tag"}; - r.add(rule1_name, rule1_tags, event_tags, rule1_filter); + r.add(rule1_name, rule1_tags, rule1_filter); - gen_event_filter *rule2_filter = new gen_event_filter(); + std::shared_ptr rule2_filter(new gen_event_filter()); string rule2_name = "two_rule"; std::set rule2_tags = {"rule2_tag"}; - r.add(rule2_name, rule2_tags, event_tags, rule2_filter); + r.add(rule2_name, rule2_tags, rule2_filter); std::set want_tags; diff --git a/userspace/engine/ruleset.cpp b/userspace/engine/ruleset.cpp index 3156320bfc6..49855358168 100644 --- a/userspace/engine/ruleset.cpp +++ b/userspace/engine/ruleset.cpp @@ -17,6 +17,8 @@ limitations under the License. #include "ruleset.h" #include "banned.h" // This raises a compilation error when certain functions are used +#include + using namespace std; falco_ruleset::falco_ruleset() @@ -25,127 +27,112 @@ falco_ruleset::falco_ruleset() falco_ruleset::~falco_ruleset() { - for(const auto &val : m_filters) - { - delete val.second->filter; - delete val.second; - } - - for(auto &ruleset : m_rulesets) - { - delete ruleset; - } - m_filters.clear(); } -falco_ruleset::ruleset_filters::ruleset_filters(): - m_num_filters(0) +falco_ruleset::ruleset_filters::ruleset_filters() { } falco_ruleset::ruleset_filters::~ruleset_filters() { - for(uint32_t i = 0; i < m_filter_by_event_tag.size(); i++) +} + +void falco_ruleset::ruleset_filters::add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr wrap) +{ + // This is O(n) but it's also uncommon + // (when loading rules only). + auto pos = std::find(wrappers.begin(), + wrappers.end(), + wrap); + + if(pos == wrappers.end()) { - if(m_filter_by_event_tag[i]) - { - delete m_filter_by_event_tag[i]; - m_filter_by_event_tag[i] = NULL; - } + wrappers.push_back(wrap); } } -void falco_ruleset::ruleset_filters::add_filter(filter_wrapper *wrap) +void falco_ruleset::ruleset_filters::remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr wrap) { + // This is O(n) but it's also uncommon + // (when loading rules only). + auto pos = std::find(wrappers.begin(), + wrappers.end(), + wrap); + if(pos != wrappers.end()) + { + wrappers.erase(pos); + } +} - bool added = false; +void falco_ruleset::ruleset_filters::add_filter(std::shared_ptr wrap) +{ + std::set fevttypes = wrap->filter->evttypes(); - for(uint32_t etag = 0; etag < wrap->event_tags.size(); etag++) + if(fevttypes.empty()) + { + // Should run for all event types + add_wrapper_to_list(m_filter_all_event_types, wrap); + } + else { - if(wrap->event_tags[etag]) + for(auto &etype : fevttypes) { - added = true; - if(m_filter_by_event_tag.size() <= etag) - { - m_filter_by_event_tag.resize(etag + 1); - } - - if(!m_filter_by_event_tag[etag]) + if(m_filter_by_event_type.size() <= etype) { - m_filter_by_event_tag[etag] = new list(); + m_filter_by_event_type.resize(etype + 1); } - m_filter_by_event_tag[etag]->push_back(wrap); + add_wrapper_to_list(m_filter_by_event_type[etype], wrap); } } - if(added) - { - m_num_filters++; - } + m_filters.insert(wrap); } -void falco_ruleset::ruleset_filters::remove_filter(filter_wrapper *wrap) +void falco_ruleset::ruleset_filters::remove_filter(std::shared_ptr wrap) { - bool removed = false; + std::set fevttypes = wrap->filter->evttypes(); - for(uint32_t etag = 0; etag < wrap->event_tags.size(); etag++) + if(fevttypes.empty()) { - if(wrap->event_tags[etag]) + remove_wrapper_from_list(m_filter_all_event_types, wrap); + } + else + { + for(auto &etype : fevttypes) { - if(etag < m_filter_by_event_tag.size()) + if( etype < m_filter_by_event_type.size() ) { - list *l = m_filter_by_event_tag[etag]; - if(l) - { - auto it = remove(l->begin(), - l->end(), - wrap); - - if(it != l->end()) - { - removed = true; - - l->erase(it, - l->end()); - - if(l->size() == 0) - { - delete l; - m_filter_by_event_tag[etag] = NULL; - } - } - } + remove_wrapper_from_list(m_filter_by_event_type[etype], wrap); } } } - if(removed) - { - m_num_filters--; - } + m_filters.erase(wrap); } uint64_t falco_ruleset::ruleset_filters::num_filters() { - return m_num_filters; + return m_filters.size(); } -bool falco_ruleset::ruleset_filters::run(gen_event *evt, uint32_t etag) +bool falco_ruleset::ruleset_filters::run(gen_event *evt) { - if(etag >= m_filter_by_event_tag.size()) + if(evt->get_type() >= m_filter_by_event_type.size()) { return false; } - list *filters = m_filter_by_event_tag[etag]; - - if(!filters) + for(auto &wrap : m_filter_by_event_type[evt->get_type()]) { - return false; + if(wrap->filter->run(evt)) + { + return true; + } } - for(auto &wrap : *filters) + // Finally, try filters that are not specific to an event type. + for(auto &wrap : m_filter_all_event_types) { if(wrap->filter->run(evt)) { @@ -156,83 +143,61 @@ bool falco_ruleset::ruleset_filters::run(gen_event *evt, uint32_t etag) return false; } -void falco_ruleset::ruleset_filters::event_tags_for_ruleset(vector &event_tags) +void falco_ruleset::ruleset_filters::evttypes_for_ruleset(std::set &evttypes) { - event_tags.assign(m_filter_by_event_tag.size(), false); + evttypes.clear(); - for(uint32_t etag = 0; etag < m_filter_by_event_tag.size(); etag++) + for(auto &wrap : m_filters) { - list *filters = m_filter_by_event_tag[etag]; - if(filters) - { - event_tags[etag] = true; - } + auto fevttypes = wrap->filter->evttypes(); + evttypes.insert(fevttypes.begin(), fevttypes.end()); } } void falco_ruleset::add(string &name, set &tags, - set &event_tags, - gen_event_filter *filter) + std::shared_ptr filter) { - filter_wrapper *wrap = new filter_wrapper(); + std::shared_ptr wrap(new filter_wrapper()); + wrap->name = name; + wrap->tags = tags; wrap->filter = filter; - for(auto &etag : event_tags) - { - wrap->event_tags.resize(etag + 1); - wrap->event_tags[etag] = true; - } - - m_filters.insert(pair(name, wrap)); - - for(const auto &tag : tags) - { - auto it = m_filter_by_event_tag.lower_bound(tag); - - if(it == m_filter_by_event_tag.end() || - it->first != tag) - { - it = m_filter_by_event_tag.emplace_hint(it, - make_pair(tag, list())); - } - - it->second.push_back(wrap); - } + m_filters.insert(wrap); } void falco_ruleset::enable(const string &substring, bool match_exact, bool enabled, uint16_t ruleset) { while(m_rulesets.size() < (size_t)ruleset + 1) { - m_rulesets.push_back(new ruleset_filters()); + m_rulesets.emplace_back(new ruleset_filters()); } - for(const auto &val : m_filters) + for(const auto &wrap : m_filters) { bool matches; if(match_exact) { - size_t pos = val.first.find(substring); + size_t pos = wrap->name.find(substring); matches = (substring == "" || (pos == 0 && - substring.size() == val.first.size())); + substring.size() == wrap->name.size())); } else { - matches = (substring == "" || (val.first.find(substring) != string::npos)); + matches = (substring == "" || (wrap->name.find(substring) != string::npos)); } if(matches) { if(enabled) { - m_rulesets[ruleset]->add_filter(val.second); + m_rulesets[ruleset]->add_filter(wrap); } else { - m_rulesets[ruleset]->remove_filter(val.second); + m_rulesets[ruleset]->remove_filter(wrap); } } } @@ -242,12 +207,18 @@ void falco_ruleset::enable_tags(const set &tags, bool enabled, uint16_t { while(m_rulesets.size() < (size_t)ruleset + 1) { - m_rulesets.push_back(new ruleset_filters()); + m_rulesets.emplace_back(new ruleset_filters()); } - for(const auto &tag : tags) + for(const auto &wrap : m_filters) { - for(const auto &wrap : m_filter_by_event_tag[tag]) + std::set intersect; + + set_intersection(tags.begin(), tags.end(), + wrap->tags.begin(), wrap->tags.end(), + inserter(intersect, intersect.begin())); + + if(!intersect.empty()) { if(enabled) { @@ -265,141 +236,28 @@ uint64_t falco_ruleset::num_rules_for_ruleset(uint16_t ruleset) { while(m_rulesets.size() < (size_t)ruleset + 1) { - m_rulesets.push_back(new ruleset_filters()); + m_rulesets.emplace_back(new ruleset_filters()); } return m_rulesets[ruleset]->num_filters(); } -bool falco_ruleset::run(gen_event *evt, uint32_t etag, uint16_t ruleset) +bool falco_ruleset::run(gen_event *evt, uint16_t ruleset) { if(m_rulesets.size() < (size_t)ruleset + 1) { return false; } - return m_rulesets[ruleset]->run(evt, etag); + return m_rulesets[ruleset]->run(evt); } -void falco_ruleset::event_tags_for_ruleset(vector &evttypes, uint16_t ruleset) +void falco_ruleset::evttypes_for_ruleset(set &evttypes, uint16_t ruleset) { if(m_rulesets.size() < (size_t)ruleset + 1) { return; } - return m_rulesets[ruleset]->event_tags_for_ruleset(evttypes); -} - -falco_sinsp_ruleset::falco_sinsp_ruleset() -{ -} - -falco_sinsp_ruleset::~falco_sinsp_ruleset() -{ -} - -void falco_sinsp_ruleset::add(string &name, - set &evttypes, - set &syscalls, - set &tags, - sinsp_filter *filter) -{ - set event_tags; - - if(evttypes.size() + syscalls.size() == 0) - { - // If no evttypes or syscalls are specified, the filter is - // enabled for all evttypes/syscalls. - for(uint32_t i = 0; i < PPM_EVENT_MAX; i++) - { - evttypes.insert(i); - } - - for(uint32_t i = 0; i < PPM_SC_MAX; i++) - { - syscalls.insert(i); - } - } - - for(auto evttype : evttypes) - { - event_tags.insert(evttype_to_event_tag(evttype)); - } - - for(auto syscallid : syscalls) - { - event_tags.insert(syscall_to_event_tag(syscallid)); - } - - falco_ruleset::add(name, tags, event_tags, (gen_event_filter *)filter); -} - -bool falco_sinsp_ruleset::run(sinsp_evt *evt, uint16_t ruleset) -{ - uint32_t etag; - - uint16_t etype = evt->get_type(); - - if(etype == PPME_GENERIC_E || etype == PPME_GENERIC_X) - { - sinsp_evt_param *parinfo = evt->get_param(0); - uint16_t syscallid = *(uint16_t *)parinfo->m_val; - - etag = syscall_to_event_tag(syscallid); - } - else - { - etag = evttype_to_event_tag(etype); - } - - return falco_ruleset::run((gen_event *)evt, etag, ruleset); -} - -void falco_sinsp_ruleset::evttypes_for_ruleset(vector &evttypes, uint16_t ruleset) -{ - vector event_tags; - - event_tags_for_ruleset(event_tags, ruleset); - - evttypes.assign(PPM_EVENT_MAX + 1, false); - - for(uint32_t etype = 0; etype < PPM_EVENT_MAX; etype++) - { - uint32_t etag = evttype_to_event_tag(etype); - - if(etag < event_tags.size() && event_tags[etag]) - { - evttypes[etype] = true; - } - } -} - -void falco_sinsp_ruleset::syscalls_for_ruleset(vector &syscalls, uint16_t ruleset) -{ - vector event_tags; - - event_tags_for_ruleset(event_tags, ruleset); - - syscalls.assign(PPM_EVENT_MAX + 1, false); - - for(uint32_t syscallid = 0; syscallid < PPM_SC_MAX; syscallid++) - { - uint32_t etag = evttype_to_event_tag(syscallid); - - if(etag < event_tags.size() && event_tags[etag]) - { - syscalls[syscallid] = true; - } - } -} - -uint32_t falco_sinsp_ruleset::evttype_to_event_tag(uint32_t evttype) -{ - return evttype; -} - -uint32_t falco_sinsp_ruleset::syscall_to_event_tag(uint32_t syscallid) -{ - return PPM_EVENT_MAX + 1 + syscallid; + return m_rulesets[ruleset]->evttypes_for_ruleset(evttypes); } diff --git a/userspace/engine/ruleset.h b/userspace/engine/ruleset.h index c41e74bef79..732d94db784 100644 --- a/userspace/engine/ruleset.h +++ b/userspace/engine/ruleset.h @@ -36,8 +36,7 @@ class falco_ruleset void add(std::string &name, std::set &tags, - std::set &event_tags, - gen_event_filter* filter); + std::shared_ptr filter); // rulesets are arbitrary numbers and should be managed by the caller. // Note that rulesets are used to index into a std::vector so @@ -65,23 +64,22 @@ class falco_ruleset uint64_t num_rules_for_ruleset(uint16_t ruleset = 0); // Match all filters against the provided event. - bool run(gen_event *evt, uint32_t etag, uint16_t ruleset = 0); + bool run(gen_event *evt, uint16_t ruleset = 0); - // Populate the provided vector, indexed by event tag, of the - // event tags associated with the given ruleset id. For - // example, event_tags[10] = true would mean that this ruleset - // relates to event tag 10. - void event_tags_for_ruleset(std::vector &event_tags, uint16_t ruleset); + // Populate the provided set of event types used by this ruleset. + void evttypes_for_ruleset(std::set &evttypes, uint16_t ruleset); private: - struct filter_wrapper { - gen_event_filter *filter; - - // Indexes from event tag to enabled/disabled. - std::vector event_tags; + class filter_wrapper { + public: + std::string name; + std::set tags; + std::shared_ptr filter; }; + typedef std::list> filter_wrapper_list; + // A group of filters all having the same ruleset class ruleset_filters { public: @@ -89,63 +87,33 @@ class falco_ruleset virtual ~ruleset_filters(); - void add_filter(filter_wrapper *wrap); - void remove_filter(filter_wrapper *wrap); + void add_filter(std::shared_ptr wrap); + void remove_filter(std::shared_ptr wrap); uint64_t num_filters(); - bool run(gen_event *evt, uint32_t etag); + bool run(gen_event *evt); - void event_tags_for_ruleset(std::vector &event_tags); + void evttypes_for_ruleset(std::set &evttypes); private: - uint64_t m_num_filters; + void add_wrapper_to_list(filter_wrapper_list &wrappers, std::shared_ptr wrap); + void remove_wrapper_from_list(filter_wrapper_list &wrappers, std::shared_ptr wrap); - // Maps from event tag to a list of filters. There can - // be multiple filters for a given event tag. - std::vector *> m_filter_by_event_tag; - - }; + // Vector indexes from event type to a set of filters. There can + // be multiple filters for a given event type. + // NOTE: This is used only when the event sub-type is 0. + std::vector m_filter_by_event_type; - std::vector m_rulesets; + filter_wrapper_list m_filter_all_event_types; - // Maps from tag to list of filters having that tag. - std::map> m_filter_by_event_tag; - - // This holds all the filters passed to add(), so they can - // be cleaned up. - std::map m_filters; -}; - -// falco_sinsp_ruleset is a specialization of falco_ruleset that -// maps sinsp evttypes/syscalls to event tags. -class falco_sinsp_ruleset : public falco_ruleset -{ -public: - falco_sinsp_ruleset(); - virtual ~falco_sinsp_ruleset(); - - void add(std::string &name, - std::set &evttypes, - std::set &syscalls, - std::set &tags, - sinsp_filter* filter); - - bool run(sinsp_evt *evt, uint16_t ruleset = 0); - - // Populate the provided vector, indexed by event type, of the - // event types associated with the given ruleset id. For - // example, evttypes[10] = true would mean that this ruleset - // relates to event type 10. - void evttypes_for_ruleset(std::vector &evttypes, uint16_t ruleset); + // All filters added. Used to make num_filters() fast. + std::set> m_filters; + }; - // Populate the provided vector, indexed by syscall code, of the - // syscall codes associated with the given ruleset id. For - // example, syscalls[10] = true would mean that this ruleset - // relates to syscall code 10. - void syscalls_for_ruleset(std::vector &syscalls, uint16_t ruleset); + // Vector indexes from ruleset id to set of rules. + std::vector> m_rulesets; -private: - uint32_t evttype_to_event_tag(uint32_t evttype); - uint32_t syscall_to_event_tag(uint32_t syscallid); + // All filters added. The set of enabled filters is held in m_rulesets + std::set> m_filters; }; From e2f5bba4bec20238c05b0660ec4c6d81e259ebed Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 16:20:24 -0700 Subject: [PATCH 02/18] Move json -> k8s audit event conversion out of falco engine Move the code that splits a json object into a list of k8s audit/json events out of falco engine and into json_evt. This, along with other changes, allows the falco engine to be more general purpose and not directly tied to the notion of syscall vs k8s audit events. Signed-off-by: Mark Stemm --- userspace/engine/falco_engine.cpp | 78 ------------------------------- userspace/engine/falco_engine.h | 10 ---- userspace/engine/json_evt.cpp | 75 +++++++++++++++++++++++++++++ userspace/engine/json_evt.h | 13 ++++++ userspace/falco/webserver.cpp | 2 +- 5 files changed, 89 insertions(+), 89 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 789b74fb04b..c539b5af1f9 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -386,84 +386,6 @@ void falco_engine::populate_rule_result(unique_ptr &res, gen } } -bool falco_engine::parse_k8s_audit_json(nlohmann::json &j, std::list &evts, bool top) -{ - // Note that nlohmann::basic_json::value can throw nlohmann::basic_json::type_error (302, 306) - try - { - // If the object is an array, call parse_k8s_audit_json again for each item. - if(j.is_array()) - { - if(top) - { - for(auto &item : j) - { - // Note we only handle a single top level array, to - // avoid excessive recursion. - if(! parse_k8s_audit_json(item, evts, false)) - { - return false; - } - } - - return true; - } - else - { - return false; - } - } - - // If the kind is EventList, split it into individual events - if(j.value("kind", "") == "EventList") - { - for(auto &je : j["items"]) - { - evts.emplace_back(); - je["kind"] = "Event"; - - uint64_t ns = 0; - if(!sinsp_utils::parse_iso_8601_utc_string(je.value(k8s_audit_time, ""), ns)) - { - return false; - } - - std::string tmp; - sinsp_utils::ts_to_string(ns, &tmp, false, true); - - evts.back().set_jevt(je, ns); - } - - return true; - } - else if(j.value("kind", "") == "Event") - { - evts.emplace_back(); - uint64_t ns = 0; - if(!sinsp_utils::parse_iso_8601_utc_string(j.value(k8s_audit_time, ""), ns)) - { - return false; - } - - evts.back().set_jevt(j, ns); - return true; - } - else - { - return false; - } - } - catch(exception &e) - { - return false; - } -} - -unique_ptr falco_engine::process_k8s_audit_event(json_event *ev) -{ - return process_k8s_audit_event(ev, m_default_ruleset_id); -} - void falco_engine::describe_rule(string *rule) { return m_rules->describe_rule(rule); diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index d975b59fb08..effa8395248 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -164,16 +164,6 @@ class falco_engine : public falco_common std::set tags; }; - // - // Given a raw json object, return a list of k8s audit event - // objects that represent the object. This method handles - // things such as EventList splitting. - // - // Returns true if the json object was recognized as a k8s - // audit event(s), false otherwise. - // - bool parse_k8s_audit_json(nlohmann::json &j, std::list &evts, bool top=true); - // // Given an event, check it against the set of rules in the // engine and if a matching rule is found, return details on diff --git a/userspace/engine/json_evt.cpp b/userspace/engine/json_evt.cpp index 36a644517b1..5eaf7ea74aa 100644 --- a/userspace/engine/json_evt.cpp +++ b/userspace/engine/json_evt.cpp @@ -50,6 +50,81 @@ uint64_t json_event::get_ts() const return m_event_ts; } +static nlohmann::json::json_pointer k8s_audit_time = "/stageTimestamp"_json_pointer; + +bool falco_k8s_audit::parse_k8s_audit_json(nlohmann::json &j, std::list &evts, bool top) +{ + // Note that nlohmann::basic_json::value can throw nlohmann::basic_json::type_error (302, 306) + try + { + // If the object is an array, call parse_k8s_audit_json again for each item. + if(j.is_array()) + { + if(top) + { + for(auto &item : j) + { + // Note we only handle a single top level array, to + // avoid excessive recursion. + if(! parse_k8s_audit_json(item, evts, false)) + { + return false; + } + } + + return true; + } + else + { + return false; + } + } + + // If the kind is EventList, split it into individual events + if(j.value("kind", "") == "EventList") + { + for(auto &je : j["items"]) + { + evts.emplace_back(); + je["kind"] = "Event"; + + uint64_t ns = 0; + if(!sinsp_utils::parse_iso_8601_utc_string(je.value(k8s_audit_time, ""), ns)) + { + return false; + } + + std::string tmp; + sinsp_utils::ts_to_string(ns, &tmp, false, true); + + evts.back().set_jevt(je, ns); + } + + return true; + } + else if(j.value("kind", "") == "Event") + { + evts.emplace_back(); + uint64_t ns = 0; + if(!sinsp_utils::parse_iso_8601_utc_string(j.value(k8s_audit_time, ""), ns)) + { + return false; + } + + evts.back().set_jevt(j, ns); + return true; + } + else + { + return false; + } + } + catch(exception &e) + { + return false; + } +} + json_event_value::json_event_value() { } diff --git a/userspace/engine/json_evt.h b/userspace/engine/json_evt.h index 34d59aecddc..2f9e1710a8d 100644 --- a/userspace/engine/json_evt.h +++ b/userspace/engine/json_evt.h @@ -57,6 +57,19 @@ class json_event : public gen_event uint64_t m_event_ts; }; +namespace falco_k8s_audit { + + // + // Given a raw json object, return a list of k8s audit event + // objects that represent the object. This method handles + // things such as EventList splitting. + // + // Returns true if the json object was recognized as a k8s + // audit event(s), false otherwise. + // + bool parse_k8s_audit_json(nlohmann::json &j, std::list &evts, bool top=true); +}; + // A class representing an extracted value or a value on the rhs of a // filter_check. This intentionally doesn't use the same types as // ppm_events_public.h to take advantage of actual classes instead of diff --git a/userspace/falco/webserver.cpp b/userspace/falco/webserver.cpp index bd9c6e49a82..7eca9dcf204 100644 --- a/userspace/falco/webserver.cpp +++ b/userspace/falco/webserver.cpp @@ -69,7 +69,7 @@ bool k8s_audit_handler::accept_data(falco_engine *engine, bool ok; try { - ok = engine->parse_k8s_audit_json(j, jevts); + ok = falco_k8s_audit::parse_k8s_audit_json(j, jevts); } catch(json::type_error &e) { From 747072e6c054e616e4cb42e1cac906c93bda6f1a Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 16:27:05 -0700 Subject: [PATCH 03/18] falco_formats only formats events now, no lua bindings Modify falco_formats to only be responsible for resolving a rule's output string or coming up with a map of field name->field values from a given output string. It relies on the changes in https://github.com/falcosecurity/libs/pull/77 to use generic formatters for a given source. Remove lua bindings to create a formatter/free a formatter. Those were unused as of the changes in https://github.com/falcosecurity/falco/pull/1451, so finally remove them now. Signed-off-by: Mark Stemm --- userspace/engine/formats.cpp | 202 ++++++------------------------ userspace/engine/formats.h | 47 +++---- userspace/falco/falco_outputs.cpp | 28 ++--- userspace/falco/falco_outputs.h | 5 +- 4 files changed, 75 insertions(+), 207 deletions(-) diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index b3078efd67b..58381776032 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -20,169 +20,49 @@ limitations under the License. #include "falco_engine.h" #include "banned.h" // This raises a compilation error when certain functions are used -sinsp *falco_formats::s_inspector = NULL; -falco_engine *falco_formats::s_engine = NULL; -bool falco_formats::s_json_output = false; -bool falco_formats::s_json_include_output_property = true; -bool falco_formats::s_json_include_tags_property = true; -std::unique_ptr falco_formats::s_formatters = NULL; - -const static struct luaL_Reg ll_falco[] = - { - {"formatter", &falco_formats::lua_formatter}, - {"free_formatter", &falco_formats::lua_free_formatter}, - {NULL, NULL}}; - -void falco_formats::init(sinsp *inspector, - falco_engine *engine, - lua_State *ls, - bool json_output, - bool json_include_output_property, - bool json_include_tags_property) +falco_formats::falco_formats(falco_engine *engine, + bool json_include_output_property, + bool json_include_tags_property) + : m_falco_engine(engine), + m_json_include_output_property(json_include_output_property), + m_json_include_tags_property(json_include_tags_property) { - s_inspector = inspector; - s_engine = engine; - s_json_output = json_output; - s_json_include_output_property = json_include_output_property; - s_json_include_tags_property = json_include_tags_property; - - // todo(leogr): we should have used std::make_unique, but we cannot since it's not C++14 - s_formatters = std::unique_ptr(new sinsp_evt_formatter_cache(s_inspector)); - - luaL_openlib(ls, "formats", ll_falco, 0); } -int falco_formats::lua_formatter(lua_State *ls) +falco_formats::~falco_formats() { - string source = luaL_checkstring(ls, -2); - string format = luaL_checkstring(ls, -1); - - try - { - if(source == "syscall") - { - sinsp_evt_formatter *formatter; - formatter = new sinsp_evt_formatter(s_inspector, format); - lua_pushnil(ls); - lua_pushlightuserdata(ls, formatter); - } - else - { - json_event_formatter *formatter; - formatter = new json_event_formatter(s_engine->json_factory(), format); - lua_pushnil(ls); - lua_pushlightuserdata(ls, formatter); - } - } - catch(exception &e) - { - std::ostringstream os; - - os << "Invalid output format '" - << format - << "': '" - << e.what() - << "'"; - - lua_pushstring(ls, os.str().c_str()); - lua_pushnil(ls); - } - - return 2; } -int falco_formats::lua_free_formatter(lua_State *ls) -{ - if(!lua_islightuserdata(ls, -1) || - !lua_isstring(ls, -2)) - - { - luaL_error(ls, "Invalid argument passed to free_formatter"); - } - - string source = luaL_checkstring(ls, -2); - - if(source == "syscall") - { - sinsp_evt_formatter *formatter = (sinsp_evt_formatter *)lua_topointer(ls, -1); - delete(formatter); - } - else - { - json_event_formatter *formatter = (json_event_formatter *)lua_topointer(ls, -1); - delete(formatter); - } - - return 0; -} - -string falco_formats::format_event(const gen_event *evt, const std::string &rule, const std::string &source, +string falco_formats::format_event(gen_event *evt, const std::string &rule, const std::string &source, const std::string &level, const std::string &format, std::set &tags) { - string line; - string json_line; - string sformat = format; - if(strcmp(source.c_str(), "syscall") == 0) - { - // This is "output" - s_formatters->tostring((sinsp_evt *)evt, sformat, &line); + std::shared_ptr formatter; - if(s_json_output) - { - sinsp_evt::param_fmt cur_fmt = s_inspector->get_buffer_format(); - switch(cur_fmt) - { - case sinsp_evt::PF_NORMAL: - s_inspector->set_buffer_format(sinsp_evt::PF_JSON); - break; - case sinsp_evt::PF_EOLS: - s_inspector->set_buffer_format(sinsp_evt::PF_JSONEOLS); - break; - case sinsp_evt::PF_HEX: - s_inspector->set_buffer_format(sinsp_evt::PF_JSONHEX); - break; - case sinsp_evt::PF_HEXASCII: - s_inspector->set_buffer_format(sinsp_evt::PF_JSONHEXASCII); - break; - case sinsp_evt::PF_BASE64: - s_inspector->set_buffer_format(sinsp_evt::PF_JSONBASE64); - break; - default: - // do nothing - break; - } - // This is output fields - s_formatters->tostring((sinsp_evt *)evt, sformat, &json_line); + formatter = m_falco_engine->create_formatter(source, format); - // The formatted string might have a leading newline. If it does, remove it. - if(json_line[0] == '\n') - { - json_line.erase(0, 1); - } - s_inspector->set_buffer_format(cur_fmt); - } - } - else + // Format the original output string, regardless of output format + formatter->tostring_withformat(evt, line, gen_event_formatter::OF_NORMAL); + + if(formatter->get_output_format() == gen_event_formatter::OF_JSON) { - json_event_formatter formatter(s_engine->json_factory(), sformat); + string json_line; - line = formatter.tostring((json_event *)evt); + // Format the event into a json object with all fields resolved + formatter->tostring(evt, json_line); - if(s_json_output) + // The formatted string might have a leading newline. If it does, remove it. + if(json_line[0] == '\n') { - json_line = formatter.tojson((json_event *)evt); + json_line.erase(0, 1); } - } - // For JSON output, the formatter returned a json-as-text - // object containing all the fields in the original format - // message as well as the event time in ns. Use this to build - // a more detailed object containing the event time, rule, - // severity, full output, and fields. - if(s_json_output) - { + // For JSON output, the formatter returned a json-as-text + // object containing all the fields in the original format + // message as well as the event time in ns. Use this to build + // a more detailed object containing the event time, rule, + // severity, full output, and fields. Json::Value event; Json::Value rule_tags; Json::FastWriter writer; @@ -204,15 +84,15 @@ string falco_formats::format_event(const gen_event *evt, const std::string &rule event["priority"] = level; event["source"] = source; - if(s_json_include_output_property) + if(m_json_include_output_property) { // This is the filled-in output line. event["output"] = line; } - - if(s_json_include_tags_property) + + if(m_json_include_tags_property) { - if (tags.size() == 0) + if (tags.size() == 0) { // This sets an empty array rule_tags = Json::arrayValue; @@ -249,19 +129,19 @@ string falco_formats::format_event(const gen_event *evt, const std::string &rule return line.c_str(); } -map falco_formats::resolve_tokens(const gen_event *evt, const std::string &source, const std::string &format) +map falco_formats::get_field_values(gen_event *evt, const std::string &source, + const std::string &format) { - string sformat = format; - map values; - if(source == "syscall") - { - s_formatters->resolve_tokens((sinsp_evt *)evt, sformat, values); - } - // k8s_audit - else + std::shared_ptr formatter; + + formatter = m_falco_engine->create_formatter(source, format); + + map ret; + + if (! formatter->get_field_values(evt, ret)) { - json_event_formatter json_formatter(s_engine->json_factory(), sformat); - values = json_formatter.tomap((json_event *)evt); + throw falco_exception("Could not extract all field values from event"); } - return values; + + return ret; } diff --git a/userspace/engine/formats.h b/userspace/engine/formats.h index 5713da05af5..2747718c2f6 100644 --- a/userspace/engine/formats.h +++ b/userspace/engine/formats.h @@ -16,7 +16,7 @@ limitations under the License. #pragma once -#include "sinsp.h" +#include extern "C" { @@ -25,37 +25,26 @@ extern "C" #include "lauxlib.h" } -#include "json_evt.h" -#include "falco_engine.h" +#include -class sinsp_evt_formatter; +#include "falco_engine.h" class falco_formats { public: - static void init(sinsp *inspector, - falco_engine *engine, - lua_State *ls, - bool json_output, - bool json_include_output_property, - bool json_include_tags_property); - - // formatter = falco.formatter(format_string) - static int lua_formatter(lua_State *ls); - - // falco.free_formatter(formatter) - static int lua_free_formatter(lua_State *ls); - - static string format_event(const gen_event *evt, const std::string &rule, const std::string &source, - const std::string &level, const std::string &format, std::set &tags); - - static map resolve_tokens(const gen_event *evt, const std::string &source, - const std::string &format); - - static sinsp *s_inspector; - static falco_engine *s_engine; - static std::unique_ptr s_formatters; - static bool s_json_output; - static bool s_json_include_output_property; - static bool s_json_include_tags_property; + falco_formats(falco_engine *engine, + bool json_include_output_property, + bool json_include_tags_property); + virtual ~falco_formats(); + + std::string format_event(gen_event *evt, const std::string &rule, const std::string &source, + const std::string &level, const std::string &format, std::set &tags); + + map get_field_values(gen_event *evt, const std::string &source, + const std::string &format); + +protected: + falco_engine *m_falco_engine; + bool m_json_include_output_property; + bool m_json_include_tags_property; }; diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index ec8541fce9e..0eae412f3b7 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -60,12 +60,13 @@ falco_outputs::~falco_outputs() } } -void falco_outputs::init(bool json_output, - bool json_include_output_property, - bool json_include_tags_property, - uint32_t timeout, - uint32_t rate, uint32_t max_burst, bool buffered, - bool time_format_iso_8601, std::string hostname) +void falco_outputs::init(falco_engine *engine, + bool json_output, + bool json_include_output_property, + bool json_include_tags_property, + uint32_t timeout, + uint32_t rate, uint32_t max_burst, bool buffered, + bool time_format_iso_8601, std::string hostname) { // Cannot be initialized more than one time. if(m_initialized) @@ -73,14 +74,9 @@ void falco_outputs::init(bool json_output, throw falco_exception("falco_outputs already initialized"); } - m_json_output = json_output; + m_formats.reset(new falco_formats(engine, json_include_output_property, json_include_tags_property)); - // Note that falco_formats is already initialized by the engine, - // and the following json options are not used within the engine. - // So we can safely update them. - falco_formats::s_json_output = json_output; - falco_formats::s_json_include_output_property = json_include_output_property; - falco_formats::s_json_include_tags_property = json_include_tags_property; + m_json_output = json_output; m_timeout = std::chrono::milliseconds(timeout); @@ -192,8 +188,8 @@ void falco_outputs::handle_event(gen_event *evt, string &rule, string &source, sformat += " " + format; } - cmsg.msg = falco_formats::format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags); - cmsg.fields = falco_formats::resolve_tokens(evt, source, sformat); + cmsg.msg = m_formats->format_event(evt, rule, source, falco_common::priority_names[priority], sformat, tags); + cmsg.fields = m_formats->get_field_values(evt, source, sformat); cmsg.tags.insert(tags.begin(), tags.end()); cmsg.type = ctrl_msg_type::CTRL_MSG_OUTPUT; @@ -332,7 +328,7 @@ void falco_outputs::worker() noexcept o->reopen(); break; default: - falco_logger::log(LOG_DEBUG, "Outputs worker received an unknown message type\n"); + falco_logger::log(LOG_DEBUG, "Outputs worker received an unknown message type\n"); } } catch(const exception &e) diff --git a/userspace/falco/falco_outputs.h b/userspace/falco/falco_outputs.h index ddd94ba9707..b58fbd6e7d0 100644 --- a/userspace/falco/falco_outputs.h +++ b/userspace/falco/falco_outputs.h @@ -25,6 +25,7 @@ limitations under the License. #include "token_bucket.h" #include "falco_engine.h" #include "outputs.h" +#include "formats.h" #include "tbb/concurrent_queue.h" // @@ -38,7 +39,8 @@ class falco_outputs falco_outputs(); virtual ~falco_outputs(); - void init(bool json_output, + void init(falco_engine *engine, + bool json_output, bool json_include_output_property, bool json_include_tags_property, uint32_t timeout, @@ -63,6 +65,7 @@ class falco_outputs void reopen_outputs(); private: + std::unique_ptr m_formats; bool m_initialized; std::vector m_outputs; From 6f565b8b32013dbcc3aa0a9d5729995dc4c345c8 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 16:48:02 -0700 Subject: [PATCH 04/18] General-purpose list_fields(), via factories Take advantage of the changes in https://github.com/falcosecurity/libs/pull/75 to have a general-purpose way to list fields for a given event source. in the engine, list_fields() now takes a source, iterates over filter factories, and calls get_fields() for each factory, printing the results. list_source_fields now calls the engine regardless of source. Signed-off-by: Mark Stemm --- userspace/engine/falco_engine.cpp | 88 +++++++++++++++---------------- userspace/engine/falco_engine.h | 3 +- userspace/engine/json_evt.cpp | 50 ++++++++++++++++-- userspace/engine/json_evt.h | 4 +- userspace/falco/falco.cpp | 13 ++--- 5 files changed, 97 insertions(+), 61 deletions(-) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index c539b5af1f9..54a79d3ea3f 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -85,61 +85,61 @@ uint32_t falco_engine::engine_version() #define CONSOLE_LINE_LEN 79 -void falco_engine::list_fields(bool names_only) +void falco_engine::list_fields(std::string &source, bool names_only) { - for(auto &chk_field : json_factory().get_fields()) + for(auto &it : m_filter_factories) { - if(!names_only) + if(source != "" && source != it.first) { - printf("\n----------------------\n"); - printf("Field Class: %s (%s)\n\n", chk_field.m_name.c_str(), chk_field.m_desc.c_str()); - if(chk_field.m_class_info != "") - { - std::string str = falco::utils::wrap_text(chk_field.m_class_info, 0, 0, CONSOLE_LINE_LEN); - printf("%s\n", str.c_str()); - } + continue; } - for(auto &field : chk_field.m_fields) + for(auto &chk_field : it.second->get_fields()) { - printf("%s", field.m_name.c_str()); - - if(names_only) - { - printf("\n"); - continue; - } - uint32_t namelen = field.m_name.size(); - - if(namelen >= DESCRIPTION_TEXT_START) + if(!names_only) { - printf("\n"); - namelen = 0; + // Add some pretty printing around deesc, but if there's no desc keep + // as an empty string. + std::string desc = chk_field.desc; + if(!desc.empty()) + { + desc = string(" (") + desc + ")"; + } + + printf("\n----------------------\n"); + printf("Field Class: %s%s\n\n", chk_field.name.c_str(), desc.c_str()); + if(chk_field.class_info != "") + { + std::string str = falco::utils::wrap_text(chk_field.class_info, 0, 0, CONSOLE_LINE_LEN); + printf("%s\n", str.c_str()); + } } - for(uint32_t l = 0; l < DESCRIPTION_TEXT_START - namelen; l++) + for(auto &field : chk_field.fields) { - printf(" "); + printf("%s", field.name.c_str()); + + if(names_only) + { + printf("\n"); + continue; + } + uint32_t namelen = field.name.size(); + + if(namelen >= DESCRIPTION_TEXT_START) + { + printf("\n"); + namelen = 0; + } + + for(uint32_t l = 0; l < DESCRIPTION_TEXT_START - namelen; l++) + { + printf(" "); + } + + std::string str = falco::utils::wrap_text(field.desc, namelen, DESCRIPTION_TEXT_START, CONSOLE_LINE_LEN); + printf("%s\n", str.c_str()); } - - std::string desc = field.m_desc; - switch(field.m_idx_mode) - { - case json_event_filter_check::IDX_REQUIRED: - case json_event_filter_check::IDX_ALLOWED: - desc += " ("; - desc += json_event_filter_check::s_index_mode_strs[field.m_idx_mode]; - desc += ", "; - desc += json_event_filter_check::s_index_type_strs[field.m_idx_type]; - desc += ")"; - break; - case json_event_filter_check::IDX_NONE: - default: - break; - }; - - std::string str = falco::utils::wrap_text(desc, namelen, DESCRIPTION_TEXT_START, CONSOLE_LINE_LEN); - printf("%s\n", str.c_str()); } } } diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index effa8395248..21c15525da1 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -57,7 +57,8 @@ class falco_engine : public falco_common static uint32_t engine_version(); // Print to stdout (using printf) a description of each field supported by this engine. - void list_fields(bool names_only=false); + // If source is non-empty, only fields for the provided source are printed. + void list_fields(std::string &source, bool names_only); // // Load rules either directly or from a filename. diff --git a/userspace/engine/json_evt.cpp b/userspace/engine/json_evt.cpp index 5eaf7ea74aa..97917d9bcdf 100644 --- a/userspace/engine/json_evt.cpp +++ b/userspace/engine/json_evt.cpp @@ -690,7 +690,7 @@ size_t json_event_filter_check::parsed_size() } } -json_event_filter_check::check_info &json_event_filter_check::get_fields() +json_event_filter_check::check_info &json_event_filter_check::get_info() { return m_info; } @@ -1432,7 +1432,7 @@ json_event_filter_factory::json_event_filter_factory() for(auto &chk : m_defined_checks) { - m_info.push_back(chk->get_fields()); + m_info.push_back(chk->get_info()); } } @@ -1464,9 +1464,51 @@ gen_event_filter_check *json_event_filter_factory::new_filtercheck(const char *f return NULL; } -std::list &json_event_filter_factory::get_fields() +std::list json_event_filter_factory::get_fields() { - return m_info; + std::list ret; + + // It's not quite copy to the public information, as m_info + // has addl info about indexing. That info is added to the + // description. + + for(auto &chk: m_defined_checks) + { + json_event_filter_check::check_info &info = chk->get_info(); + gen_event_filter_factory::filter_fieldclass_info cinfo; + + cinfo.name = info.m_name; + cinfo.desc = info.m_desc; + cinfo.class_info = info.m_class_info; + + for(auto &field : info.m_fields) + { + gen_event_filter_factory::filter_field_info info; + info.name = field.m_name; + info.desc = field.m_desc; + + switch(field.m_idx_mode) + { + case json_event_filter_check::IDX_REQUIRED: + case json_event_filter_check::IDX_ALLOWED: + info.desc += " ("; + info.desc += json_event_filter_check::s_index_mode_strs[field.m_idx_mode]; + info.desc += ", "; + info.desc += json_event_filter_check::s_index_type_strs[field.m_idx_type]; + info.desc += ")"; + break; + case json_event_filter_check::IDX_NONE: + default: + break; + }; + + cinfo.fields.emplace_back(std::move(info)); + } + + ret.emplace_back(std::move(cinfo)); + } + + return ret; } json_event_formatter::json_event_formatter(json_event_filter_factory &json_factory, std::string &format): diff --git a/userspace/engine/json_evt.h b/userspace/engine/json_evt.h index 2f9e1710a8d..457e9ca2b1a 100644 --- a/userspace/engine/json_evt.h +++ b/userspace/engine/json_evt.h @@ -189,7 +189,7 @@ class json_event_filter_check : public gen_event_filter_check // brackets (e.g. ka.image[foo]) size_t parsed_size(); - check_info &get_fields(); + check_info &get_info(); // // Allocate a new check of the same type. Must be overridden. @@ -399,7 +399,7 @@ class json_event_filter_factory : public gen_event_filter_factory gen_event_filter_check *new_filtercheck(const char *fldname); // All defined field names - std::list &get_fields(); + std::list get_fields(); private: std::list> m_defined_checks; diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 5286598eba6..83a5bd1fc20 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -424,21 +424,14 @@ static void print_all_ignored_events(sinsp *inspector) printf("\n"); } -static void list_source_fields(falco_engine *engine, bool verbose, bool names_only, std::string &source) +static void list_source_fields(falco_engine *engine, bool names_only, std::string &source) { if(source.size() > 0 && !(source == "syscall" || source == "k8s_audit")) { throw std::invalid_argument("Value for --list must be \"syscall\" or \"k8s_audit\""); } - if(source == "" || source == "syscall") - { - list_fields(verbose, false, names_only); - } - if(source == "" || source == "k8s_audit") - { - engine->list_fields(names_only); - } + engine->list_fields(source, names_only); } // @@ -781,7 +774,7 @@ int falco_init(int argc, char **argv) if(list_flds) { - list_source_fields(engine, verbose, names_only, list_flds_source); + list_source_fields(engine, names_only, list_flds_source); return EXIT_SUCCESS; } From a4dfd767154594de13e6846029e5eac0b4a6c40b Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 17:02:53 -0700 Subject: [PATCH 05/18] Make json_event_formatter a gen_event_formatter Make json_event_formatter a generic event formatter by inheriting from gen_event_formatter and implementing its methods. Most of the actual work is still done by resolve_format (previously resolve_tokens, to avoid confusion with sinsp formatter, as it behaves slightly differently). Signed-off-by: Mark Stemm --- userspace/engine/json_evt.cpp | 101 +++++++++++++++++++++++++++------- userspace/engine/json_evt.h | 56 +++++++++++++++---- 2 files changed, 127 insertions(+), 30 deletions(-) diff --git a/userspace/engine/json_evt.cpp b/userspace/engine/json_evt.cpp index 97917d9bcdf..6ea1c327f04 100644 --- a/userspace/engine/json_evt.cpp +++ b/userspace/engine/json_evt.cpp @@ -1511,31 +1511,53 @@ std::list json_event_filter_fa return ret; } -json_event_formatter::json_event_formatter(json_event_filter_factory &json_factory, std::string &format): - m_format(format), - m_json_factory(json_factory) +json_event_formatter::json_event_formatter(std::shared_ptr json_factory) + : m_output_format(OF_NORMAL), m_json_factory(json_factory) { - parse_format(); } json_event_formatter::~json_event_formatter() { } -std::string json_event_formatter::tostring(json_event *ev) +void json_event_formatter::set_format(output_format of, const std::string &format) { - std::string ret; + m_output_format = of; + m_format = format; + parse_format(); +} - std::list> resolved; +bool json_event_formatter::tostring_withformat(gen_event *gevt, std::string &output, gen_event_formatter::output_format of) +{ + json_event *ev = static_cast(gevt); - resolve_tokens(ev, resolved); + std::string ret; - for(auto &res : resolved) + if(of == OF_JSON) { - ret += res.second; + ret = tojson(ev); + return true; } + else + { + std::list> resolved; - return ret; + resolve_format(ev, resolved); + + output = ""; + + for(auto &res : resolved) + { + output += res.second; + } + + return true; + } +} + +bool json_event_formatter::tostring(gen_event *gevt, std::string &output) +{ + return tostring_withformat(gevt, output, m_output_format); } std::string json_event_formatter::tojson(json_event *ev) @@ -1545,7 +1567,7 @@ std::string json_event_formatter::tojson(json_event *ev) std::list> resolved; - resolve_tokens(ev, resolved); + resolve_format(ev, resolved); for(auto &res : resolved) { @@ -1560,12 +1582,13 @@ std::string json_event_formatter::tojson(json_event *ev) return ret.dump(); } -std::map json_event_formatter::tomap(json_event *ev) +bool json_event_formatter::get_field_values(gen_event *gevt, std::map &fields) { - std::map ret; + json_event *ev = static_cast(gevt); + std::list> res; - resolve_tokens(ev, res); + resolve_format(ev, res); for(auto &r : res) { @@ -1576,11 +1599,11 @@ std::map json_event_formatter::tomap(json_event *ev) { r.second = ""; } - ret.insert(r); + fields.insert(r); } } - return ret; + return true; } void json_event_formatter::parse_format() @@ -1602,7 +1625,7 @@ void json_event_formatter::parse_format() { // Skip the % tformat.erase(0, 1); - json_event_filter_check *chk = (json_event_filter_check *)m_json_factory.new_filtercheck(tformat.c_str()); + json_event_filter_check *chk = (json_event_filter_check *)m_json_factory->new_filtercheck(tformat.c_str()); if(!chk) { @@ -1638,7 +1661,7 @@ void json_event_formatter::parse_format() } } -void json_event_formatter::resolve_tokens(json_event *ev, std::list> &resolved) +void json_event_formatter::resolve_format(json_event *ev, std::list> &resolved) { for(auto tok : m_tokens) { @@ -1678,3 +1701,43 @@ void json_event_formatter::resolve_tokens(json_event *ev, std::list json_factory) + : m_output_format(gen_event_formatter::OF_NORMAL), m_json_factory(json_factory) +{ +} + +json_event_formatter_factory::~json_event_formatter_factory() +{ +} + +void json_event_formatter_factory::set_output_format(gen_event_formatter::output_format of) +{ + m_formatters.clear(); + + m_output_format = of; +} + +std::shared_ptr json_event_formatter_factory::create_formatter(const std::string &format) +{ + auto it = m_formatters.find(format); + + if (it != m_formatters.end()) + { + return it->second; + } + + std::shared_ptr ret; + + ret.reset(new json_event_formatter(m_json_factory)); + + ret->set_format(m_output_format, format); + m_formatters[format] = ret; + + return ret; +} diff --git a/userspace/engine/json_evt.h b/userspace/engine/json_evt.h index 457e9ca2b1a..e4c1a321b9d 100644 --- a/userspace/engine/json_evt.h +++ b/userspace/engine/json_evt.h @@ -406,22 +406,34 @@ class json_event_filter_factory : public gen_event_filter_factory std::list m_info; }; -// Unlike the other classes, this does not inherit from a shared class -// that's used both by json events and sinsp events. It might be -// worthwhile, but it would require a lot of additional work to pull -// up functionality into the generic filtercheck class. - -class json_event_formatter +class json_event_formatter : public gen_event_formatter { public: - json_event_formatter(json_event_filter_factory &factory, std::string &format); + json_event_formatter(std::shared_ptr factory); virtual ~json_event_formatter(); - std::string tostring(json_event *ev); + void set_format(output_format of, const std::string &format) override; + bool tostring(gen_event *evt, std::string &output) override; + bool tostring_withformat(gen_event *evt, std::string &output, gen_event_formatter::output_format of) override; + bool get_field_values(gen_event *evt, std::map &fields) override; + output_format get_output_format() override; + std::string tojson(json_event *ev); - std::map tomap(json_event *ev); - void resolve_tokens(json_event *ev, std::list> &resolved); + // Split the format string into a list of tuples, broken at + // output fields, where each tuple is either a block of text + // from the original format string, or a field value/pair from + // the original format string. + // + // For example, given a format string "some output + // (%proc.name)", this will fill in resolved with 3 tuples: + // - ["", "some output ("] + // - ["proc.name", "nginx"] + // - ["", ")"] + // + // This can be used either to return a resolved output string + // or a map of field name/value pairs. + void resolve_format(json_event *ev, std::list> &resolved); private: void parse_format(); @@ -441,6 +453,8 @@ class json_event_formatter std::shared_ptr check; }; + gen_event_formatter::output_format m_output_format; + // The original format string std::string m_format; @@ -449,5 +463,25 @@ class json_event_formatter std::list m_tokens; // All the filterchecks required to resolve tokens in the format string - json_event_filter_factory &m_json_factory; + std::shared_ptr m_json_factory; +}; + +class json_event_formatter_factory : public gen_event_formatter_factory +{ +public: + json_event_formatter_factory(std::shared_ptr factory); + virtual ~json_event_formatter_factory(); + + void set_output_format(gen_event_formatter::output_format of) override; + + std::shared_ptr create_formatter(const std::string &format) override; + +protected: + // Maps from output string to formatter + std::map> m_formatters; + + gen_event_formatter::output_format m_output_format; + + // All the filterchecks required to resolve tokens in the format string + std::shared_ptr m_json_factory; }; From c0d1e0ac302f9fa9a390383343d1cba03ab6cb54 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 17:16:17 -0700 Subject: [PATCH 06/18] Use factories to provide filters/formatting Instead of having hard-coded support for syscall/k8s_audit events, use the notions of filter factories/formatter factories to provide generic support for events having a given source: - Within the engine, maps m_filter_factories / m_rulesets / m_format_factories map from a given source to something that can create filters, hold filters, and create formatters for a given source. The hard-coded sinsp_factory/json_factory objects are removed. - The specific add_xxx_filter/process_xxx_event are general purpose and take an event source. - A new method create_formatter() takes a source/output format and provides a shared_ptr to a formatter than can resolve format strings. This is used by the falco outputs code. - In falco main, create the syscall/k8s_audit filter and formatter factories and pass them to the engine. Later, we might make this configurable/selective. With all of the above changes, the falco engine doesn't need a direct inspector any longer, so remove it. Signed-off-by: Mark Stemm --- userspace/engine/falco_common.cpp | 5 - userspace/engine/falco_common.h | 6 +- userspace/engine/falco_engine.cpp | 192 +++++++++++++----------------- userspace/engine/falco_engine.h | 90 ++++++-------- userspace/falco/falco.cpp | 25 +++- 5 files changed, 143 insertions(+), 175 deletions(-) diff --git a/userspace/engine/falco_common.cpp b/userspace/engine/falco_common.cpp index e64bca2188d..69068f657a7 100644 --- a/userspace/engine/falco_common.cpp +++ b/userspace/engine/falco_common.cpp @@ -48,11 +48,6 @@ falco_common::~falco_common() } } -void falco_common::set_inspector(sinsp *inspector) -{ - m_inspector = inspector; -} - void falco_common::init(const char *lua_main_filename, const char *alternate_lua_dir) { ifstream is; diff --git a/userspace/engine/falco_common.h b/userspace/engine/falco_common.h index 6f231490012..494072a86b3 100644 --- a/userspace/engine/falco_common.h +++ b/userspace/engine/falco_common.h @@ -71,8 +71,6 @@ class falco_common void init(const char *lua_main_filename, const char *alternate_lua_dir); - void set_inspector(sinsp *inspector); - // Priority levels, as a vector of strings static std::vector priority_names; @@ -94,8 +92,6 @@ class falco_common std::mutex m_ls_semaphore; - sinsp *m_inspector; - private: void add_lua_path(std::string &path); -}; \ No newline at end of file +}; diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 54a79d3ea3f..39ab9f0bce9 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -40,10 +40,8 @@ string lua_print_stats = "print_stats"; using namespace std; -nlohmann::json::json_pointer falco_engine::k8s_audit_time = "/stageTimestamp"_json_pointer; - falco_engine::falco_engine(bool seed_rng, const std::string& alternate_lua_dir) - : m_rules(NULL), m_next_ruleset_id(0), + : m_next_ruleset_id(0), m_min_priority(falco_common::PRIORITY_DEBUG), m_sampling_ratio(1), m_sampling_multiplier(0), m_replace_container_info(false) @@ -54,26 +52,16 @@ falco_engine::falco_engine(bool seed_rng, const std::string& alternate_lua_dir) falco_common::init(m_lua_main_filename.c_str(), alternate_lua_dir.c_str()); falco_rules::init(m_ls); - m_sinsp_rules.reset(new falco_sinsp_ruleset()); - m_k8s_audit_rules.reset(new falco_ruleset()); - if(seed_rng) { srandom((unsigned) getpid()); } m_default_ruleset_id = find_ruleset_id(m_default_ruleset); - - // Create this now so we can potentially list filters and exit - m_json_factory = make_shared(); } falco_engine::~falco_engine() { - if (m_rules) - { - delete m_rules; - } } uint32_t falco_engine::engine_version() @@ -153,32 +141,16 @@ void falco_engine::load_rules(const string &rules_content, bool verbose, bool al void falco_engine::load_rules(const string &rules_content, bool verbose, bool all_events, uint64_t &required_engine_version) { - // The engine must have been given an inspector by now. - if(! m_inspector) - { - throw falco_exception("No inspector provided"); - } - - if(!m_sinsp_factory) - { - m_sinsp_factory = make_shared(m_inspector); - } - if(!m_rules) { - m_rules = new falco_rules(m_inspector, - this, - m_ls); - } + m_rules.reset(new falco_rules(this, + m_ls)); - // Note that falco_formats is added to the lua state used - // by the falco engine only. Within the engine, only - // formats.formatter is used, so we can unconditionally set - // json_output to false. - bool json_output = false; - bool json_include_output_property = false; - bool json_include_tags_property = false; - falco_formats::init(m_inspector, this, m_ls, json_output, json_include_output_property, json_include_tags_property); + for(auto const &it : m_filter_factories) + { + m_rules->add_filter_factory(it.first, it.second); + } + } m_rules->load_rules(rules_content, verbose, all_events, m_extra, m_replace_container_info, m_min_priority, required_engine_version); } @@ -213,8 +185,10 @@ void falco_engine::enable_rule(const string &substring, bool enabled, const stri uint16_t ruleset_id = find_ruleset_id(ruleset); bool match_exact = false; - m_sinsp_rules->enable(substring, match_exact, enabled, ruleset_id); - m_k8s_audit_rules->enable(substring, match_exact, enabled, ruleset_id); + for(auto &it : m_rulesets) + { + it.second->enable(substring, match_exact, enabled, ruleset_id); + } } void falco_engine::enable_rule(const string &substring, bool enabled) @@ -227,8 +201,10 @@ void falco_engine::enable_rule_exact(const string &rule_name, bool enabled, cons uint16_t ruleset_id = find_ruleset_id(ruleset); bool match_exact = true; - m_sinsp_rules->enable(rule_name, match_exact, enabled, ruleset_id); - m_k8s_audit_rules->enable(rule_name, match_exact, enabled, ruleset_id); + for(auto &it : m_rulesets) + { + it.second->enable(rule_name, match_exact, enabled, ruleset_id); + } } void falco_engine::enable_rule_exact(const string &rule_name, bool enabled) @@ -240,8 +216,10 @@ void falco_engine::enable_rule_by_tag(const set &tags, bool enabled, con { uint16_t ruleset_id = find_ruleset_id(ruleset); - m_sinsp_rules->enable_tags(tags, enabled, ruleset_id); - m_k8s_audit_rules->enable_tags(tags, enabled, ruleset_id); + for(auto &it : m_rulesets) + { + it.second->enable_tags(tags, enabled, ruleset_id); + } } void falco_engine::enable_rule_by_tag(const set &tags, bool enabled) @@ -272,68 +250,85 @@ uint64_t falco_engine::num_rules_for_ruleset(const std::string &ruleset) { uint16_t ruleset_id = find_ruleset_id(ruleset); - return m_sinsp_rules->num_rules_for_ruleset(ruleset_id) + - m_k8s_audit_rules->num_rules_for_ruleset(ruleset_id); + uint64_t ret = 0; + for(auto &it : m_rulesets) + { + ret += it.second->num_rules_for_ruleset(ruleset_id); + } + + return ret; } -void falco_engine::evttypes_for_ruleset(std::vector &evttypes, const std::string &ruleset) +void falco_engine::evttypes_for_ruleset(std::string &source, std::set &evttypes, const std::string &ruleset) { uint16_t ruleset_id = find_ruleset_id(ruleset); - return m_sinsp_rules->evttypes_for_ruleset(evttypes, ruleset_id); + auto it = m_rulesets.find(source); + if(it == m_rulesets.end()) + { + string err = "Unknown event source " + source; + throw falco_exception(err); + } + + it->second->evttypes_for_ruleset(evttypes, ruleset_id); + } -void falco_engine::syscalls_for_ruleset(std::vector &syscalls, const std::string &ruleset) +std::shared_ptr falco_engine::create_formatter(const std::string &source, + const std::string &output) { - uint16_t ruleset_id = find_ruleset_id(ruleset); + auto it = m_format_factories.find(source); + + if(it == m_format_factories.end()) + { + string err = "Unknown event source " + source; + throw falco_exception(err); + } - return m_sinsp_rules->syscalls_for_ruleset(syscalls, ruleset_id); + return it->second->create_formatter(output); } -unique_ptr falco_engine::process_sinsp_event(sinsp_evt *ev, uint16_t ruleset_id) +unique_ptr falco_engine::process_event(std::string &source, gen_event *ev, uint16_t ruleset_id) { if(should_drop_evt()) { return unique_ptr(); } - if(!m_sinsp_rules->run(ev, ruleset_id)) + auto it = m_rulesets.find(source); + if(it == m_rulesets.end()) + { + string err = "Unknown event source " + source; + throw falco_exception(err); + } + + if (!it->second->run(ev, ruleset_id)) { return unique_ptr(); } unique_ptr res(new rule_result()); - res->source = "syscall"; + res->source = source; populate_rule_result(res, ev); return res; } -unique_ptr falco_engine::process_sinsp_event(sinsp_evt *ev) +unique_ptr falco_engine::process_event(std::string &source, gen_event *ev) { - return process_sinsp_event(ev, m_default_ruleset_id); + return process_event(source, ev, m_default_ruleset_id); } -unique_ptr falco_engine::process_k8s_audit_event(json_event *ev, uint16_t ruleset_id) +void falco_engine::add_source(std::string &source, + std::shared_ptr filter_factory, + std::shared_ptr formatter_factory) { - if(should_drop_evt()) - { - return unique_ptr(); - } + m_filter_factories[source] = filter_factory; + m_format_factories[source] = formatter_factory; - // All k8s audit events have the single tag "1". - if(!m_k8s_audit_rules->run((gen_event *) ev, 1, ruleset_id)) - { - return unique_ptr(); - } - - unique_ptr res(new rule_result()); - res->source = "k8s_audit"; - - populate_rule_result(res, ev); - - return res; + std::shared_ptr ruleset(new falco_ruleset()); + m_rulesets[source] = ruleset; } void falco_engine::populate_rule_result(unique_ptr &res, gen_event *ev) @@ -412,29 +407,30 @@ void falco_engine::print_stats() } -void falco_engine::add_sinsp_filter(string &rule, - set &evttypes, - set &syscalls, - set &tags, - sinsp_filter* filter) +void falco_engine::add_filter(std::shared_ptr filter, + std::string &rule, + std::string &source, + std::set &tags) { - m_sinsp_rules->add(rule, evttypes, syscalls, tags, filter); -} - -void falco_engine::add_k8s_audit_filter(string &rule, - set &tags, - json_event_filter* filter) -{ - // All k8s audit events have a single tag "1". - std::set event_tags = {1}; + auto it = m_rulesets.find(source); + if(it == m_rulesets.end()) + { + string err = "Unknown event source " + source; + throw falco_exception(err); + } - m_k8s_audit_rules->add(rule, tags, event_tags, filter); + it->second->add(rule, tags, filter); } void falco_engine::clear_filters() { - m_sinsp_rules.reset(new falco_sinsp_ruleset()); - m_k8s_audit_rules.reset(new falco_ruleset()); + m_rulesets.clear(); + + for(auto &it : m_filter_factories) + { + std::shared_ptr ruleset(new falco_ruleset()); + m_rulesets[it.first] = ruleset; + } } void falco_engine::set_sampling_ratio(uint32_t sampling_ratio) @@ -468,23 +464,3 @@ inline bool falco_engine::should_drop_evt() double coin = (random() * (1.0/RAND_MAX)); return (coin >= (1.0/(m_sampling_multiplier * m_sampling_ratio))); } - -sinsp_filter_factory &falco_engine::sinsp_factory() -{ - if(!m_sinsp_factory) - { - throw falco_exception("No sinsp factory created yet"); - } - - return *(m_sinsp_factory.get()); -} - -json_event_filter_factory &falco_engine::json_factory() -{ - if(!m_json_factory) - { - throw falco_exception("No json factory created yet"); - } - - return *(m_json_factory.get()); -} diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 21c15525da1..352bcb672fb 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -28,10 +28,7 @@ limitations under the License. #include -#include "sinsp.h" -#include "filter.h" - -#include "json_evt.h" +#include "gen_filter.h" #include "rules.h" #include "ruleset.h" @@ -153,8 +150,8 @@ class falco_engine : public falco_common // void set_extra(string &extra, bool replace_container_info); - // **Methods Related to k8s audit log events, which are - // **represented as json objects. + // Represents the result of matching an event against a set of + // rules. struct rule_result { gen_event *evt; std::string rule; @@ -176,85 +173,68 @@ class falco_engine : public falco_common // with a ruleset string. // // the returned rule_result is allocated and must be delete()d. - std::unique_ptr process_k8s_audit_event(json_event *ev, uint16_t ruleset_id); + std::unique_ptr process_event(std::string &source, gen_event *ev, uint16_t ruleset_id); // // Wrapper assuming the default ruleset // - std::unique_ptr process_k8s_audit_event(json_event *ev); + std::unique_ptr process_event(std::string &source, gen_event *ev); // - // Add a k8s_audit filter to the engine + // Configure the engine to support events with the provided + // source, with the provided filter factory and formatter factory. // - void add_k8s_audit_filter(std::string &rule, - std::set &tags, - json_event_filter* filter); + void add_source(std::string &source, + std::shared_ptr filter_factory, + std::shared_ptr formatter_factory); - // **Methods Related to Sinsp Events e.g system calls // - // Given a ruleset, fill in a bitset containing the event - // types for which this ruleset can run. + // Add a filter for the provided event source to the engine // - void evttypes_for_ruleset(std::vector &evttypes, const std::string &ruleset); + void add_filter(std::shared_ptr filter, + std::string &rule, + std::string &source, + std::set &tags); // - // Given a ruleset, fill in a bitset containing the syscalls - // for which this ruleset can run. + // Given an event source and ruleset, fill in a bitset + // containing the event types for which this ruleset can run. // - void syscalls_for_ruleset(std::vector &syscalls, const std::string &ruleset); + void evttypes_for_ruleset(std::string &source, + std::set &evttypes, + const std::string &ruleset); // - // Given an event, check it against the set of rules in the - // engine and if a matching rule is found, return details on - // the rule that matched. If no rule matched, returns NULL. - // - // When ruleset_id is provided, use the enabled/disabled status - // associated with the provided ruleset. This is only useful - // when you have previously called enable_rule/enable_rule_by_tag - // with a ruleset string. + // Given a source and output string, return an + // gen_event_formatter that can format output strings for an + // event. // - // the returned rule_result is allocated and must be delete()d. - std::unique_ptr process_sinsp_event(sinsp_evt *ev, uint16_t ruleset_id); - - // - // Wrapper assuming the default ruleset - // - std::unique_ptr process_sinsp_event(sinsp_evt *ev); - - // - // Add a filter, which is related to the specified set of - // event types/syscalls, to the engine. - // - void add_sinsp_filter(std::string &rule, - std::set &evttypes, - std::set &syscalls, - std::set &tags, - sinsp_filter* filter); - - sinsp_filter_factory &sinsp_factory(); - json_event_filter_factory &json_factory(); + std::shared_ptr create_formatter(const std::string &source, + const std::string &output); private: - static nlohmann::json::json_pointer k8s_audit_time; - // // Determine whether the given event should be matched at all // against the set of rules, given the current sampling // ratio/multiplier. // inline bool should_drop_evt(); - shared_ptr m_sinsp_factory; - shared_ptr m_json_factory; - falco_rules *m_rules; + // Maps from event source to object that can generate filters from rules + std::map> m_filter_factories; + + // Maps from event source to object that can format output strings in rules + std::map> m_format_factories; + + // Maps from event source to the set of rules for that event source + std::map> m_rulesets; + + std::unique_ptr m_rules; uint16_t m_next_ruleset_id; std::map m_known_rulesets; falco_common::priority_type m_min_priority; - std::unique_ptr m_sinsp_rules; - std::unique_ptr m_k8s_audit_rules; - void populate_rule_result(unique_ptr &res, gen_event *ev); // diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 83a5bd1fc20..826e459a5b5 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -33,6 +33,7 @@ limitations under the License. #include #include +#include #include "logger.h" #include "utils.h" @@ -258,6 +259,8 @@ uint64_t do_inspect(falco_engine *engine, uint64_t duration_start = 0; uint32_t timeouts_since_last_success_or_msg = 0; + std::string syscall_source = "syscall"; + sdropmgr.init(inspector, outputs, config.m_syscall_evt_drop_actions, @@ -371,7 +374,7 @@ uint64_t do_inspect(falco_engine *engine, // engine, which will match the event against the set // of rules. If a match is found, pass the event to // the outputs. - unique_ptr res = engine->process_sinsp_event(ev); + unique_ptr res = engine->process_event(syscall_source, ev); if(res) { outputs->handle_event(res->evt, res->rule, res->source, res->priority_num, res->format, res->tags); @@ -769,9 +772,21 @@ int falco_init(int argc, char **argv) } engine = new falco_engine(true, alternate_lua_dir); - engine->set_inspector(inspector); engine->set_extra(output_format, replace_container_info); + // Create "factories" that can create filters/formatters for + // syscalls and k8s audit events. + std::shared_ptr syscall_filter_factory(new sinsp_filter_factory(inspector)); + std::shared_ptr k8s_audit_filter_factory(new json_event_filter_factory()); + + std::shared_ptr syscall_formatter_factory(new sinsp_evt_formatter_factory(inspector)); + std::shared_ptr k8s_audit_formatter_factory(new json_event_formatter_factory(k8s_audit_filter_factory)); + + string syscall_source = "syscall"; + string k8s_audit_source = "k8s_audit"; + engine->add_source(syscall_source, syscall_filter_factory, syscall_formatter_factory); + engine->add_source(k8s_audit_source, k8s_audit_filter_factory, k8s_audit_formatter_factory); + if(list_flds) { list_source_fields(engine, names_only, list_flds_source); @@ -872,6 +887,12 @@ int falco_init(int argc, char **argv) throw std::runtime_error("Could not find configuration file at " + conf_filename); } + if(config.m_json_output) + { + syscall_formatter_factory->set_output_format(gen_event_formatter::OF_JSON); + k8s_audit_formatter_factory->set_output_format(gen_event_formatter::OF_JSON); + } + if (rules_filenames.size()) { config.m_rules_filenames = rules_filenames; From b64ea87747fe648e26121c43cef40c0d5cc3d7ac Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 17:17:52 -0700 Subject: [PATCH 07/18] Update rules loader to be general purpose, through factories Update rules loader to be more general purpose by using factories and the general purpose engine: - A lua callback create_lua_parser creates a lua_parser with a filter object of the right type. The lua parser can then iterate the AST and populate the filter object. - Like the falco engine, the rules loader is configured with a list of factories, and add_filter is now general purpose, taking a source. Given the fix in https://github.com/falcosecurity/libs/pull/72, there isn't any need to pass down the entire set of sinsp event types/syscalls and validate that all filter event types are valid. That job is now handled by the sinsp filter parsing code. add_filter now returns the number of event types used by the new filter, and if that number is excessive the lua code will return a warning. Format handling is mostly not handled by the rules loader any more. As a convienence, there's a new lua callback is_format_valid which takes a source and output string and uses the right formatter factory to create a formatter. As long as that doesn't throw an exception, the format is valid. Signed-off-by: Mark Stemm --- userspace/engine/rules.cpp | 376 +++++++++++-------------------------- userspace/engine/rules.h | 32 ++-- 2 files changed, 131 insertions(+), 277 deletions(-) diff --git a/userspace/engine/rules.cpp b/userspace/engine/rules.cpp index 14c71d8d7b2..ca1f399a841 100644 --- a/userspace/engine/rules.cpp +++ b/userspace/engine/rules.cpp @@ -30,26 +30,30 @@ extern "C" { const static struct luaL_Reg ll_falco_rules[] = { {"clear_filters", &falco_rules::clear_filters}, + {"create_lua_parser", &falco_rules::create_lua_parser}, {"add_filter", &falco_rules::add_filter}, - {"add_k8s_audit_filter", &falco_rules::add_k8s_audit_filter}, {"enable_rule", &falco_rules::enable_rule}, {"engine_version", &falco_rules::engine_version}, + {"is_format_valid", &falco_rules::is_format_valid}, {NULL, NULL}}; -falco_rules::falco_rules(sinsp* inspector, - falco_engine *engine, +falco_rules::falco_rules(falco_engine *engine, lua_State *ls) - : m_inspector(inspector), - m_engine(engine), + : m_engine(engine), m_ls(ls) { - m_sinsp_lua_parser = new lua_parser(engine->sinsp_factory(), m_ls, "filter"); - m_json_lua_parser = new lua_parser(engine->json_factory(), m_ls, "k8s_audit_filter"); +} + +void falco_rules::add_filter_factory(const std::string &source, + std::shared_ptr factory) +{ + m_filter_factories[source] = factory; } void falco_rules::init(lua_State *ls) { luaL_openlib(ls, "falco_rules", ll_falco_rules, 0); + lua_parser::register_callbacks(ls, "filter"); } int falco_rules::clear_filters(lua_State *ls) @@ -71,75 +75,61 @@ void falco_rules::clear_filters() m_engine->clear_filters(); } -int falco_rules::add_filter(lua_State *ls) +int falco_rules::create_lua_parser(lua_State *ls) { - if (! lua_islightuserdata(ls, -5) || - ! lua_isstring(ls, -4) || - ! lua_istable(ls, -3) || - ! lua_istable(ls, -2) || - ! lua_istable(ls, -1)) + if (! lua_islightuserdata(ls, -2) || + ! lua_isstring(ls, -1)) { - lua_pushstring(ls, "Invalid arguments passed to add_filter()"); + lua_pushstring(ls, "Invalid argument passed to create_lua_parser()"); lua_error(ls); } - falco_rules *rules = (falco_rules *) lua_topointer(ls, -5); - const char *rulec = lua_tostring(ls, -4); + falco_rules *rules = (falco_rules *) lua_topointer(ls, -2); + std::string source = lua_tostring(ls, -1); - set evttypes; - - lua_pushnil(ls); /* first key */ - while (lua_next(ls, -4) != 0) { - // key is at index -2, value is at index - // -1. We want the keys. - evttypes.insert(luaL_checknumber(ls, -2)); - - // Remove value, keep key for next iteration - lua_pop(ls, 1); - } - - set syscalls; - - lua_pushnil(ls); /* first key */ - while (lua_next(ls, -3) != 0) { - // key is at index -2, value is at index - // -1. We want the keys. - syscalls.insert(luaL_checknumber(ls, -2)); + std::string errstr; + lua_parser *lp = rules->create_lua_parser(source, errstr); - // Remove value, keep key for next iteration - lua_pop(ls, 1); + if(lp == NULL) { + lua_pushstring(ls, errstr.c_str()); + lua_error(ls); } - set tags; + lua_pushlightuserdata(ls, lp); + return 1; +} - lua_pushnil(ls); /* first key */ - while (lua_next(ls, -2) != 0) { - // key is at index -2, value is at index - // -1. We want the values. - tags.insert(lua_tostring(ls, -1)); +lua_parser *falco_rules::create_lua_parser(std::string &source, std::string &errstr) +{ + auto it = m_filter_factories.find(source); - // Remove value, keep key for next iteration - lua_pop(ls, 1); + if(it == m_filter_factories.end()) + { + errstr = string("Unknown event source ") + source; + return NULL; } - std::string rule = rulec; - rules->add_filter(rule, evttypes, syscalls, tags); + lua_parser *lp = new lua_parser(it->second); - return 0; + return lp; } -int falco_rules::add_k8s_audit_filter(lua_State *ls) +int falco_rules::add_filter(lua_State *ls) { - if (! lua_islightuserdata(ls, -3) || + if (! lua_islightuserdata(ls, -5) || + ! lua_islightuserdata(ls, -4) || + ! lua_isstring(ls, -3) || ! lua_isstring(ls, -2) || ! lua_istable(ls, -1)) { - lua_pushstring(ls, "Invalid arguments passed to add_k8s_audit_filter()"); + lua_pushstring(ls, "Invalid arguments passed to add_filter()"); lua_error(ls); } - falco_rules *rules = (falco_rules *) lua_topointer(ls, -3); - const char *rulec = lua_tostring(ls, -2); + falco_rules *rules = (falco_rules *) lua_topointer(ls, -5); + lua_parser *lp = (lua_parser *) lua_topointer(ls, -4); + std::string rule = lua_tostring(ls, -3); + std::string source = lua_tostring(ls, -2); set tags; @@ -153,30 +143,25 @@ int falco_rules::add_k8s_audit_filter(lua_State *ls) lua_pop(ls, 1); } - std::string rule = rulec; - rules->add_k8s_audit_filter(rule, tags); + size_t num_evttypes = lp->filter()->evttypes().size(); - return 0; -} + try { + rules->add_filter(lp->filter(), rule, source, tags); + } catch (exception &e) { + std::string errstr = string("Could not add rule to falco engine: ") + e.what(); + lua_pushstring(ls, errstr.c_str()); + lua_error(ls); + } -void falco_rules::add_filter(string &rule, set &evttypes, set &syscalls, set &tags) -{ - // While the current rule was being parsed, a sinsp_filter - // object was being populated by lua_parser. Grab that filter - // and pass it to the engine. - sinsp_filter *filter = (sinsp_filter *) m_sinsp_lua_parser->get_filter(true); + delete lp; - m_engine->add_sinsp_filter(rule, evttypes, syscalls, tags, filter); + lua_pushnumber(ls, num_evttypes); + return 1; } -void falco_rules::add_k8s_audit_filter(string &rule, set &tags) +void falco_rules::add_filter(std::shared_ptr filter, string &rule, string &source, set &tags) { - // While the current rule was being parsed, a sinsp_filter - // object was being populated by lua_parser. Grab that filter - // and pass it to the engine. - json_event_filter *filter = (json_event_filter *) m_json_lua_parser->get_filter(true); - - m_engine->add_k8s_audit_filter(rule, tags, filter); + m_engine->add_filter(filter, rule, source, tags); } int falco_rules::enable_rule(lua_State *ls) @@ -219,6 +204,62 @@ int falco_rules::engine_version(lua_State *ls) return 1; } +int falco_rules::is_format_valid(lua_State *ls) +{ + if (! lua_islightuserdata(ls, -3) || + ! lua_isstring(ls, -2) || + ! lua_isstring(ls, -1)) + { + lua_pushstring(ls, "Invalid arguments passed to is_format_valid"); + lua_error(ls); + } + + falco_rules *rules = (falco_rules *) lua_topointer(ls, -3); + string source = luaL_checkstring(ls, -2); + string format = luaL_checkstring(ls, -1); + string errstr; + + bool ret = rules->is_format_valid(source, format, errstr); + + if (!ret) + { + lua_pushstring(ls, errstr.c_str()); + } + else + { + lua_pushnil(ls); + } + + return 1; +} + +bool falco_rules::is_format_valid(const std::string &source, const std::string &format, std::string &errstr) +{ + bool ret = true; + + try + { + std::shared_ptr formatter; + + formatter = m_engine->create_formatter(source, format); + } + catch(exception &e) + { + std::ostringstream os; + + os << "Invalid output format '" + << format + << "': '" + << e.what() + << "'"; + + errstr = os.str(); + ret = false; + } + + return ret; +} + static std::list get_lua_table_values(lua_State *ls, int idx) { std::list ret; @@ -253,195 +294,6 @@ void falco_rules::load_rules(const string &rules_content, lua_getglobal(m_ls, m_lua_load_rules.c_str()); if(lua_isfunction(m_ls, -1)) { - // Create a table containing all events, so they can - // be mapped to event ids. - sinsp_evttables* einfo = m_inspector->get_event_info_tables(); - const struct ppm_event_info* etable = einfo->m_event_info; - const struct ppm_syscall_desc* stable = einfo->m_syscall_info_table; - - map events_by_name; - for(uint32_t j = 0; j < PPM_EVENT_MAX; j++) - { - auto it = events_by_name.find(etable[j].name); - - if (it == events_by_name.end()) { - events_by_name[etable[j].name] = to_string(j); - } else { - string cur = it->second; - cur += " "; - cur += to_string(j); - events_by_name[etable[j].name] = cur; - } - } - - lua_newtable(m_ls); - - for( auto kv : events_by_name) - { - lua_pushstring(m_ls, kv.first.c_str()); - lua_pushstring(m_ls, kv.second.c_str()); - lua_settable(m_ls, -3); - } - - lua_setglobal(m_ls, m_lua_events.c_str()); - - map syscalls_by_name; - for(uint32_t j = 0; j < PPM_SC_MAX; j++) - { - auto it = syscalls_by_name.find(stable[j].name); - - if (it == syscalls_by_name.end()) - { - syscalls_by_name[stable[j].name] = to_string(j); - } - else - { - string cur = it->second; - cur += " "; - cur += to_string(j); - syscalls_by_name[stable[j].name] = cur; - } - } - - lua_newtable(m_ls); - - for( auto kv : syscalls_by_name) - { - lua_pushstring(m_ls, kv.first.c_str()); - lua_pushstring(m_ls, kv.second.c_str()); - lua_settable(m_ls, -3); - } - - lua_setglobal(m_ls, m_lua_syscalls.c_str()); - - // Create a table containing the syscalls/events that - // are ignored by the kernel module. load_rules will - // return an error if any rule references one of these - // syscalls/events. - - lua_newtable(m_ls); - - for(uint32_t j = 0; j < PPM_EVENT_MAX; j++) - { - if(etable[j].flags & EF_DROP_SIMPLE_CONS) - { - lua_pushstring(m_ls, etable[j].name); - lua_pushnumber(m_ls, 1); - lua_settable(m_ls, -3); - } - } - - lua_setglobal(m_ls, m_lua_ignored_events.c_str()); - - lua_newtable(m_ls); - - for(uint32_t j = 0; j < PPM_SC_MAX; j++) - { - if(stable[j].flags & EF_DROP_SIMPLE_CONS) - { - lua_pushstring(m_ls, stable[j].name); - lua_pushnumber(m_ls, 1); - lua_settable(m_ls, -3); - } - } - - lua_setglobal(m_ls, m_lua_ignored_syscalls.c_str()); - - vector fc_plugins; - sinsp::get_filtercheck_fields_info(&fc_plugins); - - set no_argument_filters; - set argument_filters; - - for(uint32_t j = 0; j < fc_plugins.size(); j++) - { - const filter_check_info* fci = fc_plugins[j]; - - if(fci->m_flags & filter_check_info::FL_HIDDEN) - { - continue; - } - - for(int32_t k = 0; k < fci->m_nfields; k++) - { - const filtercheck_field_info* fld = &fci->m_fields[k]; - - if(fld->m_flags & EPF_TABLE_ONLY || - fld->m_flags & EPF_PRINT_ONLY) - { - continue; - } - - // Some filters can work with or without an argument - std::set flexible_filters = { - "proc.aname", - "proc.apid" - }; - - if(fld->m_flags & EPF_REQUIRES_ARGUMENT || - flexible_filters.find(fld->m_name) != flexible_filters.end()) - { - argument_filters.insert(fld->m_name); - } - - if(!(fld->m_flags & EPF_REQUIRES_ARGUMENT) || - flexible_filters.find(fld->m_name) != flexible_filters.end()) - { - no_argument_filters.insert(fld->m_name); - } - } - } - - for(auto &chk_field : m_engine->json_factory().get_fields()) - { - for(auto &field : chk_field.m_fields) - { - switch(field.m_idx_mode) - { - case json_event_filter_check::IDX_REQUIRED: - argument_filters.insert(field.m_name); - break; - case json_event_filter_check::IDX_ALLOWED: - argument_filters.insert(field.m_name); - no_argument_filters.insert(field.m_name); - break; - case json_event_filter_check::IDX_NONE: - no_argument_filters.insert(field.m_name); - break; - default: - break; - } - } - } - - // Create tables containing all filtercheck - // names. They are split into names that require - // arguments and ones that do not. - - lua_newtable(m_ls); - - for(auto &field : argument_filters) - { - lua_pushstring(m_ls, field.c_str()); - lua_pushnumber(m_ls, 1); - lua_settable(m_ls, -3); - } - - lua_setglobal(m_ls, m_lua_defined_arg_filters.c_str()); - - lua_newtable(m_ls); - - for(auto &field : no_argument_filters) - { - lua_pushstring(m_ls, field.c_str()); - lua_pushnumber(m_ls, 1); - lua_settable(m_ls, -3); - } - - lua_setglobal(m_ls, m_lua_defined_noarg_filters.c_str()); - - lua_pushlightuserdata(m_ls, m_sinsp_lua_parser); - lua_pushlightuserdata(m_ls, m_json_lua_parser); lua_pushstring(m_ls, rules_content.c_str()); lua_pushlightuserdata(m_ls, this); lua_pushboolean(m_ls, (verbose ? 1 : 0)); @@ -449,7 +301,7 @@ void falco_rules::load_rules(const string &rules_content, lua_pushstring(m_ls, extra.c_str()); lua_pushboolean(m_ls, (replace_container_info ? 1 : 0)); lua_pushnumber(m_ls, min_priority); - if(lua_pcall(m_ls, 9, 4, 0) != 0) + if(lua_pcall(m_ls, 7, 4, 0) != 0) { const char* lerr = lua_tostring(m_ls, -1); @@ -533,6 +385,4 @@ void falco_rules::describe_rule(std::string *rule) falco_rules::~falco_rules() { - delete m_sinsp_lua_parser; - delete m_json_lua_parser; } diff --git a/userspace/engine/rules.h b/userspace/engine/rules.h index 67513547137..48d5ab02aa7 100644 --- a/userspace/engine/rules.h +++ b/userspace/engine/rules.h @@ -32,41 +32,45 @@ class falco_engine; class falco_rules { public: - falco_rules(sinsp* inspector, - falco_engine *engine, + falco_rules(falco_engine *engine, lua_State *ls); ~falco_rules(); + + void add_filter_factory(const std::string &source, + std::shared_ptr factory); + void load_rules(const string &rules_content, bool verbose, bool all_events, std::string &extra, bool replace_container_info, falco_common::priority_type min_priority, uint64_t &required_engine_version); void describe_rule(string *rule); + bool is_format_valid(const std::string &source, const std::string &format, std::string &errstr); + static void init(lua_State *ls); static int clear_filters(lua_State *ls); + static int create_lua_parser(lua_State *ls); static int add_filter(lua_State *ls); - static int add_k8s_audit_filter(lua_State *ls); static int enable_rule(lua_State *ls); static int engine_version(lua_State *ls); + // err = falco_rules.is_format_valid(source, format_string) + static int is_format_valid(lua_State *ls); + private: void clear_filters(); - void add_filter(string &rule, std::set &evttypes, std::set &syscalls, std::set &tags); - void add_k8s_audit_filter(string &rule, std::set &tags); + // XXX/mstemm can I make this a shared_ptr? + lua_parser * create_lua_parser(std::string &source, std::string &errstr); + void add_filter(std::shared_ptr filter, string &rule, string &source, std::set &tags); void enable_rule(string &rule, bool enabled); - lua_parser* m_sinsp_lua_parser; - lua_parser* m_json_lua_parser; - sinsp* m_inspector; falco_engine *m_engine; lua_State* m_ls; + // Maps from event source to an object that can create rules + // for that event source. + std::map> m_filter_factories; + string m_lua_load_rules = "load_rules"; - string m_lua_ignored_syscalls = "ignored_syscalls"; - string m_lua_ignored_events = "ignored_events"; - string m_lua_defined_arg_filters = "defined_arg_filters"; - string m_lua_defined_noarg_filters = "defined_noarg_filters"; - string m_lua_events = "events"; - string m_lua_syscalls = "syscalls"; string m_lua_describe_rule = "describe_rule"; }; From 580718c60e8318357a4eb7733a36d56612b867db Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 17:26:08 -0700 Subject: [PATCH 08/18] Use new outputs interface with engine Use the new outputs interface, that uses the engine to provide a formatter. Signed-off-by: Mark Stemm --- userspace/falco/falco.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 826e459a5b5..1397aa17d1a 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -1134,14 +1134,15 @@ int falco_init(int argc, char **argv) outputs = new falco_outputs(); - outputs->init(config.m_json_output, - config.m_json_include_output_property, - config.m_json_include_tags_property, - config.m_output_timeout, - config.m_notifications_rate, config.m_notifications_max_burst, - config.m_buffered_outputs, - config.m_time_format_iso_8601, - hostname); + outputs->init(engine, + config.m_json_output, + config.m_json_include_output_property, + config.m_json_include_tags_property, + config.m_output_timeout, + config.m_notifications_rate, config.m_notifications_max_burst, + config.m_buffered_outputs, + config.m_time_format_iso_8601, + hostname); for(auto output : config.m_outputs) { From f98e6fc8815cd6400fb08e391587811430a3906f Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 17:27:12 -0700 Subject: [PATCH 09/18] Use the new falco engine interface w/ generic events Use the new falco engine interface with support for generic events instead of event-specific process_xxx_event methods. Signed-off-by: Mark Stemm --- userspace/falco/webserver.cpp | 4 +++- userspace/falco/webserver.h | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/userspace/falco/webserver.cpp b/userspace/falco/webserver.cpp index 7eca9dcf204..6098063eee2 100644 --- a/userspace/falco/webserver.cpp +++ b/userspace/falco/webserver.cpp @@ -25,6 +25,8 @@ limitations under the License. using json = nlohmann::json; using namespace std; +string k8s_audit_handler::m_k8s_audit_event_source = "k8s_audit"; + k8s_audit_handler::k8s_audit_handler(falco_engine *engine, falco_outputs *outputs): m_engine(engine), m_outputs(outputs) { @@ -87,7 +89,7 @@ bool k8s_audit_handler::accept_data(falco_engine *engine, try { - res = engine->process_k8s_audit_event(&jev); + res = engine->process_event(m_k8s_audit_event_source, &jev); } catch(...) { diff --git a/userspace/falco/webserver.h b/userspace/falco/webserver.h index 850cefee2af..c7310cb7917 100644 --- a/userspace/falco/webserver.h +++ b/userspace/falco/webserver.h @@ -35,6 +35,8 @@ class k8s_audit_handler : public CivetHandler falco_outputs *outputs, std::string &post_data, std::string &errstr); + static std::string m_k8s_audit_event_source; + private: falco_engine *m_engine; falco_outputs *m_outputs; From ec01c582084687d9e09fa5a2f03f0569fe942b47 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Wed, 25 Aug 2021 17:58:54 -0700 Subject: [PATCH 10/18] Add ability to check if field is defined Add a function is_defined_field(source, fldname) that returns whether a field with name fldname exists for the given event source. This uses the filter factory to create a filtercheck, and returns true if an object was created. This prevents having to push down the entire set of defined fields before calling load_rules(). Signed-off-by: Mark Stemm --- userspace/engine/rules.cpp | 43 ++++++++++++++++++++++++++++++++++++++ userspace/engine/rules.h | 5 +++++ 2 files changed, 48 insertions(+) diff --git a/userspace/engine/rules.cpp b/userspace/engine/rules.cpp index ca1f399a841..ba0d9ca83a9 100644 --- a/userspace/engine/rules.cpp +++ b/userspace/engine/rules.cpp @@ -35,6 +35,7 @@ const static struct luaL_Reg ll_falco_rules[] = {"enable_rule", &falco_rules::enable_rule}, {"engine_version", &falco_rules::engine_version}, {"is_format_valid", &falco_rules::is_format_valid}, + {"is_defined_field", &falco_rules::is_defined_field}, {NULL, NULL}}; falco_rules::falco_rules(falco_engine *engine, @@ -260,6 +261,48 @@ bool falco_rules::is_format_valid(const std::string &source, const std::string & return ret; } +int falco_rules::is_defined_field(lua_State *ls) +{ + if (! lua_islightuserdata(ls, -3) || + ! lua_isstring(ls, -2) || + ! lua_isstring(ls, -1)) + { + lua_pushstring(ls, "Invalid arguments passed to is_defined_field"); + lua_error(ls); + } + + falco_rules *rules = (falco_rules *) lua_topointer(ls, -3); + string source = luaL_checkstring(ls, -2); + string fldname = luaL_checkstring(ls, -1); + + bool ret = rules->is_defined_field(source, fldname); + + lua_pushboolean(ls, (ret ? 1 : 0)); + + return 1; +} + +bool falco_rules::is_defined_field(const std::string &source, const std::string &fldname) +{ + auto it = m_filter_factories.find(source); + + if(it == m_filter_factories.end()) + { + return false; + } + + auto *chk = it->second->new_filtercheck(fldname.c_str()); + + if (chk == NULL) + { + return false; + } + + delete(chk); + + return true; +} + static std::list get_lua_table_values(lua_State *ls, int idx) { std::list ret; diff --git a/userspace/engine/rules.h b/userspace/engine/rules.h index 48d5ab02aa7..717675dc4a5 100644 --- a/userspace/engine/rules.h +++ b/userspace/engine/rules.h @@ -47,6 +47,8 @@ class falco_rules bool is_format_valid(const std::string &source, const std::string &format, std::string &errstr); + bool is_defined_field(const std::string &source, const std::string &field); + static void init(lua_State *ls); static int clear_filters(lua_State *ls); static int create_lua_parser(lua_State *ls); @@ -57,6 +59,9 @@ class falco_rules // err = falco_rules.is_format_valid(source, format_string) static int is_format_valid(lua_State *ls); + // err = falco_rules.is_defined_field(source, field) + static int is_defined_field(lua_State *ls); + private: void clear_filters(); // XXX/mstemm can I make this a shared_ptr? From 341c3fdd5e4f977a0f9713e2c61a033ccf7e1390 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 26 Aug 2021 11:01:44 -0700 Subject: [PATCH 11/18] Update lua rule loading to reflect other changes Update the lua side of rule loading to reflect other changes: - install_filter renamed to create_filter_obj, and takes just a lua_parser object created via falco_rules.create_lua_parser() and uses a single lua callback "filter" instead of separate ones for syscall/k8s_audit. It can return an error, including about undefined fields - is_defined_filter, which used to be local and based on the result of sinsp_rule_utils.check_for_ignored_syscalls_events, is now a lua_callback falco_rules.is_defined_field(). - Don't need to pass down sinsp_lua_parser/json_lua_parser now, creating filters is handled via lua callbacks. - Checking for ignored syscalls/events is now done in falco itself, after loading rules. - add_xxx_filter replaced by add_filter + source. - Use is_format_valid instead of formats.formatter/formats.free_formatter. - We don't need the functions in sinsp_rule_utils any longer, so remove the file and don't import it. Signed-off-by: Mark Stemm --- userspace/engine/lua/rule_loader.lua | 192 ++++++++++----------- userspace/engine/lua/sinsp_rule_utils.lua | 199 ---------------------- 2 files changed, 90 insertions(+), 301 deletions(-) delete mode 100644 userspace/engine/lua/sinsp_rule_utils.lua diff --git a/userspace/engine/lua/rule_loader.lua b/userspace/engine/lua/rule_loader.lua index 6d24ba40623..67b979d8884 100644 --- a/userspace/engine/lua/rule_loader.lua +++ b/userspace/engine/lua/rule_loader.lua @@ -20,7 +20,6 @@ --]] -local sinsp_rule_utils = require "sinsp_rule_utils" local compiler = require "compiler" local yaml = require"lyaml" @@ -69,7 +68,7 @@ priorities = { --[[ Take a filter AST and set it up in the libsinsp runtime, using the filter API. --]] -local function install_filter(node, filter_api_lib, lua_parser, parent_bool_op) +local function create_filter_obj(node, lua_parser, parent_bool_op) local t = node.type if t == "BinaryBoolOp" then @@ -78,41 +77,84 @@ local function install_filter(node, filter_api_lib, lua_parser, parent_bool_op) -- never necessary when we have identical successive operators. so we -- avoid it as a runtime performance optimization. if (not(node.operator == parent_bool_op)) then - filter_api_lib.nest(lua_parser) -- io.write("(") + err = filter.nest(lua_parser) -- io.write("(") + if err ~= nil then + return err + end + end + + err = create_filter_obj(node.left, lua_parser, node.operator) + if err ~= nil then + return err end - install_filter(node.left, filter_api_lib, lua_parser, node.operator) - filter_api_lib.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ") - install_filter(node.right, filter_api_lib, lua_parser, node.operator) + err = filter.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ") + if err ~= nil then + return err + end + + err = create_filter_obj(node.right, lua_parser, node.operator) + if err ~= nil then + return err + end if (not (node.operator == parent_bool_op)) then - filter_api_lib.unnest(lua_parser) -- io.write(")") + err = filter.unnest(lua_parser) -- io.write(")") + if err ~= nil then + return err + end end elseif t == "UnaryBoolOp" then - filter_api_lib.nest(lua_parser) --io.write("(") - filter_api_lib.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ") - install_filter(node.argument, filter_api_lib, lua_parser) - filter_api_lib.unnest(lua_parser) -- io.write(")") + err = filter.nest(lua_parser) --io.write("(") + if err ~= nil then + return err + end + + err = filter.bool_op(lua_parser, node.operator) -- io.write(" "..node.operator.." ") + if err ~= nil then + return err + end + + err = create_filter_obj(node.argument, lua_parser) + if err ~= nil then + return err + end + + err = filter.unnest(lua_parser) -- io.write(")") + if err ~= nil then + return err + end elseif t == "BinaryRelOp" then if (node.operator == "in" or node.operator == "intersects" or node.operator == "pmatch") then elements = map(function (el) return el.value end, node.right.elements) - filter_api_lib.rel_expr(lua_parser, node.left.value, node.operator, elements, node.index) + err = filter.rel_expr(lua_parser, node.left.value, node.operator, elements, node.index) + if err ~= nil then + return err + end else - filter_api_lib.rel_expr(lua_parser, node.left.value, node.operator, node.right.value, node.index) + err = filter.rel_expr(lua_parser, node.left.value, node.operator, node.right.value, node.index) + if err ~= nil then + return err + end end -- io.write(node.left.value.." "..node.operator.." "..node.right.value) elseif t == "UnaryRelOp" then - filter_api_lib.rel_expr(lua_parser, node.argument.value, node.operator, node.index) + err = filter.rel_expr(lua_parser, node.argument.value, node.operator, node.index) + if err ~= nil then + return err + end --io.write(node.argument.value.." "..node.operator) else - error ("Unexpected type in install_filter: "..t) + return "Unexpected type in create_filter_obj: "..t end + + return nil end function set_output(output_format, state) @@ -310,7 +352,7 @@ function build_error_with_context(ctx, err) return {ret} end -function validate_exception_item_multi_fields(eitem, context) +function validate_exception_item_multi_fields(rules_mgr, source, eitem, context) local name = eitem['name'] local fields = eitem['fields'] @@ -329,7 +371,7 @@ function validate_exception_item_multi_fields(eitem, context) end end for k, fname in ipairs(fields) do - if not is_defined_filter(fname) then + if not falco_rules.is_defined_field(rules_mgr, source, fname) then return false, build_error_with_context(context, "Rule exception item "..name..": field name "..fname.." is not a supported filter field"), warnings end end @@ -340,7 +382,7 @@ function validate_exception_item_multi_fields(eitem, context) end end -function validate_exception_item_single_field(eitem, context) +function validate_exception_item_single_field(rules_mgr, source, eitem, context) local name = eitem['name'] local fields = eitem['fields'] @@ -355,7 +397,7 @@ function validate_exception_item_single_field(eitem, context) return false, build_error_with_context(context, "Rule exception item "..name..": fields and comps must both be strings"), warnings end end - if not is_defined_filter(fields) then + if not falco_rules.is_defined_field(rules_mgr, source, fields) then return false, build_error_with_context(context, "Rule exception item "..name..": field name "..fields.." is not a supported filter field"), warnings end if defined_comp_operators[comps] == nil then @@ -363,37 +405,6 @@ function validate_exception_item_single_field(eitem, context) end end -function is_defined_filter(filter) - if defined_noarg_filters[filter] ~= nil then - return true - else - bracket_idx = string.find(filter, "[", 1, true) - - if bracket_idx ~= nil then - subfilter = string.sub(filter, 1, bracket_idx-1) - - if defined_arg_filters[subfilter] ~= nil then - return true - end - end - - dot_idx = string.find(filter, ".", 1, true) - - while dot_idx ~= nil do - subfilter = string.sub(filter, 1, dot_idx-1) - - if defined_arg_filters[subfilter] ~= nil then - return true - end - - dot_idx = string.find(filter, ".", dot_idx+1, true) - end - end - - return false -end - - function load_rules_doc(rules_mgr, doc, load_state) local warnings = {} @@ -558,9 +569,9 @@ function load_rules_doc(rules_mgr, doc, load_state) -- Different handling if the fields property is a single item vs a list local valid, err if type(eitem['fields']) == "table" then - valid, err = validate_exception_item_multi_fields(eitem, v['context']) + valid, err = validate_exception_item_multi_fields(rules_mgr, v['source'], eitem, v['context']) else - valid, err = validate_exception_item_single_field(eitem, v['context']) + valid, err = validate_exception_item_single_field(rules_mgr, v['source'], eitem, v['context']) end if valid == false then @@ -771,9 +782,7 @@ end -- - required engine version. will be nil when load result is false -- - List of Errors -- - List of Warnings -function load_rules(sinsp_lua_parser, - json_lua_parser, - rules_content, +function load_rules(rules_content, rules_mgr, verbose, all_events, @@ -883,12 +892,6 @@ function load_rules(sinsp_lua_parser, return false, nil, build_error_with_context(v['context'], ast), warnings end - if v['source'] == "syscall" then - if not all_events then - sinsp_rule_utils.check_for_ignored_syscalls_events(ast, 'macro', v['condition']) - end - end - state.macros[v['macro']] = {["ast"] = ast.filter.value, ["used"] = false} end @@ -933,41 +936,13 @@ function load_rules(sinsp_lua_parser, warn_evttypes = v['warn_evttypes'] end - local status, filter_ast, filters = compiler.compile_filter(v['rule'], v['compile_condition'], - state.macros, state.lists) + local status, filter_ast = compiler.compile_filter(v['rule'], v['compile_condition'], + state.macros, state.lists) if status == false then return false, nil, build_error_with_context(v['context'], filter_ast), warnings end - local evtttypes = {} - local syscallnums = {} - - if v['source'] == "syscall" then - if not all_events then - sinsp_rule_utils.check_for_ignored_syscalls_events(filter_ast, 'rule', v['rule']) - end - - evttypes, syscallnums = sinsp_rule_utils.get_evttypes_syscalls(name, filter_ast, v['compile_condition'], warn_evttypes, verbose) - end - - -- If a filter in the rule doesn't exist, either skip the rule - -- or raise an error, depending on the value of - -- skip-if-unknown-filter. - for filter, _ in pairs(filters) do - if not is_defined_filter(filter) then - msg = "rule \""..v['rule'].."\": contains unknown filter "..filter - warnings[#warnings + 1] = msg - - if not v['skip-if-unknown-filter'] then - return false, nil, build_error_with_context(v['context'], msg), warnings - else - print("Skipping "..msg) - goto next_rule - end - end - end - if (filter_ast.type == "Rule") then state.n_rules = state.n_rules + 1 @@ -983,15 +958,30 @@ function load_rules(sinsp_lua_parser, if (v['tags'] == nil) then v['tags'] = {} end - if v['source'] == "syscall" then - install_filter(filter_ast.filter.value, filter, sinsp_lua_parser) - -- Pass the filter and event types back up - falco_rules.add_filter(rules_mgr, v['rule'], evttypes, syscallnums, v['tags']) - elseif v['source'] == "k8s_audit" then - install_filter(filter_ast.filter.value, k8s_audit_filter, json_lua_parser) + lua_parser = falco_rules.create_lua_parser(rules_mgr, v['source']) + local err = create_filter_obj(filter_ast.filter.value, lua_parser) + if err ~= nil then + + -- If a rule has a property skip-if-unknown-filter: true, + -- and the error is about an undefined field, print a + -- message but continue. + if v['skip-if-unknown-filter'] == true and string.find(err, "filter_check called with nonexistent field") ~= nil then + msg = "Rule "..v['rule']..": warning (unknown-field):" + warnings[#warnings + 1] = msg + else + msg = "Rule "..v['rule']..": error "..err + return false, nil, build_error_with_context(v['context'], msg), warnings + end - falco_rules.add_k8s_audit_filter(rules_mgr, v['rule'], v['tags']) + else + num_evttypes = falco_rules.add_filter(rules_mgr, lua_parser, v['rule'], v['source'], v['tags']) + if num_evttypes == 0 or num_evttypes > 100 then + if warn_evttypes == true then + msg = "Rule "..v['rule']..": warning (no-evttype):" + warnings[#warnings + 1] = msg + end + end end -- Rule ASTs are merged together into one big AST, with "OR" between each @@ -1042,10 +1032,8 @@ function load_rules(sinsp_lua_parser, -- Ensure that the output field is properly formatted by -- creating a formatter from it. Any error will be thrown -- up to the top level. - local err, formatter = formats.formatter(v['source'], v['output']) - if err == nil then - formats.free_formatter(v['source'], formatter) - else + local err = falco_rules.is_format_valid(rules_mgr, v['source'], v['output']) + if err ~= nil then return false, nil, build_error_with_context(v['context'], err), warnings end else diff --git a/userspace/engine/lua/sinsp_rule_utils.lua b/userspace/engine/lua/sinsp_rule_utils.lua deleted file mode 100644 index 97a994d2c89..00000000000 --- a/userspace/engine/lua/sinsp_rule_utils.lua +++ /dev/null @@ -1,199 +0,0 @@ --- Copyright (C) 2019 The Falco Authors. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -local parser = require("parser") -local sinsp_rule_utils = {} - -function sinsp_rule_utils.check_for_ignored_syscalls_events(ast, filter_type, source) - - function check_syscall(val) - if ignored_syscalls[val] then - error("Ignored syscall \""..val.."\" in "..filter_type..": "..source) - end - - end - - function check_event(val) - if ignored_events[val] then - error("Ignored event \""..val.."\" in "..filter_type..": "..source) - end - end - - function cb(node) - if node.left.type == "FieldName" and - (node.left.value == "evt.type" or - node.left.value == "syscall.type") then - - if (node.operator == "in" or - node.operator == "intersects" or - node.operator == "pmatch") then - for i, v in ipairs(node.right.elements) do - if v.type == "BareString" then - if node.left.value == "evt.type" then - check_event(v.value) - else - check_syscall(v.value) - end - end - end - else - if node.right.type == "BareString" then - if node.left.value == "evt.type" then - check_event(node.right.value) - else - check_syscall(node.right.value) - end - end - end - end - end - - parser.traverse_ast(ast, {BinaryRelOp=1}, cb) -end - --- Examine the ast and find the event types/syscalls for which the --- rule should run. All evt.type references are added as event types --- up until the first "!=" binary operator or unary not operator. If --- no event type checks are found afterward in the rule, the rule is --- considered optimized and is associated with the event type(s). --- --- Otherwise, the rule is associated with a 'catchall' category and is --- run for all event types/syscalls. (Also, a warning is printed). --- - -function sinsp_rule_utils.get_evttypes_syscalls(name, ast, source, warn_evttypes, verbose) - - local evttypes = {} - local syscallnums = {} - local evtnames = {} - local found_event = false - local found_not = false - local found_event_after_not = false - - function cb(node) - if node.type == "UnaryBoolOp" then - if node.operator == "not" then - found_not = true - end - else - if node.operator == "!=" then - found_not = true - end - if node.left.type == "FieldName" and node.left.value == "evt.type" then - found_event = true - if found_not then - found_event_after_not = true - end - if (node.operator == "in" or - node.operator == "intersects" or - node.operator == "pmatch") then - for i, v in ipairs(node.right.elements) do - if v.type == "BareString" then - - -- The event must be a known event - if events[v.value] == nil and syscalls[v.value] == nil then - error("Unknown event/syscall \""..v.value.."\" in filter: "..source) - end - - evtnames[v.value] = 1 - if events[v.value] ~= nil then - for id in string.gmatch(events[v.value], "%S+") do - evttypes[id] = 1 - end - end - - if syscalls[v.value] ~= nil then - for id in string.gmatch(syscalls[v.value], "%S+") do - syscallnums[id] = 1 - end - end - end - end - else - if node.right.type == "BareString" then - - -- The event must be a known event - if events[node.right.value] == nil and syscalls[node.right.value] == nil then - error("Unknown event/syscall \""..node.right.value.."\" in filter: "..source) - end - - evtnames[node.right.value] = 1 - if events[node.right.value] ~= nil then - for id in string.gmatch(events[node.right.value], "%S+") do - evttypes[id] = 1 - end - end - - if syscalls[node.right.value] ~= nil then - for id in string.gmatch(syscalls[node.right.value], "%S+") do - syscallnums[id] = 1 - end - end - end - end - end - end - end - - parser.traverse_ast(ast.filter.value, {BinaryRelOp=1, UnaryBoolOp=1} , cb) - - if not found_event then - if warn_evttypes == true then - io.stderr:write("Rule "..name..": warning (no-evttype):\n") - io.stderr:write(source.."\n") - io.stderr:write(" did not contain any evt.type restriction, meaning it will run for all event types.\n") - io.stderr:write(" This has a significant performance penalty. Consider adding an evt.type restriction if possible.\n") - end - evttypes = {} - syscallnums = {} - evtnames = {} - end - - if found_event_after_not then - if warn_evttypes == true then - io.stderr:write("Rule "..name..": warning (trailing-evttype):\n") - io.stderr:write(source.."\n") - io.stderr:write(" does not have all evt.type restrictions at the beginning of the condition,\n") - io.stderr:write(" or uses a negative match (i.e. \"not\"/\"!=\") for some evt.type restriction.\n") - io.stderr:write(" This has a performance penalty, as the rule can not be limited to specific event types.\n") - io.stderr:write(" Consider moving all evt.type restrictions to the beginning of the rule and/or\n") - io.stderr:write(" replacing negative matches with positive matches if possible.\n") - end - evttypes = {} - syscallnums = {} - evtnames = {} - end - - evtnames_only = {} - local num_evtnames = 0 - for name, dummy in pairs(evtnames) do - table.insert(evtnames_only, name) - num_evtnames = num_evtnames + 1 - end - - if num_evtnames == 0 then - table.insert(evtnames_only, "all") - end - - table.sort(evtnames_only) - - if verbose then - io.stderr:write("Event types/Syscalls for rule "..name..": "..table.concat(evtnames_only, ",").."\n") - end - - return evttypes, syscallnums -end - -return sinsp_rule_utils From 6fbcfd92d3336ea8a5aabd865ab22d1192d07a79 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 26 Aug 2021 11:10:43 -0700 Subject: [PATCH 12/18] Add a default ruleset version of evttypes_for_ruleset This allows for working with the default ruleset like other methods. Signed-off-by: Mark Stemm --- userspace/engine/falco_engine.cpp | 5 +++++ userspace/engine/falco_engine.h | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 39ab9f0bce9..4f04aba76d2 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -274,6 +274,11 @@ void falco_engine::evttypes_for_ruleset(std::string &source, std::set } +void falco_engine::evttypes_for_ruleset(std::string &source, std::set &evttypes) +{ + evttypes_for_ruleset(source, evttypes, m_default_ruleset); +} + std::shared_ptr falco_engine::create_formatter(const std::string &source, const std::string &output) { diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index 352bcb672fb..570df79139d 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -204,6 +204,10 @@ class falco_engine : public falco_common std::set &evttypes, const std::string &ruleset); + // Assuming default ruleset + void evttypes_for_ruleset(std::string &source, + std::set &evttypes); + // // Given a source and output string, return an // gen_event_formatter that can format output strings for an From 36f689b2a187539ce1a55247c5f38e42d69063e1 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Thu, 26 Aug 2021 11:16:38 -0700 Subject: [PATCH 13/18] Check for ignored syscall event types after loading rules This step used to be done in the lua rule loading code, but now we can get it directly from the filters, so do it in falco instead. Signed-off-by: Mark Stemm --- userspace/falco/falco.cpp | 50 ++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/userspace/falco/falco.cpp b/userspace/falco/falco.cpp index 1397aa17d1a..87bef374bd6 100644 --- a/userspace/falco/falco.cpp +++ b/userspace/falco/falco.cpp @@ -59,6 +59,9 @@ bool g_reopen_outputs = false; bool g_restart = false; bool g_daemonized = false; +static std::string syscall_source = "syscall"; +static std::string k8s_audit_source = "k8s_audit"; + // // Helper functions // @@ -259,8 +262,6 @@ uint64_t do_inspect(falco_engine *engine, uint64_t duration_start = 0; uint32_t timeouts_since_last_success_or_msg = 0; - std::string syscall_source = "syscall"; - sdropmgr.init(inspector, outputs, config.m_syscall_evt_drop_actions, @@ -427,10 +428,40 @@ static void print_all_ignored_events(sinsp *inspector) printf("\n"); } +static void check_for_ignored_events(sinsp &inspector, falco_engine &engine) +{ + std::set evttypes; + sinsp_evttables* einfo = inspector.get_event_info_tables(); + const struct ppm_event_info* etable = einfo->m_event_info; + + engine.evttypes_for_ruleset(syscall_source, evttypes); + + // Save event names so we don't warn for both the enter and exit event. + std::set warn_event_names; + + for(auto evtnum : evttypes) + { + if(evtnum == PPME_GENERIC_E || evtnum == PPME_GENERIC_X) + { + continue; + } + + if(!sinsp::simple_consumer_consider_evtnum(evtnum)) + { + std::string name = etable[evtnum].name; + if(warn_event_names.find(name) == warn_event_names.end()) + { + printf("Loaded rules use event %s, but this event is not returned unless running falco with -A\n", name.c_str()); + warn_event_names.insert(name); + } + } + } +} + static void list_source_fields(falco_engine *engine, bool names_only, std::string &source) { if(source.size() > 0 && - !(source == "syscall" || source == "k8s_audit")) + !(source == syscall_source || source == k8s_audit_source)) { throw std::invalid_argument("Value for --list must be \"syscall\" or \"k8s_audit\""); } @@ -782,8 +813,6 @@ int falco_init(int argc, char **argv) std::shared_ptr syscall_formatter_factory(new sinsp_evt_formatter_factory(inspector)); std::shared_ptr k8s_audit_formatter_factory(new json_event_formatter_factory(k8s_audit_filter_factory)); - string syscall_source = "syscall"; - string k8s_audit_source = "k8s_audit"; engine->add_source(syscall_source, syscall_filter_factory, syscall_formatter_factory); engine->add_source(k8s_audit_source, k8s_audit_filter_factory, k8s_audit_formatter_factory); @@ -798,15 +827,15 @@ int falco_init(int argc, char **argv) auto it = disable_sources.begin(); while(it != disable_sources.end()) { - if(*it != "syscall" && *it != "k8s_audit") + if(*it != syscall_source && *it != k8s_audit_source) { it = disable_sources.erase(it); continue; } ++it; } - disable_syscall = disable_sources.count("syscall") > 0; - disable_k8s_audit = disable_sources.count("k8s_audit") > 0; + disable_syscall = disable_sources.count(syscall_source) > 0; + disable_k8s_audit = disable_sources.count(k8s_audit_source) > 0; if (disable_syscall && disable_k8s_audit) { throw std::invalid_argument("The event source \"syscall\" and \"k8s_audit\" can not be disabled together"); } @@ -967,6 +996,11 @@ int falco_init(int argc, char **argv) engine->enable_rule_by_tag(enabled_rule_tags, true); } + // For syscalls, see if any event types used by the + // loaded rules are ones with the EF_DROP_SIMPLE_CONS + // label. + check_for_ignored_events(*inspector, *engine); + if(print_support) { nlohmann::json support; From 4f7e851617eada3e215007a2eaf3a1e597215e6f Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 4 Oct 2021 10:56:50 -0700 Subject: [PATCH 14/18] Update falcosecurity-libs cmake revision This has recent changes to support more general purpose event formatting. Signed-off-by: Mark Stemm --- cmake/modules/falcosecurity-libs.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/modules/falcosecurity-libs.cmake b/cmake/modules/falcosecurity-libs.cmake index 8defd40b4fc..6b488fca4f0 100644 --- a/cmake/modules/falcosecurity-libs.cmake +++ b/cmake/modules/falcosecurity-libs.cmake @@ -20,8 +20,8 @@ file(MAKE_DIRECTORY ${FALCOSECURITY_LIBS_CMAKE_WORKING_DIR}) # default below In case you want to test against another falcosecurity/libs version just pass the variable - ie., `cmake # -DFALCOSECURITY_LIBS_VERSION=dev ..` if(NOT FALCOSECURITY_LIBS_VERSION) - set(FALCOSECURITY_LIBS_VERSION "3aa7a83bf7b9e6229a3824e3fd1f4452d1e95cb4") - set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=1edb535b3778fcfb46bbeeda891f176a1bd591bebd7b89c27f04837e55a52beb") + set(FALCOSECURITY_LIBS_VERSION "a03ccfda795f2ba711b80f69cb06869f2b63121b") + set(FALCOSECURITY_LIBS_CHECKSUM "SHA256=97ce5c30b985b77e1abb04ef7037c69be176cbe2122acf67a7f6ec6b39dbdc27") endif() # cd /path/to/build && cmake /path/to/source From ffd9ddafb5f2d9758bbda53c16df07def1fe3820 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 4 Oct 2021 11:37:56 -0700 Subject: [PATCH 15/18] Update falco engine checksum This makes the output of --list a bit more precise to only include filter fields and not output fields. Signed-off-by: Mark Stemm --- userspace/engine/falco_engine_version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/engine/falco_engine_version.h b/userspace/engine/falco_engine_version.h index c0c765b5d5f..ce45e5c4df7 100644 --- a/userspace/engine/falco_engine_version.h +++ b/userspace/engine/falco_engine_version.h @@ -21,4 +21,4 @@ limitations under the License. // This is the result of running "falco --list -N | sha256sum" and // represents the fields supported by this version of falco. It's used // at build time to detect a changed set of fields. -#define FALCO_FIELDS_CHECKSUM "2f324e2e66d4b423f53600e7e0fcf2f0ff72e4a87755c490f2ae8f310aba9386" +#define FALCO_FIELDS_CHECKSUM "3153c620bf3640fab536a2d0e42aa69f10482d741ef3fa72fd8d994b0261ec55" From 498b8bce7fcb86015aa6e83259d6bd64208e3132 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 4 Oct 2021 17:29:06 -0700 Subject: [PATCH 16/18] Update automated tests to reflect evttypes behavior With the changes in https://github.com/falcosecurity/libs/pull/74, there isn't any need to warn about the order of operators and the evt.type field--the set of event types for a filter should be exact now regardless of the order of operators. So update tests that were logging those warnings to note that the warnings won't occur any more. Also, some tests more accurately *do* note that they have an overly permissive evttype (e.g. ones related to syscalls, which are uncommon and are evaluated for all event types) to reflect the new behavior. Finally, in unit tests create an actual sinsp filter instead of a gen_event_filter, which is the base class and shouldn't be created directly. Signed-off-by: Mark Stemm --- test/falco_tests.yaml | 19 +++++----------- test/rules/rule_append.yaml | 4 ++-- tests/engine/test_rulesets.cpp | 41 +++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/test/falco_tests.yaml b/test/falco_tests.yaml index df0c19fa509..efd576524da 100644 --- a/test/falco_tests.yaml +++ b/test/falco_tests.yaml @@ -32,20 +32,10 @@ trace_files: !mux - leading_not - not_equals_at_end - not_at_end - - not_before_trailing_evttype - - not_equals_before_trailing_evttype - not_equals_and_not - - not_equals_before_in - - not_before_in - - not_in_before_in - - leading_in_not_equals_before_evttype - leading_in_not_equals_at_evttype - not_with_evttypes - not_with_evttypes_addl - - not_equals_before_evttype - - not_equals_before_in_evttype - - not_before_evttype - - not_before_evttype_using_in rules_events: - no_warnings: [execve] - no_evttype: [all] @@ -1142,6 +1132,8 @@ trace_files: !mux detect_level: INFO rules_file: - rules/syscalls.yaml + rules_warning: + - detect_madvise detect_counts: - detect_madvise: 2 - detect_open: 2 @@ -1160,7 +1152,8 @@ trace_files: !mux skip_unknown_noevt: detect: False - stdout_contains: Skipping rule "Contains Unknown Event And Skipping". contains unknown filter proc.nobody + rules_warning: + - Contains Unknown Event And Skipping rules_file: - rules/skip_unknown_evt.yaml trace_file: trace_files/cat_write.scap @@ -1175,7 +1168,7 @@ trace_files: !mux exit_status: 1 stderr_contains: |+ Could not load rules file.*skip_unknown_error.yaml: 1 errors: - rule "Contains Unknown Event And Not Skipping". contains unknown filter proc.nobody + Rule Contains Unknown Event And Not Skipping: error filter_check called with nonexistent field proc.nobody --- - rule: Contains Unknown Event And Not Skipping desc: Contains an unknown event @@ -1192,7 +1185,7 @@ trace_files: !mux exit_status: 1 stderr_contains: |+ Could not load rules file .*skip_unknown_unspec.yaml: 1 errors: - rule "Contains Unknown Event And Unspecified". contains unknown filter proc.nobody + Rule Contains Unknown Event And Unspecified: error filter_check called with nonexistent field proc.nobody --- - rule: Contains Unknown Event And Unspecified desc: Contains an unknown event diff --git a/test/rules/rule_append.yaml b/test/rules/rule_append.yaml index 5441947ffcb..0a289f6e969 100644 --- a/test/rules/rule_append.yaml +++ b/test/rules/rule_append.yaml @@ -16,10 +16,10 @@ # - rule: my_rule desc: A process named cat does an open - condition: evt.type=open and fd.name=not-a-real-file + condition: (evt.type=open and fd.name=not-a-real-file) output: "An open of /dev/null was seen (command=%proc.cmdline)" priority: WARNING - rule: my_rule append: true - condition: or fd.name=/dev/null + condition: or (evt.type=open and fd.name=/dev/null) diff --git a/tests/engine/test_rulesets.cpp b/tests/engine/test_rulesets.cpp index 45e93431b86..5fbdff99e12 100644 --- a/tests/engine/test_rulesets.cpp +++ b/tests/engine/test_rulesets.cpp @@ -26,10 +26,21 @@ static uint16_t non_default_ruleset = 3; static uint16_t other_non_default_ruleset = 2; static std::set tags = {"some_tag", "some_other_tag"}; +static std::shared_ptr create_filter() +{ + // The actual contents of the filters don't matter here. + sinsp_filter_compiler compiler(NULL, "evt.type=open"); + sinsp_filter *f = compiler.compile(); + + std::shared_ptr ret(f); + + return ret; +} + TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -44,7 +55,7 @@ TEST_CASE("Should enable/disable for exact match w/ default ruleset", "[rulesets TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -63,7 +74,7 @@ TEST_CASE("Should enable/disable for exact match w/ specific ruleset", "[ruleset TEST_CASE("Should not enable for exact match different rule name", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -75,7 +86,7 @@ TEST_CASE("Should not enable for exact match different rule name", "[rulesets]") TEST_CASE("Should enable/disable for exact match w/ substring and default ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -90,7 +101,7 @@ TEST_CASE("Should enable/disable for exact match w/ substring and default rulese TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -102,7 +113,7 @@ TEST_CASE("Should not enable for substring w/ exact_match", "[rulesets]") TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -117,7 +128,7 @@ TEST_CASE("Should enable/disable for prefix match w/ default ruleset", "[ruleset TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -132,7 +143,7 @@ TEST_CASE("Should enable/disable for suffix match w/ default ruleset", "[ruleset TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -147,7 +158,7 @@ TEST_CASE("Should enable/disable for substring match w/ default ruleset", "[rule TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; r.add(rule_name, tags, filter); @@ -166,7 +177,7 @@ TEST_CASE("Should enable/disable for substring match w/ specific ruleset", "[rul TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; std::set want_tags = {"some_tag"}; @@ -182,7 +193,7 @@ TEST_CASE("Should enable/disable for tags w/ default ruleset", "[rulesets]") TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; std::set want_tags = {"some_tag"}; @@ -202,7 +213,7 @@ TEST_CASE("Should enable/disable for tags w/ specific ruleset", "[rulesets]") TEST_CASE("Should not enable for different tags", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; std::set want_tags = {"some_different_tag"}; @@ -215,7 +226,7 @@ TEST_CASE("Should not enable for different tags", "[rulesets]") TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]") { falco_ruleset r; - std::shared_ptr filter(new gen_event_filter()); + std::shared_ptr filter = create_filter(); string rule_name = "one_rule"; std::set want_tags = {"some_tag", "some_different_tag"}; @@ -231,12 +242,12 @@ TEST_CASE("Should enable/disable for overlapping tags", "[rulesets]") TEST_CASE("Should enable/disable for incremental adding tags", "[rulesets]") { falco_ruleset r; - std::shared_ptr rule1_filter(new gen_event_filter()); + std::shared_ptr rule1_filter = create_filter(); string rule1_name = "one_rule"; std::set rule1_tags = {"rule1_tag"}; r.add(rule1_name, rule1_tags, rule1_filter); - std::shared_ptr rule2_filter(new gen_event_filter()); + std::shared_ptr rule2_filter = create_filter(); string rule2_name = "two_rule"; std::set rule2_tags = {"rule2_tag"}; r.add(rule2_name, rule2_tags, rule2_filter); From 71d9fd019963b113a3b60e92fc07e5ae7ce9ee24 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 4 Oct 2021 17:31:25 -0700 Subject: [PATCH 17/18] Fix bug in macro that was masked by old evttype checking It turns out that the macro inbound_outbound had a logical bug where joining the beginning and end of the macro with "or" led to the macro matching all event types by accident. Most of the time this isn't harmful but it turns out some trace files will do operations on inet connection fds like "dup", and those get mistakenly picked up by this macro, as the fd for the event does happen to be a network connection fd. This fixes the macro to only match those event types *and* when the fd is a inet connection fd. Signed-off-by: Mark Stemm --- rules/falco_rules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/falco_rules.yaml b/rules/falco_rules.yaml index 685ade36041..5c1e8679afe 100644 --- a/rules/falco_rules.yaml +++ b/rules/falco_rules.yaml @@ -333,7 +333,7 @@ # for efficiency. - macro: inbound_outbound condition: > - ((((evt.type in (accept,listen,connect) and evt.dir=<)) or + ((((evt.type in (accept,listen,connect) and evt.dir=<)) and (fd.typechar = 4 or fd.typechar = 6)) and (fd.ip != "0.0.0.0" and fd.net != "127.0.0.0/8") and (evt.rawres >= 0 or evt.res = EINPROGRESS)) From 22c8c288c4b217560d03b3795d7d1466cca8d2a8 Mon Sep 17 00:00:00 2001 From: Mark Stemm Date: Mon, 11 Oct 2021 16:24:40 -0700 Subject: [PATCH 18/18] Change expected result for old trace file with old execve event num The trace file traces-positive/run-shell-untrusted.scap has an old execve event number (PPME_SYSCALL_EXECVE_18), which was replaced by PPME_SYSCALL_EXECVE_19 in 2018. Given the changes in https://github.com/falcosecurity/libs/pull/94, these events are now skipped. So change the test to note that *no* events will be detected. As a bit of context, event numbers won't be changing any longer--a change around the same time 298fbde8029020ce3fbddd07e2910b59cc402b8b allowed for extending existing events to add new parameters instead of having to define a new event number just to add a new parameter. So the notion of "old events" should not exist for any event created after mid-to-late 2018. Signed-off-by: Mark Stemm --- test/falco_traces.yaml.in | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/falco_traces.yaml.in b/test/falco_traces.yaml.in index a45ab6eba8e..eb757808612 100644 --- a/test/falco_traces.yaml.in +++ b/test/falco_traces.yaml.in @@ -111,12 +111,17 @@ traces: !mux detect_counts: - "Read sensitive file untrusted": 1 + # This should *not* generate any falco alerts as of the changes in + # https://github.com/falcosecurity/libs/pull/94--the execve event in + # this trace file is PPME_SYSCALL_EXECVE_18, which was deprecated by + # PPME_SYSCALL_EXECVE_19 in 2018. + # + # This activity in this trace file overlaps with the activity in + # falco-event-generator.scap so the rule is still being tested. run-shell-untrusted: trace_file: traces-positive/run-shell-untrusted.scap - detect: True + detect: False detect_level: DEBUG - detect_counts: - - "Run shell untrusted": 1 system-binaries-network-activity: trace_file: traces-positive/system-binaries-network-activity.scap