Merge pull request #2732 from DataDog/appsec-reuse-rules-when-asm-dd-…

…is-empty

[APPSEC-8867] Appsec reuse rules when ASM_DD is empty

lib/datadog/appsec/component.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative 'processor'
+require_relative 'processor/rule_merger'
+require_relative 'processor/rule_loader'
 
 module Datadog
   module AppSec
@@ -10,14 +12,27 @@ class << self
         def build_appsec_component(settings)
           return unless settings.respond_to?(:appsec) && settings.appsec.enabled
 
-          processor = create_processor
+          processor = create_processor(settings)
           new(processor: processor)
         end
 
         private
 
-        def create_processor
-          processor = Processor.new
+        def create_processor(settings)
+          rules = AppSec::Processor::RuleLoader.load_rules(ruleset: settings.appsec.ruleset)
+          return nil unless rules
+
+          data = AppSec::Processor::RuleLoader.load_data(
+            ip_denylist: settings.appsec.ip_denylist,
+            user_id_denylist: settings.appsec.user_id_denylist
+          )
+
+          ruleset = AppSec::Processor::RuleMerger.merge(
+            rules: [rules],
+            data: data,
+          )
+
+          processor = Processor.new(ruleset: ruleset)
           return nil unless processor.ready?
 
           processor

lib/datadog/appsec/processor.rb

@@ -1,5 +1,4 @@
-require_relative 'assets'
-require_relative 'processor/rule_merger'
+# frozen_string_literal: true
 
 module Datadog
   module AppSec
@@ -64,18 +63,18 @@ class AlreadyActiveContextError < StandardError; end
 
       attr_reader :ruleset_info, :addresses
 
-      def initialize(ruleset: nil)
+      def initialize(ruleset:)
         @ruleset_info = nil
         @addresses = []
         settings = Datadog::AppSec.settings
 
-        unless load_libddwaf && load_ruleset(settings, ruleset: ruleset) && create_waf_handle(settings)
+        unless load_libddwaf && create_waf_handle(settings, ruleset)
           Datadog.logger.warn { 'AppSec is disabled, see logged errors above' }
         end
       end
 
       def ready?
-        !@ruleset.nil? && !@handle.nil?
+        !@handle.nil?
       end
 
       def new_context
@@ -109,74 +108,20 @@ def finalize
 
       private
 
-      def combine_denylist_data(settings)
-        data = []
-        data << { 'rules_data' => [denylist_data('blocked_ips', settings.ip_denylist)] }
-        data << { 'rules_data' => [denylist_data('blocked_users', settings.user_id_denylist)] }
-      end
-
-      def denylist_data(id, denylist)
-        {
-          'id' => id,
-          'type' => 'data_with_expiration',
-          'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
-        }
-      end
-
       def load_libddwaf
         Processor.require_libddwaf && Processor.libddwaf_provides_waf?
       end
 
-      def load_ruleset(settings, ruleset: nil)
-        ruleset_setting = ruleset || settings.ruleset
-
-        ruleset = begin
-          case ruleset_setting
-          when :recommended, :strict
-            JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset_setting))
-          when :risky
-            Datadog.logger.warn(
-              'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
-              'The `:recommended` ruleset will be used instead.'\
-              'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
-            )
-            JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
-          when String
-            JSON.parse(File.read(ruleset_setting))
-          when File, StringIO
-            JSON.parse(ruleset_setting.read || '').tap { ruleset_setting.rewind }
-          when Hash
-            ruleset_setting
-          else
-            raise ArgumentError, "unsupported value for ruleset setting: #{ruleset_setting.inspect}"
-          end
-        rescue StandardError => e
-          Datadog.logger.error do
-            "libddwaf ruleset failed to load, ruleset: #{ruleset_setting.inspect} error: #{e.inspect}"
-          end
-
-          nil
-        end
-
-        return false if ruleset.nil?
-
-        @ruleset = RuleMerger.merge(
-          rules: [ruleset],
-          data: combine_denylist_data(settings)
-        )
-
-        true
-      end
-
-      def create_waf_handle(settings)
+      def create_waf_handle(settings, ruleset)
         # TODO: this may need to be reset if the main Datadog logging level changes after initialization
         Datadog::AppSec::WAF.logger = Datadog.logger if Datadog.logger.debug? && settings.waf_debug
 
         obfuscator_config = {
           key_regex: settings.obfuscator_key_regex,
           value_regex: settings.obfuscator_value_regex,
         }
-        @handle = Datadog::AppSec::WAF::Handle.new(@ruleset, obfuscator: obfuscator_config)
+
+        @handle = Datadog::AppSec::WAF::Handle.new(ruleset, obfuscator: obfuscator_config)
         @ruleset_info = @handle.ruleset_info
         @addresses = @handle.required_addresses
 

lib/datadog/appsec/processor/rule_loader.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../assets'
+
+module Datadog
+  module AppSec
+    class Processor
+      # RuleLoader utility modules
+      # that load appsec rules and data from  settings
+      module RuleLoader
+        class << self
+          def load_rules(ruleset:)
+            begin
+              case ruleset
+              when :recommended, :strict
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(ruleset))
+              when :risky
+                Datadog.logger.warn(
+                  'The :risky Application Security Management ruleset has been deprecated and no longer available.'\
+                  'The `:recommended` ruleset will be used instead.'\
+                  'Please remove the `appsec.ruleset = :risky` setting from your Datadog.configure block.'
+                )
+                JSON.parse(Datadog::AppSec::Assets.waf_rules(:recommended))
+              when String
+                JSON.parse(File.read(File.expand_path(ruleset)))
+              when File, StringIO
+                JSON.parse(ruleset.read || '').tap { ruleset.rewind }
+              when Hash
+                ruleset
+              else
+                raise ArgumentError, "unsupported value for ruleset setting: #{ruleset.inspect}"
+              end
+            rescue StandardError => e
+              Datadog.logger.error do
+                "libddwaf ruleset failed to load, ruleset: #{ruleset.inspect} error: #{e.inspect}"
+              end
+
+              nil
+            end
+          end
+
+          def load_data(ip_denylist: [], user_id_denylist: [])
+            data = []
+            data << { 'rules_data' => [denylist_data('blocked_ips', ip_denylist)] } if ip_denylist.any?
+            data << { 'rules_data' => [denylist_data('blocked_users', user_id_denylist)] } if user_id_denylist.any?
+
+            data.any? ? data : nil
+          end
+
+          private
+
+          def denylist_data(id, denylist)
+            {
+              'id' => id,
+              'type' => 'data_with_expiration',
+              'data' => denylist.map { |v| { 'value' => v.to_s, 'expiration' => 2**63 } }
+            }
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/core/remote/client.rb

@@ -5,6 +5,7 @@
 require_relative 'configuration'
 require_relative 'dispatcher'
 require_relative '../../appsec/processor/rule_merger'
+require_relative '../../appsec/processor/rule_loader'
 
 module Datadog
   module Core
@@ -101,6 +102,7 @@ def sync
         # rubocop:enable Metrics/AbcSize,Metrics/PerceivedComplexity
 
         class SyncError < StandardError; end
+        class ReadError < StandardError; end
 
         private
 
@@ -189,24 +191,6 @@ def capabilities_binary
           cap_to_hexs.each_with_object([]) { |hex, acc| acc << hex }.map { |e| e.to_i(16) }.pack('C*')
         end
 
-        def select_content(repository, product, config_ids = [])
-          repository.contents.select do |content|
-            content.path.product == product && (config_ids.empty? || config_ids.include?(content.path.config_id))
-          end
-        end
-
-        def parse_content(content)
-          data = content.data.read
-
-          content.data.rewind
-
-          raise ReadError, 'EOF reached' if data.nil?
-
-          JSON.parse(data)
-        end
-
-        class ReadError < StandardError; end
-
         def register_receivers
           matcher = Dispatcher::Matcher::Product.new(products)
 
@@ -215,17 +199,33 @@ def register_receivers
               Datadog.logger.debug { "remote config change: '#{change.path}'" }
             end
 
-            rules_contents = select_content(repository, 'ASM_DD')
-            rules = rules_contents.map { |content| parse_content(content) }
+            rules = []
+            data = []
+            overrides = []
+            exclusions = []
+
+            asm_data_config_types = ['blocked_ips', 'blocked_users']
+            asm_overrides_config_types = ['blocking', 'disabled_rules']
+
+            repository.contents.each do |content|
+              case content.path.product
+              when 'ASM_DD'
+                rules << parse_content(content)
+              when 'ASM_DATA'
+                data << parse_content(content) if asm_data_config_types.include?(content.path.config_id)
+              when 'ASM'
+                overrides << parse_content(content) if asm_overrides_config_types.include?(content.path.config_id)
+                exclusions << parse_content(content) if content.path.config_id == 'exclusion_filters'
+              end
+            end
 
-            data_contents = select_content(repository, 'ASM_DATA', ['blocked_ips', 'blocked_users'])
-            data = data_contents.map { |content| parse_content(content) }
+            if rules.empty?
+              settings_rules = AppSec::Processor::RuleLoader.load_rules(ruleset: Datadog.configuration.appsec.ruleset)
 
-            overrides_contents = select_content(repository, 'ASM', ['blocking', 'disabled_rules'])
-            overrides = overrides_contents.map { |content| parse_content(content) }
+              raise SyncError, 'no default rules available' unless settings_rules
 
-            exclusions_contents = select_content(repository, 'ASM', ['exclusion_filters'])
-            exclusions = exclusions_contents.map { |content| parse_content(content) }
+              rules = [settings_rules]
+            end
 
             ruleset = AppSec::Processor::RuleMerger.merge(
               rules: rules,
@@ -237,6 +237,16 @@ def register_receivers
             Datadog::AppSec.reconfigure(ruleset: ruleset)
           end
         end
+
+        def parse_content(content)
+          data = content.data.read
+
+          content.data.rewind
+
+          raise ReadError, 'EOF reached' if data.nil?
+
+          JSON.parse(data)
+        end
       end
     end
   end

sig/datadog/appsec/processor.rbs

@@ -37,7 +37,7 @@ module Datadog
       @ruleset: ::Hash[::String, untyped]
       @addresses: ::Array[::String]
 
-      def initialize: (?ruleset: ::Hash[untyped, untyped]?) -> void
+      def initialize: (ruleset: ::Hash[untyped, untyped]) -> void
       def ready?: () -> bool
       def new_context: () -> Context
       def activate_context: () -> Context
@@ -48,12 +48,8 @@ module Datadog
 
       private
 
-      def apply_denylist_data: (Configuration::Settings settings) -> untyped
-      def combine_denylist_data: (Configuration::Settings settings) -> ::Array[::Hash[::String, untyped]]
-      def denylist_data: (String id, ::Array[untyped] denylist) -> ::Hash[::String, untyped | "data_with_expiration"]
       def load_libddwaf: () -> bool
-      def load_ruleset: (Configuration::Settings settings, ?ruleset: ::Hash[untyped, untyped]?) -> bool
-      def create_waf_handle: (Configuration::Settings settings) -> bool
+      def create_waf_handle: (Configuration::Settings settings, ::Hash[String, untyped] ruleset) -> bool
 
       def self.libddwaf_provides_waf?: () -> bool
       def self.require_libddwaf: () -> bool

sig/datadog/appsec/processor/rule_loader.rbs

@@ -0,0 +1,17 @@
+module Datadog
+  module AppSec
+    class Processor
+      module RuleLoader
+        type ruleset = Symbol | String | File | StringIO | Hash[String, untyped]
+        def self.load_rules: (ruleset: ruleset) -> ::Hash[untyped, untyped]?
+
+        def self.load_data: (?ip_denylist: Array[String], ?user_id_denylist: Array[String]) -> Array[Hash[String, untyped]]?
+
+        private
+
+        def self.denylist_data: (String id, Array[String] denylist) -> ::Hash[::String, untyped]
+      end
+    end
+  end
+end
+

spec/datadog/appsec/component_spec.rb

@@ -40,6 +40,16 @@
           expect(component.processor).to be_nil
         end
       end
+
+      context 'when loading ruleset from settings fails' do
+        it 'returns a Datadog::AppSec::Component with a nil processor' do
+          expect(Datadog::AppSec::Processor::RuleLoader).to receive(:load_rules).and_return(nil)
+
+          component = described_class.build_appsec_component(settings_with_appsec)
+
+          expect(component.processor).to be_nil
+        end
+      end
     end
 
     context 'when appsec is not enabled' do