Merge pull request #238 from fluent/support-fluentd-v1-revive

Support Fluentd v1

* Rewrite models
* Depend on fluent-plugins to fetch plugins config definitions
* Construct owned plugins(parser, formatter, buffer, storage) form dynamicaly
* Use custom FormBuilder to construct form for plugin configuration
* Add models for parser, formatter, buffer, storage plugins
* Rewrite in_tail configuration wizard

Gemfile.lock

@@ -7,6 +7,10 @@ PATH
       bundler
       diff-lcs
       draper (~> 3.0)
+      fluent-plugin-elasticsearch (~> 2.10)
+      fluent-plugin-mongo (~> 1.1)
+      fluent-plugin-s3 (~> 1.1)
+      fluent-plugin-td (~> 1.0)
       fluentd (>= 1.0.0, < 2)
       font-awesome-rails
       haml-rails (~> 1.0)
@@ -74,6 +78,24 @@ GEM
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     arel (9.0.0)
+    aws-eventstream (1.0.0)
+    aws-partitions (1.87.0)
+    aws-sdk-core (3.21.2)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.13.0)
+      aws-sdk-core (~> 3, >= 3.21.2)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-sqs (1.3.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     better_errors (2.1.1)
       coderay (>= 1.0.0)
       erubis (>= 2.6.6)
@@ -83,6 +105,7 @@ GEM
       debug_inspector (>= 0.0.1)
     bootsnap (1.3.0)
       msgpack (~> 1.0)
+    bson (4.3.0)
     builder (3.2.3)
     capybara (3.0.2)
       addressable
@@ -112,14 +135,39 @@ GEM
       activemodel-serializers-xml (~> 1.0)
       activesupport (~> 5.0)
       request_store (~> 1.0)
+    elasticsearch (6.0.2)
+      elasticsearch-api (= 6.0.2)
+      elasticsearch-transport (= 6.0.2)
+    elasticsearch-api (6.0.2)
+      multi_json
+    elasticsearch-transport (6.0.2)
+      faraday
+      multi_json
     erubi (1.7.1)
     erubis (2.7.0)
+    excon (0.62.0)
     factory_bot (4.8.2)
       activesupport (>= 3.0.0)
     factory_bot_rails (4.8.2)
       factory_bot (~> 4.8.2)
       railties (>= 3.0.0)
+    faraday (0.15.2)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.23)
+    fluent-plugin-elasticsearch (2.10.1)
+      elasticsearch
+      excon
+      fluentd (>= 0.14.20)
+    fluent-plugin-mongo (1.1.1)
+      fluentd (>= 0.14.12, < 2)
+      mongo (~> 2.2.0)
+    fluent-plugin-s3 (1.1.3)
+      aws-sdk-s3 (~> 1.0)
+      aws-sdk-sqs (~> 1.0)
+      fluentd (>= 0.14.2, < 2)
+    fluent-plugin-td (1.0.0)
+      fluentd (>= 0.14.13, < 2)
+      td-client (~> 1.0)
     fluentd (1.2.1)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
@@ -159,6 +207,7 @@ GEM
     jbuilder (2.7.0)
       activesupport (>= 4.2.0)
       multi_json (>= 1.2)
+    jmespath (1.4.0)
     json (2.1.0)
     kramdown (1.16.2)
     kramdown-haml (0.0.3)
@@ -181,8 +230,11 @@ GEM
     mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.11.3)
+    mongo (2.2.7)
+      bson (~> 4.0)
     msgpack (1.2.4)
     multi_json (1.13.1)
+    multipart-post (2.0.0)
     nio4r (2.3.0)
     nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)
@@ -286,6 +338,9 @@ GEM
     strptime (0.2.3)
     sucker_punch (2.0.4)
       concurrent-ruby (~> 1.0.0)
+    td-client (1.0.6)
+      httpclient (>= 2.7)
+      msgpack (>= 0.5.6, < 2)
     temple (0.8.0)
     thor (0.20.0)
     thread_safe (0.3.6)

app/controllers/api/config_definitions_controller.rb

@@ -0,0 +1,55 @@
+class Api::ConfigDefinitionsController < ApplicationController
+  before_action :login_required
+
+  def index
+    name = params[:name]
+    type = params[:type]
+    prefix = case type
+             when "input"
+               "in"
+             when "output"
+               "out"
+             when "filter"
+               "filter"
+             when "parse"
+               "parser"
+             when "format"
+               "formatter"
+             when "parser", "formatter", "buffer", "storage"
+               type
+             end
+
+    target_class = Fluentd::Setting.const_get("#{prefix}_#{name}".classify)
+    target = target_class.new
+
+    common_options = target.common_options.map do |key|
+      h = {
+        name: key,
+        type: target.column_type(key),
+        desc: target.desc(key),
+        default: target.default(key)
+      }
+      h[:list] = target.list_of(key) if target.column_type(key) == :enum
+      h
+    end
+
+    advanced_options = target.advanced_options.map do |key|
+      h = {
+        name: key,
+        type: target.column_type(key),
+        desc: target.desc(key),
+        default: target.default(key)
+      }
+      h[:list] = target.list_of(key) if target.column_type(key) == :enum
+      h
+    end
+
+    options = {
+      type: type,
+      name: name,
+      commonOptions: common_options,
+      advancedOptions: advanced_options
+    }
+    render json: options
+  end
+end

app/controllers/api_controller.rb

@@ -19,15 +19,32 @@ def empty_json
   end
 
   def regexp_preview
-    preview = RegexpPreview.processor(params[:format]).new(params[:file], params[:format], params)
+    plugin_config = prepare_plugin_config || {}
+    preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], plugin_config)
 
-    render json: preview.matches_json
+    render json: preview.matches
+  rescue Fluent::ConfigError => ex
+    render json: { error: "#{ex.class}: #{ex.message}" }
   end
 
   def grok_to_regexp
     grok = GrokConverter.new
     grok.load_patterns
     render text: grok.convert_to_regexp(params[:grok_str]).source
   end
-end
 
+  private
+
+  def prepare_plugin_config
+    plugin_config = params[:plugin_config]
+    case params[:parse_type]
+    when "multiline"
+      plugin_config[:formats].lines.each.with_index do |line, index|
+        plugin_config["format#{index + 1}"] = line.chomp
+      end
+      plugin_config
+    else
+      plugin_config
+    end
+  end
+end

app/controllers/concerns/setting_concern.rb

@@ -10,11 +10,19 @@ module SettingConcern
 
   def show
     @setting = target_class.new(initial_params)
+    @buffer = @setting.create_buffer
+    @storage = @setting.create_storage
+    @parser = @setting.create_parser
+    @formatter = @setting.create_formatter
+    @_used_param = {}
+    @_used_section = {}
     render "shared/settings/show"
   end
 
   def finish
     @setting = target_class.new(setting_params)
+    @_used_param = {}
+    @_used_section = {}
     unless @setting.valid?
       return render "shared/settings/show"
     end
@@ -32,15 +40,23 @@ def finish
   private
 
   def setting_params
-    params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_class.const_get(:KEYS))
+    params.require(:setting).permit(*target_class.permit_params)
   end
 
   def initial_params
     target_class.initial_params
   end
 
   def target_plugin_name
-    target_class.to_s.split("::").last.underscore
+    prefix = case target_class.plugin_type
+             when "input"
+               "in"
+             when "output"
+               "out"
+             else
+               target_class.plugin_type
+             end
+    "#{prefix}_#{target_class.plugin_name}"
   end
 
   def plugin_setting_form_action_url(*args)

app/controllers/fluentd/settings/in_tail_controller.rb

@@ -4,8 +4,8 @@ class Fluentd::Settings::InTailController < ApplicationController
 
   def after_file_choose
     @setting = Fluentd::Setting::InTail.new({
-      :path => params[:path],
-      :tag => nil,
+      path: params[:path],
+      tag: nil
     })
   end
 
@@ -38,20 +38,28 @@ def finish
       return render "after_format"
     end
 
-    if @fluentd.agent.configuration.to_s.include?(@setting.to_conf.strip)
+    if @fluentd.agent.configuration.to_s.include?(@setting.to_config.to_s.strip)
       @setting.errors.add(:base, :duplicated_conf)
       return render "after_format"
     end
 
-    @fluentd.agent.config_append @setting.to_conf
+    @fluentd.agent.config_append @setting.to_config.to_s
     @fluentd.agent.restart if @fluentd.agent.running?
     redirect_to daemon_setting_path(@fluentd)
   end
 
+  def target_class
+    Fluentd::Setting::InTail
+  end
+
   private
 
   def setting_params
-    params.require(:setting).permit(:path, :format, :regexp, *Fluentd::Setting::InTail.known_formats, :tag, :rotate_wait, :pos_file, :read_from_head, :refresh_interval)
+    permit_params = target_class._types.keys
+    permit_params << :parse_type
+    section_class = Fluentd::Setting.const_get("parser_#{params.dig(:setting, :parse_type)}".classify)
+    permit_params << { parse: section_class._types.keys }
+    params.require(:setting).permit(*permit_params)
   end
 
 end

app/controllers/fluentd/settings/out_forward_controller.rb

@@ -6,13 +6,4 @@ class Fluentd::Settings::OutForwardController < ApplicationController
   def target_class
     Fluentd::Setting::OutForward
   end
-
-  def setting_params
-    params.require(:fluentd_setting_out_forward).permit(*Fluentd::Setting::OutForward::KEYS).merge(
-      params.require(:fluentd_setting_out_forward).permit(
-        :server => Fluentd::Setting::OutForward::Server::KEYS,
-        :secondary => Fluentd::Setting::OutForward::Secondary::KEYS,
-      ),
-    )
-  end
 end

app/controllers/fluentd/settings/out_s3_controller.rb

@@ -6,9 +6,4 @@ class Fluentd::Settings::OutS3Controller < ApplicationController
   def target_class
     Fluentd::Setting::OutS3
   end
-
-  def setting_params
-    params.require(:fluentd_setting_out_s3).permit(*Fluentd::Setting::OutS3::KEYS)
-  end
-
 end

app/controllers/fluentd/settings/out_tdlog_controller.rb

@@ -0,0 +1,9 @@
+class Fluentd::Settings::OutTdlogController < ApplicationController
+  include SettingConcern
+
+  private
+
+  def target_class
+    Fluentd::Setting::OutTdlog
+  end
+end