From ac8c1fa9f8eb4d1fbdcca7980855d3d741526542 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 24 May 2018 17:55:44 +0900
Subject: [PATCH 001/137] Add custom types

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/type/array.rb   | 17 +++++++++++++++++
 app/models/fluentd/setting/type/bool.rb    | 18 ++++++++++++++++++
 app/models/fluentd/setting/type/enum.rb    | 17 +++++++++++++++++
 app/models/fluentd/setting/type/hash.rb    | 17 +++++++++++++++++
 app/models/fluentd/setting/type/section.rb | 17 +++++++++++++++++
 app/models/fluentd/setting/type/size.rb    | 17 +++++++++++++++++
 app/models/fluentd/setting/type/time.rb    | 17 +++++++++++++++++
 config/initializers/types.rb               |  6 ++++++
 8 files changed, 126 insertions(+)
 create mode 100644 app/models/fluentd/setting/type/array.rb
 create mode 100644 app/models/fluentd/setting/type/bool.rb
 create mode 100644 app/models/fluentd/setting/type/enum.rb
 create mode 100644 app/models/fluentd/setting/type/hash.rb
 create mode 100644 app/models/fluentd/setting/type/section.rb
 create mode 100644 app/models/fluentd/setting/type/size.rb
 create mode 100644 app/models/fluentd/setting/type/time.rb
 create mode 100644 config/initializers/types.rb

diff --git a/app/models/fluentd/setting/type/array.rb b/app/models/fluentd/setting/type/array.rb
new file mode 100644
index 000000000..e032fc449
--- /dev/null
+++ b/app/models/fluentd/setting/type/array.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Array < ActiveModel::Type::Value
+        def type
+          :array
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/type/bool.rb b/app/models/fluentd/setting/type/bool.rb
new file mode 100644
index 000000000..b953d9be4
--- /dev/null
+++ b/app/models/fluentd/setting/type/bool.rb
@@ -0,0 +1,18 @@
+class Fluentd
+  module Setting
+    module Type
+      class Bool < ActiveModel::Type::Value
+        def type
+          :bool
+        end
+
+        private
+
+        def cast_value(value)
+          # TODO Use type converter method of Fluentd
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/type/enum.rb b/app/models/fluentd/setting/type/enum.rb
new file mode 100644
index 000000000..c2f2bd8fc
--- /dev/null
+++ b/app/models/fluentd/setting/type/enum.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Enum < ActiveModel::Type::Value
+        def type
+          :enum
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/type/hash.rb b/app/models/fluentd/setting/type/hash.rb
new file mode 100644
index 000000000..cbdfbeb06
--- /dev/null
+++ b/app/models/fluentd/setting/type/hash.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Hash < ActiveModel::Type::Value
+        def type
+          :hash
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/type/section.rb b/app/models/fluentd/setting/type/section.rb
new file mode 100644
index 000000000..2ece24d32
--- /dev/null
+++ b/app/models/fluentd/setting/type/section.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Section < ActiveModel::Type::Value
+        def type
+          :section
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/type/size.rb b/app/models/fluentd/setting/type/size.rb
new file mode 100644
index 000000000..46d8dd7aa
--- /dev/null
+++ b/app/models/fluentd/setting/type/size.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Size < ActiveModel::Type::Value
+        def type
+          :size
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/type/time.rb b/app/models/fluentd/setting/type/time.rb
new file mode 100644
index 000000000..80f1796b9
--- /dev/null
+++ b/app/models/fluentd/setting/type/time.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Time < ActiveModel::Type::Value
+        def type
+          :fluentd_time
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/types.rb b/config/initializers/types.rb
new file mode 100644
index 000000000..598f80e8c
--- /dev/null
+++ b/config/initializers/types.rb
@@ -0,0 +1,6 @@
+ActiveModel::Type.register(:array, Fluentd::Setting::Type::Array)
+ActiveModel::Type.register(:enum, Fluentd::Setting::Type::Enum)
+ActiveModel::Type.register(:bool, Fluentd::Setting::Type::Bool)
+ActiveModel::Type.register(:hash, Fluentd::Setting::Type::Hash)
+ActiveModel::Type.register(:size, Fluentd::Setting::Type::Size)
+ActiveModel::Type.register(:section, Fluentd::Setting::Type::Section)

From 4f2a06fd82edec3c0707215e62747e30ec4a996c Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 24 May 2018 18:08:33 +0900
Subject: [PATCH 002/137] Add concerns to define models for fluent-plugin

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../concerns/fluentd/setting/configurable.rb  | 103 ++++++++++++++++++
 .../concerns/fluentd/setting/pattern.rb       |  11 ++
 app/models/concerns/fluentd/setting/plugin.rb |  75 +++++++++++++
 .../concerns/fluentd/setting/plugin_config.rb |  57 ++++++++++
 .../fluentd/setting/plugin_parameter.rb       |  43 ++++++++
 .../fluentd/setting/section_parser.rb         |  21 ++++
 .../fluentd/setting/section_validator.rb      |  21 ++++
 7 files changed, 331 insertions(+)
 create mode 100644 app/models/concerns/fluentd/setting/configurable.rb
 create mode 100644 app/models/concerns/fluentd/setting/pattern.rb
 create mode 100644 app/models/concerns/fluentd/setting/plugin.rb
 create mode 100644 app/models/concerns/fluentd/setting/plugin_config.rb
 create mode 100644 app/models/concerns/fluentd/setting/plugin_parameter.rb
 create mode 100644 app/models/concerns/fluentd/setting/section_parser.rb
 create mode 100644 app/models/concerns/fluentd/setting/section_validator.rb

diff --git a/app/models/concerns/fluentd/setting/configurable.rb b/app/models/concerns/fluentd/setting/configurable.rb
new file mode 100644
index 000000000..a9fdd1988
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/configurable.rb
@@ -0,0 +1,103 @@
+class Fluentd
+  module Setting
+    module Configurable
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :_types, :_defaults, :_secrets, :_aliases, :_required
+        class_attribute :_deprecated_params, :_obsoleted_params, :_descriptions
+        class_attribute :_list, :_value_types, :_symbolize_keys
+        class_attribute :_argument_name, :_built_in_params, :_sections, :_section_params
+        self._types = {}
+        self._defaults = {}
+        self._secrets = {}
+        self._aliases = {}
+        self._required = {}
+        self._deprecated_params = {}
+        self._obsoleted_params = {}
+        self._descriptions = {}
+        self._list = {}
+        self._value_types = {}
+        self._symbolize_keys = {}
+        self._argument_name = nil
+        self._built_in_params = []
+        self._sections = {}
+        self._section_params = Hash.new {|h, k| h[k] = [] }
+      end
+
+      def initialize(attributes = {})
+        super rescue ActiveModel::UnknownAttributeError # the superclass does not know specific attributes of the model
+        self.class._sections.each do |name, klass|
+          klass.init
+          if klass.multi
+            next if attributes[name].nil?
+            attributes[name].each do |attr|
+              next unless attr
+              attr.each do |index, _attr|
+                self._section_params[name] << klass.new(_attr)
+              end
+            end
+          else
+            attr = attributes.dig(name, "0")
+            self._section_params[name] << klass.new(attr) if attr
+          end
+        end
+      end
+
+      module ClassMethods
+        # config_param :name, :string, default: "value", secret: true
+        def config_param(name, type = ActiveModel::Type::Value.new, **options)
+          # NOTE: We cannot overwrite types defined by ActiveModel in config/initializers/types.rb
+          if type == :time
+            type = Fluentd::Setting::Type::Time.new
+          end
+          if name.to_s.start_with?("@")
+            _name = name.to_s[1..-1]
+            config_param(_name.to_sym, type, **options.merge(alias: name))
+            self._built_in_params << _name
+          elsif ["id", "type", "log_level"].include?(name.to_s)
+            self._built_in_params << _name
+          else
+            attribute(name, type, **options.slice(:precision, :limit, :scale))
+            validates(name, presence: true) if options[:required]
+          end
+          self._types[name] = type
+          self._defaults[name] = options[:default] if options.key?(:default)
+          self._secrets[name] = options[:secret] if options.key?(:secret)
+          self._aliases[name] = options[:alias] if options.key?(:alias)
+          self._required[name] = options[:required] if options.key?(:required)
+          self._deprecated_params[name] = options[:deprecated] if options.key?(:deprecated)
+          self._obsoleted_params[name] = options[:obsoleted] if options.key?(:obsoleted)
+          self._list[name] = options[:list] if options.key?(:list)
+          self._value_types[name] = options[:value_types] if options.key?(:value_types)
+          self._symbolize_keys = options[:symbolize_keys] if options.key?(:symbolize_keys)
+        end
+
+        def config_section(name, **options, &block)
+          if self._sections.key?(name)
+            self._sections[name].merge(**options, &block)
+          else
+            attribute(name, :section)
+            section_class = Class.new(::Fluentd::Setting::Section)
+            section_class.section_name = name
+            section_class.required = options[:required]
+            section_class.multi = options[:multi]
+            section_class.alias = options[:alias]
+            section_class._block = block
+            self.const_set(name.to_s.classify, section_class)
+            self._sections[name] = section_class
+          end
+        end
+
+        def config_argument(name, type = ActiveModel::Type::Value.new, **options)
+          config_param(name, **options)
+          self._argument_name = name
+        end
+
+        def set_argument_name(name)
+          self._argument_name = name
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fluentd/setting/pattern.rb b/app/models/concerns/fluentd/setting/pattern.rb
new file mode 100644
index 000000000..ff0b89bbb
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/pattern.rb
@@ -0,0 +1,11 @@
+class Fluentd
+  module Setting
+    module Pattern
+      extend ActiveSupport::Concern
+
+      included do
+        config_argument(:pattern, :string)
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fluentd/setting/plugin.rb b/app/models/concerns/fluentd/setting/plugin.rb
new file mode 100644
index 000000000..351765501
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/plugin.rb
@@ -0,0 +1,75 @@
+require "fluent/plugin"
+
+class Fluentd
+  module Setting
+    module Plugin
+      extend ActiveSupport::Concern
+
+      include ActiveModel::Model
+      include ActiveModel::Attributes
+      include Fluentd::Setting::Configurable
+      include Fluentd::Setting::PluginConfig
+      include Fluentd::Setting::SectionParser
+      include Fluentd::Setting::PluginParameter
+      include Fluentd::Setting::SectionValidator
+
+      included do
+        cattr_accessor :plugin_type, :plugin_name, :config_definition
+      end
+
+      module ClassMethods
+        def register_plugin(type, name)
+          self.plugin_type = type
+          self.plugin_name = name
+
+          if ["filter", "output"].include?(type)
+            include Fluentd::Setting::Pattern
+          end
+
+          self.load_plugin_config do |_name, params|
+            params.each do |param_name, definition|
+              if definition[:section]
+                parse_section(param_name, definition)
+              else
+                config_param(param_name, definition[:type], **definition.except(:type))
+              end
+            end
+          end
+        end
+
+        def load_plugin_config
+          dumped_config = {}
+          plugin_class.ancestors.reverse_each do |klass|
+            next unless klass.respond_to?(:dump_config_definition)
+            dumped_config_definition = klass.dump_config_definition
+            dumped_config[klass.name] = dumped_config_definition unless dumped_config_definition.empty?
+          end
+          self.config_definition = dumped_config
+          dumped_config.each do |name, config|
+            yield name, config
+          end
+        end
+
+        def plugin_instance
+          @plugin_instance ||= Fluent::Plugin.__send__("new_#{plugin_type}", plugin_name)
+        end
+
+        def plugin_class
+          @plugin_class ||= plugin_instance.class
+        end
+
+        def plugin_helpers
+          @plugin_helpers ||= if plugin_instance.respond_to?(:plugin_helpers)
+                                plugin_instance.plugin_helpers
+                              else
+                                []
+                              end
+        end
+
+        def section?
+          false
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb
new file mode 100644
index 000000000..c016f8b2e
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/plugin_config.rb
@@ -0,0 +1,57 @@
+class Fluentd
+  module Setting
+    module PluginConfig
+      extend ActiveSupport::Concern
+
+      def to_config
+        name = case plugin_type
+               when "input"
+                 "source"
+               when "output"
+                 "match"
+               when "filter"
+                 "filter"
+               when "parser"
+                 "parse"
+               when "formatter"
+                 "format"
+               when "buffer"
+                 "buffer"
+               end
+        _attributes = { "@type" => self.plugin_name }.merge(attributes)
+        _attributes["@log_level"] = _attributes.delete("log_level")
+        argument = case plugin_type
+                   when "output", "filter", "buffer"
+                     _attributes.delete(self._argument_name.to_s) || ""
+                   else
+                     ""
+                   end
+        attrs, elements = parse_attributes(_attributes)
+        config_element(name, argument, attrs, elements)
+      end
+
+      def parse_attributes(attributes)
+        base_klasses = config_definition.keys
+        sections, params = attributes.partition do |key, _section_attributes|
+          base_klasses.any? do |base_klass|
+            config_definition.dig(base_klass, key.to_sym, :section) || config_definition.dig(key.to_sym, :section)
+          end
+        end
+        elements = []
+        sections.to_h.each do |key, section_params|
+          next if section_params.blank?
+          section_params.each do |index, _section_params|
+            sub_attrs, sub_elements = parse_attributes(_section_params)
+            elements << config_element(key, "", sub_attrs, sub_elements)
+          end
+        end
+        return params.to_h.reject{|key, value| value.blank? }, elements
+      end
+
+      # copy from Fluent::Test::Helpers#config_element
+      def config_element(name = 'test', argument = '', params = {}, elements = [])
+        Fluent::Config::Element.new(name, argument, params, elements)
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
new file mode 100644
index 000000000..2100541a4
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -0,0 +1,43 @@
+class Fluentd
+  module Setting
+    module PluginParameter
+      extend ActiveSupport::Concern
+
+      include Fluentd::Setting::Configurable
+
+      def column_type(name)
+        self.class._types[name]
+      end
+
+      def list_of(name)
+        self.class._list[name]
+      end
+
+      def common_options
+        []
+      end
+
+      def advanced_options
+        all_options - common_options - hidden_options
+      end
+
+      def hidden_options
+        []
+      end
+
+      def all_options
+        self.class._types.keys + self.class._sections.keys
+      end
+
+      module ClassMethods
+        def column_type(name)
+          self._types[name]
+        end
+
+        def list_of(name)
+          self._list[name]
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fluentd/setting/section_parser.rb b/app/models/concerns/fluentd/setting/section_parser.rb
new file mode 100644
index 000000000..6ae145cc7
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/section_parser.rb
@@ -0,0 +1,21 @@
+class Fluentd
+  module Setting
+    module SectionParser
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def parse_section(name, definition)
+          config_section(name, **definition.slice(:required, :multi, :alias)) do
+            definition.except(:section, :argument, :required, :multi, :alias).each do |_param_name, _definition|
+              if _definition[:section]
+                parse_section(_param_name, _definition)
+              else
+                config_param(_param_name, _definition[:type], **_definition.except(:type))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/fluentd/setting/section_validator.rb b/app/models/concerns/fluentd/setting/section_validator.rb
new file mode 100644
index 000000000..8238441b8
--- /dev/null
+++ b/app/models/concerns/fluentd/setting/section_validator.rb
@@ -0,0 +1,21 @@
+class Fluentd
+  module Setting
+    module SectionValidator
+      extend ActiveSupport::Concern
+
+      included do
+        validate :validate_sections
+      end
+
+      def validate_sections
+        self._section_params.each do |name, sections|
+          sections.each do |section|
+            if section.invalid?
+              errors.add(name, :invalid, message: section.errors.full_messages)
+            end
+          end
+        end
+      end
+    end
+  end
+end

From e51c43b156346adeb301089b1fbcdb0be85a9186 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 24 May 2018 18:15:10 +0900
Subject: [PATCH 003/137] Depend on fluent-plugins

We use plugins' definition to reconstruct plugins' model.

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 fluentd-ui.gemspec | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/fluentd-ui.gemspec b/fluentd-ui.gemspec
index f0d9d2425..dec9b7605 100644
--- a/fluentd-ui.gemspec
+++ b/fluentd-ui.gemspec
@@ -53,4 +53,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rubyzip", "~> 1.1" # API changed as Zip::ZipFile -> Zip::File since v1.0.0
   spec.add_dependency "diff-lcs"
   spec.add_dependency "webpacker"
+
+  spec.add_dependency "fluent-plugin-td", "~> 1.0"
+  spec.add_dependency "fluent-plugin-mongo", "~> 1.1"
+  spec.add_dependency "fluent-plugin-elasticsearch", "~> 2.10"
+  spec.add_dependency "fluent-plugin-s3", "~> 1.1"
 end

From a8401530ab127152624bf2c57ce9c452d610d0a7 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 24 May 2018 18:26:17 +0900
Subject: [PATCH 004/137] Use new style definition

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_forward.rb      | 23 +-----
 app/models/fluentd/setting/in_http.rb         | 22 +-----
 .../fluentd/setting/in_monitor_agent.rb       | 20 +----
 app/models/fluentd/setting/in_syslog.rb       | 23 +++---
 app/models/fluentd/setting/in_tail.rb         | 16 ++--
 .../fluentd/setting/out_elasticsearch.rb      | 26 ++-----
 app/models/fluentd/setting/out_forward.rb     | 73 ++-----------------
 app/models/fluentd/setting/out_mongo.rb       | 33 +++------
 app/models/fluentd/setting/out_s3.rb          | 34 ++-------
 app/models/fluentd/setting/out_stdout.rb      | 32 ++++----
 app/models/fluentd/setting/out_td.rb          | 48 ------------
 app/models/fluentd/setting/out_tdlog.rb       | 32 ++++++++
 app/models/fluentd/setting/section.rb         | 34 +++++++++
 13 files changed, 130 insertions(+), 286 deletions(-)
 delete mode 100644 app/models/fluentd/setting/out_td.rb
 create mode 100644 app/models/fluentd/setting/out_tdlog.rb
 create mode 100644 app/models/fluentd/setting/section.rb

diff --git a/app/models/fluentd/setting/in_forward.rb b/app/models/fluentd/setting/in_forward.rb
index 70627ad5a..c134bbad3 100644
--- a/app/models/fluentd/setting/in_forward.rb
+++ b/app/models/fluentd/setting/in_forward.rb
@@ -1,17 +1,9 @@
 class Fluentd
   module Setting
     class InForward
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :bind, :port, :linger_timeout, :chunk_size_limit, :chunk_size_warn_limit, :log_level
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      validates :bind, presence: true
-      validates :port, presence: true
+      register_plugin("input", "forward")
 
       def self.initial_params
         {
@@ -20,7 +12,6 @@ def self.initial_params
           linger_timeout: 0,
           chunk_size_limit: nil,
           chunk_size_warn_limit: nil,
-          log_level: "info",
         }
       end
 
@@ -29,16 +20,6 @@ def common_options
           :bind, :port
         ]
       end
-
-      def advanced_options
-        [
-          :linger_timeout, :chunk_size_limit, :chunk_size_warn_limit, :log_level
-        ]
-      end
-
-      def plugin_name
-        "forward"
-      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/in_http.rb b/app/models/fluentd/setting/in_http.rb
index 788a37dda..0ae0c8b2f 100644
--- a/app/models/fluentd/setting/in_http.rb
+++ b/app/models/fluentd/setting/in_http.rb
@@ -1,17 +1,9 @@
 class Fluentd
   module Setting
     class InHttp
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :bind, :port, :body_size_limit, :keepalive_timeout, :add_http_headers, :format, :log_level
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      validates :bind, presence: true
-      validates :port, presence: true
+      register_plugin("input", "http")
 
       def self.initial_params
         {
@@ -20,8 +12,6 @@ def self.initial_params
           body_size_limit: "32m",
           keepalive_timeout: "10s",
           add_http_headers: false,
-          format: "default",
-          log_level: "info",
         }
       end
 
@@ -31,15 +21,11 @@ def common_options
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :body_size_limit, :keepalive_timeout, :add_http_headers, :format, :log_level
+          :parse
         ]
       end
-
-      def plugin_name
-        "http"
-      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/in_monitor_agent.rb b/app/models/fluentd/setting/in_monitor_agent.rb
index 672b604a1..c9c123310 100644
--- a/app/models/fluentd/setting/in_monitor_agent.rb
+++ b/app/models/fluentd/setting/in_monitor_agent.rb
@@ -1,17 +1,9 @@
 class Fluentd
   module Setting
     class InMonitorAgent
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :bind, :port
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      validates :bind, presence: true
-      validates :port, presence: true
+      register_plugin("input", "monitor_agent")
 
       def self.initial_params
         {
@@ -25,14 +17,6 @@ def common_options
           :bind, :port
         ]
       end
-
-      def advanced_options
-        []
-      end
-
-      def plugin_name
-        "monitor_agent"
-      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/in_syslog.rb b/app/models/fluentd/setting/in_syslog.rb
index 2fef577ff..9366dd1fa 100644
--- a/app/models/fluentd/setting/in_syslog.rb
+++ b/app/models/fluentd/setting/in_syslog.rb
@@ -1,33 +1,28 @@
 class Fluentd
   module Setting
     class InSyslog
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :port, :bind, :tag, :types
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      validates :tag, presence: true
+      register_plugin("input", "syslog")
 
       def self.initial_params
         {
           bind: "0.0.0.0",
           port: 5140,
+          parse: {
+            "0" => {
+              type: :syslog
+            }
+          },
+          protocol_type: :udp,
         }
       end
 
       def common_options
         [
-          :tag, :bind, :port, :types,
+          :tag, :bind, :port
         ]
       end
-
-      def advanced_options
-        []
-      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb
index 90084c1fc..bfe48995f 100644
--- a/app/models/fluentd/setting/in_tail.rb
+++ b/app/models/fluentd/setting/in_tail.rb
@@ -1,14 +1,14 @@
+require "fluent/plugin/parser_multiline"
+
 class Fluentd
   module Setting
     class InTail
-      MULTI_LINE_MAX_FORMAT_COUNT = 20
+      include Fluentd::Setting::Plugin
 
-      include ActiveModel::Model
-      attr_accessor :path, :tag, :format, :regexp, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval
+      register_plugin("input", "tail")
+      # TODO support formatN ???
 
-      validates :path, presence: true
-      validates :tag, presence: true
-      #validates :format, presence: true
+      MULTI_LINE_MAX_FORMAT_COUNT = ::Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM
 
       def self.known_formats
         {
@@ -26,7 +26,7 @@ def self.known_formats
           # :grok => [:grok_str],
         }
       end
-      attr_accessor *known_formats.values.flatten.compact.uniq
+      attr_accessor(*known_formats.values.flatten.compact.uniq)
 
       def known_formats
         self.class.known_formats
@@ -103,7 +103,7 @@ def to_conf
         # NOTE: Using strip_heredoc makes more complex for format_specific_conf indent
         <<-CONFIG.gsub(/^[ ]*\n/m, "")
 <source>
-  type tail
+  @type tail
   path #{path}
   tag #{tag}
   #{certain_format_line}
diff --git a/app/models/fluentd/setting/out_elasticsearch.rb b/app/models/fluentd/setting/out_elasticsearch.rb
index bf9e78926..4df871d78 100644
--- a/app/models/fluentd/setting/out_elasticsearch.rb
+++ b/app/models/fluentd/setting/out_elasticsearch.rb
@@ -1,24 +1,9 @@
 class Fluentd
   module Setting
     class OutElasticsearch
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :match,
-        :host, :port, :index_name, :type_name,
-        :logstash_format, :logstash_prefix, :logstash_dateformat, :utc_index,
-        :hosts, :request_timeout, :include_tag_key
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      booleans :logstash_format, :utc_index, :include_tag_key
-
-      validates :match, presence: true
-      validates :host, presence: true
-      validates :port, presence: true
-      validates :index_name, presence: true
-      validates :type_name, presence: true
+      register_plugin("output", "elasticsearch")
 
       def self.initial_params
         {
@@ -34,15 +19,14 @@ def self.initial_params
 
       def common_options
         [
-          :match, :host, :port, :logstash_format,
+          :pattern, :host, :port, :logstash_format,
           :index_name, :type_name,
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :hosts, :logstash_prefix, :logstash_dateformat,
-          :utc_index, :request_timeout, :include_tag_key,
+          :secondary, :inject, :buffer
         ]
       end
     end
diff --git a/app/models/fluentd/setting/out_forward.rb b/app/models/fluentd/setting/out_forward.rb
index 1b259901d..2d3aecacd 100644
--- a/app/models/fluentd/setting/out_forward.rb
+++ b/app/models/fluentd/setting/out_forward.rb
@@ -1,70 +1,9 @@
 class Fluentd
   module Setting
     class OutForward
-      class Server
-        include Common
-        KEYS = [
-          :name, :host, :port, :weight, :standby
-        ].freeze
+      include Fluentd::Setting::Plugin
 
-        attr_accessor(*KEYS)
-
-        flags :standby
-
-        validates :host, presence: true
-        validates :port, presence: true
-      end
-
-      class Secondary
-        include Common
-        KEYS = [
-          :type, :path
-        ].freeze
-
-        attr_accessor(*KEYS)
-
-        hidden :type
-        validates :path, presence: true
-      end
-
-      include Common
-
-      KEYS = [
-        :match,
-        :send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval,
-        :phi_threshold, :hard_timeout,
-        :server, :secondary
-      ].freeze
-
-      attr_accessor(*KEYS)
-      choice :heartbeat_type, %w(udp tcp)
-      nested :server, Server, multiple: true
-      nested :secondary, Secondary
-
-      validates :match, presence: true
-      validate :validate_has_at_least_one_server
-      validate :validate_nested_values
-
-      def validate_has_at_least_one_server
-        if children_of(:server).reject{|s| s.empty_value? }.blank?
-          errors.add(:base, :out_forward_blank_server)
-        end
-      end
-
-      def validate_nested_values
-        self.class.children.inject(true) do |result, (key, _)|
-          children_of(key).each do |child|
-            if !child.empty_value? && !child.valid?
-              child.errors.full_messages.each do |message|
-                errors.add(:base, "(#{key})#{message}")
-              end
-              result = false
-            end
-            result
-          end
-          result
-        end
-      end
+      register_plugin("output", "forward")
 
       def self.initial_params
         {
@@ -78,14 +17,14 @@ def self.initial_params
 
       def common_options
         [
-          :match, :server, :secondary,
+          :pattern, :server, :secondary,
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval,
-          :phi_threshold, :hard_timeout,
+          :inject, :buffer,
+          :host, :port
         ]
       end
     end
diff --git a/app/models/fluentd/setting/out_mongo.rb b/app/models/fluentd/setting/out_mongo.rb
index 62093f48e..25afc1a8f 100644
--- a/app/models/fluentd/setting/out_mongo.rb
+++ b/app/models/fluentd/setting/out_mongo.rb
@@ -1,30 +1,14 @@
 class Fluentd
   module Setting
     class OutMongo
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :match,
-        :host, :port, :database, :collection, :capped, :capped_size, :capped_max, :user, :password, :tag_mapped,
-        :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,  :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
-      ].freeze
+      register_plugin("output", "mongo")
 
-      attr_accessor(*KEYS)
-
-      flags :capped, :tag_mapped
-
-      validates :match, presence: true
-      validates :host, presence: true
-      validates :port, presence: true
+      # NOTE: fluent-plugin-mongo defines database parameter as required parameter
+      #       But Fluentd tells us that the database parameter is not required.
       validates :database, presence: true
-      validate :validate_capped
       validate :validate_collection
-      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
-
-      def validate_capped
-        return true if capped.blank?
-        errors.add(:capped_size, :blank) if capped_size.blank?
-      end
 
       def validate_collection
         if tag_mapped.blank? && collection.blank?
@@ -43,15 +27,16 @@ def self.initial_params
 
       def common_options
         [
-          :match, :host, :port, :database, :collection,
+          :pattern, :host, :port, :database, :collection,
           :tag_mapped, :user, :password,
         ]
       end
 
-      def advanced_options
+      def hidden_options
         [
-          :capped, :capped_size, :capped_max, :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit,
-          :flush_interval, :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
+          :secondary, :inject, :buffer,
+          :include_tag_key,
+          :include_time_key
         ]
       end
     end
diff --git a/app/models/fluentd/setting/out_s3.rb b/app/models/fluentd/setting/out_s3.rb
index 28fdc3985..37053414e 100644
--- a/app/models/fluentd/setting/out_s3.rb
+++ b/app/models/fluentd/setting/out_s3.rb
@@ -1,30 +1,9 @@
 class Fluentd
   module Setting
     class OutS3
-      include ActiveModel::Model
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [
-        :match,
-        :aws_key_id, :aws_sec_key, :s3_bucket, :s3_region, :path,
-        # :reduced_redundancy, :check_apikey_on_start, :command_parameter, # not configurable?
-        :format, :include_time_key, :time_key, :delimiter, :label_delimiter,
-        :time_slice_format, :time_slice_wait, :time_format, :utc, :store_as, :proxy_uri, :use_ssl,
-        :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
-        :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      choice :format, %w(out_file json ltsv single_value)
-      choice :store_as, %w(gzip lzo lzma2 json txt)
-      choice :buffer_type, %w(memory file)
-      booleans :include_time_key, :use_ssl
-      flags :utc
-
-      validates :match, presence: true
-      validates :s3_bucket, presence: true
-      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
+      register_plugin("output", "s3")
 
       def self.initial_params
         {
@@ -35,17 +14,14 @@ def self.initial_params
 
       def common_options
         [
-          :match, :aws_key_id, :aws_sec_key,
+          :pattern, :aws_key_id, :aws_sec_key,
           :s3_region, :s3_bucket, :use_ssl, :path,
         ]
       end
 
-      def advanced_options
+      def hidden
         [
-          :format, :include_time_key, :time_key, :delimiter, :label_delimiter,
-          :utc, :time_slice_format, :time_slice_wait, :store_as, :proxy_uri,
-          :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
-          :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
+          :secondary, :inject, :buffer
         ]
       end
     end
diff --git a/app/models/fluentd/setting/out_stdout.rb b/app/models/fluentd/setting/out_stdout.rb
index 51ba3196a..ec0a5a120 100644
--- a/app/models/fluentd/setting/out_stdout.rb
+++ b/app/models/fluentd/setting/out_stdout.rb
@@ -1,36 +1,32 @@
 class Fluentd
   module Setting
     class OutStdout
-      include Common
+      include Fluentd::Setting::Plugin
 
-      KEYS = [:match, :output_type].freeze
-
-      attr_accessor(*KEYS)
-
-      choice :output_type, %w(json hash)
-
-      validates :match, presence: true
-      validates :output_type, inclusion: { in: %w(json hash) }
+      register_plugin("output", "stdout")
 
       def self.initial_params
         {
-          match: "debug.**",
-          output_type: "json",
+          pattern: "debug.**",
+          format: {
+            "0" => {
+              "@type" => "stdout",
+              "output_type" => "json"
+            }
+          }
         }
       end
 
       def common_options
         [
-          :match, :output_type
+          :pattern, :output_type
         ]
       end
 
-      def advanced_options
-        []
-      end
-
-      def plugin_name
-        "stdout"
+      def hidden_options
+        [
+          :secondary, :inject, :buffer
+        ]
       end
     end
   end
diff --git a/app/models/fluentd/setting/out_td.rb b/app/models/fluentd/setting/out_td.rb
deleted file mode 100644
index 38554ffd4..000000000
--- a/app/models/fluentd/setting/out_td.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-class Fluentd
-  module Setting
-    class OutTd
-      include ActiveModel::Model
-      include Common
-
-      KEYS = [
-        :match,
-        :apikey, :auto_create_table, :database, :table,
-        :flush_interval, :buffer_type, :buffer_path,
-      ].freeze
-
-      attr_accessor(*KEYS)
-
-      flags :auto_create_table
-
-      validates :match, presence: true
-      validates :apikey, presence: true
-      validates :auto_create_table, presence: true
-      validates :buffer_path, presence: true, if: ->{ buffer_type == "file" }
-
-      def plugin_name
-        "tdlog"
-      end
-
-      def self.initial_params
-        {
-          buffer_type: "file",
-          buffer_path: "/var/log/td-agent/buffer/td",
-          auto_create_table: true,
-          match: "td.*.*",
-        }
-      end
-
-      def common_options
-        [
-          :match, :apikey, :auto_create_table, :database, :table,
-        ]
-      end
-
-      def advanced_options
-        [
-          :flush_interval, :buffer_type, :buffer_path,
-        ]
-      end
-    end
-  end
-end
diff --git a/app/models/fluentd/setting/out_tdlog.rb b/app/models/fluentd/setting/out_tdlog.rb
new file mode 100644
index 000000000..4f0e06a4a
--- /dev/null
+++ b/app/models/fluentd/setting/out_tdlog.rb
@@ -0,0 +1,32 @@
+class Fluentd
+  module Setting
+    class OutTdlog
+      include Fluentd::Setting::Plugin
+
+      register_plugin("output", "tdlog")
+
+      def self.initial_params
+        {
+          pattern: "td.*.*",
+          buffer: {
+            "@type" => "file",
+            "path" => "/var/log/td-agent/buffer/td",
+          },
+          auto_create_table: true,
+        }
+      end
+
+      def common_options
+        [
+          :pattern, :apikey, :auto_create_table, :database, :table,
+        ]
+      end
+
+      def hidden_options
+        [
+          :secondary
+        ]
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/section.rb b/app/models/fluentd/setting/section.rb
new file mode 100644
index 000000000..4bfc7b8a8
--- /dev/null
+++ b/app/models/fluentd/setting/section.rb
@@ -0,0 +1,34 @@
+class Fluentd
+  module Setting
+    class Section
+      class << self
+        def inherited(klass)
+          klass.instance_eval do
+            include ActiveModel::Model
+            include ActiveModel::Attributes
+            include Fluentd::Setting::Configurable
+            include Fluentd::Setting::SectionParser
+            include Fluentd::Setting::PluginParameter
+
+            class_attribute :_klass, :_block
+            class_attribute :section_name, :required, :multi, :alias
+            self._klass = klass
+          end
+        end
+
+        def init
+          _klass.instance_eval(&_block)
+        end
+
+        # Don't overwrite options
+        def merge(**options)
+          _klass.instance_eval(&_block)
+        end
+
+        def section?
+          true
+        end
+      end
+    end
+  end
+end

From 03a5d86f8c238e407f4d9c9a8feaccc787fda59a Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 24 May 2018 18:28:11 +0900
Subject: [PATCH 005/137] Add specs for previous change

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../models/fluentd/setting/in_forward_spec.rb |  75 ++++++++++
 spec/models/fluentd/setting/in_http_spec.rb   |  31 +++++
 .../fluentd/setting/in_monitor_agent_spec.rb  |  31 +++++
 spec/models/fluentd/setting/in_syslog_spec.rb |  72 +++++++++-
 spec/models/fluentd/setting/in_tail_spec.rb   |  47 +++++++
 .../fluentd/setting/out_elasticsearch_spec.rb |  31 +++++
 spec/models/fluentd/setting/out_mongo_spec.rb |  47 +++++--
 spec/models/fluentd/setting/out_s3_spec.rb    | 128 ++++++++++++++++++
 .../models/fluentd/setting/out_stdout_spec.rb |  31 +++++
 spec/models/fluentd/setting/out_td_spec.rb    |  34 +++--
 10 files changed, 496 insertions(+), 31 deletions(-)
 create mode 100644 spec/models/fluentd/setting/in_forward_spec.rb
 create mode 100644 spec/models/fluentd/setting/in_http_spec.rb
 create mode 100644 spec/models/fluentd/setting/in_monitor_agent_spec.rb
 create mode 100644 spec/models/fluentd/setting/in_tail_spec.rb
 create mode 100644 spec/models/fluentd/setting/out_elasticsearch_spec.rb
 create mode 100644 spec/models/fluentd/setting/out_s3_spec.rb
 create mode 100644 spec/models/fluentd/setting/out_stdout_spec.rb

diff --git a/spec/models/fluentd/setting/in_forward_spec.rb b/spec/models/fluentd/setting/in_forward_spec.rb
new file mode 100644
index 000000000..f3ca08ff8
--- /dev/null
+++ b/spec/models/fluentd/setting/in_forward_spec.rb
@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::InForward do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {}
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      klass.new(params).should be_valid
+    end
+  end
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "forward" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "input" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type forward") }
+  end
+
+  describe "with security section" do
+    let(:valid_attributes) {
+      {
+        security: {
+          "0" => {
+            self_hostname: "test.fluentd",
+            shared_key: "secretsharedkey",
+          }
+        }
+      }
+    }
+    let(:expected) {
+      <<-CONFIG
+<source>
+  @type forward
+  <security>
+    self_hostname test.fluentd
+    shared_key secretsharedkey
+  </security>
+</source>
+      CONFIG
+    }
+    subject { instance.to_config.to_s }
+    it { should == expected }
+  end
+
+  describe "with invalid security section" do
+    let(:valid_attributes) {
+      {
+        security: {
+          "0" => {
+            self_hostname: "test.fluentd",
+          }
+        }
+      }
+    }
+    it { instance.should_not be_valid }
+    it {
+      instance.validate
+      instance.errors.full_messages.should include("Security Shared key can't be blank")
+    }
+  end
+end
+
diff --git a/spec/models/fluentd/setting/in_http_spec.rb b/spec/models/fluentd/setting/in_http_spec.rb
new file mode 100644
index 000000000..2bdd6ecfd
--- /dev/null
+++ b/spec/models/fluentd/setting/in_http_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::InHttp do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {}
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      klass.new(params).should be_valid
+    end
+  end
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "http" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "input" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type http") }
+  end
+end
diff --git a/spec/models/fluentd/setting/in_monitor_agent_spec.rb b/spec/models/fluentd/setting/in_monitor_agent_spec.rb
new file mode 100644
index 000000000..b5708e212
--- /dev/null
+++ b/spec/models/fluentd/setting/in_monitor_agent_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::InMonitorAgent do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {}
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      klass.new(params).should be_valid
+    end
+  end
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "monitor_agent" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "input" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type monitor_agent") }
+  end
+end
diff --git a/spec/models/fluentd/setting/in_syslog_spec.rb b/spec/models/fluentd/setting/in_syslog_spec.rb
index c7c3dcc71..50d9a42a7 100644
--- a/spec/models/fluentd/setting/in_syslog_spec.rb
+++ b/spec/models/fluentd/setting/in_syslog_spec.rb
@@ -17,19 +17,77 @@
     end
   end
 
-  describe "#plugin_type_name" do
-    subject { instance.plugin_type_name }
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
     it { should == "syslog" }
   end
 
-  describe "#input_plugin" do
-    it { instance.should be_input_plugin }
-    it { instance.should_not be_output_plugin }
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "input" }
   end
 
   describe "#to_config" do
-    subject { instance.to_config }
-    it { should include("type syslog") }
+    subject { instance.to_config.to_s }
+    it { should include("@type syslog") }
+  end
+
+  describe "with parse section" do
+    let(:valid_attributes) {
+      {
+        tag: "test",
+        parse: {
+          "0" => {
+            "@type" => "syslog",
+            "message_format" => "rfc5424"
+          }
+        }
+      }
+    }
+    let(:expected) {
+      <<-CONFIG
+<source>
+  @type syslog
+  tag test
+  <parse>
+    @type syslog
+    message_format rfc5424
+  </parse>
+</source>
+      CONFIG
+    }
+    subject { instance.to_config.to_s }
+    it { should == expected }
+  end
+
+  describe "with @log_level" do
+    let(:valid_attributes) {
+      {
+        tag: "test",
+        log_level: "debug",
+        parse: {
+          "0" => {
+            "@type" => "syslog",
+            "message_format" => "rfc5424"
+          }
+        }
+      }
+    }
+    let(:expected) {
+      <<-CONFIG
+<source>
+  @type syslog
+  tag test
+  @log_level debug
+  <parse>
+    @type syslog
+    message_format rfc5424
+  </parse>
+</source>
+      CONFIG
+    }
+    subject { instance.to_config.to_s }
+    it { should == expected }
   end
 end
 
diff --git a/spec/models/fluentd/setting/in_tail_spec.rb b/spec/models/fluentd/setting/in_tail_spec.rb
new file mode 100644
index 000000000..4da6ba854
--- /dev/null
+++ b/spec/models/fluentd/setting/in_tail_spec.rb
@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::InTail do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {
+      tag: "dummy.log",
+      path: "/tmp/log/dummy.log",
+    }
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      klass.new(params).should be_valid
+    end
+
+    it "should be invalid if tag parameter is missing" do
+      params = valid_attributes.dup
+      params.delete(:tag)
+      klass.new(params).should_not be_valid
+    end
+
+    it "should be invalid if path parameter is missing" do
+      params = valid_attributes.dup
+      params.delete(:path)
+      klass.new(params).should_not be_valid
+    end
+  end
+
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "tail" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "input" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type tail") }
+  end
+end
diff --git a/spec/models/fluentd/setting/out_elasticsearch_spec.rb b/spec/models/fluentd/setting/out_elasticsearch_spec.rb
new file mode 100644
index 000000000..292ea2266
--- /dev/null
+++ b/spec/models/fluentd/setting/out_elasticsearch_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::OutElasticsearch do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {}
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      klass.new(params).should be_valid
+    end
+  end
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "elasticsearch" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "output" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type elasticsearch") }
+  end
+end
diff --git a/spec/models/fluentd/setting/out_mongo_spec.rb b/spec/models/fluentd/setting/out_mongo_spec.rb
index f7436a660..56058296b 100644
--- a/spec/models/fluentd/setting/out_mongo_spec.rb
+++ b/spec/models/fluentd/setting/out_mongo_spec.rb
@@ -5,7 +5,7 @@
   let(:instance) { klass.new(valid_attributes) }
   let(:valid_attributes) {
     {
-      match: "mongo.*.*",
+      pattern: "mongo.*.*",
       host: "example.com",
       port: 12345,
       database: "mongodb",
@@ -14,27 +14,50 @@
   }
 
   describe "#valid?" do
-    it "should be invalid if tag parameter lacked" do
+    it "should be invalid if database parameter is missing" do
       params = valid_attributes.dup
-      params.delete(:match)
-      klass.new(params).should_not be_valid
+      params.delete(:database)
+      instance = klass.new(params)
+      instance.should_not be_valid
+      instance.errors.full_messages.should == ["Database can't be blank"]
+    end
+
+    it "should be invalid if collection is missing" do
+      params = {
+        pattern: "mongo.*.*",
+        host: "example.com",
+        port: 12345,
+        database: "mongodb",
+      }
+      instance = klass.new(params)
+      instance.should_not be_valid
+      instance.errors.full_messages.should == ["Collection can't be blank"]
     end
   end
 
-  describe "#plugin_type_name" do
-    subject { instance.plugin_type_name }
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
     it { should == "mongo" }
   end
 
-  describe "#input_plugin" do
-    it { instance.should_not be_input_plugin }
-    it { instance.should be_output_plugin }
+  describe "#plugin_type" do
+    it { instance.plugin_type.should == "output" }
   end
 
   describe "#to_config" do
-    subject { instance.to_config }
-    it { should include("type mongo") }
-    it { should include("tag_mapped") }
+    subject { instance.to_config.to_s }
+    let(:expected) {
+      <<-CONFIG
+<match mongo.*.*>
+  @type mongo
+  database mongodb
+  host example.com
+  port 12345
+  tag_mapped true
+</match>
+      CONFIG
+    }
+    it { should == expected}
   end
 end
 
diff --git a/spec/models/fluentd/setting/out_s3_spec.rb b/spec/models/fluentd/setting/out_s3_spec.rb
new file mode 100644
index 000000000..1dcbe24d8
--- /dev/null
+++ b/spec/models/fluentd/setting/out_s3_spec.rb
@@ -0,0 +1,128 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::OutS3 do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {
+      s3_bucket: "bucketname"
+    }
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      instance = klass.new(params)
+      instance.should be_valid
+    end
+  end
+
+  describe "#invalid?" do
+    it "should be invalid if s3_bucket parameter is missing" do
+      params = valid_attributes.dup
+      params.delete(:s3_bucket)
+      instance = klass.new(params)
+      instance.should be_invalid
+    end
+  end
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "s3" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "output" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type s3") }
+  end
+
+  describe "with assume_role_credentials" do
+    let(:valid_attributes) {
+      {
+        pattern: "s3.*",
+        s3_bucket: "bucketname",
+        assume_role_credentials: {
+          "0" => {
+            role_arn: "arn",
+            role_session_name: "session_name",
+          }
+        }
+      }
+    }
+    let(:expected) {
+      <<-CONFIG
+<match s3.*>
+  @type s3
+  s3_bucket bucketname
+  <assume_role_credentials>
+    role_arn arn
+    role_session_name session_name
+  </assume_role_credentials>
+</match>
+      CONFIG
+    }
+    subject { instance.to_config.to_s }
+    it { should == expected }
+  end
+
+  describe "with instance_profile_credentials" do
+    let(:valid_attributes) {
+      {
+        pattern: "s3.*",
+        s3_bucket: "bucketname",
+        instance_profile_credentials: {
+          "0" => {
+            port: 80
+          }
+        }
+      }
+    }
+    let(:expected) {
+      <<-CONFIG
+<match s3.*>
+  @type s3
+  s3_bucket bucketname
+  <instance_profile_credentials>
+    port 80
+  </instance_profile_credentials>
+</match>
+      CONFIG
+    }
+    subject { instance.to_config.to_s }
+    it { should == expected }
+  end
+
+  describe "with shared_credentials" do
+    let(:valid_attributes) {
+      {
+        pattern: "s3.*",
+        s3_bucket: "bucketname",
+        shared_credentials: {
+          "0" => {
+            path: "$HOME/.aws/credentials",
+            profile_name: "default",
+          }
+        }
+      }
+    }
+    let(:expected) {
+      <<-CONFIG
+<match s3.*>
+  @type s3
+  s3_bucket bucketname
+  <shared_credentials>
+    path $HOME/.aws/credentials
+    profile_name default
+  </shared_credentials>
+</match>
+      CONFIG
+    }
+    subject { instance.to_config.to_s }
+    it { should == expected }
+  end
+end
diff --git a/spec/models/fluentd/setting/out_stdout_spec.rb b/spec/models/fluentd/setting/out_stdout_spec.rb
new file mode 100644
index 000000000..b29b91fc9
--- /dev/null
+++ b/spec/models/fluentd/setting/out_stdout_spec.rb
@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Fluentd::Setting::OutStdout do
+  let(:klass) { described_class }
+  let(:instance) { klass.new(valid_attributes) }
+  let(:valid_attributes) {
+    {}
+  }
+
+  describe "#valid?" do
+    it "should be valid" do
+      params = valid_attributes.dup
+      klass.new(params).should be_valid
+    end
+  end
+
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
+    it { should == "stdout" }
+  end
+
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "output" }
+  end
+
+  describe "#to_config" do
+    subject { instance.to_config.to_s }
+    it { should include("@type stdout") }
+  end
+end
diff --git a/spec/models/fluentd/setting/out_td_spec.rb b/spec/models/fluentd/setting/out_td_spec.rb
index 59bb3167d..69bc9c1d1 100644
--- a/spec/models/fluentd/setting/out_td_spec.rb
+++ b/spec/models/fluentd/setting/out_td_spec.rb
@@ -5,34 +5,44 @@
   let(:instance) { klass.new(valid_attributes) }
   let(:valid_attributes) {
     {
-      match: "td.*.*",
+      pattern: "td.*.*",
       apikey: "APIKEY",
       auto_create_table: "true",
     }
   }
 
   describe "#valid?" do
-    it "should be invalid if tag parameter lacked" do
+    it "should be invalid if apikey is missing" do
       params = valid_attributes.dup
-      params.delete(:match)
-      klass.new(params).should_not be_valid
+      params.delete(:apikey)
+      instance = klass.new(params)
+      instance.should_not be_valid
+      instance.errors.full_messages.should == ["Apikey can't be blank"]
     end
   end
 
-  describe "#plugin_type_name" do
-    subject { instance.plugin_type_name }
+  describe "#plugin_name" do
+    subject { instance.plugin_name }
     it { should == "tdlog" }
   end
 
-  describe "#input_plugin" do
-    it { instance.should_not be_input_plugin }
-    it { instance.should be_output_plugin }
+  describe "#plugin_type" do
+    subject { instance.plugin_type }
+    it { should == "output" }
   end
 
   describe "#to_config" do
-    subject { instance.to_config }
-    it { should include("type tdlog") }
-    it { should include("APIKEY") }
+    subject { instance.to_config.to_s }
+    let(:expected) {
+      <<-CONFIG
+<match td.*.*>
+  @type tdlog
+  apikey APIKEY
+  auto_create_table true
+</match>
+      CONFIG
+    }
+    it { should == expected }
   end
 end
 

From 1bde0bae1c29543a2897b93e6e0802af9153ced0 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 24 May 2018 18:32:15 +0900
Subject: [PATCH 006/137] Rename out_td to out_tdlog

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../{out_td_controller.rb => out_tdlog_controller.rb}       | 4 ++--
 app/views/fluentd/settings/source_and_output.html.haml      | 2 +-
 config/routes.rb                                            | 2 +-
 spec/features/{out_td_spec.rb => out_tdlog_spec.rb}         | 6 +++---
 .../fluentd/setting/{out_td_spec.rb => out_tdlog_spec.rb}   | 3 +--
 5 files changed, 8 insertions(+), 9 deletions(-)
 rename app/controllers/fluentd/settings/{out_td_controller.rb => out_tdlog_controller.rb} (70%)
 rename spec/features/{out_td_spec.rb => out_tdlog_spec.rb} (82%)
 rename spec/models/fluentd/setting/{out_td_spec.rb => out_tdlog_spec.rb} (96%)

diff --git a/app/controllers/fluentd/settings/out_td_controller.rb b/app/controllers/fluentd/settings/out_tdlog_controller.rb
similarity index 70%
rename from app/controllers/fluentd/settings/out_td_controller.rb
rename to app/controllers/fluentd/settings/out_tdlog_controller.rb
index 8e3368d89..117418b9e 100644
--- a/app/controllers/fluentd/settings/out_td_controller.rb
+++ b/app/controllers/fluentd/settings/out_tdlog_controller.rb
@@ -1,10 +1,10 @@
-class Fluentd::Settings::OutTdController < ApplicationController
+class Fluentd::Settings::OutTdlogController < ApplicationController
   include SettingConcern
 
   private
 
   def target_class
-    Fluentd::Setting::OutTd
+    Fluentd::Setting::OutTdlog
   end
 
   def initial_params
diff --git a/app/views/fluentd/settings/source_and_output.html.haml b/app/views/fluentd/settings/source_and_output.html.haml
index 309674751..8c1de68b4 100644
--- a/app/views/fluentd/settings/source_and_output.html.haml
+++ b/app/views/fluentd/settings/source_and_output.html.haml
@@ -23,7 +23,7 @@
       .card-header
         %h4= t('.out')
       .card-body
-        - %w|stdout td s3 mongo elasticsearch forward|.each do |type|
+        - %w|stdout tdlog s3 mongo elasticsearch forward|.each do |type|
           %p
             = link_to(send("daemon_setting_out_#{type}_path", @fluentd)) do
               = icon('fa-file-text-o fa-lg')
diff --git a/config/routes.rb b/config/routes.rb
index c1eb8a119..31e6c5c81 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -49,7 +49,7 @@
           post "finish"
         end
 
-        resource :out_td, only: [:show], module: :settings, controller: :out_td do
+        resource :out_tdlog, only: [:show], module: :settings, controller: :out_tdlog do
           post "finish"
         end
 
diff --git a/spec/features/out_td_spec.rb b/spec/features/out_tdlog_spec.rb
similarity index 82%
rename from spec/features/out_td_spec.rb
rename to spec/features/out_tdlog_spec.rb
index 55bf9619b..3b23c44fe 100644
--- a/spec/features/out_td_spec.rb
+++ b/spec/features/out_tdlog_spec.rb
@@ -1,6 +1,6 @@
 require "spec_helper"
 
-describe "out_td", stub: :daemon do
+describe "out_tdlog", stub: :daemon do
   let(:exists_user) { build(:user) }
   let(:api_key) { "dummydummy" }
 
@@ -9,13 +9,13 @@
   end
 
   it "Shown form with filled in td.*.* on match" do
-    visit daemon_setting_out_td_path
+    visit daemon_setting_out_tdlog_path
     page.should have_css('input[name="fluentd_setting_out_td[match]"]')
   end
 
   it "Updated config after submit" do
     daemon.agent.config.should_not include(api_key)
-    visit daemon_setting_out_td_path
+    visit daemon_setting_out_tdlog_path
     within('#new_fluentd_setting_out_td') do
       fill_in "Apikey", with: api_key
     end
diff --git a/spec/models/fluentd/setting/out_td_spec.rb b/spec/models/fluentd/setting/out_tdlog_spec.rb
similarity index 96%
rename from spec/models/fluentd/setting/out_td_spec.rb
rename to spec/models/fluentd/setting/out_tdlog_spec.rb
index 69bc9c1d1..033f30d67 100644
--- a/spec/models/fluentd/setting/out_td_spec.rb
+++ b/spec/models/fluentd/setting/out_tdlog_spec.rb
@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Fluentd::Setting::OutTd do
+describe Fluentd::Setting::OutTdlog do
   let(:klass) { described_class }
   let(:instance) { klass.new(valid_attributes) }
   let(:valid_attributes) {
@@ -45,4 +45,3 @@
     it { should == expected }
   end
 end
-

From 73ee6dddc916c20939bfec3ec5e0099087ed9355 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 14 May 2018 10:21:56 +0900
Subject: [PATCH 007/137] Add partial template for in_forward setting

This change will improve usability after some more changes.

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../fluentd/settings/in_forward/_form.html.haml   | 15 +++++++++++++++
 app/views/shared/settings/show.html.haml          |  5 ++++-
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 app/views/fluentd/settings/in_forward/_form.html.haml

diff --git a/app/views/fluentd/settings/in_forward/_form.html.haml b/app/views/fluentd/settings/in_forward/_form.html.haml
new file mode 100644
index 000000000..8f4338455
--- /dev/null
+++ b/app/views/fluentd/settings/in_forward/_form.html.haml
@@ -0,0 +1,15 @@
+= render "shared/setting_errors"
+
+- # NOTE: plugin_setting_form_action_url is defined at SettingConcern
+= form_for(@setting, url: plugin_setting_form_action_url(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
+  - @setting.common_options.each do |key|
+    = field(f, key)
+
+  .well.well-sm
+    %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
+      = icon('fa-caret-down')
+      = t('terms.advanced_setting')
+    #advanced-setting.collapse
+      TODO
+
+  = f.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary pull-right"
diff --git a/app/views/shared/settings/show.html.haml b/app/views/shared/settings/show.html.haml
index 48462bc2b..89d9f0177 100644
--- a/app/views/shared/settings/show.html.haml
+++ b/app/views/shared/settings/show.html.haml
@@ -5,4 +5,7 @@
   .card-body.bg-light
     = raw t("fluentd.settings.#{target_plugin_name}.option_guide")
 
-= render "shared/settings/form"
+- if lookup_context.exists?("form", controller.controller_path, true)
+  = render "#{controller.controller_path}/form"
+- else
+  = render "shared/settings/form"

From f2f8d27f24d50c7c43383608aa157628d2b40809 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 25 May 2018 13:51:30 +0900
Subject: [PATCH 008/137] Remove setting_params

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../fluentd/settings/out_forward_controller.rb           | 9 ---------
 app/controllers/fluentd/settings/out_s3_controller.rb    | 5 -----
 2 files changed, 14 deletions(-)

diff --git a/app/controllers/fluentd/settings/out_forward_controller.rb b/app/controllers/fluentd/settings/out_forward_controller.rb
index 42a5f3c20..c438bd281 100644
--- a/app/controllers/fluentd/settings/out_forward_controller.rb
+++ b/app/controllers/fluentd/settings/out_forward_controller.rb
@@ -6,13 +6,4 @@ class Fluentd::Settings::OutForwardController < ApplicationController
   def target_class
     Fluentd::Setting::OutForward
   end
-
-  def setting_params
-    params.require(:fluentd_setting_out_forward).permit(*Fluentd::Setting::OutForward::KEYS).merge(
-      params.require(:fluentd_setting_out_forward).permit(
-        :server => Fluentd::Setting::OutForward::Server::KEYS,
-        :secondary => Fluentd::Setting::OutForward::Secondary::KEYS,
-      ),
-    )
-  end
 end
diff --git a/app/controllers/fluentd/settings/out_s3_controller.rb b/app/controllers/fluentd/settings/out_s3_controller.rb
index d8c496496..3cd6f6afe 100644
--- a/app/controllers/fluentd/settings/out_s3_controller.rb
+++ b/app/controllers/fluentd/settings/out_s3_controller.rb
@@ -6,9 +6,4 @@ class Fluentd::Settings::OutS3Controller < ApplicationController
   def target_class
     Fluentd::Setting::OutS3
   end
-
-  def setting_params
-    params.require(:fluentd_setting_out_s3).permit(*Fluentd::Setting::OutS3::KEYS)
-  end
-
 end

From eab7224ddd6359ea91b07cea209310e3468124bc Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 25 May 2018 14:07:10 +0900
Subject: [PATCH 009/137] Use plugins' config definition to construct setting
 form

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb | 42 +++++++++++-
 app/helpers/settings_helper.rb              | 73 ++++++++++++++++-----
 2 files changed, 95 insertions(+), 20 deletions(-)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index 6cf5d774a..b59acdc18 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -10,6 +10,8 @@ module SettingConcern
 
   def show
     @setting = target_class.new(initial_params)
+    @_used_param = {}
+    @_used_section = {}
     render "shared/settings/show"
   end
 
@@ -32,7 +34,7 @@ def finish
   private
 
   def setting_params
-    params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_class.const_get(:KEYS))
+    params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_plugin_params)
   end
 
   def initial_params
@@ -40,10 +42,46 @@ def initial_params
   end
 
   def target_plugin_name
-    target_class.to_s.split("::").last.underscore
+    prefix = case target_class.plugin_type
+             when "input"
+               "in"
+             when "output"
+               "out"
+             else
+               target_class.plugin_type
+             end
+    "#{prefix}_#{target_class.plugin_name}"
   end
 
   def plugin_setting_form_action_url(*args)
     send("finish_daemon_setting_#{target_plugin_name}_path", *args)
   end
+
+  def target_plugin_params
+    keys = []
+    target_class.config_definition.each do |name, definition|
+      if definition[:section]
+        keys.concat(parse_section_definition(definition))
+      else
+        keys.concat(definition.keys)
+      end
+    end
+    keys
+  end
+
+  def parse_section_definition(definition)
+    keys = []
+    definition.except(:section, :argument, :required, :multi, :alias).each do |name, _definition|
+      _keys = []
+      _definition.each do |key, __definition|
+        if __definition[:section]
+          _keys.push({ name => parse_section_definition(__definition) })
+        else
+          _keys.push(key)
+        end
+      end
+      keys.push({ name => _keys })
+    end
+    keys
+  end
 end
diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index dfee00b18..f1af4d448 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -2,26 +2,62 @@ module SettingsHelper
   def field(form, key, opts = {})
     html = '<div class="form-group">'
 
-    field_resolver(form.object.column_type(key), html, form, key, opts)
+    field_resolver(html, form, key, opts)
 
     html << "</div>"
     html.html_safe
   end
 
   private
-  def field_resolver(type, html, form, key, opts)
-    case type
-    when :hidden
-      html << form.hidden_field(key)
-    when :boolean, :flag
-      boolean_field(html, form, key, opts)
-    when :choice
-      choice_field(html, form, key, opts)
-    when :nested
-      nested_field(html, form, key, opts)
-    else
-      other_field(html, form, key, opts)
+
+  def field_resolver(html, form, key, opts)
+    plugin_class = form.object.class
+    type = plugin_class.column_type(key)
+    if type && !@_used_param.key?(key)
+      case type
+      when :enum
+        enum_field(html, form, key, opts)
+      when :bool
+        bool_field(html, form, key, opts)
+      else
+        other_field(html, form, key, opts)
+      end
+      @_used_param[key] = true
+    end
+    if plugin_class._sections[key] && !@_used_section.key?(key)
+      section_field(html, form, key, opts)
+      @_used_section[key] = true
+    end
+  end
+
+  def section_field(html, form, key, opts = {})
+    klass = form.object.class._sections[key]
+    children = form.object.__send__(key) || { "0" => {} }
+
+    children.each do |index, child|
+      open_section_div(html, klass.multi) do |_html|
+        _html << append_and_remove_links if klass.multi
+        _html << h(form.label(key))
+        _html << section_fields(form, key, index, klass, child)
+      end
+    end
+  end
+
+  def open_section_div(html, multi)
+    html << %Q!<div class="js-nested-column #{ multi ? "js-multiple" : "" } well well-sm">!
+    yield html
+    html << "</div>"
+  end
+
+  def section_fields(form, key, index, klass, child)
+    html = ""
+    object = klass.new(child)
+    form.fields_for("#{key}[#{index}]", object) do |ff|
+      klass._types.keys.each do |kk|
+        html << field(ff, kk)
+      end
     end
+    html
   end
 
   def nested_field(html, form, key, opts = {})
@@ -54,27 +90,28 @@ def nested_fields(form, key, index, klass, child)
   end
 
   def append_and_remove_links
-    %Q!<a class="btn btn-sm btn-outline-secondary js-append">#{icon('fa-plus')}</a> ! +
-    %Q!<a class="btn btn-sm btn-outline-secondary js-remove" style="display:none">#{icon('fa-minus')}</a> !
+    %Q!<a class="btn btn-xs btn-default js-append">#{icon('fa-plus')}</a> ! +
+    %Q!<a class="btn btn-xs btn-default js-remove" style="display:none">#{icon('fa-minus')}</a> !
   end
 
   def child_data(form, key)
     form.object.class.children[key]
   end
 
-  def choice_field(html, form, key, opts = {})
+  def enum_field(html, form, key, opts = {})
     html << h(form.label(key))
     html << " " # NOTE: Adding space for padding
-    html << form.select(key, form.object.values_of(key), opts)
+    html << form.select(key, form.object.list_of(key), opts)
   end
 
-  def boolean_field(html, form, key, opts = {})
+  def bool_field(html, form, key, opts = {})
     html << form.check_box(key, {}, "true", "false")
     html << " " # NOTE: Adding space for padding
     html << h(form.label(key))
   end
 
   def other_field(html, form, key, opts = {})
+    return unless form.object.respond_to?(key)
     html << h(form.label(key))
     html << form.text_field(key, class: "form-control")
   end

From c1398a3acd7e5e31076a70eee228f701fb4c7e93 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 09:53:21 +0900
Subject: [PATCH 010/137] Bundle update

* fluent-plugin-elasticsearch
*	fluent-plugin-mongo
* fluent-plugin-s3
* fluent-plugin-td

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 Gemfile.lock | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/Gemfile.lock b/Gemfile.lock
index 6f8730c1a..8c95b0359 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -7,6 +7,10 @@ PATH
       bundler
       diff-lcs
       draper (~> 3.0)
+      fluent-plugin-elasticsearch (~> 2.10)
+      fluent-plugin-mongo (~> 1.1)
+      fluent-plugin-s3 (~> 1.1)
+      fluent-plugin-td (~> 1.0)
       fluentd (>= 1.0.0, < 2)
       font-awesome-rails
       haml-rails (~> 1.0)
@@ -74,6 +78,24 @@ GEM
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     arel (9.0.0)
+    aws-eventstream (1.0.0)
+    aws-partitions (1.87.0)
+    aws-sdk-core (3.21.2)
+      aws-eventstream (~> 1.0)
+      aws-partitions (~> 1.0)
+      aws-sigv4 (~> 1.0)
+      jmespath (~> 1.0)
+    aws-sdk-kms (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.13.0)
+      aws-sdk-core (~> 3, >= 3.21.2)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-sqs (1.3.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sigv4 (1.0.2)
     better_errors (2.1.1)
       coderay (>= 1.0.0)
       erubis (>= 2.6.6)
@@ -83,6 +105,7 @@ GEM
       debug_inspector (>= 0.0.1)
     bootsnap (1.3.0)
       msgpack (~> 1.0)
+    bson (4.3.0)
     builder (3.2.3)
     capybara (3.0.2)
       addressable
@@ -112,14 +135,39 @@ GEM
       activemodel-serializers-xml (~> 1.0)
       activesupport (~> 5.0)
       request_store (~> 1.0)
+    elasticsearch (6.0.2)
+      elasticsearch-api (= 6.0.2)
+      elasticsearch-transport (= 6.0.2)
+    elasticsearch-api (6.0.2)
+      multi_json
+    elasticsearch-transport (6.0.2)
+      faraday
+      multi_json
     erubi (1.7.1)
     erubis (2.7.0)
+    excon (0.62.0)
     factory_bot (4.8.2)
       activesupport (>= 3.0.0)
     factory_bot_rails (4.8.2)
       factory_bot (~> 4.8.2)
       railties (>= 3.0.0)
+    faraday (0.15.2)
+      multipart-post (>= 1.2, < 3)
     ffi (1.9.23)
+    fluent-plugin-elasticsearch (2.10.1)
+      elasticsearch
+      excon
+      fluentd (>= 0.14.20)
+    fluent-plugin-mongo (1.1.1)
+      fluentd (>= 0.14.12, < 2)
+      mongo (~> 2.2.0)
+    fluent-plugin-s3 (1.1.3)
+      aws-sdk-s3 (~> 1.0)
+      aws-sdk-sqs (~> 1.0)
+      fluentd (>= 0.14.2, < 2)
+    fluent-plugin-td (1.0.0)
+      fluentd (>= 0.14.13, < 2)
+      td-client (~> 1.0)
     fluentd (1.2.1)
       cool.io (>= 1.4.5, < 2.0.0)
       dig_rb (~> 1.0.0)
@@ -159,6 +207,7 @@ GEM
     jbuilder (2.7.0)
       activesupport (>= 4.2.0)
       multi_json (>= 1.2)
+    jmespath (1.4.0)
     json (2.1.0)
     kramdown (1.16.2)
     kramdown-haml (0.0.3)
@@ -181,8 +230,11 @@ GEM
     mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.11.3)
+    mongo (2.2.7)
+      bson (~> 4.0)
     msgpack (1.2.4)
     multi_json (1.13.1)
+    multipart-post (2.0.0)
     nio4r (2.3.0)
     nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)
@@ -286,6 +338,9 @@ GEM
     strptime (0.2.3)
     sucker_punch (2.0.4)
       concurrent-ruby (~> 1.0.0)
+    td-client (1.0.6)
+      httpclient (>= 2.7)
+      msgpack (>= 0.5.6, < 2)
     temple (0.8.0)
     thor (0.20.0)
     thread_safe (0.3.6)

From a849fdeda3096ac450d07bd0963a5185a7976254 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 11:13:13 +0900
Subject: [PATCH 011/137] Remove unused module

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/common.rb       | 185 ---------------------
 spec/models/fluentd/setting/common_spec.rb | 178 --------------------
 2 files changed, 363 deletions(-)
 delete mode 100644 app/models/fluentd/setting/common.rb
 delete mode 100644 spec/models/fluentd/setting/common_spec.rb

diff --git a/app/models/fluentd/setting/common.rb b/app/models/fluentd/setting/common.rb
deleted file mode 100644
index faadc8e25..000000000
--- a/app/models/fluentd/setting/common.rb
+++ /dev/null
@@ -1,185 +0,0 @@
-class Fluentd
-  module Setting
-    module Common
-      extend ActiveSupport::Concern
-      include ActiveModel::Model
-
-      module ClassMethods
-        attr_accessor :values, :types, :children, :hidden_values
-
-        def choice(key, values)
-          @values ||= {}
-          @values[key] = values
-          set_type(:choice, [key])
-        end
-
-        def hidden(key)
-          set_type(:hidden, [key])
-        end
-
-        def nested(key, klass, options = {})
-          # e.g.:
-          #  <match>
-          #    type forward
-          #    <server>
-          #      ..
-          #    </server>
-          #  </match>
-          @children ||= {}
-          @children[key] = {
-            class: klass,
-            options: options,
-          }
-          set_type(:nested, [key])
-        end
-
-        def booleans(*keys)
-          # e.g.:
-          #   use_ssl true
-          #   include_time_key false
-          set_type(:boolean, keys)
-        end
-
-        def flags(*keys)
-          # e.g.:
-          #   tag_mapped
-          #   utc
-          set_type(:flag, keys)
-        end
-
-        private
-
-        def set_type(type, keys)
-          @types ||= {}
-          keys.each do |key|
-            @types[key] = type
-          end
-        end
-      end
-
-      def children_of(key)
-        meta = self.class.children[key]
-        return unless meta
-        klass = meta[:class]
-        data = send(key) || {"0" => {}}
-        children = []
-        data.each_pair do |index, attrs|
-          children << klass.new(attrs)
-        end
-        children
-      end
-
-      def child_class(key)
-        self.class.children[key][:class]
-      end
-
-      def values_of(key)
-        self.class.values.try(:[], key) || []
-      end
-
-      def column_type(key)
-        self.class.types.try(:[], key) || "string"
-      end
-
-      def conf(key)
-        case column_type(key)
-        when :boolean
-          boolenan(key)
-        when :flag
-          flag(key)
-        when :nested
-          return "" unless send(key)
-          klass = child_class(key)
-          send(key).map do |(_, child)|
-            # send("servers")
-            #
-            # "servers" => {
-            #   "0" => {
-            #     "name" => "foo",
-            #     "host" => "bar",
-            #     ..
-            #   },
-            #   "1" => {
-            #     ..
-            #   }
-            # }
-            child_instance = klass.new(child)
-            unless child_instance.empty_value?
-              "\n" + child_instance.to_config(key).gsub(/^/m, "  ")
-            end
-          end.join
-        else # including :hidden
-          print_if_present(key)
-        end
-      end
-
-      def plugin_type_name
-        # Fluentd::Setting::OutS3 -> s3
-        # Override this method if not above style
-        try(:plugin_name) || self.class.to_s.split("::").last.sub(/(In|Out)/, "").downcase
-      end
-
-      def print_if_present(key)
-        # e.g.:
-        #   path /var/log/td/aaa
-        #   user nobody
-        #   retry_limit 3
-        send(key).present? ? "#{key} #{send(key)}" : ""
-      end
-
-      def boolenan(key)
-        send(key).presence == "true" ? "#{key} true" : "#{key} false"
-      end
-
-      def flag(key)
-        send(key).presence == "true" ? key.to_s : ""
-      end
-
-      def empty_value?
-        config = ""
-        self.class.const_get(:KEYS).each do |key|
-          config << conf(key)
-        end
-        config.empty?
-      end
-
-      def input_plugin?
-        self.class.to_s.match(/::In|^In/)
-      end
-
-      def output_plugin?
-        not input_plugin?
-      end
-
-      def to_config(elm_name = nil)
-        indent = "  "
-        if elm_name
-          config = "<#{elm_name}>\n"
-        else
-          if input_plugin?
-            config = "<source>\n"
-          else
-            config = "<match #{match}>\n"
-          end
-          config << "#{indent}type #{plugin_type_name}\n"
-        end
-        self.class.const_get(:KEYS).each do |key|
-          next if key == :match
-          config << indent
-          config << conf(key)
-          config << "\n"
-        end
-        if elm_name
-          config << "</#{elm_name}>\n"
-        else
-          if input_plugin?
-            config << "</source>\n"
-          else
-            config << "</match>\n"
-          end
-        end
-        config.gsub(/^[ ]*\n/m, "")
-      end
-    end
-  end
-end
diff --git a/spec/models/fluentd/setting/common_spec.rb b/spec/models/fluentd/setting/common_spec.rb
deleted file mode 100644
index 09f8eaf2a..000000000
--- a/spec/models/fluentd/setting/common_spec.rb
+++ /dev/null
@@ -1,178 +0,0 @@
-require 'spec_helper'
-
-describe Fluentd::Setting::Common do
-  let(:klass) { Fluentd::Setting::Common }
-
-  describe "class methods" do
-    describe "#booleans" do
-      subject do
-        Class.new do
-          include Fluentd::Setting::Common
-          attr_accessor :bool1, :bool2, :foo
-          booleans :bool1, :bool2
-        end.new
-      end
-
-      it { subject.column_type(:bool1).should == :boolean }
-      it { subject.column_type(:bool2).should == :boolean }
-      it { subject.column_type(:foo).should_not == :boolean }
-    end
-
-    describe "#flags" do
-      subject do
-        Class.new do
-          include Fluentd::Setting::Common
-          attr_accessor :flag1, :flag2, :foo
-          flags :flag1, :flag2
-        end.new
-      end
-
-      it { subject.column_type(:flag1).should == :flag }
-      it { subject.column_type(:flag2).should == :flag }
-      it { subject.column_type(:foo).should_not == :flag }
-    end
-
-    describe "#hidden" do
-      subject do
-        Class.new do
-          include Fluentd::Setting::Common
-          attr_accessor :hide
-          hidden :hide
-        end.new
-      end
-
-      it { subject.column_type(:hide).should == :hidden }
-    end
-
-    describe "#choice" do
-      subject do
-        Class.new do
-          include Fluentd::Setting::Common
-          attr_accessor :choice, :foo
-          choice :choice, %w(a b c)
-        end.new
-      end
-
-      it { subject.column_type(:choice).should == :choice }
-      it { subject.values_of(:choice).should == %w(a b c) }
-      it { subject.column_type(:foo).should_not == :choice }
-    end
-
-    describe "#nested" do
-      before do
-        @child_class = Class.new do
-          include Fluentd::Setting::Common
-        end
-      end
-
-      subject do
-        child = @child_class
-        Class.new do
-          include Fluentd::Setting::Common
-          attr_accessor :child, :foo
-          nested :child, child
-        end.new
-      end
-
-      it { subject.column_type(:child).should == :nested }
-      it { subject.child_class(:child).should == @child_class }
-      it { subject.column_type(:foo).should_not == :nested }
-    end
-  end
-
-  describe "instance methods" do
-    describe "plugin name" do
-      before do
-        klass = Class.new do
-          include Fluentd::Setting::Common
-        end
-        Object.const_set(class_name, klass)
-      end
-      after { Object.send(:remove_const, class_name) }
-      subject { Object.const_get(class_name).new }
-
-      context "InFoo" do
-        let(:class_name) { "InFoo" }
-        it "plugin_type_name == foo" do
-          subject.plugin_type_name.should == "foo"
-        end
-        it "should be input_plugin" do
-          subject.should be_input_plugin
-        end
-      end
-
-      context "OutBar" do
-        let(:class_name) { "OutBar" }
-        it "plugin_type_name == bar" do
-          subject.plugin_type_name.should == "bar"
-        end
-        it "should be output_plugin" do
-          subject.should be_output_plugin
-        end
-      end
-    end
-
-    describe "generate config file" do
-      before do
-        class Child
-          include Fluentd::Setting::Common
-          KEYS = [:child_foo]
-          attr_accessor(*KEYS)
-        end
-        @klass = Class.new do
-          include Fluentd::Setting::Common
-          const_set(:KEYS, [:key1, :key2, :flag1, :hide, :ch, :child, :string])
-          attr_accessor(*const_get(:KEYS))
-          booleans :key1, :key2
-          flags :flag1
-          hidden :hide
-          choice :ch, %w(foo bar)
-          nested :child, Child
-        end
-      end
-      after { Object.send(:remove_const, :Child) }
-
-      subject { @klass.new(params).to_config("dummy") }
-
-      describe "boolean" do
-        # NOTE: "true" and "false" are the string because the are given by HTTP request
-        let(:params) { {key1: "true", key2: "false"} }
-
-        it { should include("key1 true\n") }
-        it { should include("key2 false\n") }
-      end
-
-      describe "flag" do
-        context "true" do
-          let(:params) { {flag1: "true"} }
-          it { should include("flag1\n") }
-        end
-        context "false" do
-          let(:params) { {flag1: "false"} }
-          it { should_not include("flag1\n") }
-        end
-      end
-
-      describe "hidden" do
-        let(:params) { {hide: "foo"} }
-        it { should include("hide foo\n") }
-      end
-
-      describe "choice" do
-        let(:params) { {ch: "foo"} }
-        it { should include("ch foo\n") }
-      end
-
-      describe "string" do
-        let(:params) { {string: "foobar"} }
-        it { should include("string foobar\n") }
-      end
-
-      describe "nested" do
-        let(:params) { {child: {"0" => { child_foo: "hi" }}} }
-        it { should match(%r!<child>\n\s+child_foo hi\n\s+</child>!) }
-      end
-    end
-  end
-end
-

From 7c5b420bbc2127c89c0e0a4cc2c20ce8d3652f00 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 11:35:29 +0900
Subject: [PATCH 012/137] Log unknown aatributes

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/configurable.rb | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/configurable.rb b/app/models/concerns/fluentd/setting/configurable.rb
index a9fdd1988..ef8dea982 100644
--- a/app/models/concerns/fluentd/setting/configurable.rb
+++ b/app/models/concerns/fluentd/setting/configurable.rb
@@ -26,7 +26,12 @@ module Configurable
       end
 
       def initialize(attributes = {})
-        super rescue ActiveModel::UnknownAttributeError # the superclass does not know specific attributes of the model
+        # the superclass does not know specific attributes of the model
+        begin
+          super
+        rescue ActiveModel::UnknownAttributeError => ex
+          Rails.logger.warn(ex)
+        end
         self.class._sections.each do |name, klass|
           klass.init
           if klass.multi

From b7e1ed06e775c8587d5994e13d1294d0549cd57d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 11:42:54 +0900
Subject: [PATCH 013/137] Set atttibutes log_levele aand id

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/configurable.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/configurable.rb b/app/models/concerns/fluentd/setting/configurable.rb
index ef8dea982..cab76ba80 100644
--- a/app/models/concerns/fluentd/setting/configurable.rb
+++ b/app/models/concerns/fluentd/setting/configurable.rb
@@ -61,7 +61,11 @@ def config_param(name, type = ActiveModel::Type::Value.new, **options)
             config_param(_name.to_sym, type, **options.merge(alias: name))
             self._built_in_params << _name
           elsif ["id", "type", "log_level"].include?(name.to_s)
-            self._built_in_params << _name
+            self._built_in_params << name
+            unless name == "type"
+              attribute(name, type, **options.slice(:precision, :limit, :scale))
+              validates(name, presence: true) if options[:required]
+            end
           else
             attribute(name, type, **options.slice(:precision, :limit, :scale))
             validates(name, presence: true) if options[:required]

From d67ed8f7ed0163a5930792f523d26225c7062830 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 24 Apr 2018 14:30:06 +0900
Subject: [PATCH 014/137] Display type in panel header properly

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/api/settings/_element.json.jbuilder | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/api/settings/_element.json.jbuilder b/app/views/api/settings/_element.json.jbuilder
index bc58a7d26..018ca1a69 100644
--- a/app/views/api/settings/_element.json.jbuilder
+++ b/app/views/api/settings/_element.json.jbuilder
@@ -1,6 +1,6 @@
 json.id element_id(element)
 json.name element.name
-json.type element["type"]
+json.type element["@type"] || element["type"]
 json.arg element.arg
 json.settings element
 json.content element.to_s

From 6b3584eba5244ec6ffe1a17a03803150f46eb604 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 11:53:52 +0900
Subject: [PATCH 015/137] Display caret-down sign in front of plugin name

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/vue/_setting.html.erb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/shared/vue/_setting.html.erb b/app/views/shared/vue/_setting.html.erb
index 8f424f097..1afdfffcc 100644
--- a/app/views/shared/vue/_setting.html.erb
+++ b/app/views/shared/vue/_setting.html.erb
@@ -4,9 +4,9 @@
 
 <div class="card card-primary mb-3">
   <div class="card-header" data-toggle="collapse" v-bind:href="'#'+id" v-bind:title="content">
+    <i class="fa fa-caret-down"></i>
     {{ type }}
     <span v-if="name == 'match'">({{ arg }})</span>
-    <i class="fa fa-caret-down"></i>
   </div>
   <div class="card-body collapse" v-bind:id="id">
     <pre v-if="mode != 'edit'">{{ content }}</pre>

From 9af03612daa394259f91767b7e65cbc705a2f863 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 12:05:19 +0900
Subject: [PATCH 016/137] Use `@type` in default config

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/app/models/fluentd.rb b/app/models/fluentd.rb
index d29b82b4c..695d68d43 100644
--- a/app/models/fluentd.rb
+++ b/app/models/fluentd.rb
@@ -14,28 +14,28 @@ class Fluentd
   DEFAULT_CONF = <<-CONF.strip_heredoc
     <source>
       # http://docs.fluentd.org/articles/in_forward
-      type forward
+      @type forward
       port 24224
     </source>
 
     <source>
       # http://docs.fluentd.org/articles/in_http
-      type http
+      @type http
       port 9880
     </source>
 
     <source>
-      type monitor_agent
+      @type monitor_agent
       port 24220
     </source>
     <source>
-      type debug_agent
+      @type debug_agent
       port 24230
     </source>
 
     <match debug.*>
       # http://docs.fluentd.org/articles/out_stdout
-      type stdout
+      @type stdout
     </match>
   CONF
 

From 94de13c1c2dc8d9bc3237e08a7148998948016e9 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 14:57:35 +0900
Subject: [PATCH 017/137] Initialize instance variables when finish

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index b59acdc18..c44ace433 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -17,6 +17,8 @@ def show
 
   def finish
     @setting = target_class.new(setting_params)
+    @_used_param = {}
+    @_used_section = {}
     unless @setting.valid?
       return render "shared/settings/show"
     end

From 58f5ab3b102f989a958aded6ad786ef160e84a6b Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 14:58:02 +0900
Subject: [PATCH 018/137] Construct section class properly

In previous version, run merged block before calling init.

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/section.rb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/app/models/fluentd/setting/section.rb b/app/models/fluentd/setting/section.rb
index 4bfc7b8a8..41b8f1001 100644
--- a/app/models/fluentd/setting/section.rb
+++ b/app/models/fluentd/setting/section.rb
@@ -10,19 +10,23 @@ def inherited(klass)
             include Fluentd::Setting::SectionParser
             include Fluentd::Setting::PluginParameter
 
-            class_attribute :_klass, :_block
+            class_attribute :_klass, :_block, :_blocks
             class_attribute :section_name, :required, :multi, :alias
             self._klass = klass
+            self._blocks = []
           end
         end
 
         def init
           _klass.instance_eval(&_block)
+          _blocks.each do |b|
+            _klass.instance_eval(&b)
+          end
         end
 
         # Don't overwrite options
-        def merge(**options)
-          _klass.instance_eval(&_block)
+        def merge(**options, &block)
+          _blocks << block
         end
 
         def section?

From cecbe74576eb659a1773cac73132834134019e1b Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 14:59:05 +0900
Subject: [PATCH 019/137] Overwrite options of config parameters

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../concerns/fluentd/setting/section_parser.rb  | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/section_parser.rb b/app/models/concerns/fluentd/setting/section_parser.rb
index 6ae145cc7..59396e62d 100644
--- a/app/models/concerns/fluentd/setting/section_parser.rb
+++ b/app/models/concerns/fluentd/setting/section_parser.rb
@@ -10,7 +10,22 @@ def parse_section(name, definition)
               if _definition[:section]
                 parse_section(_param_name, _definition)
               else
-                config_param(_param_name, _definition[:type], **_definition.except(:type))
+                if self._types.key?(_param_name)
+                  if _definition.key?(:default)
+                    self._defaults[_param_name] = _definition[:default]
+                    self._required[_param_name] = false
+                    self.clear_validators! # We register PresenceValidator only
+                  end
+                  self._secrets[_param_name] = _definition[:secret] if _definition.key?(:secret)
+                  self._aliases[name] = _definition[:alias] if _definition.key?(:alias)
+                  self._deprecated_params[name] = _definition[:deprecated] if _definition.key?(:deprecated)
+                  self._obsoleted_params[name] = _definition[:obsoleted] if _definition.key?(:obsoleted)
+                  self._list[name] = _definition[:list] if _definition.key?(:list)
+                  self._value_types[name] = _definition[:value_types] if _definition.key?(:value_types)
+                  self._symbolize_keys = _definition[:symbolize_keys] if _definition.key?(:symbolize_keys)
+                else
+                  config_param(_param_name, _definition[:type], **_definition.except(:type))
+                end
               end
             end
           end

From 8e11483adfaab53e4673bbe5587c9812aed42bc0 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 14:59:44 +0900
Subject: [PATCH 020/137] Use `pattern` instead of `match`

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 spec/features/fluentd/setting/out_stdout_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/features/fluentd/setting/out_stdout_spec.rb b/spec/features/fluentd/setting/out_stdout_spec.rb
index 1f2e0668d..98a70c216 100644
--- a/spec/features/fluentd/setting/out_stdout_spec.rb
+++ b/spec/features/fluentd/setting/out_stdout_spec.rb
@@ -2,6 +2,6 @@
 
 describe "out_stdout", stub: :daemon do
   before { login_with exists_user }
-  it_should_behave_like "configurable daemon settings", "out_stdout", "match", "stdout.**"
+  it_should_behave_like "configurable daemon settings", "out_stdout", "pattern", "stdout.**"
 
 end

From 66e797733fe690c6c81d7c8614aabaad758401b9 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 15:00:11 +0900
Subject: [PATCH 021/137] Permit `pattern`

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index c44ace433..460a46ba3 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -68,6 +68,10 @@ def target_plugin_params
         keys.concat(definition.keys)
       end
     end
+    # <match TAG>/<filter TAG> TAG is not appeared in config_definition
+    if ["output", "filter"].include?(target_class.plugin_type)
+      keys.push(:pattern)
+    end
     keys
   end
 

From ea61957bcee5f757a4a4c570db208b62e20a25b3 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 15:11:03 +0900
Subject: [PATCH 022/137] Use Pattern instead of Match

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 spec/features/fluentd/setting/out_forward_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/features/fluentd/setting/out_forward_spec.rb b/spec/features/fluentd/setting/out_forward_spec.rb
index 170e483ed..bbd567c7b 100644
--- a/spec/features/fluentd/setting/out_forward_spec.rb
+++ b/spec/features/fluentd/setting/out_forward_spec.rb
@@ -6,7 +6,7 @@
   let(:type) { "out_forward" }
   let(:page_url) { send("daemon_setting_#{type}_path") }
   let(:form_values) { {
-    Match: "*",
+    Pattern: "*",
     Name: "name",
     Host: "host",
     Port: "9999",

From 3e24f7eff1e2fc6e9dac8f46878750c413bf84fb Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 15:24:50 +0900
Subject: [PATCH 023/137] Use out_file as secondary output

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/out_forward.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/models/fluentd/setting/out_forward.rb b/app/models/fluentd/setting/out_forward.rb
index 2d3aecacd..623b75636 100644
--- a/app/models/fluentd/setting/out_forward.rb
+++ b/app/models/fluentd/setting/out_forward.rb
@@ -5,6 +5,10 @@ class OutForward
 
       register_plugin("output", "forward")
 
+      config_section :secondary do
+        config_param :path, :string
+      end
+
       def self.initial_params
         {
           secondary: {

From 69938874404e89de931e4d74bba70bf39b95df79 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 16:34:47 +0900
Subject: [PATCH 024/137] Define permit_params to use strong parameters

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb   | 34 +------------------
 .../fluentd/setting/plugin_parameter.rb       | 17 ++++++++++
 2 files changed, 18 insertions(+), 33 deletions(-)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index 460a46ba3..917411900 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -36,7 +36,7 @@ def finish
   private
 
   def setting_params
-    params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_plugin_params)
+    params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_class.permit_params)
   end
 
   def initial_params
@@ -58,36 +58,4 @@ def target_plugin_name
   def plugin_setting_form_action_url(*args)
     send("finish_daemon_setting_#{target_plugin_name}_path", *args)
   end
-
-  def target_plugin_params
-    keys = []
-    target_class.config_definition.each do |name, definition|
-      if definition[:section]
-        keys.concat(parse_section_definition(definition))
-      else
-        keys.concat(definition.keys)
-      end
-    end
-    # <match TAG>/<filter TAG> TAG is not appeared in config_definition
-    if ["output", "filter"].include?(target_class.plugin_type)
-      keys.push(:pattern)
-    end
-    keys
-  end
-
-  def parse_section_definition(definition)
-    keys = []
-    definition.except(:section, :argument, :required, :multi, :alias).each do |name, _definition|
-      _keys = []
-      _definition.each do |key, __definition|
-        if __definition[:section]
-          _keys.push({ name => parse_section_definition(__definition) })
-        else
-          _keys.push(key)
-        end
-      end
-      keys.push({ name => _keys })
-    end
-    keys
-  end
 end
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index 2100541a4..7c8acdd6d 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -37,6 +37,23 @@ def column_type(name)
         def list_of(name)
           self._list[name]
         end
+
+        def permit_params
+          self.new # init
+          keys = self._types.keys
+          self._sections.each do |key, section|
+            keys << _permit_section(key, section)
+          end
+          keys
+        end
+
+        def _permit_section(key, section)
+          keys = { key => section._types.keys }
+          section._sections.each do |_key, _section|
+            keys << _permit_section(_key, _section)
+          end
+          keys
+        end
       end
     end
   end

From 7a09c95b2381f87f51e6b1393d4496c75cd9a9f2 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 28 May 2018 16:35:20 +0900
Subject: [PATCH 025/137] Drop TLS support for now

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/out_forward.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/models/fluentd/setting/out_forward.rb b/app/models/fluentd/setting/out_forward.rb
index 623b75636..eb7d6c8b5 100644
--- a/app/models/fluentd/setting/out_forward.rb
+++ b/app/models/fluentd/setting/out_forward.rb
@@ -19,6 +19,11 @@ def self.initial_params
         }
       end
 
+      # TODO overwrite this method to support transport parameter and transport section
+      # def self.permit_params
+      #   super
+      # end
+
       def common_options
         [
           :pattern, :server, :secondary,
@@ -28,7 +33,9 @@ def common_options
       def hidden_options
         [
           :inject, :buffer,
-          :host, :port
+          :host, :port,
+          # We don't support TLS configuration via fluentd-ui for now.
+          :transport, :tls_version, :tls_ciphers, :tls_insecure_mode, :tls_verify_hostname, :tls_cert_path
         ]
       end
     end

From c675753db4a962565fc140f913ecbb566d1cb236 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 15:28:02 +0900
Subject: [PATCH 026/137] Set default values to in_monitor_agent

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_monitor_agent.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/models/fluentd/setting/in_monitor_agent.rb b/app/models/fluentd/setting/in_monitor_agent.rb
index c9c123310..6e6167ca4 100644
--- a/app/models/fluentd/setting/in_monitor_agent.rb
+++ b/app/models/fluentd/setting/in_monitor_agent.rb
@@ -9,12 +9,15 @@ def self.initial_params
         {
           bind: "0.0.0.0",
           port: 24220,
+          emit_interval: 60,
+          include_config: true,
+          include_retry: true
         }
       end
 
       def common_options
         [
-          :bind, :port
+          :bind, :port, :tag
         ]
       end
     end

From 5971b57f23ba52f2564fbe6f81d5739189f58fae Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 15:28:59 +0900
Subject: [PATCH 027/137] Omit values same as default value when save
 configuration

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_config.rb  | 11 ++++++++++-
 .../concerns/fluentd/setting/plugin_parameter.rb      | 10 ++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb
index c016f8b2e..dc99575f2 100644
--- a/app/models/concerns/fluentd/setting/plugin_config.rb
+++ b/app/models/concerns/fluentd/setting/plugin_config.rb
@@ -45,13 +45,22 @@ def parse_attributes(attributes)
             elements << config_element(key, "", sub_attrs, sub_elements)
           end
         end
-        return params.to_h.reject{|key, value| value.blank? }, elements
+        return params.to_h.reject{|key, value| skip?(key.to_sym, value) }, elements
       end
 
       # copy from Fluent::Test::Helpers#config_element
       def config_element(name = 'test', argument = '', params = {}, elements = [])
         Fluent::Config::Element.new(name, argument, params, elements)
       end
+
+      def skip?(key, value)
+        return true if value.blank?
+        if self._defaults.key?(key)
+          reformat_value(key, self._defaults[key]) == reformat_value(key, value)
+        else
+          false
+        end
+      end
     end
   end
 end
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index 7c8acdd6d..d42e71330 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -29,6 +29,16 @@ def all_options
         self.class._types.keys + self.class._sections.keys
       end
 
+      def reformat_value(name, value)
+        type = column_type(name)
+        type_name = if type.is_a?(Fluentd::Setting::Type::Time)
+                      :time
+                    else
+                      type
+                    end
+        Fluent::Config::REFORMAT_VALUE.call(type_name, value)
+      end
+
       module ClassMethods
         def column_type(name)
           self._types[name]

From d69aefc00305eccf3c653538e980854264040536 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 15:35:55 +0900
Subject: [PATCH 028/137] in_http: Organize options

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_http.rb | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/models/fluentd/setting/in_http.rb b/app/models/fluentd/setting/in_http.rb
index 0ae0c8b2f..70919b260 100644
--- a/app/models/fluentd/setting/in_http.rb
+++ b/app/models/fluentd/setting/in_http.rb
@@ -17,13 +17,15 @@ def self.initial_params
 
       def common_options
         [
-          :bind, :port
+          :bind, :port, :add_http_headers, :add_remote_addr
         ]
       end
 
       def hidden_options
         [
-          :parse
+          :parse,
+          :backlog,
+          :blocking_timeout,
         ]
       end
     end

From bae141196f092c2b3ec7d8b8d14930691edc509a Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 15:39:34 +0900
Subject: [PATCH 029/137] Add margin-top to fluentd.common.finish button

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/settings/_form.html.haml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index 859ad8629..229b1644c 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -14,5 +14,4 @@
         #advanced-setting.collapse
           - @setting.advanced_options.each do |key|
             = field(f, key)
-  = f.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right"
-
+  = f.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right mt-3"

From 92dd3116a32e1db1569df56f74b6c0a167cd95c4 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 15:48:33 +0900
Subject: [PATCH 030/137] Remove unused module

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/helpers/fluentd/settings_helper.rb | 2 --
 1 file changed, 2 deletions(-)
 delete mode 100644 app/helpers/fluentd/settings_helper.rb

diff --git a/app/helpers/fluentd/settings_helper.rb b/app/helpers/fluentd/settings_helper.rb
deleted file mode 100644
index 9d2e93e3d..000000000
--- a/app/helpers/fluentd/settings_helper.rb
+++ /dev/null
@@ -1,2 +0,0 @@
-module Fluentd::SettingsHelper
-end

From f9a08018e6e12036b887adf4159094b915483cde Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 16:29:19 +0900
Subject: [PATCH 031/137] Display owned plugin's type as select box

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/helpers/settings_helper.rb | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index f1af4d448..7c3ff854d 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -33,10 +33,16 @@ def field_resolver(html, form, key, opts)
   def section_field(html, form, key, opts = {})
     klass = form.object.class._sections[key]
     children = form.object.__send__(key) || { "0" => {} }
+    # <parse>/<format> section is not multiple in most cases
+    multi = if [:parse, :format].include?(key)
+              false
+            else
+              klass.multi
+            end
 
     children.each do |index, child|
-      open_section_div(html, klass.multi) do |_html|
-        _html << append_and_remove_links if klass.multi
+      open_section_div(html, multi) do |_html|
+        _html << append_and_remove_links if multi
         _html << h(form.label(key))
         _html << section_fields(form, key, index, klass, child)
       end
@@ -54,7 +60,11 @@ def section_fields(form, key, index, klass, child)
     object = klass.new(child)
     form.fields_for("#{key}[#{index}]", object) do |ff|
       klass._types.keys.each do |kk|
-        html << field(ff, kk)
+        if kk == :type
+          html << owned_plugin_type_field(ff, kk, key)
+        else
+          html << field(ff, kk)
+        end
       end
     end
     html
@@ -115,4 +125,14 @@ def other_field(html, form, key, opts = {})
     html << h(form.label(key))
     html << form.text_field(key, class: "form-control")
   end
+
+  def owned_plugin_type_field(form, key, plugin_type)
+    plugin_registry = Fluent::Plugin.const_get("#{plugin_type.to_s.upcase}_REGISTRY")
+    html = '<div class="form-group">'
+    html << form.label(key)
+    html << " " # NOTE: Adding space for padding
+    html << form.select(key, plugin_registry.map.keys)
+    html << '</div>'
+    html
+  end
 end

From 204c3d9f42cb3a3b9083050c7abe0277cb63b898 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 16:29:39 +0900
Subject: [PATCH 032/137] in_http: Display parse type

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_http.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/models/fluentd/setting/in_http.rb b/app/models/fluentd/setting/in_http.rb
index 70919b260..9cc6a50b7 100644
--- a/app/models/fluentd/setting/in_http.rb
+++ b/app/models/fluentd/setting/in_http.rb
@@ -23,7 +23,6 @@ def common_options
 
       def hidden_options
         [
-          :parse,
           :backlog,
           :blocking_timeout,
         ]

From 5750349b1e9128b6be1bc7fcd6a84a34de8714cf Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 17:13:25 +0900
Subject: [PATCH 033/137] Add models for parser plugins

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_apache.rb   | 13 +++++++++++
 app/models/fluentd/setting/parser_apache2.rb  | 13 +++++++++++
 .../fluentd/setting/parser_apache_error.rb    | 13 +++++++++++
 app/models/fluentd/setting/parser_csv.rb      | 22 +++++++++++++++++++
 app/models/fluentd/setting/parser_json.rb     | 15 +++++++++++++
 app/models/fluentd/setting/parser_ltsv.rb     | 17 ++++++++++++++
 app/models/fluentd/setting/parser_msgpack.rb  | 13 +++++++++++
 .../fluentd/setting/parser_multiline.rb       | 13 +++++++++++
 app/models/fluentd/setting/parser_nginx.rb    | 13 +++++++++++
 app/models/fluentd/setting/parser_none.rb     | 13 +++++++++++
 app/models/fluentd/setting/parser_regexp.rb   | 13 +++++++++++
 app/models/fluentd/setting/parser_syslog.rb   | 13 +++++++++++
 app/models/fluentd/setting/parser_tsv.rb      | 13 +++++++++++
 13 files changed, 184 insertions(+)
 create mode 100644 app/models/fluentd/setting/parser_apache.rb
 create mode 100644 app/models/fluentd/setting/parser_apache2.rb
 create mode 100644 app/models/fluentd/setting/parser_apache_error.rb
 create mode 100644 app/models/fluentd/setting/parser_csv.rb
 create mode 100644 app/models/fluentd/setting/parser_json.rb
 create mode 100644 app/models/fluentd/setting/parser_ltsv.rb
 create mode 100644 app/models/fluentd/setting/parser_msgpack.rb
 create mode 100644 app/models/fluentd/setting/parser_multiline.rb
 create mode 100644 app/models/fluentd/setting/parser_nginx.rb
 create mode 100644 app/models/fluentd/setting/parser_none.rb
 create mode 100644 app/models/fluentd/setting/parser_regexp.rb
 create mode 100644 app/models/fluentd/setting/parser_syslog.rb
 create mode 100644 app/models/fluentd/setting/parser_tsv.rb

diff --git a/app/models/fluentd/setting/parser_apache.rb b/app/models/fluentd/setting/parser_apache.rb
new file mode 100644
index 000000000..b13d46731
--- /dev/null
+++ b/app/models/fluentd/setting/parser_apache.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserApache
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "apache")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_apache2.rb b/app/models/fluentd/setting/parser_apache2.rb
new file mode 100644
index 000000000..9cda33397
--- /dev/null
+++ b/app/models/fluentd/setting/parser_apache2.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserApache2
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "apache2")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_apache_error.rb b/app/models/fluentd/setting/parser_apache_error.rb
new file mode 100644
index 000000000..91671c04d
--- /dev/null
+++ b/app/models/fluentd/setting/parser_apache_error.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserApacheError
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "apache_error")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_csv.rb b/app/models/fluentd/setting/parser_csv.rb
new file mode 100644
index 000000000..74facc27c
--- /dev/null
+++ b/app/models/fluentd/setting/parser_csv.rb
@@ -0,0 +1,22 @@
+class Fluentd
+  module Setting
+    module ParserCsv
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "csv")
+
+      def self.initial_params
+        {
+          keys: nil,
+          delimiter: ","
+        }
+      end
+
+      def common_options
+        [
+          :keys, :delimiter
+        ]
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_json.rb b/app/models/fluentd/setting/parser_json.rb
new file mode 100644
index 000000000..4bdbc2179
--- /dev/null
+++ b/app/models/fluentd/setting/parser_json.rb
@@ -0,0 +1,15 @@
+class Fluentd
+  module Setting
+    module ParserJson
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "json")
+
+      def self.initial_params
+        {
+          json_parser: "oj"
+        }
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_ltsv.rb b/app/models/fluentd/setting/parser_ltsv.rb
new file mode 100644
index 000000000..4cb16c87d
--- /dev/null
+++ b/app/models/fluentd/setting/parser_ltsv.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module ParserLtsv
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "ltsv")
+
+      def self.initial_params
+        {
+          delimiter: "\t",
+          delimiter_pattern: nil,
+          label_delimiter: ":"
+        }
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_msgpack.rb b/app/models/fluentd/setting/parser_msgpack.rb
new file mode 100644
index 000000000..467e35923
--- /dev/null
+++ b/app/models/fluentd/setting/parser_msgpack.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserMsgpack
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "msgpack")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_multiline.rb b/app/models/fluentd/setting/parser_multiline.rb
new file mode 100644
index 000000000..f6a31125a
--- /dev/null
+++ b/app/models/fluentd/setting/parser_multiline.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserMultiline
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "multiline")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_nginx.rb b/app/models/fluentd/setting/parser_nginx.rb
new file mode 100644
index 000000000..9a409380d
--- /dev/null
+++ b/app/models/fluentd/setting/parser_nginx.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserNginx
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "nginx")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_none.rb b/app/models/fluentd/setting/parser_none.rb
new file mode 100644
index 000000000..5514b3138
--- /dev/null
+++ b/app/models/fluentd/setting/parser_none.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserNone
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "none")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_regexp.rb b/app/models/fluentd/setting/parser_regexp.rb
new file mode 100644
index 000000000..68ca7641f
--- /dev/null
+++ b/app/models/fluentd/setting/parser_regexp.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserRegexp
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "regexp")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_syslog.rb b/app/models/fluentd/setting/parser_syslog.rb
new file mode 100644
index 000000000..cf048d796
--- /dev/null
+++ b/app/models/fluentd/setting/parser_syslog.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserSyslog
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "syslog")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/parser_tsv.rb b/app/models/fluentd/setting/parser_tsv.rb
new file mode 100644
index 000000000..7c618bb1b
--- /dev/null
+++ b/app/models/fluentd/setting/parser_tsv.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module ParserTsv
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "tsv")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end

From 59ac41185cbe41e4f69ab509f0ee0bceb5c19654 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 5 Jun 2018 17:18:54 +0900
Subject: [PATCH 034/137] Add models for formatter plugins

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/formatter_csv.rb         | 13 +++++++++++++
 app/models/fluentd/setting/formatter_hash.rb        | 13 +++++++++++++
 app/models/fluentd/setting/formatter_json.rb        | 13 +++++++++++++
 app/models/fluentd/setting/formatter_ltsv.rb        | 13 +++++++++++++
 app/models/fluentd/setting/formatter_msgpack.rb     | 13 +++++++++++++
 app/models/fluentd/setting/formatter_out_file.rb    | 13 +++++++++++++
 .../fluentd/setting/formatter_single_value.rb       | 13 +++++++++++++
 app/models/fluentd/setting/formatter_stdout.rb      | 13 +++++++++++++
 app/models/fluentd/setting/formatter_tsv.rb         | 13 +++++++++++++
 9 files changed, 117 insertions(+)
 create mode 100644 app/models/fluentd/setting/formatter_csv.rb
 create mode 100644 app/models/fluentd/setting/formatter_hash.rb
 create mode 100644 app/models/fluentd/setting/formatter_json.rb
 create mode 100644 app/models/fluentd/setting/formatter_ltsv.rb
 create mode 100644 app/models/fluentd/setting/formatter_msgpack.rb
 create mode 100644 app/models/fluentd/setting/formatter_out_file.rb
 create mode 100644 app/models/fluentd/setting/formatter_single_value.rb
 create mode 100644 app/models/fluentd/setting/formatter_stdout.rb
 create mode 100644 app/models/fluentd/setting/formatter_tsv.rb

diff --git a/app/models/fluentd/setting/formatter_csv.rb b/app/models/fluentd/setting/formatter_csv.rb
new file mode 100644
index 000000000..4c1893538
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_csv.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterCsv
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "csv")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_hash.rb b/app/models/fluentd/setting/formatter_hash.rb
new file mode 100644
index 000000000..68dfc9e32
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_hash.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterHash
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "hash")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_json.rb b/app/models/fluentd/setting/formatter_json.rb
new file mode 100644
index 000000000..28ca08b1f
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_json.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterJson
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "json")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_ltsv.rb b/app/models/fluentd/setting/formatter_ltsv.rb
new file mode 100644
index 000000000..c60eda408
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_ltsv.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterLtsv
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "ltsv")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_msgpack.rb b/app/models/fluentd/setting/formatter_msgpack.rb
new file mode 100644
index 000000000..5bdfedb15
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_msgpack.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterMsgpack
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "msgpack")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_out_file.rb b/app/models/fluentd/setting/formatter_out_file.rb
new file mode 100644
index 000000000..3e28e6af6
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_out_file.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterOutFile
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "out_file")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_single_value.rb b/app/models/fluentd/setting/formatter_single_value.rb
new file mode 100644
index 000000000..03bfd902f
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_single_value.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterSingleValue
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "single_value")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_stdout.rb b/app/models/fluentd/setting/formatter_stdout.rb
new file mode 100644
index 000000000..5f6d0da4f
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_stdout.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterStdout
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "stdout")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/formatter_tsv.rb b/app/models/fluentd/setting/formatter_tsv.rb
new file mode 100644
index 000000000..e31ab77c6
--- /dev/null
+++ b/app/models/fluentd/setting/formatter_tsv.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    module FormatterTsv
+      include Fluentd::Setting::Plugin
+
+      register_plugin("formatter", "tsv")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end

From 893bafa548d5e5ae96bfff456f0c2f8fa48afbdc Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 6 Jun 2018 11:28:31 +0900
Subject: [PATCH 035/137] Indent

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/helpers/settings_helper.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index 7c3ff854d..3d8268e4f 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -127,7 +127,13 @@ def other_field(html, form, key, opts = {})
   end
 
   def owned_plugin_type_field(form, key, plugin_type)
-    plugin_registry = Fluent::Plugin.const_get("#{plugin_type.to_s.upcase}_REGISTRY")
+    registry_type = case plugin_type
+                    when :parse
+                      "PARSER_REGISTRY"
+                    when :format
+                      "FORMATTER_REGISTRY"
+                    end
+    plugin_registry = Fluent::Plugin.const_get("#{registry_type}")
     html = '<div class="form-group">'
     html << form.label(key)
     html << " " # NOTE: Adding space for padding

From 0edf43fc0cd5ec05f84f8a7ddc5c3c10ee1fa9e1 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 6 Jun 2018 11:29:15 +0900
Subject: [PATCH 036/137] Set class to select

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/helpers/settings_helper.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index 3d8268e4f..aca9fdf67 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -111,7 +111,7 @@ def child_data(form, key)
   def enum_field(html, form, key, opts = {})
     html << h(form.label(key))
     html << " " # NOTE: Adding space for padding
-    html << form.select(key, form.object.list_of(key), opts)
+    html << form.select(key, form.object.list_of(key), opts, { class: "enum" })
   end
 
   def bool_field(html, form, key, opts = {})
@@ -137,7 +137,7 @@ def owned_plugin_type_field(form, key, plugin_type)
     html = '<div class="form-group">'
     html << form.label(key)
     html << " " # NOTE: Adding space for padding
-    html << form.select(key, plugin_registry.map.keys)
+    html << form.select(key, plugin_registry.map.keys, {}, { class: "owned" })
     html << '</div>'
     html
   end

From ed3ede60fbbcbecaaa4a8d097ebb235c6fe7f6f2 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 6 Jun 2018 18:04:46 +0900
Subject: [PATCH 037/137] Fix typos

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/formatter_csv.rb          | 2 +-
 app/models/fluentd/setting/formatter_hash.rb         | 2 +-
 app/models/fluentd/setting/formatter_json.rb         | 2 +-
 app/models/fluentd/setting/formatter_ltsv.rb         | 2 +-
 app/models/fluentd/setting/formatter_msgpack.rb      | 2 +-
 app/models/fluentd/setting/formatter_out_file.rb     | 2 +-
 app/models/fluentd/setting/formatter_single_value.rb | 2 +-
 app/models/fluentd/setting/formatter_stdout.rb       | 2 +-
 app/models/fluentd/setting/formatter_tsv.rb          | 2 +-
 app/models/fluentd/setting/parser_apache.rb          | 2 +-
 app/models/fluentd/setting/parser_apache2.rb         | 2 +-
 app/models/fluentd/setting/parser_apache_error.rb    | 2 +-
 app/models/fluentd/setting/parser_csv.rb             | 2 +-
 app/models/fluentd/setting/parser_json.rb            | 2 +-
 app/models/fluentd/setting/parser_ltsv.rb            | 2 +-
 app/models/fluentd/setting/parser_msgpack.rb         | 2 +-
 app/models/fluentd/setting/parser_multiline.rb       | 2 +-
 app/models/fluentd/setting/parser_nginx.rb           | 2 +-
 app/models/fluentd/setting/parser_none.rb            | 2 +-
 app/models/fluentd/setting/parser_regexp.rb          | 2 +-
 app/models/fluentd/setting/parser_syslog.rb          | 2 +-
 app/models/fluentd/setting/parser_tsv.rb             | 2 +-
 22 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/app/models/fluentd/setting/formatter_csv.rb b/app/models/fluentd/setting/formatter_csv.rb
index 4c1893538..0a81b4c25 100644
--- a/app/models/fluentd/setting/formatter_csv.rb
+++ b/app/models/fluentd/setting/formatter_csv.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterCsv
+    class FormatterCsv
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "csv")
diff --git a/app/models/fluentd/setting/formatter_hash.rb b/app/models/fluentd/setting/formatter_hash.rb
index 68dfc9e32..b5e6f75e3 100644
--- a/app/models/fluentd/setting/formatter_hash.rb
+++ b/app/models/fluentd/setting/formatter_hash.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterHash
+    class FormatterHash
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "hash")
diff --git a/app/models/fluentd/setting/formatter_json.rb b/app/models/fluentd/setting/formatter_json.rb
index 28ca08b1f..61a14a4b2 100644
--- a/app/models/fluentd/setting/formatter_json.rb
+++ b/app/models/fluentd/setting/formatter_json.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterJson
+    class FormatterJson
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "json")
diff --git a/app/models/fluentd/setting/formatter_ltsv.rb b/app/models/fluentd/setting/formatter_ltsv.rb
index c60eda408..204c498a0 100644
--- a/app/models/fluentd/setting/formatter_ltsv.rb
+++ b/app/models/fluentd/setting/formatter_ltsv.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterLtsv
+    class FormatterLtsv
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "ltsv")
diff --git a/app/models/fluentd/setting/formatter_msgpack.rb b/app/models/fluentd/setting/formatter_msgpack.rb
index 5bdfedb15..482cfb406 100644
--- a/app/models/fluentd/setting/formatter_msgpack.rb
+++ b/app/models/fluentd/setting/formatter_msgpack.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterMsgpack
+    class FormatterMsgpack
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "msgpack")
diff --git a/app/models/fluentd/setting/formatter_out_file.rb b/app/models/fluentd/setting/formatter_out_file.rb
index 3e28e6af6..289d751cb 100644
--- a/app/models/fluentd/setting/formatter_out_file.rb
+++ b/app/models/fluentd/setting/formatter_out_file.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterOutFile
+    class FormatterOutFile
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "out_file")
diff --git a/app/models/fluentd/setting/formatter_single_value.rb b/app/models/fluentd/setting/formatter_single_value.rb
index 03bfd902f..65c3603eb 100644
--- a/app/models/fluentd/setting/formatter_single_value.rb
+++ b/app/models/fluentd/setting/formatter_single_value.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterSingleValue
+    class FormatterSingleValue
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "single_value")
diff --git a/app/models/fluentd/setting/formatter_stdout.rb b/app/models/fluentd/setting/formatter_stdout.rb
index 5f6d0da4f..f2d06dd72 100644
--- a/app/models/fluentd/setting/formatter_stdout.rb
+++ b/app/models/fluentd/setting/formatter_stdout.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterStdout
+    class FormatterStdout
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "stdout")
diff --git a/app/models/fluentd/setting/formatter_tsv.rb b/app/models/fluentd/setting/formatter_tsv.rb
index e31ab77c6..9a4461cea 100644
--- a/app/models/fluentd/setting/formatter_tsv.rb
+++ b/app/models/fluentd/setting/formatter_tsv.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module FormatterTsv
+    class FormatterTsv
       include Fluentd::Setting::Plugin
 
       register_plugin("formatter", "tsv")
diff --git a/app/models/fluentd/setting/parser_apache.rb b/app/models/fluentd/setting/parser_apache.rb
index b13d46731..6bdf6c498 100644
--- a/app/models/fluentd/setting/parser_apache.rb
+++ b/app/models/fluentd/setting/parser_apache.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserApache
+    class ParserApache
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "apache")
diff --git a/app/models/fluentd/setting/parser_apache2.rb b/app/models/fluentd/setting/parser_apache2.rb
index 9cda33397..76480ae6b 100644
--- a/app/models/fluentd/setting/parser_apache2.rb
+++ b/app/models/fluentd/setting/parser_apache2.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserApache2
+    class ParserApache2
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "apache2")
diff --git a/app/models/fluentd/setting/parser_apache_error.rb b/app/models/fluentd/setting/parser_apache_error.rb
index 91671c04d..43fa488ee 100644
--- a/app/models/fluentd/setting/parser_apache_error.rb
+++ b/app/models/fluentd/setting/parser_apache_error.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserApacheError
+    class ParserApacheError
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "apache_error")
diff --git a/app/models/fluentd/setting/parser_csv.rb b/app/models/fluentd/setting/parser_csv.rb
index 74facc27c..45ee3220d 100644
--- a/app/models/fluentd/setting/parser_csv.rb
+++ b/app/models/fluentd/setting/parser_csv.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserCsv
+    class ParserCsv
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "csv")
diff --git a/app/models/fluentd/setting/parser_json.rb b/app/models/fluentd/setting/parser_json.rb
index 4bdbc2179..83edc5dec 100644
--- a/app/models/fluentd/setting/parser_json.rb
+++ b/app/models/fluentd/setting/parser_json.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserJson
+    class ParserJson
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "json")
diff --git a/app/models/fluentd/setting/parser_ltsv.rb b/app/models/fluentd/setting/parser_ltsv.rb
index 4cb16c87d..2f9a313d2 100644
--- a/app/models/fluentd/setting/parser_ltsv.rb
+++ b/app/models/fluentd/setting/parser_ltsv.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserLtsv
+    class ParserLtsv
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "ltsv")
diff --git a/app/models/fluentd/setting/parser_msgpack.rb b/app/models/fluentd/setting/parser_msgpack.rb
index 467e35923..c89070ef0 100644
--- a/app/models/fluentd/setting/parser_msgpack.rb
+++ b/app/models/fluentd/setting/parser_msgpack.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserMsgpack
+    class ParserMsgpack
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "msgpack")
diff --git a/app/models/fluentd/setting/parser_multiline.rb b/app/models/fluentd/setting/parser_multiline.rb
index f6a31125a..783540684 100644
--- a/app/models/fluentd/setting/parser_multiline.rb
+++ b/app/models/fluentd/setting/parser_multiline.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserMultiline
+    class ParserMultiline
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "multiline")
diff --git a/app/models/fluentd/setting/parser_nginx.rb b/app/models/fluentd/setting/parser_nginx.rb
index 9a409380d..f06d5e055 100644
--- a/app/models/fluentd/setting/parser_nginx.rb
+++ b/app/models/fluentd/setting/parser_nginx.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserNginx
+    class ParserNginx
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "nginx")
diff --git a/app/models/fluentd/setting/parser_none.rb b/app/models/fluentd/setting/parser_none.rb
index 5514b3138..f41833804 100644
--- a/app/models/fluentd/setting/parser_none.rb
+++ b/app/models/fluentd/setting/parser_none.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserNone
+    class ParserNone
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "none")
diff --git a/app/models/fluentd/setting/parser_regexp.rb b/app/models/fluentd/setting/parser_regexp.rb
index 68ca7641f..0e9ccfbfc 100644
--- a/app/models/fluentd/setting/parser_regexp.rb
+++ b/app/models/fluentd/setting/parser_regexp.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserRegexp
+    class ParserRegexp
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "regexp")
diff --git a/app/models/fluentd/setting/parser_syslog.rb b/app/models/fluentd/setting/parser_syslog.rb
index cf048d796..9c88312a9 100644
--- a/app/models/fluentd/setting/parser_syslog.rb
+++ b/app/models/fluentd/setting/parser_syslog.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserSyslog
+    class ParserSyslog
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "syslog")
diff --git a/app/models/fluentd/setting/parser_tsv.rb b/app/models/fluentd/setting/parser_tsv.rb
index 7c618bb1b..18c334d1d 100644
--- a/app/models/fluentd/setting/parser_tsv.rb
+++ b/app/models/fluentd/setting/parser_tsv.rb
@@ -1,6 +1,6 @@
 class Fluentd
   module Setting
-    module ParserTsv
+    class ParserTsv
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "tsv")

From 9f75208c8842ca418c9fce71a17b8d79a0cc7f6d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 6 Jun 2018 18:05:22 +0900
Subject: [PATCH 038/137] Add parser_in_http

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_in_http.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 app/models/fluentd/setting/parser_in_http.rb

diff --git a/app/models/fluentd/setting/parser_in_http.rb b/app/models/fluentd/setting/parser_in_http.rb
new file mode 100644
index 000000000..b48bb143a
--- /dev/null
+++ b/app/models/fluentd/setting/parser_in_http.rb
@@ -0,0 +1,13 @@
+class Fluentd
+  module Setting
+    class ParserInHttp
+      include Fluentd::Setting::Plugin
+
+      register_plugin("parser", "in_http")
+
+      def self.initial_params
+        {}
+      end
+    end
+  end
+end

From 12c56811d24291b9e84134db13a19334d1925751 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:37:49 +0900
Subject: [PATCH 039/137] Add parameters to some parser plugins

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_json.rb   | 10 ++++++++++
 app/models/fluentd/setting/parser_syslog.rb | 16 ++++++++++++++++
 app/models/fluentd/setting/parser_tsv.rb    |  6 ++++++
 3 files changed, 32 insertions(+)

diff --git a/app/models/fluentd/setting/parser_json.rb b/app/models/fluentd/setting/parser_json.rb
index 83edc5dec..168c12739 100644
--- a/app/models/fluentd/setting/parser_json.rb
+++ b/app/models/fluentd/setting/parser_json.rb
@@ -10,6 +10,16 @@ def self.initial_params
           json_parser: "oj"
         }
       end
+
+      def common_options
+        []
+      end
+
+      def advanced_options
+        [
+          :json_parser
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/parser_syslog.rb b/app/models/fluentd/setting/parser_syslog.rb
index 9c88312a9..d3f9eeee5 100644
--- a/app/models/fluentd/setting/parser_syslog.rb
+++ b/app/models/fluentd/setting/parser_syslog.rb
@@ -4,10 +4,26 @@ class ParserSyslog
       include Fluentd::Setting::Plugin
 
       register_plugin("parser", "syslog")
+      # Overwrite type of time_format
+      config_param(:time_format, :string)
 
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :time_format,
+          :with_priority,
+        ]
+      end
+
+      def advanced_options
+        [
+          :message_format,
+          :rfc5424_time_format
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/parser_tsv.rb b/app/models/fluentd/setting/parser_tsv.rb
index 18c334d1d..fbb31258b 100644
--- a/app/models/fluentd/setting/parser_tsv.rb
+++ b/app/models/fluentd/setting/parser_tsv.rb
@@ -8,6 +8,12 @@ class ParserTsv
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :keys, :delimiter
+        ]
+      end
     end
   end
 end

From 6128141918b4039845f8d737e2eaf748854effee Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:38:24 +0900
Subject: [PATCH 040/137] Load Fluentd plugins

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 config/environment.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/config/environment.rb b/config/environment.rb
index 426333bb4..822deae95 100644
--- a/config/environment.rb
+++ b/config/environment.rb
@@ -1,5 +1,8 @@
 # Load the Rails application.
 require_relative 'application'
 
+# Load fluentd libraries & plugins
+require "fluent/load"
+
 # Initialize the Rails application.
 Rails.application.initialize!

From 0cb0484a6b93c1e0f84940395515a6cce06e27ac Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:39:05 +0900
Subject: [PATCH 041/137] Add parse_type and format_type attribute to owner
 plugins if needed

In previous version, the owner plugin that has parse section, it will define parse parameter twice.
The first one is String type and second one is Hash type. It will cause type mismatch error.

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/models/concerns/fluentd/setting/plugin.rb b/app/models/concerns/fluentd/setting/plugin.rb
index 351765501..9b2ad4977 100644
--- a/app/models/concerns/fluentd/setting/plugin.rb
+++ b/app/models/concerns/fluentd/setting/plugin.rb
@@ -30,6 +30,9 @@ def register_plugin(type, name)
             params.each do |param_name, definition|
               if definition[:section]
                 parse_section(param_name, definition)
+                if %i(parse format).include?(param_name)
+                  attribute("#{param_name}_type", :string)
+                end
               else
                 config_param(param_name, definition[:type], **definition.except(:type))
               end

From f6a3d9de3da2021f0a43ea062246d9d4d3f52204 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:42:57 +0900
Subject: [PATCH 042/137] Add methods to support parse section and format
 section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../fluentd/setting/plugin_parameter.rb       | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index d42e71330..c3a9527e4 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -29,6 +29,26 @@ def all_options
         self.class._types.keys + self.class._sections.keys
       end
 
+      def have_parse_section?
+        self.class._sections.key?(:parse)
+      end
+
+      def have_format_section?
+        self.class._sections.key?(:format)
+      end
+
+      def create_parser
+        return unless have_parse_section?
+        parser_class = Fluentd::Setting.const_get("parser_#{parse_type}".classify)
+        parser_class.new(parse["0"].except("type"))
+      end
+
+      def create_formatter
+        return unless have_format_section?
+        formatter_class = Fluentd::Setting.const_get("formatter_#{format_type}".classify)
+        formatter_class.new(format["0"].except("type"))
+      end
+
       def reformat_value(name, value)
         type = column_type(name)
         type_name = if type.is_a?(Fluentd::Setting::Type::Time)

From 0d01c76f4b08a67c85726418ee2c6372fd961e1a Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:43:35 +0900
Subject: [PATCH 043/137] Add FluentdFormBuilder

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/form_builders/fluentd_form_builder.rb | 76 +++++++++++++++++++++++
 config/application.rb                     |  2 +-
 2 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 app/form_builders/fluentd_form_builder.rb

diff --git a/app/form_builders/fluentd_form_builder.rb b/app/form_builders/fluentd_form_builder.rb
new file mode 100644
index 000000000..9b204d1ac
--- /dev/null
+++ b/app/form_builders/fluentd_form_builder.rb
@@ -0,0 +1,76 @@
+class FluentdFormBuilder < ActionView::Helpers::FormBuilder
+  include ActionView::Helpers::TagHelper
+  include ActionView::Context
+
+  def field(key, options = {})
+    plugin_class = object.class
+    content_tag(:div, class: "form-group") do
+      if plugin_class._sections[key]
+        render_section(key, options)
+      else
+        resolve_field(key, options)
+      end
+    end
+  end
+
+  private
+
+  def resolve_field(key, options = {})
+    plugin_class = object.class
+    column_type = plugin_class.column_type(key)
+    case column_type
+    when :enum
+      enum_field(key, options)
+    when :bool
+      bool_field(key, options)
+    else
+      other_field(key, options)
+    end
+  end
+
+  def enum_field(key, options)
+    label(key) + select(key, object.list_of(key), options, { class: "enum" })
+  end
+
+  def bool_field(key, options)
+    check_box(key, {}, "true", "false") + " " + label(key)
+  end
+
+  def other_field(key, options)
+    return unless object.respond_to?(key)
+    label(key) + text_field(key, class: "form-control")
+  end
+
+  def render_section(key, options)
+    section_class = object.class._sections[key]
+    children = object.__send__(key) || { "0" => {} }
+    html = ""
+
+    children.each do |index, child|
+      html << content_tag("div", class: "js-nested-column #{section_class.multi ? "js-multiple" : ""}") do
+        _html = ""
+        _html << append_and_remove_links if section_class.multi
+        _html << label(key)
+        _html << section_fields(key, index, section_class, child)
+        _html.html_safe
+      end
+    end
+    html.html_safe
+  end
+
+  def section_fields(key, index, section_class, child)
+    section = section_class.new(child)
+    fields("#{key}[#{index}]", model: section) do |section_form|
+      html = ""
+      section_class._types.keys.each do |section_key|
+        html << section_form.field(section_key)
+      end
+      html.html_safe
+    end
+  end
+
+  def append_and_remove_links
+    %Q!<a class="btn btn-xs btn-default js-append">#{icon('fa-plus')}</a> ! +
+    %Q!<a class="btn btn-xs btn-default js-remove" style="display:none">#{icon('fa-minus')}</a> !
+  end
+end
diff --git a/config/application.rb b/config/application.rb
index b93dc7a50..a0ee75f6d 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -46,7 +46,7 @@ class Application < Rails::Application
     # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
     config.i18n.default_locale = 'en'
     config.i18n.available_locales = %i(en ja)
-    config.autoload_paths += %W(#{config.root}/lib)
+    config.autoload_paths += %W(#{config.root}/lib #{Rails.root}/app/form_builders)
 
     config.active_job.queue_adapter = :sucker_punch
 

From 5f76e4ec63c2dd664646a9cab839606a72e3a040 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:44:57 +0900
Subject: [PATCH 044/137] Add partial template for parse and format section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/settings/_formatter_form.html.haml | 7 +++++++
 app/views/shared/settings/_parser_form.html.haml    | 7 +++++++
 2 files changed, 14 insertions(+)
 create mode 100644 app/views/shared/settings/_formatter_form.html.haml
 create mode 100644 app/views/shared/settings/_parser_form.html.haml

diff --git a/app/views/shared/settings/_formatter_form.html.haml b/app/views/shared/settings/_formatter_form.html.haml
new file mode 100644
index 000000000..0b85f5bfd
--- /dev/null
+++ b/app/views/shared/settings/_formatter_form.html.haml
@@ -0,0 +1,7 @@
+#format-section.form-group.card.bg-light.mb-3
+  .card-body
+    = form.label(:format_type, "Parse")
+    = form.select(:format_type, Fluent::Plugin::FORMATTER_REGISTRY.map.keys, {}, { class: "parse form-control" })
+    = form.fields("format[0]", model: formatter, class: ".form-group", builder: FluentdFormBuilder) do |formatter_form|
+      - formatter_form.object.common_options.each do |key|
+        = formatter_form.field(key)
diff --git a/app/views/shared/settings/_parser_form.html.haml b/app/views/shared/settings/_parser_form.html.haml
new file mode 100644
index 000000000..04642f088
--- /dev/null
+++ b/app/views/shared/settings/_parser_form.html.haml
@@ -0,0 +1,7 @@
+#parse-section.form-group.card.bg-light.mb-3
+  .card-body
+    = form.label(:parse_type, "Parse")
+    = form.select(:parse_type, Fluent::Plugin::PARSER_REGISTRY.map.keys, {}, { class: "parse form-control" })
+    = form.fields("parse[0]", model: parser, class: ".form-group", builder: FluentdFormBuilder) do |parser_form|
+      - parser_form.object.common_options.each do |key|
+        = parser_form.field(key)

From f1204bd2877110b05819c0c82a9e32790ff6505e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:45:51 +0900
Subject: [PATCH 045/137] Use new partial template and form_with

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/settings/_form.html.haml | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index 229b1644c..a15e5b580 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -1,9 +1,15 @@
 = render "shared/setting_errors"
 
 - # NOTE: plugin_setting_form_action_url is defined at SettingConcern
-= form_for(@setting, url: plugin_setting_form_action_url(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
+= form_with(model: @setting, url: plugin_setting_form_action_url(@fluentd), class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |form|
   - @setting.common_options.each do |key|
-    = field(f, key)
+    = form.field(key)
+
+  - if @setting.have_parse_section?
+    = render "shared/settings/parser_form", form: form, parser: @parser
+
+  - if @setting.have_format_section?
+    = render "shared/settings/formatter_form", form: form, formatter: @formatter
 
   - if @setting.advanced_options.present?
     .card.bg-light
@@ -13,5 +19,6 @@
           = t('terms.advanced_setting')
         #advanced-setting.collapse
           - @setting.advanced_options.each do |key|
-            = field(f, key)
-  = f.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right mt-3"
+            = form.field(key)
+
+  = form.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right mt-3"

From 2e3f24224aaf8e387fafd7f922e012769c410763 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:46:20 +0900
Subject: [PATCH 046/137] Add owned_plugin_section.js

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_section.js | 35 ++++++++++++++++++++
 app/views/shared/settings/show.html.haml     |  1 +
 2 files changed, 36 insertions(+)
 create mode 100644 app/javascript/packs/owned_plugin_section.js

diff --git a/app/javascript/packs/owned_plugin_section.js b/app/javascript/packs/owned_plugin_section.js
new file mode 100644
index 000000000..64b51236c
--- /dev/null
+++ b/app/javascript/packs/owned_plugin_section.js
@@ -0,0 +1,35 @@
+import "lodash/lodash"
+$(document).ready(() => {
+  let configUrl = (name) => {
+    return `/daemon/setting/${name}/configure`
+  }
+  function ownedSectionOnChange() {
+    const pluginName = _.last(document.documentURI.replace(/\/configure$/, "").split("/"))
+    $("#parse-section select").on("change", (event) => {
+      $.ajax({
+        url: configUrl(pluginName),
+        method: "GET",
+        data: {
+          parse_type: $(event.target).val()
+        }
+      }).then((data) => {
+        $("#parse-section").html($(data).find("#parse-section").html())
+        ownedSectionOnChange()
+      })
+    })
+    $("#format-section select").on("change", (event) => {
+      $.ajax({
+        url: configUrl(pluginName),
+        method: "GET",
+        data: {
+          format_type: $(event.target).val()
+        }
+      }).then((data) => {
+        $("#format-section").html($(data).find("#format-section").html())
+        ownedSectionOnChange()
+      })
+    })
+  }
+
+  ownedSectionOnChange()
+})
diff --git a/app/views/shared/settings/show.html.haml b/app/views/shared/settings/show.html.haml
index 89d9f0177..6c16c75de 100644
--- a/app/views/shared/settings/show.html.haml
+++ b/app/views/shared/settings/show.html.haml
@@ -1,5 +1,6 @@
 - page_title t("fluentd.settings.#{target_plugin_name}.show.page_title")
 - add_javascript_pack_tag("nested_settings")
+- add_javascript_pack_tag("owned_plugin_section")
 
 .card.mb-3
   .card-body.bg-light

From 89296a331d4fe411e7811e335dad045b6acf1be3 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:46:46 +0900
Subject: [PATCH 047/137] Organize some input plugins' parameters

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_http.rb   | 7 +++++++
 app/models/fluentd/setting/in_syslog.rb | 9 ++++++++-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/app/models/fluentd/setting/in_http.rb b/app/models/fluentd/setting/in_http.rb
index 9cc6a50b7..be3052880 100644
--- a/app/models/fluentd/setting/in_http.rb
+++ b/app/models/fluentd/setting/in_http.rb
@@ -12,6 +12,12 @@ def self.initial_params
           body_size_limit: "32m",
           keepalive_timeout: "10s",
           add_http_headers: false,
+          parse_type: "in_http",
+          parse: {
+            "0" => {
+              "type" => "in_http"
+            }
+          }
         }
       end
 
@@ -23,6 +29,7 @@ def common_options
 
       def hidden_options
         [
+          :parse,
           :backlog,
           :blocking_timeout,
         ]
diff --git a/app/models/fluentd/setting/in_syslog.rb b/app/models/fluentd/setting/in_syslog.rb
index 9366dd1fa..8ed0f9d8c 100644
--- a/app/models/fluentd/setting/in_syslog.rb
+++ b/app/models/fluentd/setting/in_syslog.rb
@@ -9,9 +9,10 @@ def self.initial_params
         {
           bind: "0.0.0.0",
           port: 5140,
+          parse_type: "syslog",
           parse: {
             "0" => {
-              type: :syslog
+              "type" => "syslog"
             }
           },
           protocol_type: :udp,
@@ -23,6 +24,12 @@ def common_options
           :tag, :bind, :port
         ]
       end
+
+      def hidden_options
+        [
+          :parse
+        ]
+      end
     end
   end
 end

From 421cbdd1af27e216e05c8f210e1c40295409a09e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:47:15 +0900
Subject: [PATCH 048/137] Avoid unsupported type error

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_parameter.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index c3a9527e4..179746cc7 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -51,6 +51,8 @@ def create_formatter
 
       def reformat_value(name, value)
         type = column_type(name)
+        return value if type == :enum
+        return value if type == :regexp
         type_name = if type.is_a?(Fluentd::Setting::Type::Time)
                       :time
                     else

From 6ef0380b3eaa854cdff3c8535b0b4488ed666f60 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 16:47:52 +0900
Subject: [PATCH 049/137] Add configure action to configure parse/format
 section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb | 10 ++++++++++
 config/routes.rb                            |  5 +++++
 2 files changed, 15 insertions(+)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index 917411900..a254622d7 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -10,11 +10,21 @@ module SettingConcern
 
   def show
     @setting = target_class.new(initial_params)
+    @parser = @setting.create_parser
+    @formatter = @setting.create_formatter
     @_used_param = {}
     @_used_section = {}
     render "shared/settings/show"
   end
 
+  def configure
+    owned_keys = %i(parse_type format_type)
+    @setting = target_class.new(initial_params.merge(params.permit(*owned_keys).slice(*owned_keys)))
+    @parser = @setting.create_parser
+    @formatter = @setting.create_formatter
+    render "shared/settings/show"
+  end
+
   def finish
     @setting = target_class.new(setting_params)
     @_used_param = {}
diff --git a/config/routes.rb b/config/routes.rb
index 31e6c5c81..14d5464fd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -20,12 +20,14 @@
 
         resource :in_tail, only: [:show], module: :settings, controller: :in_tail do
           post "after_file_choose"
+          # get "configure"
           post "after_format"
           post "confirm"
           post "finish"
         end
 
         resource :in_syslog, only: [:show], module: :settings, controller: :in_syslog do
+          get "configure"
           post "finish"
         end
 
@@ -34,6 +36,7 @@
         end
 
         resource :in_http, only: [:show], module: :settings, controller: :in_http do
+          get "configure"
           post "finish"
         end
 
@@ -42,6 +45,7 @@
         end
 
         resource :out_stdout, only: [:show], module: :settings, controller: :out_stdout do
+          get "configure"
           post "finish"
         end
 
@@ -54,6 +58,7 @@
         end
 
         resource :out_s3, only: [:show], module: :settings, controller: :out_s3 do
+          get "configure"
           post "finish"
         end
 

From e4a33d21aea90373e921e2fada6ca61a38a62c8f Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 17:23:22 +0900
Subject: [PATCH 050/137] Add Fluentd::Setting::Type::Regexp

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/type/regexp.rb | 17 +++++++++++++++++
 config/initializers/types.rb              |  1 +
 2 files changed, 18 insertions(+)
 create mode 100644 app/models/fluentd/setting/type/regexp.rb

diff --git a/app/models/fluentd/setting/type/regexp.rb b/app/models/fluentd/setting/type/regexp.rb
new file mode 100644
index 000000000..8210b14a4
--- /dev/null
+++ b/app/models/fluentd/setting/type/regexp.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    module Type
+      class Regexp < ActiveModel::Type::Value
+        def type
+          :regexp
+        end
+
+        private
+
+        def cast_value(value)
+          value
+        end
+      end
+    end
+  end
+end
diff --git a/config/initializers/types.rb b/config/initializers/types.rb
index 598f80e8c..1e20ff6fa 100644
--- a/config/initializers/types.rb
+++ b/config/initializers/types.rb
@@ -2,5 +2,6 @@
 ActiveModel::Type.register(:enum, Fluentd::Setting::Type::Enum)
 ActiveModel::Type.register(:bool, Fluentd::Setting::Type::Bool)
 ActiveModel::Type.register(:hash, Fluentd::Setting::Type::Hash)
+ActiveModel::Type.register(:regexp, Fluentd::Setting::Type::Hash)
 ActiveModel::Type.register(:size, Fluentd::Setting::Type::Size)
 ActiveModel::Type.register(:section, Fluentd::Setting::Type::Section)

From adc128487fcc352b9b81d9fe6ec8a73d76560ef7 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 17:25:07 +0900
Subject: [PATCH 051/137] Define options for parser_regexp

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_regexp.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/app/models/fluentd/setting/parser_regexp.rb b/app/models/fluentd/setting/parser_regexp.rb
index 0e9ccfbfc..b1b9f2355 100644
--- a/app/models/fluentd/setting/parser_regexp.rb
+++ b/app/models/fluentd/setting/parser_regexp.rb
@@ -8,6 +8,19 @@ class ParserRegexp
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :expression
+        ]
+      end
+
+      def hidden_options
+        [
+          :ignorecase,
+          :multiline
+        ]
+      end
     end
   end
 end

From acf4004b798a5bae20e0786fceb6124ebb02dd83 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 17:27:33 +0900
Subject: [PATCH 052/137] Define options for parser_ltsv

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_ltsv.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/app/models/fluentd/setting/parser_ltsv.rb b/app/models/fluentd/setting/parser_ltsv.rb
index 2f9a313d2..b4cc82937 100644
--- a/app/models/fluentd/setting/parser_ltsv.rb
+++ b/app/models/fluentd/setting/parser_ltsv.rb
@@ -12,6 +12,19 @@ def self.initial_params
           label_delimiter: ":"
         }
       end
+
+      def common_options
+        [
+          :delimiter,
+          :label_delimiter
+        ]
+      end
+
+      def advanced_options
+        [
+          :delimiter_pattern
+        ]
+      end
     end
   end
 end

From 20613928c5716edc9007b760b0e090cea470719d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 17:36:44 +0900
Subject: [PATCH 053/137] Define options for parser_multiline

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_multiline.rb | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/app/models/fluentd/setting/parser_multiline.rb b/app/models/fluentd/setting/parser_multiline.rb
index 783540684..0cc7e551f 100644
--- a/app/models/fluentd/setting/parser_multiline.rb
+++ b/app/models/fluentd/setting/parser_multiline.rb
@@ -5,9 +5,20 @@ class ParserMultiline
 
       register_plugin("parser", "multiline")
 
+      FORMAT_MAX_NUM = 20
+
+      (1..FORMAT_MAX_NUM).each do |n|
+        config_param("format#{n}", :string)
+      end
+
       def self.initial_params
         {}
       end
+
+      def common_options
+        [:format_firstline] +
+          (1..FORMAT_MAX_NUM).to_a.map{|n| "format#{n}".to_sym }
+      end
     end
   end
 end

From 35049a7d7cc7a6df3d0fae3df2debf36ccc2a88e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 17:36:56 +0900
Subject: [PATCH 054/137] Define options for parser_none

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/parser_none.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/models/fluentd/setting/parser_none.rb b/app/models/fluentd/setting/parser_none.rb
index f41833804..d914200b2 100644
--- a/app/models/fluentd/setting/parser_none.rb
+++ b/app/models/fluentd/setting/parser_none.rb
@@ -8,6 +8,12 @@ class ParserNone
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :message_key
+        ]
+      end
     end
   end
 end

From 448b5fd1300be8d74e92adc815047286d144afde Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 7 Jun 2018 17:41:55 +0900
Subject: [PATCH 055/137] Define options for formatter plugins

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/formatter_csv.rb          |  8 ++++++++
 app/models/fluentd/setting/formatter_hash.rb         |  6 ++++++
 app/models/fluentd/setting/formatter_json.rb         | 12 ++++++++++++
 app/models/fluentd/setting/formatter_ltsv.rb         |  8 ++++++++
 app/models/fluentd/setting/formatter_out_file.rb     |  8 ++++++++
 app/models/fluentd/setting/formatter_single_value.rb |  7 +++++++
 app/models/fluentd/setting/formatter_stdout.rb       |  6 ++++++
 app/models/fluentd/setting/formatter_tsv.rb          |  8 ++++++++
 8 files changed, 63 insertions(+)

diff --git a/app/models/fluentd/setting/formatter_csv.rb b/app/models/fluentd/setting/formatter_csv.rb
index 0a81b4c25..c567f0ef6 100644
--- a/app/models/fluentd/setting/formatter_csv.rb
+++ b/app/models/fluentd/setting/formatter_csv.rb
@@ -8,6 +8,14 @@ class FormatterCsv
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :delimiter,
+          :fields,
+          :add_newline
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_hash.rb b/app/models/fluentd/setting/formatter_hash.rb
index b5e6f75e3..dc61f45fc 100644
--- a/app/models/fluentd/setting/formatter_hash.rb
+++ b/app/models/fluentd/setting/formatter_hash.rb
@@ -8,6 +8,12 @@ class FormatterHash
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :add_newline
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_json.rb b/app/models/fluentd/setting/formatter_json.rb
index 61a14a4b2..87c2d4b6f 100644
--- a/app/models/fluentd/setting/formatter_json.rb
+++ b/app/models/fluentd/setting/formatter_json.rb
@@ -8,6 +8,18 @@ class FormatterJson
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :add_newline
+        ]
+      end
+
+      def advanced_options
+        [
+          :json_parser
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_ltsv.rb b/app/models/fluentd/setting/formatter_ltsv.rb
index 204c498a0..84189a13b 100644
--- a/app/models/fluentd/setting/formatter_ltsv.rb
+++ b/app/models/fluentd/setting/formatter_ltsv.rb
@@ -8,6 +8,14 @@ class FormatterLtsv
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :delimiter,
+          :label_delimiter,
+          :add_newline
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_out_file.rb b/app/models/fluentd/setting/formatter_out_file.rb
index 289d751cb..40239bdf1 100644
--- a/app/models/fluentd/setting/formatter_out_file.rb
+++ b/app/models/fluentd/setting/formatter_out_file.rb
@@ -8,6 +8,14 @@ class FormatterOutFile
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :output_time,
+          :output_tag,
+          :delimiter
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_single_value.rb b/app/models/fluentd/setting/formatter_single_value.rb
index 65c3603eb..9839093ea 100644
--- a/app/models/fluentd/setting/formatter_single_value.rb
+++ b/app/models/fluentd/setting/formatter_single_value.rb
@@ -8,6 +8,13 @@ class FormatterSingleValue
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :message_key,
+          :add_newline
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_stdout.rb b/app/models/fluentd/setting/formatter_stdout.rb
index f2d06dd72..1e392370c 100644
--- a/app/models/fluentd/setting/formatter_stdout.rb
+++ b/app/models/fluentd/setting/formatter_stdout.rb
@@ -8,6 +8,12 @@ class FormatterStdout
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :output_type
+        ]
+      end
     end
   end
 end
diff --git a/app/models/fluentd/setting/formatter_tsv.rb b/app/models/fluentd/setting/formatter_tsv.rb
index 9a4461cea..24c0b1f17 100644
--- a/app/models/fluentd/setting/formatter_tsv.rb
+++ b/app/models/fluentd/setting/formatter_tsv.rb
@@ -8,6 +8,14 @@ class FormatterTsv
       def self.initial_params
         {}
       end
+
+      def common_options
+        [
+          :keys,
+          :delimiter,
+          :add_newline
+        ]
+      end
     end
   end
 end

From 9cbc616624e80f9f286a9ec7525af4544642832d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 09:37:01 +0900
Subject: [PATCH 056/137] Support buffer section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb   |  2 ++
 .../fluentd/setting/plugin_parameter.rb       | 10 +++++++++
 app/models/fluentd/setting/buffer_file.rb     | 21 +++++++++++++++++++
 app/models/fluentd/setting/buffer_memory.rb   | 17 +++++++++++++++
 .../shared/settings/_buffer_form.html.haml    |  7 +++++++
 app/views/shared/settings/_form.html.haml     |  3 +++
 6 files changed, 60 insertions(+)
 create mode 100644 app/models/fluentd/setting/buffer_file.rb
 create mode 100644 app/models/fluentd/setting/buffer_memory.rb
 create mode 100644 app/views/shared/settings/_buffer_form.html.haml

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index a254622d7..32854a9f7 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -10,6 +10,7 @@ module SettingConcern
 
   def show
     @setting = target_class.new(initial_params)
+    @buffer = @setting.create_buffer
     @parser = @setting.create_parser
     @formatter = @setting.create_formatter
     @_used_param = {}
@@ -20,6 +21,7 @@ def show
   def configure
     owned_keys = %i(parse_type format_type)
     @setting = target_class.new(initial_params.merge(params.permit(*owned_keys).slice(*owned_keys)))
+    @buffer = @setting.create_buffer
     @parser = @setting.create_parser
     @formatter = @setting.create_formatter
     render "shared/settings/show"
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index 179746cc7..f388525d5 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -29,6 +29,10 @@ def all_options
         self.class._types.keys + self.class._sections.keys
       end
 
+      def have_buffer_section?
+        self.class._sections.key?(:buffer)
+      end
+
       def have_parse_section?
         self.class._sections.key?(:parse)
       end
@@ -37,6 +41,12 @@ def have_format_section?
         self.class._sections.key?(:format)
       end
 
+      def create_buffer
+        return unless have_buffer_section?
+        buffer_class = Fluentd::Setting.const_get("buffer_#{buffer_type}".classify)
+        buffer_class.new(buffer["0"].except("type"))
+      end
+
       def create_parser
         return unless have_parse_section?
         parser_class = Fluentd::Setting.const_get("parser_#{parse_type}".classify)
diff --git a/app/models/fluentd/setting/buffer_file.rb b/app/models/fluentd/setting/buffer_file.rb
new file mode 100644
index 000000000..31a74f7d0
--- /dev/null
+++ b/app/models/fluentd/setting/buffer_file.rb
@@ -0,0 +1,21 @@
+class Fluentd
+  module Setting
+    class BufferFile
+      include Fluentd::Setting::Plugin
+
+      register_plugin("buffer", "file")
+
+      def self.initial_params
+        {}
+      end
+
+      def common_options
+        [
+          :path,
+          :file_permission,
+          :dir_permission
+        ]
+      end
+    end
+  end
+end
diff --git a/app/models/fluentd/setting/buffer_memory.rb b/app/models/fluentd/setting/buffer_memory.rb
new file mode 100644
index 000000000..9abe3a9f3
--- /dev/null
+++ b/app/models/fluentd/setting/buffer_memory.rb
@@ -0,0 +1,17 @@
+class Fluentd
+  module Setting
+    class BufferMemory
+      include Fluentd::Setting::Plugin
+
+      register_plugin("buffer", "memory")
+
+      def self.initial_params
+        {}
+      end
+
+      def common_options
+        []
+      end
+    end
+  end
+end
diff --git a/app/views/shared/settings/_buffer_form.html.haml b/app/views/shared/settings/_buffer_form.html.haml
new file mode 100644
index 000000000..a88d5d741
--- /dev/null
+++ b/app/views/shared/settings/_buffer_form.html.haml
@@ -0,0 +1,7 @@
+#buffer-section.form-group.card.bg-light.mb-3
+  .card-body
+    = form.label(:buffer_type, "Buffer")
+    = form.select(:buffer_type, Fluent::Plugin::BUFFER_REGISTRY.map.keys, {}, { class: "buffer form-control" })
+    = form.fields("buffer[0]", model: buffer, class: ".form-group", builder: FluentdFormBuilder) do |buffer_form|
+      - buffer_form.object.common_options.each do |key|
+        = buffer_form.field(key)
diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index a15e5b580..514328de3 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -5,6 +5,9 @@
   - @setting.common_options.each do |key|
     = form.field(key)
 
+  - if @setting.have_buffer_section?
+    = render "shared/settings/buffer_form", form: form, buffer: @buffer
+
   - if @setting.have_parse_section?
     = render "shared/settings/parser_form", form: form, parser: @parser
 

From b14da4174fea2eb5928b81320fabde7b22c0ac4a Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 09:40:00 +0900
Subject: [PATCH 057/137] Support storage section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb   |  2 ++
 .../fluentd/setting/plugin_parameter.rb       | 10 +++++++++
 app/models/fluentd/setting/storage_local.rb   | 22 +++++++++++++++++++
 app/views/shared/settings/_form.html.haml     |  3 +++
 .../shared/settings/_storage_form.html.haml   |  7 ++++++
 5 files changed, 44 insertions(+)
 create mode 100644 app/models/fluentd/setting/storage_local.rb
 create mode 100644 app/views/shared/settings/_storage_form.html.haml

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index 32854a9f7..c63360130 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -11,6 +11,7 @@ module SettingConcern
   def show
     @setting = target_class.new(initial_params)
     @buffer = @setting.create_buffer
+    @storage = @setting.create_storage
     @parser = @setting.create_parser
     @formatter = @setting.create_formatter
     @_used_param = {}
@@ -22,6 +23,7 @@ def configure
     owned_keys = %i(parse_type format_type)
     @setting = target_class.new(initial_params.merge(params.permit(*owned_keys).slice(*owned_keys)))
     @buffer = @setting.create_buffer
+    @storage = @setting.create_storage
     @parser = @setting.create_parser
     @formatter = @setting.create_formatter
     render "shared/settings/show"
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index f388525d5..26edeff0c 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -33,6 +33,10 @@ def have_buffer_section?
         self.class._sections.key?(:buffer)
       end
 
+      def have_storage_section?
+        self.class._sections.key?(:storage)
+      end
+
       def have_parse_section?
         self.class._sections.key?(:parse)
       end
@@ -47,6 +51,12 @@ def create_buffer
         buffer_class.new(buffer["0"].except("type"))
       end
 
+      def create_storage
+        return unless have_storage_section?
+        storage_class = Fluentd::Setting.const_get("storage_#{storage_type}".classify)
+        storage_class.new(storage["0"].except("type"))
+      end
+
       def create_parser
         return unless have_parse_section?
         parser_class = Fluentd::Setting.const_get("parser_#{parse_type}".classify)
diff --git a/app/models/fluentd/setting/storage_local.rb b/app/models/fluentd/setting/storage_local.rb
new file mode 100644
index 000000000..a6cc0f1f3
--- /dev/null
+++ b/app/models/fluentd/setting/storage_local.rb
@@ -0,0 +1,22 @@
+class Fluentd
+  module Setting
+    class StorageLocal
+      include Fluentd::Setting::Plugin
+
+      register_plugin("storage", "local")
+
+      def self.initial_params
+        {}
+      end
+
+      def common_options
+        [
+          :path,
+          :mode,
+          :dir_mode,
+          :pretty_print
+        ]
+      end
+    end
+  end
+end
diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index 514328de3..d81f899f1 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -8,6 +8,9 @@
   - if @setting.have_buffer_section?
     = render "shared/settings/buffer_form", form: form, buffer: @buffer
 
+  - if @setting.have_storage_section?
+    = render "shared/settings/storage_form", form: form, storage: @storage
+
   - if @setting.have_parse_section?
     = render "shared/settings/parser_form", form: form, parser: @parser
 
diff --git a/app/views/shared/settings/_storage_form.html.haml b/app/views/shared/settings/_storage_form.html.haml
new file mode 100644
index 000000000..e91651f0d
--- /dev/null
+++ b/app/views/shared/settings/_storage_form.html.haml
@@ -0,0 +1,7 @@
+#storage-section.form-group.card.bg-light.mb-3
+  .card-body
+    = form.label(:storage_type, "Storage")
+    = form.select(:storage_type, Fluent::Plugin::STORAGE_REGISTRY.map.keys, {}, { class: "storage form-control" })
+    = form.fields("storage[0]", model: storage, class: ".form-group", builder: FluentdFormBuilder) do |storage_form|
+      - storage_form.object.common_options.each do |key|
+        = storage_form.field(key)

From f4ceb7bd9aebc31a6d35091a6d23f62c84269070 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 10:12:07 +0900
Subject: [PATCH 058/137] Display description on tooltip

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/form_builders/fluentd_form_builder.rb             | 11 +++++++----
 app/javascript/packs/tooltip.js                       |  3 +++
 app/models/concerns/fluentd/setting/configurable.rb   |  1 +
 .../concerns/fluentd/setting/plugin_parameter.rb      |  4 ++++
 app/views/shared/settings/show.html.haml              |  1 +
 5 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 app/javascript/packs/tooltip.js

diff --git a/app/form_builders/fluentd_form_builder.rb b/app/form_builders/fluentd_form_builder.rb
index 9b204d1ac..d05f1f178 100644
--- a/app/form_builders/fluentd_form_builder.rb
+++ b/app/form_builders/fluentd_form_builder.rb
@@ -29,16 +29,19 @@ def resolve_field(key, options = {})
   end
 
   def enum_field(key, options)
-    label(key) + select(key, object.list_of(key), options, { class: "enum" })
+    label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) +
+      select(key, object.list_of(key), options, { class: "enum" })
   end
 
   def bool_field(key, options)
-    check_box(key, {}, "true", "false") + " " + label(key)
+    check_box(key, {}, "true", "false") + " " +
+      label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key))
   end
 
   def other_field(key, options)
     return unless object.respond_to?(key)
-    label(key) + text_field(key, class: "form-control")
+    label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) +
+      text_field(key, class: "form-control")
   end
 
   def render_section(key, options)
@@ -50,7 +53,7 @@ def render_section(key, options)
       html << content_tag("div", class: "js-nested-column #{section_class.multi ? "js-multiple" : ""}") do
         _html = ""
         _html << append_and_remove_links if section_class.multi
-        _html << label(key)
+        _html << label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key))
         _html << section_fields(key, index, section_class, child)
         _html.html_safe
       end
diff --git a/app/javascript/packs/tooltip.js b/app/javascript/packs/tooltip.js
new file mode 100644
index 000000000..3f3560fc8
--- /dev/null
+++ b/app/javascript/packs/tooltip.js
@@ -0,0 +1,3 @@
+$(document).ready(() => {
+  $("label[data-toggle=tooltip]").tooltip()
+})
diff --git a/app/models/concerns/fluentd/setting/configurable.rb b/app/models/concerns/fluentd/setting/configurable.rb
index cab76ba80..669067ed0 100644
--- a/app/models/concerns/fluentd/setting/configurable.rb
+++ b/app/models/concerns/fluentd/setting/configurable.rb
@@ -71,6 +71,7 @@ def config_param(name, type = ActiveModel::Type::Value.new, **options)
             validates(name, presence: true) if options[:required]
           end
           self._types[name] = type
+          self._descriptions[name] = options[:desc] if options.key?(:desc)
           self._defaults[name] = options[:default] if options.key?(:default)
           self._secrets[name] = options[:secret] if options.key?(:secret)
           self._aliases[name] = options[:alias] if options.key?(:alias)
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index 26edeff0c..e009f0ded 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -13,6 +13,10 @@ def list_of(name)
         self.class._list[name]
       end
 
+      def desc(name)
+        self.class._descriptions[name]
+      end
+
       def common_options
         []
       end
diff --git a/app/views/shared/settings/show.html.haml b/app/views/shared/settings/show.html.haml
index 6c16c75de..cc7753ac7 100644
--- a/app/views/shared/settings/show.html.haml
+++ b/app/views/shared/settings/show.html.haml
@@ -1,6 +1,7 @@
 - page_title t("fluentd.settings.#{target_plugin_name}.show.page_title")
 - add_javascript_pack_tag("nested_settings")
 - add_javascript_pack_tag("owned_plugin_section")
+- add_javascript_pack_tag("tooltip")
 
 .card.mb-3
   .card-body.bg-light

From daf5bf05be58fb51e1a3382f49f51b8a0bc1f178 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 10:18:54 +0900
Subject: [PATCH 059/137] Add buffer_type/storage_type attribute to owner
 plugins that have buffer/storage section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/plugin.rb b/app/models/concerns/fluentd/setting/plugin.rb
index 9b2ad4977..05464e7bb 100644
--- a/app/models/concerns/fluentd/setting/plugin.rb
+++ b/app/models/concerns/fluentd/setting/plugin.rb
@@ -30,7 +30,7 @@ def register_plugin(type, name)
             params.each do |param_name, definition|
               if definition[:section]
                 parse_section(param_name, definition)
-                if %i(parse format).include?(param_name)
+                if %i(buffer storage parse format).include?(param_name)
                   attribute("#{param_name}_type", :string)
                 end
               else

From 97ef81a7ad853685be65567ed6bfe1f7515a44c0 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 10:19:02 +0900
Subject: [PATCH 060/137] Handle onChange event of buffer/storage section
 configuration

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_section.js | 27 ++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/app/javascript/packs/owned_plugin_section.js b/app/javascript/packs/owned_plugin_section.js
index 64b51236c..75e9f9962 100644
--- a/app/javascript/packs/owned_plugin_section.js
+++ b/app/javascript/packs/owned_plugin_section.js
@@ -5,6 +5,32 @@ $(document).ready(() => {
   }
   function ownedSectionOnChange() {
     const pluginName = _.last(document.documentURI.replace(/\/configure$/, "").split("/"))
+    $("#buffer-section select").on("change", (event) => {
+      $.ajax({
+        url: configUrl(pluginName),
+        method: "GET",
+        data: {
+          buffer_type: $(event.target).val()
+        }
+      }).then((data) => {
+        $("#buffer-section").html($(data).find("#buffer-section").html())
+        ownedSectionOnChange()
+      })
+    })
+
+    $("#storage-section select").on("change", (event) => {
+      $.ajax({
+        url: configUrl(pluginName),
+        method: "GET",
+        data: {
+          storage_type: $(event.target).val()
+        }
+      }).then((data) => {
+        $("#storage-section").html($(data).find("#storage-section").html())
+        ownedSectionOnChange()
+      })
+    })
+
     $("#parse-section select").on("change", (event) => {
       $.ajax({
         url: configUrl(pluginName),
@@ -17,6 +43,7 @@ $(document).ready(() => {
         ownedSectionOnChange()
       })
     })
+
     $("#format-section select").on("change", (event) => {
       $.ajax({
         url: configUrl(pluginName),

From c7695cf0b3060b485828a0d76a5958fcac960076 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 13:21:50 +0900
Subject: [PATCH 061/137] Fix errors

* format -> parse_type
* to_conf -> to_config
* stub some code

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/_form.html.haml  | 13 ++++++-------
 .../settings/in_tail/after_file_choose.html.haml    |  2 +-
 .../fluentd/settings/in_tail/confirm.html.haml      |  2 +-
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index df3f1f523..d93184e79 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -6,17 +6,17 @@
   = f.hidden_field :path
   = f.text_field :path, class: "form-control", disabled: true
 .form-group
-  = f.label :format
-  = f.hidden_field :format
-  = f.text_field :format, class: "form-control", disabled: true
-- if @setting.known_formats[@setting.format.to_sym]
-  - @setting.known_formats[@setting.format.to_sym].each do |key|
+  = f.label :parse_type
+  = f.hidden_field :parse_type
+  = f.text_field :parse_type, class: "form-control", disabled: true
+- if @setting.known_formats[@setting.parse_type&.to_sym || "syslog"]
+  - @setting.known_formats[@setting.parse_type&.to_sym || "syslog"].each do |key|
     %label= key
     = f.hidden_field key
     = @setting.send(key)
     %br
 - else
-  %label= @setting.format
+  %label= @setting.parse_type
   = f.hidden_field :regexp
   = @setting.regexp
 .form-group
@@ -39,4 +39,3 @@
     .form-group
       = f.label :refresh_interval
       = f.text_field :refresh_interval, class: "form-control"
-
diff --git a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml
index 74834d999..73866fb19 100644
--- a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml
+++ b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml
@@ -10,7 +10,7 @@
     = f.label :path
     = f.hidden_field :path
     = f.text_field :path, class: "form-control", disabled: true
-  = render partial: "shared/vue/in_tail_format", locals: { file: f.object.path, formats: @setting.known_formats, initialSelected: f.object.format || @setting.guess_format }
+  = render partial: "shared/vue/in_tail_format", locals: { file: f.object.path, formats: @setting.known_formats, initialSelected: f.object.parse_type || @setting.guess_format }
 
   .card
     %pre.card-body= file_tail(@setting.path, Settings.in_tail_preview_line_count).join("\n")
diff --git a/app/views/fluentd/settings/in_tail/confirm.html.haml b/app/views/fluentd/settings/in_tail/confirm.html.haml
index 76748a1ef..ae5aa1688 100644
--- a/app/views/fluentd/settings/in_tail/confirm.html.haml
+++ b/app/views/fluentd/settings/in_tail/confirm.html.haml
@@ -5,7 +5,7 @@
 = form_for(@setting, as: "setting", url: finish_daemon_setting_in_tail_path(@fluentd)) do |f|
   = render partial: "form", locals: { f: f }
 
-  %pre= @setting.to_conf
+  %pre= @setting.to_config
 
   %p
     = f.submit t('fluentd.common.finish')   , class: "btn btn-lg btn-primary float-right"

From db062ee391413528b70db3baaf5bf9d259b867fa Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 13:28:52 +0900
Subject: [PATCH 062/137] Set scope to make shot parameter name

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb | 2 +-
 app/views/shared/settings/_form.html.haml   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index c63360130..1a2eb4212 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -50,7 +50,7 @@ def finish
   private
 
   def setting_params
-    params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_class.permit_params)
+    params.require(:setting).permit(*target_class.permit_params)
   end
 
   def initial_params
diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index d81f899f1..9fcf43570 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -1,7 +1,7 @@
 = render "shared/setting_errors"
 
 - # NOTE: plugin_setting_form_action_url is defined at SettingConcern
-= form_with(model: @setting, url: plugin_setting_form_action_url(@fluentd), class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |form|
+= form_with(model: @setting, scope: :setting, url: plugin_setting_form_action_url(@fluentd), class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |form|
   - @setting.common_options.each do |key|
     = form.field(key)
 

From a6d27c0b147366ea189dc0ec55935ea95d73a5d6 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 8 Jun 2018 13:43:10 +0900
Subject: [PATCH 063/137] Remove unused method

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_tail.rb | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb
index bfe48995f..7d3df53ed 100644
--- a/app/models/fluentd/setting/in_tail.rb
+++ b/app/models/fluentd/setting/in_tail.rb
@@ -98,24 +98,6 @@ def grok
             grok
           end
       end
-
-      def to_conf
-        # NOTE: Using strip_heredoc makes more complex for format_specific_conf indent
-        <<-CONFIG.gsub(/^[ ]*\n/m, "")
-<source>
-  @type tail
-  path #{path}
-  tag #{tag}
-  #{certain_format_line}
-#{format_specific_conf}
-
-  #{read_from_head.to_i.zero? ? "" : "read_from_head true"}
-  #{pos_file.present? ? "pos_file #{pos_file}" : ""}
-  #{rotate_wait.present? ? "rotate_wait #{rotate_wait}" : ""}
-  #{refresh_interval.present? ? "refresh_interval #{refresh_interval}" : ""}
-</source>
-        CONFIG
-      end
     end
   end
 end

From 322bfefab3426d113ff28ab87bab11c8f11e0b5b Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 12:32:51 +0900
Subject: [PATCH 064/137] Add Api::ConfigDefinitionsController to fetch plugin
 definitions

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../api/config_definitions_controller.rb      | 53 +++++++++++++++++++
 config/routes.rb                              |  1 +
 2 files changed, 54 insertions(+)
 create mode 100644 app/controllers/api/config_definitions_controller.rb

diff --git a/app/controllers/api/config_definitions_controller.rb b/app/controllers/api/config_definitions_controller.rb
new file mode 100644
index 000000000..fe79521f7
--- /dev/null
+++ b/app/controllers/api/config_definitions_controller.rb
@@ -0,0 +1,53 @@
+class Api::ConfigDefinitionsController < ApplicationController
+  before_action :login_required
+
+  def index
+    name = params[:name]
+    type = params[:type]
+    prefix = case type
+             when "input"
+               "in"
+             when "output"
+               "out"
+             when "filter"
+               "filter"
+             when "parse"
+               "parser"
+             when "format"
+               "formatter"
+             when "parser", "formatter", "buffer", "storage"
+               type
+             end
+
+    target_class = Fluentd::Setting.const_get("#{prefix}_#{name}".classify)
+    target = target_class.new
+
+    common_options = target.common_options.map do |key|
+      h = {
+        name: key,
+        type: target.column_type(key),
+        desc: target.desc(key)
+      }
+      h[:list] = target.list_of(key) if target.column_type(key) == :enum
+      h
+    end
+
+    advanced_options = target.advanced_options.map do |key|
+      h = {
+        name: key,
+        type: target.column_type(key),
+        desc: target.desc(key)
+      }
+      h[:list] = target.list_of(key) if target.column_type(key) == :enum
+      h
+    end
+
+    options = {
+      type: type,
+      name: name,
+      commonOptions: common_options,
+      advancedOptions: advanced_options
+    }
+    render json: options
+  end
+end
diff --git a/config/routes.rb b/config/routes.rb
index 14d5464fd..6e3ddeb8e 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -129,5 +129,6 @@
     post "grok_to_regexp"
 
     resources :settings, only: [:index, :show, :update, :destroy], defaults: { format: "json" }
+    resources :config_definitions, only: [:index], defaults: { format: "json" }
   end
 end

From 1e763acfcd65400d2a07277fa8b72f92bf865e8d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 12:34:35 +0900
Subject: [PATCH 065/137] Enable tooltip

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/application.js      | 4 ++++
 app/javascript/packs/tooltip.js          | 3 ---
 app/views/shared/settings/show.html.haml | 1 -
 3 files changed, 4 insertions(+), 4 deletions(-)
 delete mode 100644 app/javascript/packs/tooltip.js

diff --git a/app/javascript/packs/application.js b/app/javascript/packs/application.js
index 25c74dbfb..2fbbe6cc9 100644
--- a/app/javascript/packs/application.js
+++ b/app/javascript/packs/application.js
@@ -34,3 +34,7 @@ Vue.filter('to_json', function (value) {
 window.Vue = Vue
 
 import '../stylesheets/application.scss'
+
+$(document).ready(() => {
+  $("[data-toggle=tooltip]").tooltip()
+})
diff --git a/app/javascript/packs/tooltip.js b/app/javascript/packs/tooltip.js
deleted file mode 100644
index 3f3560fc8..000000000
--- a/app/javascript/packs/tooltip.js
+++ /dev/null
@@ -1,3 +0,0 @@
-$(document).ready(() => {
-  $("label[data-toggle=tooltip]").tooltip()
-})
diff --git a/app/views/shared/settings/show.html.haml b/app/views/shared/settings/show.html.haml
index cc7753ac7..6c16c75de 100644
--- a/app/views/shared/settings/show.html.haml
+++ b/app/views/shared/settings/show.html.haml
@@ -1,7 +1,6 @@
 - page_title t("fluentd.settings.#{target_plugin_name}.show.page_title")
 - add_javascript_pack_tag("nested_settings")
 - add_javascript_pack_tag("owned_plugin_section")
-- add_javascript_pack_tag("tooltip")
 
 .card.mb-3
   .card-body.bg-light

From 893cb2f2c4d1dc7a09abe25cb8ab0375657cc0a1 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 12:37:30 +0900
Subject: [PATCH 066/137] Use Vue.js component to handle owned plugin
 configuration

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/plugin_setting.js        | 95 +++++++++++++++++++
 app/views/shared/settings/_form.html.haml     | 64 ++++++++-----
 .../shared/vue/_owned_plugin_form.html.haml   | 44 +++++++++
 3 files changed, 181 insertions(+), 22 deletions(-)
 create mode 100644 app/javascript/packs/plugin_setting.js
 create mode 100644 app/views/shared/vue/_owned_plugin_form.html.haml

diff --git a/app/javascript/packs/plugin_setting.js b/app/javascript/packs/plugin_setting.js
new file mode 100644
index 000000000..a488d10d6
--- /dev/null
+++ b/app/javascript/packs/plugin_setting.js
@@ -0,0 +1,95 @@
+'use strict'
+import 'lodash/lodash'
+import 'popper.js/dist/popper'
+import 'bootstrap/dist/js/bootstrap'
+
+window.addEventListener('load', () => {
+  const OwnedPluginForm = {
+    template: "#vue-owned-plugin-form",
+    props: [
+      "id",
+      "optionsJson",
+      "initialPluginName",
+      "pluginType",
+      "pluginLabel"
+    ],
+    data: () => {
+      return {
+        pluginName: "",
+        options: [],
+        commonOptions: [],
+        advancedOptions: []
+      }
+    },
+
+    computed: {
+      token: function() {
+        return Rails.csrfToken()
+      }
+    },
+
+    mounted: function() {
+      this.options = JSON.parse(this.optionsJson)
+      this.pluginName = this.initialPluginName
+      this.$on("hook:updated", () => {
+        console.log("hook:updated")
+        $("[data-toggle=tooltip]").tooltip("dispose")
+        $("[data-toggle=tooltip]").tooltip("enable")
+      })
+      this.$once("data-loaded", () => {
+        this.updateSection()
+      })
+      this.$emit("data-loaded")
+    },
+
+    methods: {
+      onChange: function() {
+        this.updateSection()
+      },
+
+      updateSection: function() {
+        $.ajax({
+          method: "GET",
+          url: "/api/config_definitions",
+          headers: {
+            'X-CSRF-Token': this.token
+          },
+          data: {
+            type: this.pluginType,
+            name: this.pluginName
+          }
+        }).then((data) => {
+          this.commonOptions = data.commonOptions
+        })
+      },
+
+      selectId: function(pluginType) {
+        return `setting_${pluginType}_type`
+      },
+      selectClass: function(pluginType) {
+        return `${pluginType} form-control`
+      },
+      selectName: function(pluginType) {
+        return `setting[${pluginType}_type]`
+      },
+      inputId: function(pluginName, option) {
+        return `setting_${pluginName}_0__${option.name}`
+      },
+      inputName: function(pluginName, option) {
+        return `setting[${pluginName}[0]][${option.name}]`
+      }
+    }
+  }
+
+  new Vue({
+    el: '#plugin-setting',
+    data: () => {
+      return {}
+    },
+    components: {
+      'owned-plugin-form': OwnedPluginForm
+    },
+    methods: {
+    }
+  })
+})
diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index 9fcf43570..b3c6a6cb7 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -1,30 +1,50 @@
 = render "shared/setting_errors"
+= render "shared/vue/owned_plugin_form"
 
-- # NOTE: plugin_setting_form_action_url is defined at SettingConcern
-= form_with(model: @setting, scope: :setting, url: plugin_setting_form_action_url(@fluentd), class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |form|
-  - @setting.common_options.each do |key|
-    = form.field(key)
+= javascript_pack_tag("plugin_setting")
 
-  - if @setting.have_buffer_section?
-    = render "shared/settings/buffer_form", form: form, buffer: @buffer
+#plugin-setting
+  - # NOTE: plugin_setting_form_action_url is defined at SettingConcern
+  = form_with(model: @setting, scope: :setting, url: plugin_setting_form_action_url(@fluentd), class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |form|
+    - @setting.common_options.each do |key|
+      = form.field(key)
 
-  - if @setting.have_storage_section?
-    = render "shared/settings/storage_form", form: form, storage: @storage
+    - if @setting.have_buffer_section?
+      %owned-plugin-form{"v-bind:id" => "'buffer-section'",
+                         "v-bind:options-json" => "'#{Fluent::Plugin::BUFFER_REGISTRY.map.keys.to_json}'",
+                         "v-bind:initial-plugin-name" => "'#{@setting.buffer_type}'",
+                         "v-bind:plugin-type" => "'buffer'",
+                         "v-bind:plugin-label" => "'Buffer'"}
 
-  - if @setting.have_parse_section?
-    = render "shared/settings/parser_form", form: form, parser: @parser
+    - if @setting.have_storage_section?
+      %owned-plugin-form{"v-bind:id" => "'storage-section'",
+                         "v-bind:options-json" => "'#{Fluent::Plugin::STORAGE_REGISTRY.map.keys.to_json}'",
+                         "v-bind:initial-plugin-name" => "'#{@setting.storage_type}'",
+                         "v-bind:plugin-type" => "'storage'",
+                         "v-bind:plugin-label" => "'Storage'"}
 
-  - if @setting.have_format_section?
-    = render "shared/settings/formatter_form", form: form, formatter: @formatter
+    - if @setting.have_parse_section?
+      %owned-plugin-form{"v-bind:id" => "'parse-section'",
+                         "v-bind:options-json" => "'#{Fluent::Plugin::PARSER_REGISTRY.map.keys.to_json}'",
+                         "v-bind:initial-plugin-name" => "'#{@setting.parse_type}'",
+                         "v-bind:plugin-type" => "'parse'",
+                         "v-bind:plugin-label" => "'Parse'"}
 
-  - if @setting.advanced_options.present?
-    .card.bg-light
-      .card-body
-        %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
-          = icon('fa-caret-down')
-          = t('terms.advanced_setting')
-        #advanced-setting.collapse
-          - @setting.advanced_options.each do |key|
-            = form.field(key)
+    - if @setting.have_format_section?
+      %owned-plugin-form{"v-bind:id" => "'format-section'",
+                         "v-bind:options-json" => "'#{Fluent::Plugin::FORMATTER_REGISTRY.map.keys.to_json}'",
+                         "v-bind:initial-plugin-name" => "'#{@setting.format_type}'",
+                         "v-bind:plugin-type" => "'format'",
+                         "v-bind:plugin-label" => "'Format'"}
 
-  = form.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right mt-3"
+    - if @setting.advanced_options.present?
+      .card.bg-light
+        .card-body
+          %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
+            = icon('fa-caret-down')
+            = t('terms.advanced_setting')
+          #advanced-setting.collapse
+            - @setting.advanced_options.each do |key|
+              = form.field(key)
+
+    = form.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right mt-3"
diff --git a/app/views/shared/vue/_owned_plugin_form.html.haml b/app/views/shared/vue/_owned_plugin_form.html.haml
new file mode 100644
index 000000000..f79c295d1
--- /dev/null
+++ b/app/views/shared/vue/_owned_plugin_form.html.haml
@@ -0,0 +1,44 @@
+%script{type: "text/x-template", id: "vue-owned-plugin-form"}
+  .form-group.card.bg-light.mb-3{"v-bind:id" => "id"}
+    .card-body
+      %label{"v-bind:for" => "selectId(pluginType)"}
+        {{ pluginLabel }}
+      %select{"v-bind:id" => "selectId(pluginType)",
+              "v-bind:class" => "selectClass(pluginType)",
+              "v-bind:name" => "selectName(pluginType)",
+              "v-model" => "pluginName",
+              "v-on:change" => "onChange"}
+        %option{"v-for" => "option in options",
+                "v-bind:value" => "option",
+                "v-bind:selected" => "pluginName===option"}
+          {{ option }}
+      .form-group{ "v-for" => "option in commonOptions" }
+        %template{"v-if" => 'option.type==="enum"'}
+          %label{"v-bind:for" => "inputId(pluginName, option)",
+                 "data-toggle" => "tooltip",
+                 "data-placement" => "right",
+                 "v-bind:title" => "option.desc"}
+            {{ option.name }}
+          %input{"v-bind:id" => "inputId(pluginName, option)",
+                 "v-bind:name" => "inputName(pluginName, option)",
+                 "type" => "text",
+                 "class" => "form-control"}
+        %template{"v-else-if" => 'option.type==="bool"'}
+          %input{"v-bind:id" => "inputId(pluginName, option)",
+                 "v-bind:name" => "inputName(pluginName, option)",
+                 "type" => "checkbox"}
+          %label{"v-bind:for" => "inputId(pluginName, option)",
+                 "data-toggle" => "tooltip",
+                 "data-placement" => "right",
+                 "v-bind:title" => "option.desc"}
+            {{ option.name }}
+        %template(v-else)
+          %label{"v-bind:for" => "inputId(pluginName, option)",
+                 "data-toggle" => "tooltip",
+                 "data-placement" => "right",
+                 "v-bind:title" => "option.desc"}
+            {{ option.name }}
+          %input{"v-bind:id" => "inputId(pluginName, option)",
+                 "v-bind:name" => "inputName(pluginName, option)",
+                 "type" => "text",
+                 "class" => "form-control"}

From 643a2279d51c166769d4676e1d87b66cfd992d29 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 12:39:17 +0900
Subject: [PATCH 067/137] Remove unused action

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/concerns/setting_concern.rb | 10 ----------
 config/routes.rb                            |  5 -----
 2 files changed, 15 deletions(-)

diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb
index 1a2eb4212..47dee4d0e 100644
--- a/app/controllers/concerns/setting_concern.rb
+++ b/app/controllers/concerns/setting_concern.rb
@@ -19,16 +19,6 @@ def show
     render "shared/settings/show"
   end
 
-  def configure
-    owned_keys = %i(parse_type format_type)
-    @setting = target_class.new(initial_params.merge(params.permit(*owned_keys).slice(*owned_keys)))
-    @buffer = @setting.create_buffer
-    @storage = @setting.create_storage
-    @parser = @setting.create_parser
-    @formatter = @setting.create_formatter
-    render "shared/settings/show"
-  end
-
   def finish
     @setting = target_class.new(setting_params)
     @_used_param = {}
diff --git a/config/routes.rb b/config/routes.rb
index 6e3ddeb8e..93d29718f 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -20,14 +20,12 @@
 
         resource :in_tail, only: [:show], module: :settings, controller: :in_tail do
           post "after_file_choose"
-          # get "configure"
           post "after_format"
           post "confirm"
           post "finish"
         end
 
         resource :in_syslog, only: [:show], module: :settings, controller: :in_syslog do
-          get "configure"
           post "finish"
         end
 
@@ -36,7 +34,6 @@
         end
 
         resource :in_http, only: [:show], module: :settings, controller: :in_http do
-          get "configure"
           post "finish"
         end
 
@@ -45,7 +42,6 @@
         end
 
         resource :out_stdout, only: [:show], module: :settings, controller: :out_stdout do
-          get "configure"
           post "finish"
         end
 
@@ -58,7 +54,6 @@
         end
 
         resource :out_s3, only: [:show], module: :settings, controller: :out_s3 do
-          get "configure"
           post "finish"
         end
 

From 75f6d0f156159e5bc7f43509bbee9af5dd4b246c Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 12:44:56 +0900
Subject: [PATCH 068/137] Organize tooltip position

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/_global_nav.html.erb | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/app/views/shared/_global_nav.html.erb b/app/views/shared/_global_nav.html.erb
index 314a08dbe..be3b492a9 100644
--- a/app/views/shared/_global_nav.html.erb
+++ b/app/views/shared/_global_nav.html.erb
@@ -1,6 +1,6 @@
 <ul class="navbar-nav navbar-sidenav" id="side-menu">
-  <li class="nav-item" data-toggle="tooltip" data-placement="right" title="Fluentd">
-    <a class="section">
+  <li class="nav-item">
+    <a class="section" data-toggle="tooltip" data-placement="right" title="Fluentd">
       <%= icon("fa-puzzle-piece fa-fw") << "fluentd" %>
     </a>
     <ul class="sidenav-second-level">
@@ -29,8 +29,8 @@
   </li>
 
   <% if fluentd_exists? %>
-  <li class="nav-item" data-toggle="tooltip" data-placement="right" title="Plugins">
-    <a class="section">
+  <li class="nav-item">
+    <a class="section" data-toggle="tooltip" data-placement="right" title="Plugins">
       <%= icon("fa-cogs fa-fw") << t('terms.plugins') %>
     </a>
     <ul class="sidenav-second-level">
@@ -47,8 +47,8 @@
   </li>
   <% end %>
 
-  <li class="nav-item" data-toggle="tooltip" data-placement="right" title="Misc">
-    <a class="section">
+  <li class="nav-item">
+    <a class="section" data-toggle="tooltip" data-placement="right" title="Misc">
       <%= icon('fa-coffee fa-fw') << t('terms.misc') %>
     </a>
     <ul class="sidenav-second-level">
@@ -67,8 +67,8 @@
     </ul>
   </li>
 
-  <li class="nav-item" data-toggle="tooltip" data-placement="right" title="Language">
-    <a class="section">
+  <li class="nav-item">
+    <a class="section" data-toggle="tooltip" data-placement="right" title="Language">
       <%= icon('fa-font') << t('terms.languages') %>
     </a>
     <ul class="sidenav-second-level">

From 54a9dc52e98bf05cfc903a960a28dbd03b833d20 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 12:51:07 +0900
Subject: [PATCH 069/137] Remove unused partial templates

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/settings/_buffer_form.html.haml    | 7 -------
 app/views/shared/settings/_formatter_form.html.haml | 7 -------
 app/views/shared/settings/_parser_form.html.haml    | 7 -------
 app/views/shared/settings/_storage_form.html.haml   | 7 -------
 4 files changed, 28 deletions(-)
 delete mode 100644 app/views/shared/settings/_buffer_form.html.haml
 delete mode 100644 app/views/shared/settings/_formatter_form.html.haml
 delete mode 100644 app/views/shared/settings/_parser_form.html.haml
 delete mode 100644 app/views/shared/settings/_storage_form.html.haml

diff --git a/app/views/shared/settings/_buffer_form.html.haml b/app/views/shared/settings/_buffer_form.html.haml
deleted file mode 100644
index a88d5d741..000000000
--- a/app/views/shared/settings/_buffer_form.html.haml
+++ /dev/null
@@ -1,7 +0,0 @@
-#buffer-section.form-group.card.bg-light.mb-3
-  .card-body
-    = form.label(:buffer_type, "Buffer")
-    = form.select(:buffer_type, Fluent::Plugin::BUFFER_REGISTRY.map.keys, {}, { class: "buffer form-control" })
-    = form.fields("buffer[0]", model: buffer, class: ".form-group", builder: FluentdFormBuilder) do |buffer_form|
-      - buffer_form.object.common_options.each do |key|
-        = buffer_form.field(key)
diff --git a/app/views/shared/settings/_formatter_form.html.haml b/app/views/shared/settings/_formatter_form.html.haml
deleted file mode 100644
index 0b85f5bfd..000000000
--- a/app/views/shared/settings/_formatter_form.html.haml
+++ /dev/null
@@ -1,7 +0,0 @@
-#format-section.form-group.card.bg-light.mb-3
-  .card-body
-    = form.label(:format_type, "Parse")
-    = form.select(:format_type, Fluent::Plugin::FORMATTER_REGISTRY.map.keys, {}, { class: "parse form-control" })
-    = form.fields("format[0]", model: formatter, class: ".form-group", builder: FluentdFormBuilder) do |formatter_form|
-      - formatter_form.object.common_options.each do |key|
-        = formatter_form.field(key)
diff --git a/app/views/shared/settings/_parser_form.html.haml b/app/views/shared/settings/_parser_form.html.haml
deleted file mode 100644
index 04642f088..000000000
--- a/app/views/shared/settings/_parser_form.html.haml
+++ /dev/null
@@ -1,7 +0,0 @@
-#parse-section.form-group.card.bg-light.mb-3
-  .card-body
-    = form.label(:parse_type, "Parse")
-    = form.select(:parse_type, Fluent::Plugin::PARSER_REGISTRY.map.keys, {}, { class: "parse form-control" })
-    = form.fields("parse[0]", model: parser, class: ".form-group", builder: FluentdFormBuilder) do |parser_form|
-      - parser_form.object.common_options.each do |key|
-        = parser_form.field(key)
diff --git a/app/views/shared/settings/_storage_form.html.haml b/app/views/shared/settings/_storage_form.html.haml
deleted file mode 100644
index e91651f0d..000000000
--- a/app/views/shared/settings/_storage_form.html.haml
+++ /dev/null
@@ -1,7 +0,0 @@
-#storage-section.form-group.card.bg-light.mb-3
-  .card-body
-    = form.label(:storage_type, "Storage")
-    = form.select(:storage_type, Fluent::Plugin::STORAGE_REGISTRY.map.keys, {}, { class: "storage form-control" })
-    = form.fields("storage[0]", model: storage, class: ".form-group", builder: FluentdFormBuilder) do |storage_form|
-      - storage_form.object.common_options.each do |key|
-        = storage_form.field(key)

From fad62d812f84d8b08a57e8d834148ce0a0e20994 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 14:44:54 +0900
Subject: [PATCH 070/137] Use new Hash syntax

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/fluentd/settings/in_tail_controller.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/fluentd/settings/in_tail_controller.rb b/app/controllers/fluentd/settings/in_tail_controller.rb
index 85d849c0b..38b660fd5 100644
--- a/app/controllers/fluentd/settings/in_tail_controller.rb
+++ b/app/controllers/fluentd/settings/in_tail_controller.rb
@@ -4,8 +4,8 @@ class Fluentd::Settings::InTailController < ApplicationController
 
   def after_file_choose
     @setting = Fluentd::Setting::InTail.new({
-      :path => params[:path],
-      :tag => nil,
+      path: params[:path],
+      tag: nil
     })
   end
 

From b642961fdb30ca5b8788609f93bfdb74786ae2a0 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 15:09:07 +0900
Subject: [PATCH 071/137] Remove an unused file

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_section.js | 62 --------------------
 app/views/shared/settings/show.html.haml     |  1 -
 2 files changed, 63 deletions(-)
 delete mode 100644 app/javascript/packs/owned_plugin_section.js

diff --git a/app/javascript/packs/owned_plugin_section.js b/app/javascript/packs/owned_plugin_section.js
deleted file mode 100644
index 75e9f9962..000000000
--- a/app/javascript/packs/owned_plugin_section.js
+++ /dev/null
@@ -1,62 +0,0 @@
-import "lodash/lodash"
-$(document).ready(() => {
-  let configUrl = (name) => {
-    return `/daemon/setting/${name}/configure`
-  }
-  function ownedSectionOnChange() {
-    const pluginName = _.last(document.documentURI.replace(/\/configure$/, "").split("/"))
-    $("#buffer-section select").on("change", (event) => {
-      $.ajax({
-        url: configUrl(pluginName),
-        method: "GET",
-        data: {
-          buffer_type: $(event.target).val()
-        }
-      }).then((data) => {
-        $("#buffer-section").html($(data).find("#buffer-section").html())
-        ownedSectionOnChange()
-      })
-    })
-
-    $("#storage-section select").on("change", (event) => {
-      $.ajax({
-        url: configUrl(pluginName),
-        method: "GET",
-        data: {
-          storage_type: $(event.target).val()
-        }
-      }).then((data) => {
-        $("#storage-section").html($(data).find("#storage-section").html())
-        ownedSectionOnChange()
-      })
-    })
-
-    $("#parse-section select").on("change", (event) => {
-      $.ajax({
-        url: configUrl(pluginName),
-        method: "GET",
-        data: {
-          parse_type: $(event.target).val()
-        }
-      }).then((data) => {
-        $("#parse-section").html($(data).find("#parse-section").html())
-        ownedSectionOnChange()
-      })
-    })
-
-    $("#format-section select").on("change", (event) => {
-      $.ajax({
-        url: configUrl(pluginName),
-        method: "GET",
-        data: {
-          format_type: $(event.target).val()
-        }
-      }).then((data) => {
-        $("#format-section").html($(data).find("#format-section").html())
-        ownedSectionOnChange()
-      })
-    })
-  }
-
-  ownedSectionOnChange()
-})
diff --git a/app/views/shared/settings/show.html.haml b/app/views/shared/settings/show.html.haml
index 6c16c75de..89d9f0177 100644
--- a/app/views/shared/settings/show.html.haml
+++ b/app/views/shared/settings/show.html.haml
@@ -1,6 +1,5 @@
 - page_title t("fluentd.settings.#{target_plugin_name}.show.page_title")
 - add_javascript_pack_tag("nested_settings")
-- add_javascript_pack_tag("owned_plugin_section")
 
 .card.mb-3
   .card-body.bg-light

From 620372dd5ddc98f5454a37e667d831a37aeddb60 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 15:21:42 +0900
Subject: [PATCH 072/137] Extract owned_plugin_form.js

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_form.js | 79 +++++++++++++++++++++++
 app/javascript/packs/plugin_setting.js    | 78 +---------------------
 2 files changed, 80 insertions(+), 77 deletions(-)
 create mode 100644 app/javascript/packs/owned_plugin_form.js

diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
new file mode 100644
index 000000000..318d9ae57
--- /dev/null
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -0,0 +1,79 @@
+'use strict'
+const OwnedPluginForm = {
+  template: "#vue-owned-plugin-form",
+  props: [
+    "id",
+    "optionsJson",
+    "initialPluginName",
+    "pluginType",
+    "pluginLabel"
+  ],
+  data: () => {
+    return {
+      pluginName: "",
+      options: [],
+      commonOptions: [],
+      advancedOptions: []
+    }
+  },
+
+  computed: {
+    token: function() {
+      return Rails.csrfToken()
+    }
+  },
+
+  mounted: function() {
+    this.options = JSON.parse(this.optionsJson)
+    this.pluginName = this.initialPluginName
+    this.$on("hook:updated", () => {
+      console.log("hook:updated")
+      $("[data-toggle=tooltip]").tooltip("dispose")
+      $("[data-toggle=tooltip]").tooltip("enable")
+    })
+    this.$once("data-loaded", () => {
+      this.updateSection()
+    })
+    this.$emit("data-loaded")
+  },
+
+  methods: {
+    onChange: function() {
+      this.updateSection()
+    },
+
+    updateSection: function() {
+      $.ajax({
+        method: "GET",
+        url: "/api/config_definitions",
+        headers: {
+          'X-CSRF-Token': this.token
+        },
+        data: {
+          type: this.pluginType,
+          name: this.pluginName
+        }
+      }).then((data) => {
+        this.commonOptions = data.commonOptions
+      })
+    },
+
+    selectId: function(pluginType) {
+      return `setting_${pluginType}_type`
+    },
+    selectClass: function(pluginType) {
+      return `${pluginType} form-control`
+    },
+    selectName: function(pluginType) {
+      return `setting[${pluginType}_type]`
+    },
+    inputId: function(pluginName, option) {
+      return `setting_${pluginName}_0__${option.name}`
+    },
+    inputName: function(pluginName, option) {
+      return `setting[${pluginName}[0]][${option.name}]`
+    }
+  }
+}
+
+export { OwnedPluginForm as default }
diff --git a/app/javascript/packs/plugin_setting.js b/app/javascript/packs/plugin_setting.js
index a488d10d6..675d4c94a 100644
--- a/app/javascript/packs/plugin_setting.js
+++ b/app/javascript/packs/plugin_setting.js
@@ -2,85 +2,9 @@
 import 'lodash/lodash'
 import 'popper.js/dist/popper'
 import 'bootstrap/dist/js/bootstrap'
+import OwnedPluginForm from './owned_plugin_form'
 
 window.addEventListener('load', () => {
-  const OwnedPluginForm = {
-    template: "#vue-owned-plugin-form",
-    props: [
-      "id",
-      "optionsJson",
-      "initialPluginName",
-      "pluginType",
-      "pluginLabel"
-    ],
-    data: () => {
-      return {
-        pluginName: "",
-        options: [],
-        commonOptions: [],
-        advancedOptions: []
-      }
-    },
-
-    computed: {
-      token: function() {
-        return Rails.csrfToken()
-      }
-    },
-
-    mounted: function() {
-      this.options = JSON.parse(this.optionsJson)
-      this.pluginName = this.initialPluginName
-      this.$on("hook:updated", () => {
-        console.log("hook:updated")
-        $("[data-toggle=tooltip]").tooltip("dispose")
-        $("[data-toggle=tooltip]").tooltip("enable")
-      })
-      this.$once("data-loaded", () => {
-        this.updateSection()
-      })
-      this.$emit("data-loaded")
-    },
-
-    methods: {
-      onChange: function() {
-        this.updateSection()
-      },
-
-      updateSection: function() {
-        $.ajax({
-          method: "GET",
-          url: "/api/config_definitions",
-          headers: {
-            'X-CSRF-Token': this.token
-          },
-          data: {
-            type: this.pluginType,
-            name: this.pluginName
-          }
-        }).then((data) => {
-          this.commonOptions = data.commonOptions
-        })
-      },
-
-      selectId: function(pluginType) {
-        return `setting_${pluginType}_type`
-      },
-      selectClass: function(pluginType) {
-        return `${pluginType} form-control`
-      },
-      selectName: function(pluginType) {
-        return `setting[${pluginType}_type]`
-      },
-      inputId: function(pluginName, option) {
-        return `setting_${pluginName}_0__${option.name}`
-      },
-      inputName: function(pluginName, option) {
-        return `setting[${pluginName}[0]][${option.name}]`
-      }
-    }
-  }
-
   new Vue({
     el: '#plugin-setting',
     data: () => {

From 9d1c4f9d7b4556982a3a35fe62bb1019601cb862 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 16:52:26 +0900
Subject: [PATCH 073/137] Remove unused methods

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_tail.rb | 21 ---------------------
 1 file changed, 21 deletions(-)

diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb
index 7d3df53ed..0affe835d 100644
--- a/app/models/fluentd/setting/in_tail.rb
+++ b/app/models/fluentd/setting/in_tail.rb
@@ -10,27 +10,6 @@ class InTail
 
       MULTI_LINE_MAX_FORMAT_COUNT = ::Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM
 
-      def self.known_formats
-        {
-          :apache2 => [:time_format],
-          :nginx => [:time_format],
-          :syslog => [:time_format],
-          :tsv => [:keys, :time_key],
-          :csv => [:keys, :time_key],
-          :ltsv => [:delimiter, :time_key],
-          :json => [:time_key],
-          :regexp => [:time_format, :regexp],
-          :multiline => [:format_firstline] + (1..MULTI_LINE_MAX_FORMAT_COUNT).map{|n| "format#{n}".to_sym }
-          # TODO: Grok could generate Regexp including \d, \s, etc. fluentd config parser raise error with them for escape sequence check.
-          #       TBD How to handle Grok/Regexp later, just comment out for hide
-          # :grok => [:grok_str],
-        }
-      end
-      attr_accessor(*known_formats.values.flatten.compact.uniq)
-
-      def known_formats
-        self.class.known_formats
-      end
 
       def guess_format
         case path

From 4eb54f0725793abbdfddb0803c058a40db5dd82d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 16:52:37 +0900
Subject: [PATCH 074/137] Rename guess_format -> guess_parse_type

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_tail.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb
index 0affe835d..855f2228e 100644
--- a/app/models/fluentd/setting/in_tail.rb
+++ b/app/models/fluentd/setting/in_tail.rb
@@ -10,8 +10,7 @@ class InTail
 
       MULTI_LINE_MAX_FORMAT_COUNT = ::Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM
 
-
-      def guess_format
+      def guess_parse_type
         case path
         when /\.json$/
           :json

From d80afa5e8a37976eb7292c6643a5f414cb6946a2 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 17:23:27 +0900
Subject: [PATCH 075/137] Fix typos

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_form.js      |  8 ++++----
 .../shared/vue/_owned_plugin_form.html.haml    | 18 +++++++++---------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index 318d9ae57..226489892 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -67,11 +67,11 @@ const OwnedPluginForm = {
     selectName: function(pluginType) {
       return `setting[${pluginType}_type]`
     },
-    inputId: function(pluginName, option) {
-      return `setting_${pluginName}_0__${option.name}`
+    inputId: function(pluginType, option) {
+      return `setting_${pluginType}_0__${option.name}`
     },
-    inputName: function(pluginName, option) {
-      return `setting[${pluginName}[0]][${option.name}]`
+    inputName: function(pluginType, option) {
+      return `setting[${pluginType}[0]][${option.name}]`
     }
   }
 }
diff --git a/app/views/shared/vue/_owned_plugin_form.html.haml b/app/views/shared/vue/_owned_plugin_form.html.haml
index f79c295d1..8e56b21d1 100644
--- a/app/views/shared/vue/_owned_plugin_form.html.haml
+++ b/app/views/shared/vue/_owned_plugin_form.html.haml
@@ -14,31 +14,31 @@
           {{ option }}
       .form-group{ "v-for" => "option in commonOptions" }
         %template{"v-if" => 'option.type==="enum"'}
-          %label{"v-bind:for" => "inputId(pluginName, option)",
+          %label{"v-bind:for" => "inputId(pluginType, option)",
                  "data-toggle" => "tooltip",
                  "data-placement" => "right",
                  "v-bind:title" => "option.desc"}
             {{ option.name }}
-          %input{"v-bind:id" => "inputId(pluginName, option)",
-                 "v-bind:name" => "inputName(pluginName, option)",
+          %input{"v-bind:id" => "inputId(pluginType, option)",
+                 "v-bind:name" => "inputName(pluginType, option)",
                  "type" => "text",
                  "class" => "form-control"}
         %template{"v-else-if" => 'option.type==="bool"'}
-          %input{"v-bind:id" => "inputId(pluginName, option)",
-                 "v-bind:name" => "inputName(pluginName, option)",
+          %input{"v-bind:id" => "inputId(pluginType, option)",
+                 "v-bind:name" => "inputName(pluginType, option)",
                  "type" => "checkbox"}
-          %label{"v-bind:for" => "inputId(pluginName, option)",
+          %label{"v-bind:for" => "inputId(pluginType, option)",
                  "data-toggle" => "tooltip",
                  "data-placement" => "right",
                  "v-bind:title" => "option.desc"}
             {{ option.name }}
         %template(v-else)
-          %label{"v-bind:for" => "inputId(pluginName, option)",
+          %label{"v-bind:for" => "inputId(pluginType, option)",
                  "data-toggle" => "tooltip",
                  "data-placement" => "right",
                  "v-bind:title" => "option.desc"}
             {{ option.name }}
-          %input{"v-bind:id" => "inputId(pluginName, option)",
-                 "v-bind:name" => "inputName(pluginName, option)",
+          %input{"v-bind:id" => "inputId(pluginType, option)",
+                 "v-bind:name" => "inputName(pluginType, option)",
                  "type" => "text",
                  "class" => "form-control"}

From 6e8f5a29793046a59e4c98c3505fde5845344d8c Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 17:30:48 +0900
Subject: [PATCH 076/137] Set default value to /api/config_definitions' return
 value

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api/config_definitions_controller.rb    | 6 ++++--
 app/models/concerns/fluentd/setting/plugin_parameter.rb | 4 ++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/controllers/api/config_definitions_controller.rb b/app/controllers/api/config_definitions_controller.rb
index fe79521f7..6b5384d7a 100644
--- a/app/controllers/api/config_definitions_controller.rb
+++ b/app/controllers/api/config_definitions_controller.rb
@@ -26,7 +26,8 @@ def index
       h = {
         name: key,
         type: target.column_type(key),
-        desc: target.desc(key)
+        desc: target.desc(key),
+        default: target.default(key)
       }
       h[:list] = target.list_of(key) if target.column_type(key) == :enum
       h
@@ -36,7 +37,8 @@ def index
       h = {
         name: key,
         type: target.column_type(key),
-        desc: target.desc(key)
+        desc: target.desc(key),
+        default: target.default(key)
       }
       h[:list] = target.list_of(key) if target.column_type(key) == :enum
       h
diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index e009f0ded..5dc701ffe 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -17,6 +17,10 @@ def desc(name)
         self.class._descriptions[name]
       end
 
+      def default(name)
+        reformat_value(name, self.class._defaults[name])
+      end
+
       def common_options
         []
       end

From f194aae8809cdcdeccef00f755e4aa721270d6c5 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 18:02:46 +0900
Subject: [PATCH 077/137] Set default value to owned plugin form

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_form.js         |  7 +++++++
 app/views/shared/vue/_owned_plugin_form.html.haml | 13 +++++++++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index 226489892..d74019f81 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -72,6 +72,13 @@ const OwnedPluginForm = {
     },
     inputName: function(pluginType, option) {
       return `setting[${pluginType}[0]][${option.name}]`
+    },
+    checked: function(checked) {
+      if (checked === true || checked === "true") {
+        return "checked"
+      } else {
+        return ""
+      }
     }
   }
 }
diff --git a/app/views/shared/vue/_owned_plugin_form.html.haml b/app/views/shared/vue/_owned_plugin_form.html.haml
index 8e56b21d1..b70915d9e 100644
--- a/app/views/shared/vue/_owned_plugin_form.html.haml
+++ b/app/views/shared/vue/_owned_plugin_form.html.haml
@@ -19,13 +19,17 @@
                  "data-placement" => "right",
                  "v-bind:title" => "option.desc"}
             {{ option.name }}
-          %input{"v-bind:id" => "inputId(pluginType, option)",
-                 "v-bind:name" => "inputName(pluginType, option)",
-                 "type" => "text",
-                 "class" => "form-control"}
+          %select{"v-bind:id" => "inputId(pluginType, option)",
+                  "v-bind:name" => "inputName(pluginType, option)",
+                  "class" => "form-control"}
+            %option{"v-for" => "item in option.list",
+                    "v-bind:value" => "item",
+                    "v-bind:selected" => "option.default === item"}
+              {{ item }}
         %template{"v-else-if" => 'option.type==="bool"'}
           %input{"v-bind:id" => "inputId(pluginType, option)",
                  "v-bind:name" => "inputName(pluginType, option)",
+                 "v-bind:checked" => "checked(option.default)",
                  "type" => "checkbox"}
           %label{"v-bind:for" => "inputId(pluginType, option)",
                  "data-toggle" => "tooltip",
@@ -40,5 +44,6 @@
             {{ option.name }}
           %input{"v-bind:id" => "inputId(pluginType, option)",
                  "v-bind:name" => "inputName(pluginType, option)",
+                 "v-bind:value" => "option.default",
                  "type" => "text",
                  "class" => "form-control"}

From 72d377b51a0e4cd5696bf59418d52638f994fde5 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 18:09:09 +0900
Subject: [PATCH 078/137] Handle time_key without error

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_parameter.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index 5dc701ffe..355b6eaa6 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -79,6 +79,7 @@ def create_formatter
 
       def reformat_value(name, value)
         type = column_type(name)
+        return value if type.nil? # name == :time_key
         return value if type == :enum
         return value if type == :regexp
         type_name = if type.is_a?(Fluentd::Setting::Type::Time)

From 3a80c940f85aaadd8a59727fb86b5e85210e1603 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Tue, 12 Jun 2018 18:10:13 +0900
Subject: [PATCH 079/137] Render shared/vue/in_tail_parse to set configuration
 for in_tail's parse section

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/after_file_choose.html.haml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml
index 73866fb19..0e6db6506 100644
--- a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml
+++ b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml
@@ -10,7 +10,7 @@
     = f.label :path
     = f.hidden_field :path
     = f.text_field :path, class: "form-control", disabled: true
-  = render partial: "shared/vue/in_tail_format", locals: { file: f.object.path, formats: @setting.known_formats, initialSelected: f.object.parse_type || @setting.guess_format }
+  = render "shared/vue/in_tail_parse", setting: @setting
 
   .card
     %pre.card-body= file_tail(@setting.path, Settings.in_tail_preview_line_count).join("\n")

From 089ce6ba20411656ff474bb81779e5f8e8bfdb35 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 13 Jun 2018 14:03:11 +0900
Subject: [PATCH 080/137] Add missing files

---
 app/javascript/packs/in_tail_parse.js         | 120 ++++++++++++++++++
 app/views/shared/vue/_in_tail_parse.html.haml |  17 +++
 2 files changed, 137 insertions(+)
 create mode 100644 app/javascript/packs/in_tail_parse.js
 create mode 100644 app/views/shared/vue/_in_tail_parse.html.haml

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
new file mode 100644
index 000000000..f7e98f80c
--- /dev/null
+++ b/app/javascript/packs/in_tail_parse.js
@@ -0,0 +1,120 @@
+'use strict'
+import 'lodash/lodash'
+import 'popper.js/dist/popper'
+import 'bootstrap/dist/js/bootstrap'
+import OwnedPluginForm from './owned_plugin_form'
+
+$(document).ready(() => {
+  new Vue({
+    el: '#in-tail-parse',
+    props: [
+      "targetFile",
+      "parseType",
+    ],
+    data: () => {
+      return {
+        highlightedLines: null
+      }
+    },
+    computed: {
+      token: function() {
+        return Rails.csrfToken()
+      }
+    },
+    components: {
+      'owned-plugin-form': OwnedPluginForm
+    },
+    watch: {
+      'params.setting.regexp': function() {
+        this.preview();
+      },
+      'parseType': function() {
+        this.preview();
+      },
+    },
+    methods: {
+      updateHighlightedLines: function(matches) {
+        if (!matches) {
+          this.highlightedLines = null
+          return
+        }
+
+        let $container = $('<div>')
+        _.each(matches, (match) => {
+          const colors = ["#ff9", "#cff", "#fcf", "#dfd"]
+          const whole = match.whole
+          let html = ""
+          let _matches = []
+          let lastPos = 0
+
+          _.each(match.matches, (m) => {
+            let matched = m.matched
+            if (!matched) {
+              return
+            }
+            // Ignore empty matched with "foobar".match(/foo(.*?)bar/)[1] #=> ""
+            if (matched.length === 0) {
+              return
+            }
+            // rotate color
+            let currentColor = colors.shift()
+            colors.push(currentColor)
+
+            // create highlighted range HTML
+            let $highlighted = $("<span>").text(matched)
+            $highlighted.attr({
+              "class": "regexp-preview",
+              "data-toggle": "tooltip",
+              "data-placement": "top",
+              "title": match.key,
+              'style': 'background-color:' + currentColor
+            })
+            let highlightedHtml = $highlighted.wrap("<div>").parent().html()
+
+            let pos = {
+              start: match.pos[0],
+              ent: match.pos[1]
+            }
+            if (pos.start > 0) {
+              html += _.escape(whole.substring(lastPos, pos.start))
+            }
+            html += highlightedHtml
+            lastPos = pos.end
+          })
+          html += whole.substring(lastPos)
+
+          $container.append(html)
+          $container.append("<br>")
+        })
+
+        this.highlightedLines = $container.html()
+      },
+
+      preview: function() {
+        $.ajax({
+          method: "POST",
+          url: "/api/regexp_preview",
+          headers: {
+            'X-CSRF-Token': token
+          },
+          data: {
+            regexp: this.params.setting.regexp,
+            time_format: this.params.setting.time_format,
+            format: _.isEmpty(this.parseType) ? "regexp" : this.parseType,
+            params: this.params.setting,
+            file: this.targetFile
+          }
+        }).then(
+          (result) => {
+            this.params = _merge(this.params, result.params)
+            this.updateHighlightedLines(result.matches)
+          },
+          (error) => {
+            if (error.stack) {
+              console.error(error.stack)
+            }
+          })
+      }
+    }
+  })
+})
diff --git a/app/views/shared/vue/_in_tail_parse.html.haml b/app/views/shared/vue/_in_tail_parse.html.haml
new file mode 100644
index 000000000..a96809bd3
--- /dev/null
+++ b/app/views/shared/vue/_in_tail_parse.html.haml
@@ -0,0 +1,17 @@
+- add_javascript_pack_tag("in_tail_parse")
+
+= render "shared/vue/owned_plugin_form"
+
+#in-tail-parse{"v-bind:target-file" => "'#{setting.path}'"}
+  %owned-plugin-form{"v-bind:id" => "'parse-section'",
+                     "v-bind:options-json" => "'#{Fluent::Plugin::PARSER_REGISTRY.map.keys.to_json}'",
+                     "v-bind:initial-plugin-name" => "'#{setting.parse_type || setting.guess_parse_type}'",
+                     "v-bind:plugin-type" => "'parse'",
+                     "v-bind:plugin-label" => "'Parse'"}
+
+  .card.mb-3
+    %pre{"class" => "card-body", "v-html" => "highlightedLines"}
+
+  .card.md-3
+    %p.card-body.pb-0
+      =raw(t("fluentd.settings.in_tail_option_guide"))

From 838a3e484a1d45b45990a80c07a161e5c4245829 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 13 Jun 2018 17:07:21 +0900
Subject: [PATCH 081/137] Change argument name format to parse_type

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api_controller.rb |  3 +--
 lib/regexp_preview.rb             |  2 +-
 lib/regexp_preview/single_line.rb | 12 ++++++------
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index c1623904f..631c03c39 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -19,7 +19,7 @@ def empty_json
   end
 
   def regexp_preview
-    preview = RegexpPreview.processor(params[:format]).new(params[:file], params[:format], params)
+    preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], params)
 
     render json: preview.matches_json
   end
@@ -30,4 +30,3 @@ def grok_to_regexp
     render text: grok.convert_to_regexp(params[:grok_str]).source
   end
 end
-
diff --git a/lib/regexp_preview.rb b/lib/regexp_preview.rb
index bb0c81ff6..8591d5d9d 100644
--- a/lib/regexp_preview.rb
+++ b/lib/regexp_preview.rb
@@ -6,7 +6,7 @@
 require "regexp_preview/multi_line"
 
 module RegexpPreview
-  def self.processor(format)
+  def self.processor(parse_type)
     case format
     when "multiline"
       RegexpPreview::MultiLine
diff --git a/lib/regexp_preview/single_line.rb b/lib/regexp_preview/single_line.rb
index 5ef193bbd..22a47d8b2 100644
--- a/lib/regexp_preview/single_line.rb
+++ b/lib/regexp_preview/single_line.rb
@@ -2,13 +2,13 @@ module RegexpPreview
   class SingleLine
     attr_reader :file, :format, :params, :regexp, :time_format
 
-    def initialize(file, format, params = {})
+    def initialize(file, parse_type, params = {})
       @file = file
-      @format = format
+      @parse_type = parse_type
       @time_format = params[:time_format]
       @params = params
 
-      case format
+      case parse_type
       when "regexp"
         @regexp = Regexp.new(params[:regexp])
         @time_format = nil
@@ -16,8 +16,8 @@ def initialize(file, format, params = {})
         @regexp = nil
         @time_format = nil
       else # apache, nginx, etc
-        parser_plugin = Fluent::Plugin.new_parser(format)
-        raise "Unknown format '#{format}'" unless parser_plugin
+        parser_plugin = Fluent::Plugin.new_parser(parse_type)
+        raise "Unknown parse type '#{parse_type}'" unless parser_plugin
         parser_plugin.configure(Fluent::Config::Element.new('ROOT', '', {}, [])) # NOTE: SyslogParser define @regexp in configure method so call it to grab Regexp object
         @regexp = parser_plugin.instance_variable_get(:@regexp)
         @time_format = parser_plugin.time_format
@@ -28,7 +28,7 @@ def matches_json
       {
         params: {
           setting: {
-            # NOTE: regexp and time_format are used when format == 'apache' || 'nginx' || etc.
+            # NOTE: regexp and time_format are used when parse_type == 'apache' || 'nginx' || etc.
             regexp: regexp.try(:source),
             time_format: time_format,
           }

From a0ecbfa00485a793baf7ebfe483c344cd5994925 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 13 Jun 2018 17:08:12 +0900
Subject: [PATCH 082/137] Use v-model for expression and time_format

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../shared/vue/_owned_plugin_form.html.haml   | 23 +++++++++++++++----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/app/views/shared/vue/_owned_plugin_form.html.haml b/app/views/shared/vue/_owned_plugin_form.html.haml
index b70915d9e..160b91c97 100644
--- a/app/views/shared/vue/_owned_plugin_form.html.haml
+++ b/app/views/shared/vue/_owned_plugin_form.html.haml
@@ -42,8 +42,21 @@
                  "data-placement" => "right",
                  "v-bind:title" => "option.desc"}
             {{ option.name }}
-          %input{"v-bind:id" => "inputId(pluginType, option)",
-                 "v-bind:name" => "inputName(pluginType, option)",
-                 "v-bind:value" => "option.default",
-                 "type" => "text",
-                 "class" => "form-control"}
+          %template{"v-if" => 'option.name === "expression"'}
+            %input{"v-bind:id" => "inputId(pluginType, option)",
+                   "v-bind:name" => "inputName(pluginType, option)",
+                   "v-model.lazy" => "expression",
+                   "type" => "text",
+                   "class" => "form-control"}
+          %template{"v-else-if" => 'option.name === "time_format"'}
+            %input{"v-bind:id" => "inputId(pluginType, option)",
+                   "v-bind:name" => "inputName(pluginType, option)",
+                   "v-model.lazy" => "timeFormat",
+                   "type" => "text",
+                   "class" => "form-control"}
+          %template(v-else)
+            %input{"v-bind:id" => "inputId(pluginType, option)",
+                   "v-bind:name" => "inputName(pluginType, option)",
+                   "v-bind:value" => "option.default",
+                   "type" => "text",
+                   "class" => "form-control"}

From a806ac7b7c178cc8a9c6d0040b4858edc5166610 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Wed, 13 Jun 2018 17:09:12 +0900
Subject: [PATCH 083/137] Add custom events to handle data changed in child
 component

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js         | 39 ++++++++++++----
 app/javascript/packs/owned_plugin_form.js     | 46 ++++++++++++++++++-
 app/views/shared/vue/_in_tail_parse.html.haml |  4 +-
 3 files changed, 78 insertions(+), 11 deletions(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index f7e98f80c..9a8213353 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -25,14 +25,31 @@ $(document).ready(() => {
       'owned-plugin-form': OwnedPluginForm
     },
     watch: {
-      'params.setting.regexp': function() {
-        this.preview();
+      'parse.expression': function() {
+        console.log('parse.expression')
+        this.preview()
+      },
+      'parse.timeFormat': function() {
+        console.log('parse.timeFormat')
+        this.preview()
       },
       'parseType': function() {
-        this.preview();
+        this.preview()
       },
     },
+    mounted: function() {
+      this.parse = {}
+    },
     methods: {
+      onChangePluginName: function(name) {
+        console.log("onChangePluginName")
+        this.parseType = name
+      },
+      onChangeParseConfig: function(data) {
+        console.log("onChangeParseConfig")
+        _.merge(this.parse, data)
+        this.preview()
+      },
       updateHighlightedLines: function(matches) {
         if (!matches) {
           this.highlightedLines = null
@@ -91,17 +108,21 @@ $(document).ready(() => {
       },
 
       preview: function() {
-        $.ajax({
+        console.log("preview!!!!")
+        if (this.previewAjax) {
+          this.previewAjax.abort()
+        }
+
+        this.previewAjax = $.ajax({
           method: "POST",
           url: "/api/regexp_preview",
           headers: {
-            'X-CSRF-Token': token
+            'X-CSRF-Token': this.token
           },
           data: {
-            regexp: this.params.setting.regexp,
-            time_format: this.params.setting.time_format,
-            format: _.isEmpty(this.parseType) ? "regexp" : this.parseType,
-            params: this.params.setting,
+            expression: this.parse.expression,
+            time_format: this.parse.timeFormat,
+            parse_type: _.isEmpty(this.parseType) ? "regexp" : this.parseType,
             file: this.targetFile
           }
         }).then(
diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index d74019f81..d9f736d11 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -13,7 +13,11 @@ const OwnedPluginForm = {
       pluginName: "",
       options: [],
       commonOptions: [],
-      advancedOptions: []
+      advancedOptions: [],
+      expression: null,
+      timeFormat: null,
+      unwatchExpression: null,
+      unwatchTimeFormat: null
     }
   },
 
@@ -40,6 +44,9 @@ const OwnedPluginForm = {
   methods: {
     onChange: function() {
       this.updateSection()
+      if (this.pluginType === "parse") {
+        this.$emit("change-plugin-name", this.pluginName)
+      }
     },
 
     updateSection: function() {
@@ -55,6 +62,43 @@ const OwnedPluginForm = {
         }
       }).then((data) => {
         this.commonOptions = data.commonOptions
+        let foundExpression = false
+        let foundTimeFormat = false
+        _.each(this.commonOptions, (option) => {
+          if (option.name === "expression") {
+            foundExpression = true
+            this.expression = option.default
+            this.unwatchExpression = this.$watch("expression", (newValue, oldValue) => {
+              console.log(newValue)
+              this.$emit("change-parse-config", {
+                expression: this.expression,
+                timeFormat: this.timeFormat
+              })
+            })
+          }
+          if (option.name === "time_format") {
+            foundTimeFormat = true
+            this.timeFormat = option.default
+            this.unwatchTimeFormat = this.$watch("timeFormat", (newValue, oldValue) => {
+              console.log({"watch time_format": newValue})
+              this.$emit("change-parse-config", {
+                expression: this.expression,
+                timeFormat: this.timeFormat
+              })
+            })
+          }
+
+          if (!foundExpression && this.unwatchExpression) {
+            this.expression = null
+            this.unwatchExpression()
+            this.unwatchExpression = null
+          }
+          if (!foundTimeFormat && this.unwatchTimeFormat) {
+            this.timeFormat = null
+            this.unwatchTimeFormat()
+            this.unwatchTimeFormat = null
+          }
+        })
       })
     },
 
diff --git a/app/views/shared/vue/_in_tail_parse.html.haml b/app/views/shared/vue/_in_tail_parse.html.haml
index a96809bd3..35c7b5232 100644
--- a/app/views/shared/vue/_in_tail_parse.html.haml
+++ b/app/views/shared/vue/_in_tail_parse.html.haml
@@ -7,7 +7,9 @@
                      "v-bind:options-json" => "'#{Fluent::Plugin::PARSER_REGISTRY.map.keys.to_json}'",
                      "v-bind:initial-plugin-name" => "'#{setting.parse_type || setting.guess_parse_type}'",
                      "v-bind:plugin-type" => "'parse'",
-                     "v-bind:plugin-label" => "'Parse'"}
+                     "v-bind:plugin-label" => "'Parse'",
+                     "v-on:change-parse-config" => "onChangeParseConfig",
+                     "v-on:change-plugin-name" => "onChangePluginName"}
 
   .card.mb-3
     %pre{"class" => "card-body", "v-html" => "highlightedLines"}

From bdea8c7ce501c712af3616a83264ac88f0a49355 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 09:44:39 +0900
Subject: [PATCH 084/137] Follow the parameter name change

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 lib/regexp_preview.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/regexp_preview.rb b/lib/regexp_preview.rb
index 8591d5d9d..e534ccc33 100644
--- a/lib/regexp_preview.rb
+++ b/lib/regexp_preview.rb
@@ -7,7 +7,7 @@
 
 module RegexpPreview
   def self.processor(parse_type)
-    case format
+    case parse_type
     when "multiline"
       RegexpPreview::MultiLine
     else

From d351691dc0dccb64df083bb8d718bf656ec16701 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:19:51 +0900
Subject: [PATCH 085/137] Fix a typo

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 9a8213353..7f5007d21 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -90,7 +90,7 @@ $(document).ready(() => {
 
             let pos = {
               start: match.pos[0],
-              ent: match.pos[1]
+              end: match.pos[1]
             }
             if (pos.start > 0) {
               html += _.escape(whole.substring(lastPos, pos.start))

From e4e2da31c2444a187ab18cb2031ef80de433269f Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:20:04 +0900
Subject: [PATCH 086/137] Use function instead of arrow function

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 7f5007d21..532a2fd4c 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -11,7 +11,7 @@ $(document).ready(() => {
       "targetFile",
       "parseType",
     ],
-    data: () => {
+    data: function() {
       return {
         highlightedLines: null
       }

From e2fd7a967bf82431a38ddc9c4658f6a74fc8bcd1 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:20:25 +0900
Subject: [PATCH 087/137] Fix a typo

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 532a2fd4c..0ef889e56 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -127,7 +127,7 @@ $(document).ready(() => {
           }
         }).then(
           (result) => {
-            this.params = _merge(this.params, result.params)
+            this.params = _.merge(this.params, result.params)
             this.updateHighlightedLines(result.matches)
           },
           (error) => {

From c513c35d2a3a9fa87762a79ace8d89f00a157891 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:20:36 +0900
Subject: [PATCH 088/137] Change variable name targetFile to path

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js         | 9 ++++++---
 app/views/shared/vue/_in_tail_parse.html.haml | 2 +-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 0ef889e56..2d46d05c5 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -8,8 +8,8 @@ $(document).ready(() => {
   new Vue({
     el: '#in-tail-parse',
     props: [
-      "targetFile",
-      "parseType",
+      "path",
+      "parseType"
     ],
     data: function() {
       return {
@@ -37,6 +37,9 @@ $(document).ready(() => {
         this.preview()
       },
     },
+    beforeMount: function() {
+      this.path = this.$el.attributes.path.nodeValue;
+    },
     mounted: function() {
       this.parse = {}
     },
@@ -123,7 +126,7 @@ $(document).ready(() => {
             expression: this.parse.expression,
             time_format: this.parse.timeFormat,
             parse_type: _.isEmpty(this.parseType) ? "regexp" : this.parseType,
-            file: this.targetFile
+            file: this.path
           }
         }).then(
           (result) => {
diff --git a/app/views/shared/vue/_in_tail_parse.html.haml b/app/views/shared/vue/_in_tail_parse.html.haml
index 35c7b5232..be481e5b3 100644
--- a/app/views/shared/vue/_in_tail_parse.html.haml
+++ b/app/views/shared/vue/_in_tail_parse.html.haml
@@ -2,7 +2,7 @@
 
 = render "shared/vue/owned_plugin_form"
 
-#in-tail-parse{"v-bind:target-file" => "'#{setting.path}'"}
+#in-tail-parse{"path" => setting.path}
   %owned-plugin-form{"v-bind:id" => "'parse-section'",
                      "v-bind:options-json" => "'#{Fluent::Plugin::PARSER_REGISTRY.map.keys.to_json}'",
                      "v-bind:initial-plugin-name" => "'#{setting.parse_type || setting.guess_parse_type}'",

From 38d4fba7e5a39f0d94858ecc30f8549ec3bb28c4 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:21:09 +0900
Subject: [PATCH 089/137] Rewrite RegexpPreview::SingleLine using Fluentd's
 parser plugin instance

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 lib/regexp_preview/single_line.rb           |  86 +++++-----
 spec/lib/regexp_preview/single_line_spec.rb | 176 ++++++++++----------
 2 files changed, 129 insertions(+), 133 deletions(-)

diff --git a/lib/regexp_preview/single_line.rb b/lib/regexp_preview/single_line.rb
index 22a47d8b2..3af686fa6 100644
--- a/lib/regexp_preview/single_line.rb
+++ b/lib/regexp_preview/single_line.rb
@@ -1,65 +1,59 @@
 module RegexpPreview
   class SingleLine
-    attr_reader :file, :format, :params, :regexp, :time_format
+    attr_reader :path, :plugin_name, :plugin_config, :plugin
 
-    def initialize(file, parse_type, params = {})
-      @file = file
-      @parse_type = parse_type
-      @time_format = params[:time_format]
-      @params = params
+    def initialize(path, plugin_name, plugin_config = {})
+      @path = path
+      @plugin_name = plugin_name.to_sym
+      @plugin_config = plugin_config
 
-      case parse_type
-      when "regexp"
-        @regexp = Regexp.new(params[:regexp])
-        @time_format = nil
-      when "ltsv", "json", "csv", "tsv"
-        @regexp = nil
-        @time_format = nil
-      else # apache, nginx, etc
-        parser_plugin = Fluent::Plugin.new_parser(parse_type)
-        raise "Unknown parse type '#{parse_type}'" unless parser_plugin
-        parser_plugin.configure(Fluent::Config::Element.new('ROOT', '', {}, [])) # NOTE: SyslogParser define @regexp in configure method so call it to grab Regexp object
-        @regexp = parser_plugin.instance_variable_get(:@regexp)
-        @time_format = parser_plugin.time_format
+      config = Fluent::Config::Element.new("ROOT", "", @plugin_config, [])
+      @plugin = Fluent::Plugin.new_parser(@plugin_name).tap do |instance|
+        instance.configure(config)
       end
     end
 
-    def matches_json
+    def matches
       {
-        params: {
-          setting: {
-            # NOTE: regexp and time_format are used when parse_type == 'apache' || 'nginx' || etc.
-            regexp: regexp.try(:source),
-            time_format: time_format,
-          }
-        },
-        matches: matches.compact,
+        pluginConfig: @plugin_config,
+        matches: _matches
       }
     end
 
     private
 
-    def matches
-      return [] unless @regexp # such as ltsv, json, etc
-      reader = FileReverseReader.new(File.open(file))
-      matches = reader.tail(Settings.in_tail_preview_line_count).map do |line|
-        result = {
-          :whole => line,
-          :matches => [],
-        }
-        match = line.match(regexp)
-        next result unless match
-
-        match.names.each_with_index do |name, index|
-          result[:matches] << {
-            key: name,
-            matched: match[name],
-            pos: match.offset(index + 1),
+    def _matches
+      return [] if %i(json csv tsv ltsv).include?(@plugin_name)
+      begin
+        io = File.open(path)
+        reader = FileReverseReader.new(io)
+        parsed_lines = reader.tail(Settings.in_tail_preview_line_count).map do |line|
+          parsed = {
+            whole: line,
+            matches: []
           }
+          @plugin.parse(line) do |time, record|
+            next unless record
+            last_pos = 0
+            record.each do |key, value|
+              start = line.index(value, last_pos)
+              finish = start + value.bytesize
+              last_pos = finish + 1
+              parsed[:matches] << {
+                key: key,
+                matched: value,
+                pos: [start, finish]
+              }
+            end
+          end
+          parsed
+        end
+        parsed_lines.reject do |parsed|
+          parsed[:matches].blank?
         end
-        result
+      ensure
+        io.close
       end
-      matches
     end
   end
 end
diff --git a/spec/lib/regexp_preview/single_line_spec.rb b/spec/lib/regexp_preview/single_line_spec.rb
index a1ee90c19..9540f5d61 100644
--- a/spec/lib/regexp_preview/single_line_spec.rb
+++ b/spec/lib/regexp_preview/single_line_spec.rb
@@ -2,123 +2,109 @@
 
 describe RegexpPreview::SingleLine do
   describe ".initialize" do
-    subject { RegexpPreview::SingleLine.new("log_file.log", format, params) }
-
-    describe "format" do
-      let :params do
-        {
-          regexp: "(?<category>\[.+\])",
-          time_format: "%y/%m/%d",
-        }
-      end
+    subject { RegexpPreview::SingleLine.new("log_file.log", plugin_name, plugin_config) }
 
-      shared_examples "should set regexp and time_format from selected format" do
-        it do
-          expect(subject.regexp).to eq regexp
-          expect(subject.time_format).to eq time_format
-          expect(subject.params).to eq params
-        end
+    describe "parse" do
+      let :plugin_config do
+        {}
       end
 
-      shared_examples "should set params only" do
-        include_examples "should set regexp and time_format from selected format" do
-          let(:regexp) { nil }
-          let(:time_format) { nil }
+      shared_examples "should create parser plugin instance from selected plugin name" do
+        it do
+          expect(subject.plugin).to(be_an_instance_of(plugin_class))
         end
       end
 
       context "regexp" do
-        let(:format) { "regexp" }
-
-        it 'should set regexp from params' do
-          expect(subject.regexp).to eq /#{params[:regexp]}/
-          expect(subject.time_format).to be_nil
-          expect(subject.params).to eq params
+        let(:plugin_name) { "regexp" }
+        let(:plugin_class) { Fluent::Plugin::RegexpParser }
+        let :plugin_config do
+          {
+            "expression" => "(?<category>\[.+\])",
+            "time_format" => "%y/%m/%d",
+          }
         end
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "ltsv" do
-        let(:format) { "ltsv" }
+        let(:plugin_name) { "ltsv" }
+        let(:plugin_class) { Fluent::Plugin::LabeledTSVParser }
 
-        include_examples "should set params only"
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "json" do
-        let(:format) { "json" }
+        let(:plugin_name) { "json" }
+        let(:plugin_class) { Fluent::Plugin::JSONParser }
 
-        include_examples "should set params only"
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "csv" do
-        let(:format) { "csv" }
+        let(:plugin_name) { "csv" }
+        let(:plugin_class) { Fluent::Plugin::CSVParser }
+        let(:plugin_config) do
+          {
+            "keys" => "column1,column2"
+          }
+        end
 
-        include_examples "should set params only"
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "tsv" do
-        let(:format) { "tsv" }
+        let(:plugin_name) { "tsv" }
+        let(:plugin_class) { Fluent::Plugin::TSVParser }
+        let(:plugin_config) do
+          {
+            "keys" => "column1,column2"
+          }
+        end
 
-        include_examples "should set params only"
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "syslog" do # "apache", "nginx", etc
-        let(:format) { "syslog" }
+        let(:plugin_name) { "syslog" }
+        let(:plugin_class) { Fluent::Plugin::SyslogParser }
 
-        include_examples "should set regexp and time_format from selected format" do
-          let(:regexp) do
-            /^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\[]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$/
-          end
-          let(:time_format) { "%b %d %H:%M:%S" }
-        end
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "apache" do
-        let(:format) { "apache" }
+        let(:plugin_name) { "apache" }
+        let(:plugin_class) { Fluent::Plugin::ApacheParser }
 
-        include_examples "should set regexp and time_format from selected format" do
-          let(:regexp) do
-            /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
-          end
-          let(:time_format) { "%d/%b/%Y:%H:%M:%S %z" }
-        end
+        include_examples("should create parser plugin instance from selected plugin name")
       end
 
       context "nginx" do
-        let(:format) { "nginx" }
+        let(:plugin_name) { "nginx" }
+        let(:plugin_class) { Fluent::Plugin::NginxParser }
 
-        include_examples "should set regexp and time_format from selected format" do
-          let(:regexp) do
-            /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+(?<http_x_forwarded_for>[^ ]+))?)?$/
-          end
-          let(:time_format) { "%d/%b/%Y:%H:%M:%S %z" }
-        end
+        include_examples("should create parser plugin instance from selected plugin name")
       end
     end
   end
 
-  describe "#matches_json" do
+  describe "#matches" do
     let(:logfile) { File.expand_path(logfile_path, Rails.root) }
-    let :params do
-      {
-        regexp: "(?<regexp>bar)", # bar from error0.log
-        time_format: "time_format",
-      }
-    end
-
-    subject { RegexpPreview::SingleLine.new(logfile, format, params).matches_json }
+    subject { RegexpPreview::SingleLine.new(logfile, plugin_name, plugin_config).matches }
 
-    describe "format" do
+    describe "parse" do
       context "regexp" do
-        let(:format) { "regexp" }
+        let(:plugin_name) { "regexp" }
         let(:logfile_path) { "./spec/support/fixtures/error0.log" }
-
-        it 'should have regexp only in [:params][:setting]' do
-          setting_json = {
-            regexp: params[:regexp],
-            time_format: nil
+        let :plugin_config do
+          {
+            "expression" => "(?<regexp>bar)", # bar from error0.log
+            "time_format" => "time_format",
           }
+        end
 
-          expect(subject[:params][:setting]).to eq setting_json
+        it 'should have regexp only in [:params][:setting]' do
+          expect(subject[:pluginConfig]).to eq plugin_config
         end
 
         it 'should include matches info' do
@@ -133,16 +119,12 @@
       end
 
       context "csv" do
-        let(:format) { "csv" }
+        let(:plugin_name) { "csv" }
         let(:logfile_path) { "./spec/support/fixtures/error0.log" }
-
-        it 'should not have regexp and time_format in [:params][:setting]' do
-          setting_json = {
-            regexp: nil,
-            time_format: nil
+        let :plugin_config do
+          {
+            "keys" => "column1,column2"
           }
-
-          expect(subject[:params][:setting]).to eq setting_json
         end
 
         it 'should not have matches_info' do
@@ -151,26 +133,46 @@
       end
 
       context "syslog" do
-        let(:format) { "syslog" }
         let(:logfile_path) { "./spec/support/fixtures/error4.log" }
+        let(:plugin_name) { "syslog" }
+        let(:plugin_config) do
+          {
+            "time_format" => "%Y-%m-%d %H:%M:%S %z",
+            "keep_time_key" => true
+          }
+        end
 
-        it 'should set regexp and time_format from syslog format' do
-          setting_json = {
-            regexp: "^(?<time>[^ ]*\\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[^ :\\[]*)(?:\\[(?<pid>[0-9]+)\\])?(?:[^\\:]*\\:)? *(?<message>.*)$",
-            time_format: "%b %d %H:%M:%S",
+        it 'should include matches info' do
+          matches_info = {
+            whole: "2014-05-27 10:54:37 +0900 [info]: listening fluent socket on 0.0.0.0:24224",
+            matches: [
+              { key: "time", matched: "2014-05-27 10:54:37 +0900", pos: [0, 25] },
+              { key: "host", matched: "[info]:", pos: [26, 33] },
+              { key: "ident", matched: "listening", pos: [34, 43] },
+              { key: "message", matched: "24224", pos: [69, 74] }
+            ]
           }
 
-          expect(subject[:params][:setting]).to eq setting_json
+          expect(subject[:matches]).to include matches_info
+        end
+      end
+
+      context "syslog when keep_time_key is false" do
+        let(:logfile_path) { "./spec/support/fixtures/error4.log" }
+        let(:plugin_name) { "syslog" }
+        let(:plugin_config) do
+          {
+            "time_format" => "%Y-%m-%d %H:%M:%S %z",
+            "keep_time_key" => false
+          }
         end
 
         it 'should include matches info' do
           matches_info = {
             whole: "2014-05-27 10:54:37 +0900 [info]: listening fluent socket on 0.0.0.0:24224",
             matches: [
-              { key: "time", matched: "2014-05-27 10:54:37 +0900", pos: [0, 25] },
               { key: "host", matched: "[info]:", pos: [26, 33] },
               { key: "ident", matched: "listening", pos: [34, 43] },
-              { key: "pid", matched: nil, pos: [nil, nil] },
               { key: "message", matched: "24224", pos: [69, 74] }
             ]
           }

From 55f7ebe91b4743ca9060f6ebe2141077a0c96a57 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:37:50 +0900
Subject: [PATCH 090/137] Follow the previous change

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index 631c03c39..fc2e4d1fa 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -21,7 +21,7 @@ def empty_json
   def regexp_preview
     preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], params)
 
-    render json: preview.matches_json
+    render json: preview.matches
   end
 
   def grok_to_regexp

From 951b4440e7273ce9c693296120e24a3c83809e30 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:38:15 +0900
Subject: [PATCH 091/137] Remove useless code

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 2d46d05c5..5369906e3 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -130,7 +130,6 @@ $(document).ready(() => {
           }
         }).then(
           (result) => {
-            this.params = _.merge(this.params, result.params)
             this.updateHighlightedLines(result.matches)
           },
           (error) => {

From 809c2a0c476bfabcda1b60dc9eda6783a1a279d2 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:38:29 +0900
Subject: [PATCH 092/137] Fix typos

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 5369906e3..3b96407d7 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -86,14 +86,13 @@ $(document).ready(() => {
               "class": "regexp-preview",
               "data-toggle": "tooltip",
               "data-placement": "top",
-              "title": match.key,
+              "title": m.key,
               'style': 'background-color:' + currentColor
             })
             let highlightedHtml = $highlighted.wrap("<div>").parent().html()
-
             let pos = {
-              start: match.pos[0],
-              end: match.pos[1]
+              start: m.pos[0],
+              end: m.pos[1]
             }
             if (pos.start > 0) {
               html += _.escape(whole.substring(lastPos, pos.start))

From b96a6ed95fbeca7bc5e508ce0c53090550295873 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 14:38:34 +0900
Subject: [PATCH 093/137] Enable tooltip

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 3b96407d7..83ef645d5 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -42,6 +42,10 @@ $(document).ready(() => {
     },
     mounted: function() {
       this.parse = {}
+      this.$on("hook:updated", () => {
+        $("[data-toggle=tooltip]").tooltip("dispose")
+        $("[data-toggle=tooltip]").tooltip("enable")
+      })
     },
     methods: {
       onChangePluginName: function(name) {
@@ -107,6 +111,7 @@ $(document).ready(() => {
         })
 
         this.highlightedLines = $container.html()
+        this.$emit("hook:updated")
       },
 
       preview: function() {

From 81be7792bd3f5bd8ff93e61fe7b30d18a5798c68 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 15:33:52 +0900
Subject: [PATCH 094/137] Rewrite RegexpPreview::MultiLine using Fluentd's
 parser plugin

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 lib/regexp_preview/multi_line.rb           | 116 ++++++++++-----------
 spec/lib/regexp_preview/multi_line_spec.rb |  63 ++++++-----
 2 files changed, 82 insertions(+), 97 deletions(-)

diff --git a/lib/regexp_preview/multi_line.rb b/lib/regexp_preview/multi_line.rb
index b8fc6e544..be304935b 100644
--- a/lib/regexp_preview/multi_line.rb
+++ b/lib/regexp_preview/multi_line.rb
@@ -1,80 +1,70 @@
 module RegexpPreview
   class MultiLine
-    attr_reader :file, :format, :params
+    attr_reader :path, :plugin_name, :plugin_config, :plugin
 
-    def initialize(file, format, params = {})
-      @file = file
-      @format = format
-      @params = params[:params]
+    def initialize(path, plugin_name, plugin_config = {})
+      @path = path
+      @plugin_name = plugin_name
+      @plugin_config = plugin_config
+
+      config = Fluent::Config::Element.new("ROOT", "", @plugin_config, [])
+      @plugin = Fluent::Plugin.new_parser(@plugin_name).tap do |instance|
+        instance.configure(config)
+      end
     end
 
-    def matches_json
+    def matches
       {
-        params: {
-          setting: { # for vue.js
-            regexp: nil,
-            time_format: nil,
-          }
-        },
-        matches: matches.compact,
+        pluginConfig: @plugin_config,
+        matches: _matches,
       }
     end
 
     private
 
-    def matches
-      return [] if patterns.empty?
-      reader = FileReverseReader.new(File.open(file))
-      result = []
-      target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" }
-      whole_string = target_lines.join
-      re_firstline = Regexp.new(params[:format_firstline])
-      indexes = []
-      cur = 0
-      while first_index = whole_string.index(re_firstline, cur)
-        indexes << first_index
-        cur = first_index + 1
-      end
-      indexes.each_with_index do |index, i|
-        next_index = indexes[i + 1] || -1
-        chunk = whole_string[index...next_index]
-        ret = detect_chunk(chunk)
-        next unless ret
-        result << ret
-      end
-      result
-    end
-
-    def detect_chunk(chunk)
-      whole = ""
-      matches = []
-      offset = 0
-      patterns.each do |pat|
-        match = chunk.match(pat)
-        return nil unless match
-        offset = chunk.index(pat)
-        return nil if offset > 0
-        chunk = chunk[match[0].length..-1]
-        match.names.each_with_index do |name, index|
-          matches << {
-            key: name,
-            matched: match[name],
-            pos: match.offset(index + 1).map{|pos| pos + whole.length},
+    def _matches
+      begin
+        io = File.open(path)
+        reader = FileReverseReader.new(io)
+        parserd_chunks = []
+        target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" }
+        whole_string = target_lines.join
+        firstline_regex = Regexp.new(plugin_config["format_firstline"][1..-2])
+        indexes = []
+        cur = 0
+        while first_index = whole_string.index(firstline_regex, cur)
+          indexes << first_index
+          cur = first_index + 1
+        end
+        indexes.each_with_index do |index, i|
+          next_index = indexes[i + 1] || -1
+          chunk = whole_string[index...next_index]
+          parsed = {
+            whole: chunk,
+            matches: []
           }
+          @plugin.parse(chunk) do |time, record|
+            next unless record
+            last_pos = 0
+            record.each do |key, value|
+              start = chunk.index(value, last_pos)
+              finish = start + value.bytesize
+              last_pos = finish
+              parsed[:matches] << {
+                key: key,
+                matched: value,
+                pos: [start, finish]
+              }
+            end
+          end
+          parserd_chunks << parsed
         end
-        whole << match[0]
+        parserd_chunks.reject do |parsed|
+          parsed[:matches].blank?
+        end
+      ensure
+        io.close
       end
-      {
-        whole: whole,
-        matches: matches,
-      }
-    end
-
-    def patterns
-      @patterns ||= (1..20).map do |n|
-        params["format#{n}"].presence
-      end.compact.map {|pattern| Regexp.new(pattern, Regexp::MULTILINE)}
     end
   end
 end
-
diff --git a/spec/lib/regexp_preview/multi_line_spec.rb b/spec/lib/regexp_preview/multi_line_spec.rb
index 97b40ea7e..4e5927dc7 100644
--- a/spec/lib/regexp_preview/multi_line_spec.rb
+++ b/spec/lib/regexp_preview/multi_line_spec.rb
@@ -1,33 +1,29 @@
 require 'spec_helper'
 
 describe RegexpPreview::MultiLine do
-  describe "#matches_json" do
-    subject { parser.matches_json }
-    let(:parser) { RegexpPreview::MultiLine.new(target_path, "multiline", params) }
+  describe "#matches" do
+    subject { parser.matches }
+    let(:parser) { RegexpPreview::MultiLine.new(target_path, "multiline", plugin_config) }
 
     describe "simple usage" do
       let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
 
-      let :params do
-        params = {
-          format_firstline: "foo",
-          time_format: "time_format",
+      let :plugin_config do
+        plugin_config = {
+          "format_firstline" => "/foo/",
+          "time_format" => "time_format",
         }
-        params["format1"] = "(?<foo>foo)\n"
-        params["format2"] = "(?<bar>bar)"
+        plugin_config["format1"] = "/(?<foo>foo)\n/"
+        plugin_config["format2"] = "/(?<bar>bar)/"
         3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-          params["format#{i}"] = ""
+          plugin_config["format#{i}"] = "//"
         end
-        { params: params }
-      end
-
-      it 'should not have regexp and time_format in [:params][:setting]' do
-        expect(subject[:params][:setting]).to eq(regexp: nil, time_format: nil)
+        plugin_config
       end
 
       it "should include matches info" do
         matches_info = {
-          whole: "foo\nbar",
+          whole: "foo\nbar\nbaz\n1\n2\n3\n4\n5\n6\n10\n11\n12",
           matches: [
             {
               key: "foo", matched: "foo", pos: [0, 3]
@@ -37,24 +33,23 @@
             }
           ]
         }
-
         expect(subject[:matches]).to include matches_info
       end
     end
 
     describe "detect only continuos patterns" do
       let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
-      let(:params) do
-        params = {
-          format_firstline: "foo",
-          time_format: "time_format",
+      let(:plugin_config) do
+        plugin_config = {
+          "format_firstline" => "/foo/",
+          "time_format" => "time_format",
         }
-        params["format1"] = "(?<foo>foo)\n"
-        params["format2"] = "(?<bar>baz)"
+        plugin_config["format1"] = "/(?<foo>foo)\n/"
+        plugin_config["format2"] = "/(?<bar>baz)/"
         3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-          params["format#{i}"] = ""
+          plugin_config["format#{i}"] = "//"
         end
-        { params: params }
+        plugin_config
       end
 
       it "shouldn't match" do
@@ -66,21 +61,21 @@
       # http://docs.fluentd.org/articles/in_tail
       let(:target_path) { File.expand_path("./spec/support/fixtures/multiline_example.log", Rails.root) }
 
-      let :params do
-        params = {
-          format_firstline: "\\d{4}-\\d{1,2}-\\d{1,2}",
-          "format1" => "^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)",
-          time_format: "time_format",
+      let :plugin_config do
+        plugin_config = {
+          "format_firstline" => "/\\d{4}-\\d{1,2}-\\d{1,2}/",
+          "format1" => "/^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)/",
+          "time_format" => "%Y-%m-%d %H:%M:%S",
+          "keep_time_key" => true
         }
         2.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-          params["format#{i}"] = ""
+          plugin_config["format#{i}"] = "//"
         end
-        { params: params }
+        plugin_config
       end
 
       it "should include matches info" do
-        matches_info = 
-          [
+        matches_info = [
             {
               whole: "2013-3-03 14:27:33 [main] INFO  Main - Start\n",
               matches: [

From 9b0abd9338906d4beaf42e4b8dd5f0150eb80272 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 15:36:26 +0900
Subject: [PATCH 095/137] Remove garbage

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 lib/regexp_preview/single_line.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/regexp_preview/single_line.rb b/lib/regexp_preview/single_line.rb
index 3af686fa6..6349aa6ed 100644
--- a/lib/regexp_preview/single_line.rb
+++ b/lib/regexp_preview/single_line.rb
@@ -38,7 +38,7 @@ def _matches
             record.each do |key, value|
               start = line.index(value, last_pos)
               finish = start + value.bytesize
-              last_pos = finish + 1
+              last_pos = finish
               parsed[:matches] << {
                 key: key,
                 matched: value,

From ef9c554b74d23767bca3dfd09fbc965cbf370dab Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 16:02:01 +0900
Subject: [PATCH 096/137] Handle API error

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 83ef645d5..74725475d 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -137,6 +137,8 @@ $(document).ready(() => {
             this.updateHighlightedLines(result.matches)
           },
           (error) => {
+            this.highlightedLines = null
+            console.error(error.responseText)
             if (error.stack) {
               console.error(error.stack)
             }

From e6d2c4048da1c4904ca3cecc88de9d45d3373ebd Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Thu, 14 Jun 2018 16:02:24 +0900
Subject: [PATCH 097/137] Run one Ajax request at once

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 74725475d..ed84874b0 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -116,7 +116,7 @@ $(document).ready(() => {
 
       preview: function() {
         console.log("preview!!!!")
-        if (this.previewAjax) {
+        if (this.previewAjax && this.previewAjax.state() === "pending") {
           this.previewAjax.abort()
         }
 

From cbbd169ae0f46bc131b615f4c4fab86ae72af174 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 10:14:43 +0900
Subject: [PATCH 098/137] Add Vue component for parser_multiline

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/parser_multiline_form.js | 37 +++++++++++++++++++
 .../vue/_parser_multiline_form.html.haml      |  8 ++++
 2 files changed, 45 insertions(+)
 create mode 100644 app/javascript/packs/parser_multiline_form.js
 create mode 100644 app/views/shared/vue/_parser_multiline_form.html.haml

diff --git a/app/javascript/packs/parser_multiline_form.js b/app/javascript/packs/parser_multiline_form.js
new file mode 100644
index 000000000..3d932b356
--- /dev/null
+++ b/app/javascript/packs/parser_multiline_form.js
@@ -0,0 +1,37 @@
+'use strict'
+const ParserMultilineForm = {
+  template: "#vue-parser-multiline-form",
+  props: [
+    "pluginType"
+  ],
+
+  data: function() {
+    return {
+      formats: ""
+    }
+  },
+
+  watch: {
+    "formats": function(newValue, oldValue) {
+      console.log(`watch formats: ${newValue}`)
+      this.$emit("change-formats", newValue)
+    }
+  },
+
+  methods: {
+    hiddenId: function(pluginType, n) {
+      return `setting_${pluginType}_0__format${n}`
+    },
+    hiddenName: function(pluginType, n) {
+      return `setting[${pluginType}[0]][format${n}]`
+    },
+    textareaId: function(pluginType) {
+      return `setting_${pluginType}_0__formats`
+    },
+    textareaName: function(pluginType) {
+      return `setting[${pluginType}[0]][formats]`
+    }
+  }
+}
+
+export { ParserMultilineForm as default }
diff --git a/app/views/shared/vue/_parser_multiline_form.html.haml b/app/views/shared/vue/_parser_multiline_form.html.haml
new file mode 100644
index 000000000..7c6427440
--- /dev/null
+++ b/app/views/shared/vue/_parser_multiline_form.html.haml
@@ -0,0 +1,8 @@
+%script{type: "text/x-template", id: "vue-parser-multiline-form"}
+  .form-group
+    %label.mt-3{"v-bind:for" => "textareaId(pluginType)"}
+      Formats
+    %textarea.form-control{"v-bind:id" => "textareaId(pluginType)",
+                           "v-bind:name" => "textareaName(pluginType)",
+                           "v-model.lazy" => "formats",
+                           "rows" => 20}

From 81b063a0edae8cec595f50fff153d7e069796daa Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 10:15:29 +0900
Subject: [PATCH 099/137] Use ParserMultilineForm

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/owned_plugin_form.js         | 6 ++++++
 app/views/shared/vue/_owned_plugin_form.html.haml | 7 ++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index d9f736d11..a2bb00982 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -1,6 +1,12 @@
 'use strict'
+
+import ParserMultilineForm from './parser_multiline_form'
+
 const OwnedPluginForm = {
   template: "#vue-owned-plugin-form",
+  components: {
+    "parser-multiline-form": ParserMultilineForm,
+  },
   props: [
     "id",
     "optionsJson",
diff --git a/app/views/shared/vue/_owned_plugin_form.html.haml b/app/views/shared/vue/_owned_plugin_form.html.haml
index 160b91c97..82372f98a 100644
--- a/app/views/shared/vue/_owned_plugin_form.html.haml
+++ b/app/views/shared/vue/_owned_plugin_form.html.haml
@@ -1,3 +1,5 @@
+= render "shared/vue/parser_multiline_form"
+
 %script{type: "text/x-template", id: "vue-owned-plugin-form"}
   .form-group.card.bg-light.mb-3{"v-bind:id" => "id"}
     .card-body
@@ -12,7 +14,10 @@
                 "v-bind:value" => "option",
                 "v-bind:selected" => "pluginName===option"}
           {{ option }}
-      .form-group{ "v-for" => "option in commonOptions" }
+      %parser-multiline-form{"v-if" => 'pluginName==="multiline"',
+                             "v-bind:plugin-type" => "pluginType",
+                             "v-on:change-formats" => "onChangeFormats"}
+      .form-group(v-else){"v-for" => "option in commonOptions"}
         %template{"v-if" => 'option.type==="enum"'}
           %label{"v-bind:for" => "inputId(pluginType, option)",
                  "data-toggle" => "tooltip",

From 076e692ab0a5175b1e047d03c68b65f42be8004e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 10:15:46 +0900
Subject: [PATCH 100/137] Handle custom event change-formats

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js         | 10 ++++++++--
 app/javascript/packs/owned_plugin_form.js     |  5 +++++
 app/views/shared/vue/_in_tail_parse.html.haml |  3 ++-
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index ed84874b0..d0d0392e1 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -57,6 +57,11 @@ $(document).ready(() => {
         _.merge(this.parse, data)
         this.preview()
       },
+      onChangeFormats: function(formats) {
+        console.log(`in_tail_parse:onChangeFormats: ${formats}`)
+        this.parse.formats = formats
+        this.preview()
+      },
       updateHighlightedLines: function(matches) {
         if (!matches) {
           this.highlightedLines = null
@@ -130,7 +135,8 @@ $(document).ready(() => {
             expression: this.parse.expression,
             time_format: this.parse.timeFormat,
             parse_type: _.isEmpty(this.parseType) ? "regexp" : this.parseType,
-            file: this.path
+            file: this.path,
+            plugin_config: this.parse
           }
         }).then(
           (result) => {
@@ -138,7 +144,7 @@ $(document).ready(() => {
           },
           (error) => {
             this.highlightedLines = null
-            console.error(error.responseText)
+            // console.error(error.responseText)
             if (error.stack) {
               console.error(error.stack)
             }
diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index a2bb00982..22d161297 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -55,6 +55,11 @@ const OwnedPluginForm = {
       }
     },
 
+    onChangeFormats: function(newValue, oldValue) {
+      console.log(`ownedPluginForm:onChangeFormats: ${newValue}`)
+      this.$emit("change-formats", newValue)
+    },
+
     updateSection: function() {
       $.ajax({
         method: "GET",
diff --git a/app/views/shared/vue/_in_tail_parse.html.haml b/app/views/shared/vue/_in_tail_parse.html.haml
index be481e5b3..81fe0a67c 100644
--- a/app/views/shared/vue/_in_tail_parse.html.haml
+++ b/app/views/shared/vue/_in_tail_parse.html.haml
@@ -9,7 +9,8 @@
                      "v-bind:plugin-type" => "'parse'",
                      "v-bind:plugin-label" => "'Parse'",
                      "v-on:change-parse-config" => "onChangeParseConfig",
-                     "v-on:change-plugin-name" => "onChangePluginName"}
+                     "v-on:change-plugin-name" => "onChangePluginName",
+                     "v-on:change-formats" => "onChangeFormats"}
 
   .card.mb-3
     %pre{"class" => "card-body", "v-html" => "highlightedLines"}

From e7580f42f91217f8e06ba2bab4f6b11656d6685f Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:00:05 +0900
Subject: [PATCH 101/137] Use underscored name for query parameter

Because it is Rails way.

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js     | 4 ++--
 app/javascript/packs/owned_plugin_form.js | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index d0d0392e1..02ef0c58b 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -29,8 +29,8 @@ $(document).ready(() => {
         console.log('parse.expression')
         this.preview()
       },
-      'parse.timeFormat': function() {
-        console.log('parse.timeFormat')
+      'parse.time_format': function() {
+        console.log(`parse.time_format: ${this.parse.time_format}`)
         this.preview()
       },
       'parseType': function() {
diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index 22d161297..676366888 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -82,8 +82,8 @@ const OwnedPluginForm = {
             this.unwatchExpression = this.$watch("expression", (newValue, oldValue) => {
               console.log(newValue)
               this.$emit("change-parse-config", {
-                expression: this.expression,
-                timeFormat: this.timeFormat
+                "expression": this.expression,
+                "time_format": this.timeFormat
               })
             })
           }
@@ -93,8 +93,8 @@ const OwnedPluginForm = {
             this.unwatchTimeFormat = this.$watch("timeFormat", (newValue, oldValue) => {
               console.log({"watch time_format": newValue})
               this.$emit("change-parse-config", {
-                expression: this.expression,
-                timeFormat: this.timeFormat
+                "expression": this.expression,
+                "time_format": this.timeFormat
               })
             })
           }

From 404f2929f80dcba0f8aedc67b0a6a683db45503b Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:01:19 +0900
Subject: [PATCH 102/137] Move parse plugin config under plugin_config

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api_controller.rb     | 2 +-
 app/javascript/packs/in_tail_parse.js | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index fc2e4d1fa..684718596 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -19,7 +19,7 @@ def empty_json
   end
 
   def regexp_preview
-    preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], params)
+    preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], params[:plugin_config])
 
     render json: preview.matches
   end
diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 02ef0c58b..6cfc91e09 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -26,7 +26,7 @@ $(document).ready(() => {
     },
     watch: {
       'parse.expression': function() {
-        console.log('parse.expression')
+        console.log(`parse.expression: ${this.parse.expression}`)
         this.preview()
       },
       'parse.time_format': function() {
@@ -53,7 +53,7 @@ $(document).ready(() => {
         this.parseType = name
       },
       onChangeParseConfig: function(data) {
-        console.log("onChangeParseConfig")
+        console.log(`onChangeParseConfig: ${data}`)
         _.merge(this.parse, data)
         this.preview()
       },
@@ -132,8 +132,6 @@ $(document).ready(() => {
             'X-CSRF-Token': this.token
           },
           data: {
-            expression: this.parse.expression,
-            time_format: this.parse.timeFormat,
             parse_type: _.isEmpty(this.parseType) ? "regexp" : this.parseType,
             file: this.path,
             plugin_config: this.parse

From d1781d9ce5502c9f786726830cd2b4083ff93bcf Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:07:29 +0900
Subject: [PATCH 103/137] Clear parse plugin configuration when parser plugin
 type is changed

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 6cfc91e09..ab86332cf 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -51,6 +51,7 @@ $(document).ready(() => {
       onChangePluginName: function(name) {
         console.log("onChangePluginName")
         this.parseType = name
+        this.parse = {} // clear parser plugin configuration
       },
       onChangeParseConfig: function(data) {
         console.log(`onChangeParseConfig: ${data}`)

From 39bbcf8449169c743f6d37141ef6f532d26be418 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:47:36 +0900
Subject: [PATCH 104/137] Remove unused methods

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/parser_multiline_form.js | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/app/javascript/packs/parser_multiline_form.js b/app/javascript/packs/parser_multiline_form.js
index 3d932b356..143afae23 100644
--- a/app/javascript/packs/parser_multiline_form.js
+++ b/app/javascript/packs/parser_multiline_form.js
@@ -19,12 +19,6 @@ const ParserMultilineForm = {
   },
 
   methods: {
-    hiddenId: function(pluginType, n) {
-      return `setting_${pluginType}_0__format${n}`
-    },
-    hiddenName: function(pluginType, n) {
-      return `setting[${pluginType}[0]][format${n}]`
-    },
     textareaId: function(pluginType) {
       return `setting_${pluginType}_0__formats`
     },

From 0e4f972872cd9065415caedfbe3e18b32b661a59 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:48:12 +0900
Subject: [PATCH 105/137] Add format_firstline

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/parser_multiline_form.js         | 1 +
 app/views/shared/vue/_parser_multiline_form.html.haml | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/app/javascript/packs/parser_multiline_form.js b/app/javascript/packs/parser_multiline_form.js
index 143afae23..72e10db1a 100644
--- a/app/javascript/packs/parser_multiline_form.js
+++ b/app/javascript/packs/parser_multiline_form.js
@@ -7,6 +7,7 @@ const ParserMultilineForm = {
 
   data: function() {
     return {
+      formatFirstline: "",
       formats: ""
     }
   },
diff --git a/app/views/shared/vue/_parser_multiline_form.html.haml b/app/views/shared/vue/_parser_multiline_form.html.haml
index 7c6427440..c8c34ebbf 100644
--- a/app/views/shared/vue/_parser_multiline_form.html.haml
+++ b/app/views/shared/vue/_parser_multiline_form.html.haml
@@ -1,5 +1,11 @@
 %script{type: "text/x-template", id: "vue-parser-multiline-form"}
   .form-group
+    %label{"for" => "setting_parse_0__format_firstline"}
+      Format first line
+    %input.form-control{"type" => "text",
+                        "id" => "setting_parse_0__format_firstline",
+                        "name" => "setting[parse[0]][format_firstline]",
+                        "v-model.lazy" => "formatFirstline"}
     %label.mt-3{"v-bind:for" => "textareaId(pluginType)"}
       Formats
     %textarea.form-control{"v-bind:id" => "textareaId(pluginType)",

From 10373a2ae9ce27cdbc4b106c2d406b3f38e4c23a Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:48:24 +0900
Subject: [PATCH 106/137] Add prepare_plugin_config

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api_controller.rb | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index 684718596..25164076c 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -19,7 +19,8 @@ def empty_json
   end
 
   def regexp_preview
-    preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], params[:plugin_config])
+    plugin_config = prepare_plugin_config
+    preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], plugin_config)
 
     render json: preview.matches
   end
@@ -29,4 +30,19 @@ def grok_to_regexp
     grok.load_patterns
     render text: grok.convert_to_regexp(params[:grok_str]).source
   end
+
+  private
+
+  def prepare_plugin_config
+    plugin_config = params[:plugin_config]
+    case params[:parse_type]
+    when "multiline"
+      plugin_config[:formats].lines.each.with_index do |line, index|
+        plugin_config["format#{index + 1}"] = line.chomp
+      end
+      plugin_config
+    else
+      plugin_config
+    end
+  end
 end

From b70cc1c0fbc4e1e708554cae4745542a5642e102 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:48:48 +0900
Subject: [PATCH 107/137] Set formats and format_firstline to plugin_config

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_parse.js         |  8 ++++----
 app/javascript/packs/owned_plugin_form.js     |  6 +++---
 app/javascript/packs/parser_multiline_form.js | 12 +++++++++++-
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index ab86332cf..60124bde6 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -54,13 +54,13 @@ $(document).ready(() => {
         this.parse = {} // clear parser plugin configuration
       },
       onChangeParseConfig: function(data) {
-        console.log(`onChangeParseConfig: ${data}`)
+        console.log("onChangeParseConfig", data)
         _.merge(this.parse, data)
         this.preview()
       },
-      onChangeFormats: function(formats) {
-        console.log(`in_tail_parse:onChangeFormats: ${formats}`)
-        this.parse.formats = formats
+      onChangeFormats: function(data) {
+        console.log("in_tail_parse:onChangeFormats", data)
+        _.merge(this.parse, data)
         this.preview()
       },
       updateHighlightedLines: function(matches) {
diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js
index 676366888..35b7f981a 100644
--- a/app/javascript/packs/owned_plugin_form.js
+++ b/app/javascript/packs/owned_plugin_form.js
@@ -55,9 +55,9 @@ const OwnedPluginForm = {
       }
     },
 
-    onChangeFormats: function(newValue, oldValue) {
-      console.log(`ownedPluginForm:onChangeFormats: ${newValue}`)
-      this.$emit("change-formats", newValue)
+    onChangeFormats: function(data) {
+      console.log("ownedPluginForm:onChangeFormats", data)
+      this.$emit("change-formats", data)
     },
 
     updateSection: function() {
diff --git a/app/javascript/packs/parser_multiline_form.js b/app/javascript/packs/parser_multiline_form.js
index 72e10db1a..6500595d0 100644
--- a/app/javascript/packs/parser_multiline_form.js
+++ b/app/javascript/packs/parser_multiline_form.js
@@ -13,9 +13,19 @@ const ParserMultilineForm = {
   },
 
   watch: {
+    "formatFirstLine": function(newValue, oldValue) {
+      console.log(`watch formatFirstLine: ${newValue}`)
+      this.$emit("change-formats", {
+        "format_firstline": this.formatFirstline,
+        "formats": this.formats
+      })
+    },
     "formats": function(newValue, oldValue) {
       console.log(`watch formats: ${newValue}`)
-      this.$emit("change-formats", newValue)
+      this.$emit("change-formats", {
+        "format_firstline": this.formatFirstline,
+        "formats": this.formats
+      })
     }
   },
 

From 8c708c0df488ba9c9ba7011fbaa2fd3d80a4b904 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 11:54:12 +0900
Subject: [PATCH 108/137] Remove unused code

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/in_tail_format.js        | 174 ------------------
 app/views/shared/vue/_in_tail_format.html.erb |  43 -----
 2 files changed, 217 deletions(-)
 delete mode 100644 app/javascript/packs/in_tail_format.js
 delete mode 100644 app/views/shared/vue/_in_tail_format.html.erb

diff --git a/app/javascript/packs/in_tail_format.js b/app/javascript/packs/in_tail_format.js
deleted file mode 100644
index 2ba758db0..000000000
--- a/app/javascript/packs/in_tail_format.js
+++ /dev/null
@@ -1,174 +0,0 @@
-'use strict'
-import 'lodash/lodash'
-
-window.addEventListener('load', () => {
-  const maxFormatCount = 20;
-
-  new Vue({
-    el: "#in_tail_format",
-    props: ["formatOptionsJson", "initialSelected", "targetFile", "paramsJson"],
-    data: {
-      previewProcessing: false,
-      format: "",
-      highlightedLines: null,
-    },
-
-    computed: {
-      options: {
-        get: function(){
-          return this.formatOptions[this.format];
-        },
-      },
-      selectableFormats: {
-        get: function() {
-          return Object.keys(this.formatOptions);
-        }
-      },
-      useTextArea: function() {
-        return this.format === "multiline";
-      }
-    },
-
-    beforeMount: function() {
-      this.formatOptions = JSON.parse(this.$el.attributes.formatOptionsJson.nodeValue);
-      this.format = this.$el.attributes.initialSelected.nodeValue;
-
-      // initialize params
-      // NOTE: if `params.setting.foo` is undefined, Vue can't binding with v-model="params.setting.foo"
-      var params = JSON.parse(this.$el.attributes.paramsJson.nodeValue);
-      if(!params.setting) {
-        params.setting = {};
-      }
-
-      var formats = _.chain(_.range(1, maxFormatCount + 1)).map(function(i) {return params.setting["format" + String(i)];}).compact().value();
-      params.setting.formats = formats.join("\n");
-
-      _.each(this.formatOptions, function(options){
-        _.each(options, function(key){
-          if(!params.setting.hasOwnProperty(key)){
-            params.setting[key] = "";
-          }
-        });
-      });
-      this.params = params;
-
-      this.targetFile = this.$el.attributes.targetFile.nodeValue;
-    },
-    mounted: function(){
-      this.$emit("data-loaded");
-    },
-
-    watch: {
-      'params.setting.formats': function(formats) {
-        _.range(1, maxFormatCount + 1).forEach(()=> {params.setting["format" + String(i)] = "";});
-
-        _.compact(formats.split("\n")).forEach((formatLine, index)=> {
-          params.setting["format" + String(index + 1)] = formatLine;
-        })
-      },
-      'params.setting.regexp': function() {
-        this.preview();
-      },
-      'format': function() {
-        this.preview();
-      },
-    },
-
-    methods: {
-      onKeyup: function(ev){
-        var el = ev.target;
-        if(el.name.match(/\[format/)){
-          this.preview();
-        }
-      },
-      updateHighlightedLines: function() {
-        if(!this.regexpMatches) {
-          this.highlightedLines = null;
-          return;
-        }
-
-        var $container = jQuery('<div>');
-        _.each(this.regexpMatches, function(match){
-          var colors = [
-            "#ff9", "#cff", "#fcf", "#dfd"
-          ];
-          var whole = match.whole;
-          var html = "";
-          var matches = [];
-
-          var lastPos = 0;
-          _.each(match.matches, function(match) {
-            var matched = match.matched;
-            if(!matched) return;
-            if(matched.length === 0) return; // Ignore empty matched with "foobar".match(/foo(.*?)bar/)[1] #=> ""
-
-            // rotated highlight color
-            var currentColor = colors.shift();
-            colors.push(currentColor);
-
-            // create highlighted range HTML
-            var $highlighted = jQuery('<span>').text(matched);
-            $highlighted.attr({
-              "class": "regexp-preview",
-              "data-toggle": "tooltip",
-              "data-placement": "top",
-              "title": match.key,
-              'style': 'background-color:' + currentColor
-            });
-            var highlightedHtml = $highlighted.wrap('<div>').parent().html();
-
-            var pos = {
-              "start": match.pos[0],
-              "end": match.pos[1]
-            };
-            if(pos.start > 0) {
-              html += _.escape(whole.substring(lastPos, pos.start));
-            }
-            html += highlightedHtml;
-            lastPos = pos.end;
-          });
-          html += whole.substring(lastPos);
-
-          $container.append(html);
-          $container.append("<br />");
-        });
-
-        this.highlightedLines = $container.html();
-      },
-
-      preview: function(){
-        if(this.previewAjax) {
-          this.previewAjax.abort();
-        }
-
-        const token = document.getElementsByName("csrf-token")[0].getAttribute('content')
-
-        var self = this;
-        new Promise(function(resolve, reject) {
-          self.previewAjax = $.ajax({
-            method: "POST",
-            url: "/api/regexp_preview",
-            headers: {
-              'X-CSRF-Token': token
-            },
-            data: {
-              regexp: self.params.setting.regexp,
-              time_format: self.params.setting.time_format,
-              format: _.isEmpty(self.format) ? "regexp" : self.format,
-              params: self.params.setting,
-              file: self.targetFile
-            }
-          }).done(resolve).fail(reject);
-        }).then(function(result){
-          self.params = _.merge(self.params, result.params);
-          self.regexpMatches = result.matches;
-          self.updateHighlightedLines();
-        })["catch"](function(error){
-          if(error.stack) {
-            console.error(error.stack);
-          }
-        });
-      },
-    }
-  });
-});
diff --git a/app/views/shared/vue/_in_tail_format.html.erb b/app/views/shared/vue/_in_tail_format.html.erb
deleted file mode 100644
index bf8b6fed9..000000000
--- a/app/views/shared/vue/_in_tail_format.html.erb
+++ /dev/null
@@ -1,43 +0,0 @@
-<%- add_javascript_pack_tag("in_tail_format") %>
-
-<div id="in_tail_format"
-  formatOptionsJson="<%= formats.to_json %>"
-  initialSelected="<%= initialSelected %>"
-  targetFile="<%= file %>"
-  paramsJson="<%= params.to_json %>"
-  >
-
-  <div class="form-inline form-group">
-    <label for="in_tail_setting_format">format</label>
-    <select id="in_tail_setting_format" name="setting[format]" v-model="format" class="form-control">
-      <option v-for="selectableFormat in selectableFormats" v-bind:value="selectableFormat" v-bind:selected="format==selectableFormat" v-if="selectableFormats">{{ selectableFormat }}</option>
-    </select>
-  </div>
-  <div class="form-inline form-group" v-for="option in options">
-    <label v-bind:for="'in_tail_setting_' + option" v-if="!useTextArea">{{ option }} </label>
-    <input v-bind:id="'in_tail_setting_' + option" v-bind:type="useTextArea ? 'hidden' : 'text'" v-bind:name="'setting[' + option + ']'" v-model="params.setting[option]" v-on:keyup="onKeyup" size="100%" class="form-control" />
-  </div>
-
-  <div v-if="useTextArea">
-    <div class="form-inline form-group">
-      <label for="in_tail_setting_format_firstline">format_firstline</label>
-      <input id="in_tail_setting_format_firstline" type="text" name="setting[format_firstline]" v-model="params.setting['format_firstline']" v-on:keyup="onKeyup" size="100%" class="form-control" />
-    </div>
-
-    <div class="form-group">
-      <p class="alert alert-warning"><%= t("fluentd.settings.in_tail.notice_for_multiline_limit") %></p>
-      <label for="in_tail_setting_formats">formats</label>
-      <textarea id="in_tail_setting_formats" type="text" name="setting[formats]" v-model="params.setting['formats']" v-on:keyup="onKeyup" rows='20' size='100%' class="form-control"></textarea>
-    </div>
-  </div>
-
-  <div class="card mb-3">
-    <pre class="card-body" v-html="highlightedLines"></pre>
-  </div>
-
-  <div class="card mb-3">
-    <p class="card-body pb-0">
-      <%= raw t('fluentd.settings.in_tail_option_guide') %>
-    </p>
-  </div>
-</div>

From 1cd2282212a0c2f36984637a908417e791902bac Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 12:36:45 +0900
Subject: [PATCH 109/137] Set empty hash if params[:plugin_config] is null

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index 25164076c..24ea26522 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -19,7 +19,7 @@ def empty_json
   end
 
   def regexp_preview
-    plugin_config = prepare_plugin_config
+    plugin_config = prepare_plugin_config || {}
     preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], plugin_config)
 
     render json: preview.matches

From fe32983c0fd13a21b4387c281daadd0602e8925c Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:19:51 +0900
Subject: [PATCH 110/137] Remove unused methods

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_tail.rb | 37 ---------------------------
 1 file changed, 37 deletions(-)

diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb
index 855f2228e..89de568d0 100644
--- a/app/models/fluentd/setting/in_tail.rb
+++ b/app/models/fluentd/setting/in_tail.rb
@@ -31,43 +31,6 @@ def guess_parse_type
         end
       end
 
-      def extra_format_options
-        self.class.known_formats[format.to_sym] || []
-      end
-
-      def format_specific_conf
-        return "" if %w(grok regexp).include?(format)
-
-        indent = " " * 2
-        format_specific_conf = ""
-
-        if format.to_sym == :multiline
-          known_formats[:multiline].each do |key|
-            value = send(key)
-            if value.present?
-              format_specific_conf << "#{indent}#{key} /#{value}/\n"
-            end
-          end
-        else
-          extra_format_options.each do |key|
-            format_specific_conf << "#{indent}#{key} #{send(key)}\n"
-          end
-        end
-
-        format_specific_conf
-      end
-
-      def certain_format_line
-        case format
-        when "grok"
-          "format /#{grok.convert_to_regexp(grok_str).source.gsub("/", "\\/")}/ # grok: '#{grok_str}'"
-        when "regexp"
-          "format /#{regexp}/"
-        else
-          "format #{format}"
-        end
-      end
-
       def grok
         @grok ||=
           begin

From c55038ed170b207910296643cc151eece1358892 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:20:05 +0900
Subject: [PATCH 111/137] Rescue Fluent::ConfigError

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/api_controller.rb     | 2 ++
 app/javascript/packs/in_tail_parse.js | 7 ++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb
index 24ea26522..11445a0b5 100644
--- a/app/controllers/api_controller.rb
+++ b/app/controllers/api_controller.rb
@@ -23,6 +23,8 @@ def regexp_preview
     preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], plugin_config)
 
     render json: preview.matches
+  rescue Fluent::ConfigError => ex
+    render json: { error: "#{ex.class}: #{ex.message}" }
   end
 
   def grok_to_regexp
diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js
index 60124bde6..9cf205a06 100644
--- a/app/javascript/packs/in_tail_parse.js
+++ b/app/javascript/packs/in_tail_parse.js
@@ -139,7 +139,12 @@ $(document).ready(() => {
           }
         }).then(
           (result) => {
-            this.updateHighlightedLines(result.matches)
+            if (result.matches) {
+              this.updateHighlightedLines(result.matches)
+            } else {
+              console.error(result.error)
+              this.previewError = result.error
+            }
           },
           (error) => {
             this.highlightedLines = null

From 600d93461d84173110193e10f6ac0267f86dd6e1 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:20:53 +0900
Subject: [PATCH 112/137] Update setting_params to apply strong parameter

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/fluentd/settings/in_tail_controller.rb | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/controllers/fluentd/settings/in_tail_controller.rb b/app/controllers/fluentd/settings/in_tail_controller.rb
index 38b660fd5..d7615d438 100644
--- a/app/controllers/fluentd/settings/in_tail_controller.rb
+++ b/app/controllers/fluentd/settings/in_tail_controller.rb
@@ -48,10 +48,18 @@ def finish
     redirect_to daemon_setting_path(@fluentd)
   end
 
+  def target_class
+    Fluentd::Setting::InTail
+  end
+
   private
 
   def setting_params
-    params.require(:setting).permit(:path, :format, :regexp, *Fluentd::Setting::InTail.known_formats, :tag, :rotate_wait, :pos_file, :read_from_head, :refresh_interval)
+    permit_params = target_class._types.keys
+    permit_params << :parse_type
+    section_class = Fluentd::Setting.const_get("parser_#{params.dig(:setting, :parse_type)}".classify)
+    permit_params << { parse: section_class._types.keys }
+    params.require(:setting).permit(*permit_params)
   end
 
 end

From 908c4efebe05e2823ac226e4b727b2bf0f9b5137 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:24:39 +0900
Subject: [PATCH 113/137] Use local variables instead of instance variables

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../fluentd/settings/in_tail/_form.html.haml      | 15 ++++-----------
 .../settings/in_tail/after_format.html.haml       |  2 +-
 2 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index d93184e79..93f4c31ca 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -1,4 +1,4 @@
-- @setting.errors.full_messages.each do |e|
+- setting.errors.full_messages.each do |e|
   .alert.alert-danger= e
 
 .form-group
@@ -9,16 +9,9 @@
   = f.label :parse_type
   = f.hidden_field :parse_type
   = f.text_field :parse_type, class: "form-control", disabled: true
-- if @setting.known_formats[@setting.parse_type&.to_sym || "syslog"]
-  - @setting.known_formats[@setting.parse_type&.to_sym || "syslog"].each do |key|
-    %label= key
-    = f.hidden_field key
-    = @setting.send(key)
-    %br
-- else
-  %label= @setting.parse_type
-  = f.hidden_field :regexp
-  = @setting.regexp
+%label= setting.parse_type
+= f.hidden_field :regexp
+= setting.regexp
 .form-group
   = f.label :tag
   = f.text_field :tag, class: "form-control"
diff --git a/app/views/fluentd/settings/in_tail/after_format.html.haml b/app/views/fluentd/settings/in_tail/after_format.html.haml
index 4c521ff12..13ee2dcd4 100644
--- a/app/views/fluentd/settings/in_tail/after_format.html.haml
+++ b/app/views/fluentd/settings/in_tail/after_format.html.haml
@@ -3,7 +3,7 @@
 = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentds)
 
 = form_for(@setting, as: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd)) do |f|
-  = render partial: "form", locals: { f: f }
+  = render partial: "form", locals: { f: f, setting: @setting }
 
   %p
     = f.submit t('terms.next'), class: "btn btn-lg btn-primary float-right"

From 8962784d4bff41e98c3e728625f88b1bd0675c65 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:37:43 +0900
Subject: [PATCH 114/137] Use form_with instead of form_for

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/after_format.html.haml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/fluentd/settings/in_tail/after_format.html.haml b/app/views/fluentd/settings/in_tail/after_format.html.haml
index 13ee2dcd4..fc42864d5 100644
--- a/app/views/fluentd/settings/in_tail/after_format.html.haml
+++ b/app/views/fluentd/settings/in_tail/after_format.html.haml
@@ -2,7 +2,7 @@
 
 = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentds)
 
-= form_for(@setting, as: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd)) do |f|
+= form_with(model: @setting, scope: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd), builder: FluentdFormBuilder) do |f|
   = render partial: "form", locals: { f: f, setting: @setting }
 
   %p

From 193fe57a22c47459082335c8972d09886cbaa185 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:38:18 +0900
Subject: [PATCH 115/137] Use FluentdFormBuilder's method

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../fluentd/settings/in_tail/_form.html.haml  | 22 ++++++++-----------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index 93f4c31ca..24db245fe 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -9,26 +9,22 @@
   = f.label :parse_type
   = f.hidden_field :parse_type
   = f.text_field :parse_type, class: "form-control", disabled: true
-%label= setting.parse_type
-= f.hidden_field :regexp
-= setting.regexp
+
+.form-group
+  %label= setting.parse_type
+
 .form-group
-  = f.label :tag
-  = f.text_field :tag, class: "form-control"
+  = f.field(:tag)
 .form-group
-  = f.label :pos_file
-  = f.text_field :pos_file, class: "form-control"
+  = f.field(:pos_file)
 .well.well-sm
   %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
     = icon('fa-caret-down')
     = t('terms.advanced_setting')
   #advanced-setting.collapse
     .form-group
-      = f.label :rotate_wait
-      = f.text_field :rotate_wait, class: "form-control"
+      = f.field(:rotate_wait)
     .form-group
-      = f.check_box :read_from_head
-      = f.label :read_from_head
+      = f.field(:read_from_head)
     .form-group
-      = f.label :refresh_interval
-      = f.text_field :refresh_interval, class: "form-control"
+      = f.field(:refresh_interval)

From 51e76524d9275aec9bf188f6613e233c1a053227 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 14:52:20 +0900
Subject: [PATCH 116/137] Rename `f` to `form`

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 .../fluentd/settings/in_tail/_form.html.haml  | 22 +++++++++----------
 .../settings/in_tail/after_format.html.haml   |  8 +++----
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index 24db245fe..ff8e60b88 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -2,29 +2,29 @@
   .alert.alert-danger= e
 
 .form-group
-  = f.label :path
-  = f.hidden_field :path
-  = f.text_field :path, class: "form-control", disabled: true
+  = form.label :path
+  = form.hidden_field :path
+  = form.text_field :path, class: "form-control", disabled: true
 .form-group
-  = f.label :parse_type
-  = f.hidden_field :parse_type
-  = f.text_field :parse_type, class: "form-control", disabled: true
+  = form.label :parse_type
+  = form.hidden_field :parse_type
+  = form.text_field :parse_type, class: "form-control", disabled: true
 
 .form-group
   %label= setting.parse_type
 
 .form-group
-  = f.field(:tag)
+  = form.field(:tag)
 .form-group
-  = f.field(:pos_file)
+  = form.field(:pos_file)
 .well.well-sm
   %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
     = icon('fa-caret-down')
     = t('terms.advanced_setting')
   #advanced-setting.collapse
     .form-group
-      = f.field(:rotate_wait)
+      = form.field(:rotate_wait)
     .form-group
-      = f.field(:read_from_head)
+      = form.field(:read_from_head)
     .form-group
-      = f.field(:refresh_interval)
+      = form.field(:refresh_interval)
diff --git a/app/views/fluentd/settings/in_tail/after_format.html.haml b/app/views/fluentd/settings/in_tail/after_format.html.haml
index fc42864d5..fa2194ecd 100644
--- a/app/views/fluentd/settings/in_tail/after_format.html.haml
+++ b/app/views/fluentd/settings/in_tail/after_format.html.haml
@@ -2,9 +2,9 @@
 
 = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentds)
 
-= form_with(model: @setting, scope: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd), builder: FluentdFormBuilder) do |f|
-  = render partial: "form", locals: { f: f, setting: @setting }
+= form_with(model: @setting, scope: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd), builder: FluentdFormBuilder) do |form|
+  = render partial: "form", locals: { form: form, setting: @setting }
 
   %p
-    = f.submit t('terms.next'), class: "btn btn-lg btn-primary float-right"
-    = f.submit t('terms.prev'), class: "btn btn-lg btn-secondary", name: "back"
+    = form.submit t('terms.next'), class: "btn btn-lg btn-primary float-right"
+    = form.submit t('terms.prev'), class: "btn btn-lg btn-secondary", name: "back"

From e0557c5697fdea0293ada0249f02d53b4f6d29d3 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 15:00:38 +0900
Subject: [PATCH 117/137] Support options

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/form_builders/fluentd_form_builder.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/form_builders/fluentd_form_builder.rb b/app/form_builders/fluentd_form_builder.rb
index d05f1f178..cbec0e540 100644
--- a/app/form_builders/fluentd_form_builder.rb
+++ b/app/form_builders/fluentd_form_builder.rb
@@ -34,14 +34,14 @@ def enum_field(key, options)
   end
 
   def bool_field(key, options)
-    check_box(key, {}, "true", "false") + " " +
+    check_box(key, options, "true", "false") + " " +
       label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key))
   end
 
   def other_field(key, options)
     return unless object.respond_to?(key)
     label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) +
-      text_field(key, class: "form-control")
+      text_field(key, class: "form-control", **options)
   end
 
   def render_section(key, options)

From 90a88ec5d1102731d9422e03304bd1f80acbd345 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 15:16:46 +0900
Subject: [PATCH 118/137] Display parser common options

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/_form.html.haml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index ff8e60b88..5b72ab8d8 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -12,6 +12,10 @@
 
 .form-group
   %label= setting.parse_type
+  - parse_config = setting.create_parser
+  = form.fields("parse[0]", model: parse_config, builder: FluentdFormBuilder) do |f|
+    - f.object.common_options.each do |key|
+      = f.field(key, disabled: true)
 
 .form-group
   = form.field(:tag)

From c8fde4618bc1b83a5fd792f4d3863defd799419d Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:18:54 +0900
Subject: [PATCH 119/137] Indent

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/_form.html.haml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index 5b72ab8d8..67f182714 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -17,10 +17,9 @@
     - f.object.common_options.each do |key|
       = f.field(key, disabled: true)
 
-.form-group
-  = form.field(:tag)
-.form-group
-  = form.field(:pos_file)
+= form.field(:tag)
+= form.field(:pos_file)
+
 .well.well-sm
   %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
     = icon('fa-caret-down')

From dc44aaf952181a20b30ac990b9f694d4e4eb7395 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:19:03 +0900
Subject: [PATCH 120/137] Add missing hidden field

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/_form.html.haml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index 67f182714..78ed46571 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -15,6 +15,7 @@
   - parse_config = setting.create_parser
   = form.fields("parse[0]", model: parse_config, builder: FluentdFormBuilder) do |f|
     - f.object.common_options.each do |key|
+      = f.hidden_field(key)
       = f.field(key, disabled: true)
 
 = form.field(:tag)

From f8d1a5c379b4c7f1bd80f23017bf1b17aa94112e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:20:18 +0900
Subject: [PATCH 121/137] Use to_config instead of to_conf

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/fluentd/settings/in_tail_controller.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/controllers/fluentd/settings/in_tail_controller.rb b/app/controllers/fluentd/settings/in_tail_controller.rb
index d7615d438..0da439b6e 100644
--- a/app/controllers/fluentd/settings/in_tail_controller.rb
+++ b/app/controllers/fluentd/settings/in_tail_controller.rb
@@ -38,12 +38,12 @@ def finish
       return render "after_format"
     end
 
-    if @fluentd.agent.configuration.to_s.include?(@setting.to_conf.strip)
+    if @fluentd.agent.configuration.to_s.include?(@setting.to_config.to_s.strip)
       @setting.errors.add(:base, :duplicated_conf)
       return render "after_format"
     end
 
-    @fluentd.agent.config_append @setting.to_conf
+    @fluentd.agent.config_append @setting.to_config.to_s
     @fluentd.agent.restart if @fluentd.agent.running?
     redirect_to daemon_setting_path(@fluentd)
   end

From 6c36d5e81867a9d46e8f781852f0ab40057be90f Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:20:40 +0900
Subject: [PATCH 122/137] Use POST instead of XHR

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/after_format.html.haml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/fluentd/settings/in_tail/after_format.html.haml b/app/views/fluentd/settings/in_tail/after_format.html.haml
index fa2194ecd..ed3ad219a 100644
--- a/app/views/fluentd/settings/in_tail/after_format.html.haml
+++ b/app/views/fluentd/settings/in_tail/after_format.html.haml
@@ -2,7 +2,7 @@
 
 = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentds)
 
-= form_with(model: @setting, scope: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd), builder: FluentdFormBuilder) do |form|
+= form_with(model: @setting, scope: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd), local: true, builder: FluentdFormBuilder) do |form|
   = render partial: "form", locals: { form: form, setting: @setting }
 
   %p

From 8644549fce5afb3cc9272aa2d7125fdee9f47139 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:21:10 +0900
Subject: [PATCH 123/137] Apply changes same as after_format

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/confirm.html.haml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/views/fluentd/settings/in_tail/confirm.html.haml b/app/views/fluentd/settings/in_tail/confirm.html.haml
index ae5aa1688..cf0f7719e 100644
--- a/app/views/fluentd/settings/in_tail/confirm.html.haml
+++ b/app/views/fluentd/settings/in_tail/confirm.html.haml
@@ -2,12 +2,12 @@
 
 = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentd)
 
-= form_for(@setting, as: "setting", url: finish_daemon_setting_in_tail_path(@fluentd)) do |f|
-  = render partial: "form", locals: { f: f }
+= form_with(model: @setting, scope: "setting", url: finish_daemon_setting_in_tail_path(@fluentd), local: true, builder: FluentdFormBuilder) do |form|
+  = render partial: "form", locals: { form: form, setting: @setting }
 
   %pre= @setting.to_config
 
   %p
-    = f.submit t('fluentd.common.finish')   , class: "btn btn-lg btn-primary float-right"
-    = f.submit t('terms.prev'),   class: "btn btn-lg btn-secondary", name: "back"
+    = form.submit t('fluentd.common.finish')   , class: "btn btn-lg btn-primary float-right"
+    = form.submit t('terms.prev'),   class: "btn btn-lg btn-secondary", name: "back"
     .clearfix.float-right= t('terms.notice_restart_for_config_edit', brand: fluentd_ui_brand)

From 2f2b388d577beb847afdb720cd1dbf92220dd269 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:31:45 +0900
Subject: [PATCH 124/137] Support storage

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_config.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb
index dc99575f2..59ffadd7b 100644
--- a/app/models/concerns/fluentd/setting/plugin_config.rb
+++ b/app/models/concerns/fluentd/setting/plugin_config.rb
@@ -17,6 +17,8 @@ def to_config
                  "format"
                when "buffer"
                  "buffer"
+               when "storage"
+                 "storage"
                end
         _attributes = { "@type" => self.plugin_name }.merge(attributes)
         _attributes["@log_level"] = _attributes.delete("log_level")

From 215da19bdab1440474baff72b71b0f398cdf7166 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:32:01 +0900
Subject: [PATCH 125/137] Remove parse_type like attributes from owner plugin
 configuration

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_config.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb
index 59ffadd7b..453a4a357 100644
--- a/app/models/concerns/fluentd/setting/plugin_config.rb
+++ b/app/models/concerns/fluentd/setting/plugin_config.rb
@@ -20,7 +20,10 @@ def to_config
                when "storage"
                  "storage"
                end
-        _attributes = { "@type" => self.plugin_name }.merge(attributes)
+        _attributes = attributes.reject do |key, value|
+          %w(parse_type format_type buffer_type storage_type).include?(key.to_s)
+        end
+        _attributes = { "@type" => self.plugin_name }.merge(_attributes)
         _attributes["@log_level"] = _attributes.delete("log_level")
         argument = case plugin_type
                    when "output", "filter", "buffer"

From f1d32695a44b0ed5ecc601ce5557109c8b504ed3 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:32:36 +0900
Subject: [PATCH 126/137] Dump `@type` attribute of owned plugins

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_config.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb
index 453a4a357..c2814bb11 100644
--- a/app/models/concerns/fluentd/setting/plugin_config.rb
+++ b/app/models/concerns/fluentd/setting/plugin_config.rb
@@ -44,6 +44,9 @@ def parse_attributes(attributes)
         end
         elements = []
         sections.to_h.each do |key, section_params|
+          if %w(parse format buffer storage).include?(key)
+            section_params["0"] = { "@type" => self.attributes["#{key}_type"] }.merge(section_params["0"])
+          end
           next if section_params.blank?
           section_params.each do |index, _section_params|
             sub_attrs, sub_elements = parse_attributes(_section_params)

From 41b18ffba112f7bcb49e34a98265ef542d4d188e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 16:42:15 +0900
Subject: [PATCH 127/137] Add hidden field for owned plugin type

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_tail/_form.html.haml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml
index 78ed46571..cd789ee8a 100644
--- a/app/views/fluentd/settings/in_tail/_form.html.haml
+++ b/app/views/fluentd/settings/in_tail/_form.html.haml
@@ -13,6 +13,7 @@
 .form-group
   %label= setting.parse_type
   - parse_config = setting.create_parser
+  = hidden_field_tag("setting[parse[0]][type]", form.object.parse_type)
   = form.fields("parse[0]", model: parse_config, builder: FluentdFormBuilder) do |f|
     - f.object.common_options.each do |key|
       = f.hidden_field(key)

From fce12c5cb5f8d5d7731c2c6da69021c0047b3fed Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Fri, 15 Jun 2018 17:38:41 +0900
Subject: [PATCH 128/137] Display tooltip and guide for parser_multiline

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/javascript/packs/parser_multiline_form.js       | 13 +++++++++++--
 app/views/shared/vue/_owned_plugin_form.html.haml   |  1 +
 .../shared/vue/_parser_multiline_form.html.haml     |  7 ++++++-
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/app/javascript/packs/parser_multiline_form.js b/app/javascript/packs/parser_multiline_form.js
index 6500595d0..e80f0a703 100644
--- a/app/javascript/packs/parser_multiline_form.js
+++ b/app/javascript/packs/parser_multiline_form.js
@@ -1,14 +1,17 @@
 'use strict'
+import 'lodash/lodash'
 const ParserMultilineForm = {
   template: "#vue-parser-multiline-form",
   props: [
-    "pluginType"
+    "pluginType",
+    "commonOptions"
   ],
 
   data: function() {
     return {
       formatFirstline: "",
-      formats: ""
+      formats: "",
+      formatFirstlineDesc: ""
     }
   },
 
@@ -26,6 +29,12 @@ const ParserMultilineForm = {
         "format_firstline": this.formatFirstline,
         "formats": this.formats
       })
+    },
+    "commonOptions": function(newValue, oldValue) {
+      const option = _.find(newValue, (o) => {
+        return o.name === "format_firstline"
+      })
+      this.formatFirstlineDesc = option.desc
     }
   },
 
diff --git a/app/views/shared/vue/_owned_plugin_form.html.haml b/app/views/shared/vue/_owned_plugin_form.html.haml
index 82372f98a..537f61bb5 100644
--- a/app/views/shared/vue/_owned_plugin_form.html.haml
+++ b/app/views/shared/vue/_owned_plugin_form.html.haml
@@ -16,6 +16,7 @@
           {{ option }}
       %parser-multiline-form{"v-if" => 'pluginName==="multiline"',
                              "v-bind:plugin-type" => "pluginType",
+                             "v-bind:common-options" => "commonOptions",
                              "v-on:change-formats" => "onChangeFormats"}
       .form-group(v-else){"v-for" => "option in commonOptions"}
         %template{"v-if" => 'option.type==="enum"'}
diff --git a/app/views/shared/vue/_parser_multiline_form.html.haml b/app/views/shared/vue/_parser_multiline_form.html.haml
index c8c34ebbf..c7e12f392 100644
--- a/app/views/shared/vue/_parser_multiline_form.html.haml
+++ b/app/views/shared/vue/_parser_multiline_form.html.haml
@@ -1,6 +1,9 @@
 %script{type: "text/x-template", id: "vue-parser-multiline-form"}
   .form-group
-    %label{"for" => "setting_parse_0__format_firstline"}
+    %label{"for" => "setting_parse_0__format_firstline",
+           "data-toggle" => "tooltip",
+           "data-placement" => "right",
+           "v-bind:title" => "formatFirstlineDesc"}
       Format first line
     %input.form-control{"type" => "text",
                         "id" => "setting_parse_0__format_firstline",
@@ -8,6 +11,8 @@
                         "v-model.lazy" => "formatFirstline"}
     %label.mt-3{"v-bind:for" => "textareaId(pluginType)"}
       Formats
+    %p.alert.alert-warning
+      = t("fluentd.settings.in_tail.notice_for_multiline_limit")
     %textarea.form-control{"v-bind:id" => "textareaId(pluginType)",
                            "v-bind:name" => "textareaName(pluginType)",
                            "v-model.lazy" => "formats",

From e7666771738f54cb0acdb84caa93f75246de2040 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 11:22:34 +0900
Subject: [PATCH 129/137] Use FluentdFormBuilder

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_forward/_form.html.haml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/views/fluentd/settings/in_forward/_form.html.haml b/app/views/fluentd/settings/in_forward/_form.html.haml
index 8f4338455..a51f079d7 100644
--- a/app/views/fluentd/settings/in_forward/_form.html.haml
+++ b/app/views/fluentd/settings/in_forward/_form.html.haml
@@ -1,15 +1,16 @@
 = render "shared/setting_errors"
 
 - # NOTE: plugin_setting_form_action_url is defined at SettingConcern
-= form_for(@setting, url: plugin_setting_form_action_url(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
+= form_with(model: @setting, scope: "setting", url: plugin_setting_form_action_url(@fluentd), local: true, class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |f|
   - @setting.common_options.each do |key|
-    = field(f, key)
+    = f.field(key)
 
   .well.well-sm
     %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
       = icon('fa-caret-down')
       = t('terms.advanced_setting')
     #advanced-setting.collapse
-      TODO
+      - @setting.advanced_options.each do |key|
+        = f.field(key)
 
   = f.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary pull-right"

From d58640f6eececc7b306c77f9aa4634c32eae67e5 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 11:22:52 +0900
Subject: [PATCH 130/137] Drop TLS support for now

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_forward.rb | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/app/models/fluentd/setting/in_forward.rb b/app/models/fluentd/setting/in_forward.rb
index c134bbad3..40bf949d4 100644
--- a/app/models/fluentd/setting/in_forward.rb
+++ b/app/models/fluentd/setting/in_forward.rb
@@ -20,6 +20,13 @@ def common_options
           :bind, :port
         ]
       end
+
+      def hidden_options
+        [
+          # We don't support TLS configuration via fluentd-ui for now.
+          :transport
+        ]
+      end
     end
   end
 end

From 15530c223c25a627317e06f42def9a3ab26bc09e Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 11:26:35 +0900
Subject: [PATCH 131/137] Handle nested section properly

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/concerns/fluentd/setting/plugin_parameter.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb
index 355b6eaa6..62367fd4c 100644
--- a/app/models/concerns/fluentd/setting/plugin_parameter.rb
+++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb
@@ -111,7 +111,7 @@ def permit_params
         def _permit_section(key, section)
           keys = { key => section._types.keys }
           section._sections.each do |_key, _section|
-            keys << _permit_section(_key, _section)
+            keys[key] << _permit_section(_key, _section)
           end
           keys
         end

From 2b577a1fe93b29ff7f96d3ef26615c192d9ed9a7 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 11:59:15 +0900
Subject: [PATCH 132/137] Hide hidden options

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/in_forward.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/models/fluentd/setting/in_forward.rb b/app/models/fluentd/setting/in_forward.rb
index 40bf949d4..db1e5e3fb 100644
--- a/app/models/fluentd/setting/in_forward.rb
+++ b/app/models/fluentd/setting/in_forward.rb
@@ -24,7 +24,9 @@ def common_options
       def hidden_options
         [
           # We don't support TLS configuration via fluentd-ui for now.
-          :transport
+          :transport,
+          :backlog,
+          :blocking_timeout,
         ]
       end
     end

From 05f89523d14052833e7d056937a7c40c320b1cb8 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 14:36:30 +0900
Subject: [PATCH 133/137] Simplify markup

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/shared/settings/_form.html.haml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/app/views/shared/settings/_form.html.haml b/app/views/shared/settings/_form.html.haml
index b3c6a6cb7..99c3dde0e 100644
--- a/app/views/shared/settings/_form.html.haml
+++ b/app/views/shared/settings/_form.html.haml
@@ -38,13 +38,12 @@
                          "v-bind:plugin-label" => "'Format'"}
 
     - if @setting.advanced_options.present?
-      .card.bg-light
-        .card-body
-          %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
-            = icon('fa-caret-down')
-            = t('terms.advanced_setting')
-          #advanced-setting.collapse
-            - @setting.advanced_options.each do |key|
-              = form.field(key)
+      .card.card-body.bg-light
+        %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
+          = icon('fa-caret-down')
+          = t('terms.advanced_setting')
+        #advanced-setting.collapse
+          - @setting.advanced_options.each do |key|
+            = form.field(key)
 
     = form.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary float-right mt-3"

From 085226e98424ef1b04100c24add42b8ad04228f1 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 14:36:46 +0900
Subject: [PATCH 134/137] Use card

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/views/fluentd/settings/in_forward/_form.html.haml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/views/fluentd/settings/in_forward/_form.html.haml b/app/views/fluentd/settings/in_forward/_form.html.haml
index a51f079d7..6ea374144 100644
--- a/app/views/fluentd/settings/in_forward/_form.html.haml
+++ b/app/views/fluentd/settings/in_forward/_form.html.haml
@@ -5,7 +5,7 @@
   - @setting.common_options.each do |key|
     = f.field(key)
 
-  .well.well-sm
+  .card.card-body.bg-light
     %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
       = icon('fa-caret-down')
       = t('terms.advanced_setting')

From 1145123296da05d4fd8fdd98fe75d4683c9b1270 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 14:37:16 +0900
Subject: [PATCH 135/137] Remove unnecessary code

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/controllers/fluentd/settings/out_tdlog_controller.rb | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/app/controllers/fluentd/settings/out_tdlog_controller.rb b/app/controllers/fluentd/settings/out_tdlog_controller.rb
index 117418b9e..dac9b14f0 100644
--- a/app/controllers/fluentd/settings/out_tdlog_controller.rb
+++ b/app/controllers/fluentd/settings/out_tdlog_controller.rb
@@ -6,13 +6,4 @@ class Fluentd::Settings::OutTdlogController < ApplicationController
   def target_class
     Fluentd::Setting::OutTdlog
   end
-
-  def initial_params
-    {
-      buffer_type: "file",
-      buffer_path: "/var/log/td-agent/buffer/td",
-      auto_create_table: true,
-      match: "td.*.*",
-    }
-  end
 end

From cac395695b999843c58958678b70e155c4937896 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 14:37:46 +0900
Subject: [PATCH 136/137] Add icon method

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/form_builders/fluentd_form_builder.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/form_builders/fluentd_form_builder.rb b/app/form_builders/fluentd_form_builder.rb
index cbec0e540..07c3a2a2e 100644
--- a/app/form_builders/fluentd_form_builder.rb
+++ b/app/form_builders/fluentd_form_builder.rb
@@ -76,4 +76,8 @@ def append_and_remove_links
     %Q!<a class="btn btn-xs btn-default js-append">#{icon('fa-plus')}</a> ! +
     %Q!<a class="btn btn-xs btn-default js-remove" style="display:none">#{icon('fa-minus')}</a> !
   end
+
+  def icon(classes, inner=nil)
+    %Q!<i class="fa #{classes}">#{inner}</i> !.html_safe
+  end
 end

From 8b776469b5d9ad1f4383dc61bbeff1ac4ec71241 Mon Sep 17 00:00:00 2001
From: Kenji Okimoto <okimoto@clear-code.com>
Date: Mon, 18 Jun 2018 14:38:07 +0900
Subject: [PATCH 137/137] Organize initial parameters

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>
---
 app/models/fluentd/setting/buffer_file.rb       |  4 +++-
 app/models/fluentd/setting/out_elasticsearch.rb |  7 +++++++
 app/models/fluentd/setting/out_forward.rb       |  8 +++++++-
 app/models/fluentd/setting/out_mongo.rb         | 13 +++++++++++--
 app/models/fluentd/setting/out_s3.rb            | 14 +++++++++++++-
 app/models/fluentd/setting/out_stdout.rb        |  7 +++++++
 app/models/fluentd/setting/out_tdlog.rb         |  7 +++++--
 7 files changed, 53 insertions(+), 7 deletions(-)

diff --git a/app/models/fluentd/setting/buffer_file.rb b/app/models/fluentd/setting/buffer_file.rb
index 31a74f7d0..6be83f0ad 100644
--- a/app/models/fluentd/setting/buffer_file.rb
+++ b/app/models/fluentd/setting/buffer_file.rb
@@ -6,7 +6,9 @@ class BufferFile
       register_plugin("buffer", "file")
 
       def self.initial_params
-        {}
+        {
+          path: ""
+        }
       end
 
       def common_options
diff --git a/app/models/fluentd/setting/out_elasticsearch.rb b/app/models/fluentd/setting/out_elasticsearch.rb
index 4df871d78..b1566f38b 100644
--- a/app/models/fluentd/setting/out_elasticsearch.rb
+++ b/app/models/fluentd/setting/out_elasticsearch.rb
@@ -14,6 +14,13 @@ def self.initial_params
           logstash_format: true,
           include_tag_key: false,
           utc_index: true,
+          buffer_type: "file",
+          buffer: {
+            "0" => {
+              "type" => "file",
+              "path" => "/var/log/td-agent/buffer/elasticsearch",
+             }
+          },
         }
       end
 
diff --git a/app/models/fluentd/setting/out_forward.rb b/app/models/fluentd/setting/out_forward.rb
index eb7d6c8b5..d6bdcc350 100644
--- a/app/models/fluentd/setting/out_forward.rb
+++ b/app/models/fluentd/setting/out_forward.rb
@@ -11,9 +11,15 @@ class OutForward
 
       def self.initial_params
         {
+          buffer_type: "memory",
+          buffer: {
+            "0" => {
+              "type" => "memory",
+            }
+          },
           secondary: {
             "0" => {
-              type: "file",
+              "type" => "file",
             }
           }
         }
diff --git a/app/models/fluentd/setting/out_mongo.rb b/app/models/fluentd/setting/out_mongo.rb
index 25afc1a8f..6338cb91d 100644
--- a/app/models/fluentd/setting/out_mongo.rb
+++ b/app/models/fluentd/setting/out_mongo.rb
@@ -4,6 +4,8 @@ class OutMongo
       include Fluentd::Setting::Plugin
 
       register_plugin("output", "mongo")
+      config_param(:capped, :bool, default: false)
+      config_param(:capped_size, :size, default: nil)
 
       # NOTE: fluent-plugin-mongo defines database parameter as required parameter
       #       But Fluentd tells us that the database parameter is not required.
@@ -12,7 +14,7 @@ class OutMongo
 
       def validate_collection
         if tag_mapped.blank? && collection.blank?
-          errors.add(:collection, :blank) 
+          errors.add(:collection, :blank)
         end
       end
 
@@ -22,7 +24,14 @@ def self.initial_params
           port: 27017,
           capped: true,
           capped_size: "100m",
-        }
+          buffer_type: "file",
+          buffer: {
+            "0" => {
+              "type" => "file",
+              "path" => "/var/log/td-agent/buffer/mongo",
+            }
+          },
+         }
       end
 
       def common_options
diff --git a/app/models/fluentd/setting/out_s3.rb b/app/models/fluentd/setting/out_s3.rb
index 37053414e..fedca81aa 100644
--- a/app/models/fluentd/setting/out_s3.rb
+++ b/app/models/fluentd/setting/out_s3.rb
@@ -8,7 +8,19 @@ class OutS3
       def self.initial_params
         {
           s3_region: "us-west-1",
-          use_ssl: true,
+          buffer_type: "file",
+          buffer: {
+            "0" => {
+              "type" => "file",
+              "path" => "/var/log/td-agent/buffer/s3",
+            }
+          },
+          format_type: "out_file",
+          format: {
+            "0" => {
+              "type" => "out_file"
+            }
+          }
         }
       end
 
diff --git a/app/models/fluentd/setting/out_stdout.rb b/app/models/fluentd/setting/out_stdout.rb
index ec0a5a120..0337b97f0 100644
--- a/app/models/fluentd/setting/out_stdout.rb
+++ b/app/models/fluentd/setting/out_stdout.rb
@@ -8,6 +8,13 @@ class OutStdout
       def self.initial_params
         {
           pattern: "debug.**",
+          buffer_type: "memory",
+          buffer: {
+            "0" => {
+              "type" => "memory",
+            }
+          },
+          format_type: "stdout",
           format: {
             "0" => {
               "@type" => "stdout",
diff --git a/app/models/fluentd/setting/out_tdlog.rb b/app/models/fluentd/setting/out_tdlog.rb
index 4f0e06a4a..65177a7a8 100644
--- a/app/models/fluentd/setting/out_tdlog.rb
+++ b/app/models/fluentd/setting/out_tdlog.rb
@@ -8,9 +8,12 @@ class OutTdlog
       def self.initial_params
         {
           pattern: "td.*.*",
+          buffer_type: "file",
           buffer: {
-            "@type" => "file",
-            "path" => "/var/log/td-agent/buffer/td",
+            "0" => {
+              "@type" => "file",
+              "path" => "/var/log/td-agent/buffer/td",
+            }
           },
           auto_create_table: true,
         }