diff --git a/Gemfile.lock b/Gemfile.lock index 6f8730c1a..8c95b0359 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,6 +7,10 @@ PATH bundler diff-lcs draper (~> 3.0) + fluent-plugin-elasticsearch (~> 2.10) + fluent-plugin-mongo (~> 1.1) + fluent-plugin-s3 (~> 1.1) + fluent-plugin-td (~> 1.0) fluentd (>= 1.0.0, < 2) font-awesome-rails haml-rails (~> 1.0) @@ -74,6 +78,24 @@ GEM addressable (2.5.2) public_suffix (>= 2.0.2, < 4.0) arel (9.0.0) + aws-eventstream (1.0.0) + aws-partitions (1.87.0) + aws-sdk-core (3.21.2) + aws-eventstream (~> 1.0) + aws-partitions (~> 1.0) + aws-sigv4 (~> 1.0) + jmespath (~> 1.0) + aws-sdk-kms (1.5.0) + aws-sdk-core (~> 3) + aws-sigv4 (~> 1.0) + aws-sdk-s3 (1.13.0) + aws-sdk-core (~> 3, >= 3.21.2) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.0) + aws-sdk-sqs (1.3.0) + aws-sdk-core (~> 3) + aws-sigv4 (~> 1.0) + aws-sigv4 (1.0.2) better_errors (2.1.1) coderay (>= 1.0.0) erubis (>= 2.6.6) @@ -83,6 +105,7 @@ GEM debug_inspector (>= 0.0.1) bootsnap (1.3.0) msgpack (~> 1.0) + bson (4.3.0) builder (3.2.3) capybara (3.0.2) addressable @@ -112,14 +135,39 @@ GEM activemodel-serializers-xml (~> 1.0) activesupport (~> 5.0) request_store (~> 1.0) + elasticsearch (6.0.2) + elasticsearch-api (= 6.0.2) + elasticsearch-transport (= 6.0.2) + elasticsearch-api (6.0.2) + multi_json + elasticsearch-transport (6.0.2) + faraday + multi_json erubi (1.7.1) erubis (2.7.0) + excon (0.62.0) factory_bot (4.8.2) activesupport (>= 3.0.0) factory_bot_rails (4.8.2) factory_bot (~> 4.8.2) railties (>= 3.0.0) + faraday (0.15.2) + multipart-post (>= 1.2, < 3) ffi (1.9.23) + fluent-plugin-elasticsearch (2.10.1) + elasticsearch + excon + fluentd (>= 0.14.20) + fluent-plugin-mongo (1.1.1) + fluentd (>= 0.14.12, < 2) + mongo (~> 2.2.0) + fluent-plugin-s3 (1.1.3) + aws-sdk-s3 (~> 1.0) + aws-sdk-sqs (~> 1.0) + fluentd (>= 0.14.2, < 2) + fluent-plugin-td (1.0.0) + fluentd (>= 0.14.13, < 2) + td-client (~> 1.0) fluentd (1.2.1) cool.io (>= 1.4.5, < 2.0.0) dig_rb (~> 1.0.0) @@ -159,6 +207,7 @@ GEM jbuilder (2.7.0) activesupport (>= 4.2.0) multi_json (>= 1.2) + jmespath (1.4.0) json (2.1.0) kramdown (1.16.2) kramdown-haml (0.0.3) @@ -181,8 +230,11 @@ GEM mini_mime (1.0.0) mini_portile2 (2.3.0) minitest (5.11.3) + mongo (2.2.7) + bson (~> 4.0) msgpack (1.2.4) multi_json (1.13.1) + multipart-post (2.0.0) nio4r (2.3.0) nokogiri (1.8.2) mini_portile2 (~> 2.3.0) @@ -286,6 +338,9 @@ GEM strptime (0.2.3) sucker_punch (2.0.4) concurrent-ruby (~> 1.0.0) + td-client (1.0.6) + httpclient (>= 2.7) + msgpack (>= 0.5.6, < 2) temple (0.8.0) thor (0.20.0) thread_safe (0.3.6) diff --git a/app/controllers/api/config_definitions_controller.rb b/app/controllers/api/config_definitions_controller.rb new file mode 100644 index 000000000..6b5384d7a --- /dev/null +++ b/app/controllers/api/config_definitions_controller.rb @@ -0,0 +1,55 @@ +class Api::ConfigDefinitionsController < ApplicationController + before_action :login_required + + def index + name = params[:name] + type = params[:type] + prefix = case type + when "input" + "in" + when "output" + "out" + when "filter" + "filter" + when "parse" + "parser" + when "format" + "formatter" + when "parser", "formatter", "buffer", "storage" + type + end + + target_class = Fluentd::Setting.const_get("#{prefix}_#{name}".classify) + target = target_class.new + + common_options = target.common_options.map do |key| + h = { + name: key, + type: target.column_type(key), + desc: target.desc(key), + default: target.default(key) + } + h[:list] = target.list_of(key) if target.column_type(key) == :enum + h + end + + advanced_options = target.advanced_options.map do |key| + h = { + name: key, + type: target.column_type(key), + desc: target.desc(key), + default: target.default(key) + } + h[:list] = target.list_of(key) if target.column_type(key) == :enum + h + end + + options = { + type: type, + name: name, + commonOptions: common_options, + advancedOptions: advanced_options + } + render json: options + end +end diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb index c1623904f..11445a0b5 100644 --- a/app/controllers/api_controller.rb +++ b/app/controllers/api_controller.rb @@ -19,9 +19,12 @@ def empty_json end def regexp_preview - preview = RegexpPreview.processor(params[:format]).new(params[:file], params[:format], params) + plugin_config = prepare_plugin_config || {} + preview = RegexpPreview.processor(params[:parse_type]).new(params[:file], params[:parse_type], plugin_config) - render json: preview.matches_json + render json: preview.matches + rescue Fluent::ConfigError => ex + render json: { error: "#{ex.class}: #{ex.message}" } end def grok_to_regexp @@ -29,5 +32,19 @@ def grok_to_regexp grok.load_patterns render text: grok.convert_to_regexp(params[:grok_str]).source end -end + private + + def prepare_plugin_config + plugin_config = params[:plugin_config] + case params[:parse_type] + when "multiline" + plugin_config[:formats].lines.each.with_index do |line, index| + plugin_config["format#{index + 1}"] = line.chomp + end + plugin_config + else + plugin_config + end + end +end diff --git a/app/controllers/concerns/setting_concern.rb b/app/controllers/concerns/setting_concern.rb index 6cf5d774a..47dee4d0e 100644 --- a/app/controllers/concerns/setting_concern.rb +++ b/app/controllers/concerns/setting_concern.rb @@ -10,11 +10,19 @@ module SettingConcern def show @setting = target_class.new(initial_params) + @buffer = @setting.create_buffer + @storage = @setting.create_storage + @parser = @setting.create_parser + @formatter = @setting.create_formatter + @_used_param = {} + @_used_section = {} render "shared/settings/show" end def finish @setting = target_class.new(setting_params) + @_used_param = {} + @_used_section = {} unless @setting.valid? return render "shared/settings/show" end @@ -32,7 +40,7 @@ def finish private def setting_params - params.require(target_class.to_s.underscore.gsub("/", "_")).permit(*target_class.const_get(:KEYS)) + params.require(:setting).permit(*target_class.permit_params) end def initial_params @@ -40,7 +48,15 @@ def initial_params end def target_plugin_name - target_class.to_s.split("::").last.underscore + prefix = case target_class.plugin_type + when "input" + "in" + when "output" + "out" + else + target_class.plugin_type + end + "#{prefix}_#{target_class.plugin_name}" end def plugin_setting_form_action_url(*args) diff --git a/app/controllers/fluentd/settings/in_tail_controller.rb b/app/controllers/fluentd/settings/in_tail_controller.rb index 85d849c0b..0da439b6e 100644 --- a/app/controllers/fluentd/settings/in_tail_controller.rb +++ b/app/controllers/fluentd/settings/in_tail_controller.rb @@ -4,8 +4,8 @@ class Fluentd::Settings::InTailController < ApplicationController def after_file_choose @setting = Fluentd::Setting::InTail.new({ - :path => params[:path], - :tag => nil, + path: params[:path], + tag: nil }) end @@ -38,20 +38,28 @@ def finish return render "after_format" end - if @fluentd.agent.configuration.to_s.include?(@setting.to_conf.strip) + if @fluentd.agent.configuration.to_s.include?(@setting.to_config.to_s.strip) @setting.errors.add(:base, :duplicated_conf) return render "after_format" end - @fluentd.agent.config_append @setting.to_conf + @fluentd.agent.config_append @setting.to_config.to_s @fluentd.agent.restart if @fluentd.agent.running? redirect_to daemon_setting_path(@fluentd) end + def target_class + Fluentd::Setting::InTail + end + private def setting_params - params.require(:setting).permit(:path, :format, :regexp, *Fluentd::Setting::InTail.known_formats, :tag, :rotate_wait, :pos_file, :read_from_head, :refresh_interval) + permit_params = target_class._types.keys + permit_params << :parse_type + section_class = Fluentd::Setting.const_get("parser_#{params.dig(:setting, :parse_type)}".classify) + permit_params << { parse: section_class._types.keys } + params.require(:setting).permit(*permit_params) end end diff --git a/app/controllers/fluentd/settings/out_forward_controller.rb b/app/controllers/fluentd/settings/out_forward_controller.rb index 42a5f3c20..c438bd281 100644 --- a/app/controllers/fluentd/settings/out_forward_controller.rb +++ b/app/controllers/fluentd/settings/out_forward_controller.rb @@ -6,13 +6,4 @@ class Fluentd::Settings::OutForwardController < ApplicationController def target_class Fluentd::Setting::OutForward end - - def setting_params - params.require(:fluentd_setting_out_forward).permit(*Fluentd::Setting::OutForward::KEYS).merge( - params.require(:fluentd_setting_out_forward).permit( - :server => Fluentd::Setting::OutForward::Server::KEYS, - :secondary => Fluentd::Setting::OutForward::Secondary::KEYS, - ), - ) - end end diff --git a/app/controllers/fluentd/settings/out_s3_controller.rb b/app/controllers/fluentd/settings/out_s3_controller.rb index d8c496496..3cd6f6afe 100644 --- a/app/controllers/fluentd/settings/out_s3_controller.rb +++ b/app/controllers/fluentd/settings/out_s3_controller.rb @@ -6,9 +6,4 @@ class Fluentd::Settings::OutS3Controller < ApplicationController def target_class Fluentd::Setting::OutS3 end - - def setting_params - params.require(:fluentd_setting_out_s3).permit(*Fluentd::Setting::OutS3::KEYS) - end - end diff --git a/app/controllers/fluentd/settings/out_td_controller.rb b/app/controllers/fluentd/settings/out_td_controller.rb deleted file mode 100644 index 8e3368d89..000000000 --- a/app/controllers/fluentd/settings/out_td_controller.rb +++ /dev/null @@ -1,18 +0,0 @@ -class Fluentd::Settings::OutTdController < ApplicationController - include SettingConcern - - private - - def target_class - Fluentd::Setting::OutTd - end - - def initial_params - { - buffer_type: "file", - buffer_path: "/var/log/td-agent/buffer/td", - auto_create_table: true, - match: "td.*.*", - } - end -end diff --git a/app/controllers/fluentd/settings/out_tdlog_controller.rb b/app/controllers/fluentd/settings/out_tdlog_controller.rb new file mode 100644 index 000000000..dac9b14f0 --- /dev/null +++ b/app/controllers/fluentd/settings/out_tdlog_controller.rb @@ -0,0 +1,9 @@ +class Fluentd::Settings::OutTdlogController < ApplicationController + include SettingConcern + + private + + def target_class + Fluentd::Setting::OutTdlog + end +end diff --git a/app/form_builders/fluentd_form_builder.rb b/app/form_builders/fluentd_form_builder.rb new file mode 100644 index 000000000..07c3a2a2e --- /dev/null +++ b/app/form_builders/fluentd_form_builder.rb @@ -0,0 +1,83 @@ +class FluentdFormBuilder < ActionView::Helpers::FormBuilder + include ActionView::Helpers::TagHelper + include ActionView::Context + + def field(key, options = {}) + plugin_class = object.class + content_tag(:div, class: "form-group") do + if plugin_class._sections[key] + render_section(key, options) + else + resolve_field(key, options) + end + end + end + + private + + def resolve_field(key, options = {}) + plugin_class = object.class + column_type = plugin_class.column_type(key) + case column_type + when :enum + enum_field(key, options) + when :bool + bool_field(key, options) + else + other_field(key, options) + end + end + + def enum_field(key, options) + label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) + + select(key, object.list_of(key), options, { class: "enum" }) + end + + def bool_field(key, options) + check_box(key, options, "true", "false") + " " + + label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) + end + + def other_field(key, options) + return unless object.respond_to?(key) + label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) + + text_field(key, class: "form-control", **options) + end + + def render_section(key, options) + section_class = object.class._sections[key] + children = object.__send__(key) || { "0" => {} } + html = "" + + children.each do |index, child| + html << content_tag("div", class: "js-nested-column #{section_class.multi ? "js-multiple" : ""}") do + _html = "" + _html << append_and_remove_links if section_class.multi + _html << label(key, nil, data: { toggle: "tooltip", placement: "right" }, title: object.desc(key)) + _html << section_fields(key, index, section_class, child) + _html.html_safe + end + end + html.html_safe + end + + def section_fields(key, index, section_class, child) + section = section_class.new(child) + fields("#{key}[#{index}]", model: section) do |section_form| + html = "" + section_class._types.keys.each do |section_key| + html << section_form.field(section_key) + end + html.html_safe + end + end + + def append_and_remove_links + %Q!#{icon('fa-plus')} ! + + %Q! ! + end + + def icon(classes, inner=nil) + %Q!#{inner} !.html_safe + end +end diff --git a/app/helpers/fluentd/settings_helper.rb b/app/helpers/fluentd/settings_helper.rb deleted file mode 100644 index 9d2e93e3d..000000000 --- a/app/helpers/fluentd/settings_helper.rb +++ /dev/null @@ -1,2 +0,0 @@ -module Fluentd::SettingsHelper -end diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index dfee00b18..aca9fdf67 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -2,28 +2,74 @@ module SettingsHelper def field(form, key, opts = {}) html = '
' - field_resolver(form.object.column_type(key), html, form, key, opts) + field_resolver(html, form, key, opts) html << "
" html.html_safe end private - def field_resolver(type, html, form, key, opts) - case type - when :hidden - html << form.hidden_field(key) - when :boolean, :flag - boolean_field(html, form, key, opts) - when :choice - choice_field(html, form, key, opts) - when :nested - nested_field(html, form, key, opts) - else - other_field(html, form, key, opts) + + def field_resolver(html, form, key, opts) + plugin_class = form.object.class + type = plugin_class.column_type(key) + if type && !@_used_param.key?(key) + case type + when :enum + enum_field(html, form, key, opts) + when :bool + bool_field(html, form, key, opts) + else + other_field(html, form, key, opts) + end + @_used_param[key] = true + end + if plugin_class._sections[key] && !@_used_section.key?(key) + section_field(html, form, key, opts) + @_used_section[key] = true + end + end + + def section_field(html, form, key, opts = {}) + klass = form.object.class._sections[key] + children = form.object.__send__(key) || { "0" => {} } + # / section is not multiple in most cases + multi = if [:parse, :format].include?(key) + false + else + klass.multi + end + + children.each do |index, child| + open_section_div(html, multi) do |_html| + _html << append_and_remove_links if multi + _html << h(form.label(key)) + _html << section_fields(form, key, index, klass, child) + end end end + def open_section_div(html, multi) + html << %Q!
! + yield html + html << "
" + end + + def section_fields(form, key, index, klass, child) + html = "" + object = klass.new(child) + form.fields_for("#{key}[#{index}]", object) do |ff| + klass._types.keys.each do |kk| + if kk == :type + html << owned_plugin_type_field(ff, kk, key) + else + html << field(ff, kk) + end + end + end + html + end + def nested_field(html, form, key, opts = {}) klass = child_data(form, key)[:class] options = child_data(form, key)[:options] @@ -54,28 +100,45 @@ def nested_fields(form, key, index, klass, child) end def append_and_remove_links - %Q!#{icon('fa-plus')} ! + - %Q! ! + %Q!#{icon('fa-plus')} ! + + %Q! ! end def child_data(form, key) form.object.class.children[key] end - def choice_field(html, form, key, opts = {}) + def enum_field(html, form, key, opts = {}) html << h(form.label(key)) html << " " # NOTE: Adding space for padding - html << form.select(key, form.object.values_of(key), opts) + html << form.select(key, form.object.list_of(key), opts, { class: "enum" }) end - def boolean_field(html, form, key, opts = {}) + def bool_field(html, form, key, opts = {}) html << form.check_box(key, {}, "true", "false") html << " " # NOTE: Adding space for padding html << h(form.label(key)) end def other_field(html, form, key, opts = {}) + return unless form.object.respond_to?(key) html << h(form.label(key)) html << form.text_field(key, class: "form-control") end + + def owned_plugin_type_field(form, key, plugin_type) + registry_type = case plugin_type + when :parse + "PARSER_REGISTRY" + when :format + "FORMATTER_REGISTRY" + end + plugin_registry = Fluent::Plugin.const_get("#{registry_type}") + html = '
' + html << form.label(key) + html << " " # NOTE: Adding space for padding + html << form.select(key, plugin_registry.map.keys, {}, { class: "owned" }) + html << '
' + html + end end diff --git a/app/javascript/packs/application.js b/app/javascript/packs/application.js index 25c74dbfb..2fbbe6cc9 100644 --- a/app/javascript/packs/application.js +++ b/app/javascript/packs/application.js @@ -34,3 +34,7 @@ Vue.filter('to_json', function (value) { window.Vue = Vue import '../stylesheets/application.scss' + +$(document).ready(() => { + $("[data-toggle=tooltip]").tooltip() +}) diff --git a/app/javascript/packs/in_tail_format.js b/app/javascript/packs/in_tail_format.js deleted file mode 100644 index 2ba758db0..000000000 --- a/app/javascript/packs/in_tail_format.js +++ /dev/null @@ -1,174 +0,0 @@ -'use strict' -import 'lodash/lodash' - -window.addEventListener('load', () => { - const maxFormatCount = 20; - - new Vue({ - el: "#in_tail_format", - props: ["formatOptionsJson", "initialSelected", "targetFile", "paramsJson"], - data: { - previewProcessing: false, - format: "", - highlightedLines: null, - }, - - computed: { - options: { - get: function(){ - return this.formatOptions[this.format]; - }, - }, - selectableFormats: { - get: function() { - return Object.keys(this.formatOptions); - } - }, - useTextArea: function() { - return this.format === "multiline"; - } - }, - - beforeMount: function() { - this.formatOptions = JSON.parse(this.$el.attributes.formatOptionsJson.nodeValue); - this.format = this.$el.attributes.initialSelected.nodeValue; - - // initialize params - // NOTE: if `params.setting.foo` is undefined, Vue can't binding with v-model="params.setting.foo" - var params = JSON.parse(this.$el.attributes.paramsJson.nodeValue); - if(!params.setting) { - params.setting = {}; - } - - var formats = _.chain(_.range(1, maxFormatCount + 1)).map(function(i) {return params.setting["format" + String(i)];}).compact().value(); - params.setting.formats = formats.join("\n"); - - _.each(this.formatOptions, function(options){ - _.each(options, function(key){ - if(!params.setting.hasOwnProperty(key)){ - params.setting[key] = ""; - } - }); - }); - this.params = params; - - this.targetFile = this.$el.attributes.targetFile.nodeValue; - }, - mounted: function(){ - this.$emit("data-loaded"); - }, - - watch: { - 'params.setting.formats': function(formats) { - _.range(1, maxFormatCount + 1).forEach(()=> {params.setting["format" + String(i)] = "";}); - - _.compact(formats.split("\n")).forEach((formatLine, index)=> { - params.setting["format" + String(index + 1)] = formatLine; - }) - }, - 'params.setting.regexp': function() { - this.preview(); - }, - 'format': function() { - this.preview(); - }, - }, - - methods: { - onKeyup: function(ev){ - var el = ev.target; - if(el.name.match(/\[format/)){ - this.preview(); - } - }, - updateHighlightedLines: function() { - if(!this.regexpMatches) { - this.highlightedLines = null; - return; - } - - var $container = jQuery('
'); - _.each(this.regexpMatches, function(match){ - var colors = [ - "#ff9", "#cff", "#fcf", "#dfd" - ]; - var whole = match.whole; - var html = ""; - var matches = []; - - var lastPos = 0; - _.each(match.matches, function(match) { - var matched = match.matched; - if(!matched) return; - if(matched.length === 0) return; // Ignore empty matched with "foobar".match(/foo(.*?)bar/)[1] #=> "" - - // rotated highlight color - var currentColor = colors.shift(); - colors.push(currentColor); - - // create highlighted range HTML - var $highlighted = jQuery('').text(matched); - $highlighted.attr({ - "class": "regexp-preview", - "data-toggle": "tooltip", - "data-placement": "top", - "title": match.key, - 'style': 'background-color:' + currentColor - }); - var highlightedHtml = $highlighted.wrap('
').parent().html(); - - var pos = { - "start": match.pos[0], - "end": match.pos[1] - }; - if(pos.start > 0) { - html += _.escape(whole.substring(lastPos, pos.start)); - } - html += highlightedHtml; - lastPos = pos.end; - }); - html += whole.substring(lastPos); - - $container.append(html); - $container.append("
"); - }); - - this.highlightedLines = $container.html(); - }, - - preview: function(){ - if(this.previewAjax) { - this.previewAjax.abort(); - } - - const token = document.getElementsByName("csrf-token")[0].getAttribute('content') - - var self = this; - new Promise(function(resolve, reject) { - self.previewAjax = $.ajax({ - method: "POST", - url: "/api/regexp_preview", - headers: { - 'X-CSRF-Token': token - }, - data: { - regexp: self.params.setting.regexp, - time_format: self.params.setting.time_format, - format: _.isEmpty(self.format) ? "regexp" : self.format, - params: self.params.setting, - file: self.targetFile - } - }).done(resolve).fail(reject); - }).then(function(result){ - self.params = _.merge(self.params, result.params); - self.regexpMatches = result.matches; - self.updateHighlightedLines(); - })["catch"](function(error){ - if(error.stack) { - console.error(error.stack); - } - }); - }, - } - }); -}); diff --git a/app/javascript/packs/in_tail_parse.js b/app/javascript/packs/in_tail_parse.js new file mode 100644 index 000000000..9cf205a06 --- /dev/null +++ b/app/javascript/packs/in_tail_parse.js @@ -0,0 +1,159 @@ +'use strict' +import 'lodash/lodash' +import 'popper.js/dist/popper' +import 'bootstrap/dist/js/bootstrap' +import OwnedPluginForm from './owned_plugin_form' + +$(document).ready(() => { + new Vue({ + el: '#in-tail-parse', + props: [ + "path", + "parseType" + ], + data: function() { + return { + highlightedLines: null + } + }, + computed: { + token: function() { + return Rails.csrfToken() + } + }, + components: { + 'owned-plugin-form': OwnedPluginForm + }, + watch: { + 'parse.expression': function() { + console.log(`parse.expression: ${this.parse.expression}`) + this.preview() + }, + 'parse.time_format': function() { + console.log(`parse.time_format: ${this.parse.time_format}`) + this.preview() + }, + 'parseType': function() { + this.preview() + }, + }, + beforeMount: function() { + this.path = this.$el.attributes.path.nodeValue; + }, + mounted: function() { + this.parse = {} + this.$on("hook:updated", () => { + $("[data-toggle=tooltip]").tooltip("dispose") + $("[data-toggle=tooltip]").tooltip("enable") + }) + }, + methods: { + onChangePluginName: function(name) { + console.log("onChangePluginName") + this.parseType = name + this.parse = {} // clear parser plugin configuration + }, + onChangeParseConfig: function(data) { + console.log("onChangeParseConfig", data) + _.merge(this.parse, data) + this.preview() + }, + onChangeFormats: function(data) { + console.log("in_tail_parse:onChangeFormats", data) + _.merge(this.parse, data) + this.preview() + }, + updateHighlightedLines: function(matches) { + if (!matches) { + this.highlightedLines = null + return + } + + let $container = $('
') + _.each(matches, (match) => { + const colors = ["#ff9", "#cff", "#fcf", "#dfd"] + const whole = match.whole + let html = "" + let _matches = [] + let lastPos = 0 + + _.each(match.matches, (m) => { + let matched = m.matched + if (!matched) { + return + } + // Ignore empty matched with "foobar".match(/foo(.*?)bar/)[1] #=> "" + if (matched.length === 0) { + return + } + // rotate color + let currentColor = colors.shift() + colors.push(currentColor) + + // create highlighted range HTML + let $highlighted = $("").text(matched) + $highlighted.attr({ + "class": "regexp-preview", + "data-toggle": "tooltip", + "data-placement": "top", + "title": m.key, + 'style': 'background-color:' + currentColor + }) + let highlightedHtml = $highlighted.wrap("
").parent().html() + let pos = { + start: m.pos[0], + end: m.pos[1] + } + if (pos.start > 0) { + html += _.escape(whole.substring(lastPos, pos.start)) + } + html += highlightedHtml + lastPos = pos.end + }) + html += whole.substring(lastPos) + + $container.append(html) + $container.append("
") + }) + + this.highlightedLines = $container.html() + this.$emit("hook:updated") + }, + + preview: function() { + console.log("preview!!!!") + if (this.previewAjax && this.previewAjax.state() === "pending") { + this.previewAjax.abort() + } + + this.previewAjax = $.ajax({ + method: "POST", + url: "/api/regexp_preview", + headers: { + 'X-CSRF-Token': this.token + }, + data: { + parse_type: _.isEmpty(this.parseType) ? "regexp" : this.parseType, + file: this.path, + plugin_config: this.parse + } + }).then( + (result) => { + if (result.matches) { + this.updateHighlightedLines(result.matches) + } else { + console.error(result.error) + this.previewError = result.error + } + }, + (error) => { + this.highlightedLines = null + // console.error(error.responseText) + if (error.stack) { + console.error(error.stack) + } + }) + } + } + }) +}) diff --git a/app/javascript/packs/owned_plugin_form.js b/app/javascript/packs/owned_plugin_form.js new file mode 100644 index 000000000..35b7f981a --- /dev/null +++ b/app/javascript/packs/owned_plugin_form.js @@ -0,0 +1,141 @@ +'use strict' + +import ParserMultilineForm from './parser_multiline_form' + +const OwnedPluginForm = { + template: "#vue-owned-plugin-form", + components: { + "parser-multiline-form": ParserMultilineForm, + }, + props: [ + "id", + "optionsJson", + "initialPluginName", + "pluginType", + "pluginLabel" + ], + data: () => { + return { + pluginName: "", + options: [], + commonOptions: [], + advancedOptions: [], + expression: null, + timeFormat: null, + unwatchExpression: null, + unwatchTimeFormat: null + } + }, + + computed: { + token: function() { + return Rails.csrfToken() + } + }, + + mounted: function() { + this.options = JSON.parse(this.optionsJson) + this.pluginName = this.initialPluginName + this.$on("hook:updated", () => { + console.log("hook:updated") + $("[data-toggle=tooltip]").tooltip("dispose") + $("[data-toggle=tooltip]").tooltip("enable") + }) + this.$once("data-loaded", () => { + this.updateSection() + }) + this.$emit("data-loaded") + }, + + methods: { + onChange: function() { + this.updateSection() + if (this.pluginType === "parse") { + this.$emit("change-plugin-name", this.pluginName) + } + }, + + onChangeFormats: function(data) { + console.log("ownedPluginForm:onChangeFormats", data) + this.$emit("change-formats", data) + }, + + updateSection: function() { + $.ajax({ + method: "GET", + url: "/api/config_definitions", + headers: { + 'X-CSRF-Token': this.token + }, + data: { + type: this.pluginType, + name: this.pluginName + } + }).then((data) => { + this.commonOptions = data.commonOptions + let foundExpression = false + let foundTimeFormat = false + _.each(this.commonOptions, (option) => { + if (option.name === "expression") { + foundExpression = true + this.expression = option.default + this.unwatchExpression = this.$watch("expression", (newValue, oldValue) => { + console.log(newValue) + this.$emit("change-parse-config", { + "expression": this.expression, + "time_format": this.timeFormat + }) + }) + } + if (option.name === "time_format") { + foundTimeFormat = true + this.timeFormat = option.default + this.unwatchTimeFormat = this.$watch("timeFormat", (newValue, oldValue) => { + console.log({"watch time_format": newValue}) + this.$emit("change-parse-config", { + "expression": this.expression, + "time_format": this.timeFormat + }) + }) + } + + if (!foundExpression && this.unwatchExpression) { + this.expression = null + this.unwatchExpression() + this.unwatchExpression = null + } + if (!foundTimeFormat && this.unwatchTimeFormat) { + this.timeFormat = null + this.unwatchTimeFormat() + this.unwatchTimeFormat = null + } + }) + }) + }, + + selectId: function(pluginType) { + return `setting_${pluginType}_type` + }, + selectClass: function(pluginType) { + return `${pluginType} form-control` + }, + selectName: function(pluginType) { + return `setting[${pluginType}_type]` + }, + inputId: function(pluginType, option) { + return `setting_${pluginType}_0__${option.name}` + }, + inputName: function(pluginType, option) { + return `setting[${pluginType}[0]][${option.name}]` + }, + checked: function(checked) { + if (checked === true || checked === "true") { + return "checked" + } else { + return "" + } + } + } +} + +export { OwnedPluginForm as default } diff --git a/app/javascript/packs/parser_multiline_form.js b/app/javascript/packs/parser_multiline_form.js new file mode 100644 index 000000000..e80f0a703 --- /dev/null +++ b/app/javascript/packs/parser_multiline_form.js @@ -0,0 +1,51 @@ +'use strict' +import 'lodash/lodash' +const ParserMultilineForm = { + template: "#vue-parser-multiline-form", + props: [ + "pluginType", + "commonOptions" + ], + + data: function() { + return { + formatFirstline: "", + formats: "", + formatFirstlineDesc: "" + } + }, + + watch: { + "formatFirstLine": function(newValue, oldValue) { + console.log(`watch formatFirstLine: ${newValue}`) + this.$emit("change-formats", { + "format_firstline": this.formatFirstline, + "formats": this.formats + }) + }, + "formats": function(newValue, oldValue) { + console.log(`watch formats: ${newValue}`) + this.$emit("change-formats", { + "format_firstline": this.formatFirstline, + "formats": this.formats + }) + }, + "commonOptions": function(newValue, oldValue) { + const option = _.find(newValue, (o) => { + return o.name === "format_firstline" + }) + this.formatFirstlineDesc = option.desc + } + }, + + methods: { + textareaId: function(pluginType) { + return `setting_${pluginType}_0__formats` + }, + textareaName: function(pluginType) { + return `setting[${pluginType}[0]][formats]` + } + } +} + +export { ParserMultilineForm as default } diff --git a/app/javascript/packs/plugin_setting.js b/app/javascript/packs/plugin_setting.js new file mode 100644 index 000000000..675d4c94a --- /dev/null +++ b/app/javascript/packs/plugin_setting.js @@ -0,0 +1,19 @@ +'use strict' +import 'lodash/lodash' +import 'popper.js/dist/popper' +import 'bootstrap/dist/js/bootstrap' +import OwnedPluginForm from './owned_plugin_form' + +window.addEventListener('load', () => { + new Vue({ + el: '#plugin-setting', + data: () => { + return {} + }, + components: { + 'owned-plugin-form': OwnedPluginForm + }, + methods: { + } + }) +}) diff --git a/app/models/concerns/fluentd/setting/configurable.rb b/app/models/concerns/fluentd/setting/configurable.rb new file mode 100644 index 000000000..669067ed0 --- /dev/null +++ b/app/models/concerns/fluentd/setting/configurable.rb @@ -0,0 +1,113 @@ +class Fluentd + module Setting + module Configurable + extend ActiveSupport::Concern + + included do + class_attribute :_types, :_defaults, :_secrets, :_aliases, :_required + class_attribute :_deprecated_params, :_obsoleted_params, :_descriptions + class_attribute :_list, :_value_types, :_symbolize_keys + class_attribute :_argument_name, :_built_in_params, :_sections, :_section_params + self._types = {} + self._defaults = {} + self._secrets = {} + self._aliases = {} + self._required = {} + self._deprecated_params = {} + self._obsoleted_params = {} + self._descriptions = {} + self._list = {} + self._value_types = {} + self._symbolize_keys = {} + self._argument_name = nil + self._built_in_params = [] + self._sections = {} + self._section_params = Hash.new {|h, k| h[k] = [] } + end + + def initialize(attributes = {}) + # the superclass does not know specific attributes of the model + begin + super + rescue ActiveModel::UnknownAttributeError => ex + Rails.logger.warn(ex) + end + self.class._sections.each do |name, klass| + klass.init + if klass.multi + next if attributes[name].nil? + attributes[name].each do |attr| + next unless attr + attr.each do |index, _attr| + self._section_params[name] << klass.new(_attr) + end + end + else + attr = attributes.dig(name, "0") + self._section_params[name] << klass.new(attr) if attr + end + end + end + + module ClassMethods + # config_param :name, :string, default: "value", secret: true + def config_param(name, type = ActiveModel::Type::Value.new, **options) + # NOTE: We cannot overwrite types defined by ActiveModel in config/initializers/types.rb + if type == :time + type = Fluentd::Setting::Type::Time.new + end + if name.to_s.start_with?("@") + _name = name.to_s[1..-1] + config_param(_name.to_sym, type, **options.merge(alias: name)) + self._built_in_params << _name + elsif ["id", "type", "log_level"].include?(name.to_s) + self._built_in_params << name + unless name == "type" + attribute(name, type, **options.slice(:precision, :limit, :scale)) + validates(name, presence: true) if options[:required] + end + else + attribute(name, type, **options.slice(:precision, :limit, :scale)) + validates(name, presence: true) if options[:required] + end + self._types[name] = type + self._descriptions[name] = options[:desc] if options.key?(:desc) + self._defaults[name] = options[:default] if options.key?(:default) + self._secrets[name] = options[:secret] if options.key?(:secret) + self._aliases[name] = options[:alias] if options.key?(:alias) + self._required[name] = options[:required] if options.key?(:required) + self._deprecated_params[name] = options[:deprecated] if options.key?(:deprecated) + self._obsoleted_params[name] = options[:obsoleted] if options.key?(:obsoleted) + self._list[name] = options[:list] if options.key?(:list) + self._value_types[name] = options[:value_types] if options.key?(:value_types) + self._symbolize_keys = options[:symbolize_keys] if options.key?(:symbolize_keys) + end + + def config_section(name, **options, &block) + if self._sections.key?(name) + self._sections[name].merge(**options, &block) + else + attribute(name, :section) + section_class = Class.new(::Fluentd::Setting::Section) + section_class.section_name = name + section_class.required = options[:required] + section_class.multi = options[:multi] + section_class.alias = options[:alias] + section_class._block = block + self.const_set(name.to_s.classify, section_class) + self._sections[name] = section_class + end + end + + def config_argument(name, type = ActiveModel::Type::Value.new, **options) + config_param(name, **options) + self._argument_name = name + end + + def set_argument_name(name) + self._argument_name = name + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/pattern.rb b/app/models/concerns/fluentd/setting/pattern.rb new file mode 100644 index 000000000..ff0b89bbb --- /dev/null +++ b/app/models/concerns/fluentd/setting/pattern.rb @@ -0,0 +1,11 @@ +class Fluentd + module Setting + module Pattern + extend ActiveSupport::Concern + + included do + config_argument(:pattern, :string) + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/plugin.rb b/app/models/concerns/fluentd/setting/plugin.rb new file mode 100644 index 000000000..05464e7bb --- /dev/null +++ b/app/models/concerns/fluentd/setting/plugin.rb @@ -0,0 +1,78 @@ +require "fluent/plugin" + +class Fluentd + module Setting + module Plugin + extend ActiveSupport::Concern + + include ActiveModel::Model + include ActiveModel::Attributes + include Fluentd::Setting::Configurable + include Fluentd::Setting::PluginConfig + include Fluentd::Setting::SectionParser + include Fluentd::Setting::PluginParameter + include Fluentd::Setting::SectionValidator + + included do + cattr_accessor :plugin_type, :plugin_name, :config_definition + end + + module ClassMethods + def register_plugin(type, name) + self.plugin_type = type + self.plugin_name = name + + if ["filter", "output"].include?(type) + include Fluentd::Setting::Pattern + end + + self.load_plugin_config do |_name, params| + params.each do |param_name, definition| + if definition[:section] + parse_section(param_name, definition) + if %i(buffer storage parse format).include?(param_name) + attribute("#{param_name}_type", :string) + end + else + config_param(param_name, definition[:type], **definition.except(:type)) + end + end + end + end + + def load_plugin_config + dumped_config = {} + plugin_class.ancestors.reverse_each do |klass| + next unless klass.respond_to?(:dump_config_definition) + dumped_config_definition = klass.dump_config_definition + dumped_config[klass.name] = dumped_config_definition unless dumped_config_definition.empty? + end + self.config_definition = dumped_config + dumped_config.each do |name, config| + yield name, config + end + end + + def plugin_instance + @plugin_instance ||= Fluent::Plugin.__send__("new_#{plugin_type}", plugin_name) + end + + def plugin_class + @plugin_class ||= plugin_instance.class + end + + def plugin_helpers + @plugin_helpers ||= if plugin_instance.respond_to?(:plugin_helpers) + plugin_instance.plugin_helpers + else + [] + end + end + + def section? + false + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb new file mode 100644 index 000000000..c2814bb11 --- /dev/null +++ b/app/models/concerns/fluentd/setting/plugin_config.rb @@ -0,0 +1,74 @@ +class Fluentd + module Setting + module PluginConfig + extend ActiveSupport::Concern + + def to_config + name = case plugin_type + when "input" + "source" + when "output" + "match" + when "filter" + "filter" + when "parser" + "parse" + when "formatter" + "format" + when "buffer" + "buffer" + when "storage" + "storage" + end + _attributes = attributes.reject do |key, value| + %w(parse_type format_type buffer_type storage_type).include?(key.to_s) + end + _attributes = { "@type" => self.plugin_name }.merge(_attributes) + _attributes["@log_level"] = _attributes.delete("log_level") + argument = case plugin_type + when "output", "filter", "buffer" + _attributes.delete(self._argument_name.to_s) || "" + else + "" + end + attrs, elements = parse_attributes(_attributes) + config_element(name, argument, attrs, elements) + end + + def parse_attributes(attributes) + base_klasses = config_definition.keys + sections, params = attributes.partition do |key, _section_attributes| + base_klasses.any? do |base_klass| + config_definition.dig(base_klass, key.to_sym, :section) || config_definition.dig(key.to_sym, :section) + end + end + elements = [] + sections.to_h.each do |key, section_params| + if %w(parse format buffer storage).include?(key) + section_params["0"] = { "@type" => self.attributes["#{key}_type"] }.merge(section_params["0"]) + end + next if section_params.blank? + section_params.each do |index, _section_params| + sub_attrs, sub_elements = parse_attributes(_section_params) + elements << config_element(key, "", sub_attrs, sub_elements) + end + end + return params.to_h.reject{|key, value| skip?(key.to_sym, value) }, elements + end + + # copy from Fluent::Test::Helpers#config_element + def config_element(name = 'test', argument = '', params = {}, elements = []) + Fluent::Config::Element.new(name, argument, params, elements) + end + + def skip?(key, value) + return true if value.blank? + if self._defaults.key?(key) + reformat_value(key, self._defaults[key]) == reformat_value(key, value) + else + false + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb new file mode 100644 index 000000000..62367fd4c --- /dev/null +++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb @@ -0,0 +1,121 @@ +class Fluentd + module Setting + module PluginParameter + extend ActiveSupport::Concern + + include Fluentd::Setting::Configurable + + def column_type(name) + self.class._types[name] + end + + def list_of(name) + self.class._list[name] + end + + def desc(name) + self.class._descriptions[name] + end + + def default(name) + reformat_value(name, self.class._defaults[name]) + end + + def common_options + [] + end + + def advanced_options + all_options - common_options - hidden_options + end + + def hidden_options + [] + end + + def all_options + self.class._types.keys + self.class._sections.keys + end + + def have_buffer_section? + self.class._sections.key?(:buffer) + end + + def have_storage_section? + self.class._sections.key?(:storage) + end + + def have_parse_section? + self.class._sections.key?(:parse) + end + + def have_format_section? + self.class._sections.key?(:format) + end + + def create_buffer + return unless have_buffer_section? + buffer_class = Fluentd::Setting.const_get("buffer_#{buffer_type}".classify) + buffer_class.new(buffer["0"].except("type")) + end + + def create_storage + return unless have_storage_section? + storage_class = Fluentd::Setting.const_get("storage_#{storage_type}".classify) + storage_class.new(storage["0"].except("type")) + end + + def create_parser + return unless have_parse_section? + parser_class = Fluentd::Setting.const_get("parser_#{parse_type}".classify) + parser_class.new(parse["0"].except("type")) + end + + def create_formatter + return unless have_format_section? + formatter_class = Fluentd::Setting.const_get("formatter_#{format_type}".classify) + formatter_class.new(format["0"].except("type")) + end + + def reformat_value(name, value) + type = column_type(name) + return value if type.nil? # name == :time_key + return value if type == :enum + return value if type == :regexp + type_name = if type.is_a?(Fluentd::Setting::Type::Time) + :time + else + type + end + Fluent::Config::REFORMAT_VALUE.call(type_name, value) + end + + module ClassMethods + def column_type(name) + self._types[name] + end + + def list_of(name) + self._list[name] + end + + def permit_params + self.new # init + keys = self._types.keys + self._sections.each do |key, section| + keys << _permit_section(key, section) + end + keys + end + + def _permit_section(key, section) + keys = { key => section._types.keys } + section._sections.each do |_key, _section| + keys[key] << _permit_section(_key, _section) + end + keys + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/section_parser.rb b/app/models/concerns/fluentd/setting/section_parser.rb new file mode 100644 index 000000000..59396e62d --- /dev/null +++ b/app/models/concerns/fluentd/setting/section_parser.rb @@ -0,0 +1,36 @@ +class Fluentd + module Setting + module SectionParser + extend ActiveSupport::Concern + + module ClassMethods + def parse_section(name, definition) + config_section(name, **definition.slice(:required, :multi, :alias)) do + definition.except(:section, :argument, :required, :multi, :alias).each do |_param_name, _definition| + if _definition[:section] + parse_section(_param_name, _definition) + else + if self._types.key?(_param_name) + if _definition.key?(:default) + self._defaults[_param_name] = _definition[:default] + self._required[_param_name] = false + self.clear_validators! # We register PresenceValidator only + end + self._secrets[_param_name] = _definition[:secret] if _definition.key?(:secret) + self._aliases[name] = _definition[:alias] if _definition.key?(:alias) + self._deprecated_params[name] = _definition[:deprecated] if _definition.key?(:deprecated) + self._obsoleted_params[name] = _definition[:obsoleted] if _definition.key?(:obsoleted) + self._list[name] = _definition[:list] if _definition.key?(:list) + self._value_types[name] = _definition[:value_types] if _definition.key?(:value_types) + self._symbolize_keys = _definition[:symbolize_keys] if _definition.key?(:symbolize_keys) + else + config_param(_param_name, _definition[:type], **_definition.except(:type)) + end + end + end + end + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/section_validator.rb b/app/models/concerns/fluentd/setting/section_validator.rb new file mode 100644 index 000000000..8238441b8 --- /dev/null +++ b/app/models/concerns/fluentd/setting/section_validator.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + module SectionValidator + extend ActiveSupport::Concern + + included do + validate :validate_sections + end + + def validate_sections + self._section_params.each do |name, sections| + sections.each do |section| + if section.invalid? + errors.add(name, :invalid, message: section.errors.full_messages) + end + end + end + end + end + end +end diff --git a/app/models/fluentd.rb b/app/models/fluentd.rb index d29b82b4c..695d68d43 100644 --- a/app/models/fluentd.rb +++ b/app/models/fluentd.rb @@ -14,28 +14,28 @@ class Fluentd DEFAULT_CONF = <<-CONF.strip_heredoc # http://docs.fluentd.org/articles/in_forward - type forward + @type forward port 24224 # http://docs.fluentd.org/articles/in_http - type http + @type http port 9880 - type monitor_agent + @type monitor_agent port 24220 - type debug_agent + @type debug_agent port 24230 # http://docs.fluentd.org/articles/out_stdout - type stdout + @type stdout CONF diff --git a/app/models/fluentd/setting/buffer_file.rb b/app/models/fluentd/setting/buffer_file.rb new file mode 100644 index 000000000..6be83f0ad --- /dev/null +++ b/app/models/fluentd/setting/buffer_file.rb @@ -0,0 +1,23 @@ +class Fluentd + module Setting + class BufferFile + include Fluentd::Setting::Plugin + + register_plugin("buffer", "file") + + def self.initial_params + { + path: "" + } + end + + def common_options + [ + :path, + :file_permission, + :dir_permission + ] + end + end + end +end diff --git a/app/models/fluentd/setting/buffer_memory.rb b/app/models/fluentd/setting/buffer_memory.rb new file mode 100644 index 000000000..9abe3a9f3 --- /dev/null +++ b/app/models/fluentd/setting/buffer_memory.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + class BufferMemory + include Fluentd::Setting::Plugin + + register_plugin("buffer", "memory") + + def self.initial_params + {} + end + + def common_options + [] + end + end + end +end diff --git a/app/models/fluentd/setting/common.rb b/app/models/fluentd/setting/common.rb deleted file mode 100644 index faadc8e25..000000000 --- a/app/models/fluentd/setting/common.rb +++ /dev/null @@ -1,185 +0,0 @@ -class Fluentd - module Setting - module Common - extend ActiveSupport::Concern - include ActiveModel::Model - - module ClassMethods - attr_accessor :values, :types, :children, :hidden_values - - def choice(key, values) - @values ||= {} - @values[key] = values - set_type(:choice, [key]) - end - - def hidden(key) - set_type(:hidden, [key]) - end - - def nested(key, klass, options = {}) - # e.g.: - # - # type forward - # - # .. - # - # - @children ||= {} - @children[key] = { - class: klass, - options: options, - } - set_type(:nested, [key]) - end - - def booleans(*keys) - # e.g.: - # use_ssl true - # include_time_key false - set_type(:boolean, keys) - end - - def flags(*keys) - # e.g.: - # tag_mapped - # utc - set_type(:flag, keys) - end - - private - - def set_type(type, keys) - @types ||= {} - keys.each do |key| - @types[key] = type - end - end - end - - def children_of(key) - meta = self.class.children[key] - return unless meta - klass = meta[:class] - data = send(key) || {"0" => {}} - children = [] - data.each_pair do |index, attrs| - children << klass.new(attrs) - end - children - end - - def child_class(key) - self.class.children[key][:class] - end - - def values_of(key) - self.class.values.try(:[], key) || [] - end - - def column_type(key) - self.class.types.try(:[], key) || "string" - end - - def conf(key) - case column_type(key) - when :boolean - boolenan(key) - when :flag - flag(key) - when :nested - return "" unless send(key) - klass = child_class(key) - send(key).map do |(_, child)| - # send("servers") - # - # "servers" => { - # "0" => { - # "name" => "foo", - # "host" => "bar", - # .. - # }, - # "1" => { - # .. - # } - # } - child_instance = klass.new(child) - unless child_instance.empty_value? - "\n" + child_instance.to_config(key).gsub(/^/m, " ") - end - end.join - else # including :hidden - print_if_present(key) - end - end - - def plugin_type_name - # Fluentd::Setting::OutS3 -> s3 - # Override this method if not above style - try(:plugin_name) || self.class.to_s.split("::").last.sub(/(In|Out)/, "").downcase - end - - def print_if_present(key) - # e.g.: - # path /var/log/td/aaa - # user nobody - # retry_limit 3 - send(key).present? ? "#{key} #{send(key)}" : "" - end - - def boolenan(key) - send(key).presence == "true" ? "#{key} true" : "#{key} false" - end - - def flag(key) - send(key).presence == "true" ? key.to_s : "" - end - - def empty_value? - config = "" - self.class.const_get(:KEYS).each do |key| - config << conf(key) - end - config.empty? - end - - def input_plugin? - self.class.to_s.match(/::In|^In/) - end - - def output_plugin? - not input_plugin? - end - - def to_config(elm_name = nil) - indent = " " - if elm_name - config = "<#{elm_name}>\n" - else - if input_plugin? - config = "\n" - else - config = "\n" - end - config << "#{indent}type #{plugin_type_name}\n" - end - self.class.const_get(:KEYS).each do |key| - next if key == :match - config << indent - config << conf(key) - config << "\n" - end - if elm_name - config << "\n" - else - if input_plugin? - config << "\n" - else - config << "\n" - end - end - config.gsub(/^[ ]*\n/m, "") - end - end - end -end diff --git a/app/models/fluentd/setting/formatter_csv.rb b/app/models/fluentd/setting/formatter_csv.rb new file mode 100644 index 000000000..c567f0ef6 --- /dev/null +++ b/app/models/fluentd/setting/formatter_csv.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + class FormatterCsv + include Fluentd::Setting::Plugin + + register_plugin("formatter", "csv") + + def self.initial_params + {} + end + + def common_options + [ + :delimiter, + :fields, + :add_newline + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_hash.rb b/app/models/fluentd/setting/formatter_hash.rb new file mode 100644 index 000000000..dc61f45fc --- /dev/null +++ b/app/models/fluentd/setting/formatter_hash.rb @@ -0,0 +1,19 @@ +class Fluentd + module Setting + class FormatterHash + include Fluentd::Setting::Plugin + + register_plugin("formatter", "hash") + + def self.initial_params + {} + end + + def common_options + [ + :add_newline + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_json.rb b/app/models/fluentd/setting/formatter_json.rb new file mode 100644 index 000000000..87c2d4b6f --- /dev/null +++ b/app/models/fluentd/setting/formatter_json.rb @@ -0,0 +1,25 @@ +class Fluentd + module Setting + class FormatterJson + include Fluentd::Setting::Plugin + + register_plugin("formatter", "json") + + def self.initial_params + {} + end + + def common_options + [ + :add_newline + ] + end + + def advanced_options + [ + :json_parser + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_ltsv.rb b/app/models/fluentd/setting/formatter_ltsv.rb new file mode 100644 index 000000000..84189a13b --- /dev/null +++ b/app/models/fluentd/setting/formatter_ltsv.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + class FormatterLtsv + include Fluentd::Setting::Plugin + + register_plugin("formatter", "ltsv") + + def self.initial_params + {} + end + + def common_options + [ + :delimiter, + :label_delimiter, + :add_newline + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_msgpack.rb b/app/models/fluentd/setting/formatter_msgpack.rb new file mode 100644 index 000000000..482cfb406 --- /dev/null +++ b/app/models/fluentd/setting/formatter_msgpack.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class FormatterMsgpack + include Fluentd::Setting::Plugin + + register_plugin("formatter", "msgpack") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_out_file.rb b/app/models/fluentd/setting/formatter_out_file.rb new file mode 100644 index 000000000..40239bdf1 --- /dev/null +++ b/app/models/fluentd/setting/formatter_out_file.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + class FormatterOutFile + include Fluentd::Setting::Plugin + + register_plugin("formatter", "out_file") + + def self.initial_params + {} + end + + def common_options + [ + :output_time, + :output_tag, + :delimiter + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_single_value.rb b/app/models/fluentd/setting/formatter_single_value.rb new file mode 100644 index 000000000..9839093ea --- /dev/null +++ b/app/models/fluentd/setting/formatter_single_value.rb @@ -0,0 +1,20 @@ +class Fluentd + module Setting + class FormatterSingleValue + include Fluentd::Setting::Plugin + + register_plugin("formatter", "single_value") + + def self.initial_params + {} + end + + def common_options + [ + :message_key, + :add_newline + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_stdout.rb b/app/models/fluentd/setting/formatter_stdout.rb new file mode 100644 index 000000000..1e392370c --- /dev/null +++ b/app/models/fluentd/setting/formatter_stdout.rb @@ -0,0 +1,19 @@ +class Fluentd + module Setting + class FormatterStdout + include Fluentd::Setting::Plugin + + register_plugin("formatter", "stdout") + + def self.initial_params + {} + end + + def common_options + [ + :output_type + ] + end + end + end +end diff --git a/app/models/fluentd/setting/formatter_tsv.rb b/app/models/fluentd/setting/formatter_tsv.rb new file mode 100644 index 000000000..24c0b1f17 --- /dev/null +++ b/app/models/fluentd/setting/formatter_tsv.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + class FormatterTsv + include Fluentd::Setting::Plugin + + register_plugin("formatter", "tsv") + + def self.initial_params + {} + end + + def common_options + [ + :keys, + :delimiter, + :add_newline + ] + end + end + end +end diff --git a/app/models/fluentd/setting/in_forward.rb b/app/models/fluentd/setting/in_forward.rb index 70627ad5a..db1e5e3fb 100644 --- a/app/models/fluentd/setting/in_forward.rb +++ b/app/models/fluentd/setting/in_forward.rb @@ -1,17 +1,9 @@ class Fluentd module Setting class InForward - include ActiveModel::Model - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :bind, :port, :linger_timeout, :chunk_size_limit, :chunk_size_warn_limit, :log_level - ].freeze - - attr_accessor(*KEYS) - - validates :bind, presence: true - validates :port, presence: true + register_plugin("input", "forward") def self.initial_params { @@ -20,7 +12,6 @@ def self.initial_params linger_timeout: 0, chunk_size_limit: nil, chunk_size_warn_limit: nil, - log_level: "info", } end @@ -30,15 +21,14 @@ def common_options ] end - def advanced_options + def hidden_options [ - :linger_timeout, :chunk_size_limit, :chunk_size_warn_limit, :log_level + # We don't support TLS configuration via fluentd-ui for now. + :transport, + :backlog, + :blocking_timeout, ] end - - def plugin_name - "forward" - end end end end diff --git a/app/models/fluentd/setting/in_http.rb b/app/models/fluentd/setting/in_http.rb index 788a37dda..be3052880 100644 --- a/app/models/fluentd/setting/in_http.rb +++ b/app/models/fluentd/setting/in_http.rb @@ -1,17 +1,9 @@ class Fluentd module Setting class InHttp - include ActiveModel::Model - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :bind, :port, :body_size_limit, :keepalive_timeout, :add_http_headers, :format, :log_level - ].freeze - - attr_accessor(*KEYS) - - validates :bind, presence: true - validates :port, presence: true + register_plugin("input", "http") def self.initial_params { @@ -20,26 +12,28 @@ def self.initial_params body_size_limit: "32m", keepalive_timeout: "10s", add_http_headers: false, - format: "default", - log_level: "info", + parse_type: "in_http", + parse: { + "0" => { + "type" => "in_http" + } + } } end def common_options [ - :bind, :port + :bind, :port, :add_http_headers, :add_remote_addr ] end - def advanced_options + def hidden_options [ - :body_size_limit, :keepalive_timeout, :add_http_headers, :format, :log_level + :parse, + :backlog, + :blocking_timeout, ] end - - def plugin_name - "http" - end end end end diff --git a/app/models/fluentd/setting/in_monitor_agent.rb b/app/models/fluentd/setting/in_monitor_agent.rb index 672b604a1..6e6167ca4 100644 --- a/app/models/fluentd/setting/in_monitor_agent.rb +++ b/app/models/fluentd/setting/in_monitor_agent.rb @@ -1,38 +1,25 @@ class Fluentd module Setting class InMonitorAgent - include ActiveModel::Model - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :bind, :port - ].freeze - - attr_accessor(*KEYS) - - validates :bind, presence: true - validates :port, presence: true + register_plugin("input", "monitor_agent") def self.initial_params { bind: "0.0.0.0", port: 24220, + emit_interval: 60, + include_config: true, + include_retry: true } end def common_options [ - :bind, :port + :bind, :port, :tag ] end - - def advanced_options - [] - end - - def plugin_name - "monitor_agent" - end end end end diff --git a/app/models/fluentd/setting/in_syslog.rb b/app/models/fluentd/setting/in_syslog.rb index 2fef577ff..8ed0f9d8c 100644 --- a/app/models/fluentd/setting/in_syslog.rb +++ b/app/models/fluentd/setting/in_syslog.rb @@ -1,32 +1,34 @@ class Fluentd module Setting class InSyslog - include ActiveModel::Model - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :port, :bind, :tag, :types - ].freeze - - attr_accessor(*KEYS) - - validates :tag, presence: true + register_plugin("input", "syslog") def self.initial_params { bind: "0.0.0.0", port: 5140, + parse_type: "syslog", + parse: { + "0" => { + "type" => "syslog" + } + }, + protocol_type: :udp, } end def common_options [ - :tag, :bind, :port, :types, + :tag, :bind, :port ] end - def advanced_options - [] + def hidden_options + [ + :parse + ] end end end diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb index 90084c1fc..89de568d0 100644 --- a/app/models/fluentd/setting/in_tail.rb +++ b/app/models/fluentd/setting/in_tail.rb @@ -1,38 +1,16 @@ +require "fluent/plugin/parser_multiline" + class Fluentd module Setting class InTail - MULTI_LINE_MAX_FORMAT_COUNT = 20 - - include ActiveModel::Model - attr_accessor :path, :tag, :format, :regexp, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval - - validates :path, presence: true - validates :tag, presence: true - #validates :format, presence: true + include Fluentd::Setting::Plugin - def self.known_formats - { - :apache2 => [:time_format], - :nginx => [:time_format], - :syslog => [:time_format], - :tsv => [:keys, :time_key], - :csv => [:keys, :time_key], - :ltsv => [:delimiter, :time_key], - :json => [:time_key], - :regexp => [:time_format, :regexp], - :multiline => [:format_firstline] + (1..MULTI_LINE_MAX_FORMAT_COUNT).map{|n| "format#{n}".to_sym } - # TODO: Grok could generate Regexp including \d, \s, etc. fluentd config parser raise error with them for escape sequence check. - # TBD How to handle Grok/Regexp later, just comment out for hide - # :grok => [:grok_str], - } - end - attr_accessor *known_formats.values.flatten.compact.uniq + register_plugin("input", "tail") + # TODO support formatN ??? - def known_formats - self.class.known_formats - end + MULTI_LINE_MAX_FORMAT_COUNT = ::Fluent::Plugin::MultilineParser::FORMAT_MAX_NUM - def guess_format + def guess_parse_type case path when /\.json$/ :json @@ -53,43 +31,6 @@ def guess_format end end - def extra_format_options - self.class.known_formats[format.to_sym] || [] - end - - def format_specific_conf - return "" if %w(grok regexp).include?(format) - - indent = " " * 2 - format_specific_conf = "" - - if format.to_sym == :multiline - known_formats[:multiline].each do |key| - value = send(key) - if value.present? - format_specific_conf << "#{indent}#{key} /#{value}/\n" - end - end - else - extra_format_options.each do |key| - format_specific_conf << "#{indent}#{key} #{send(key)}\n" - end - end - - format_specific_conf - end - - def certain_format_line - case format - when "grok" - "format /#{grok.convert_to_regexp(grok_str).source.gsub("/", "\\/")}/ # grok: '#{grok_str}'" - when "regexp" - "format /#{regexp}/" - else - "format #{format}" - end - end - def grok @grok ||= begin @@ -98,24 +39,6 @@ def grok grok end end - - def to_conf - # NOTE: Using strip_heredoc makes more complex for format_specific_conf indent - <<-CONFIG.gsub(/^[ ]*\n/m, "") - - type tail - path #{path} - tag #{tag} - #{certain_format_line} -#{format_specific_conf} - - #{read_from_head.to_i.zero? ? "" : "read_from_head true"} - #{pos_file.present? ? "pos_file #{pos_file}" : ""} - #{rotate_wait.present? ? "rotate_wait #{rotate_wait}" : ""} - #{refresh_interval.present? ? "refresh_interval #{refresh_interval}" : ""} - - CONFIG - end end end end diff --git a/app/models/fluentd/setting/out_elasticsearch.rb b/app/models/fluentd/setting/out_elasticsearch.rb index bf9e78926..b1566f38b 100644 --- a/app/models/fluentd/setting/out_elasticsearch.rb +++ b/app/models/fluentd/setting/out_elasticsearch.rb @@ -1,24 +1,9 @@ class Fluentd module Setting class OutElasticsearch - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :match, - :host, :port, :index_name, :type_name, - :logstash_format, :logstash_prefix, :logstash_dateformat, :utc_index, - :hosts, :request_timeout, :include_tag_key - ].freeze - - attr_accessor(*KEYS) - - booleans :logstash_format, :utc_index, :include_tag_key - - validates :match, presence: true - validates :host, presence: true - validates :port, presence: true - validates :index_name, presence: true - validates :type_name, presence: true + register_plugin("output", "elasticsearch") def self.initial_params { @@ -29,20 +14,26 @@ def self.initial_params logstash_format: true, include_tag_key: false, utc_index: true, + buffer_type: "file", + buffer: { + "0" => { + "type" => "file", + "path" => "/var/log/td-agent/buffer/elasticsearch", + } + }, } end def common_options [ - :match, :host, :port, :logstash_format, + :pattern, :host, :port, :logstash_format, :index_name, :type_name, ] end - def advanced_options + def hidden_options [ - :hosts, :logstash_prefix, :logstash_dateformat, - :utc_index, :request_timeout, :include_tag_key, + :secondary, :inject, :buffer ] end end diff --git a/app/models/fluentd/setting/out_forward.rb b/app/models/fluentd/setting/out_forward.rb index 1b259901d..d6bdcc350 100644 --- a/app/models/fluentd/setting/out_forward.rb +++ b/app/models/fluentd/setting/out_forward.rb @@ -1,91 +1,47 @@ class Fluentd module Setting class OutForward - class Server - include Common - KEYS = [ - :name, :host, :port, :weight, :standby - ].freeze + include Fluentd::Setting::Plugin - attr_accessor(*KEYS) + register_plugin("output", "forward") - flags :standby - - validates :host, presence: true - validates :port, presence: true - end - - class Secondary - include Common - KEYS = [ - :type, :path - ].freeze - - attr_accessor(*KEYS) - - hidden :type - validates :path, presence: true - end - - include Common - - KEYS = [ - :match, - :send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval, - :phi_threshold, :hard_timeout, - :server, :secondary - ].freeze - - attr_accessor(*KEYS) - choice :heartbeat_type, %w(udp tcp) - nested :server, Server, multiple: true - nested :secondary, Secondary - - validates :match, presence: true - validate :validate_has_at_least_one_server - validate :validate_nested_values - - def validate_has_at_least_one_server - if children_of(:server).reject{|s| s.empty_value? }.blank? - errors.add(:base, :out_forward_blank_server) - end - end - - def validate_nested_values - self.class.children.inject(true) do |result, (key, _)| - children_of(key).each do |child| - if !child.empty_value? && !child.valid? - child.errors.full_messages.each do |message| - errors.add(:base, "(#{key})#{message}") - end - result = false - end - result - end - result - end + config_section :secondary do + config_param :path, :string end def self.initial_params { + buffer_type: "memory", + buffer: { + "0" => { + "type" => "memory", + } + }, secondary: { "0" => { - type: "file", + "type" => "file", } } } end + # TODO overwrite this method to support transport parameter and transport section + # def self.permit_params + # super + # end + def common_options [ - :match, :server, :secondary, + :pattern, :server, :secondary, ] end - def advanced_options + def hidden_options [ - :send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval, - :phi_threshold, :hard_timeout, + :inject, :buffer, + :host, :port, + # We don't support TLS configuration via fluentd-ui for now. + :transport, :tls_version, :tls_ciphers, :tls_insecure_mode, :tls_verify_hostname, :tls_cert_path ] end end diff --git a/app/models/fluentd/setting/out_mongo.rb b/app/models/fluentd/setting/out_mongo.rb index 62093f48e..6338cb91d 100644 --- a/app/models/fluentd/setting/out_mongo.rb +++ b/app/models/fluentd/setting/out_mongo.rb @@ -1,34 +1,20 @@ class Fluentd module Setting class OutMongo - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :match, - :host, :port, :database, :collection, :capped, :capped_size, :capped_max, :user, :password, :tag_mapped, - :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval, :retry_wait, :retry_limit, :max_retry_wait, :num_threads, - ].freeze + register_plugin("output", "mongo") + config_param(:capped, :bool, default: false) + config_param(:capped_size, :size, default: nil) - attr_accessor(*KEYS) - - flags :capped, :tag_mapped - - validates :match, presence: true - validates :host, presence: true - validates :port, presence: true + # NOTE: fluent-plugin-mongo defines database parameter as required parameter + # But Fluentd tells us that the database parameter is not required. validates :database, presence: true - validate :validate_capped validate :validate_collection - validates :buffer_path, presence: true, if: ->{ buffer_type == "file" } - - def validate_capped - return true if capped.blank? - errors.add(:capped_size, :blank) if capped_size.blank? - end def validate_collection if tag_mapped.blank? && collection.blank? - errors.add(:collection, :blank) + errors.add(:collection, :blank) end end @@ -38,20 +24,28 @@ def self.initial_params port: 27017, capped: true, capped_size: "100m", - } + buffer_type: "file", + buffer: { + "0" => { + "type" => "file", + "path" => "/var/log/td-agent/buffer/mongo", + } + }, + } end def common_options [ - :match, :host, :port, :database, :collection, + :pattern, :host, :port, :database, :collection, :tag_mapped, :user, :password, ] end - def advanced_options + def hidden_options [ - :capped, :capped_size, :capped_max, :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, - :flush_interval, :retry_wait, :retry_limit, :max_retry_wait, :num_threads, + :secondary, :inject, :buffer, + :include_tag_key, + :include_time_key ] end end diff --git a/app/models/fluentd/setting/out_s3.rb b/app/models/fluentd/setting/out_s3.rb index 28fdc3985..fedca81aa 100644 --- a/app/models/fluentd/setting/out_s3.rb +++ b/app/models/fluentd/setting/out_s3.rb @@ -1,51 +1,39 @@ class Fluentd module Setting class OutS3 - include ActiveModel::Model - include Common + include Fluentd::Setting::Plugin - KEYS = [ - :match, - :aws_key_id, :aws_sec_key, :s3_bucket, :s3_region, :path, - # :reduced_redundancy, :check_apikey_on_start, :command_parameter, # not configurable? - :format, :include_time_key, :time_key, :delimiter, :label_delimiter, - :time_slice_format, :time_slice_wait, :time_format, :utc, :store_as, :proxy_uri, :use_ssl, - :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval, - :retry_wait, :retry_limit, :max_retry_wait, :num_threads, - ].freeze - - attr_accessor(*KEYS) - - choice :format, %w(out_file json ltsv single_value) - choice :store_as, %w(gzip lzo lzma2 json txt) - choice :buffer_type, %w(memory file) - booleans :include_time_key, :use_ssl - flags :utc - - validates :match, presence: true - validates :s3_bucket, presence: true - validates :buffer_path, presence: true, if: ->{ buffer_type == "file" } + register_plugin("output", "s3") def self.initial_params { s3_region: "us-west-1", - use_ssl: true, + buffer_type: "file", + buffer: { + "0" => { + "type" => "file", + "path" => "/var/log/td-agent/buffer/s3", + } + }, + format_type: "out_file", + format: { + "0" => { + "type" => "out_file" + } + } } end def common_options [ - :match, :aws_key_id, :aws_sec_key, + :pattern, :aws_key_id, :aws_sec_key, :s3_region, :s3_bucket, :use_ssl, :path, ] end - def advanced_options + def hidden [ - :format, :include_time_key, :time_key, :delimiter, :label_delimiter, - :utc, :time_slice_format, :time_slice_wait, :store_as, :proxy_uri, - :buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval, - :retry_wait, :retry_limit, :max_retry_wait, :num_threads, + :secondary, :inject, :buffer ] end end diff --git a/app/models/fluentd/setting/out_stdout.rb b/app/models/fluentd/setting/out_stdout.rb index 51ba3196a..0337b97f0 100644 --- a/app/models/fluentd/setting/out_stdout.rb +++ b/app/models/fluentd/setting/out_stdout.rb @@ -1,36 +1,39 @@ class Fluentd module Setting class OutStdout - include Common + include Fluentd::Setting::Plugin - KEYS = [:match, :output_type].freeze - - attr_accessor(*KEYS) - - choice :output_type, %w(json hash) - - validates :match, presence: true - validates :output_type, inclusion: { in: %w(json hash) } + register_plugin("output", "stdout") def self.initial_params { - match: "debug.**", - output_type: "json", + pattern: "debug.**", + buffer_type: "memory", + buffer: { + "0" => { + "type" => "memory", + } + }, + format_type: "stdout", + format: { + "0" => { + "@type" => "stdout", + "output_type" => "json" + } + } } end def common_options [ - :match, :output_type + :pattern, :output_type ] end - def advanced_options - [] - end - - def plugin_name - "stdout" + def hidden_options + [ + :secondary, :inject, :buffer + ] end end end diff --git a/app/models/fluentd/setting/out_td.rb b/app/models/fluentd/setting/out_td.rb deleted file mode 100644 index 38554ffd4..000000000 --- a/app/models/fluentd/setting/out_td.rb +++ /dev/null @@ -1,48 +0,0 @@ -class Fluentd - module Setting - class OutTd - include ActiveModel::Model - include Common - - KEYS = [ - :match, - :apikey, :auto_create_table, :database, :table, - :flush_interval, :buffer_type, :buffer_path, - ].freeze - - attr_accessor(*KEYS) - - flags :auto_create_table - - validates :match, presence: true - validates :apikey, presence: true - validates :auto_create_table, presence: true - validates :buffer_path, presence: true, if: ->{ buffer_type == "file" } - - def plugin_name - "tdlog" - end - - def self.initial_params - { - buffer_type: "file", - buffer_path: "/var/log/td-agent/buffer/td", - auto_create_table: true, - match: "td.*.*", - } - end - - def common_options - [ - :match, :apikey, :auto_create_table, :database, :table, - ] - end - - def advanced_options - [ - :flush_interval, :buffer_type, :buffer_path, - ] - end - end - end -end diff --git a/app/models/fluentd/setting/out_tdlog.rb b/app/models/fluentd/setting/out_tdlog.rb new file mode 100644 index 000000000..65177a7a8 --- /dev/null +++ b/app/models/fluentd/setting/out_tdlog.rb @@ -0,0 +1,35 @@ +class Fluentd + module Setting + class OutTdlog + include Fluentd::Setting::Plugin + + register_plugin("output", "tdlog") + + def self.initial_params + { + pattern: "td.*.*", + buffer_type: "file", + buffer: { + "0" => { + "@type" => "file", + "path" => "/var/log/td-agent/buffer/td", + } + }, + auto_create_table: true, + } + end + + def common_options + [ + :pattern, :apikey, :auto_create_table, :database, :table, + ] + end + + def hidden_options + [ + :secondary + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_apache.rb b/app/models/fluentd/setting/parser_apache.rb new file mode 100644 index 000000000..6bdf6c498 --- /dev/null +++ b/app/models/fluentd/setting/parser_apache.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class ParserApache + include Fluentd::Setting::Plugin + + register_plugin("parser", "apache") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/parser_apache2.rb b/app/models/fluentd/setting/parser_apache2.rb new file mode 100644 index 000000000..76480ae6b --- /dev/null +++ b/app/models/fluentd/setting/parser_apache2.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class ParserApache2 + include Fluentd::Setting::Plugin + + register_plugin("parser", "apache2") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/parser_apache_error.rb b/app/models/fluentd/setting/parser_apache_error.rb new file mode 100644 index 000000000..43fa488ee --- /dev/null +++ b/app/models/fluentd/setting/parser_apache_error.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class ParserApacheError + include Fluentd::Setting::Plugin + + register_plugin("parser", "apache_error") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/parser_csv.rb b/app/models/fluentd/setting/parser_csv.rb new file mode 100644 index 000000000..45ee3220d --- /dev/null +++ b/app/models/fluentd/setting/parser_csv.rb @@ -0,0 +1,22 @@ +class Fluentd + module Setting + class ParserCsv + include Fluentd::Setting::Plugin + + register_plugin("parser", "csv") + + def self.initial_params + { + keys: nil, + delimiter: "," + } + end + + def common_options + [ + :keys, :delimiter + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_in_http.rb b/app/models/fluentd/setting/parser_in_http.rb new file mode 100644 index 000000000..b48bb143a --- /dev/null +++ b/app/models/fluentd/setting/parser_in_http.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class ParserInHttp + include Fluentd::Setting::Plugin + + register_plugin("parser", "in_http") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/parser_json.rb b/app/models/fluentd/setting/parser_json.rb new file mode 100644 index 000000000..168c12739 --- /dev/null +++ b/app/models/fluentd/setting/parser_json.rb @@ -0,0 +1,25 @@ +class Fluentd + module Setting + class ParserJson + include Fluentd::Setting::Plugin + + register_plugin("parser", "json") + + def self.initial_params + { + json_parser: "oj" + } + end + + def common_options + [] + end + + def advanced_options + [ + :json_parser + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_ltsv.rb b/app/models/fluentd/setting/parser_ltsv.rb new file mode 100644 index 000000000..b4cc82937 --- /dev/null +++ b/app/models/fluentd/setting/parser_ltsv.rb @@ -0,0 +1,30 @@ +class Fluentd + module Setting + class ParserLtsv + include Fluentd::Setting::Plugin + + register_plugin("parser", "ltsv") + + def self.initial_params + { + delimiter: "\t", + delimiter_pattern: nil, + label_delimiter: ":" + } + end + + def common_options + [ + :delimiter, + :label_delimiter + ] + end + + def advanced_options + [ + :delimiter_pattern + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_msgpack.rb b/app/models/fluentd/setting/parser_msgpack.rb new file mode 100644 index 000000000..c89070ef0 --- /dev/null +++ b/app/models/fluentd/setting/parser_msgpack.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class ParserMsgpack + include Fluentd::Setting::Plugin + + register_plugin("parser", "msgpack") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/parser_multiline.rb b/app/models/fluentd/setting/parser_multiline.rb new file mode 100644 index 000000000..0cc7e551f --- /dev/null +++ b/app/models/fluentd/setting/parser_multiline.rb @@ -0,0 +1,24 @@ +class Fluentd + module Setting + class ParserMultiline + include Fluentd::Setting::Plugin + + register_plugin("parser", "multiline") + + FORMAT_MAX_NUM = 20 + + (1..FORMAT_MAX_NUM).each do |n| + config_param("format#{n}", :string) + end + + def self.initial_params + {} + end + + def common_options + [:format_firstline] + + (1..FORMAT_MAX_NUM).to_a.map{|n| "format#{n}".to_sym } + end + end + end +end diff --git a/app/models/fluentd/setting/parser_nginx.rb b/app/models/fluentd/setting/parser_nginx.rb new file mode 100644 index 000000000..f06d5e055 --- /dev/null +++ b/app/models/fluentd/setting/parser_nginx.rb @@ -0,0 +1,13 @@ +class Fluentd + module Setting + class ParserNginx + include Fluentd::Setting::Plugin + + register_plugin("parser", "nginx") + + def self.initial_params + {} + end + end + end +end diff --git a/app/models/fluentd/setting/parser_none.rb b/app/models/fluentd/setting/parser_none.rb new file mode 100644 index 000000000..d914200b2 --- /dev/null +++ b/app/models/fluentd/setting/parser_none.rb @@ -0,0 +1,19 @@ +class Fluentd + module Setting + class ParserNone + include Fluentd::Setting::Plugin + + register_plugin("parser", "none") + + def self.initial_params + {} + end + + def common_options + [ + :message_key + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_regexp.rb b/app/models/fluentd/setting/parser_regexp.rb new file mode 100644 index 000000000..b1b9f2355 --- /dev/null +++ b/app/models/fluentd/setting/parser_regexp.rb @@ -0,0 +1,26 @@ +class Fluentd + module Setting + class ParserRegexp + include Fluentd::Setting::Plugin + + register_plugin("parser", "regexp") + + def self.initial_params + {} + end + + def common_options + [ + :expression + ] + end + + def hidden_options + [ + :ignorecase, + :multiline + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_syslog.rb b/app/models/fluentd/setting/parser_syslog.rb new file mode 100644 index 000000000..d3f9eeee5 --- /dev/null +++ b/app/models/fluentd/setting/parser_syslog.rb @@ -0,0 +1,29 @@ +class Fluentd + module Setting + class ParserSyslog + include Fluentd::Setting::Plugin + + register_plugin("parser", "syslog") + # Overwrite type of time_format + config_param(:time_format, :string) + + def self.initial_params + {} + end + + def common_options + [ + :time_format, + :with_priority, + ] + end + + def advanced_options + [ + :message_format, + :rfc5424_time_format + ] + end + end + end +end diff --git a/app/models/fluentd/setting/parser_tsv.rb b/app/models/fluentd/setting/parser_tsv.rb new file mode 100644 index 000000000..fbb31258b --- /dev/null +++ b/app/models/fluentd/setting/parser_tsv.rb @@ -0,0 +1,19 @@ +class Fluentd + module Setting + class ParserTsv + include Fluentd::Setting::Plugin + + register_plugin("parser", "tsv") + + def self.initial_params + {} + end + + def common_options + [ + :keys, :delimiter + ] + end + end + end +end diff --git a/app/models/fluentd/setting/section.rb b/app/models/fluentd/setting/section.rb new file mode 100644 index 000000000..41b8f1001 --- /dev/null +++ b/app/models/fluentd/setting/section.rb @@ -0,0 +1,38 @@ +class Fluentd + module Setting + class Section + class << self + def inherited(klass) + klass.instance_eval do + include ActiveModel::Model + include ActiveModel::Attributes + include Fluentd::Setting::Configurable + include Fluentd::Setting::SectionParser + include Fluentd::Setting::PluginParameter + + class_attribute :_klass, :_block, :_blocks + class_attribute :section_name, :required, :multi, :alias + self._klass = klass + self._blocks = [] + end + end + + def init + _klass.instance_eval(&_block) + _blocks.each do |b| + _klass.instance_eval(&b) + end + end + + # Don't overwrite options + def merge(**options, &block) + _blocks << block + end + + def section? + true + end + end + end + end +end diff --git a/app/models/fluentd/setting/storage_local.rb b/app/models/fluentd/setting/storage_local.rb new file mode 100644 index 000000000..a6cc0f1f3 --- /dev/null +++ b/app/models/fluentd/setting/storage_local.rb @@ -0,0 +1,22 @@ +class Fluentd + module Setting + class StorageLocal + include Fluentd::Setting::Plugin + + register_plugin("storage", "local") + + def self.initial_params + {} + end + + def common_options + [ + :path, + :mode, + :dir_mode, + :pretty_print + ] + end + end + end +end diff --git a/app/models/fluentd/setting/type/array.rb b/app/models/fluentd/setting/type/array.rb new file mode 100644 index 000000000..e032fc449 --- /dev/null +++ b/app/models/fluentd/setting/type/array.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Array < ActiveModel::Type::Value + def type + :array + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/bool.rb b/app/models/fluentd/setting/type/bool.rb new file mode 100644 index 000000000..b953d9be4 --- /dev/null +++ b/app/models/fluentd/setting/type/bool.rb @@ -0,0 +1,18 @@ +class Fluentd + module Setting + module Type + class Bool < ActiveModel::Type::Value + def type + :bool + end + + private + + def cast_value(value) + # TODO Use type converter method of Fluentd + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/enum.rb b/app/models/fluentd/setting/type/enum.rb new file mode 100644 index 000000000..c2f2bd8fc --- /dev/null +++ b/app/models/fluentd/setting/type/enum.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Enum < ActiveModel::Type::Value + def type + :enum + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/hash.rb b/app/models/fluentd/setting/type/hash.rb new file mode 100644 index 000000000..cbdfbeb06 --- /dev/null +++ b/app/models/fluentd/setting/type/hash.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Hash < ActiveModel::Type::Value + def type + :hash + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/regexp.rb b/app/models/fluentd/setting/type/regexp.rb new file mode 100644 index 000000000..8210b14a4 --- /dev/null +++ b/app/models/fluentd/setting/type/regexp.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Regexp < ActiveModel::Type::Value + def type + :regexp + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/section.rb b/app/models/fluentd/setting/type/section.rb new file mode 100644 index 000000000..2ece24d32 --- /dev/null +++ b/app/models/fluentd/setting/type/section.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Section < ActiveModel::Type::Value + def type + :section + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/size.rb b/app/models/fluentd/setting/type/size.rb new file mode 100644 index 000000000..46d8dd7aa --- /dev/null +++ b/app/models/fluentd/setting/type/size.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Size < ActiveModel::Type::Value + def type + :size + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/models/fluentd/setting/type/time.rb b/app/models/fluentd/setting/type/time.rb new file mode 100644 index 000000000..80f1796b9 --- /dev/null +++ b/app/models/fluentd/setting/type/time.rb @@ -0,0 +1,17 @@ +class Fluentd + module Setting + module Type + class Time < ActiveModel::Type::Value + def type + :fluentd_time + end + + private + + def cast_value(value) + value + end + end + end + end +end diff --git a/app/views/api/settings/_element.json.jbuilder b/app/views/api/settings/_element.json.jbuilder index bc58a7d26..018ca1a69 100644 --- a/app/views/api/settings/_element.json.jbuilder +++ b/app/views/api/settings/_element.json.jbuilder @@ -1,6 +1,6 @@ json.id element_id(element) json.name element.name -json.type element["type"] +json.type element["@type"] || element["type"] json.arg element.arg json.settings element json.content element.to_s diff --git a/app/views/fluentd/settings/in_forward/_form.html.haml b/app/views/fluentd/settings/in_forward/_form.html.haml new file mode 100644 index 000000000..6ea374144 --- /dev/null +++ b/app/views/fluentd/settings/in_forward/_form.html.haml @@ -0,0 +1,16 @@ += render "shared/setting_errors" + +- # NOTE: plugin_setting_form_action_url is defined at SettingConcern += form_with(model: @setting, scope: "setting", url: plugin_setting_form_action_url(@fluentd), local: true, class: "ignore-rails-error-div", builder: FluentdFormBuilder) do |f| + - @setting.common_options.each do |key| + = f.field(key) + + .card.card-body.bg-light + %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"} + = icon('fa-caret-down') + = t('terms.advanced_setting') + #advanced-setting.collapse + - @setting.advanced_options.each do |key| + = f.field(key) + + = f.submit t('fluentd.common.finish'), class: "btn btn-lg btn-primary pull-right" diff --git a/app/views/fluentd/settings/in_tail/_form.html.haml b/app/views/fluentd/settings/in_tail/_form.html.haml index df3f1f523..cd789ee8a 100644 --- a/app/views/fluentd/settings/in_tail/_form.html.haml +++ b/app/views/fluentd/settings/in_tail/_form.html.haml @@ -1,42 +1,35 @@ -- @setting.errors.full_messages.each do |e| +- setting.errors.full_messages.each do |e| .alert.alert-danger= e .form-group - = f.label :path - = f.hidden_field :path - = f.text_field :path, class: "form-control", disabled: true + = form.label :path + = form.hidden_field :path + = form.text_field :path, class: "form-control", disabled: true .form-group - = f.label :format - = f.hidden_field :format - = f.text_field :format, class: "form-control", disabled: true -- if @setting.known_formats[@setting.format.to_sym] - - @setting.known_formats[@setting.format.to_sym].each do |key| - %label= key - = f.hidden_field key - = @setting.send(key) - %br -- else - %label= @setting.format - = f.hidden_field :regexp - = @setting.regexp -.form-group - = f.label :tag - = f.text_field :tag, class: "form-control" + = form.label :parse_type + = form.hidden_field :parse_type + = form.text_field :parse_type, class: "form-control", disabled: true + .form-group - = f.label :pos_file - = f.text_field :pos_file, class: "form-control" + %label= setting.parse_type + - parse_config = setting.create_parser + = hidden_field_tag("setting[parse[0]][type]", form.object.parse_type) + = form.fields("parse[0]", model: parse_config, builder: FluentdFormBuilder) do |f| + - f.object.common_options.each do |key| + = f.hidden_field(key) + = f.field(key, disabled: true) + += form.field(:tag) += form.field(:pos_file) + .well.well-sm %h4{"data-toggle" => "collapse", "href" => "#advanced-setting"} = icon('fa-caret-down') = t('terms.advanced_setting') #advanced-setting.collapse .form-group - = f.label :rotate_wait - = f.text_field :rotate_wait, class: "form-control" + = form.field(:rotate_wait) .form-group - = f.check_box :read_from_head - = f.label :read_from_head + = form.field(:read_from_head) .form-group - = f.label :refresh_interval - = f.text_field :refresh_interval, class: "form-control" - + = form.field(:refresh_interval) diff --git a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml index 74834d999..0e6db6506 100644 --- a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml +++ b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml @@ -10,7 +10,7 @@ = f.label :path = f.hidden_field :path = f.text_field :path, class: "form-control", disabled: true - = render partial: "shared/vue/in_tail_format", locals: { file: f.object.path, formats: @setting.known_formats, initialSelected: f.object.format || @setting.guess_format } + = render "shared/vue/in_tail_parse", setting: @setting .card %pre.card-body= file_tail(@setting.path, Settings.in_tail_preview_line_count).join("\n") diff --git a/app/views/fluentd/settings/in_tail/after_format.html.haml b/app/views/fluentd/settings/in_tail/after_format.html.haml index 4c521ff12..ed3ad219a 100644 --- a/app/views/fluentd/settings/in_tail/after_format.html.haml +++ b/app/views/fluentd/settings/in_tail/after_format.html.haml @@ -2,9 +2,9 @@ = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentds) -= form_for(@setting, as: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd)) do |f| - = render partial: "form", locals: { f: f } += form_with(model: @setting, scope: "setting", url: confirm_daemon_setting_in_tail_path(@fluentd), local: true, builder: FluentdFormBuilder) do |form| + = render partial: "form", locals: { form: form, setting: @setting } %p - = f.submit t('terms.next'), class: "btn btn-lg btn-primary float-right" - = f.submit t('terms.prev'), class: "btn btn-lg btn-secondary", name: "back" + = form.submit t('terms.next'), class: "btn btn-lg btn-primary float-right" + = form.submit t('terms.prev'), class: "btn btn-lg btn-secondary", name: "back" diff --git a/app/views/fluentd/settings/in_tail/confirm.html.haml b/app/views/fluentd/settings/in_tail/confirm.html.haml index 76748a1ef..cf0f7719e 100644 --- a/app/views/fluentd/settings/in_tail/confirm.html.haml +++ b/app/views/fluentd/settings/in_tail/confirm.html.haml @@ -2,12 +2,12 @@ = link_to t('fluentd.settings.in_tail.restart_from_first'), daemon_setting_in_tail_path(@fluentd) -= form_for(@setting, as: "setting", url: finish_daemon_setting_in_tail_path(@fluentd)) do |f| - = render partial: "form", locals: { f: f } += form_with(model: @setting, scope: "setting", url: finish_daemon_setting_in_tail_path(@fluentd), local: true, builder: FluentdFormBuilder) do |form| + = render partial: "form", locals: { form: form, setting: @setting } - %pre= @setting.to_conf + %pre= @setting.to_config %p - = f.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary float-right" - = f.submit t('terms.prev'), class: "btn btn-lg btn-secondary", name: "back" + = form.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary float-right" + = form.submit t('terms.prev'), class: "btn btn-lg btn-secondary", name: "back" .clearfix.float-right= t('terms.notice_restart_for_config_edit', brand: fluentd_ui_brand) diff --git a/app/views/fluentd/settings/source_and_output.html.haml b/app/views/fluentd/settings/source_and_output.html.haml index 309674751..8c1de68b4 100644 --- a/app/views/fluentd/settings/source_and_output.html.haml +++ b/app/views/fluentd/settings/source_and_output.html.haml @@ -23,7 +23,7 @@ .card-header %h4= t('.out') .card-body - - %w|stdout td s3 mongo elasticsearch forward|.each do |type| + - %w|stdout tdlog s3 mongo elasticsearch forward|.each do |type| %p = link_to(send("daemon_setting_out_#{type}_path", @fluentd)) do = icon('fa-file-text-o fa-lg') diff --git a/app/views/shared/_global_nav.html.erb b/app/views/shared/_global_nav.html.erb index 314a08dbe..be3b492a9 100644 --- a/app/views/shared/_global_nav.html.erb +++ b/app/views/shared/_global_nav.html.erb @@ -1,6 +1,6 @@