Merge pull request #209 from fluent/fix208

Fix #208 for multiline preview

lib/regexp_preview/multi_line.rb

@@ -27,32 +27,42 @@ def matches
       reader = FileReverseReader.new(File.open(file))
       result = []
       target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" }
-      target_lines.each_with_index do |line, line_no|
-        if line.match(params[:format_firstline])
-          lines = target_lines[line_no, patterns.length]
-          next if lines.length < patterns.length
-          ret = detect_chunk(lines)
-          next unless ret
-          result << ret
-        end
+      whole_string = target_lines.join
+      re_firstline = Regexp.new(params[:format_firstline])
+      indexes = []
+      cur = 0
+      while first_index = whole_string.index(re_firstline, cur)
+        indexes << first_index
+        cur = first_index + 1
+      end
+      indexes.each_with_index do |index, i|
+        next_index = indexes[i + 1] || -1
+        chunk = whole_string[index...next_index]
+        ret = detect_chunk(chunk)
+        next unless ret
+        result << ret
       end
       result
     end
 
-    def detect_chunk(lines)
+    def detect_chunk(chunk)
       whole = ""
       matches = []
-      lines.each_with_index do |line, i|
-        match = line.match(patterns[i])
+      offset = 0
+      patterns.each do |pat|
+        match = chunk.match(pat)
         return nil unless match
+        offset = chunk.index(pat)
+        return nil if offset > 0
+        chunk = chunk[match[0].length..-1]
         match.names.each_with_index do |name, index|
           matches << {
             key: name,
             matched: match[name],
             pos: match.offset(index + 1).map{|pos| pos + whole.length},
           }
         end
-        whole << line
+        whole << match[0]
       end
       {
         whole: whole,
@@ -63,7 +73,7 @@ def detect_chunk(lines)
     def patterns
       @patterns ||= (1..20).map do |n|
         params["format#{n}"].presence
-      end.compact.map {|pattern| Regexp.new(pattern)}
+      end.compact.map {|pattern| Regexp.new(pattern, Regexp::MULTILINE)}
     end
   end
 end

spec/lib/regexp_preview/multi_line_spec.rb

@@ -2,39 +2,115 @@
 
 describe RegexpPreview::MultiLine do
   describe "#matches_json" do
-    subject { RegexpPreview::MultiLine.new(File.expand_path("./spec/support/fixtures/error0.log", Rails.root), "multiline", params).matches_json }
-
-    let :params do
-      params = {
-        format_firstline: ".+",
-        time_format: "time_format",
-      }
-      params["format1"] = "(?<foo>foo)"
-      params["format2"] = "(?<bar>bar)"
-      3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-        params["format#{i}"] = ""
+    subject { parser.matches_json }
+    let(:parser) { RegexpPreview::MultiLine.new(target_path, "multiline", params) }
+
+    describe "simple usage" do
+      let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
+
+      let :params do
+        params = {
+          format_firstline: "foo",
+          time_format: "time_format",
+        }
+        params["format1"] = "(?<foo>foo)\n"
+        params["format2"] = "(?<bar>bar)"
+        3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
+          params["format#{i}"] = ""
+        end
+        { params: params }
+      end
+
+      it 'should not have regexp and time_format in [:params][:setting]' do
+        expect(subject[:params][:setting]).to eq(regexp: nil, time_format: nil)
+      end
+
+      it "should include matches info" do
+        matches_info = {
+          whole: "foo\nbar",
+          matches: [
+            {
+              key: "foo", matched: "foo", pos: [0, 3]
+            },
+            {
+              key: "bar", matched: "bar", pos: [4, 7]
+            }
+          ]
+        }
+
+        expect(subject[:matches]).to include matches_info
       end
-      { params: params }
     end
 
-    it 'should not have regexp and time_format in [:params][:setting]' do
-      expect(subject[:params][:setting]).to eq({ regexp: nil, time_format: nil })
+    describe "detect only continuos patterns" do
+      let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
+      let(:params) do
+        params = {
+          format_firstline: "foo",
+          time_format: "time_format",
+        }
+        params["format1"] = "(?<foo>foo)\n"
+        params["format2"] = "(?<bar>baz)"
+        3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
+          params["format#{i}"] = ""
+        end
+        { params: params }
+      end
+
+      it "shouldn't match" do
+        expect(subject[:matches]).to eq []
+      end
     end
 
-    it "should include matches info" do
-      matches_info = {
-        whole: "foo\nbar\n",
-        matches: [
-          {
-            key: "foo", matched: "foo", pos: [0, 3]
-          },
-          {
-            key: "bar", matched: "bar", pos: [4, 7]
-          }
-        ]
-      }
+    describe "example on document" do
+      # http://docs.fluentd.org/articles/in_tail
+      let(:target_path) { File.expand_path("./spec/support/fixtures/multiline_example.log", Rails.root) }
 
-      expect(subject[:matches]).to include matches_info
+      let :params do
+        params = {
+          format_firstline: "\\d{4}-\\d{1,2}-\\d{1,2}",
+          "format1" => "^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)",
+          time_format: "time_format",
+        }
+        2.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
+          params["format#{i}"] = ""
+        end
+        { params: params }
+      end
+
+      it "should include matches info" do
+        matches_info = 
+          [
+            {
+              whole: "2013-3-03 14:27:33 [main] INFO  Main - Start\n",
+              matches: [
+                {key: "time", matched: "2013-3-03 14:27:33", pos: [0, 18]},
+                {key: "thread", matched: "main", pos: [20, 24]},
+                {key: "level", matched: "INFO", pos: [26, 30]},
+                {key: "message", matched: "  Main - Start\n", pos: [30, 45]}
+              ]
+            },
+            {
+              whole: "2013-3-03 14:27:33 [main] ERROR Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n",
+              matches: [
+                {key: "time", matched: "2013-3-03 14:27:33", pos: [0, 18]},
+                {key: "thread", matched: "main", pos: [20, 24]},
+                {key: "level", matched: "ERROR", pos: [26, 31]},
+                {key: "message", matched: " Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n", pos: [31, 136]},
+              ]
+            },
+            {
+              whole: "2013-3-03 14:27:33 [main] INFO  Main - End",
+              matches: [
+                {key: "time", matched: "2013-3-03 14:27:33", pos: [0, 18]},
+                {key: "thread", matched: "main", pos: [20, 24]},
+                {key: "level", matched: "INFO", pos: [26, 30]},
+                {key: "message", matched: "  Main - End", pos: [30, 42]},
+              ]
+            }
+        ]
+        expect(subject[:matches]).to eq matches_info
+      end
     end
   end
 end

spec/support/fixtures/multiline_example.log

@@ -0,0 +1,5 @@
+2013-3-03 14:27:33 [main] INFO  Main - Start
+2013-3-03 14:27:33 [main] ERROR Main - Exception
+javax.management.RuntimeErrorException: null
+    at Main.main(Main.java:16) ~[bin/:na]
+2013-3-03 14:27:33 [main] INFO  Main - End