From 33656e29f4624e2304254bb94405f8419b4c9a7c Mon Sep 17 00:00:00 2001
From: uu59 <k@uu59.org>
Date: Fri, 9 Dec 2016 16:18:28 +0900
Subject: [PATCH 1/3] Fix #208 for multiline preview

---
 lib/regexp_preview/multi_line.rb            |  36 ++++--
 spec/lib/regexp_preview/multi_line_spec.rb  | 130 ++++++++++++++++----
 spec/support/fixtures/multiline_example.log |   5 +
 3 files changed, 131 insertions(+), 40 deletions(-)
 create mode 100644 spec/support/fixtures/multiline_example.log

diff --git a/lib/regexp_preview/multi_line.rb b/lib/regexp_preview/multi_line.rb
index 7c291bb2f..b8fc6e544 100644
--- a/lib/regexp_preview/multi_line.rb
+++ b/lib/regexp_preview/multi_line.rb
@@ -27,24 +27,34 @@ def matches
       reader = FileReverseReader.new(File.open(file))
       result = []
       target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" }
-      target_lines.each_with_index do |line, line_no|
-        if line.match(params[:format_firstline])
-          lines = target_lines[line_no, patterns.length]
-          next if lines.length < patterns.length
-          ret = detect_chunk(lines)
-          next unless ret
-          result << ret
-        end
+      whole_string = target_lines.join
+      re_firstline = Regexp.new(params[:format_firstline])
+      indexes = []
+      cur = 0
+      while first_index = whole_string.index(re_firstline, cur)
+        indexes << first_index
+        cur = first_index + 1
+      end
+      indexes.each_with_index do |index, i|
+        next_index = indexes[i + 1] || -1
+        chunk = whole_string[index...next_index]
+        ret = detect_chunk(chunk)
+        next unless ret
+        result << ret
       end
       result
     end
 
-    def detect_chunk(lines)
+    def detect_chunk(chunk)
       whole = ""
       matches = []
-      lines.each_with_index do |line, i|
-        match = line.match(patterns[i])
+      offset = 0
+      patterns.each do |pat|
+        match = chunk.match(pat)
         return nil unless match
+        offset = chunk.index(pat)
+        return nil if offset > 0
+        chunk = chunk[match[0].length..-1]
         match.names.each_with_index do |name, index|
           matches << {
             key: name,
@@ -52,7 +62,7 @@ def detect_chunk(lines)
             pos: match.offset(index + 1).map{|pos| pos + whole.length},
           }
         end
-        whole << line
+        whole << match[0]
       end
       {
         whole: whole,
@@ -63,7 +73,7 @@ def detect_chunk(lines)
     def patterns
       @patterns ||= (1..20).map do |n|
         params["format#{n}"].presence
-      end.compact.map {|pattern| Regexp.new(pattern)}
+      end.compact.map {|pattern| Regexp.new(pattern, Regexp::MULTILINE)}
     end
   end
 end
diff --git a/spec/lib/regexp_preview/multi_line_spec.rb b/spec/lib/regexp_preview/multi_line_spec.rb
index 100c6ff0a..012d2a448 100644
--- a/spec/lib/regexp_preview/multi_line_spec.rb
+++ b/spec/lib/regexp_preview/multi_line_spec.rb
@@ -2,39 +2,115 @@
 
 describe RegexpPreview::MultiLine do
   describe "#matches_json" do
-    subject { RegexpPreview::MultiLine.new(File.expand_path("./spec/support/fixtures/error0.log", Rails.root), "multiline", params).matches_json }
-
-    let :params do
-      params = {
-        format_firstline: ".+",
-        time_format: "time_format",
-      }
-      params["format1"] = "(?<foo>foo)"
-      params["format2"] = "(?<bar>bar)"
-      3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
-        params["format#{i}"] = ""
+    subject { parser.matches_json }
+    let(:parser) { RegexpPreview::MultiLine.new(target_path, "multiline", params) }
+
+    describe "simple usage" do
+      let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
+
+      let :params do
+        params = {
+          format_firstline: "foo",
+          time_format: "time_format",
+        }
+        params["format1"] = "(?<foo>foo)\n"
+        params["format2"] = "(?<bar>bar)"
+        3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
+          params["format#{i}"] = ""
+        end
+        { params: params }
+      end
+
+      it 'should not have regexp and time_format in [:params][:setting]' do
+        expect(subject[:params][:setting]).to eq({ regexp: nil, time_format: nil })
+      end
+
+      it "should include matches info" do
+        matches_info = {
+          whole: "foo\nbar",
+          matches: [
+            {
+              key: "foo", matched: "foo", pos: [0, 3]
+            },
+            {
+              key: "bar", matched: "bar", pos: [4, 7]
+            }
+          ]
+        }
+
+        expect(subject[:matches]).to include matches_info
       end
-      { params: params }
     end
 
-    it 'should not have regexp and time_format in [:params][:setting]' do
-      expect(subject[:params][:setting]).to eq({ regexp: nil, time_format: nil })
+    describe "detect only continuos patterns" do
+      let(:target_path) { File.expand_path("./spec/support/fixtures/error0.log", Rails.root) }
+      let(:params) do
+        params = {
+          format_firstline: "foo",
+          time_format: "time_format",
+        }
+        params["format1"] = "(?<foo>foo)\n"
+        params["format2"] = "(?<bar>baz)"
+        3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
+          params["format#{i}"] = ""
+        end
+        { params: params }
+      end
+
+      it "shouldn't match" do
+        expect(subject[:matches]).to eq []
+      end
     end
 
-    it "should include matches info" do
-      matches_info = {
-        whole: "foo\nbar\n",
-        matches: [
-          {
-            key: "foo", matched: "foo", pos: [0, 3]
-          },
-          {
-            key: "bar", matched: "bar", pos: [4, 7]
-          }
-        ]
-      }
+    describe "example on document" do
+      # http://docs.fluentd.org/articles/in_tail
+      let(:target_path) { File.expand_path("./spec/support/fixtures/multiline_example.log", Rails.root) }
 
-      expect(subject[:matches]).to include matches_info
+      let :params do
+        params = {
+          format_firstline: "\\d{4}-\\d{1,2}-\\d{1,2}",
+          "format1" => "^(?<time>\\d{4}-\\d{1,2}-\\d{1,2} \\d{1,2}:\\d{1,2}:\\d{1,2}) \\[(?<thread>.*)\\] (?<level>[^\\s]+)(?<message>.*)",
+          time_format: "time_format",
+        }
+        2.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i|
+          params["format#{i}"] = ""
+        end
+        { params: params }
+      end
+
+      it "should include matches info" do
+        matches_info = 
+          [
+            {
+              :whole => "2013-3-03 14:27:33 [main] INFO  Main - Start\n",
+              :matches => [
+                {:key => "time", :matched => "2013-3-03 14:27:33", :pos => [0, 18]},
+                {:key => "thread", :matched => "main", :pos => [20, 24]},
+                {:key => "level", :matched => "INFO", :pos => [26, 30]},
+                {:key => "message", :matched => "  Main - Start\n", :pos => [30, 45]}
+              ]
+            },
+            {
+              :whole => "2013-3-03 14:27:33 [main] ERROR Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n",
+              :matches => [
+                {:key => "time", :matched => "2013-3-03 14:27:33", :pos => [0, 18]},
+                {:key => "thread", :matched => "main", :pos => [20, 24]},
+                {:key => "level", :matched => "ERROR", :pos => [26, 31]},
+                {:key => "message", :matched => " Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n", :pos => [31, 136]},
+              ]
+            },
+            {
+              :whole => "2013-3-03 14:27:33 [main] INFO  Main - End",
+              :matches => [
+                {:key => "time", :matched => "2013-3-03 14:27:33", :pos => [0, 18]},
+                {:key => "thread", :matched => "main", :pos => [20, 24]},
+                {:key => "level", :matched => "INFO", :pos => [26, 30]},
+                {:key => "message", :matched => "  Main - End", :pos => [30, 42]},
+              ]
+            }
+        ]
+        expect(subject[:matches]).to eq matches_info
+      end
     end
   end
 end
diff --git a/spec/support/fixtures/multiline_example.log b/spec/support/fixtures/multiline_example.log
new file mode 100644
index 000000000..a6e4ef21e
--- /dev/null
+++ b/spec/support/fixtures/multiline_example.log
@@ -0,0 +1,5 @@
+2013-3-03 14:27:33 [main] INFO  Main - Start
+2013-3-03 14:27:33 [main] ERROR Main - Exception
+javax.management.RuntimeErrorException: null
+    at Main.main(Main.java:16) ~[bin/:na]
+2013-3-03 14:27:33 [main] INFO  Main - End

From 8e391682885c72c801769fa1c6d4e3107f42650f Mon Sep 17 00:00:00 2001
From: uu59 <k@uu59.org>
Date: Mon, 12 Dec 2016 18:04:10 +0900
Subject: [PATCH 2/3] cosme: Use newer Hash syntax

---
 spec/lib/regexp_preview/multi_line_spec.rb | 36 +++++++++++-----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/spec/lib/regexp_preview/multi_line_spec.rb b/spec/lib/regexp_preview/multi_line_spec.rb
index 012d2a448..2b9053ca2 100644
--- a/spec/lib/regexp_preview/multi_line_spec.rb
+++ b/spec/lib/regexp_preview/multi_line_spec.rb
@@ -82,30 +82,30 @@
         matches_info = 
           [
             {
-              :whole => "2013-3-03 14:27:33 [main] INFO  Main - Start\n",
-              :matches => [
-                {:key => "time", :matched => "2013-3-03 14:27:33", :pos => [0, 18]},
-                {:key => "thread", :matched => "main", :pos => [20, 24]},
-                {:key => "level", :matched => "INFO", :pos => [26, 30]},
-                {:key => "message", :matched => "  Main - Start\n", :pos => [30, 45]}
+              whole: "2013-3-03 14:27:33 [main] INFO  Main - Start\n",
+              matches: [
+                {key: "time", matched: "2013-3-03 14:27:33", pos: [0, 18]},
+                {key: "thread", matched: "main", pos: [20, 24]},
+                {key: "level", matched: "INFO", pos: [26, 30]},
+                {key: "message", matched: "  Main - Start\n", pos: [30, 45]}
               ]
             },
             {
-              :whole => "2013-3-03 14:27:33 [main] ERROR Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n",
-              :matches => [
-                {:key => "time", :matched => "2013-3-03 14:27:33", :pos => [0, 18]},
-                {:key => "thread", :matched => "main", :pos => [20, 24]},
-                {:key => "level", :matched => "ERROR", :pos => [26, 31]},
-                {:key => "message", :matched => " Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n", :pos => [31, 136]},
+              whole: "2013-3-03 14:27:33 [main] ERROR Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n",
+              matches: [
+                {key: "time", matched: "2013-3-03 14:27:33", pos: [0, 18]},
+                {key: "thread", matched: "main", pos: [20, 24]},
+                {key: "level", matched: "ERROR", pos: [26, 31]},
+                {key: "message", matched: " Main - Exception\njavax.management.RuntimeErrorException: null\n    at Main.main(Main.java:16) ~[bin/:na]\n", pos: [31, 136]},
               ]
             },
             {
-              :whole => "2013-3-03 14:27:33 [main] INFO  Main - End",
-              :matches => [
-                {:key => "time", :matched => "2013-3-03 14:27:33", :pos => [0, 18]},
-                {:key => "thread", :matched => "main", :pos => [20, 24]},
-                {:key => "level", :matched => "INFO", :pos => [26, 30]},
-                {:key => "message", :matched => "  Main - End", :pos => [30, 42]},
+              whole: "2013-3-03 14:27:33 [main] INFO  Main - End",
+              matches: [
+                {key: "time", matched: "2013-3-03 14:27:33", pos: [0, 18]},
+                {key: "thread", matched: "main", pos: [20, 24]},
+                {key: "level", matched: "INFO", pos: [26, 30]},
+                {key: "message", matched: "  Main - End", pos: [30, 42]},
               ]
             }
         ]

From 902a4887469ad1b26337dfe41f2eed93f60dc2e6 Mon Sep 17 00:00:00 2001
From: uu59 <k@uu59.org>
Date: Mon, 12 Dec 2016 18:04:31 +0900
Subject: [PATCH 3/3] cosme: Remove unnecessary {}

---
 spec/lib/regexp_preview/multi_line_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/lib/regexp_preview/multi_line_spec.rb b/spec/lib/regexp_preview/multi_line_spec.rb
index 2b9053ca2..97b40ea7e 100644
--- a/spec/lib/regexp_preview/multi_line_spec.rb
+++ b/spec/lib/regexp_preview/multi_line_spec.rb
@@ -22,7 +22,7 @@
       end
 
       it 'should not have regexp and time_format in [:params][:setting]' do
-        expect(subject[:params][:setting]).to eq({ regexp: nil, time_format: nil })
+        expect(subject[:params][:setting]).to eq(regexp: nil, time_format: nil)
       end
 
       it "should include matches info" do