Doctype deprecation

CHANGELOG.md

@@ -1,3 +1,7 @@
+### 9.0.0
+  - Change default document type to 'doc' from 'logs' to align with beats and reflect the generic nature of logstash.
+  - Deprecate 'document_type' option
+
 ### 8.2.2
   - Use `#response_body` instead of `#body` when debugging response from the server #679
 

docs/index.asciidoc

@@ -236,11 +236,14 @@ Elasticsearch with the same ID.
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
+  * This option is deprecated
 
-The document type to write events to. Generally you should try to write only
+Note: This option is deprecated due to the https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal of types in Logstash 6.0].
+It will be removed in the next major version of Logstash.
+This sets the document type to write events to. Generally you should try to write only
 similar events to the same 'type'. String expansion `%{foo}` works here.
 Unless you set 'document_type', the event 'type' will be used if it exists
-otherwise the document type will be assigned the value of 'logs'
+otherwise the document type will be assigned the value of 'doc'.
 
 [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
 ===== `failure_type_logging_whitelist` 

lib/logstash/outputs/elasticsearch/common.rb

@@ -204,7 +204,7 @@ def get_event_type(event)
       type = if @document_type
                event.sprintf(@document_type)
              else
-               event.get("type") || "logs"
+               event.get("type") || "doc"
              end
 
       if !(type.is_a?(String) || type.is_a?(Numeric))

lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -12,11 +12,9 @@ def self.included(mod)
       # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
       mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
 
-      # The document type to write events to. Generally you should try to write only
-      # similar events to the same 'type'. String expansion `%{foo}` works here.
-      # Unless you set 'document_type', the event 'type' will be used if it exists
-      # otherwise the document type will be assigned the value of 'logs'
-      mod.config :document_type, :validate => :string
+      mod.config :document_type, 
+        :validate => :string, 
+        :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
 
       # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
       # Logstash's startup if one with the name `template_name` does not already exist.

logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '8.2.2'
+  s.version         = '9.0.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/integration/outputs/delete_spec.rb

@@ -40,25 +40,25 @@
       it "should ignore non-monotonic external version updates" do
         id = "ev2"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-        r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+        r = es.get(:index => 'logstash-delete', :type => 'doc', :id => id, :refresh => true)
         expect(r['_version']).to eq(99)
         expect(r['_source']['message']).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 98)])
-        r2 = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+        r2 = es.get(:index => 'logstash-delete', :type => 'doc', :id => id, :refresh => true)
         expect(r2['_version']).to eq(99)
         expect(r2['_source']['message']).to eq('foo')
       end
 
       it "should commit monotonic external version updates" do
         id = "ev3"
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "index", "message" => "foo", "my_version" => 99)])
-        r = es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true)
+        r = es.get(:index => 'logstash-delete', :type => 'doc', :id => id, :refresh => true)
         expect(r['_version']).to eq(99)
         expect(r['_source']['message']).to eq('foo')
 
         subject.multi_receive([LogStash::Event.new("my_id" => id, "my_action" => "delete", "message" => "foo", "my_version" => 100)])
-        expect { es.get(:index => 'logstash-delete', :type => 'logs', :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+        expect { es.get(:index => 'logstash-delete', :type => 'doc', :id => id, :refresh => true) }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
       end
     end
   end

spec/integration/outputs/groovy_update_spec.rb

@@ -25,7 +25,7 @@ def get_es_output( options={} )
       @es.indices.delete(:index => "*") rescue nil
       @es.index(
         :index => 'logstash-update',
-        :type => 'logs',
+        :type => 'doc',
         :id => "123",
         :body => { :message => 'Test', :counter => 1 }
       )
@@ -37,15 +37,15 @@ def get_es_output( options={} )
         subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
         subject.register
         subject.multi_receive([LogStash::Event.new("count" => 2)])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 3
       end
 
       it "should increment a counter with event/doc '[data][count]' nested variable" do
         subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
         subject.register
         subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 4
       end
 
@@ -58,7 +58,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 4
       end
 
@@ -72,7 +72,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 4
       end
 
@@ -86,7 +86,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["counter"] } == 3
       end
 
@@ -100,7 +100,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("count" => 4 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 5
       end
     end
@@ -110,15 +110,15 @@ def get_es_output( options={} )
         subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
         subject.register
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["message"] } == 'upsert message'
       end
 
       it "should create new documents with event/doc as upsert" do
         subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
         subject.register
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["message"] } == 'sample message here'
       end
 
@@ -133,7 +133,7 @@ def get_es_output( options={} )
         subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
         subject.register
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["message"] } == 'upsert message'
       end
 
@@ -142,7 +142,7 @@ def get_es_output( options={} )
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 1)])
         @es.indices.refresh
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["counter"] } == 1
       end
     end

spec/integration/outputs/index_spec.rb

@@ -113,8 +113,8 @@
     it_behaves_like("an indexer")
   end
 
-  describe "an indexer with no type value set (default to logs)", :integration => true do
-    let(:type) { "logs" }
+  describe "an indexer with no type value set (default to doc)", :integration => true do
+    let(:type) { "doc" }
     let(:config) {
       {
         "hosts" => get_host_port,

spec/integration/outputs/index_version_spec.rb

@@ -38,11 +38,11 @@
 
         it "should default to ES version" do
           subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foo")])
-          r = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
+          r = es.get(:index => 'logstash-index', :type => 'doc', :id => "123", :refresh => true)
           expect(r["_version"]).to eq(1)
           expect(r["_source"]["message"]).to eq('foo')
           subject.multi_receive([LogStash::Event.new("my_id" => "123", "message" => "foobar")])
-          r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => "123", :refresh => true)
+          r2 = es.get(:index => 'logstash-index', :type => 'doc', :id => "123", :refresh => true)
           expect(r2["_version"]).to eq(2)
           expect(r2["_source"]["message"]).to eq('foobar')
         end  
@@ -66,33 +66,33 @@
         it "should respect the external version" do
           id = "ev1"
           subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-          r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+          r = es.get(:index => 'logstash-index', :type => 'doc', :id => id, :refresh => true)
           expect(r["_version"]).to eq(99)
           expect(r["_source"]["message"]).to eq('foo')
         end
 
         it "should ignore non-monotonic external version updates" do
           id = "ev2"
           subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-          r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+          r = es.get(:index => 'logstash-index', :type => 'doc', :id => id, :refresh => true)
           expect(r["_version"]).to eq(99)
           expect(r["_source"]["message"]).to eq('foo')
 
           subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "98", "message" => "foo")])
-          r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+          r2 = es.get(:index => 'logstash-index', :type => 'doc', :id => id, :refresh => true)
           expect(r2["_version"]).to eq(99)
           expect(r2["_source"]["message"]).to eq('foo')
         end
 
         it "should commit monotonic external version updates" do
           id = "ev3"
           subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "99", "message" => "foo")])
-          r = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+          r = es.get(:index => 'logstash-index', :type => 'doc', :id => id, :refresh => true)
           expect(r["_version"]).to eq(99)
           expect(r["_source"]["message"]).to eq('foo')
 
           subject.multi_receive([LogStash::Event.new("my_id" => id, "my_version" => "100", "message" => "foo")])
-          r2 = es.get(:index => 'logstash-index', :type => 'logs', :id => id, :refresh => true)
+          r2 = es.get(:index => 'logstash-index', :type => 'doc', :id => id, :refresh => true)
           expect(r2["_version"]).to eq(100)
           expect(r2["_source"]["message"]).to eq('foo')
         end

spec/integration/outputs/painless_update_spec.rb

@@ -27,7 +27,7 @@ def get_es_output( options={} )
       @es.indices.delete(:index => "*") rescue nil
       @es.index(
         :index => 'logstash-update',
-        :type => 'logs',
+        :type => 'doc',
         :id => "123",
         :body => { :message => 'Test', :counter => 1 }
       )
@@ -41,15 +41,15 @@ def get_es_output( options={} )
             subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
             subject.register
             subject.multi_receive([LogStash::Event.new("count" => 2)])
-            r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+            r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
             insist { r["_source"]["counter"] } == 3
           end
 
           it "should increment a counter with event/doc '[data][count]' nested variable" do
             subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
             subject.register
             subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
-            r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+            r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
             insist { r["_source"]["counter"] } == 4
           end
         end
@@ -63,7 +63,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 4
       end
 
@@ -76,7 +76,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
         insist { r["_source"]["counter"] } == 4
       end
 
@@ -89,7 +89,7 @@ def get_es_output( options={} )
         })
         subject.register
         subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["counter"] } == 3
       end
 
@@ -114,7 +114,7 @@ def get_es_output( options={} )
           subject = get_es_output(plugin_parameters)
           subject.register
           subject.multi_receive([LogStash::Event.new("count" => 4 )])
-          r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+          r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "123", :refresh => true)
           insist { r["_source"]["counter"] } == 5
         end
       end
@@ -125,15 +125,15 @@ def get_es_output( options={} )
         subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
         subject.register
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["message"] } == 'upsert message'
       end
 
       it "should create new documents with event/doc as upsert" do
         subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
         subject.register
         subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
         insist { r["_source"]["message"] } == 'sample message here'
       end
 
@@ -150,7 +150,7 @@ def get_es_output( options={} )
             subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
             subject.register
             subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-            r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+            r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
             insist { r["_source"]["message"] } == 'upsert message'
           end
 
@@ -159,7 +159,7 @@ def get_es_output( options={} )
             subject.register
             subject.multi_receive([LogStash::Event.new("counter" => 1)])
             @es.indices.refresh
-            r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+            r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
             insist { r["_source"]["counter"] } == 1
           end
         end
@@ -170,7 +170,7 @@ def get_es_output( options={} )
           subject = get_es_output({ 'document_id' => "456", 'script' => 'ctx._source.counter = params.event.counter', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'inline' })
           subject.register
           subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-          r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+          r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
           insist { r["_source"]["message"] } == 'upsert message'
         end
 
@@ -179,7 +179,7 @@ def get_es_output( options={} )
           subject.register
           subject.multi_receive([LogStash::Event.new("counter" => 1)])
           @es.indices.refresh
-          r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+          r = @es.get(:index => 'logstash-update', :type => 'doc', :id => "456", :refresh => true)
           insist { r["_source"]["counter"] } == 1
         end
       end