Add support for non-keyed, regular hash functions

Fixes #18 Fixes #37
logstash-plugins · Jun 19, 2018 · 119e941 · 119e941
1 parent ecca47b
commit 119e941
Show file tree

Hide file tree

Showing 3 changed files with 128 additions and 38 deletions.
diff --git a/docs/index.asciidoc b/docs/index.asciidoc
@@ -27,9 +27,6 @@ This can e.g. be used to create consistent document ids when inserting
 events into Elasticsearch, allowing events in Logstash to cause existing
 documents to be updated rather than new documents to be created.
 
-NOTE: When using any method other than 'UUID', 'PUNCTUATION' or 'MURMUR3'
-you must set the key, otherwise the plugin will raise an exception
-
 NOTE: When the `target` option is set to `UUID` the result won't be
 a consistent hash but a random
 https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID].
@@ -99,8 +96,7 @@ source fields given.
   * There is no default value for this setting.
 
 When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
-Key is required with all methods except  `MURMUR3`, `PUNCTUATION` or `UUID`.
-With other methods fill in the HMAC key.
+With other methods, optionally fill in the HMAC key.
 
 [id="plugins-{type}s-{plugin}-method"]
 ===== `method` 
@@ -111,10 +107,12 @@ With other methods fill in the HMAC key.
 
 The fingerprint method to use.
 
-If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` the
-cryptographic keyed-hash function with the same name will be used to
-generate the fingerprint. If set to `MURMUR3` the non-cryptographic
-MurmurHash function will be used.
+If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
+the cryptographic hash function with the same name will be used to generate
+the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
+be used.
+
+If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
 
 If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
 the hash value will be the masked-out address using the number of bits

diff --git a/lib/logstash/filters/fingerprint.rb b/lib/logstash/filters/fingerprint.rb
@@ -34,8 +34,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   config :target, :validate => :string, :default => 'fingerprint'
 
   # When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
-  # Key is required with all methods except  `MURMUR3`, `PUNCTUATION` or `UUID`.
-  # With other methods fill in the HMAC key.
+  # With other methods, optionally fill in the HMAC key.
   config :key, :validate => :string
 
   # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
@@ -44,10 +43,12 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
 
   # The fingerprint method to use.
   #
-  # If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` the
-  # cryptographic keyed-hash function with the same name will be used to
-  # generate the fingerprint. If set to `MURMUR3` the non-cryptographic
-  # MurmurHash function will be used.
+  # If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
+  # the cryptographic hash function with the same name will be used to generate
+  # the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
+  # be used.
+  #
+  # If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
   #
   # If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
   # the hash value will be the masked-out address using the number of bits
@@ -98,14 +99,6 @@ class << self; alias_method :fingerprint, :fingerprint_murmur3; end
     when :PUNCTUATION
       # nothing
     else
-      if @key.nil?
-        raise LogStash::ConfigurationError, I18n.t(
-          "logstash.runner.configuration.invalid_plugin_register",
-          :plugin => "filter",
-          :type => "fingerprint",
-          :error => "Key value is empty. Please fill in an encryption key"
-        )
-      end
       class << self; alias_method :fingerprint, :fingerprint_openssl; end
       @digest = select_digest(@method)
     end
@@ -161,11 +154,19 @@ def fingerprint_ipv4_network(ip_string)
 
   def fingerprint_openssl(data)
     # in JRuby 1.7.11 outputs as ASCII-8BIT
-    if @base64encode
-      hash  = OpenSSL::HMAC.digest(@digest, @key, data.to_s)
-      Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
+    if @key.nil?
+      if @base64encode
+        @digest.base64digest(data.to_s).force_encoding(Encoding::UTF_8)
+      else
+        @digest.hexdigest(data.to_s).force_encoding(Encoding::UTF_8)
+      end
     else
-      OpenSSL::HMAC.hexdigest(@digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
+      if @base64encode
+        hash = OpenSSL::HMAC.digest(@digest, @key, data.to_s)
+        Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
+      else
+        OpenSSL::HMAC.hexdigest(@digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
+      end
     end
   end
 

diff --git a/spec/filters/fingerprint_spec.rb b/spec/filters/fingerprint_spec.rb
@@ -35,7 +35,22 @@
     end
   end
 
-  describe "fingerprint string with SHA1 alogrithm" do
+  describe "fingerprint string with SHA1 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA1'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "3a5076c520b4b463f43806896ea0b3978d09dcae"
+    end
+  end
+
+  describe "fingerprint string with SHA1 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -51,7 +66,7 @@
     end
   end
 
-  describe "fingerprint string with SHA1 alogrithm on all event fields" do
+  describe "fingerprint string with SHA1 HMAC algorithm on all event fields" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -68,7 +83,23 @@
     end
   end
 
-  describe "fingerprint string with SHA1 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA1 algorithm and base64 encoding" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA1'
+          base64encode => true
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "OlB2xSC0tGP0OAaJbqCzl40J3K4="
+    end
+  end
+
+  describe "fingerprint string with SHA1 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -85,7 +116,22 @@
     end
   end
 
-  describe "fingerprint string with SHA256 alogrithm" do
+  describe "fingerprint string with SHA256 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA256'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "4dabcab210766e35f03e77120e6986d6e6d4752b2a9ff22980b9253d026080d8"
+    end
+  end
+
+  describe "fingerprint string with SHA256 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -101,7 +147,7 @@
     end
   end
 
-  describe "fingerprint string with SHA256 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA256 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -118,7 +164,22 @@
     end
   end
 
-  describe "fingerprint string with SHA384 alogrithm" do
+  describe "fingerprint string with SHA384 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA384'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "fd605b0a3af3e04ce0d7a0b0d9c48d67a12dab811f60072e6eae84e35d567793ffb68a1807536f11c90874065c2a4392"
+    end
+  end
+
+  describe "fingerprint string with SHA384 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -134,7 +195,7 @@
     end
   end
 
-  describe "fingerprint string with SHA384 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA384 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -151,7 +212,22 @@
     end
   end
 
-  describe "fingerprint string with SHA512 alogrithm" do
+  describe "fingerprint string with SHA512 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA512'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "5468e2dc64ea92b617782aae884b35af60041ac9e168a283615b6a462c54c13d42fa9542cce9b7d76a8124ac6616818905e3e5dd35d6e519f77c3b517558639a"
+    end
+  end
+
+  describe "fingerprint string with SHA512 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -167,7 +243,7 @@
     end
   end
 
-  describe "fingerprint string with SHA512 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA512 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -184,7 +260,22 @@
     end
   end
 
-  describe "fingerprint string with MD5 alogrithm" do
+  describe "fingerprint string with MD5 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'MD5'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "ccdd8d3d940a01b2fb3258c059924c0d"
+    end
+  end
+
+  describe "fingerprint string with MD5 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -200,7 +291,7 @@
     end
   end
 
-  describe "fingerprint string with MD5 alogrithm and base64 encoding" do
+  describe "fingerprint string with MD5 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {