Avoid usage of legacy algorithms on libssl-3.0+ (#53)

Single DES and MD4 are considered legacy algorithms in OpenSSL/libssl-3.0.
They can be enabled by adjusting the openssl configuration file or by using the new provider concept of OpenSSL-3.
Editing the configuration must be done by each use of rubyntlm, which is very inconvenient.
The provider API is not yet supported by the ruby binding to OpenSSL.

So it's better to avoid the legacy algorithms at all.
The single DES algorithm can easily implemented by a two key 3-DES run.
The md4 implementation is taken from here:
  https://gist.github.com/tprynn/5419da1a2ad8935c1fff
And the rc4 implementation is taken from here and modified:
  https://github.com/caiges/Ruby-RC4/blob/082fce56ab707dc77442709357cf176f7b3b6f22/lib/rc4.rb

lib/net/ntlm.rb

@@ -59,6 +59,8 @@
 require 'net/ntlm/message/type3'
 
 require 'net/ntlm/encode_util'
+require 'net/ntlm/md4'
+require 'net/ntlm/rc4'
 
 require 'net/ntlm/client'
 require 'net/ntlm/channel_binding'
@@ -125,9 +127,9 @@ def gen_keys(str)
 
       def apply_des(plain, keys)
         keys.map {|k|
-          dec = OpenSSL::Cipher.new("des-cbc").encrypt
+          dec = OpenSSL::Cipher.new("des-ede-cbc").encrypt
           dec.padding = 0
-          dec.key = k
+          dec.key = k + k
           dec.update(plain) + dec.final
         }
       end
@@ -147,7 +149,7 @@ def ntlm_hash(password, opt = {})
         unless opt[:unicode]
           pwd = EncodeUtil.encode_utf16le(pwd)
         end
-        OpenSSL::Digest::MD4.digest pwd
+        Net::NTLM::Md4.digest pwd
       end
 
       # Generate a NTLMv2 Hash

lib/net/ntlm/client/session.rb

@@ -36,11 +36,8 @@ def authenticate!
         t3 = Message::Type3.create type3_opts
         if negotiate_key_exchange?
           t3.enable(:session_key)
-          rc4 = OpenSSL::Cipher.new("rc4")
-          rc4.encrypt
-          rc4.key = user_session_key
-          sk = rc4.update exported_session_key
-          sk << rc4.final
+          rc4 = Net::NTLM::Rc4.new(user_session_key)
+          sk = rc4.encrypt exported_session_key
           t3.session_key = sk
         end
         t3
@@ -50,7 +47,7 @@ def exported_session_key
         @exported_session_key ||=
           begin
             if negotiate_key_exchange?
-              OpenSSL::Cipher.new("rc4").random_key
+              OpenSSL::Random.random_bytes(16)
             else
               user_session_key
             end
@@ -61,8 +58,7 @@ def sign_message(message)
         seq = sequence
         sig = OpenSSL::HMAC.digest(OpenSSL::Digest::MD5.new, client_sign_key, "#{seq}#{message}")[0..7]
         if negotiate_key_exchange?
-          sig = client_cipher.update sig
-          sig << client_cipher.final
+          sig = client_cipher.encrypt sig
         end
         "#{VERSION_MAGIC}#{sig}#{seq}"
       end
@@ -71,20 +67,17 @@ def verify_signature(signature, message)
         seq = signature[-4..-1]
         sig = OpenSSL::HMAC.digest(OpenSSL::Digest::MD5.new, server_sign_key, "#{seq}#{message}")[0..7]
         if negotiate_key_exchange?
-          sig = server_cipher.update sig
-          sig << server_cipher.final
+          sig = server_cipher.encrypt sig
         end
         "#{VERSION_MAGIC}#{sig}#{seq}" == signature
       end
 
       def seal_message(message)
-        emessage = client_cipher.update(message)
-        emessage + client_cipher.final
+        client_cipher.encrypt(message)
       end
 
       def unseal_message(emessage)
-        message = server_cipher.update(emessage)
-        message + server_cipher.final
+        server_cipher.encrypt(emessage)
       end
 
       private
@@ -123,23 +116,11 @@ def server_seal_key
       end
 
       def client_cipher
-        @client_cipher ||=
-          begin
-            rc4 = OpenSSL::Cipher.new("rc4")
-            rc4.encrypt
-            rc4.key = client_seal_key
-            rc4
-          end
+        @client_cipher ||= Net::NTLM::Rc4.new(client_seal_key)
       end
 
       def server_cipher
-        @server_cipher ||=
-          begin
-            rc4 = OpenSSL::Cipher.new("rc4")
-            rc4.decrypt
-            rc4.key = server_seal_key
-            rc4
-          end
+        @server_cipher ||= Net::NTLM::Rc4.new(server_seal_key)
       end
 
       def client_challenge

lib/net/ntlm/md4.rb

@@ -0,0 +1,80 @@
+require 'openssl'
+
+module Net
+module NTLM
+
+  class Md4
+
+    begin
+      OpenSSL::Digest::MD4.digest("")
+    rescue
+      # libssl-3.0+ doesn't support legacy MD4 -> use our own implementation
+
+      require 'stringio'
+
+      def self.digest(string)
+        # functions
+        mask = (1 << 32) - 1
+        f = proc {|x, y, z| x & y | x.^(mask) & z}
+        g = proc {|x, y, z| x & y | x & z | y & z}
+        h = proc {|x, y, z| x ^ y ^ z}
+        r = proc {|v, s| (v << s).&(mask) | (v.&(mask) >> (32 - s))}
+
+        # initial hash
+        a, b, c, d = 0x67452301, 0xefcdab89, 0x98badcfe, 0x10325476
+
+        bit_len = string.size << 3
+        string += "\x80"
+        while (string.size % 64) != 56
+          string += "\0"
+        end
+        string = string.force_encoding('ascii-8bit') + [bit_len & mask, bit_len >> 32].pack("V2")
+
+        if string.size % 64 != 0
+          fail "failed to pad to correct length"
+        end
+
+        io = StringIO.new(string)
+        block = ""
+
+        while io.read(64, block)
+          x = block.unpack("V16")
+
+          # Process this block.
+          aa, bb, cc, dd = a, b, c, d
+          [0, 4, 8, 12].each {|i|
+            a = r[a + f[b, c, d] + x[i],  3]; i += 1
+            d = r[d + f[a, b, c] + x[i],  7]; i += 1
+            c = r[c + f[d, a, b] + x[i], 11]; i += 1
+            b = r[b + f[c, d, a] + x[i], 19]
+          }
+          [0, 1, 2, 3].each {|i|
+            a = r[a + g[b, c, d] + x[i] + 0x5a827999,  3]; i += 4
+            d = r[d + g[a, b, c] + x[i] + 0x5a827999,  5]; i += 4
+            c = r[c + g[d, a, b] + x[i] + 0x5a827999,  9]; i += 4
+            b = r[b + g[c, d, a] + x[i] + 0x5a827999, 13]
+          }
+          [0, 2, 1, 3].each {|i|
+            a = r[a + h[b, c, d] + x[i] + 0x6ed9eba1,  3]; i += 8
+            d = r[d + h[a, b, c] + x[i] + 0x6ed9eba1,  9]; i -= 4
+            c = r[c + h[d, a, b] + x[i] + 0x6ed9eba1, 11]; i += 8
+            b = r[b + h[c, d, a] + x[i] + 0x6ed9eba1, 15]
+          }
+          a = (a + aa) & mask
+          b = (b + bb) & mask
+          c = (c + cc) & mask
+          d = (d + dd) & mask
+        end
+
+        [a, b, c, d].pack("V4")
+      end
+
+    else
+      # Openssl/libssl provides MD4, so we can use it.
+      def self.digest(string)
+        OpenSSL::Digest::MD4.digest(string)
+      end
+    end
+  end
+end
+end

lib/net/ntlm/rc4.rb

@@ -0,0 +1,59 @@
+require 'openssl'
+
+module Net
+module NTLM
+
+  begin
+    OpenSSL::Cipher.new("rc4")
+  rescue
+    # libssl-3.0+ doesn't support legacy Rc4 -> use our own implementation
+
+    class Rc4
+      def initialize(str)
+        raise ArgumentError, "RC4: Key supplied is blank"  if str.eql?('')
+        initialize_state(str)
+        @q1, @q2 = 0, 0
+      end
+
+      def encrypt(text)
+        text.each_byte.map do |b|
+          @q1 = (@q1 + 1) % 256
+          @q2 = (@q2 + @state[@q1]) % 256
+          @state[@q1], @state[@q2] = @state[@q2], @state[@q1]
+          b ^ @state[(@state[@q1] + @state[@q2]) % 256]
+        end.pack("C*")
+      end
+
+      private
+
+      # The initial state which is then modified by the key-scheduling algorithm
+      INITIAL_STATE = (0..255).to_a
+
+      # Performs the key-scheduling algorithm to initialize the state.
+      def initialize_state(key)
+        i = j = 0
+        @state = INITIAL_STATE.dup
+        key_length = key.length
+        while i < 256
+          j = (j + @state[i] + key.getbyte(i % key_length)) % 256
+          @state[i], @state[j] = @state[j], @state[i]
+          i += 1
+        end
+      end
+    end
+
+  else
+    # Openssl/libssl provides RC4, so we can use it.
+    class Rc4
+      def initialize(str)
+        @ci = OpenSSL::Cipher.new("rc4")
+        @ci.key = str
+      end
+
+      def encrypt(text)
+        @ci.update(text) + @ci.final
+      end
+    end
+  end
+end
+end