From 8aaa391859bbdb08f75fbeed4819fef1c73185df Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Thu, 11 Feb 2016 13:42:17 -0500 Subject: [PATCH] Provide an alternative implementation of Net::SSH::KnownHost in ssh_options As discussed in https://github.com/capistrano/sshkit/issues/326#issuecomment-179284023 Net::SSH re-parse the known_hosts files every time it needs to lookup for a known key. This alternative implementation parse it once and for all, and cache the result. --- README.md | 13 ++ lib/sshkit/all.rb | 1 + lib/sshkit/backends/netssh.rb | 5 +- lib/sshkit/backends/netssh/known_hosts.rb | 141 ++++++++++++++++++++++ test/known_hosts/github | 1 + test/known_hosts/github_hash | 1 + test/unit/backends/test_netssh.rb | 24 ++++ 7 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 lib/sshkit/backends/netssh/known_hosts.rb create mode 100644 test/known_hosts/github create mode 100644 test/known_hosts/github_hash diff --git a/README.md b/README.md index 52c0ebb4..801b42b4 100644 --- a/README.md +++ b/README.md @@ -508,6 +508,19 @@ pooling behaviour entirely by setting the idle_timeout to zero: SSHKit::Backend::Netssh.pool.idle_timeout = 0 # disabled ``` +## Known hosts caching + +If you connect to many hosts with the `Netssh` backend, looking up `~/.ssh/known_hosts` can significantly impact performances. +You can mitigate this by using SSHKit's lookup caching like this: + +```ruby +SSHKit::Backend::Netssh.configure do |ssh| + ssh.ssh_options = { + known_hosts: SSHKit::Backend::Netssh::KnownHosts.new, + } +end +``` + ## Tunneling and other related SSH themes In order to do special gymnasitcs with SSH, tunneling, aliasing, complex options, etc with SSHKit it is possible to use [the underlying Net::SSH API](https://github.com/capistrano/sshkit/blob/master/EXAMPLES.md#setting-global-ssh-options) however in many cases it is preferred to use the system SSH configuration file at [`~/.ssh/config`](http://man.cx/ssh_config). This allows you to have personal configuration tied to your machine that does not have to be committed with the repository. If this is not suitable (everyone on the team needs a proxy command, or some special aliasing) a file in the same format can be placed in the project directory at `~/yourproject/.ssh/config`, this will be merged with the system settings in `~/.ssh/config`, and with any configuration specified in [`SSHKit::Backend::Netssh.config.ssh_options`](https://github.com/capistrano/sshkit/blob/master/lib/sshkit/backends/netssh.rb#L133). diff --git a/lib/sshkit/all.rb b/lib/sshkit/all.rb index 24f64879..3739cf65 100644 --- a/lib/sshkit/all.rb +++ b/lib/sshkit/all.rb @@ -35,5 +35,6 @@ require_relative 'backends/connection_pool' require_relative 'backends/printer' require_relative 'backends/netssh' +require_relative 'backends/netssh/known_hosts' require_relative 'backends/local' require_relative 'backends/skipper' diff --git a/lib/sshkit/backends/netssh.rb b/lib/sshkit/backends/netssh.rb index 92336e46..f48aed75 100644 --- a/lib/sshkit/backends/netssh.rb +++ b/lib/sshkit/backends/netssh.rb @@ -1,4 +1,6 @@ require 'English' +require 'strscan' +require 'mutex_m' require 'net/ssh' require 'net/scp' @@ -21,13 +23,12 @@ module SSHKit module Backend class Netssh < Abstract - class Configuration attr_accessor :connection_timeout, :pty attr_writer :ssh_options def ssh_options - @ssh_options || {} + @ssh_options ||= {} end end diff --git a/lib/sshkit/backends/netssh/known_hosts.rb b/lib/sshkit/backends/netssh/known_hosts.rb new file mode 100644 index 00000000..ec03fd00 --- /dev/null +++ b/lib/sshkit/backends/netssh/known_hosts.rb @@ -0,0 +1,141 @@ +module SSHKit + + module Backend + + class Netssh < Abstract + + class KnownHostsKeys + include Mutex_m + + def initialize(path) + super() + @path = File.expand_path(path) + @hosts_keys = nil + end + + def keys_for(hostlist) + keys, hashes = hosts_keys, hosts_hashes + parse_file unless keys && hashes + keys, hashes = hosts_keys, hosts_hashes + + hostlist.split(',').each do |host| + key_list = keys[host] + return key_list if key_list + + hashes.each do |(hmac, salt), hash_keys| + if OpenSSL::HMAC.digest(sha1, salt, host) == hmac + return hash_keys + end + end + end + + [] + end + + private + + attr_reader :path + attr_accessor :hosts_keys, :hosts_hashes + + def sha1 + @sha1 ||= OpenSSL::Digest.new('sha1') + end + + def parse_file + synchronize do + return if hosts_keys && hosts_hashes + + unless File.readable?(path) + self.hosts_keys = {} + self.hosts_hashes = [] + return + end + + new_keys = {} + new_hashes = [] + File.open(path) do |file| + scanner = StringScanner.new("") + file.each_line do |line| + scanner.string = line + parse_line(scanner, new_keys, new_hashes) + end + end + self.hosts_keys = new_keys + self.hosts_hashes = new_hashes + end + end + + def parse_line(scanner, hosts_keys, hosts_hashes) + return if empty_line?(scanner) + + hostlist = parse_hostlist(scanner) + return unless supported_type?(scanner) + key = parse_key(scanner) + + if hostlist.size == 1 && hostlist.first =~ /\A\|1(\|.+){2}\z/ + hosts_hashes << [parse_host_hash(hostlist.first), key] + else + hostlist.each do |host| + (hosts_keys[host] ||= []) << key + end + end + end + + def parse_host_hash(line) + _, _, salt, hmac = line.split('|') + [Base64.decode64(hmac), Base64.decode64(salt)] + end + + def empty_line?(scanner) + scanner.skip(/\s*/) + scanner.match?(/$|#/) + end + + def parse_hostlist(scanner) + scanner.skip(/\s*/) + scanner.scan(/\S+/).split(',') + end + + def supported_type?(scanner) + scanner.skip(/\s*/) + Net::SSH::KnownHosts::SUPPORTED_TYPE.include?(scanner.scan(/\S+/)) + end + + def parse_key(scanner) + scanner.skip(/\s*/) + Net::SSH::Buffer.new(scanner.rest.unpack("m*").first).read_key + end + end + + class KnownHosts + include Mutex_m + + def initialize + super() + @files = {} + end + + def search_for(host, options = {}) + keys = ::Net::SSH::KnownHosts.hostfiles(options).map do |path| + known_hosts_file(path).keys_for(host) + end.flatten + ::Net::SSH::HostKeys.new(keys, host, self, options) + end + + def add(*args) + ::Net::SSH::KnownHosts.add(*args) + synchronize { @files = {} } + end + + private + + def known_hosts_file(path) + @files[path] || synchronize { @files[path] ||= KnownHostsKeys.new(path) } + end + end + + end + + end + +end \ No newline at end of file diff --git a/test/known_hosts/github b/test/known_hosts/github new file mode 100644 index 00000000..138000ba --- /dev/null +++ b/test/known_hosts/github @@ -0,0 +1 @@ +github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== \ No newline at end of file diff --git a/test/known_hosts/github_hash b/test/known_hosts/github_hash new file mode 100644 index 00000000..b1a7020d --- /dev/null +++ b/test/known_hosts/github_hash @@ -0,0 +1 @@ +|1|eKp+6E0rZ3lONgsIziurXEnaIik=|rcQB/rlJMUquUyFta64KugPjX4o= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ== diff --git a/test/unit/backends/test_netssh.rb b/test/unit/backends/test_netssh.rb index 13f82060..f641af73 100644 --- a/test/unit/backends/test_netssh.rb +++ b/test/unit/backends/test_netssh.rb @@ -1,4 +1,5 @@ require 'helper' +require 'tempfile' module SSHKit module Backend @@ -53,6 +54,29 @@ def test_transfer_summarizer end end + if Net::SSH::Version::CURRENT >= Net::SSH::Version[3, 1, 0] + def test_known_hosts_for_when_all_hosts_are_recognized + perform_known_hosts_test("github") + end + + def test_known_hosts_for_when_an_host_hash_is_recognized + perform_known_hosts_test("github_hash") + end + end + + private + + def perform_known_hosts_test(hostfile) + source = File.join(File.dirname(__FILE__), '../../known_hosts', hostfile) + kh = Netssh::KnownHosts.new + keys = kh.search_for('github.com', user_known_hosts_file: source, global_known_hosts_file: Tempfile.new('sshkit-test').path) + + assert_instance_of ::Net::SSH::HostKeys, keys + assert_equal(1, keys.count) + keys.each do |key| + assert_equal("ssh-rsa", key.ssh_type) + end + end end end end