From 9e9b69976d60b8c9260dbba8c59d11c15f0b3834 Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Wed, 13 Apr 2016 01:32:38 +0100 Subject: [PATCH 1/4] WIP Use HAProxy as to route Postgresql and ElasticSearch connections This provides two new configuration options: use_chef_backend chef_backend_members When use_chef_backend is true, HAProxy will be enabled with a configuration that will route requests to Postgresql and Elasticsearch to the current Chef Backend leader. --- omnibus/config/projects/chef-server.rb | 1 + omnibus/config/software/haproxy.rb | 71 ++++++++++++++ .../private-chef/attributes/default.rb | 20 ++++ .../libraries/preflight_checks.rb | 7 +- .../private-chef/libraries/private_chef.rb | 9 +- .../private-chef/recipes/default.rb | 4 + .../private-chef/recipes/haproxy.rb | 96 +++++++++++++++++++ .../templates/default/haproxy.cfg.erb | 23 +++++ .../templates/default/sv-haproxy-log-run.erb | 3 + .../templates/default/sv-haproxy-run.erb | 4 + 10 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 omnibus/config/software/haproxy.rb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-log-run.erb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-run.erb diff --git a/omnibus/config/projects/chef-server.rb b/omnibus/config/projects/chef-server.rb index 46672e93df..2d22c19da5 100644 --- a/omnibus/config/projects/chef-server.rb +++ b/omnibus/config/projects/chef-server.rb @@ -52,6 +52,7 @@ dependency "rabbitmq" dependency "redis" # dynamic routing controls dependency "opscode-solr4" +dependency "haproxy" dependency "opscode-expander" dependency "pg-gem" # used by private-chef-ctl reconfigure diff --git a/omnibus/config/software/haproxy.rb b/omnibus/config/software/haproxy.rb new file mode 100644 index 0000000000..e3e8092da2 --- /dev/null +++ b/omnibus/config/software/haproxy.rb @@ -0,0 +1,71 @@ +# +# Copyright 2016 Chef Software, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name "haproxy" +default_version "1.6.4" + +dependency "zlib" +dependency "pcre" +dependency "openssl" + +license "GPLv2" +license_file "LICENSE" + +# HTTPS is available but certificate validation fails on OS X +source url: "http://www.haproxy.org/download/1.6/src/haproxy-#{version}.tar.gz", + sha256: "e5fa3c604f1fe9ecb6974ccda6705c105ebee14b3a913069fb08f00e860cd230" + +relative_path "haproxy-#{version}" + +build do + env = with_standard_compiler_flags(with_embedded_path) + # + # Many of these are the same environment variables that debian sets + # PREFIX and TARGET are mandatory to get it building under omnibus. + # + build_options = { + "PREFIX" => "#{install_dir}/embedded", + # Use libpcre for regex, libpcre > 8.32 required + # for JIT + "USE_PCRE" => "1", + "USE_PCRE_JIT" => "1", + "USE_ZLIB" => "1", + "USE_OPENSSL" => "1", + } + # Required to resolve hostnames to IPv6 addresses + # off-by-default because of prolems on older glibc's + # TODO(ssd): Should we turn this off on RHEL5? + build_options['USE_GETADDRINFO'] = "1" + if intel? + build_options["USE_REGPARM"] = "1" + end + build_options['TARGET'] = if ohai["kernel"] && ohai["kernel"]["name"] == "Linux" + version = Gem::Version.new(String(ohai["kernel"]["release"]).split("-").first) + case + when version >= Gem::Version.new("2.6.28") + "linux2628" + when version >= Gem::Version.new("2.6") + "linux26" + else + "linux24e" + end + else + "generic" + end + build_args = "" + build_options.each { |k,v| build_args << " #{k}=#{v}"} + make "haproxy #{build_args}", env: env + make "install #{build_args}", env: env +end diff --git a/omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb b/omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb index 854b8281b1..5f725220f7 100755 --- a/omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb @@ -93,6 +93,26 @@ #### default['private_chef']['server-api-version'] = 0 +#### +# HAProxy +# +# HAProxy is only used when use_chef_backend is true. All Postgresql +# and Elasticsearch requests are routed to it and then forwarded to +# the current chef-backend leader. +#### +default['private_chef']['haproxy']['enable'] = true +default['private_chef']['haproxy']['ha'] = false +default['private_chef']['haproxy']['dir'] = "/var/opt/opscode/haproxy" +default['private_chef']['haproxy']['log_directory'] = "/var/log/opscode/haproxy" +default['private_chef']['haproxy']['log_rotation']['file_maxbytes'] = 104857600 +default['private_chef']['haproxy']['log_rotation']['num_to_keep'] = 10 +default['private_chef']['haproxy']['listen'] = '0.0.0.0' +default['private_chef']['haproxy']['local_postgresql_port'] = 5432 +default['private_chef']['haproxy']['remote_postgresql_port'] = 5432 +default['private_chef']['haproxy']['local_elasticsearch_port'] = 9200 +default['private_chef']['haproxy']['remote_elasticsearch_port'] = 9200 +default['private_chef']['haproxy']['leaderl_healthcheck_port'] = 7331 +default['private_chef']['haproxy']['etcd_port'] = 2379 #### # RabbitMQ diff --git a/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb b/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb index 8fd0ec9cef..122da1a824 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb @@ -126,7 +126,12 @@ def initialize(node) def run! begin BootstrapPreflightValidator.new(node).run! - PostgresqlPreflightValidator.new(node).run! + # When Chef Backend is configured, this is too early to verify + # postgresql accessibility since we need to configure HAProxy + # first + if ! PrivateChef['use_chef_backend'] + PostgresqlPreflightValidator.new(node).run! + end SolrPreflightValidator.new(node).run! BookshelfPreflightValidator.new(node).run! rescue PreflightValidationFailed => e diff --git a/omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb b/omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb index 6343a26555..64b8d128de 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb @@ -23,6 +23,9 @@ module PrivateChef # Set this for default org mode default_orgname nil + use_chef_backend false + chef_backend_members [] + addons Mash.new rabbitmq Mash.new external_rabbitmq Mash.new @@ -212,6 +215,8 @@ def generate_hash "ldap", "user", "ha", + "use_chef_backend", + "chef_backend_members", "disabled_plugins", "enabled_plugins", "license", @@ -224,7 +229,9 @@ def generate_hash (default_keys | keys_from_extensions).each do |key| # @todo: Just pick a naming convention and adhere to it # consistently - rkey = if key =~ /^oc_/ || key == "redis_lb" + rkey = if key =~ /^oc_/ || key == "redis_lb" || + key == "use_chef_backend" || + key == "chef_backend_members" key # leave oc_* keys as is else key.gsub('_', '-') diff --git a/omnibus/files/private-chef-cookbooks/private-chef/recipes/default.rb b/omnibus/files/private-chef-cookbooks/private-chef/recipes/default.rb index 3c410b8417..db176b8dbb 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/recipes/default.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/recipes/default.rb @@ -121,6 +121,10 @@ # Run plugins first, mostly for ha include_recipe "private-chef::plugin_chef_run" +if node['private_chef']['use_chef_backend'] + include_recipe "private-chef::haproxy" +end + # Configure Services [ "rabbitmq", diff --git a/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb b/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb new file mode 100644 index 0000000000..a87bba0a62 --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb @@ -0,0 +1,96 @@ +# Copyright:: Copyright (c) 2012 Opscode, Inc. +# License:: Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +haproxy_dir = node['private_chef']['haproxy']['dir'] +haproxy_log_dir = node['private_chef']['haproxy']['log_directory'] +[ + haproxy_dir, + haproxy_log_dir +].each do |dir_name| + directory dir_name do + owner OmnibusHelper.new(node).ownership['owner'] + group OmnibusHelper.new(node).ownership['group'] + mode node['private_chef']['service_dir_perms'] + recursive true + end +end + +# +# We need to configure haproxy with the members in our Chef Backend +# cluster. The user should have provided an array of chef-backend +# members. We iterate through this array and try to contact the member +# on etcd to get the most up-to-date list. If any part of that fails, +# we fall-back to the user-configured list of hosts. +# +def get_chef_backend_cluster_members + members = nil + ret = {} + node['private_chef']['chef_backend_members'].each do |member| + begin + client = Chef::HTTP.new("http://#{member}:#{node['private_chef']['haproxy']['etcd_port']}") + members = JSON.parse(client.get("/v2/members"))["members"] + if members && !members.empty? + break + else + next + end + rescue + next + end + end + if members + members.each do |m| + ret[m["name"]] = URI.parse(m["peerURLs"].first).host + end + ret + else + nil + end +end + +def configured_members + ret = {} + node['private_chef']['chef_backend_members'].each_with_index do |member, i| + ret["#{backend}#{i}"] = member + end + ret +end + +chef_backend_members = begin + Chef::Log.info("Attempting Chef Backend Member Discovery") + if members = get_chef_backend_cluster_members + Chef::Log.info("Using Chef Backend members discovered via etcd") + members + else + Chef::Log.info("Member discovery failed") + Chef::Log.info("Using statically configured member list") + configured_members + end + rescue e + Chef::Log.info("member discoverry failed: #{e}") + Chef::Log.info("Using statically configured member list") + configured_members + end + +template File.join(haproxy_dir, "haproxy.cfg") do + source "haproxy.cfg.erb" + owner OmnibusHelper.new(node).ownership['owner'] + group OmnibusHelper.new(node).ownership['group'] + mode "600" + variables(node['private_chef']['haproxy'].to_hash.merge(chef_backend_members: chef_backend_members)) + notifies :restart, 'runit_service[haproxy]' +end + +component_runit_service "haproxy" diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb new file mode 100644 index 0000000000..d8adbe428a --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb @@ -0,0 +1,23 @@ +frontend postgresql + bind <%= @listen %>:<%= @local_postgresql_port %> + mode tcp + default_backend chef_backend_postgresql + +frontend elasticsearch + bind <%= @listen %>:<%= @local_elasticsearch_port %> + mode tcp + default_backend chef_backend_elasticsearch + +backend chef_backend_postgresql + mode tcp + option httpchk GET /leader HTTP/1.1\r\nHost:localhost:<%= @leaderl_healthcheck_port %>\r\n\r\n + <% @chef_backend_members.each do |name, ip| -%> + server <%= name %> <%= ip %>:<%= @remote_postgresql_port %> check port <%= @leaderl_healthcheck_port %> rise 1 fall 1 + <% end -%> + +backend chef_backend_elasticsearch + mode tcp + option httpchk GET /leader HTTP/1.1\r\nHost:localhost:<%= @leaderl_healthcheck_port %>\r\n\r\n + <% @chef_backend_members.each do |name, ip| -%> + server <%= name %> <%= ip %>:<%= @remote_elasticsearch_port %> check port <%= @leaderl_healthcheck_port %> rise 1 fall 1 + <% end -%> \ No newline at end of file diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-log-run.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-log-run.erb new file mode 100644 index 0000000000..f709559608 --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-log-run.erb @@ -0,0 +1,3 @@ +#!/bin/sh +exec chpst -U <%= node['private_chef']['user']['username'] %> -u <%= node['private_chef']['user']['username'] %> \ + svlogd -tt <%= @options[:log_directory] %> diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-run.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-run.erb new file mode 100644 index 0000000000..b25dbb49f3 --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/sv-haproxy-run.erb @@ -0,0 +1,4 @@ +#!/bin/sh +exec 2>&1 +cd <%= node['private_chef']['haproxy']['dir'] %> +exec chpst -P -u <%= node['private_chef']['user']['username'] %> -U <%= node['private_chef']['user']['username'] %> /opt/opscode/embedded/sbin/haproxy -f haproxy.cfg From e6470d4ae8e1ad56132d98a98bace647f83c985e Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Fri, 15 Apr 2016 13:22:19 +0100 Subject: [PATCH 2/4] [omnibus] Better handle bootstrapping with HAProxy - The BootstrapValidator checks can't run because they require postgresql to be up. - HAProxy needs some time to mark the non-leader backends as down. Thus we wait for that to happen or error out if it doesn't happen in a reasonable amount of time. --- .../libraries/preflight_checks.rb | 2 +- .../private-chef/recipes/haproxy.rb | 106 ++++++++++++++++++ .../templates/default/haproxy.cfg.erb | 3 + 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb b/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb index 122da1a824..8c8b9424ab 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb @@ -125,11 +125,11 @@ def initialize(node) # the defaults set in the recipe. def run! begin - BootstrapPreflightValidator.new(node).run! # When Chef Backend is configured, this is too early to verify # postgresql accessibility since we need to configure HAProxy # first if ! PrivateChef['use_chef_backend'] + BootstrapPreflightValidator.new(node).run! PostgresqlPreflightValidator.new(node).run! end SolrPreflightValidator.new(node).run! diff --git a/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb b/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb index a87bba0a62..61e1d6ef7e 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb @@ -94,3 +94,109 @@ def configured_members end component_runit_service "haproxy" + + +# On startup, all backend servers will be marked as UP. +# +# As the heathcheck on the non-leaders fails, they will be marked as +# down. This usually takes 2-3 seconds, which is long enough for the +# reconfigure to proceed to attempting to bootstrap erchef, which +# won't work if we end up talking to a follower in read-only mode. +# +# Here, we wait for the HAProxy stats output to confirm that only one +# server in each backend group (the leader) is marked as up to avoid +# trying to bootstrap against the read-only follower. +ruby_block "wait for haproxy status socket" do + block do + connected = false + 10.times do + if ::File.exist?("/var/opt/opscode/haproxy/haproxy.sock") + require 'socket' + begin + UNIXSocket.new("/var/opt/opscode/haproxy/haproxy.sock") + connected = true + break + rescue + sleep 1 + end + else + sleep 1 + end + end + + if !connected + Chef::Log.fatal("HAProxy status socket never appeared properly!") + Chef::Log.fatal("See /var/log/opscode/haproxy/current for more information") + Kernel.exit! 1 + end + end +end + +ruby_block "wait for backend leader to stabilize" do + block do + stable = false + 10.times do + require 'socket' + begin + s = UNIXSocket.new("/var/opt/opscode/haproxy/haproxy.sock") + s.puts "show stat;quit" + _header = s.gets + table = [] + up_servers = { + "chef_backend_elasticsearch" => [], + "chef_backend_postgresql" => [], + } + while line = s.gets + table << line + end + + table.each do |l| + # show stat returns a csv formatted output. For now we do + # the parsing ourselves since we have simple needs. + split_line = l.split(",") + pxname = split_line[0] # Name of the service (ex: chef_backend_elasticsearch) + svname = split_line[1] # Name of this server or "BACKEND" or "FRONTEND" + state = split_line[17] # UP/DOWN/MAINT + + # Ignore lines of the table not related to backend server entries + if svname != "BACKEND" && state == "UP" && + ["chef_backend_elasticsearch", "chef_backend_postgresql"].include?(pxname) + up_servers[pxname] << svname + end + end + + # We expect the status checks to fail on all but 1 backend + # (the current leader) thus we wait for that to be the case. + if up_servers["chef_backend_elasticsearch"].count == 1 && + up_servers["chef_backend_postgresql"].count == 1 + stable = true + break + else + Chef::Log.warn("HAProxy still inconsistent:") + Chef::Log.warn(" Postgresql servers UP:") + up_servers["chef_backend_postgresql"].each do |server_name| + Chef::Log.warn(" -#{server_name}") + end + Chef::Log.warn(" Elasticsearch servers UP:") + up_servers["chef_backend_elasticsearch"].each do |server_name| + Chef::Log.warn(" -#{server_name}") + end + Chef::Log.warn("Retrying in 2 seconds") + sleep 2 + end + rescue StandardError => e + Chef::Log.warn("Error attempting to verify HAProxy State:") + Chef::Log.warn(" #{e}") + Chef::Log.warn("Retrying in 2 seconds") + sleep 2 + end + end + + if !stable + Chef::Log.fatal("HAProxy still showing multiple active backends") + Chef::Log.fatal("Please check /var/log/opscode/haproxy/current locally for problems.") + Chef::Log.fatal("Please check your backend cluster's status for problems.") + Kernel.exit! 1 + end + end +end diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb index d8adbe428a..70ea0a5ab7 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb @@ -1,3 +1,6 @@ +global + stats socket /var/opt/opscode/haproxy/haproxy.sock mode 600 + frontend postgresql bind <%= @listen %>:<%= @local_postgresql_port %> mode tcp From a3e3675a63ef787d4c849e06bfb1ef743b5daa0b Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Tue, 19 Apr 2016 13:49:59 +0100 Subject: [PATCH 3/4] [omnibus] Refactor HAProxy cookbook and increase test coverage - Move code for parsing HAProxy status socket responses into a HAProxyStatus class. - Move code for determining backend cluster members into a ChefBackend module. --- .../private-chef/libraries/chef_backend.rb | 28 +++++ .../private-chef/libraries/haproxy.rb | 55 ++++++++ .../private-chef/recipes/haproxy.rb | 89 ++++--------- .../spec/libraries/chef_backend_spec.rb | 42 +++++++ .../spec/libraries/haproxy_spec.rb | 118 ++++++++++++++++++ 5 files changed, 270 insertions(+), 62 deletions(-) create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/libraries/chef_backend.rb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/libraries/haproxy.rb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/chef_backend_spec.rb create mode 100644 omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/haproxy_spec.rb diff --git a/omnibus/files/private-chef-cookbooks/private-chef/libraries/chef_backend.rb b/omnibus/files/private-chef-cookbooks/private-chef/libraries/chef_backend.rb new file mode 100644 index 0000000000..482f624ff8 --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/libraries/chef_backend.rb @@ -0,0 +1,28 @@ +require 'json' +require 'uri' +require 'chef/http' + +module ChefBackend + ETCD_MEMBERS_URL = "/v2/members" + def self.configured_members(node) + ret = {} + node['private_chef']['chef_backend_members'].each_with_index do |member, i| + ret["backend#{i}"] = member + end + ret + end + + def self.etcd_members(ip, port) + ret = {} + raw_members = JSON.parse(etcd_get(ETCD_MEMBERS_URL, ip, port))["members"] + raw_members.each do |m| + ret[m["name"]] = URI.parse(m["peerURLs"].first).host + end + ret + end + + def self.etcd_get(url, ip, port) + client = Chef::HTTP.new("http://#{ip}:#{port}") + client.get(url) + end +end diff --git a/omnibus/files/private-chef-cookbooks/private-chef/libraries/haproxy.rb b/omnibus/files/private-chef-cookbooks/private-chef/libraries/haproxy.rb new file mode 100644 index 0000000000..0f6a62fed5 --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/libraries/haproxy.rb @@ -0,0 +1,55 @@ +class HAProxyStatus + + attr_accessor :socket + def initialize(sock) + @socket = sock + end + + def server_stats + stats(" -1 4 -1") + end + + def stats(args=nil) + socket.puts("show stat#{args}") + parse_stats_table(read_until_end(socket)) + end + + private + def read_until_end(socket) + ret = [] + while line = socket.gets + ret << line + end + ret + end + + def parse_stats_table(table) + return [] if table.empty? + header, *data = table + header = transform_header(header) + data.map do |line| + parse_status_line(line, header) + end.compact + end + + def transform_header(line) + columns = line.split(",").map(&:strip) + columns[0] = columns[0].gsub("# ", "") + columns + end + + # Incomplete parser for the output of show stats; + # + # Currently only returns the pxname, svname, status + # + def parse_status_line(line, header) + split_line = line.split(",").map(&:strip) + if split_line.first == "" && split_line.length == 1 # Empty line + nil + else + { pxname: split_line[header.index("pxname")], + svname: split_line[header.index("svname")], + status: split_line[header.index("status")] } + end + end +end diff --git a/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb b/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb index 61e1d6ef7e..302f540fa8 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb +++ b/omnibus/files/private-chef-cookbooks/private-chef/recipes/haproxy.rb @@ -14,7 +14,9 @@ # limitations under the License. # haproxy_dir = node['private_chef']['haproxy']['dir'] +haproxy_socket = ::File.join(haproxy_dir, "haproxy.sock") haproxy_log_dir = node['private_chef']['haproxy']['log_directory'] + [ haproxy_dir, haproxy_log_dir @@ -36,36 +38,17 @@ # def get_chef_backend_cluster_members members = nil - ret = {} node['private_chef']['chef_backend_members'].each do |member| begin - client = Chef::HTTP.new("http://#{member}:#{node['private_chef']['haproxy']['etcd_port']}") - members = JSON.parse(client.get("/v2/members"))["members"] - if members && !members.empty? - break - else - next - end - rescue - next + members = ChefBackend.etcd_members(member, node['private_chef']['haproxy']['etcd_port']) + break if members && !members.empty? + rescue StandardError => e + Chef::Log.warn("Error attempting to get cluster members from #{member}:") + Chef::Log.warn(" #{e}") + Chef::Log.warn("Trying next configured chef_backend member.") end end - if members - members.each do |m| - ret[m["name"]] = URI.parse(m["peerURLs"].first).host - end - ret - else - nil - end -end - -def configured_members - ret = {} - node['private_chef']['chef_backend_members'].each_with_index do |member, i| - ret["#{backend}#{i}"] = member - end - ret + members end chef_backend_members = begin @@ -74,14 +57,14 @@ def configured_members Chef::Log.info("Using Chef Backend members discovered via etcd") members else - Chef::Log.info("Member discovery failed") - Chef::Log.info("Using statically configured member list") - configured_members + Chef::Log.warn("Member discovery failed") + Chef::Log.warn("Using statically configured member list") + ChefBackend.configured_members(node) end rescue e - Chef::Log.info("member discoverry failed: #{e}") - Chef::Log.info("Using statically configured member list") - configured_members + Chef::Log.warn("member discoverry failed: #{e}") + Chef::Log.warn("Using statically configured member list") + ChefBackend.configured_members(node) end template File.join(haproxy_dir, "haproxy.cfg") do @@ -95,7 +78,6 @@ def configured_members component_runit_service "haproxy" - # On startup, all backend servers will be marked as UP. # # As the heathcheck on the non-leaders fails, they will be marked as @@ -110,10 +92,10 @@ def configured_members block do connected = false 10.times do - if ::File.exist?("/var/opt/opscode/haproxy/haproxy.sock") + if ::File.exist?(haproxy_socket) require 'socket' begin - UNIXSocket.new("/var/opt/opscode/haproxy/haproxy.sock") + UNIXSocket.new(haproxy_socket) connected = true break rescue @@ -136,49 +118,32 @@ def configured_members block do stable = false 10.times do - require 'socket' begin - s = UNIXSocket.new("/var/opt/opscode/haproxy/haproxy.sock") - s.puts "show stat;quit" - _header = s.gets - table = [] - up_servers = { + require 'socket' + s = HAProxyStatus.new(UNIXSocket.new(haproxy_socket)) + active_servers = { "chef_backend_elasticsearch" => [], - "chef_backend_postgresql" => [], + "chef_backend_postgresql" => [] } - while line = s.gets - table << line - end - - table.each do |l| - # show stat returns a csv formatted output. For now we do - # the parsing ourselves since we have simple needs. - split_line = l.split(",") - pxname = split_line[0] # Name of the service (ex: chef_backend_elasticsearch) - svname = split_line[1] # Name of this server or "BACKEND" or "FRONTEND" - state = split_line[17] # UP/DOWN/MAINT - # Ignore lines of the table not related to backend server entries - if svname != "BACKEND" && state == "UP" && - ["chef_backend_elasticsearch", "chef_backend_postgresql"].include?(pxname) - up_servers[pxname] << svname - end + s.server_stats.each do |server| + active_servers[server[:pxname]] << server[:svname] if server[:status] == "UP" end # We expect the status checks to fail on all but 1 backend # (the current leader) thus we wait for that to be the case. - if up_servers["chef_backend_elasticsearch"].count == 1 && - up_servers["chef_backend_postgresql"].count == 1 + if active_servers["chef_backend_elasticsearch"].count == 1 && + active_servers["chef_backend_postgresql"].count == 1 stable = true break else Chef::Log.warn("HAProxy still inconsistent:") Chef::Log.warn(" Postgresql servers UP:") - up_servers["chef_backend_postgresql"].each do |server_name| + active_servers["chef_backend_postgresql"].each do |server_name| Chef::Log.warn(" -#{server_name}") end Chef::Log.warn(" Elasticsearch servers UP:") - up_servers["chef_backend_elasticsearch"].each do |server_name| + active_servers["chef_backend_elasticsearch"].each do |server_name| Chef::Log.warn(" -#{server_name}") end Chef::Log.warn("Retrying in 2 seconds") diff --git a/omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/chef_backend_spec.rb b/omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/chef_backend_spec.rb new file mode 100644 index 0000000000..ebe6592762 --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/chef_backend_spec.rb @@ -0,0 +1,42 @@ +require_relative '../../libraries/chef_backend.rb' + +describe ChefBackend do + describe "#configured_members" do + it "Returns a hash of name => IP based on the chef_backend_members config" do + node = {"private_chef" => {"chef_backend_members" => ["1.2.3.4", "2.3.4.5"]}} + expect(ChefBackend.configured_members(node)).to eq({ + "backend0" => "1.2.3.4", + "backend1" => "2.3.4.5" + }) + end + end + + describe "#etcd_members" do + let(:http_client) { double("Chef::HTTP") } + let(:members_response) do + < "192.168.33.216", + "89cc3a0c2e3e51bdc5db1f97f63deb99" => "192.168.33.217", + "7caffa2c440aa682a2abbf4902d35fbd" => "192.168.33.215" + } + end + + it "makes a request /v2/members on the give host and port" do + expect(Chef::HTTP).to receive(:new).with("http://1.2.3.4:99").and_return(http_client) + expect(http_client).to receive(:get).with("/v2/members").and_return(members_response) + ChefBackend.etcd_members("1.2.3.4", 99) + end + + it "parses the raw response, returning a hash of name => ip" do + expect(Chef::HTTP).to receive(:new).with("http://1.2.3.4:99").and_return(http_client) + expect(http_client).to receive(:get).with("/v2/members").and_return(members_response) + expect(ChefBackend.etcd_members("1.2.3.4", 99)).to eq(parsed_response) + end + end +end diff --git a/omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/haproxy_spec.rb b/omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/haproxy_spec.rb new file mode 100644 index 0000000000..0e61c99e2e --- /dev/null +++ b/omnibus/files/private-chef-cookbooks/private-chef/spec/libraries/haproxy_spec.rb @@ -0,0 +1,118 @@ +require_relative '../../libraries/haproxy.rb' + +# +# FakeSocket pretends to be an HAProxy staus socket. It understand +# the following commands: +# +# - show stat +# - show stat -1 4 -1 +# +class FakeSocket + FULL_OUTPUT = <"postgresql", :svname=>"FRONTEND", :status=>"OPEN"}, + {:pxname=>"elasticsearch", :svname=>"FRONTEND", :status=>"OPEN"}, + {:pxname=>"chef_backend_postgresql", + :svname=>"4ba4ae0a6686cf6b67590e621459f1e2", + :status=>"DOWN"}, + {:pxname=>"chef_backend_postgresql", + :svname=>"89cc3a0c2e3e51bdc5db1f97f63deb99", + :status=>"DOWN"}, + {:pxname=>"chef_backend_postgresql", + :svname=>"7caffa2c440aa682a2abbf4902d35fbd", + :status=>"UP"}, + {:pxname=>"chef_backend_postgresql", :svname=>"BACKEND", :status=>"UP"}, + {:pxname=>"chef_backend_elasticsearch", + :svname=>"4ba4ae0a6686cf6b67590e621459f1e2", + :status=>"DOWN"}, + {:pxname=>"chef_backend_elasticsearch", + :svname=>"89cc3a0c2e3e51bdc5db1f97f63deb99", + :status=>"DOWN"}, + {:pxname=>"chef_backend_elasticsearch", + :svname=>"7caffa2c440aa682a2abbf4902d35fbd", + :status=>"UP"}, + {:pxname=>"chef_backend_elasticsearch", :svname=>"BACKEND", :status=>"UP"}]) + end + end + + describe "#server_stats" do + it "sends the 'show stat' with the correct arguments" do + expect(socket).to receive(:puts).with("show stat -1 4 -1") + subject.server_stats + end + + it "returns the parsed version of each line of data" do + expect(subject.server_stats).to eq([{:pxname=>"chef_backend_postgresql", + :svname=>"4ba4ae0a6686cf6b67590e621459f1e2", + :status=>"DOWN"}, + {:pxname=>"chef_backend_postgresql", + :svname=>"89cc3a0c2e3e51bdc5db1f97f63deb99", + :status=>"DOWN"}, + {:pxname=>"chef_backend_postgresql", + :svname=>"7caffa2c440aa682a2abbf4902d35fbd", + :status=>"UP"}, + {:pxname=>"chef_backend_elasticsearch", + :svname=>"4ba4ae0a6686cf6b67590e621459f1e2", + :status=>"DOWN"}, + {:pxname=>"chef_backend_elasticsearch", + :svname=>"89cc3a0c2e3e51bdc5db1f97f63deb99", + :status=>"DOWN"}, + {:pxname=>"chef_backend_elasticsearch", + :svname=>"7caffa2c440aa682a2abbf4902d35fbd", + :status=>"UP"}]) + end + end +end From b48d02908cdc3b34de61db0feeb49e2f2c8726b4 Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Wed, 20 Apr 2016 11:45:15 +0100 Subject: [PATCH 4/4] [haproxy] Use default-server to simplify configuration --- .../private-chef/templates/default/haproxy.cfg.erb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb index 70ea0a5ab7..5d82ceb87a 100644 --- a/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb +++ b/omnibus/files/private-chef-cookbooks/private-chef/templates/default/haproxy.cfg.erb @@ -14,13 +14,15 @@ frontend elasticsearch backend chef_backend_postgresql mode tcp option httpchk GET /leader HTTP/1.1\r\nHost:localhost:<%= @leaderl_healthcheck_port %>\r\n\r\n + default-server inter 2s rise 1 fall 1 <% @chef_backend_members.each do |name, ip| -%> - server <%= name %> <%= ip %>:<%= @remote_postgresql_port %> check port <%= @leaderl_healthcheck_port %> rise 1 fall 1 + server <%= name %> <%= ip %>:<%= @remote_postgresql_port %> check port <%= @leaderl_healthcheck_port %> <% end -%> backend chef_backend_elasticsearch mode tcp option httpchk GET /leader HTTP/1.1\r\nHost:localhost:<%= @leaderl_healthcheck_port %>\r\n\r\n + default-server inter 2s rise 1 fall 1 <% @chef_backend_members.each do |name, ip| -%> - server <%= name %> <%= ip %>:<%= @remote_elasticsearch_port %> check port <%= @leaderl_healthcheck_port %> rise 1 fall 1 + server <%= name %> <%= ip %>:<%= @remote_elasticsearch_port %> check port <%= @leaderl_healthcheck_port %> <% end -%> \ No newline at end of file