Use HAProxy to route Postgresql and ElasticSearch connections

omnibus/config/projects/chef-server.rb

@@ -52,6 +52,7 @@
 dependency "rabbitmq"
 dependency "redis" # dynamic routing controls
 dependency "opscode-solr4"
+dependency "haproxy"
 dependency "opscode-expander"
 dependency "pg-gem" # used by private-chef-ctl reconfigure
 

omnibus/config/software/haproxy.rb

@@ -0,0 +1,71 @@
+#
+# Copyright 2016 Chef Software, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name "haproxy"
+default_version "1.6.4"
+
+dependency "zlib"
+dependency "pcre"
+dependency "openssl"
+
+license "GPLv2"
+license_file "LICENSE"
+
+# HTTPS is available but certificate validation fails on OS X
+source url: "http://www.haproxy.org/download/1.6/src/haproxy-#{version}.tar.gz",
+       sha256: "e5fa3c604f1fe9ecb6974ccda6705c105ebee14b3a913069fb08f00e860cd230"
+
+relative_path "haproxy-#{version}"
+
+build do
+  env = with_standard_compiler_flags(with_embedded_path)
+  #
+  # Many of these are the same environment variables that debian sets
+  # PREFIX and TARGET are mandatory to get it building under omnibus.
+  #
+  build_options = {
+   "PREFIX" => "#{install_dir}/embedded",
+   # Use libpcre for regex, libpcre > 8.32 required
+   # for JIT
+   "USE_PCRE" => "1",
+   "USE_PCRE_JIT" => "1",
+   "USE_ZLIB" => "1",
+   "USE_OPENSSL" => "1",
+  }
+  # Required to resolve hostnames to IPv6 addresses
+  # off-by-default because of prolems on older glibc's
+  # TODO(ssd): Should we turn this off on RHEL5?
+  build_options['USE_GETADDRINFO'] = "1"
+  if intel?
+    build_options["USE_REGPARM"] = "1"
+  end
+  build_options['TARGET'] = if ohai["kernel"] && ohai["kernel"]["name"] == "Linux"
+                              version = Gem::Version.new(String(ohai["kernel"]["release"]).split("-").first)
+                              case
+                              when version >= Gem::Version.new("2.6.28")
+                                "linux2628"
+                              when version >= Gem::Version.new("2.6")
+                                "linux26"
+                              else
+                                "linux24e"
+                              end
+                            else
+                              "generic"
+                            end
+  build_args = ""
+  build_options.each { |k,v| build_args << " #{k}=#{v}"}
+  make "haproxy #{build_args}", env: env
+  make "install #{build_args}", env: env
+end

omnibus/files/private-chef-cookbooks/private-chef/attributes/default.rb

@@ -93,6 +93,26 @@
 ####
 default['private_chef']['server-api-version'] = 0
 
+####
+# HAProxy
+#
+# HAProxy is only used when use_chef_backend is true. All Postgresql
+# and Elasticsearch requests are routed to it and then forwarded to
+# the current chef-backend leader.
+####
+default['private_chef']['haproxy']['enable'] = true
+default['private_chef']['haproxy']['ha'] = false
+default['private_chef']['haproxy']['dir'] = "/var/opt/opscode/haproxy"
+default['private_chef']['haproxy']['log_directory'] = "/var/log/opscode/haproxy"
+default['private_chef']['haproxy']['log_rotation']['file_maxbytes'] = 104857600
+default['private_chef']['haproxy']['log_rotation']['num_to_keep'] = 10
+default['private_chef']['haproxy']['listen'] = '0.0.0.0'
+default['private_chef']['haproxy']['local_postgresql_port'] = 5432
+default['private_chef']['haproxy']['remote_postgresql_port'] = 5432
+default['private_chef']['haproxy']['local_elasticsearch_port'] = 9200
+default['private_chef']['haproxy']['remote_elasticsearch_port'] = 9200
+default['private_chef']['haproxy']['leaderl_healthcheck_port'] = 7331
+default['private_chef']['haproxy']['etcd_port'] = 2379
 
 ####
 # RabbitMQ

omnibus/files/private-chef-cookbooks/private-chef/libraries/chef_backend.rb

@@ -0,0 +1,28 @@
+require 'json'
+require 'uri'
+require 'chef/http'
+
+module ChefBackend
+  ETCD_MEMBERS_URL = "/v2/members"
+  def self.configured_members(node)
+    ret = {}
+    node['private_chef']['chef_backend_members'].each_with_index do |member, i|
+      ret["backend#{i}"] = member
+    end
+    ret
+  end
+
+  def self.etcd_members(ip, port)
+    ret = {}
+    raw_members = JSON.parse(etcd_get(ETCD_MEMBERS_URL, ip, port))["members"]
+    raw_members.each do |m|
+      ret[m["name"]] = URI.parse(m["peerURLs"].first).host
+    end
+    ret
+  end
+
+  def self.etcd_get(url, ip, port)
+    client = Chef::HTTP.new("http://#{ip}:#{port}")
+    client.get(url)
+  end
+end

omnibus/files/private-chef-cookbooks/private-chef/libraries/haproxy.rb

@@ -0,0 +1,55 @@
+class HAProxyStatus
+
+  attr_accessor :socket
+  def initialize(sock)
+    @socket = sock
+  end
+
+  def server_stats
+    stats(" -1 4 -1")
+  end
+
+  def stats(args=nil)
+    socket.puts("show stat#{args}")
+    parse_stats_table(read_until_end(socket))
+  end
+
+  private
+  def read_until_end(socket)
+    ret = []
+    while line = socket.gets
+      ret << line
+    end
+    ret
+  end
+
+  def parse_stats_table(table)
+    return [] if table.empty?
+    header, *data = table
+    header = transform_header(header)
+    data.map do |line|
+      parse_status_line(line, header)
+    end.compact
+  end
+
+  def transform_header(line)
+    columns = line.split(",").map(&:strip)
+    columns[0] = columns[0].gsub("# ", "")
+    columns
+  end
+
+  # Incomplete parser for the output of show stats;
+  #
+  # Currently only returns the pxname, svname, status
+  #
+  def parse_status_line(line, header)
+    split_line = line.split(",").map(&:strip)
+    if split_line.first == "" && split_line.length == 1 # Empty line
+      nil
+    else
+      { pxname: split_line[header.index("pxname")],
+        svname: split_line[header.index("svname")],
+        status: split_line[header.index("status")] }
+    end
+  end
+end

omnibus/files/private-chef-cookbooks/private-chef/libraries/preflight_checks.rb

@@ -125,8 +125,13 @@ def initialize(node)
   # the defaults set in the recipe.
   def run!
     begin
-      BootstrapPreflightValidator.new(node).run!
-      PostgresqlPreflightValidator.new(node).run!
+      # When Chef Backend is configured, this is too early to verify
+      # postgresql accessibility since we need to configure HAProxy
+      # first
+      if ! PrivateChef['use_chef_backend']
+        BootstrapPreflightValidator.new(node).run!
+        PostgresqlPreflightValidator.new(node).run!
+      end
       SolrPreflightValidator.new(node).run!
       BookshelfPreflightValidator.new(node).run!
     rescue PreflightValidationFailed => e

omnibus/files/private-chef-cookbooks/private-chef/libraries/private_chef.rb

@@ -23,6 +23,9 @@ module PrivateChef
   # Set this for default org mode
   default_orgname nil
 
+  use_chef_backend false
+  chef_backend_members []
+
   addons Mash.new
   rabbitmq Mash.new
   external_rabbitmq Mash.new
@@ -212,6 +215,8 @@ def generate_hash
         "ldap",
         "user",
         "ha",
+        "use_chef_backend",
+        "chef_backend_members",
         "disabled_plugins",
         "enabled_plugins",
         "license",
@@ -224,7 +229,9 @@ def generate_hash
       (default_keys | keys_from_extensions).each do |key|
         # @todo: Just pick a naming convention and adhere to it
         # consistently
-        rkey = if key =~ /^oc_/ || key == "redis_lb"
+        rkey = if key =~ /^oc_/ || key == "redis_lb" ||
+                  key == "use_chef_backend" ||
+                  key == "chef_backend_members"
                  key # leave oc_* keys as is
                else
                  key.gsub('_', '-')

omnibus/files/private-chef-cookbooks/private-chef/recipes/default.rb

@@ -121,6 +121,10 @@
 # Run plugins first, mostly for ha
 include_recipe "private-chef::plugin_chef_run"
 
+if node['private_chef']['use_chef_backend']
+  include_recipe "private-chef::haproxy"
+end
+
 # Configure Services
 [
   "rabbitmq",