feat(conf) add support to configure nginx directives via kong.conf

kong-0.13.1-0.rockspec

@@ -76,6 +76,7 @@ build = {
     ["kong.cmd.version"] = "kong/cmd/version.lua",
     ["kong.cmd.utils.log"] = "kong/cmd/utils/log.lua",
     ["kong.cmd.utils.kill"] = "kong/cmd/utils/kill.lua",
+    ["kong.cmd.utils.env"] = "kong/cmd/utils/env.lua",
     ["kong.cmd.utils.nginx_signals"] = "kong/cmd/utils/nginx_signals.lua",
     ["kong.cmd.utils.prefix_handler"] = "kong/cmd/utils/prefix_handler.lua",
 

kong/cmd/utils/env.lua

@@ -0,0 +1,35 @@
+local pl_utils = require "pl.utils"
+local log = require "kong.cmd.utils.log"
+local fmt = string.format
+
+
+local cmd = [[ printenv ]]
+
+
+-- returns table and err
+local function read_all()
+  log.debug("reading environment variables: %s", cmd)
+  local vars = {}
+  local success, ret_code, stdout, stderr = pl_utils.executeex(cmd)
+  if not success or ret_code ~= 0 then
+    return nil, fmt("could not read environment variables (exit code: %d): %s",
+                    ret_code, stderr)
+  end
+
+  for line in stdout:gmatch("[^\r\n]+") do
+    local i = string.find(line, "=") -- match first =
+    if i then
+      local k = string.sub(line, 1, i - 1)
+      local v = string.sub(line, i + 1)
+      if k and v then
+        vars[k] = v
+      end
+    end
+  end
+  return vars
+end
+
+
+return {
+  read_all = read_all,
+}

kong/cmd/utils/nginx_signals.lua

@@ -107,6 +107,25 @@ function _M.start(kong_conf)
   return true
 end
 
+function _M.check_conf(kong_conf)
+  local nginx_bin, err = _M.find_nginx_bin()
+  if not nginx_bin then
+    return nil, err
+  end
+
+  local cmd = fmt("KONG_NGINX_CONF_CHECK=true %s -t -p %s -c %s",
+                  nginx_bin, kong_conf.prefix, "nginx.conf")
+  log.debug("checking nginx conf: %s", cmd)
+
+  local ok, retcode, _, stderr = pl_utils.executeex(cmd)
+  if not ok then
+    return false, ("failed to validate nginx configuration " ..
+                   "(exit code %d):\n%s"):format(retcode, stderr)
+  end
+
+  return true
+end
+
 function _M.stop(kong_conf)
   return send_signal(kong_conf, "TERM")
 end

kong/cmd/utils/prefix_handler.lua

@@ -13,6 +13,7 @@ local log = require "kong.cmd.utils.log"
 local constants = require "kong.constants"
 local ffi = require "ffi"
 local fmt = string.format
+local nginx_signals = require "kong.cmd.utils.nginx_signals"
 
 local function gen_default_ssl_cert(kong_config, admin)
   -- create SSL folder
@@ -194,8 +195,8 @@ local function prepare_prefix(kong_config, nginx_custom_template_path)
       return nil, err
     end
   end
-  if not pl_path.exists(kong_config.nginx_admin_acc_logs) then
-    local ok, err = pl_file.write(kong_config.nginx_admin_acc_logs, "")
+  if not pl_path.exists(kong_config.admin_acc_logs) then
+    local ok, err = pl_file.write(kong_config.admin_acc_logs, "")
     if not ok then
       return nil, err
     end
@@ -254,6 +255,12 @@ local function prepare_prefix(kong_config, nginx_custom_template_path)
   end
   pl_file.write(kong_config.nginx_kong_conf, nginx_kong_conf)
 
+  -- check if nginx conf is valid
+  local _, err = nginx_signals.check_conf(kong_config)
+  if err then
+    return nil, err
+  end
+
   -- write kong.conf in prefix (for workers and CLI)
   local buf = {
     "# *************************",

kong/conf_loader.lua

@@ -9,6 +9,7 @@ local pl_path = require "pl.path"
 local tablex = require "pl.tablex"
 local utils = require "kong.tools.utils"
 local log = require "kong.cmd.utils.log"
+local env = require "kong.cmd.utils.env"
 local ip = require "kong.tools.ip"
 local ciphers = require "kong.tools.ciphers"
 
@@ -28,11 +29,17 @@ local header_key_to_name = {
   [string.lower(headers.UPSTREAM_STATUS)] = headers.UPSTREAM_STATUS,
 }
 
+local CONF_KEY_PREFIXES = {
+  ["nginx_http_directives"] = "nginx_http_",
+  ["nginx_proxy_directives"] = "nginx_proxy_",
+  ["nginx_admin_directives"] = "nginx_admin_",
+}
+
 local PREFIX_PATHS = {
   nginx_pid = {"pids", "nginx.pid"},
   nginx_err_logs = {"logs", "error.log"},
   nginx_acc_logs = {"logs", "access.log"},
-  nginx_admin_acc_logs = {"logs", "admin_access.log"},
+  admin_acc_logs = {"logs", "admin_access.log"},
   nginx_conf = {"nginx.conf"},
   nginx_kong_conf = {"nginx-kong.conf"}
   ;
@@ -448,6 +455,24 @@ local function parse_listeners(values)
   return list
 end
 
+local function parse_nginx_directives(prefix, conf)
+  local directives = {}
+
+  if type(prefix) ~= "string" or type(conf) ~= "table" then
+    return directives
+  end
+
+  for k, v in pairs(conf) do
+    if type(k) == "string" then
+      local _, _, directive= string.find(k, prefix .. "(.+)")
+      if directive then
+        directives[directive] = v
+      end
+    end
+  end
+  return directives
+end
+
 --- Load Kong configuration
 -- The loaded configuration will have all properties from the default config
 -- merged with the (optionally) specified config file, environment variables
@@ -514,6 +539,50 @@ local function load(path, custom_conf)
   -- Merging & validation
   -----------------------
 
+  -- find dynamic keys that need to be loaded
+  do
+    local dynamic_keys = {}
+    local function find_dynamic_keys(t, prefix)
+      if not t then
+        return
+      end
+
+      for k, v in pairs(t) do
+        local _, _, directive = string.find(k, "(" .. prefix .. ".+)")
+        if directive then
+          dynamic_keys[directive] = true
+          if tonumber(v) then
+            t[k] = string.format("%q", v)
+          end
+        end
+      end
+    end
+
+    local env_vars, err = env.read_all()
+    if err then
+      return nil, err
+    end
+
+    local kong_env_vars = {}
+    for k, v in pairs(env_vars) do
+      local clean_k = string.lower(k)
+      local kong_var = string.match(clean_k, "^kong_(.+)")
+      if kong_var then
+        kong_env_vars[kong_var] = true
+      end
+    end
+
+    for _, prefix in pairs(CONF_KEY_PREFIXES) do
+      find_dynamic_keys(custom_conf, prefix)
+      find_dynamic_keys(kong_env_vars, prefix)
+      find_dynamic_keys(from_file_conf, prefix)
+    end
+
+    -- union (add dynamic keys to `defaults` to prevent removal of the keys
+    -- during the intersection that happens later)
+    defaults = tablex.merge(defaults, dynamic_keys, true)
+  end
+
   -- merge default conf with file conf, ENV variables and arg conf (with precedence)
   local conf = tablex.pairmap(overrides, defaults, from_file_conf, custom_conf)
 
@@ -525,6 +594,11 @@ local function load(path, custom_conf)
 
   conf = tablex.merge(conf, defaults) -- intersection (remove extraneous properties)
 
+  -- nginx directives from conf
+  for block, prefix in pairs(CONF_KEY_PREFIXES) do
+    conf[block] = parse_nginx_directives(prefix, conf)
+  end
+
   -- print alphabetically-sorted values
   do
     local conf_arr = {}

kong/init.lua

@@ -40,6 +40,14 @@ do
                    "directive is defined.")
     end
   end
+
+  -- if we're running nginx -t then don't initialize
+  if os.getenv("KONG_NGINX_CONF_CHECK") then
+    return {
+      init = function()
+      end,
+    }
+  end
 end
 
 require("kong.globalpatches")()

kong/templates/nginx_kong.lua

@@ -44,6 +44,10 @@ lua_ssl_trusted_certificate '${{LUA_SSL_TRUSTED_CERTIFICATE}}';
 lua_ssl_verify_depth ${{LUA_SSL_VERIFY_DEPTH}};
 > end
 
+> for k, v in pairs(nginx_http_directives)  do
+    $(k) $(v);
+> end
+
 init_by_lua_block {
     kong = require 'kong'
     kong.init()
@@ -101,6 +105,10 @@ server {
     set_real_ip_from   $(trusted_ips[i]);
 > end
 
+> for k, v in pairs(nginx_proxy_directives)  do
+    $(k) $(v);
+> end
+
     location / {
         set $upstream_host               '';
         set $upstream_upgrade            '';
@@ -180,6 +188,10 @@ server {
     ssl_ciphers ${{SSL_CIPHERS}};
 > end
 
+> for k, v in pairs(nginx_admin_directives)  do
+    $(k) $(v);
+> end
+
     location / {
         default_type application/json;
         content_by_lua_block {

spec/01-unit/002-conf_loader_spec.lua

@@ -143,7 +143,7 @@ describe("Configuration loader", function()
     assert.equal("/usr/local/kong/pids/nginx.pid", conf.nginx_pid)
     assert.equal("/usr/local/kong/logs/error.log", conf.nginx_err_logs)
     assert.equal("/usr/local/kong/logs/access.log", conf.nginx_acc_logs)
-    assert.equal("/usr/local/kong/logs/admin_access.log", conf.nginx_admin_acc_logs)
+    assert.equal("/usr/local/kong/logs/admin_access.log", conf.admin_acc_logs)
     assert.equal("/usr/local/kong/nginx.conf", conf.nginx_conf)
     assert.equal("/usr/local/kong/nginx-kong.conf", conf.nginx_kong_conf)
     assert.equal("/usr/local/kong/.kong_env", conf.kong_env)
@@ -170,6 +170,29 @@ describe("Configuration loader", function()
     local conf = assert(conf_loader("spec/fixtures/to-strip.conf"))
     assert.equal("test#123", conf.pg_password)
   end)
+  it("loads flexible prefix based configs from a file", function()
+    local conf = assert(conf_loader("spec/fixtures/nginx-directives.conf", nil))
+    assert.equal("custom_cache 5m",
+                 conf.nginx_http_directives["lua_shared_dict"])
+    assert.equal("8 24k",
+                 conf.nginx_http_directives["large_client_header_buffers"])
+  end)
+  it("quotes numeric flexible prefix based configs", function()
+    local conf = assert(conf_loader(nil, {
+      ["nginx_http_max_pending_timers"] = 4096,
+    }))
+    assert.equal("\"4096\"", conf.nginx_http_directives["max_pending_timers"])
+  end)
+  it("accepts flexible config values with highest precedence", function()
+    local conf = assert(conf_loader("spec/fixtures/nginx-directives.conf", {
+      ["nginx_http_large_client_header_buffers"] = "4 16k",
+      ["nginx_http_lua_shared_dict"] = "custom_cache 2m",
+    }))
+    assert.equal("custom_cache 2m",
+                 conf.nginx_http_directives["lua_shared_dict"])
+    assert.equal("4 16k",
+                 conf.nginx_http_directives["large_client_header_buffers"])
+  end)
 
   describe("nginx_user", function()
     it("is nil by default", function()

spec/01-unit/003-prefix_handler_spec.lua

@@ -379,7 +379,7 @@ describe("NGINX conf compiler", function()
       assert.truthy(exists(tmp_config.nginx_kong_conf))
       assert.truthy(exists(tmp_config.nginx_err_logs))
       assert.truthy(exists(tmp_config.nginx_acc_logs))
-      assert.truthy(exists(tmp_config.nginx_admin_acc_logs))
+      assert.truthy(exists(tmp_config.admin_acc_logs))
     end)
     it("dumps Kong conf", function()
       assert(prefix_handler.prepare_prefix(tmp_config))

spec/02-integration/02-cmd/09-prepare_spec.lua

@@ -45,5 +45,12 @@ describe("kong prepare", function()
       assert.is_string(stderr)
       assert.matches("Error: no file at: foobar.conf", stderr, nil, true)
     end)
+    it("on invalid nginx directive", function()
+      local ok, stderr = helpers.kong_exec("prepare --conf spec/fixtures/invalid_nginx_directives.conf" ..
+                                           " -p " .. TEST_PREFIX)
+      assert.False(ok)
+      assert.is_string(stderr)
+      assert.matches("[emerg] unknown directive \"random_directive\"", stderr, nil, true)
+    end)
   end)
 end)

spec/02-integration/05-proxy/15-custom_nginx_directive_spec.lua

@@ -0,0 +1,58 @@
+local helpers = require "spec.helpers"
+
+
+describe("Custom NGINX directives", function()
+  local proxy_client
+  local bp
+
+  local function start(config)
+    return function()
+      bp.routes:insert {
+        hosts = { "headers-inspect.com" },
+      }
+
+      config = config or {}
+      config.nginx_conf = "spec/fixtures/custom_nginx.template"
+
+      assert(helpers.start_kong(config))
+    end
+  end
+
+  setup(function()
+    bp = helpers.get_db_utils()
+  end)
+
+  before_each(function()
+    proxy_client = helpers.proxy_client()
+  end)
+
+  after_each(function()
+    if proxy_client then
+      proxy_client:close()
+    end
+  end)
+
+  describe("with config value 'nginx_proxy_add_header=foo-header bar-value'", function()
+
+    setup(start{
+      ["nginx_proxy_add_header"] = "foo-header bar-value"
+    })
+
+    teardown(helpers.stop_kong)
+
+    it("header 'foo-header' should be inserted", function()
+      local res = assert(proxy_client:send {
+        method  = "GET",
+        path    = "/get",
+        headers = {
+          host  = "headers-inspect.com",
+        }
+      })
+
+      assert.res_status(200, res)
+      assert.equal("bar-value", res.headers["foo-header"])
+    end)
+
+  end)
+end)
+