diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index bc3c6949c454..0d8d2c61725d 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -23,5 +23,5 @@ SUMMARY_GOES_HERE - Kong debug-level startup logs (`$ kong start --vv`) - Kong error logs (`/logs/error.log`) - Kong configuration (the output of a GET request to Kong's Admin port - see - https://getkong.org/docs/latest/admin-api/#endpoint) + https://docs.konghq.com/latest/admin-api/#retrieve-node-information) - Operating system diff --git a/.travis.yml b/.travis.yml index 2037da7059f3..7c0a79723e1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ addons: env: global: - LUAROCKS=3.0.4 - - OPENSSL=1.1.1 + - OPENSSL=1.1.1a - OPENRESTY_BASE=1.13.6.2 - OPENRESTY_LATEST=1.13.6.2 - OPENRESTY=$OPENRESTY_BASE diff --git a/CHANGELOG.md b/CHANGELOG.md index 13fb4ad25cdd..4d14e096dacc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2297,7 +2297,7 @@ This release includes support for PostgreSQL as Kong's primary datastore! > **internal** > - replace globals with singleton pattern thanks to [@mars](https://github.com/mars). -> - fixed resolution mismatches when using deep paths in the path resolver thanks to [siddharthkchatterjee](https://github.com/siddharthkchatterjee) +> - fixed resolution mismatches when using deep paths in the path resolver. [Back to TOC](#table-of-contents) @@ -2366,7 +2366,7 @@ This release contains tiny bug fixes that were especially annoying for complex C We would recommended to consult the suggested [0.6 upgrade path](https://github.com/Kong/kong/blob/master/UPGRADE.md#upgrade-to-06x) for this release. -- [Serf](https://www.serfdom.io) is now a Kong dependency. It allows Kong nodes to communicate between each other opening the way to many features and improvements. +- [Serf](https://www.serf.io/) is now a Kong dependency. It allows Kong nodes to communicate between each other opening the way to many features and improvements. - The configuration file changed. Some properties were renamed, others were moved, and some are new. We would recommend checking out the new default configuration file. - Drop the Lua 5.1 dependency which was only used by the CLI. The CLI now runs with LuaJIT, which is consistent with other Kong components (Luarocks and OpenResty) already relying on LuaJIT. Make sure the LuaJIT interpreter is included in your `$PATH`. [#799](https://github.com/Kong/kong/pull/799) @@ -2443,7 +2443,7 @@ Other additions include: > ***internal*** > - Event bus for local and cluster-wide events propagation. Plans for this event bus is to be widely used among Kong in the future. -> - The Kong Public Lua API (Lua helpers integrated in Kong such as DAO and Admin API helpers) is now documented with [ldoc](http://stevedonovan.github.io/ldoc/) format and published on [the online documentation](https://getkong.org/docs/latest/lua-reference/). +> - The Kong Public Lua API (Lua helpers integrated in Kong such as DAO and Admin API helpers) is now documented with [ldoc](http://stevedonovan.github.io/ldoc/). > - Work has been done to restore the reliability of the CI platforms. > - Migrations can now execute DML queries (instead of DDL queries only). Handy for migrations implying plugin configuration changes, plugins renamings etc... [#770](https://github.com/Kong/kong/pull/770) @@ -2519,8 +2519,7 @@ With new plugins, many improvements and bug fixes, this release comes with break ### Breaking changes -Several breaking changes are introduced. You will have to slightly change your configuration file and a migration script will take care of updating your database cluster. **Please follow the instructions in [UPDATE.md](/UPDATE.md#update-to-kong-050) for an update without downtime**. - +Several breaking changes are introduced. You will have to slightly change your configuration file and a migration script will take care of updating your database cluster. **Please follow the instructions in [UPGRADE.md](/UPGRADE.md#update-to-kong-050) for an update without downtime.** - Many plugins were renamed due to new naming conventions for consistency. [#480](https://github.com/Kong/kong/issues/480) - In the configuration file, the Cassandra `hosts` property was renamed to `contact_points`. [#513](https://github.com/Kong/kong/issues/513) - Properties belonging to APIs entities have been renamed for clarity. [#513](https://github.com/Kong/kong/issues/513) diff --git a/kong.conf.default b/kong.conf.default index d5da7f26aa79..57928be4d89f 100644 --- a/kong.conf.default +++ b/kong.conf.default @@ -409,15 +409,20 @@ #cassandra_consistency = ONE # Consistency setting to use when reading/ # writing to the Cassandra cluster. -#cassandra_lb_policy = RoundRobin # Load balancing policy to use when - # distributing queries across your Cassandra - # cluster. - # Accepted values are: - # `RoundRobin`, `RequestRoundRobin`, - # `DCAwareRoundRobin`, and - # `RequestDCAwareRoundRobin`. - # Prefer the later if and only if you are - # using a multi-datacenter cluster. +#cassandra_lb_policy = RequestRoundRobin # Load balancing policy to use when + # distributing queries across your + # Cassandra cluster. + # Accepted values are: + # `RoundRobin`, `RequestRoundRobin`, + # `DCAwareRoundRobin`, and + # `RequestDCAwareRoundRobin`. + # Policies prefixed with "Request" + # make efficient use of established + # connections throughout the same + # request. + # Prefer "DCAware" policies if and + # only if you are using a + # multi-datacenter cluster. #cassandra_local_datacenter = # When using the `DCAwareRoundRobin` # or `RequestDCAwareRoundRobin` load diff --git a/kong/dao/migrations/cassandra.lua b/kong/dao/migrations/cassandra.lua index 37921f90f783..91f0889ae005 100644 --- a/kong/dao/migrations/cassandra.lua +++ b/kong/dao/migrations/cassandra.lua @@ -274,7 +274,7 @@ return { ALTER TABLE apis ADD http_if_terminated boolean; ]], down = [[ - DROP INDEX ssl_servers_names_ssl_certificate_idx; + DROP INDEX ssl_servers_names_ssl_certificate_id_idx; DROP TABLE ssl_certificates; DROP TABLE ssl_servers_names; @@ -770,7 +770,7 @@ return { }, { name = "2018-03-27-002500_drop_old_ssl_tables", up = [[ - DROP INDEX ssl_servers_names_ssl_certificate_id_idx; + DROP INDEX IF EXISTS ssl_servers_names_ssl_certificate_id_idx; DROP TABLE ssl_certificates; DROP TABLE ssl_servers_names; ]], diff --git a/kong/plugins/acl/handler.lua b/kong/plugins/acl/handler.lua index f27dbda16ef6..e0184abb3216 100644 --- a/kong/plugins/acl/handler.lua +++ b/kong/plugins/acl/handler.lua @@ -40,6 +40,7 @@ function ACLHandler:access(conf) config.type = (conf.blacklist or EMPTY)[1] and BLACK or WHITE config.groups = config.type == BLACK and conf.blacklist or conf.whitelist config.cache = setmetatable({}, mt_cache) + config_cache[conf] = config end -- get the consumer/credentials diff --git a/kong/plugins/aws-lambda/handler.lua b/kong/plugins/aws-lambda/handler.lua index 09141d2f805f..5ea073b7a814 100644 --- a/kong/plugins/aws-lambda/handler.lua +++ b/kong/plugins/aws-lambda/handler.lua @@ -90,6 +90,7 @@ function AWSLambdaHandler:access(conf) AWSLambdaHandler.super.access(self) local upstream_body = kong.table.new(0, 6) + local var = ngx.var if conf.forward_request_body or conf.forward_request_headers or conf.forward_request_method or conf.forward_request_uri @@ -199,6 +200,14 @@ function AWSLambdaHandler:access(conf) local content = res:read_body() local headers = res.headers + if var.http2 then + headers["Connection"] = nil + headers["Keep-Alive"] = nil + headers["Proxy-Connection"] = nil + headers["Upgrade"] = nil + headers["Transfer-Encoding"] = nil + end + local ok, err = client:set_keepalive(conf.keepalive) if not ok then kong.log.err(err) diff --git a/kong/plugins/cors/handler.lua b/kong/plugins/cors/handler.lua index f596bfde5f99..258078d8bd39 100644 --- a/kong/plugins/cors/handler.lua +++ b/kong/plugins/cors/handler.lua @@ -1,4 +1,6 @@ local BasePlugin = require "kong.plugins.base_plugin" +local lrucache = require "resty.lrucache" +local url = require "socket.url" local kong = kong @@ -18,6 +20,36 @@ CorsHandler.PRIORITY = 2000 CorsHandler.VERSION = "1.0.0" +-- per-plugin cache of normalized origins for runtime comparison +local mt_cache = { __mode = "k" } +local config_cache = setmetatable({}, mt_cache) + + +-- per-worker cache of parsed requests origins with 1000 slots +local normalized_req_domains = lrucache.new(10e3) + + +local function normalize_origin(domain) + local parsed_obj = assert(url.parse(domain)) + if not parsed_obj.host then + return domain + end + + local port = parsed_obj.port + if not port and parsed_obj.scheme then + if parsed_obj.scheme == "http" then + port = 80 + + elseif parsed_obj.scheme == "https" then + port = 443 + end + end + + return (parsed_obj.scheme and parsed_obj.scheme .. "://" or "") .. + parsed_obj.host .. (port and ":" .. port or "") +end + + local function configure_origin(conf) local n_origins = conf.origins ~= nil and #conf.origins or 0 local set_header = kong.response.set_header @@ -51,8 +83,26 @@ local function configure_origin(conf) local req_origin = kong.request.get_header("origin") if req_origin then - for _, domain in ipairs(conf.origins) do - local from, _, err = re_find(req_origin, domain, "jo") + local normalized_domains = config_cache[conf] + if not normalized_domains then + normalized_domains = {} + + for _, domain in ipairs(conf.origins) do + table.insert(normalized_domains, normalize_origin(domain)) + end + + config_cache[conf] = normalized_domains + end + + local normalized_req_origin = normalized_req_domains:get(req_origin) + if not normalized_req_origin then + normalized_req_origin = normalize_origin(req_origin) + normalized_req_domains:set(req_origin, normalized_req_origin) + end + + for _, normalized_domain in ipairs(normalized_domains) do + local from, _, err = re_find(normalized_req_origin, + normalized_domain .. "$", "ajo") if err then kong.log.err("could not search for domain: ", err) end diff --git a/kong/plugins/rate-limiting/policies/init.lua b/kong/plugins/rate-limiting/policies/init.lua index ef01cb7b97e1..ba3d66fedd4c 100644 --- a/kong/plugins/rate-limiting/policies/init.lua +++ b/kong/plugins/rate-limiting/policies/init.lua @@ -57,6 +57,9 @@ local get_local_key = function(conf, identifier, period, period_date) end +local sock_opts = {} + + local EXPIRATIONS = { second = 1, minute = 60, @@ -147,7 +150,13 @@ return { increment = function(conf, limits, identifier, current_timestamp, value) local red = redis:new() red:set_timeout(conf.redis_timeout) - local ok, err = red:connect(conf.redis_host, conf.redis_port) + -- use a special pool name only if redis_database is set to non-zero + -- otherwise use the default pool name host:port + sock_opts.pool = conf.redis_database and + conf.redis_host .. ":" .. conf.redis_port .. + ":" .. conf.redis_database + local ok, err = red:connect(conf.redis_host, conf.redis_port, + sock_opts) if not ok then kong.log.err("failed to connect to Redis: ", err) return nil, err @@ -159,27 +168,24 @@ return { return nil, err end - if times == 0 and is_present(conf.redis_password) then - local ok, err = red:auth(conf.redis_password) - if not ok then - kong.log.err("failed to auth Redis: ", err) - return nil, err + if times == 0 then + if is_present(conf.redis_password) then + local ok, err = red:auth(conf.redis_password) + if not ok then + kong.log.err("failed to auth Redis: ", err) + return nil, err + end end - end - if times ~= 0 or conf.redis_database then - -- The connection pool is shared between multiple instances of this - -- plugin, and instances of the response-ratelimiting plugin. - -- Because there isn't a way for us to know which Redis database a given - -- socket is connected to without a roundtrip, we force the retrieved - -- socket to select the desired database. - -- When the connection is fresh and the database is the default one, we - -- can skip this roundtrip. + if conf.redis_database ~= 0 then + -- Only call select first time, since we know the connection is shared + -- between instances that use the same redis database - local ok, err = red:select(conf.redis_database or 0) - if not ok then - kong.log.err("failed to change Redis database: ", err) - return nil, err + local ok, err = red:select(conf.redis_database) + if not ok then + kong.log.err("failed to change Redis database: ", err) + return nil, err + end end end diff --git a/kong/plugins/response-ratelimiting/policies/init.lua b/kong/plugins/response-ratelimiting/policies/init.lua index 94c7aef0cd0f..57e46f7a04f9 100644 --- a/kong/plugins/response-ratelimiting/policies/init.lua +++ b/kong/plugins/response-ratelimiting/policies/init.lua @@ -56,6 +56,9 @@ local get_local_key = function(conf, identifier, period_date, name, period) end +local sock_opts = {} + + local EXPIRATIONS = { second = 1, minute = 60, @@ -143,7 +146,13 @@ return { increment = function(conf, identifier, current_timestamp, value, name) local red = redis:new() red:set_timeout(conf.redis_timeout) - local ok, err = red:connect(conf.redis_host, conf.redis_port) + -- use a special pool name only if redis_database is set to non-zero + -- otherwise use the default pool name host:port + sock_opts.pool = conf.redis_database and + conf.redis_host .. ":" .. conf.redis_port .. + ":" .. conf.redis_database + local ok, err = red:connect(conf.redis_host, conf.redis_port, + sock_opts) if not ok then kong.log.err("failed to connect to Redis: ", err) return nil, err @@ -155,27 +164,24 @@ return { return nil, err end - if times == 0 and is_present(conf.redis_password) then - local ok, err = red:auth(conf.redis_password) - if not ok then - kong.log.err("failed to auth Redis: ", err) - return nil, err + if times == 0 then + if is_present(conf.redis_password) then + local ok, err = red:auth(conf.redis_password) + if not ok then + kong.log.err("failed to auth Redis: ", err) + return nil, err + end end - end - if times ~= 0 or conf.redis_database then - -- The connection pool is shared between multiple instances of this - -- plugin, and instances of the response-ratelimiting plugin. - -- Because there isn't a way for us to know which Redis database a given - -- socket is connected to without a roundtrip, we force the retrieved - -- socket to select the desired database. - -- When the connection is fresh and the database is the default one, we - -- can skip this roundtrip. + if conf.redis_database ~= 0 then + -- Only call select first time, since we know the connection is shared + -- between instances that use the same redis database - local ok, err = red:select(conf.redis_database or 0) - if not ok then - kong.log.err("failed to change Redis database: ", err) - return nil, err + local ok, err = red:select(conf.redis_database) + if not ok then + kong.log.err("failed to change Redis database: ", err) + return nil, err + end end end diff --git a/kong/templates/kong_defaults.lua b/kong/templates/kong_defaults.lua index 80a3cd633557..0050232accd5 100644 --- a/kong/templates/kong_defaults.lua +++ b/kong/templates/kong_defaults.lua @@ -54,7 +54,7 @@ cassandra_ssl_verify = off cassandra_username = kong cassandra_password = NONE cassandra_consistency = ONE -cassandra_lb_policy = RoundRobin +cassandra_lb_policy = RequestRoundRobin cassandra_local_datacenter = NONE cassandra_repl_strategy = SimpleStrategy cassandra_repl_factor = 1 diff --git a/spec-old-api/03-plugins/14-cors/01-access_spec.lua b/spec-old-api/03-plugins/14-cors/01-access_spec.lua index b95a996ca34b..e1044570322e 100644 --- a/spec-old-api/03-plugins/14-cors/01-access_spec.lua +++ b/spec-old-api/03-plugins/14-cors/01-access_spec.lua @@ -326,11 +326,11 @@ describe("Plugin: cors (access)", function() method = "GET", headers = { ["Host"] = "cors6.com", - ["Origin"] = "http://www.example.com" + ["Origin"] = "example.com" } }) assert.res_status(200, res) - assert.equal("http://www.example.com", res.headers["Access-Control-Allow-Origin"]) + assert.equal("example.com", res.headers["Access-Control-Allow-Origin"]) local domains = { ["example.com"] = true, @@ -359,7 +359,7 @@ describe("Plugin: cors (access)", function() method = "GET", headers = { ["Host"] = "cors6.com", - ["Origin"] = "http://www.example.net" + ["Origin"] = "example.net" } }) assert.res_status(200, res) diff --git a/spec/03-plugins/14-cors/01-access_spec.lua b/spec/03-plugins/14-cors/01-access_spec.lua index f37bc4fd2585..6a4f74f49fcd 100644 --- a/spec/03-plugins/14-cors/01-access_spec.lua +++ b/spec/03-plugins/14-cors/01-access_spec.lua @@ -44,17 +44,25 @@ for _, strategy in helpers.each_strategy() do hosts = { "cors9.com" }, }) + local route10 = bp.routes:insert({ + hosts = { "cors10.com" }, + }) + + local route11 = bp.routes:insert({ + hosts = { "cors11.com" }, + }) + local mock_service = bp.services:insert { host = "127.0.0.2", port = 26865, } - local route10 = bp.routes:insert { + local route_timeout = bp.routes:insert { hosts = { "cors-timeout.com" }, service = mock_service, } - local route11 = bp.routes:insert { + local route_error = bp.routes:insert { hosts = { "cors-error.com" }, } @@ -149,6 +157,22 @@ for _, strategy in helpers.each_strategy() do bp.plugins:insert { name = "cors", route = { id = route10.id }, + config = { + origins = { "http://my-site.com", "http://my-other-site.com" }, + } + } + + bp.plugins:insert { + name = "cors", + route = { id = route11.id }, + config = { + origins = { "http://my-site.com", "https://my-other-site.com:9000" }, + } + } + + bp.plugins:insert { + name = "cors", + route = { id = route_timeout.id }, config = { origins = { "example.com" }, methods = { "GET" }, @@ -161,7 +185,7 @@ for _, strategy in helpers.each_strategy() do bp.plugins:insert { name = "cors", - route = { id = route11.id }, + route = { id = route_error.id }, config = { origins = { "example.com" }, methods = { "GET" }, @@ -174,7 +198,7 @@ for _, strategy in helpers.each_strategy() do bp.plugins:insert { name = "error-generator-post", - route = { id = route11.id }, + route = { id = route_error.id }, config = { access = true, }, @@ -298,6 +322,44 @@ for _, strategy in helpers.each_strategy() do assert.equal("0", res.headers["Content-Length"]) assert.equal("origin,accepts", res.headers["Access-Control-Allow-Headers"]) end) + + it("properly validates flat strings", function() + -- Legitimate origins + local res = assert(proxy_client:send { + method = "OPTIONS", + headers = { + ["Host"] = "cors10.com", + ["Origin"] = "http://my-site.com" + } + }) + + assert.res_status(200, res) + assert.equal("http://my-site.com", res.headers["Access-Control-Allow-Origin"]) + + -- Illegitimate origins + res = assert(proxy_client:send { + method = "OPTIONS", + headers = { + ["Host"] = "cors10.com", + ["Origin"] = "http://bad-guys.com" + } + }) + + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + + -- Tricky illegitimate origins + res = assert(proxy_client:send { + method = "OPTIONS", + headers = { + ["Host"] = "cors10.com", + ["Origin"] = "http://my-site.com.bad-guys.com" + } + }) + + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + end) end) describe("HTTP method: others", function() @@ -409,11 +471,11 @@ for _, strategy in helpers.each_strategy() do method = "GET", headers = { ["Host"] = "cors6.com", - ["Origin"] = "http://www.example.com" + ["Origin"] = "example.com" } }) assert.res_status(200, res) - assert.equal("http://www.example.com", res.headers["Access-Control-Allow-Origin"]) + assert.equal("example.com", res.headers["Access-Control-Allow-Origin"]) assert.equal("Origin", res.headers["Vary"]) local domains = { @@ -439,6 +501,91 @@ for _, strategy in helpers.each_strategy() do end end) + it("does not automatically parse the host", function() + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors6.com", + ["Origin"] = "http://example.com" + } + }) + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + + -- With a different transport too + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors6.com", + ["Origin"] = "https://example.com" + } + }) + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + end) + + it("validates scheme and port", function() + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors11.com", + ["Origin"] = "http://my-site.com" + } + }) + assert.res_status(200, res) + assert.equals("http://my-site.com", res.headers["Access-Control-Allow-Origin"]) + + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors11.com", + ["Origin"] = "http://my-site.com:80" + } + }) + assert.res_status(200, res) + assert.equals("http://my-site.com:80", res.headers["Access-Control-Allow-Origin"]) + + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors11.com", + ["Origin"] = "http://my-site.com:8000" + } + }) + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + + res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors11.com", + ["Origin"] = "https://my-site.com" + } + }) + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors11.com", + ["Origin"] = "https://my-other-site.com:9000" + } + }) + assert.res_status(200, res) + assert.equals("https://my-other-site.com:9000", res.headers["Access-Control-Allow-Origin"]) + + local res = assert(proxy_client:send { + method = "GET", + headers = { + ["Host"] = "cors11.com", + ["Origin"] = "https://my-other-site.com:9001" + } + }) + assert.res_status(200, res) + assert.is_nil(res.headers["Access-Control-Allow-Origin"]) + end) + it("does not sets CORS orgin if origin host is not in origin_domains list", function() local res = assert(proxy_client:send { method = "GET",