From 3b13eb9c86219a4d8e299c3d9af02b799f2458ab Mon Sep 17 00:00:00 2001 From: Jean-Michael Cyr Date: Tue, 10 Jan 2017 15:31:49 -0500 Subject: [PATCH 1/2] Summary Here's some metadata plugin that allows kong to modify Headers and Querystring based on two source of metadata, the persistent ones in database / redis and the ones in the transitory store. Transitory store could allow other plugins to prepare metadata on the fly when kong receive a Request. (View Tieske suggestion here #550) One of the test cover the use case by feeding the transitory store with fixtures. I've added a debug.lua library for my POC, I should probably remove it if the pull request ever gets approved. Also I believe my request_headers_factory and request_querystring_factory could be use in the request-transformer plugin, so it might be wise to move in /kong/tools or something like that ? Comments are welcome, i've never done Lua / OpenResty before. Full changelog Implement metadata-insertion plugin Issues resolved Fix #550 --- kong-0.9.7-0.rockspec | 4 + kong/plugins/metadata-insertion/access.lua | 170 +++++++ kong/plugins/metadata-insertion/api.lua | 19 + kong/plugins/metadata-insertion/daos.lua | 16 + .../factory/request_headers_factory.lua | 50 ++ .../factory/request_querystring_factory.lua | 42 ++ kong/plugins/metadata-insertion/handler.lua | 15 + kong/plugins/metadata-insertion/hooks.lua | 17 + .../migrations/cassandra.lua | 21 + .../migrations/postgres.lua | 28 ++ kong/plugins/metadata-insertion/schema.lua | 43 ++ .../plugins/metadata-insertion/tool/debug.lua | 52 ++ .../0001-metadata-insertion/access_spec.lua | 457 ++++++++++++++++++ .../metadata-transitory-store/handler.lua | 35 ++ .../metadata-transitory-store/schema.lua | 3 + spec/kong_tests.conf | 2 + 16 files changed, 974 insertions(+) create mode 100644 kong/plugins/metadata-insertion/access.lua create mode 100644 kong/plugins/metadata-insertion/api.lua create mode 100644 kong/plugins/metadata-insertion/daos.lua create mode 100644 kong/plugins/metadata-insertion/factory/request_headers_factory.lua create mode 100644 kong/plugins/metadata-insertion/factory/request_querystring_factory.lua create mode 100644 kong/plugins/metadata-insertion/handler.lua create mode 100644 kong/plugins/metadata-insertion/hooks.lua create mode 100644 kong/plugins/metadata-insertion/migrations/cassandra.lua create mode 100644 kong/plugins/metadata-insertion/migrations/postgres.lua create mode 100644 kong/plugins/metadata-insertion/schema.lua create mode 100644 kong/plugins/metadata-insertion/tool/debug.lua create mode 100644 spec/03-plugins/0001-metadata-insertion/access_spec.lua create mode 100644 spec/fixtures/kong/plugins/metadata-transitory-store/handler.lua create mode 100644 spec/fixtures/kong/plugins/metadata-transitory-store/schema.lua diff --git a/kong-0.9.7-0.rockspec b/kong-0.9.7-0.rockspec index c3909872b3f..2dcc19f9be8 100644 --- a/kong-0.9.7-0.rockspec +++ b/kong-0.9.7-0.rockspec @@ -272,5 +272,9 @@ build = { ["kong.plugins.aws-lambda.handler"] = "kong/plugins/aws-lambda/handler.lua", ["kong.plugins.aws-lambda.schema"] = "kong/plugins/aws-lambda/schema.lua", ["kong.plugins.aws-lambda.v4"] = "kong/plugins/aws-lambda/v4.lua", + + ["kong.plugins.metadata-insertion.handler"] = "kong/plugins/metadata-insertion/handler.lua", + ["kong.plugins.metadata-insertion.access"] = "kong/plugins/metadata-insertion/access.lua", + ["kong.plugins.metadata-insertion.schema"] = "kong/plugins/metadata-insertion/schema.lua", } } diff --git a/kong/plugins/metadata-insertion/access.lua b/kong/plugins/metadata-insertion/access.lua new file mode 100644 index 00000000000..4cf15da9c9c --- /dev/null +++ b/kong/plugins/metadata-insertion/access.lua @@ -0,0 +1,170 @@ +local _M = {} +local request_querystring_factory = require "kong.plugins.metadata-insertion.factory.request_querystring_factory" +local request_headers_factory = require "kong.plugins.metadata-insertion.factory.request_headers_factory" +-- local debug = require "kong.plugins.metadata-insertion.tool.debug" +local cache = require "kong.tools.database_cache" +local singletons = require "kong.singletons" +local responses = require "kong.tools.responses" +local currentUserMetadata + +local function getPersistentMetadata() + -- retrieve metadata from cache or database for current user + currentUserMetadata = cache.get_or_set("metadata_keyvaluestore." .. ngx.ctx.authenticated_consumer.id, nil, function() + local metadata, err = singletons.dao.metadata_keyvaluestore:find_all({ consumer_id = ngx.ctx.authenticated_consumer.id }) + if err then + return responses.send_HTTP_INTERNAL_SERVER_ERROR(err) + end + return metadata + end) + + if currentUserMetadata then + return currentUserMetadata + end + + return {} +end + +local function resolveParameterValuePlaceholderWithMetadata(dataProvisioningName) + for _, elem in ipairs(currentUserMetadata) do + if elem.key == dataProvisioningName then + return elem.value + end + end + error("This API needs metadata that the current user does not provide.") +end + +local function retrieveMetadataForConfigToken(querystringModifier) + local args = {} + + for stringChunk in string.gmatch(querystringModifier, "%S+") do + table.insert(args, stringChunk) + end + + assert(table.getn(args) == 2, "Invalid format") + + -- Prepare arg name + local argName = args[1] + argName = argName:gsub(":", "") + + -- Prepare arg value (replace placeholder name by metadata value) + local argValuePlaceholderName = args[2] + argValuePlaceholderName = argValuePlaceholderName:gsub("%%", "") + local parameterValue = resolveParameterValuePlaceholderWithMetadata(argValuePlaceholderName) + return argName, parameterValue +end + + +local function appendMetadataFromTransitoryStore(currentUserMetadata) + + -- loop through transitory store and add the metadata in memory + -- transitory store take precedence over persitent metadata + if ngx.ctx.metadata_transitory_store and type(ngx.ctx.metadata_transitory_store) == "table" then + + for _, transitoryStoreElem in ipairs(ngx.ctx.metadata_transitory_store) do + + local persistentMetadataFound = false + + -- replace persitent metadata with transitory store if it exist in both place + for index, persistentMetadataElem in ipairs(currentUserMetadata) do + if persistentMetadataElem.key == transitoryStoreElem.key then + currentUserMetadata[index] = transitoryStoreElem + persistentMetadataFound = true + end + end + + -- if nothing found in persistent metadata, let's make sure we add the transitory store element as new + if persistentMetadataFound == false then + table.insert(currentUserMetadata, transitoryStoreElem) + end + end + end +end + +local function updateQuerystring(confDataInsertion) + + local RequestQuerystringFactory = request_querystring_factory:new() + + RequestQuerystringFactory:mergeArgsWithRequestArgs() + + -- Remove querystring(s) + if confDataInsertion.remove and confDataInsertion.remove.querystring then + for _, key in ipairs(confDataInsertion.remove.querystring) do + RequestQuerystringFactory:removeArgByKey(key) + end + end + + -- Replace querystring(s) + if confDataInsertion.replace and confDataInsertion.replace.querystring then + for _, querystringModifier in pairs(confDataInsertion.replace.querystring) do + local parameterName, parameterValue = retrieveMetadataForConfigToken(querystringModifier) + RequestQuerystringFactory:replaceArgByKey(parameterName, parameterValue) + end + end + + -- Add querystring(s) + if confDataInsertion.add and confDataInsertion.add.querystring then + for _, querystringModifier in pairs(confDataInsertion.add.querystring) do + local parameterName, parameterValue = retrieveMetadataForConfigToken(querystringModifier) + RequestQuerystringFactory:add(parameterName, parameterValue) + end + end + + RequestQuerystringFactory:persist() +end + +local function updateHeaders(confDataInsertion) + + local RequestHeadersFactory = request_headers_factory:new() + + RequestHeadersFactory:mergeArgsWithRequestArgs() + + -- Remove querystring(s) + if confDataInsertion.remove and confDataInsertion.remove.headers then + for _, key in ipairs(confDataInsertion.remove.headers) do + RequestHeadersFactory:removeArgByKey(key) + end + end + + -- Replace querystring(s) + if confDataInsertion.replace and confDataInsertion.replace.headers then + for _, headersModifier in pairs(confDataInsertion.replace.headers) do + local parameterName, parameterValue = retrieveMetadataForConfigToken(headersModifier) + RequestHeadersFactory:replaceArgByKey(parameterName, parameterValue) + end + end + + -- Add querystring(s) + if confDataInsertion.add and confDataInsertion.add.headers then + for _, headersModifier in pairs(confDataInsertion.add.headers) do + local parameterName, parameterValue = retrieveMetadataForConfigToken(headersModifier) + RequestHeadersFactory:add(parameterName, parameterValue) + end + end + + RequestHeadersFactory:persist() +end + +function _M.execute(conf) + + if ngx.ctx.authenticated_consumer == nil then + return responses.send_HTTP_UNAUTHORIZED("Metadata plugin can't be used without having an authenticated user.") + end + + currentUserMetadata = getPersistentMetadata() + appendMetadataFromTransitoryStore(currentUserMetadata) + + local _, err = pcall(function() + + ----------------------------- + -- Data Insertion Processing + ----------------------------- + updateQuerystring(conf) + updateHeaders(conf) + end) + + if err then + return responses.send_HTTP_BAD_REQUEST(err) + end +end + +return _M \ No newline at end of file diff --git a/kong/plugins/metadata-insertion/api.lua b/kong/plugins/metadata-insertion/api.lua new file mode 100644 index 00000000000..27e731b87b7 --- /dev/null +++ b/kong/plugins/metadata-insertion/api.lua @@ -0,0 +1,19 @@ +local crud = require "kong.api.crud_helpers" + +return { + ["/consumers/:username_or_id/metadata/"] = { + before = function(self, dao_factory, helpers) + crud.find_consumer_by_username_or_id(self, dao_factory, helpers) + self.params.consumer_id = self.consumer.id + end, + GET = function(self, dao_factory) + crud.paginated_set(self, dao_factory.metadata_keyvaluestore) + end, + PUT = function(self, dao_factory) + crud.put(self.params, dao_factory.metadata_keyvaluestore) + end, + POST = function(self, dao_factory) + crud.post(self.params, dao_factory.metadata_keyvaluestore) + end + } +} diff --git a/kong/plugins/metadata-insertion/daos.lua b/kong/plugins/metadata-insertion/daos.lua new file mode 100644 index 00000000000..85eb4beabc3 --- /dev/null +++ b/kong/plugins/metadata-insertion/daos.lua @@ -0,0 +1,16 @@ +local SCHEMA = { + primary_key = { "id" }, + table = "metadata_keyvaluestore", + fields = { + id = { type = "id", dao_insert_value = true }, + created_at = { type = "timestamp", immutable = true, dao_insert_value = true }, + consumer_id = { type = "id", required = true, foreign = "consumers:id" }, + key = { type = "string", required = true }, + value = { type = "string", required = true } + }, + marshall_event = function(self, t) + return { id = t.id, consumer_id = t.consumer_id, key = t.key } + end +} + +return { metadata_keyvaluestore = SCHEMA } diff --git a/kong/plugins/metadata-insertion/factory/request_headers_factory.lua b/kong/plugins/metadata-insertion/factory/request_headers_factory.lua new file mode 100644 index 00000000000..17bb40474f5 --- /dev/null +++ b/kong/plugins/metadata-insertion/factory/request_headers_factory.lua @@ -0,0 +1,50 @@ +local req_set_header = ngx.req.set_header +local req_get_headers = ngx.req.get_headers +local HOST = "host" +local RequestHeadersFactory = {} +local args = {} + +function RequestHeadersFactory:new() + return RequestHeadersFactory +end + +function RequestHeadersFactory:getArgs() + return args +end + +function RequestHeadersFactory:mergeArgsWithRequestArgs() + local requestArgs = req_get_headers() + for k, v in pairs(requestArgs) do + args[k] = v + end +end + +function RequestHeadersFactory:persist() + for key, value in pairs(args) do + if key:lower() ~= HOST then + if value == "metadata_asked_for_removal" then + req_set_header(key, nil) + else + req_set_header(key, value) + end + end + end +end + +function RequestHeadersFactory:removeArgByKey(key) + if args[key] then + args[key] = "metadata_asked_for_removal" + end +end + +function RequestHeadersFactory:replaceArgByKey(key, value) + if args[key] then + args[key] = value + end +end + +function RequestHeadersFactory:add(key, value) + args[key] = value +end + +return RequestHeadersFactory \ No newline at end of file diff --git a/kong/plugins/metadata-insertion/factory/request_querystring_factory.lua b/kong/plugins/metadata-insertion/factory/request_querystring_factory.lua new file mode 100644 index 00000000000..8e5d6d4be05 --- /dev/null +++ b/kong/plugins/metadata-insertion/factory/request_querystring_factory.lua @@ -0,0 +1,42 @@ +local req_get_uri_args = ngx.req.get_uri_args +local req_set_uri_args = ngx.req.set_uri_args + +local RequestQuerystringFactory = {} +local args = {} + +function RequestQuerystringFactory:new() + return RequestQuerystringFactory +end + +function RequestQuerystringFactory:getArgs() + return args +end + +function RequestQuerystringFactory:mergeArgsWithRequestArgs() + local requestArgs = req_get_uri_args() + for k, v in pairs(requestArgs) do + args[k] = v + end +end + +function RequestQuerystringFactory:persist() + req_set_uri_args(args) +end + +function RequestQuerystringFactory:removeArgByKey(key) + if args[key] then + args[key] = nil + end +end + +function RequestQuerystringFactory:replaceArgByKey(key, value) + if args[key] then + args[key] = value + end +end + +function RequestQuerystringFactory:add(key, value) + args[key] = value +end + +return RequestQuerystringFactory \ No newline at end of file diff --git a/kong/plugins/metadata-insertion/handler.lua b/kong/plugins/metadata-insertion/handler.lua new file mode 100644 index 00000000000..fc9566195fa --- /dev/null +++ b/kong/plugins/metadata-insertion/handler.lua @@ -0,0 +1,15 @@ +local BasePlugin = require "kong.plugins.base_plugin" +local access = require "kong.plugins.metadata-insertion.access" + +local MetadataInsertionHandler = BasePlugin:extend() + +function MetadataInsertionHandler:new() + MetadataInsertionHandler.super.new(self, "metadata-insertion") +end + +function MetadataInsertionHandler:access(conf) + MetadataInsertionHandler.super.access(self) + access.execute(conf) +end + +return MetadataInsertionHandler diff --git a/kong/plugins/metadata-insertion/hooks.lua b/kong/plugins/metadata-insertion/hooks.lua new file mode 100644 index 00000000000..6285226becb --- /dev/null +++ b/kong/plugins/metadata-insertion/hooks.lua @@ -0,0 +1,17 @@ +local events = require "kong.core.events" +local cache = require "kong.tools.database_cache" + +local function invalidate(message_t) + if message_t.collection == "metadata_keyvaluestore" then + cache.delete(cache.metadata_keyvaluestore(message_t.old_entity and message_t.old_entity.key or message_t.entity.key)) + end +end + +return { + [events.TYPES.ENTITY_UPDATED] = function(message_t) + invalidate(message_t) + end, + [events.TYPES.ENTITY_DELETED] = function(message_t) + invalidate(message_t) + end +} diff --git a/kong/plugins/metadata-insertion/migrations/cassandra.lua b/kong/plugins/metadata-insertion/migrations/cassandra.lua new file mode 100644 index 00000000000..24287b9ecee --- /dev/null +++ b/kong/plugins/metadata-insertion/migrations/cassandra.lua @@ -0,0 +1,21 @@ +return { + { + name = "2017-01-05-000000_init_metadata", + up = [[ + CREATE TABLE IF NOT EXISTS metadata_keyvaluestore( + id uuid, + consumer_id uuid, + key text, + value text, + created_at timestamp, + PRIMARY KEY (id) + ); + + CREATE INDEX IF NOT EXISTS ON metadata_keyvaluestore(key); + CREATE INDEX IF NOT EXISTS metadata_keyvaluestore_consumer_id ON metadata_keyvaluestore(consumer_id); + ]], + down = [[ + DROP TABLE metadata_keyvaluestore; + ]] + } +} diff --git a/kong/plugins/metadata-insertion/migrations/postgres.lua b/kong/plugins/metadata-insertion/migrations/postgres.lua new file mode 100644 index 00000000000..64172ee69d4 --- /dev/null +++ b/kong/plugins/metadata-insertion/migrations/postgres.lua @@ -0,0 +1,28 @@ +return { + { + name = "2017-01-05-000000_init_metadata", + up = [[ + CREATE TABLE IF NOT EXISTS metadata_keyvaluestore( + id uuid, + consumer_id uuid REFERENCES consumers (id) ON DELETE CASCADE, + key text, + value text, + created_at timestamp without time zone default (CURRENT_TIMESTAMP(0) at time zone 'utc'), + PRIMARY KEY (id) + ); + + DO $$ + BEGIN + IF (SELECT to_regclass('metadata_keyvaluestore_key_idx')) IS NULL THEN + CREATE INDEX metadata_keyvaluestore_key_idx ON metadata_keyvaluestore(key); + END IF; + IF (SELECT to_regclass('metadata_keyvaluestore_consumer_idx')) IS NULL THEN + CREATE INDEX metadata_keyvaluestore_consumer_idx ON metadata_keyvaluestore(consumer_id); + END IF; + END$$; + ]], + down = [[ + DROP TABLE metadata_keyvaluestore; + ]] + } +} diff --git a/kong/plugins/metadata-insertion/schema.lua b/kong/plugins/metadata-insertion/schema.lua new file mode 100644 index 00000000000..a0ddd588efa --- /dev/null +++ b/kong/plugins/metadata-insertion/schema.lua @@ -0,0 +1,43 @@ +local find = string.find +local function check_for_value(value) + for i, entry in ipairs(value) do + local ok = find(entry, ":") + if not ok then + return false, "key '" .. entry .. "' has no value" + end + end + return true +end + +return { + no_consumer = true, + fields = { + add = { + type = "table", + schema = { + fields = { + querystring = { type = "array", default = {}, func = check_for_value }, + headers = { type = "array", default = {}, func = check_for_value } + } + } + }, + replace = { + type = "table", + schema = { + fields = { + querystring = { type = "array", default = {}, func = check_for_value }, + headers = { type = "array", default = {}, func = check_for_value } + } + } + }, + remove = { + type = "table", + schema = { + fields = { + querystring = { type = "array" }, + headers = { type = "array" } + } + } + } + } +} diff --git a/kong/plugins/metadata-insertion/tool/debug.lua b/kong/plugins/metadata-insertion/tool/debug.lua new file mode 100644 index 00000000000..bec1a72c141 --- /dev/null +++ b/kong/plugins/metadata-insertion/tool/debug.lua @@ -0,0 +1,52 @@ +return { + print_r = function(t) + local print_r_cache = {} + local function sub_print_r(t, indent) + if (print_r_cache[tostring(t)]) then + print(indent .. "*" .. tostring(t)) + else + print_r_cache[tostring(t)] = true + if (type(t) == "table") then + for pos, val in pairs(t) do + if (type(val) == "table") then + print(indent .. "[" .. pos .. "] => " .. tostring(t) .. " {") + sub_print_r(val, indent .. string.rep(" ", string.len(pos) + 8)) + print(indent .. string.rep(" ", string.len(pos) + 6) .. "}") + else + print(indent .. "[" .. pos .. "] => " .. tostring(val)) + end + end + else + print(indent .. tostring(t)) + end + end + end + + sub_print_r(t, " ") + end, + log_r = function(t) + local print_r_cache = {} + local function sub_print_r(t, indent) + if (print_r_cache[tostring(t)]) then + ngx.log(ngx.ERR, "=== : " .. indent .. "*" .. tostring(t)) + else + print_r_cache[tostring(t)] = true + if (type(t) == "table") then + for pos, val in pairs(t) do + if (type(val) == "table") then + ngx.log(ngx.ERR, "=== : " .. indent .. "[" .. pos .. "] => " .. tostring(t) .. " {") + sub_print_r(val, indent .. string.rep(" ", string.len(pos) + 8)) + ngx.log(ngx.ERR, "=== : " .. indent .. string.rep(" ", string.len(pos) + 6) .. "}") + else + ngx.log(ngx.ERR, "=== : " .. indent .. "[" .. pos .. "] => " .. tostring(val)) + end + end + else + ngx.log(ngx.ERR, "=== : " .. indent .. tostring(t)) + end + end + end + + sub_print_r(t, " ") + end +} \ No newline at end of file diff --git a/spec/03-plugins/0001-metadata-insertion/access_spec.lua b/spec/03-plugins/0001-metadata-insertion/access_spec.lua new file mode 100644 index 00000000000..f1788d18699 --- /dev/null +++ b/spec/03-plugins/0001-metadata-insertion/access_spec.lua @@ -0,0 +1,457 @@ +local helpers = require "spec.helpers" +local client +local admin_client +local cjson = require "cjson" +local consumer + local debug = require "kong.plugins.metadata-insertion.tool.debug" + +local function setConsumerDummyData() + + -- SETUP DUMMY DATA + consumer = assert(helpers.dao.consumers:insert { + username = "bob" + }) + + assert(helpers.dao.keyauth_credentials:insert { + consumer_id = consumer.id, + key = "bob-api-key" + }) + + assert(helpers.dao.metadata_keyvaluestore:insert { + consumer_id = consumer.id, + key = "location", + value = "europe" + }) + + assert(helpers.dao.metadata_keyvaluestore:insert { + consumer_id = consumer.id, + key = "third_party_api_key", + value = "some-generic-api-key" + }) +end + +describe("Metadata-Insertion Plugin", function() + + teardown(function() + if client then client:close() end + if admin_client then admin_client:close() end + end) + + before_each(function() + assert(helpers.start_kong()) + admin_client = helpers.admin_client() + client = helpers.proxy_client() + end) + + after_each(function() + assert(helpers.stop_kong()) + if client then client:close() end + if admin_client then admin_client:close() end + end) + + describe("Response", function() + + it("Should return metadata previously created", function() + + setConsumerDummyData() + + local res = admin_client:send { + method = "GET", + path = "/consumers/bob/metadata", + headers = { + ["Content-Type"] = "application/json" + } + } + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + assert(json.data[1].key == "location", "Invalid parameter name") + assert(json.data[1].value == "europe", "Invalid value") + assert(json.data[2].key == "third_party_api_key", "Invalid parameter name") + assert(json.data[2].value == "some-generic-api-key", "Invalid value") + end) + + it("Should return metadata previously created with crud API access point", function() + + -- SETUP DUMMY DATA + consumer = assert(helpers.dao.consumers:insert { + username = "bob" + }) + + assert(helpers.dao.keyauth_credentials:insert { + consumer_id = consumer.id, + key = "bob-api-key" + }) + + assert(admin_client:send { + method = "POST", + path = "/consumers/" .. consumer.id .. "/metadata/", + body = { + key = "some-field", + value = "some-value" + }, + headers = { + ["Content-Type"] = "application/json" + } + }) + + if admin_client then admin_client:close() end + admin_client = helpers.admin_client() + + local res = admin_client:send { + method = "GET", + path = "/consumers/bob/metadata", + headers = { + ["Content-Type"] = "application/json" + } + } + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + + assert(json.data[1].key == "some-field", "Metadata added with POST request seems to be missing") + assert(json.data[1].value == "some-value", "Metadata added with POST request seems to be missing") + + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + assert(helpers.dao.plugins:insert { + name = "key-auth", + api_id = api1.id, + config = { + hide_credentials = true + } + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + add = { + querystring = { + "location: %some-field%" + }, + headers = { + "location: %some-field%" + } + } + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/request?apikey=bob-api-key", + headers = { + ["Host"] = "mockbin.com" + } + })) + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + + assert(json.headers.location == "some-value", "Metadata added with POST request seems to not work properly") + assert(json.queryString.location == "some-value", "Metadata added with POST request seems to not work properly") + end) + + it("Should transform request with consumer metadata", function() + + setConsumerDummyData() + + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + assert(helpers.dao.plugins:insert { + name = "key-auth", + api_id = api1.id, + config = { + hide_credentials = true + } + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + add = { + querystring = { + "location: %location%", + "apikey: %third_party_api_key%" + }, + headers = { + "user_country: %location%", + } + } + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/request?apikey=bob-api-key", + headers = { + ["Host"] = "mockbin.com" + } + })) + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + + assert(json.headers.user_country == "europe", "Invalid user_country in headers") + assert(json.queryString.apikey == "some-generic-api-key", "Invalid api key in Querystring") + assert(json.queryString.location == "europe", "Invalid location in QueryString") + end) + + it("Should throw error when trying to access an API without the proper metadata available", function() + + consumer = assert(helpers.dao.consumers:insert { + username = "empty-user" + }) + + assert(helpers.dao.keyauth_credentials:insert { + consumer_id = consumer.id, + key = "empty-user-api-key" + }) + + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + assert(helpers.dao.plugins:insert { + name = "key-auth", + api_id = api1.id, + config = { + hide_credentials = true + } + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + add = { + querystring = { + "location: %location%", + "apikey: %third_party_api_key%" + }, + headers = { + "user_country: %location%", + } + } + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/request?apikey=empty-user-api-key", + headers = { + ["Host"] = "mockbin.com" + } + })) + + assert.res_status(400, res) + end) + + it("Should replace value from headers and querystring", function() + + setConsumerDummyData() + + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + assert(helpers.dao.plugins:insert { + name = "key-auth", + api_id = api1.id, + config = { + hide_credentials = true + } + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + replace = { + querystring = { + "location: %location%" + }, + headers = { + "third-party-api-key: %third_party_api_key%" + } + } + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/request?apikey=bob-api-key&location=bob_location_will_be_overwritten", + headers = { + ["Host"] = "mockbin.com", + ["third-party-api-key"] = "will_be_overwritten" + } + })) + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + assert(json.queryString.location == "europe", "Invalid location in QueryString") + assert(json.headers["third-party-api-key"] == "some-generic-api-key", "Invalid third-party-api-key in headers") + end) + + it("Should remove value from headers and querystring", function() + + setConsumerDummyData() + + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + assert(helpers.dao.plugins:insert { + name = "key-auth", + api_id = api1.id, + config = { + hide_credentials = true + } + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + remove = { + querystring = { + "location" + }, + headers = { + "third-party-api-key" + } + } + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/request?apikey=bob-api-key&location=will_be_removed", + headers = { + ["Host"] = "mockbin.com", + ["third-party-api-key"] = "will_be_removed" + } + })) + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + assert(json.queryString.location == nil, "Location in querystring should be nil") + assert(json.headers["third-party-api-key"] == nil, "Invalid third-party-api-key in headers should be nil") + end) + + it("Should trigger forbidden access when no user is authenticated", function() + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + replace = { + querystring = { + "whatever: %whatever%" + } + } + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/status", + headers = { + ["Host"] = "mockbin.com" + } + })) + + assert.res_status(401, res) + end) + + it("Should transform request with consumer metadata but prioritise value from metadata transitory store", function() + + -- must re-init the kong instance with new parameters + assert(helpers.stop_kong()) + if client then client:close() end + if admin_client then admin_client:close() end + + assert(helpers.start_kong({ + custom_plugins = "metadata-insertion,metadata-transitory-store", + lua_package_path = "?/init.lua;./kong/?.lua;./spec/fixtures/?.lua" + })) + + client = helpers.proxy_client() + admin_client = helpers.admin_client() + + local api1 = assert(helpers.dao.apis:insert { + request_host = "mockbin.com", + upstream_url = "http://www.mockbin.com" + }) + + setConsumerDummyData() + + assert(helpers.dao.plugins:insert { + name = "key-auth", + api_id = api1.id, + config = { + hide_credentials = true + } + }) + + assert(helpers.dao.plugins:insert { + api_id = api1.id, + name = "metadata-insertion", + config = { + add = { + querystring = { + "location: %location%", + "apikey: %third_party_api_key%", + "field_only_available_in_transitory_store: %field_only_available_in_transitory_store%" + }, + headers = { + "user_country: %location%", + "field_only_available_in_transitory_store: %field_only_available_in_transitory_store%" + } + } + } + }) + + assert(admin_client:send { + method = "POST", + path = "/apis/" .. api1.id .. "/plugins/", + body = { + name = "metadata-transitory-store" + }, + headers = { + ["Content-Type"] = "application/json" + } + }) + + local res = assert(client:send({ + method = "GET", + path = "/request?apikey=bob-api-key", + headers = { + ["Host"] = "mockbin.com" + } + })) + + local body = assert.res_status(200, res) + local json = cjson.decode(body) + + assert(json.headers.user_country == "location-from-transitory", "Invalid user_country in headers") + assert(json.queryString.apikey == "api-key-from-transitory", "Invalid apikey in QueryString") + assert(json.queryString.location == "location-from-transitory", "Invalid location in QueryString") + assert(json.queryString.field_only_available_in_transitory_store == "field_only_available_in_transitory_store", "Exclusive field in transitory store not added to metadata") + end) + end) +end) \ No newline at end of file diff --git a/spec/fixtures/kong/plugins/metadata-transitory-store/handler.lua b/spec/fixtures/kong/plugins/metadata-transitory-store/handler.lua new file mode 100644 index 00000000000..250c32c8ae6 --- /dev/null +++ b/spec/fixtures/kong/plugins/metadata-transitory-store/handler.lua @@ -0,0 +1,35 @@ +local BasePlugin = require "kong.plugins.base_plugin" +-- local debug = require "kong.plugins.metadata-insertion.tool.debug" +local MetadataTransitoryStoreHandler = BasePlugin:extend() + +MetadataTransitoryStoreHandler.PRIORITY = 100 + +function MetadataTransitoryStoreHandler:new() + MetadataTransitoryStoreHandler.super.new(self, "metadata-transitory-store") +end + +function MetadataTransitoryStoreHandler:init_worker() + MetadataTransitoryStoreHandler.super.init_worker(self) +end + +function MetadataTransitoryStoreHandler:access(conf) + MetadataTransitoryStoreHandler.super.access(self) + + -- add data in metadata transitory store + ngx.ctx.metadata_transitory_store = { + { + key = "location", + value = "location-from-transitory" + }, + { + key = "third_party_api_key", + value = "api-key-from-transitory" + }, + { + key = "field_only_available_in_transitory_store", + value = "field_only_available_in_transitory_store" + }, + } +end + +return MetadataTransitoryStoreHandler diff --git a/spec/fixtures/kong/plugins/metadata-transitory-store/schema.lua b/spec/fixtures/kong/plugins/metadata-transitory-store/schema.lua new file mode 100644 index 00000000000..a896976a118 --- /dev/null +++ b/spec/fixtures/kong/plugins/metadata-transitory-store/schema.lua @@ -0,0 +1,3 @@ +return { + fields = {} +} \ No newline at end of file diff --git a/spec/kong_tests.conf b/spec/kong_tests.conf index aaff89561f1..ecf8a02e730 100644 --- a/spec/kong_tests.conf +++ b/spec/kong_tests.conf @@ -28,3 +28,5 @@ nginx_optimizations = off prefix = servroot log_level = debug + +custom_plugins = metadata-insertion \ No newline at end of file From 788a5035e37dce17b1649d8e85c3cc045231227d Mon Sep 17 00:00:00 2001 From: Jean-Michael Cyr Date: Tue, 10 Jan 2017 15:46:10 -0500 Subject: [PATCH 2/2] - refactor first test of metadata-insertion plugin because it was not working all the time. The actual index order might change depending on generated unique id for the consumer_id field. --- spec/03-plugins/0001-metadata-insertion/access_spec.lua | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spec/03-plugins/0001-metadata-insertion/access_spec.lua b/spec/03-plugins/0001-metadata-insertion/access_spec.lua index f1788d18699..065be54505c 100644 --- a/spec/03-plugins/0001-metadata-insertion/access_spec.lua +++ b/spec/03-plugins/0001-metadata-insertion/access_spec.lua @@ -3,7 +3,7 @@ local client local admin_client local cjson = require "cjson" local consumer - local debug = require "kong.plugins.metadata-insertion.tool.debug" +-- local debug = require "kong.plugins.metadata-insertion.tool.debug" local function setConsumerDummyData() @@ -65,10 +65,7 @@ describe("Metadata-Insertion Plugin", function() local body = assert.res_status(200, res) local json = cjson.decode(body) - assert(json.data[1].key == "location", "Invalid parameter name") - assert(json.data[1].value == "europe", "Invalid value") - assert(json.data[2].key == "third_party_api_key", "Invalid parameter name") - assert(json.data[2].value == "some-generic-api-key", "Invalid value") + assert(json.total == 2, "Invalid number of metadata for the current test") end) it("Should return metadata previously created with crud API access point", function()