Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: use luasocket instead of curl in etcd.lua #2965

Merged
merged 7 commits into from
Jan 7, 2021
Merged

feat: use luasocket instead of curl in etcd.lua #2965

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
83df02b
feat: use luasocket instead of curl in etcd.lua (#2818)
starsz Dec 22, 2020
edd1a31
update: fix ci
starsz Dec 23, 2020
d731433
update: fix test
starsz Dec 23, 2020
7429657
update: fix test
starsz Dec 23, 2020
02cc565
update: fix test
starsz Dec 23, 2020
2ffba56
update: enrich request comment
starsz Dec 24, 2020
2f9106b
update: fix etcd error handle
starsz Dec 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions .travis/apisix_cli_test/test_main.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -925,3 +925,57 @@ if ! echo "$out" | grep "Admin API can only be used with etcd config_center"; th
fi

echo "passed: Admin API can only be used with etcd config_center"

# Check etcd connect refused
git checkout conf/config.yaml

echo '
etcd:
host:
- "http://127.0.0.1:2389"
prefix: "/apisix"
' > conf/config.yaml

out=$(make init 2>&1 || true)
if ! echo "$out" | grep "connection refused"; then
echo "failed: apisix should echo \"connection refused\""
exit 1
fi

echo "passed: Show connection refused info successfully"

# check etcd auth error
git checkout conf/config.yaml

export ETCDCTL_API=3
etcdctl version
etcdctl --endpoints=127.0.0.1:2379 user add "root:apache-api6"
etcdctl --endpoints=127.0.0.1:2379 role add root
etcdctl --endpoints=127.0.0.1:2379 user grant-role root root
etcdctl --endpoints=127.0.0.1:2379 user get root
etcdctl --endpoints=127.0.0.1:2379 auth enable
etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 del /apisix --prefix

echo '
etcd:
host:
- "http://127.0.0.1:2379"
prefix: "/apisix"
timeout: 30
user: root
password: apache-api7
' > conf/config.yaml

out=$(make init 2>&1 || true)
if ! echo "$out" | grep "invalid user ID or password"; then
echo "failed: should echo \"invalid user ID or password\""
exit 1
fi

echo "passed: show password error successfully"

# clean etcd auth
etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 auth disable
etcdctl --endpoints=127.0.0.1:2379 role delete root
etcdctl --endpoints=127.0.0.1:2379 user delete root

99 changes: 61 additions & 38 deletions apisix/cli/etcd.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,20 @@ local base64_encode = require("base64").encode
local dkjson = require("dkjson")
local util = require("apisix.cli.util")
local file = require("apisix.cli.file")
local http = require("socket.http")
local ltn12 = require("ltn12")

local type = type
local ipairs = ipairs
local print = print
local tonumber = tonumber
local str_format = string.format
local table_concat = table.concat

local _M = {}

-- Timeout for all I/O operations
http.TIMEOUT = 3

local function parse_semantic_version(ver)
local errmsg = "invalid semantic version: " .. ver
Expand Down Expand Up @@ -106,9 +111,6 @@ function _M.init(env, show_output)

local etcd_conf = yaml_conf.etcd

local timeout = etcd_conf.timeout or 3
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should allow configuring timeout in the new way

local uri

-- convert old single etcd config to multiple etcd config
if type(yaml_conf.etcd.host) == "string" then
yaml_conf.etcd.host = {yaml_conf.etcd.host}
Expand All @@ -132,22 +134,23 @@ function _M.init(env, show_output)

-- check the etcd cluster version
for index, host in ipairs(yaml_conf.etcd.host) do
uri = host .. "/version"
local cmd = str_format("curl -s -m %d %s", timeout * 2, uri)
local res = util.execute_cmd(cmd)
local errmsg = str_format("got malformed version message: \"%s\" from etcd\n",
res)
local version_url = host .. "/version"
local errmsg

local body, _, err = dkjson.decode(res)
if err then
local res, err = http.request(version_url)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Look like this library doesn't support max-timeout? Only operation level timeout is supported.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right. The library doesn't support max-timeout.

if err and type(err) == "string" then
errmsg = str_format("request etcd endpoint \'%s\' error, %s\n", version_url, err)
util.die(errmsg)
end

local cluster_version = body["etcdcluster"]
if not cluster_version then
local body, _, err = dkjson.decode(res)
if err or (body and not body["etcdcluster"]) then
errmsg = str_format("got malformed version message: \"%s\" from etcd \"%s\"\n", res,
version_url)
util.die(errmsg)
end

local cluster_version = body["etcdcluster"]
if compare_semantic_version(cluster_version, env.min_etcd_version) then
util.die("etcd cluster version ", cluster_version,
" is less than the required version ",
Expand All @@ -160,31 +163,38 @@ function _M.init(env, show_output)
for index, host in ipairs(yaml_conf.etcd.host) do
local is_success = true

local token_head = ""
local errmsg
local auth_token
local user = yaml_conf.etcd.user
local password = yaml_conf.etcd.password
if user and password then
local uri_auth = host .. "/v3/auth/authenticate"
local auth_url = host .. "/v3/auth/authenticate"
local json_auth = {
name = etcd_conf.user,
password = etcd_conf.password
}
local post_json_auth = dkjson.encode(json_auth)
local cmd_auth = "curl -s " .. uri_auth .. " -X POST -d '" ..
post_json_auth .. "' --connect-timeout " .. timeout
.. " --max-time " .. timeout * 2 .. " --retry 1 2>&1"

local res_auth = util.execute_cmd(cmd_auth)
local body_auth, _, err_auth = dkjson.decode(res_auth)
if err_auth then
util.die(cmd_auth, "\n", res_auth)
local post_json_auth = dkjson.encode(json_auth)
local response_body = {}
local _, err = http.request{url = auth_url, method = "POST",
source = ltn12.source.string(post_json_auth),
sink = ltn12.sink.table(response_body),
headers = {["Content-Length"] = #post_json_auth}}
-- err is string type
if err and type(err) == "string" then
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Better to add more info about the err handle, so that people can know what you are doing without looking into luasocket's doc.

Copy link
Contributor Author

@starsz starsz Dec 24, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@spacewander Sorry, I can't get your point.
That we need to distinguish the connection refused and timeout

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I mean, why we need to check the type of err? We need to clarify it in the comment, so that people can know it without searching luasocket's doc.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. I got it now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added.

Copy link
Member

@spacewander spacewander Dec 25, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would be better to use if not res then handle err? Since the first argument is nil when error happened. This way is more normal.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well. Agree with you.

errmsg = str_format("request etcd endpoint \"%s\" error, %s\n", auth_url, err)
util.die(errmsg)
end

if not body_auth.token then
util.die(cmd_auth, "\n", res_auth)
local res_auth = table_concat(response_body)
local body_auth, _, err_auth = dkjson.decode(res_auth)
if err_auth or (body_auth and not body_auth["token"]) then
errmsg = str_format("got malformed auth message: \"%s\" from etcd \"%s\"\n",
res_auth, auth_url)
util.die(errmsg)
end

token_head = " -H 'Authorization: " .. body_auth.token .. "'"
auth_token = body_auth.token
end


Expand All @@ -195,31 +205,44 @@ function _M.init(env, show_output)

local key = (etcd_conf.prefix or "") .. dir_name .. "/"

local uri = host .. "/v3/kv/put"
local put_url = host .. "/v3/kv/put"
local post_json = '{"value":"' .. base64_encode("init_dir")
.. '", "key":"' .. base64_encode(key) .. '"}'
local cmd = "curl " .. uri .. token_head .. " -X POST -d '" .. post_json
.. "' --connect-timeout " .. timeout
.. " --max-time " .. timeout * 2 .. " --retry 1 2>&1"

local res = util.execute_cmd(cmd)
if res:find("404 page not found", 1, true) then
util.die("gRPC gateway is not enabled in your etcd cluster, ",
"which is required by Apache APISIX.", "\n")
local response_body = {}
local headers = {["Content-Length"] = #post_json}
if auth_token then
headers["Authorization"] = auth_token
end

local _, err = http.request{url = put_url, method = "POST",
source = ltn12.source.string(post_json),
sink = ltn12.sink.table(response_body),
headers = headers}
if err and type(err) == "string" then
errmsg = str_format("request etcd endpoint \"%s\" error, %s\n", put_url, err)
util.die(errmsg)
end

local res_put = table_concat(response_body)
if res_put:find("404 page not found", 1, true) then
errmsg = str_format("gRPC gateway is not enabled in etcd cluster \"%s\",",
"which is required by Apache APISIX\n")
util.die(errmsg)
end

if res:find("error", 1, true) then
if res_put:find("error", 1, true) then
is_success = false
if (index == host_count) then
util.die(cmd, "\n", res)
errmsg = str_format("got malformed key-put message: \"%s\" from etcd \"%s\"\n",
res_put, put_url)
util.die(errmsg)
end

break
end

if show_output then
print(cmd)
print(res)
print(res_put)
end
end

Expand Down
1 change: 1 addition & 0 deletions rockspec/apisix-master-0.rockspec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ dependencies = {
"resty-redis-cluster = 1.02-4",
"lua-resty-expr = 1.0.0",
"graphql = 0.0.2",
"luasocket = 3.0rc1-2",
}

build = {
Expand Down