Skip to content

Commit

Permalink
feat(clickhouse-logger): support multi endpoints
Browse files Browse the repository at this point in the history
  • Loading branch information
@qizhendong1
qizhendong1 committed Jul 24, 2022
1 parent 3b1cda7 commit 7f2da81
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 20 deletions.
10 changes: 6 additions & 4 deletions apisix/plugins/clickhouse-logger.lua
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ local core = require("apisix.core")
local http = require("resty.http")
local url = require("net.url")
local plugin = require("apisix.plugin")
local math_random = math.random

local ngx = ngx
local tostring = tostring
Expand All @@ -31,7 +32,7 @@ local batch_processor_manager = bp_manager_mod.new(plugin_name)
local schema = {
type = "object",
properties = {
endpoint_addr = core.schema.uri_def,
endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1},
user = {type = "string", default = ""},
password = {type = "string", default = ""},
database = {type = "string", default = ""},
Expand All @@ -40,7 +41,7 @@ local schema = {
name = {type = "string", default = "clickhouse logger"},
ssl_verify = {type = "boolean", default = true},
},
required = {"endpoint_addr", "user", "password", "database", "logtable"}
required = {"endpoint_addrs", "user", "password", "database", "logtable"}
}


Expand Down Expand Up @@ -72,11 +73,12 @@ end
local function send_http_data(conf, log_message)
local err_msg
local res = true
local url_decoded = url.parse(conf.endpoint_addr)
local selected_endpoint_addr = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)]
local url_decoded = url.parse(selected_endpoint_addr)
local host = url_decoded.host
local port = url_decoded.port

core.log.info("sending a batch logs to ", conf.endpoint_addr)
core.log.info("sending a batch logs to ", selected_endpoint_addr)

if not port then
if url_decoded.scheme == "https" then
Expand Down
5 changes: 3 additions & 2 deletions docs/en/latest/plugins/clickhouse-logger.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic

| Name | Type | Required | Default | Valid values | Description |
|---------------|---------|----------|---------------------|--------------|----------------------------------------------------------------|
| endpoint_addr | string | True | | | ClickHouse endpoint. |
| endpoint_addr | array | True | | | ClickHouse endpoints. |
| database | string | True | | | Name of the database to store the logs. |
| logtable | string | True | | | Table name to store the logs. |
| user | string | True | | | ClickHouse username. |
Expand Down Expand Up @@ -96,6 +96,7 @@ Now, if you run `select * from default.test;`, you will get the following row:

## Enabling the Plugin

If multiple endpoints are configured, they will be written randomly.
The example below shows how you can enable the Plugin on a specific Route:

```shell
Expand All @@ -107,7 +108,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
"password": "a",
"database": "default",
"logtable": "test",
"endpoint_addr": "http://127.0.0.1:8123"
"endpoint_addr": ["http://127.0.0.1:8123"]
}
},
"upstream": {
Expand Down
21 changes: 11 additions & 10 deletions docs/zh/latest/plugins/clickhouse-logger.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,22 @@ title: clickhouse-logger

## 属性

| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
| endpoint_addr | string | 必须 | | | `clickhouse` 服务器的 endpoint。 |
| database | string | 必须 | | | 使用的数据库。 |
| logtable | string | 必须 | | | 写入的表名 。 |
| user | string | 必须 | | | clickhouse 的用户。 |
| password | string | 必须 | | | clickhouse 的密码 。 |
| timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 |
| name | string | 可选 | "clickhouse logger" | | 标识 logger 的唯一标识符。 |
| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。 |
| endpoint_addr | 数组 | 必须 | | | `clickhouse` 服务器的 endpoints。 |
| database | string | 必须 | | | 使用的数据库。 |
| logtable | string | 必须 | | | 写入的表名 。 |
| user | string | 必须 | | | clickhouse 的用户。 |
| password | string | 必须 | | | clickhouse 的密码 。 |
| timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 |
| name | string | 可选 | "clickhouse logger" | | 标识 logger 的唯一标识符。 |
| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。 |

本插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据，如需了解或自定义批处理器相关参数设置，请参考 [Batch-Processor](../batch-processor.md#配置) 配置部分。

## 如何开启

如果配置多个 endpoints 将随机写入。
这是有关如何为特定路由启用 `clickhouse-logger` 插件的示例。

```shell
Expand All @@ -53,7 +54,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
"password": "a",
"database": "default",
"logtable": "test",
"endpoint_addr": "http://127.0.0.1:8123"
"endpoint_addr": ["http://127.0.0.1:8123"]
}
},
"upstream": {
Expand Down
75 changes: 71 additions & 4 deletions t/plugin/clickhouse-logger.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,18 @@ add_block_preprocessor(sub {
ngx.say("ok")
}
}
location /clickhouse-logger/test1 {
content_by_lua_block {
ngx.req.read_body()
local data = ngx.req.get_body_data()
local headers = ngx.req.get_headers()
ngx.log(ngx.WARN, "clickhouse body: ", data)
for k, v in pairs(headers) do
ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v)
end
ngx.say("ok")
}
}
}
_EOC_

Expand All @@ -70,7 +82,7 @@ __DATA__
password = "a",
database = "default",
logtable = "t",
endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test",
endpoint_addrs = {"http://127.0.0.1:10420/clickhouse-logger/test"},
max_retry_count = 1,
name = "clickhouse logger",
ssl_verify = false
Expand All @@ -97,7 +109,7 @@ passed
password = "a",
database = "default",
logtable = "t",
endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test"
endpoint_addrs = {"http://127.0.0.1:10420/clickhouse-logger/test"}
})
if not ok then
Expand Down Expand Up @@ -131,7 +143,7 @@ passed
}
}
--- response_body
property "endpoint_addr" is required
property "endpoint_addrs" is required
Expand All @@ -149,7 +161,7 @@ property "endpoint_addr" is required
"password": "a",
"database": "default",
"logtable": "t",
"endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test",
"endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test"],
"batch_max_size":1,
"inactive_timeout":1
}
Expand Down Expand Up @@ -186,3 +198,58 @@ clickhouse headers: x-clickhouse-key:a
clickhouse headers: x-clickhouse-user:default
clickhouse headers: x-clickhouse-database:default
--- wait: 5
=== TEST 6: add plugin on routes using multi clickhouse-logger
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"plugins": {
"clickhouse-logger": {
"user": "default",
"password": "a",
"database": "default",
"logtable": "t",
"endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test",
"http://127.0.0.1:10420/clickhouse-logger/test1"],
"batch_max_size":1,
"inactive_timeout":1
}
},
"upstream": {
"nodes": {
"127.0.0.1:1982": 1
},
"type": "roundrobin"
},
"uri": "/opentracing"
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- response_body
passed
=== TEST 7: access local server
--- request
GET /opentracing
--- response_body
opentracing
--- error_log
clickhouse body: INSERT INTO t FORMAT JSONEachRow
clickhouse headers: x-clickhouse-key:a
clickhouse headers: x-clickhouse-user:default
clickhouse headers: x-clickhouse-database:default
--- wait: 5

0 comments on commit 7f2da81

Please sign in to comment.