feat: support multiple kubernets clusters discovery (#7895)

apache · Sep 15, 2022 · 5a630b8 · 5a630b8
1 parent ac2c1b3
commit 5a630b8
Show file tree

Hide file tree

Showing 10 changed files with 2,122 additions and 800 deletions.
diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml
@@ -44,22 +44,17 @@ jobs:
 
       - name: Setup kubernetes cluster
         run: |
-          KIND_VERSION="v0.11.1"
-          KUBECTL_VERSION="v1.22.0"
-          curl -Lo ./kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-$(uname)-amd64"
-          curl -Lo ./kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
-          chmod +x ./kind
-          chmod +x ./kubectl
+          KUBERNETES_VERSION="v1.22.7"
 
-          ./kind create cluster --name apisix-test --config ./t/kubernetes/configs/kind.yaml
+          kind create cluster --name apisix-test --config ./t/kubernetes/configs/kind.yaml --image kindest/node:${KUBERNETES_VERSION}
 
-          ./kubectl wait --for=condition=Ready nodes --all --timeout=180s
+          kubectl wait --for=condition=Ready nodes --all --timeout=180s
 
-          ./kubectl apply -f ./t/kubernetes/configs/account.yaml
+          kubectl apply -f ./t/kubernetes/configs/account.yaml
 
-          ./kubectl apply -f ./t/kubernetes/configs/endpoint.yaml
+          kubectl apply -f ./t/kubernetes/configs/endpoint.yaml
 
-          KUBERNETES_CLIENT_TOKEN_CONTENT=$(./kubectl get secrets | grep apisix-test | awk '{system("./kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}')
+          KUBERNETES_CLIENT_TOKEN_CONTENT=$(kubectl get secrets | grep apisix-test | awk '{system("kubectl get secret -o jsonpath={.data.token} "$1" | base64 --decode")}')
 
           KUBERNETES_CLIENT_TOKEN_DIR="/tmp/var/run/secrets/kubernetes.io/serviceaccount"
 
@@ -73,7 +68,7 @@ jobs:
           echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}"
           echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE}
 
-          ./kubectl proxy -p 6445 &
+          kubectl proxy -p 6445 &
 
       - name: Linux Install
         run: |

diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
@@ -238,8 +238,11 @@ http {
     lua_shared_dict balancer-ewma-last-touched-at {* http.lua_shared_dict["balancer-ewma-last-touched-at"] *};
     lua_shared_dict etcd-cluster-health-check {* http.lua_shared_dict["etcd-cluster-health-check"] *}; # etcd health check
 
-    {% if enabled_discoveries["kubernetes"] then %}
-    lua_shared_dict kubernetes {* http.lua_shared_dict["kubernetes"] *};
+    # for discovery shared dict
+    {% if discovery_shared_dicts then %}
+    {% for key, size in pairs(discovery_shared_dicts) do %}
+    lua_shared_dict {*key*} {*size*};
+    {% end %}
     {% end %}
 
     {% if enabled_discoveries["tars"] then %}

diff --git a/apisix/cli/ops.lua b/apisix/cli/ops.lua
@@ -662,36 +662,52 @@ Please modify "admin_key" in conf/config.yaml .
         end
     end
 
-    -- inject kubernetes discovery environment variable
+    -- inject kubernetes discovery shared dict and environment variable
     if enabled_discoveries["kubernetes"] then
 
-        local kubernetes_conf = yaml_conf.discovery["kubernetes"]
+        if not sys_conf["discovery_shared_dicts"] then
+            sys_conf["discovery_shared_dicts"] = {}
+        end
 
-        local keys = {
-            kubernetes_conf.service.host,
-            kubernetes_conf.service.port,
-        }
+        local kubernetes_conf = yaml_conf.discovery["kubernetes"]
 
-        if kubernetes_conf.client.token then
-            table_insert(keys, kubernetes_conf.client.token)
-        end
+        local inject_environment = function(conf, envs)
+            local keys = {
+                conf.service.host,
+                conf.service.port,
+            }
 
-        if kubernetes_conf.client.token_file then
-            table_insert(keys, kubernetes_conf.client.token_file)
-        end
+            if conf.client.token then
+                table_insert(keys, conf.client.token)
+            end
 
-        local envs = {}
+            if conf.client.token_file then
+                table_insert(keys, conf.client.token_file)
+            end
 
-        for _, key in ipairs(keys) do
-            if #key > 3 then
-                local first, second = str_byte(key, 1, 2)
-                if first == str_byte('$') and second == str_byte('{') then
-                    local last = str_byte(key, #key)
-                    if last == str_byte('}') then
-                        envs[str_sub(key, 3, #key - 1)] = ""
+            for _, key in ipairs(keys) do
+                if #key > 3 then
+                    local first, second = str_byte(key, 1, 2)
+                    if first == str_byte('$') and second == str_byte('{') then
+                        local last = str_byte(key, #key)
+                        if last == str_byte('}') then
+                            envs[str_sub(key, 3, #key - 1)] = ""
+                        end
                     end
                 end
             end
+
+        end
+
+        local envs = {}
+        if #kubernetes_conf == 0 then
+            sys_conf["discovery_shared_dicts"]["kubernetes"] = kubernetes_conf.shared_size
+            inject_environment(kubernetes_conf, envs)
+        else
+            for _, item in ipairs(kubernetes_conf) do
+                sys_conf["discovery_shared_dicts"]["kubernetes-" .. item.id] = item.shared_size
+                inject_environment(item, envs)
+            end
         end
 
         if not sys_conf["envs"] then
@@ -701,6 +717,7 @@ Please modify "admin_key" in conf/config.yaml .
         for item in pairs(envs) do
             table_insert(sys_conf["envs"], item)
         end
+
     end
 
     -- fix up lua path