WIP: Replace ingress-nginx with higress. (#4831)

* WIP: Replace `ingress-nginx` to `higress`. Signed-off-by: zzjin <[email protected]> * feat: update higress version * feat: controller autoscalling and gateway daemonset * feat: Optimize H2 flow control buffer size * chore: bump higress version to 2.0.0 * chore: no need to install istio --------- Signed-off-by: zzjin <[email protected]> Co-authored-by: zijiren233 <[email protected]>
labring · Sep 13, 2024 · 7ef586d · 7ef586d
1 parent bf2fe19
commit 7ef586d
Showing 1 changed file with 151 additions and 26 deletions.
diff --git a/scripts/cloud/install.sh b/scripts/cloud/install.sh
@@ -31,7 +31,7 @@ cilium_version=${cilium_version:-"1.14.8"}
 cert_manager_version=${cert_manager_version:-"1.14.6"}
 helm_version=${helm_version:-"3.14.1"}
 openebs_version=${openebs_version:-"3.10.0"}
-ingress_nginx_version=${ingress_nginx_version:-"1.9.4"}
+higress_version=${higress_version:-"2.0.0"}
 kubeblocks_version=${kubeblocks_version:-"0.8.2"}
 metrics_server_version=${metrics_server_version:-"0.6.4"}
 victoria_metrics_k8s_stack_version=${victoria_metrics_k8s_stack_version:-"1.96.0"}
@@ -60,9 +60,8 @@ PROMPTS_EN=(
     ["choose_language"]="Please choose a language: "
     ["enter_choice"]="Please enter your choice (zh/en): "
     ["k8s_installation"]="Installing Kubernetes cluster."
-    ["ingress_installation"]="Installing Ingress-nginx-controller and Kubeblocks."
+    ["partner_installation"]="Installing Higress and Kubeblocks."
     ["installing_monitoring"]="Installing kubernetes monitoring."
-    ["patching_ingress"]="Modifying the tolerance of Ingress-nginx-controller to allow it to run on the master node."
     ["installing_cloud"]="Installing Sealos Cloud."
     ["avx_not_supported"]="CPU does not support AVX instruction set."
     ["ssh_private_key"]="Please enter the ssh private key path (Press enter to use the default value: '/root/.ssh/id_rsa'): "
@@ -71,6 +70,7 @@ PROMPTS_EN=(
     ["cilium_requirement"]="Using Cilium as the network plugin, the host system must meet the following requirements:
 1. Hosts with AMD64 or AArch64 architecture;
 2. Linux kernel> = 4.19.57 or equivalent version (e.g., 4.18 on RHEL8)."
+    ["optimizing_h2_buffer"]="Optimizing the size of the H2 flow control buffer."
     ["mongo_avx_requirement"]="MongoDB 5.0 version depends on a CPU that supports the AVX instruction set. The current environment does not support AVX, so it has been switched to MongoDB 4.4 version. For more information, see: https://www.mongodb.com/docs/v5.0/administration/production-notes/"
     ["enable_acme"]="Do you want to enable ACME to automatically obtain certificates (Press n to use the self-signed certificate provided by Sealos)? (y/n): "
     ["acmedns_registration_failed"]="ACME DNS registration failed. Please check if the acmedns-host: '${GREEN}%s${RESET}' is correct."
@@ -86,7 +86,7 @@ Options:
   --cert-manager-version            # Cert Manager version (default: 1.14.6)
   --helm-version                    # Helm version (default: 3.14.1)
   --openebs-version                 # OpenEBS version (default: 3.10.0)
-  --ingress-nginx-version           # Ingress Nginx version (default: 1.9.4)
+  --higress-version                 # Higress version (default: 2.0.0)
   --kubeblocks-version              # Kubeblocks version (default: 0.8.2)
   --metrics-server-version          # Metrics Server version (default: 0.6.4)
   --cloud-version                   # Sealos Cloud version (default: latest)
@@ -129,9 +129,8 @@ PROMPTS_CN=(
     ["choose_language"]="请选择语言: "
     ["enter_choice"]="请输入您的选择 (zh/en): "
     ["k8s_installation"]="正在安装 Kubernetes 集群."
-    ["ingress_installation"]="正在安装 Ingress-nginx-controller 和 Kubeblocks."
+    ["partner_installation"]="正在安装 Higress 和 Kubeblocks."
     ["installing_monitoring"]="正在安装 kubernetes 监控."
-    ["patching_ingress"]="正在修改 Ingress-nginx-controller 的容忍度, 以允许它在主节点上运行."
     ["installing_cloud"]="正在安装 Sealos Cloud."
     ["avx_not_supported"]="CPU 不支持 AVX 指令集."
     ["ssh_private_key"]="请输入 ssh 私钥路径 (回车使用默认值: '/root/.ssh/id_rsa'): "
@@ -140,6 +139,7 @@ PROMPTS_CN=(
     ["cilium_requirement"]="正在使用 Cilium 作为网络插件, 主机系统必须满足以下要求:
 1.具有AMD64或AArch64架构的主机;
 2.Linux内核> = 4.19.57或等效版本 (例如, 在RHEL8上为4.18)."
+    ["optimizing_h2_buffer"]="正在优化H2流控缓冲区大小."
     ["mongo_avx_requirement"]="MongoDB 5.0版本依赖支持 AVX 指令集的 CPU, 当前环境不支持 AVX, 已切换为 MongoDB 4.4版本, 更多信息查看: https://www.mongodb.com/docs/v5.0/administration/production-notes/"
     ["enable_acme"]="是否启用 ACME 自动获取证书（输入 n 使用 Sealos 提供的自签证书）? (y/n): "
     ["acmedns_registration_failed"]="注册 ACME DNS 失败, 请检查 acmedns-host: '${GREEN}%s${RESET}' 是否正确."
@@ -155,7 +155,7 @@ Options:
   --cert-manager-version          # Cert Manager版本 (默认: 1.14.6)
   --helm-version                  # Helm版本 (默认: 3.14.1)
   --openebs-version               # OpenEBS版本 (默认: 3.10.0)
-  --ingress-nginx-version         # Ingress Nginx版本 (默认: 1.9.4)
+  --higress-version               # Higress版本 (默认: 2.0.0)
   --kubeblocks-version            # Kubeblocks版本 (默认: 0.8.2)
   --metrics-server-version        # Metrics Server版本 (默认: 0.6.4)
   --cloud-version                 # Sealos Cloud版本 (默认: latest)
@@ -271,7 +271,7 @@ init() {
     pull_image "cert-manager" "v${cert_manager_version#v:-1.14.6}"
     pull_image "helm" "v${helm_version#v:-3.14.1}"
     pull_image "openebs" "v${openebs_version#v:-3.10.0}"
-    pull_image "ingress-nginx" "v${ingress_nginx_version#v:-1.9.4}"
+    pull_image "higress" "v${higress_version#v:-2.0.0}"
     pull_image "kubeblocks" "v${kubeblocks_version#v:-0.8.2}"
     pull_image "kubeblocks-redis" "v${kubeblocks_version#v:-0.8.2}"
     pull_image "kubeblocks-apecloud-mysql" "v${kubeblocks_version#v:-0.8.2}"
@@ -401,24 +401,151 @@ spec:
         echo "$tls_config" > $CLOUD_DIR/tls-secret.yaml
     fi
 
-    ingress_config="
+    higress_config="
 apiVersion: apps.sealos.io/v1beta1
 kind: Config
 metadata:
-  creationTimestamp: null
-  name: ingress-nginx-config
+  name: higress-config
 spec:
   data: |
-    controller:
+    global:
+      ingressClass: nginx
+      enableStatus: false
+      enableGatewayAPI: false
+      disableAlpnH2: false
+      enableIstioAPI: true
+      enableSRDS: true
+    gateway:
+      httpsPort: ${cloud_port:-443}
       hostNetwork: true
-      kind: DaemonSet
       service:
         type: NodePort
-  match: ${image_registry}/${image_repository}/ingress-nginx:v${ingress_nginx_version#v:-1.9.4}
-  path: charts/ingress-nginx/values.yaml
+      kind: DaemonSet
+      tolerations:
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+      resources:
+        requests:
+          cpu: 256m
+          memory: 256Mi
+        limits:
+          memory: 4Gi
+    controller:
+      autoscaling:
+        enabled: true
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ''
+      resources:
+        requests:
+          cpu: 256m
+          memory: 256Mi
+  match: ${image_registry}/${image_repository}/higress:v${higress_version#v:-2.0.0}
+  path: charts/higress/charts/higress-core/values.yaml
+  strategy: merge
+"
+    echo "$higress_config" > $CLOUD_DIR/higress-config.yaml
+    higress_console_config="
+apiVersion: apps.sealos.io/v1beta1
+kind: Config
+metadata:
+  name: higress-console-config
+spec:
+  data: |
+    replicaCount: 0
+  match: ${image_registry}/${image_repository}/higress:v${higress_version#v:-2.0.0}
+  path: charts/higress/charts/higress-console/values.yaml
   strategy: merge
 "
-    echo "$ingress_config" > $CLOUD_DIR/ingress-nginx-config.yaml
+    echo "$higress_console_config" > $CLOUD_DIR/higress-console-config.yaml
+
+    higress_https_config="
+apiVersion: v1
+data:
+  cert: |
+    automaticHttps: false
+    fallbackForInvalidSecret: true
+    acmeIssuer:
+    - email: [email protected]
+      name: letsencrypt
+    renewBeforeDays: 1
+    credentialConfig:
+    - domains:
+        - '*.$cloud_domain'
+        - '$cloud_domain'
+      tlsSecret: sealos-system/wildcard-cert
+kind: ConfigMap
+metadata:
+  name: higress-https
+  namespace: higress-system
+"
+    echo "$higress_https_config" > $CLOUD_DIR/higress-https.yaml
+
+    higress_plugins_config="
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name:  hcm-options
+  namespace: higress-system
+spec:
+  configPatches:
+  - applyTo: NETWORK_FILTER
+    match:
+      context: GATEWAY
+      listener:
+        filterChain:
+          filter:
+            name: envoy.filters.network.http_connection_manager
+    patch:
+      operation: MERGE
+      value:
+        name: envoy.filters.network.http_connection_manager
+        typed_config:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          max_request_headers_kb: 8192
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name: global-route-config
+  namespace: higress-system
+spec:
+  configPatches:
+  - applyTo: ROUTE_CONFIGURATION
+    match:
+      context: GATEWAY
+    patch:
+      operation: MERGE
+      value:
+        request_headers_to_add:
+        - append: false
+          header:
+            key: x-real-ip
+            value: '%REQ(X-ENVOY-EXTERNAL-ADDRESS)%'
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: EnvoyFilter
+metadata:
+  name:  tailscale-options
+  namespace: higress-system
+spec:
+  configPatches:
+  - applyTo: NETWORK_FILTER
+    match:
+      context: GATEWAY
+      listener:
+        filterChain:
+          filter:
+            name: envoy.filters.network.http_connection_manager
+    patch:
+      operation: MERGE
+      value:
+        typed_config:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          upgrade_configs:
+            - upgrade_type: tailscale-control-protocol
+"
+    echo "$higress_plugins_config" > $CLOUD_DIR/higress-plugins.yaml
 
     backuprepo='
     apiVersion: dataprotection.kubeblocks.io/v1alpha1
@@ -632,8 +759,12 @@ EOF
     get_prompt "installing_monitoring"
     sealos run "${image_registry}/${image_repository}/victoria-metrics-k8s-stack:v${victoria_metrics_k8s_stack_version#v:-1.96.0}"
 
-    get_prompt "ingress_installation"
-    sealos run ${image_registry}/${image_repository}/ingress-nginx:v${ingress_nginx_version#v:-1.9.4} --config-file $CLOUD_DIR/ingress-nginx-config.yaml
+    get_prompt "partner_installation"
+    sealos run ${image_registry}/${image_repository}/higress:v${higress_version#v:-2.0.0} --config-file $CLOUD_DIR/higress-config.yaml --config-file $CLOUD_DIR/higress-console-config.yaml
+    kubectl apply -f $CLOUD_DIR/higress-https.yaml
+    kubectl apply -f $CLOUD_DIR/higress-plugins.yaml
+    get_prompt "optimizing_h2_buffer"
+    kubectl patch cm higress-config -n higress-system -p '{"data":{"higress":"downstream:\n  http2:\n    initialConnectionWindowSize: 4194304\n    initialStreamWindowSize: 524288"}}' --type=merge
 
     sealos run ${image_registry}/${image_repository}/kubeblocks:v${kubeblocks_version#v:-0.8.2}
     sealos run ${image_registry}/${image_repository}/kubeblocks-apecloud-mysql:v${kubeblocks_version#v:-0.8.2} \
@@ -653,12 +784,6 @@ EOF
     kubectl patch vmagent -n vm victoria-metrics-k8s-stack --type merge -p '{"spec":{"additionalScrapeConfigs":{"key":"prometheus-additional.yaml","name":"additional-scrape-configs"}}}'
     kubectl rollout restart deploy -n vm vmagent-victoria-metrics-k8s-stack || true
 
-    get_prompt "patching_ingress"
-    kubectl patch cm -n ingress-nginx ingress-nginx-controller --patch '{"data":{"allow-snippet-annotations":"true","annotation-value-word-blocklist":"load_module,lua_package,_by_lua,location,root,proxy_pass,serviceaccount"}}'
-    kubectl -n ingress-nginx patch ds ingress-nginx-controller -p '{"spec":{"template":{"spec":{"tolerations":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists","effect":"NoSchedule"}]}}}}'
-    kubectl get daemonset ingress-nginx-controller -n ingress-nginx -o json | grep https-port= >/dev/null || kubectl patch daemonset ingress-nginx-controller -n ingress-nginx --type='json' -p="[{'op': 'add', 'path': '/spec/template/spec/containers/0/args/-', 'value': '--https-port=${cloud_port:-443}'}]"
-    kubectl get daemonset ingress-nginx-controller -n ingress-nginx -o json | grep default-ssl-certificate= >/dev/null || kubectl patch daemonset ingress-nginx-controller -n ingress-nginx --type='json' -p="[{'op': 'add', 'path': '/spec/template/spec/containers/0/args/-', 'value': '--default-ssl-certificate=sealos-system/wildcard-cert'}]"
-
     get_prompt "installing_cloud"
 
     setMongoVersion
@@ -693,7 +818,7 @@ for i in "$@"; do
   --cert-manager-version=*) cert_manager_version="${i#*=}"; shift ;;
   --helm-version=*) helm_version="${i#*=}"; shift ;;
   --openebs-version=*) openebs_version="${i#*=}"; shift ;;
-  --ingress-nginx-version=*) ingress_nginx_version="${i#*=}"; shift ;;
+  --higress-version=*) higress_version="${i#*=}"; shift ;;
   --kubeblocks-version=*) kubeblocks_version="${i#*=}"; shift ;;
   --metrics-server-version=*) metrics_server_version="${i#*=}"; shift ;;
   --cloud-version=*) cloud_version="${i#*=}"; shift ;;
@@ -725,7 +850,7 @@ for i in "$@"; do
   --cert-manager-version | cert-manager-version | \
   --helm-version | helm-version | \
   --openebs-version | openebs-version | \
-  --ingress-nginx-version | ingress-nginx-version | \
+  --higress-version | higress-version | \
   --kubeblocks-version | kubeblocks-version | \
   --metrics-server-version | metrics-server-version | \
   --cloud-version | cloud-version | \