Fix database controller memory leak

src/database-controller/config/database-controller.yaml

@@ -39,6 +39,10 @@ db-poller-max-heap-mb: 4096
 framework-watcher-max-rpc-concurrency: 150
 # Framework watcher max heap size in MB
 framework-watcher-max-heap-mb: 8192
+# Timeout seconds of watching in framework watcher.
+# Larger timeout can reduce list impact caused to K8S api server,
+# but increase memory usage.
+framework-watcher-watch-timeout-seconds: 86400
 
 # cluster event watcher
 # Max rpc concurrency for cluster event watcher
@@ -49,3 +53,7 @@ cluster-event-watcher-max-heap-mb: 2048
 cluster-event-max-db-connection: 40
 # Max disk usage in internal storage for cluster event watcher
 cluster-event-watcher-max-disk-usage-percent: 80
+# Timeout seconds of watching in cluster event watcher.
+# Larger timeout can reduce list impact caused to K8S api server,
+# but increase memory usage.
+cluster-event-watcher-watch-timeout-seconds: 86400

src/database-controller/deploy/database-controller.yaml.template

@@ -157,6 +157,8 @@ spec:
         # Max rpc concurrency for framework watcher
         - name: MAX_RPC_CONCURRENCY
           value: "{{ cluster_cfg['database-controller']['framework-watcher-max-rpc-concurrency'] }}"
+        - name: WATCH_TIMEOUT_SECONDS
+          value: "{{ cluster_cfg['database-controller']['framework-watcher-watch-timeout-seconds'] }}"
         command: ["node", "--max-old-space-size={{ cluster_cfg['database-controller']['framework-watcher-max-heap-mb'] }}", "watcher/framework/index.js"]
       - name: cluster-event-watcher
         image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}database-controller:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
@@ -211,6 +213,8 @@ spec:
           value: "/paiInternal/storage"
         - name: MAX_DISK_USAGE_PERCENT
           value:  "{{ cluster_cfg['database-controller']['cluster-event-watcher-max-disk-usage-percent'] }}"
+        - name: WATCH_TIMEOUT_SECONDS
+          value: "{{ cluster_cfg['database-controller']['cluster-event-watcher-watch-timeout-seconds'] }}"
         command: ["node", "--max-old-space-size={{ cluster_cfg['database-controller']['cluster-event-watcher-max-heap-mb'] }}", "watcher/cluster-event/index.js"]
       - name: poller
         image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}database-controller:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}

src/database-controller/src/watcher/cluster-event/config.js

@@ -21,6 +21,9 @@ const configSchema = Joi.object()
     maxDiskUsagePercent: Joi.number()
       .integer()
       .required(),
+    watchTimeoutSeconds: Joi.number()
+      .integer()
+      .required(),
   })
   .required();
 
@@ -31,6 +34,7 @@ const config = {
   diskPath: process.env.DISK_PATH,
   diskCheckIntervalSecond: 60,
   maxDiskUsagePercent: parseInt(process.env.MAX_DISK_USAGE_PERCENT),
+  watchTimeoutSeconds: parseInt(process.env.WATCH_TIMEOUT_SECONDS),
 };
 
 const { error, value } = Joi.validate(config, configSchema);

src/database-controller/src/watcher/cluster-event/index.js

@@ -104,7 +104,7 @@ async function assertDiskUsageHealthy() {
 }
 
 function startInformer() {
-  const informer = getEventInformer();
+  const informer = getEventInformer(config.watchTimeoutSeconds);
 
   informer.on('add', apiObject => {
     eventHandler('ADDED', apiObject);

src/database-controller/src/watcher/framework/config.js

@@ -13,12 +13,16 @@ const configSchema = Joi.object()
     maxRpcConcurrency: Joi.number()
       .integer()
       .required(),
+    watchTimeoutSeconds: Joi.number()
+      .integer()
+      .required(),
   })
   .required();
 
 const config = {
   writeMergerUrl: process.env.WRITE_MERGER_URL,
   maxRpcConcurrency: parseInt(process.env.MAX_RPC_CONCURRENCY),
+  watchTimeoutSeconds: parseInt(process.env.WATCH_TIMEOUT_SECONDS),
 };
 
 const { error, value } = Joi.validate(config, configSchema);

src/database-controller/src/watcher/framework/index.js

@@ -57,7 +57,7 @@ const eventHandler = (eventType, apiObject) => {
   });
 };
 
-const informer = getFrameworkInformer();
+const informer = getFrameworkInformer(config.watchTimeoutSeconds);
 
 informer.on('add', apiObject => {
   eventHandler('ADDED', apiObject);