feat(server): support runtime pod affinity, add task switcher env (#846)

deploy/scripts/install-on-linux.sh

@@ -56,6 +56,9 @@ NODENAME=$(kubectl get nodes -ojsonpath='{.items[0].metadata.name}')
 kubectl taint node $NODENAME node-role.kubernetes.io/master-
 kubectl taint node $NODENAME node-role.kubernetes.io/control-plane-
 
+# label master node as a app node
+kubectl label node $NODENAME laf.dev/node.type=app
+
 # install required components
 sealos run labring/openebs:v1.9.0
 sealos run labring/cert-manager:v1.8.0

server/src/application/application-task.service.ts

@@ -14,7 +14,7 @@ import { DatabaseService } from '../database/database.service'
 import { ClusterService } from 'src/region/cluster/cluster.service'
 import { RegionService } from 'src/region/region.service'
 import { RuntimeDomainService } from 'src/gateway/runtime-domain.service'
-import { TASK_LOCK_INIT_TIME } from 'src/constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { SystemDatabase } from 'src/database/system-database'
 
 @Injectable()
@@ -32,6 +32,10 @@ export class ApplicationTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_APPLICATION_TASK) {
+      return
+    }
+
     // Phase `Creating` -> `Created`
     this.handleCreatingPhase()
 

server/src/constants.ts

@@ -53,6 +53,24 @@ export class ServerConfig {
     return process.env.SYSTEM_NAMESPACE || 'laf-system'
   }
 
+  /* switcher of task controllers */
+
+  static get DISABLED_INSTANCE_TASK() {
+    return process.env.DISABLED_INSTANCE_TASK || false
+  }
+
+  static get DISABLED_APPLICATION_TASK() {
+    return process.env.DISABLED_APPLICATION_TASK || false
+  }
+
+  static get DISABLED_GATEWAY_TASK() {
+    return process.env.DISABLED_GATEWAY_TASK || false
+  }
+
+  static get DISABLED_STORAGE_TASK() {
+    return process.env.DISABLED_STORAGE_TASK || false
+  }
+
   static get APPID_LENGTH(): number {
     return parseInt(process.env.APPID_LENGTH || '6')
   }
@@ -123,22 +141,15 @@ export class ServerConfig {
   }
 }
 
-export class ResourceLabelKey {
-  static get USER_ID() {
-    return 'laf.dev/user.id'
-  }
-
-  static get APP_ID() {
-    return 'laf.dev/appid'
-  }
-
-  static get NAMESPACE_TYPE() {
-    return 'laf.dev/namespace.type'
-  }
-
-  static get BUNDLE() {
-    return 'laf.dev/bundle'
-  }
+export const LABEL_KEY_USER_ID = 'laf.dev/user.id'
+export const LABEL_KEY_APP_ID = 'laf.dev/appid'
+export const LABEL_KEY_NAMESPACE_TYPE = 'laf.dev/namespace.type'
+export const LABEL_KEY_BUNDLE = 'laf.dev/bundle'
+export const LABEL_KEY_NODE_TYPE = 'laf.dev/node.type'
+export enum NodeType {
+  Runtime = 'runtime',
+  Database = 'database',
+  Storage = 'storage',
 }
 
 // Runtime constants

server/src/gateway/bucket-domain-task.service.ts

@@ -5,7 +5,7 @@ import { ApisixService } from './apisix.service'
 import * as assert from 'node:assert'
 import { Cron, CronExpression } from '@nestjs/schedule'
 import { times } from 'lodash'
-import { TASK_LOCK_INIT_TIME } from 'src/constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { SystemDatabase } from 'src/database/system-database'
 
 @Injectable()
@@ -21,6 +21,10 @@ export class BucketDomainTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_GATEWAY_TASK) {
+      return
+    }
+
     // Phase `Creating` -> `Created`
     times(this.concurrency, () => this.handleCreatingPhase())
 

server/src/gateway/runtime-domain-task.service.ts

@@ -4,7 +4,7 @@ import { RegionService } from '../region/region.service'
 import { ApisixService } from './apisix.service'
 import * as assert from 'node:assert'
 import { Cron, CronExpression } from '@nestjs/schedule'
-import { TASK_LOCK_INIT_TIME } from '../constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from '../constants'
 import { SystemDatabase } from '../database/system-database'
 
 @Injectable()
@@ -19,6 +19,10 @@ export class RuntimeDomainTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_GATEWAY_TASK) {
+      return
+    }
+
     // Phase `Creating` -> `Created`
     this.handleCreatingPhase()
 

server/src/gateway/website-task.service.ts

@@ -7,7 +7,7 @@ import {
   WebsiteHosting,
 } from '@prisma/client'
 import { times } from 'lodash'
-import { TASK_LOCK_INIT_TIME } from 'src/constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { SystemDatabase } from 'src/database/system-database'
 import { RegionService } from 'src/region/region.service'
 import * as assert from 'node:assert'
@@ -26,6 +26,10 @@ export class WebsiteTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_GATEWAY_TASK) {
+      return
+    }
+
     // Phase `Creating` -> `Created`
     times(this.concurrency, () => this.handleCreatingPhase())
 

server/src/instance/instance-task.service.ts

@@ -4,7 +4,7 @@ import { Application, ApplicationPhase, ApplicationState } from '@prisma/client'
 import { isConditionTrue } from '../utils/getter'
 import { InstanceService } from './instance.service'
 import { times } from 'lodash'
-import { TASK_LOCK_INIT_TIME } from 'src/constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { SystemDatabase } from 'src/database/system-database'
 import { ClusterService } from 'src/region/cluster/cluster.service'
 
@@ -21,6 +21,10 @@ export class InstanceTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_INSTANCE_TASK) {
+      return
+    }
+
     // Phase `Created` | `Stopped` ->  `Starting`
     times(this.concurrency, () => this.handleRunningState())
 

server/src/instance/instance.service.ts

@@ -1,7 +1,13 @@
 import { V1Deployment } from '@kubernetes/client-node'
 import { Injectable, Logger } from '@nestjs/common'
 import { GetApplicationNamespaceByAppId } from '../utils/getter'
-import { MB, ResourceLabelKey } from '../constants'
+import {
+  LABEL_KEY_APP_ID,
+  LABEL_KEY_BUNDLE,
+  LABEL_KEY_NODE_TYPE,
+  MB,
+  NodeType,
+} from '../constants'
 import { PrismaService } from '../prisma/prisma.service'
 import { StorageService } from '../storage/storage.service'
 import { DatabaseService } from 'src/database/database.service'
@@ -24,7 +30,7 @@ export class InstanceService {
 
   async create(app: Application) {
     const appid = app.appid
-    const labels = { [ResourceLabelKey.APP_ID]: appid }
+    const labels = { [LABEL_KEY_APP_ID]: appid }
     const region = await this.regionService.findByAppId(appid)
     const appWithRegion = { ...app, region } as ApplicationWithRegion
 
@@ -51,7 +57,7 @@ export class InstanceService {
     })
 
     // add bundle label
-    labels[ResourceLabelKey.BUNDLE] = app.bundle.name
+    labels[LABEL_KEY_BUNDLE] = app.bundle.name
 
     // prepare params
     const limitMemory = app.bundle.resource.limitMemory
@@ -184,8 +190,41 @@ export class InstanceService {
               emptyDir: {},
             },
           ],
-        },
-      },
+          affinity: {
+            nodeAffinity: {
+              // required to schedule on runtime node
+              requiredDuringSchedulingIgnoredDuringExecution: {
+                nodeSelectorTerms: [
+                  {
+                    matchExpressions: [
+                      {
+                        key: LABEL_KEY_NODE_TYPE,
+                        operator: 'In',
+                        values: [NodeType.Runtime],
+                      },
+                    ],
+                  },
+                ],
+              },
+              // preferred to schedule on bundle matched node
+              preferredDuringSchedulingIgnoredDuringExecution: [
+                {
+                  weight: 10,
+                  preference: {
+                    matchExpressions: [
+                      {
+                        key: LABEL_KEY_BUNDLE,
+                        operator: 'In',
+                        values: [app.bundle.name],
+                      },
+                    ],
+                  },
+                },
+              ],
+            }, // end of nodeAffinity {}
+          }, // end of affinity {}
+        }, // end of spec {}
+      }, // end of template {}
     }
 
     const appsV1Api = this.clusterService.makeAppsV1Api(app.region)

server/src/region/cluster/cluster.service.ts

@@ -3,9 +3,13 @@ import { KubernetesObject } from '@kubernetes/client-node'
 import * as k8s from '@kubernetes/client-node'
 import { Region } from '@prisma/client'
 import { GetApplicationNamespaceByAppId } from 'src/utils/getter'
-import { ResourceLabelKey } from 'src/constants'
 import { compare } from 'fast-json-patch'
 import { GroupVersionKind } from 'src/region/cluster/types'
+import {
+  LABEL_KEY_APP_ID,
+  LABEL_KEY_NAMESPACE_TYPE,
+  LABEL_KEY_USER_ID,
+} from 'src/constants'
 
 @Injectable()
 export class ClusterService {
@@ -38,9 +42,9 @@ export class ClusterService {
       namespace.metadata = new k8s.V1ObjectMeta()
       namespace.metadata.name = GetApplicationNamespaceByAppId(appid)
       namespace.metadata.labels = {
-        [ResourceLabelKey.APP_ID]: appid,
-        [ResourceLabelKey.NAMESPACE_TYPE]: 'app',
-        [ResourceLabelKey.USER_ID]: userid,
+        [LABEL_KEY_APP_ID]: appid,
+        [LABEL_KEY_NAMESPACE_TYPE]: 'app',
+        [LABEL_KEY_USER_ID]: userid,
       }
       const coreV1Api = this.makeCoreV1Api(region)
 

server/src/storage/bucket-task.service.ts

@@ -4,7 +4,7 @@ import { RegionService } from 'src/region/region.service'
 import * as assert from 'node:assert'
 import { Cron, CronExpression } from '@nestjs/schedule'
 import { times } from 'lodash'
-import { TASK_LOCK_INIT_TIME } from 'src/constants'
+import { ServerConfig, TASK_LOCK_INIT_TIME } from 'src/constants'
 import { SystemDatabase } from 'src/database/system-database'
 import { MinioService } from './minio/minio.service'
 import { BucketDomainService } from 'src/gateway/bucket-domain.service'
@@ -23,6 +23,10 @@ export class BucketTaskService {
 
   @Cron(CronExpression.EVERY_SECOND)
   async tick() {
+    if (ServerConfig.DISABLED_STORAGE_TASK) {
+      return
+    }
+
     // Phase `Creating` -> `Created`
     times(this.concurrency, () => this.handleCreatingPhase())