feat: Add realtime update feature

.github/workflows/validate-api.yaml

@@ -51,7 +51,7 @@ jobs:
       - name: Unit tests
         run: |
           pnpm run db:generate-types
-          pnpm run test:api
+          pnpm run unit:api
 
       - name: E2E tests
         env:

apps/api/jest.e2e-config.ts

@@ -1,5 +1,6 @@
 /* eslint-disable */
 export default {
+  forceExit: true,
   displayName: 'api',
   preset: '../../jest.preset.js',
   testEnvironment: 'node',

apps/api/package.json

@@ -20,9 +20,12 @@
     "@nestjs/passport": "^10.0.3",
     "@nestjs/platform-express": "^10.0.0",
     "@nestjs/platform-fastify": "^10.3.3",
+    "@nestjs/platform-socket.io": "^10.3.7",
     "@nestjs/schedule": "^4.0.1",
     "@nestjs/swagger": "^7.3.0",
+    "@nestjs/websockets": "^10.3.7",
     "@prisma/client": "^5.10.1",
+    "@socket.io/redis-adapter": "^8.3.0",
     "@supabase/supabase-js": "^2.39.6",
     "@types/uuid": "^9.0.8",
     "chalk": "^4.1.2",
@@ -34,8 +37,10 @@
     "passport-github2": "^0.1.12",
     "passport-google-oauth20": "^2.0.0",
     "prisma": "^5.10.1",
+    "redis": "^4.6.13",
     "reflect-metadata": "^0.2.0",
     "rxjs": "^7.8.1",
+    "socket.io": "^4.7.5",
     "uuid": "^9.0.1"
   },
   "devDependencies": {

apps/api/src/app/app.module.ts

@@ -18,13 +18,17 @@ import { ApiKeyGuard } from '../auth/guard/api-key/api-key.guard'
 import { EventModule } from '../event/event.module'
 import { VariableModule } from '../variable/variable.module'
 import { ApprovalModule } from '../approval/approval.module'
+import { SocketModule } from '../socket/socket.module'
+import { ProviderModule } from '../provider/provider.module'
+import { ScheduleModule } from '@nestjs/schedule'
 
 @Module({
   controllers: [AppController],
   imports: [
     ConfigModule.forRoot({
       isGlobal: true
     }),
+    ScheduleModule.forRoot(),
     PassportModule,
     AuthModule,
     PrismaModule,
@@ -38,7 +42,9 @@ import { ApprovalModule } from '../approval/approval.module'
     WorkspaceRoleModule,
     EventModule,
     VariableModule,
-    ApprovalModule
+    ApprovalModule,
+    SocketModule,
+    ProviderModule
   ],
   providers: [
     {

apps/api/src/approval/controller/approval.controller.spec.ts

@@ -10,12 +10,17 @@ import { ApprovalService } from '../service/approval.service'
 import { MAIL_SERVICE } from '../../mail/services/interface.service'
 import { MockMailService } from '../../mail/services/mock.service'
 import { JwtService } from '@nestjs/jwt'
+import { REDIS_CLIENT } from '../../provider/redis.provider'
+import { RedisClientType } from 'redis'
+import { mockDeep } from 'jest-mock-extended'
+import { ProviderModule } from '../../provider/provider.module'
 
 describe('ApprovalController', () => {
   let controller: ApprovalController
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
+      imports: [ProviderModule],
       controllers: [ApprovalController],
       providers: [
         ApprovalService,
@@ -31,7 +36,10 @@ describe('ApprovalController', () => {
           useClass: MockMailService
         }
       ]
-    }).compile()
+    })
+      .overrideProvider(REDIS_CLIENT)
+      .useValue(mockDeep<RedisClientType>())
+      .compile()
 
     controller = module.get<ApprovalController>(ApprovalController)
   })

apps/api/src/approval/service/approval.service.spec.ts

@@ -9,12 +9,17 @@ import { SecretService } from '../../secret/service/secret.service'
 import { MAIL_SERVICE } from '../../mail/services/interface.service'
 import { MockMailService } from '../../mail/services/mock.service'
 import { JwtService } from '@nestjs/jwt'
+import { REDIS_CLIENT } from '../../provider/redis.provider'
+import { RedisClientType } from 'redis'
+import { mockDeep } from 'jest-mock-extended'
+import { ProviderModule } from '../../provider/provider.module'
 
 describe('ApprovalService', () => {
   let service: ApprovalService
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
+      imports: [ProviderModule],
       providers: [
         ApprovalService,
         PrismaService,
@@ -29,7 +34,10 @@ describe('ApprovalService', () => {
           useClass: MockMailService
         }
       ]
-    }).compile()
+    })
+      .overrideProvider(REDIS_CLIENT)
+      .useValue(mockDeep<RedisClientType>())
+      .compile()
 
     service = module.get<ApprovalService>(ApprovalService)
   })

apps/api/src/auth/guard/auth/auth.guard.ts

@@ -6,7 +6,6 @@ import {
   UnauthorizedException
 } from '@nestjs/common'
 import { JwtService } from '@nestjs/jwt'
-import { Request } from 'express'
 import { Reflector } from '@nestjs/core'
 import { IS_PUBLIC_KEY } from '../../../decorators/public.decorator'
 import { PrismaService } from '../../../prisma/prisma.service'
@@ -42,6 +41,10 @@ export class AuthGuard implements CanActivate {
     const request = context.switchToHttp().getRequest()
     const authType = this.getAuthType(request)
 
+    if (process.env.NODE_ENV !== 'e2e' && authType === 'NONE') {
+      throw new ForbiddenException('No authentication provided')
+    }
+
     // In case the environment is e2e, we want to authenticate the user using the email
     // else we want to authenticate the user using the JWT token.
     if (authType !== 'API_KEY' && process.env.NODE_ENV === 'e2e') {
@@ -129,25 +132,28 @@ export class AuthGuard implements CanActivate {
     return true
   }
 
-  private getAuthType(request: Request): 'JWT' | 'API_KEY' | 'NONE' {
-    if (request.headers[X_KEYSHADE_TOKEN]) {
+  private getAuthType(request: any): 'JWT' | 'API_KEY' | 'NONE' {
+    const headers = request.headers || request.handshake.headers // For websockets
+    if (headers[X_KEYSHADE_TOKEN]) {
       return 'API_KEY'
     }
-    if (request.headers[AUTHORIZATION]) {
+    if (headers[AUTHORIZATION]) {
       return 'JWT'
     }
     return 'NONE'
   }
 
-  private extractTokenFromHeader(request: Request): string | undefined {
-    const [type, token] = request.headers.authorization?.split(' ') ?? []
+  private extractTokenFromHeader(request: any): string | undefined {
+    const headers = request.headers || request.handshake.headers // For websockets
+    const [type, token] = headers.authorization?.split(' ') ?? []
     return type === 'Bearer' ? token : undefined
   }
 
-  private extractApiKeyFromHeader(request: Request): string | undefined {
-    if (Array.isArray(request.headers[X_KEYSHADE_TOKEN])) {
+  private extractApiKeyFromHeader(request: any): string | undefined {
+    const headers = request.headers || request.handshake.headers // For websockets
+    if (Array.isArray(headers[X_KEYSHADE_TOKEN])) {
       throw new Error('Bad auth')
     }
-    return request.headers[X_KEYSHADE_TOKEN]
+    return headers[X_KEYSHADE_TOKEN]
   }
 }

apps/api/src/common/environment.ts

@@ -0,0 +1,3 @@
+export const Environment = {
+  DATABASE_URL: process.env.DATABASE_URL!
+}

apps/api/src/main.ts

@@ -13,6 +13,7 @@ import { QueryTransformPipe } from './common/query.transform.pipe'
 import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'
 import * as Sentry from '@sentry/node'
 import { ProfilingIntegration } from '@sentry/profiling-node'
+import { RedisIoAdapter } from './socket/redis.adapter'
 
 export const sentryEnv = process.env.SENTRY_ENV || 'production'
 
@@ -78,11 +79,16 @@ async function initializeSentry() {
 async function initializeNestApp() {
   const logger = new CustomLogger()
   const app = await NestFactory.create(AppModule, {
-    logger
+    logger,
+    cors: false
   })
   app.use(Sentry.Handlers.requestHandler())
   app.use(Sentry.Handlers.tracingHandler())
 
+  const redisIpoAdapter = new RedisIoAdapter(app)
+  await redisIpoAdapter.connectToRedis()
+  app.useWebSocketAdapter(redisIpoAdapter)
+
   const globalPrefix = 'api'
   app.setGlobalPrefix(globalPrefix)
   app.useGlobalPipes(

apps/api/src/prisma/migrations/20240410130019_add_change_notification_socket_map/migration.sql

@@ -0,0 +1,11 @@
+-- CreateTable
+CREATE TABLE "ChangeNotificationSocketMap" (
+    "id" TEXT NOT NULL,
+    "socketId" TEXT NOT NULL,
+    "environmentId" TEXT NOT NULL,
+
+    CONSTRAINT "ChangeNotificationSocketMap_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "ChangeNotificationSocketMap_environmentId_socketId_idx" ON "ChangeNotificationSocketMap"("environmentId", "socketId");

apps/api/src/prisma/schema.prisma

@@ -455,3 +455,11 @@ model Approval {
 
   @@index([itemType, itemId])
 }
+
+model ChangeNotificationSocketMap {
+  id            String @id @default(cuid())
+  socketId      String
+  environmentId String
+  
+  @@index([environmentId, socketId])
+}

apps/api/src/provider/provider.module.ts

@@ -0,0 +1,14 @@
+import { Global, Module } from '@nestjs/common'
+import { REDIS_CLIENT, RedisProvider } from './redis.provider'
+
+@Global()
+@Module({
+  exports: [
+    {
+      provide: REDIS_CLIENT,
+      useValue: RedisProvider
+    }
+  ],
+  providers: [RedisProvider]
+})
+export class ProviderModule {}

apps/api/src/provider/redis.provider.ts

@@ -0,0 +1,40 @@
+import { Logger, Provider } from '@nestjs/common'
+import { exit } from 'process'
+import * as redis from 'redis'
+
+export const REDIS_CLIENT = 'RedisClient'
+
+export const RedisProvider: Provider = {
+  provide: REDIS_CLIENT,
+  useFactory: async () => {
+    const logger = new Logger('RedisProvider')
+    if (!process.env.REDIS_URL) {
+      logger.error('Redis credentials are not set. Stopping the application.')
+      exit(1)
+    }
+
+    const subscriber = redis.createClient({
+      url: process.env.REDIS_URL,
+      password: process.env.REDIS_PASSWORD
+    })
+    const publisher = redis.createClient({
+      url: process.env.REDIS_URL,
+      password: process.env.REDIS_PASSWORD
+    })
+
+    publisher.on('error', (error) => {
+      logger.error('Redis client error:', error)
+    })
+    await publisher.connect()
+
+    subscriber.on('error', (error) => {
+      logger.error('Redis client error:', error)
+    })
+    await subscriber.connect()
+
+    return {
+      subscriber,
+      publisher
+    }
+  }
+}

apps/api/src/secret/controller/secret.controller.spec.ts

@@ -5,12 +5,16 @@ import { MockMailService } from '../../mail/services/mock.service'
 import { SecretService } from '../service/secret.service'
 import { PrismaService } from '../../prisma/prisma.service'
 import { mockDeep } from 'jest-mock-extended'
+import { REDIS_CLIENT } from '../../provider/redis.provider'
+import { RedisClientType } from 'redis'
+import { ProviderModule } from '../../provider/provider.module'
 
 describe('SecretController', () => {
   let controller: SecretController
 
   beforeEach(async () => {
     const module: TestingModule = await Test.createTestingModule({
+      imports: [ProviderModule],
       controllers: [SecretController],
       providers: [
         PrismaService,
@@ -21,6 +25,8 @@ describe('SecretController', () => {
         SecretService
       ]
     })
+      .overrideProvider(REDIS_CLIENT)
+      .useValue(mockDeep<RedisClientType>())
       .overrideProvider(PrismaService)
       .useValue(mockDeep<PrismaService>())
       .compile()

apps/api/src/secret/secret.e2e.spec.ts

@@ -32,6 +32,9 @@ import { v4 } from 'uuid'
 import fetchEvents from '../common/fetch-events'
 import { SecretService } from './service/secret.service'
 import { EventService } from '../event/service/event.service'
+import { REDIS_CLIENT } from '../provider/redis.provider'
+import { RedisClientType } from 'redis'
+import { mockDeep } from 'jest-mock-extended'
 
 describe('Secret Controller Tests', () => {
   let app: NestFastifyApplication
@@ -63,6 +66,8 @@ describe('Secret Controller Tests', () => {
     })
       .overrideProvider(MAIL_SERVICE)
       .useClass(MockMailService)
+      .overrideProvider(REDIS_CLIENT)
+      .useValue(mockDeep<RedisClientType>())
       .compile()
 
     app = moduleRef.createNestApplication<NestFastifyApplication>(