fix: crypto transpiling issue #9405 (#9443)

* feat: added ability to bundle multiple files to avoid max size exception

* chore: added bundle folder to ignore

* feat: added polyfill package for crypto

* feat: added global hack to get around polyfill issue

* refactor: remove unused split property

packages/deployment-service/.gitignore

@@ -32,3 +32,5 @@ ormconfig.json
 
 .cdk.staging
 cdk.out
+
+bundle/

packages/deployment-service/cdk/lib/cdk-stack.ts

@@ -45,6 +45,7 @@ type FunctionSetup = {
   queues?: sqs.IQueue[]
   topic?: Topic
   role?: Role
+  entrypoint: string,
 }
 
 const getNumberOfMigrations = async () => {
@@ -109,17 +110,20 @@ export const createStack = async () => {
       ],
       timeout: 900,
       RAM: 2048,
+      entrypoint: 'bundle/sqs.zip',
     },
     appEvents: {
       handler: createFileLoc('sqs', 'handle'),
       policies: [...policies.commonBackendPolicies, policies.cloudFrontPolicy, policies.route53Policy],
       queues: [queues[QueueNames.APP_EVENTS]],
+      entrypoint: 'bundle/sqs.zip',
     },
     sns: {
       handler: createFileLoc('sns', 'handle'),
       policies: [...policies.commonBackendPolicies],
       topic: codebuildSnsTopic,
       timeout: 900,
+      entrypoint: 'bundle/sns.zip',
     },
     httpApi: {
       handler: createFileLoc('http', 'handler'),
@@ -136,6 +140,7 @@ export const createStack = async () => {
         },
         headers: ['Content-Type', 'api-version', 'X-Api-Key'],
       },
+      entrypoint: 'bundle/http.zip',
     },
     http: {
       handler: createFileLoc('http', 'handler'),
@@ -153,6 +158,7 @@ export const createStack = async () => {
         headers: ['Content-Type', 'Authorization', 'api-version'],
         authorizer: true,
       },
+      entrypoint: 'bundle/http.zip',
     },
   }
 
@@ -184,9 +190,9 @@ export const createStack = async () => {
 
   for (const [name, options] of Object.entries(functionSetups)) {
     const lambda = createLambda({
+      entrypoint: options.entrypoint,
       stack,
       name,
-      entrypoint: path.resolve('bundle.zip'),
       handler: options.handler,
       env,
       vpc,
@@ -223,7 +229,7 @@ export const createStack = async () => {
   const migrationHandler = createLambda({
     stack,
     name: 'cloud-deployment-migration',
-    entrypoint: path.resolve('bundle.zip'),
+    entrypoint: 'bundle/migration-run.zip',
     handler: createFileLoc('migration-run', 'migrationRun'),
     env,
     vpc,

packages/deployment-service/package.json

@@ -11,7 +11,7 @@
     "@nestjs/passport": "^8.2.2",
     "@nestjs/platform-express": "^9.4.2",
     "@nestjs/typeorm": "^8.1.4",
-    "@octokit/app": "^12.0.7",
+    "@octokit/app": "^14.0.0",
     "@reapit/api-key-verify": "workspace:packages/api-key-verify",
     "@reapit/connect-session": "workspace:packages/connect-session",
     "@reapit/utils-nest": "workspace:packages/utils-nest",
@@ -73,6 +73,7 @@
     "@typescript-eslint/parser": "^5.59.7",
     "concurrently": "^6.5.1",
     "cross-env": "^7.0.3",
+    "esbuild-plugins-node-modules-polyfill": "^1.2.0",
     "eslint": "^8.41.0",
     "eslint-plugin-prettier": "^4.2.1",
     "jest": "^29.5.0",

packages/deployment-service/src/sqs.ts

@@ -3,6 +3,10 @@ import { AppModule } from './app-module'
 import { WorkflowHandlerProvider } from './events/workflow-handler-provider'
 import { SQSHandler } from 'aws-lambda'
 import { INestMicroservice } from '@nestjs/common'
+const crypto = require('crypto')
+
+global.crypto = crypto
+
 
 let app: INestMicroservice
 

packages/deployment-service/tsup-zip.config.json

@@ -0,0 +1,3 @@
+{
+  "bundleFiles": ["dist/http.js", "dist/sqs.js", "dist/sns.js", "dist/migration-run.js"]
+}

packages/deployment-service/tsup.config.js

@@ -1,18 +1,31 @@
 import { defineConfig } from 'tsup'
 import fs from 'fs'
 import config from './config.json'
+import { nodeModulesPolyfillPlugin } from 'esbuild-plugins-node-modules-polyfill';
 
 const pkgJson = JSON.parse(fs.readFileSync('package.json', 'utf-8'))
 
 export default defineConfig({
-  entry: ['src/http.ts', 'src/sqs.ts', 'src/sns.ts'],
+  entry: ['src/http.ts', 'src/sqs.ts', 'src/sns.ts', 'src/migration-run.ts'],
   target: 'node18',
   clean: true,
   minify: config.NODE_ENV === 'production',
   esbuildOptions: (opts) => {
     opts.resolveExtensions = ['.ts', '.mjs', '.js']
+    opts.plugins = [
+      ...opts.plugins,
+      nodeModulesPolyfillPlugin({
+        // modules: ['crypto'],
+        global: {
+          crypto: true,
+        },
+      }),
+    ]
   },
-  noExternal: Object.keys(pkgJson.dependencies),
+  noExternal: [
+    ...Object.keys(pkgJson.dependencies),
+    'crypto',
+  ],
   external: [
     '@nestjs/microservices',
     '@nestjs/websockets',
@@ -26,5 +39,6 @@ export default defineConfig({
     'mqtt',
     'amqplib',
     'redis',
+    'crypto',
   ],
 })

packages/ts-scripts/src/tsup-zip/index.ts

@@ -24,9 +24,10 @@ const findConfig = () => {
   return JSON.parse(fs.readFileSync(path.resolve('tsup-zip.config.json')).toString())
 }
 
-const bundle = (includes: string[], packageFolder: string) => {
+const bundle = (packageFolder: string) => {
   const tmpDir = fs.mkdtempSync([os.tmpdir(), `${packageFolder}-build-`].join(path.sep))
   const config = findConfig()
+  const includes = config.bundleFiles || ['dist']
 
   fs.mkdirSync(`${tmpDir}/node_modules`)
   ;(config['bundle-packages'] || []).forEach((folder) => {
@@ -37,12 +38,35 @@ const bundle = (includes: string[], packageFolder: string) => {
   fs.mkdirSync(`${tmpDir}/packages/${packageFolder}`)
 
   includes.forEach((folder) => {
+    if (folder.includes('/')) {
+      const resolveEndFolder = folder.split('/')
+      resolveEndFolder.pop()
+      if (!fs.existsSync(path.resolve(tmpDir, 'packages', packageFolder, resolveEndFolder.join('/')))) {
+        fs.mkdirSync(path.resolve(tmpDir, 'packages', packageFolder, resolveEndFolder.join('/')))
+      }
+    }
     execSync(`cp -r ${path.resolve(process.cwd(), folder)} ${tmpDir}/packages/${packageFolder}/${folder}`)
   })
 
-  execSync(`cd ${tmpDir} && zip -q -r ${path.resolve('bundle.zip')} .`)
+  if (includes.length >= 2) {
+    if (fs.existsSync(`${process.cwd()}/bundle`)) {
+      fs.rmSync(`${process.cwd()}/bundle`, {
+        force: true,
+        recursive: true,
+      })
+    }
+    fs.mkdirSync(`${process.cwd()}/bundle`)
+    includes.forEach(file => {
+      const parts = file.split('/')
+      const fullFileName = parts.pop()
+      const [fileName] = fullFileName.split('.')
+      execSync(`cd ${tmpDir} && zip -q -r ${path.resolve('bundle', fileName)} ./packages/${packageFolder}/${parts.join('/')} ./node_modules`)
+    })
+  } else {
+    execSync(`cd ${tmpDir} && zip -q -r ${path.resolve('bundle.zip')} .`)
+  }
 }
 
 const [folderPackage] = process.argv.slice(2)
 
-bundle(['dist'], folderPackage)
+bundle(folderPackage)