Skip to content

Commit

Permalink
perf: various performance improvements (#4959)
Browse files Browse the repository at this point in the history 
* perf: speed up `splitLast` function

* perf: skip function wrapping if profiler is disabled

* perf: index files into a tree structure to get files for a subdirectory more quickly

* perf: increase max-semi-space-size for less GC interruptions at the cost of some more memory

See https://github.com/nodejs/node/blob/main/doc/api/cli.md#useful-v8-options
Also see https://www.alibabacloud.com/blog/better-node-application-performance-through-gc-optimization_595119 and nodejs/node#42511 for some details on impact

* refactor: replace Bluebird with native Promises (part 1)

* refactor: replace bluebird with native promises (part 2)

* refactor: replace bluebird with native promises (part 3)

* chore: make the linter happy after all the bluebird replacements

* fix: splitLast behavior with no index found

* chore: linter again

* chore: cleanup file-tree and add header

* refactor: replace pLimit with pMap
  • Loading branch information
@TimBeyer
TimBeyer authored Aug 22, 2023
1 parent 7f67071 commit a2c5f6e
Show file tree
Hide file tree
Showing 80 changed files with 1,887 additions and 1,623 deletions.
2 changes: 1 addition & 1 deletion bin/garden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

repo_root=$(cd `dirname $0` && cd .. && pwd)

node --max-old-space-size=4096 ${repo_root}/cli/bin/garden "$@"
node --max-old-space-size=4096 --max-semi-space-size=64 ${repo_root}/cli/bin/garden "$@"
2 changes: 1 addition & 1 deletion cli/bin/garden-debug
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/usr/bin/env node --inspect --stack-trace-limit=1000
#!/usr/bin/env node --inspect --stack-trace-limit=1000 --max-semi-space-size=64

require("./garden")
33 changes: 17 additions & 16 deletions cli/src/add-version-files.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import { GitHandler } from "@garden-io/core/build/src/vcs/git"
import { Garden } from "@garden-io/core/build/src/garden"
import { LogLevel, RootLogger } from "@garden-io/core/build/src/logger/logger"
import { resolve, relative } from "path"
import Bluebird from "bluebird"
import { STATIC_DIR, GARDEN_VERSIONFILE_NAME } from "@garden-io/core/build/src/constants"
import { writeTreeVersionFile } from "@garden-io/core/build/src/vcs/vcs"
import { TreeCache } from "@garden-io/core/build/src/cache"
Expand All @@ -28,24 +27,26 @@ async function addVersionFiles() {

const moduleConfigs = await garden.getRawModuleConfigs()

return Bluebird.map(moduleConfigs, async (config) => {
const path = config.path
const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)
return Promise.all(
moduleConfigs.map(async (config) => {
const path = config.path
const versionFilePath = resolve(path, GARDEN_VERSIONFILE_NAME)

const vcsHandler = new GitHandler({
garden,
projectRoot: STATIC_DIR,
gardenDirPath: garden.gardenDirPath,
ignoreFile: garden.dotIgnoreFile,
cache: new TreeCache(),
})
const treeVersion = await vcsHandler.getTreeVersion({ log: garden.log, projectName: garden.projectName, config })
const vcsHandler = new GitHandler({
garden,
projectRoot: STATIC_DIR,
gardenDirPath: garden.gardenDirPath,
ignoreFile: garden.dotIgnoreFile,
cache: new TreeCache(),
})
const treeVersion = await vcsHandler.getTreeVersion({ log: garden.log, projectName: garden.projectName, config })

// eslint-disable-next-line no-console
console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)
// eslint-disable-next-line no-console
console.log(`${config.name} -> ${relative(STATIC_DIR, versionFilePath)}`)

return writeTreeVersionFile(path, treeVersion)
})
return writeTreeVersionFile(path, treeVersion)
})
)
}

if (require.main === module) {
Expand Down
97 changes: 51 additions & 46 deletions cli/src/build-pkg.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import chalk from "chalk"
import { getAbi } from "node-abi"
import { resolve, relative, join } from "path"
import Bluebird from "bluebird"
import { STATIC_DIR, GARDEN_CLI_ROOT, GARDEN_CORE_ROOT } from "@garden-io/core/build/src/constants"
import { remove, mkdirp, copy, writeFile } from "fs-extra"
import { exec, getPackageVersion, sleep } from "@garden-io/core/build/src/util/util"
Expand All @@ -29,7 +28,7 @@ const pkgPath = resolve(repoRoot, "cli", "node_modules", ".bin", "pkg")
const distPath = resolve(repoRoot, "dist")

// Allow larger heap size than default
const nodeOptions = ["max-old-space-size=4096"]
const nodeOptions = ["max-old-space-size=4096", "max-semi-space-size=64"]

/* eslint-disable no-console */

Expand Down Expand Up @@ -95,48 +94,52 @@ async function buildBinaries(args: string[]) {
const workspaces = JSON.parse(JSON.parse(res).data)

console.log(chalk.cyan("Copying packages"))
await Bluebird.map(Object.entries(workspaces), async ([name, info]: [string, any]) => {
const sourcePath = resolve(repoRoot, info.location)
const targetPath = resolve(tmpDirPath, info.location)
await remove(targetPath)
await mkdirp(targetPath)
await exec("rsync", [
"-r",
"-L",
"--exclude=node_modules",
"--exclude=tmp",
"--exclude=test",
sourcePath,
resolve(targetPath, ".."),
])

console.log(chalk.green(" ✓ " + name))
})
await Promise.all(
Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
const sourcePath = resolve(repoRoot, info.location)
const targetPath = resolve(tmpDirPath, info.location)
await remove(targetPath)
await mkdirp(targetPath)
await exec("rsync", [
"-r",
"-L",
"--exclude=node_modules",
"--exclude=tmp",
"--exclude=test",
sourcePath,
resolve(targetPath, ".."),
])

console.log(chalk.green(" ✓ " + name))
})
)

// Edit all the packages to have them directly link any internal dependencies
console.log(chalk.cyan("Modifying package.json files for direct installation"))
await Bluebird.map(Object.entries(workspaces), async ([name, info]: [string, any]) => {
const packageRoot = resolve(tmpDirPath, info.location)
const packageJsonPath = resolve(packageRoot, "package.json")
const packageJson = require(packageJsonPath)

for (const depName of info.workspaceDependencies) {
const depInfo = workspaces[depName]
const targetRoot = resolve(tmpDirPath, depInfo.location)
const relPath = relative(packageRoot, targetRoot)
packageJson.dependencies[depName] = "file:" + relPath
}

if (version === "edge" || version.startsWith("edge-")) {
const gitHash = await exec("git", ["rev-parse", "--short", "HEAD"])
packageJson.version = `${packageJson.version}-${version}-${gitHash.stdout}`
console.log("Set package version to " + packageJson.version)
}

await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))

console.log(chalk.green(" ✓ " + name))
})
await Promise.all(
Object.entries(workspaces).map(async ([name, info]: [string, any]) => {
const packageRoot = resolve(tmpDirPath, info.location)
const packageJsonPath = resolve(packageRoot, "package.json")
const packageJson = require(packageJsonPath)

for (const depName of info.workspaceDependencies) {
const depInfo = workspaces[depName]
const targetRoot = resolve(tmpDirPath, depInfo.location)
const relPath = relative(packageRoot, targetRoot)
packageJson.dependencies[depName] = "file:" + relPath
}

if (version === "edge" || version.startsWith("edge-")) {
const gitHash = await exec("git", ["rev-parse", "--short", "HEAD"])
packageJson.version = `${packageJson.version}-${version}-${gitHash.stdout}`
console.log("Set package version to " + packageJson.version)
}

await writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2))

console.log(chalk.green(" ✓ " + name))
})
)

// Run yarn install in the cli package
console.log(chalk.cyan("Installing packages in @garden-io/cli package"))
Expand All @@ -146,11 +149,13 @@ async function buildBinaries(args: string[]) {
// Run pkg and pack up each platform binary
console.log(chalk.cyan("Packaging garden binaries"))

await Bluebird.map(Object.entries(selected), async ([targetName, spec]) => {
await spec.handler({ targetName, sourcePath: cliPath, pkgType: spec.pkgType, version })
await sleep(5000) // Work around concurrency bug in pkg...
console.log(chalk.green(" ✓ " + targetName))
})
await Promise.all(
Object.entries(selected).map(async ([targetName, spec]) => {
await spec.handler({ targetName, sourcePath: cliPath, pkgType: spec.pkgType, version })
await sleep(5000) // Work around concurrency bug in pkg...
console.log(chalk.green(" ✓ " + targetName))
})
)

console.log(chalk.green.bold("Done!"))
}
Expand Down
8 changes: 5 additions & 3 deletions core/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@
"async": "^3.2.4",
"@scg82/exit-hook": "^3.4.1",
"async-lock": "^1.4.0",
"bluebird": "^3.7.2",
"certpem": "^1.1.3",
"chalk": "^4.1.2",
"chokidar": "^3.5.3",
Expand Down Expand Up @@ -113,8 +112,12 @@
"normalize-path": "^3.0.0",
"normalize-url": "^5.3.1",
"open": "^8.4.2",
"p-filter": "^2.1.0",
"p-queue": "^6.0.0",
"p-retry": "^4.6.2",
"p-limit": "^3.1.0",
"p-map": "^4.0.0",
"p-props": "^4.0.0",
"parse-duration": "^1.0.2",
"parse-git-config": "^3.0.0",
"path-is-inside": "^1.0.2",
Expand Down Expand Up @@ -166,7 +169,6 @@
"@types/analytics-node": "^3.1.10",
"@types/async": "^3.2.18",
"@types/async-lock": "^1.4.0",
"@types/bluebird": "^3.5.38",
"@types/chai": "^4.3.4",
"@types/circular-json": "^0.4.0",
"@types/cross-spawn": "^6.0.2",
Expand Down Expand Up @@ -278,4 +280,4 @@
]
},
"gitHead": "b0647221a4d2ff06952bae58000b104215aed922"
}
}
121 changes: 64 additions & 57 deletions core/src/actions/helpers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/

import Bluebird from "bluebird"
import chalk from "chalk"
import { fromPairs, memoize } from "lodash"
import { joi } from "../config/common"
Expand Down Expand Up @@ -158,22 +157,24 @@ export async function getDeployStatusPayloads({
const actions = graph.getDeploys()

return fromPairs(
await Bluebird.map(actions, async (action) => {
const startedAt = new Date().toISOString()
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const { result } = await router.deploy.getStatus({ action, log: actionLog, graph })
await Promise.all(
actions.map(async (action) => {
const startedAt = new Date().toISOString()
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const { result } = await router.deploy.getStatus({ action, log: actionLog, graph })

const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<DeployStatusForEventPayload>
const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<DeployStatusForEventPayload>

return [action.name, payload]
})
return [action.name, payload]
})
)
)
}

Expand All @@ -191,22 +192,24 @@ export async function getBuildStatusPayloads({
const actions = graph.getBuilds()

return fromPairs(
await Bluebird.map(actions, async (action) => {
const startedAt = new Date().toISOString()
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const { result } = await router.build.getStatus({ action, log: actionLog, graph })
await Promise.all(
actions.map(async (action) => {
const startedAt = new Date().toISOString()
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const { result } = await router.build.getStatus({ action, log: actionLog, graph })

const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<BuildStatusForEventPayload>
const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<BuildStatusForEventPayload>

return [action.name, payload]
})
return [action.name, payload]
})
)
)
}

Expand All @@ -224,20 +227,22 @@ export async function getTestStatusPayloads({
const actions = graph.getTests()

return fromPairs(
await Bluebird.map(actions, async (action) => {
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const startedAt = new Date().toISOString()
const { result } = await router.test.getResult({ action, log: actionLog, graph })
const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<RunStatusForEventPayload>
return [action.name, payload]
})
await Promise.all(
actions.map(async (action) => {
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const startedAt = new Date().toISOString()
const { result } = await router.test.getResult({ action, log: actionLog, graph })
const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<RunStatusForEventPayload>
return [action.name, payload]
})
)
)
}

Expand All @@ -255,21 +260,23 @@ export async function getRunStatusPayloads({
const actions = graph.getRuns()

return fromPairs(
await Bluebird.map(actions, async (action) => {
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const startedAt = new Date().toISOString()
const { result } = await router.run.getResult({ action, log: actionLog, graph })
await Promise.all(
actions.map(async (action) => {
const actionLog = createActionLog({ log, actionName: action.name, actionKind: action.kind })
const startedAt = new Date().toISOString()
const { result } = await router.run.getResult({ action, log: actionLog, graph })

const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<RunStatusForEventPayload>
const payload = makeActionCompletePayload({
result,
operation: "getStatus",
startedAt,
force: false,
action,
sessionId,
}) as ActionStatusPayload<RunStatusForEventPayload>

return [action.name, payload]
})
return [action.name, payload]
})
)
)
}
Loading

0 comments on commit a2c5f6e

Please sign in to comment.