Skip to content

Commit

Permalink
feat: add option to run git pulls in parallel
Browse files Browse the repository at this point in the history 
update-remote commands now take a --parallel flag that allows git updates to happen in parallel. This will automatically reject any Git prompt, such as username / password

Co-authored-by: Ryan Hair <[email protected]>
  • Loading branch information
@ryanhair
ryanhair and Ryan Hair authored Sep 15, 2022
1 parent 38b5578 commit 5554a3d
Show file tree
Hide file tree
Showing 8 changed files with 199 additions and 47 deletions.
18 changes: 16 additions & 2 deletions core/src/commands/update-remote/all.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,28 @@ import { updateRemoteModules } from "./modules"
import { SourceConfig, projectSourceSchema, moduleSourceSchema } from "../../config/project"
import { printHeader } from "../../logger/util"
import { joi, joiArray } from "../../config/common"
import { BooleanParameter } from "../../cli/params"

export interface UpdateRemoteAllResult {
projectSources: SourceConfig[]
moduleSources: SourceConfig[]
}

export class UpdateRemoteAllCommand extends Command {
const updateRemoteAllOptions = {
parallel: new BooleanParameter({
help: "Allow git updates to happen in parallel",
defaultValue: false,
}),
}

type Opts = typeof updateRemoteAllOptions

export class UpdateRemoteAllCommand extends Command<{}, Opts> {
name = "all"
help = "Update all remote sources and modules."

options = updateRemoteAllOptions

outputsSchema = () =>
joi.object().keys({
projectSources: joiArray(projectSourceSchema()).description("A list of all configured external project sources."),
Expand All @@ -42,16 +54,18 @@ export class UpdateRemoteAllCommand extends Command {
printHeader(headerLog, "Update remote sources and modules", "hammer_and_wrench")
}

async action({ garden, log }: CommandParams): Promise<CommandResult<UpdateRemoteAllResult>> {
async action({ garden, log, opts }: CommandParams<{}, Opts>): Promise<CommandResult<UpdateRemoteAllResult>> {
const { result: projectSources } = await updateRemoteSources({
garden,
log,
args: { sources: undefined },
opts: { parallel: opts.parallel },
})
const { result: moduleSources } = await updateRemoteModules({
garden,
log,
args: { modules: undefined },
opts: { parallel: opts.parallel },
})

return {
Expand Down
33 changes: 26 additions & 7 deletions core/src/commands/update-remote/modules.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import { printHeader } from "../../logger/util"
import { Garden } from "../../garden"
import { LogEntry } from "../../logger/log-entry"
import { joiArray, joi } from "../../config/common"
import { StringsParameter, ParameterValues } from "../../cli/params"
import { StringsParameter, ParameterValues, BooleanParameter } from "../../cli/params"

const updateRemoteModulesArguments = {
modules: new StringsParameter({
Expand All @@ -29,14 +29,25 @@ const updateRemoteModulesArguments = {

type Args = typeof updateRemoteModulesArguments

const updateRemoteModulesOptions = {
parallel: new BooleanParameter({
help:
"Allow git updates to happen in parallel. This will automatically reject any Git prompt, such as username / password",
defaultValue: false,
}),
}

type Opts = typeof updateRemoteModulesOptions

interface Output {
sources: SourceConfig[]
}

export class UpdateRemoteModulesCommand extends Command<Args> {
export class UpdateRemoteModulesCommand extends Command<Args, Opts> {
name = "modules"
help = "Update remote modules."
arguments = updateRemoteModulesArguments
options = updateRemoteModulesOptions

outputsSchema = () =>
joi.object().keys({
Expand All @@ -57,19 +68,21 @@ export class UpdateRemoteModulesCommand extends Command<Args> {
printHeader(headerLog, "Update remote modules", "hammer_and_wrench")
}

async action({ garden, log, args }: CommandParams<Args>): Promise<CommandResult<Output>> {
return updateRemoteModules({ garden, log, args })
async action({ garden, log, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<Output>> {
return updateRemoteModules({ garden, log, args, opts })
}
}

export async function updateRemoteModules({
garden,
log,
args,
opts,
}: {
garden: Garden
log: LogEntry
args: ParameterValues<Args>
opts: ParameterValues<Opts>
}) {
const { modules: moduleNames } = args
const graph = await garden.getConfigGraph({ log, emit: false })
Expand All @@ -92,16 +105,22 @@ export async function updateRemoteModules({
})
}

// TODO Update remotes in parallel. Currently not possible since updating might
// trigger a username and password prompt from git.
const promises: Promise<void>[] = []
for (const { name, repositoryUrl } of moduleSources) {
await garden.vcs.updateRemoteSource({
const promise = garden.vcs.updateRemoteSource({
name,
url: repositoryUrl,
sourceType: "module",
log,
failOnPrompt: opts.parallel,
})
if (opts.parallel) {
promises.push(promise)
} else {
await promise
}
}
await Promise.all(promises)

await pruneRemoteSources({
gardenDirPath: garden.gardenDirPath,
Expand Down
33 changes: 26 additions & 7 deletions core/src/commands/update-remote/sources.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { printHeader } from "../../logger/util"
import { Garden } from "../../garden"
import { LogEntry } from "../../logger/log-entry"
import { joiArray, joi } from "../../config/common"
import { StringsParameter, ParameterValues } from "../../cli/params"
import { StringsParameter, ParameterValues, BooleanParameter } from "../../cli/params"

const updateRemoteSourcesArguments = {
sources: new StringsParameter({
Expand All @@ -28,14 +28,25 @@ const updateRemoteSourcesArguments = {

type Args = typeof updateRemoteSourcesArguments

const updateRemoteSourcesOptions = {
parallel: new BooleanParameter({
help:
"Allow git updates to happen in parallel. This will automatically reject any Git prompt, such as username / password",
defaultValue: false,
}),
}

type Opts = typeof updateRemoteSourcesOptions

interface Output {
sources: SourceConfig[]
}

export class UpdateRemoteSourcesCommand extends Command<Args> {
export class UpdateRemoteSourcesCommand extends Command<Args, Opts> {
name = "sources"
help = "Update remote sources."
arguments = updateRemoteSourcesArguments
opts = updateRemoteSourcesOptions

outputsSchema = () =>
joi.object().keys({
Expand All @@ -55,19 +66,21 @@ export class UpdateRemoteSourcesCommand extends Command<Args> {
printHeader(headerLog, "Update remote sources", "hammer_and_wrench")
}

async action({ garden, log, args }: CommandParams<Args>): Promise<CommandResult<Output>> {
return updateRemoteSources({ garden, log, args })
async action({ garden, log, args, opts }: CommandParams<Args, Opts>): Promise<CommandResult<Output>> {
return updateRemoteSources({ garden, log, args, opts })
}
}

export async function updateRemoteSources({
garden,
log,
args,
opts,
}: {
garden: Garden
log: LogEntry
args: ParameterValues<Args>
opts: ParameterValues<Opts>
}) {
const { sources } = args

Expand All @@ -88,16 +101,22 @@ export async function updateRemoteSources({
)
}

// TODO Update remotes in parallel. Currently not possible since updating might
// trigger a username and password prompt from git.
const promises: Promise<void>[] = []
for (const { name, repositoryUrl } of selectedSources) {
await garden.vcs.updateRemoteSource({
const promise = garden.vcs.updateRemoteSource({
name,
url: repositoryUrl,
sourceType: "project",
log,
failOnPrompt: opts.parallel,
})
if (opts.parallel) {
promises.push(promise)
} else {
await promise
}
}
await Promise.all(promises)

await pruneRemoteSources({
gardenDirPath: garden.gardenDirPath,
Expand Down
46 changes: 26 additions & 20 deletions core/src/vcs/git.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,14 @@ export class GitHandler extends VcsHandler {
this.gitSafeDirsRead = false
}

gitCli(log: LogEntry, cwd: string): GitCli {
gitCli(log: LogEntry, cwd: string, failOnPrompt = false): GitCli {
return async (...args: (string | undefined)[]) => {
log.silly(`Calling git with args '${args.join(" ")}' in ${cwd}`)
const { stdout } = await exec("git", args.filter(isString), { cwd, maxBuffer: 10 * 1024 * 1024 })
const { stdout } = await exec("git", args.filter(isString), {
cwd,
maxBuffer: 10 * 1024 * 1024,
env: failOnPrompt ? { GIT_TERMINAL_PROMPT: "0", GIT_ASKPASS: "true" } : undefined,
})
return stdout.split("\n").filter((line) => line.length > 0)
}
}
Expand Down Expand Up @@ -131,17 +135,17 @@ export class GitHandler extends VcsHandler {
* Git has stricter repository ownerships checks since 2.36.0,
* see https://github.blog/2022-04-18-highlights-from-git-2-36/ for more details.
*/
private async ensureSafeDirGitRepo(log: LogEntry, path: string): Promise<void> {
private async ensureSafeDirGitRepo(log: LogEntry, path: string, failOnPrompt = false): Promise<void> {
if (this.gitSafeDirs.has(path)) {
return
}

const git = this.gitCli(log, path)
const git = this.gitCli(log, path, failOnPrompt)

if (!this.gitSafeDirsRead) {
await gitConfigAsyncLock.acquire(".gitconfig", async () => {
if (!this.gitSafeDirsRead) {
const gitCli = this.gitCli(log, path)
const gitCli = this.gitCli(log, path, failOnPrompt)
try {
const safeDirectories = await gitCli("config", "--get-all", "safe.directory")
safeDirectories.forEach((safeDir) => this.gitSafeDirs.add(safeDir))
Expand Down Expand Up @@ -192,15 +196,15 @@ export class GitHandler extends VcsHandler {
this.gitSafeDirs.add(path)
}

async getRepoRoot(log: LogEntry, path: string) {
async getRepoRoot(log: LogEntry, path: string, failOnPrompt = false) {
if (this.repoRoots.has(path)) {
return this.repoRoots.get(path)
}

await this.ensureSafeDirGitRepo(log, STATIC_DIR)
await this.ensureSafeDirGitRepo(log, path)
await this.ensureSafeDirGitRepo(log, STATIC_DIR, failOnPrompt)
await this.ensureSafeDirGitRepo(log, path, failOnPrompt)

const git = this.gitCli(log, path)
const git = this.gitCli(log, path, failOnPrompt)

try {
const repoRoot = (await git("rev-parse", "--show-toplevel"))[0]
Expand All @@ -227,6 +231,7 @@ export class GitHandler extends VcsHandler {
include,
exclude,
filter,
failOnPrompt = false,
}: GetFilesParams): Promise<VcsFile[]> {
if (include && include.length === 0) {
// No need to proceed, nothing should be included
Expand Down Expand Up @@ -260,8 +265,8 @@ export class GitHandler extends VcsHandler {
}
}

const git = this.gitCli(log, path)
const gitRoot = await this.getRepoRoot(log, path)
const git = this.gitCli(log, path, failOnPrompt)
const gitRoot = await this.getRepoRoot(log, path, failOnPrompt)

// List modified files, so that we can ensure we have the right hash for them later
const modified = new Set(
Expand Down Expand Up @@ -568,15 +573,16 @@ export class GitHandler extends VcsHandler {
remoteSourcesPath: string,
repositoryUrl: string,
hash: string,
absPath: string
absPath: string,
failOnPrompt = false
) {
const git = this.gitCli(log, remoteSourcesPath)
const git = this.gitCli(log, remoteSourcesPath, failOnPrompt)
// Use `--recursive` to include submodules
return git("clone", "--recursive", "--depth=1", `--branch=${hash}`, repositoryUrl, absPath)
}

// TODO Better auth handling
async ensureRemoteSource({ url, name, log, sourceType }: RemoteSourceParams): Promise<string> {
async ensureRemoteSource({ url, name, log, sourceType, failOnPrompt = false }: RemoteSourceParams): Promise<string> {
const remoteSourcesPath = join(this.gardenDirPath, this.getRemoteSourcesDirname(sourceType))
await ensureDir(remoteSourcesPath)

Expand All @@ -588,7 +594,7 @@ export class GitHandler extends VcsHandler {
const { repositoryUrl, hash } = parseGitUrl(url)

try {
await this.cloneRemoteSource(log, remoteSourcesPath, repositoryUrl, hash, absPath)
await this.cloneRemoteSource(log, remoteSourcesPath, repositoryUrl, hash, absPath, failOnPrompt)
} catch (err) {
entry.setError()
throw new RuntimeError(`Downloading remote ${sourceType} failed with error: \n\n${err}`, {
Expand All @@ -603,12 +609,12 @@ export class GitHandler extends VcsHandler {
return absPath
}

async updateRemoteSource({ url, name, sourceType, log }: RemoteSourceParams) {
async updateRemoteSource({ url, name, sourceType, log, failOnPrompt = false }: RemoteSourceParams) {
const absPath = join(this.gardenDirPath, this.getRemoteSourceRelPath(name, url, sourceType))
const git = this.gitCli(log, absPath)
const git = this.gitCli(log, absPath, failOnPrompt)
const { repositoryUrl, hash } = parseGitUrl(url)

await this.ensureRemoteSource({ url, name, sourceType, log })
await this.ensureRemoteSource({ url, name, sourceType, log, failOnPrompt })

const entry = log.info({ section: name, msg: "Getting remote state", status: "active" })
await git("remote", "update")
Expand Down Expand Up @@ -703,8 +709,8 @@ export class GitHandler extends VcsHandler {
return submodules
}

async getPathInfo(log: LogEntry, path: string): Promise<VcsInfo> {
const git = this.gitCli(log, path)
async getPathInfo(log: LogEntry, path: string, failOnPrompt = false): Promise<VcsInfo> {
const git = this.gitCli(log, path, failOnPrompt)

const output: VcsInfo = {
branch: "",
Expand Down
2 changes: 2 additions & 0 deletions core/src/vcs/vcs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,13 +92,15 @@ export interface GetFilesParams {
include?: string[]
exclude?: string[]
filter?: (path: string) => boolean
failOnPrompt?: boolean
}

export interface RemoteSourceParams {
url: string
name: string
sourceType: ExternalSourceType
log: LogEntry
failOnPrompt?: boolean
}

export interface VcsFile {
Expand Down
Loading

0 comments on commit 5554a3d

Please sign in to comment.