Yarn- Enhancements to Artifactory Integration and auto-installation f…

…or yarn projects (jfrog#1015)

artifactory/commands/utils/npmcmdutils.go

@@ -165,7 +165,16 @@ func createRestoreFileFunc(filePath, backupFileName string) func() error {
 		backupPath := filepath.Join(filepath.Dir(filePath), backupFileName)
 		if _, err := os.Stat(backupPath); err != nil {
 			if os.IsNotExist(err) {
-				err = os.Remove(filePath)
+				// We verify the existence of the file in the specified filePath before initiating its deletion in order to prevent errors that might occur when attempting to remove a non-existent file
+				var fileExists bool
+				fileExists, err = fileutils.IsFileExists(filePath, false)
+				if err != nil {
+					err = fmt.Errorf("failed to check for the existence of '%s' before deleting the file: %s", filePath, err.Error())
+					return errorutils.CheckError(err)
+				}
+				if fileExists {
+					err = os.Remove(filePath)
+				}
 				return errorutils.CheckError(err)
 			}
 			return errorutils.CheckErrorf(createRestoreErrorPrefix(filePath, backupPath) + err.Error())

artifactory/commands/yarn/yarn.go

@@ -25,6 +25,7 @@ const (
 	YarnrcFileName       = ".yarnrc.yml"
 	YarnrcBackupFileName = "jfrog.yarnrc.backup"
 	NpmScopesConfigName  = "npmScopes"
+	YarnLockFileName     = "yarn.lock"
 	//#nosec G101
 	yarnNpmRegistryServerEnv = "YARN_NPM_REGISTRY_SERVER"
 	yarnNpmAuthIndent        = "YARN_NPM_AUTH_IDENT"

go.mod

@@ -101,4 +101,4 @@ require (
 
 // replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go dev
 
-// replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go dev
+replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go v1.8.9-0.20231119150101-5cfbe8fca39e

go.sum

@@ -196,8 +196,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jedib0t/go-pretty/v6 v6.4.8 h1:HiNzyMSEpsBaduKhmK+CwcpulEeBrTmxutz4oX/oWkg=
 github.com/jedib0t/go-pretty/v6 v6.4.8/go.mod h1:Ndk3ase2CkQbXLLNf5QDHoYb6J9WtVfmHZu9n8rk2xs=
-github.com/jfrog/build-info-go v1.9.15 h1:DN7DKZq6H5FlHfL3Lu8fo4t2INgczRgT09dJiZjJ1oo=
-github.com/jfrog/build-info-go v1.9.15/go.mod h1:XVFk2rCYhIdc7+hIGE8TC3le5PPM+xYHU22udoE2b7Q=
+github.com/jfrog/build-info-go v1.8.9-0.20231119150101-5cfbe8fca39e h1:yhy4z08QtckwUfVs0W931wjYUif/Gfv46QazrgHqQrE=
+github.com/jfrog/build-info-go v1.8.9-0.20231119150101-5cfbe8fca39e/go.mod h1:XVFk2rCYhIdc7+hIGE8TC3le5PPM+xYHU22udoE2b7Q=
 github.com/jfrog/gofrog v1.3.1 h1:QqAwQXCVReT724uga1AYqG/ZyrNQ6f+iTxmzkb+YFQk=
 github.com/jfrog/gofrog v1.3.1/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=

xray/commands/audit/sca/yarn/yarn.go

@@ -1,16 +1,40 @@
 package yarn
 
 import (
+	"errors"
+	"fmt"
+	"github.com/jfrog/build-info-go/build"
 	biUtils "github.com/jfrog/build-info-go/build/utils"
+	"github.com/jfrog/gofrog/version"
+	rtutils "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/utils"
+	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/yarn"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/commands/audit/sca"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/utils"
 	"github.com/jfrog/jfrog-client-go/utils/errorutils"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	"github.com/jfrog/jfrog-client-go/utils/log"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
+	"path/filepath"
 )
 
-func BuildDependencyTree() (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) {
+const (
+	// Do not execute any scripts defined in the project package.json and its dependencies.
+	v1IgnoreScriptsFlag = "--ignore-scripts"
+	// Run yarn install without printing installation log.
+	v1SilentFlag = "--silent"
+	// Disable interactive prompts, like when there’s an invalid version of a dependency.
+	v1NonInteractiveFlag = "--non-interactive"
+	// Skips linking and fetch only packages that are missing from yarn.lock file
+	v2UpdateLockfileFlag = "--mode=update-lockfile"
+	// Ignores any build scripts
+	v2SkipBuildFlag     = "--mode=skip-build"
+	yarnV2Version       = "2.0.0"
+	nodeModulesRepoName = "node_modules"
+)
+
+func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps []string, err error) {
 	currentDir, err := coreutils.GetWorkingDirectory()
 	if err != nil {
 		return
@@ -24,6 +48,21 @@ func BuildDependencyTree() (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps [
 	if errorutils.CheckError(err) != nil {
 		return
 	}
+
+	projectInstalled, err := isYarnProjectInstalled(currentDir)
+	if err != nil {
+		return
+	}
+
+	if !projectInstalled || len(params.InstallCommandArgs()) != 0 {
+		// In case project is not "installed" or in case the user has provided an 'install' command to run
+		err = configureYarnResolutionServerAndRunInstall(params, currentDir, executablePath)
+		if err != nil {
+			err = fmt.Errorf("failed to configure an Artifactory resolution server or running and install command: %s", err.Error())
+			return
+		}
+	}
+
 	// Calculate Yarn dependencies
 	dependenciesMap, root, err := biUtils.GetYarnDependencies(executablePath, currentDir, packageInfo, log.Logger)
 	if err != nil {
@@ -35,6 +74,103 @@ func BuildDependencyTree() (dependencyTrees []*xrayUtils.GraphNode, uniqueDeps [
 	return
 }
 
+// Sets up Artifactory server configurations for dependency resolution, if such were provided by the user.
+// Executes the user's 'install' command or a default 'install' command if none was specified.
+func configureYarnResolutionServerAndRunInstall(params utils.AuditParams, curWd, yarnExecPath string) (err error) {
+	depsRepo := params.DepsRepo()
+	if depsRepo == "" {
+		// Run install without configuring an Artifactory server
+		return runYarnInstallAccordingToVersion(curWd, yarnExecPath, params.InstallCommandArgs())
+	}
+
+	var serverDetails *config.ServerDetails
+	serverDetails, err = params.ServerDetails()
+	if err != nil {
+		err = fmt.Errorf("failed to get server details while building yarn dependency tree: %s", err.Error())
+		return
+	}
+
+	// If an Artifactory resolution repository was provided we first configure to resolve from it and only then run the 'install' command
+	restoreYarnrcFunc, err := rtutils.BackupFile(filepath.Join(curWd, yarn.YarnrcFileName), yarn.YarnrcBackupFileName)
+	if err != nil {
+		return
+	}
+
+	registry, repoAuthIdent, err := yarn.GetYarnAuthDetails(serverDetails, depsRepo)
+	if err != nil {
+		err = errors.Join(err, restoreYarnrcFunc())
+		return
+	}
+
+	backupEnvMap, err := yarn.ModifyYarnConfigurations(yarnExecPath, registry, repoAuthIdent)
+	if err != nil {
+		if len(backupEnvMap) > 0 {
+			err = errors.Join(err, yarn.RestoreConfigurationsFromBackup(backupEnvMap, restoreYarnrcFunc))
+		} else {
+			err = errors.Join(err, restoreYarnrcFunc())
+		}
+		return
+	}
+	defer func() {
+		err = errors.Join(err, yarn.RestoreConfigurationsFromBackup(backupEnvMap, restoreYarnrcFunc))
+	}()
+
+	return runYarnInstallAccordingToVersion(curWd, yarnExecPath, params.InstallCommandArgs())
+}
+
+// Verifies the project's installation status by examining the presence of the yarn.lock file.
+// Notice!: If alterations are made manually in the package.json file, it necessitates a manual update to the yarn.lock file as well.
+func isYarnProjectInstalled(currentDir string) (projectInstalled bool, err error) {
+	yarnLockExits, err := fileutils.IsFileExists(filepath.Join(currentDir, yarn.YarnLockFileName), false)
+	if err != nil {
+		err = fmt.Errorf("failed to check the existence of '%s' file: %s", filepath.Join(currentDir, yarn.YarnLockFileName), err.Error())
+		return
+	}
+	projectInstalled = yarnLockExits
+	return
+}
+
+// Executes the user-defined 'install' command; if absent, defaults to running an 'install' command with specific flags suited to the current yarn version.
+func runYarnInstallAccordingToVersion(curWd, yarnExecPath string, installCommandArgs []string) (err error) {
+	// If the installCommandArgs in the params is not empty, it signifies that the user has provided it, and 'install' is already included as one of the arguments
+	installCommandProvidedFromUser := len(installCommandArgs) != 0
+
+	// Upon receiving a user-provided 'install' command, we execute the command exactly as provided
+	if installCommandProvidedFromUser {
+		return build.RunYarnCommand(yarnExecPath, curWd, installCommandArgs...)
+	}
+
+	installCommandArgs = []string{"install"}
+	executableVersionStr, err := biUtils.GetVersion(yarnExecPath, curWd)
+	if err != nil {
+		return
+	}
+
+	isYarnV1 := version.NewVersion(executableVersionStr).Compare(yarnV2Version) > 0
+
+	if isYarnV1 {
+		// When executing 'yarn install...', the node_modules directory is automatically generated.
+		// If it did not exist prior to the 'install' command, we aim to remove it.
+		nodeModulesFullPath := filepath.Join(curWd, nodeModulesRepoName)
+		var nodeModulesDirExists bool
+		nodeModulesDirExists, err = fileutils.IsDirExists(nodeModulesFullPath, false)
+		if err != nil {
+			err = fmt.Errorf("failed while checking for existence of node_modules directory: %s", err.Error())
+			return
+		}
+		if !nodeModulesDirExists {
+			defer func() {
+				err = errors.Join(err, fileutils.RemoveTempDir(nodeModulesFullPath))
+			}()
+		}
+
+		installCommandArgs = append(installCommandArgs, v1IgnoreScriptsFlag, v1SilentFlag, v1NonInteractiveFlag)
+	} else {
+		installCommandArgs = append(installCommandArgs, v2UpdateLockfileFlag, v2SkipBuildFlag)
+	}
+	return build.RunYarnCommand(yarnExecPath, curWd, installCommandArgs...)
+}
+
 // Parse the dependencies into a Xray dependency tree format
 func parseYarnDependenciesMap(dependencies map[string]*biUtils.YarnDependency, rootXrayId string) (*xrayUtils.GraphNode, []string) {
 	treeMap := make(map[string][]string)

xray/commands/audit/sca/yarn/yarn_test.go

@@ -1,13 +1,15 @@
 package yarn
 
 import (
+	"github.com/jfrog/build-info-go/build"
+	biutils "github.com/jfrog/build-info-go/build/utils"
+	utils2 "github.com/jfrog/build-info-go/utils"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 	"github.com/jfrog/jfrog-cli-core/v2/xray/utils"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
 	"github.com/stretchr/testify/assert"
+	"path/filepath"
 	"testing"
-
-	biutils "github.com/jfrog/build-info-go/build/utils"
-	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 )
 
 func TestParseYarnDependenciesList(t *testing.T) {
@@ -49,3 +51,53 @@ func TestParseYarnDependenciesList(t *testing.T) {
 	assert.ElementsMatch(t, uniqueDeps, expectedUniqueDeps, "First is actual, Second is Expected")
 	assert.True(t, tests.CompareTree(expectedTree, xrayDependenciesTree), "expected:", expectedTree.Nodes, "got:", xrayDependenciesTree.Nodes)
 }
+
+func TestIsYarnProjectInstalled(t *testing.T) {
+	tempDirPath, createTempDirCallback := tests.CreateTempDirWithCallbackAndAssert(t)
+	defer createTempDirCallback()
+	yarnProjectPath := filepath.Join("..", "..", "..", "testdata", "yarn-project")
+	assert.NoError(t, utils2.CopyDir(yarnProjectPath, tempDirPath, false, nil))
+	projectInstalled, err := isYarnProjectInstalled(tempDirPath)
+	assert.NoError(t, err)
+	assert.False(t, projectInstalled)
+	executablePath, err := biutils.GetYarnExecutable()
+	assert.NoError(t, err)
+
+	// We install the project and check again to verify we get the correct answer
+	assert.NoError(t, build.RunYarnCommand(executablePath, tempDirPath, "install"))
+	projectInstalled, err = isYarnProjectInstalled(tempDirPath)
+	assert.NoError(t, err)
+	assert.True(t, projectInstalled)
+}
+
+func TestRunYarnInstallAccordingToVersion(t *testing.T) {
+	executeRunYarnInstallAccordingToVersionAndVerifyInstallation(t, "1.22.19", []string{})
+	executeRunYarnInstallAccordingToVersionAndVerifyInstallation(t, "", []string{})
+	executeRunYarnInstallAccordingToVersionAndVerifyInstallation(t, "", []string{"install", "--mode=update-lockfile"})
+}
+
+func executeRunYarnInstallAccordingToVersionAndVerifyInstallation(t *testing.T, version string, params []string) {
+	tempDirPath, createTempDirCallback := tests.CreateTempDirWithCallbackAndAssert(t)
+	defer createTempDirCallback()
+	yarnProjectPath := filepath.Join("..", "..", "..", "testdata", "yarn-project")
+	assert.NoError(t, utils2.CopyDir(yarnProjectPath, tempDirPath, false, nil))
+
+	executablePath, err := biutils.GetYarnExecutable()
+	assert.NoError(t, err)
+
+	if version != "" {
+		assert.NoError(t, build.RunYarnCommand(executablePath, tempDirPath, "set", "version", version))
+	}
+
+	err = runYarnInstallAccordingToVersion(tempDirPath, executablePath, params)
+	if err != nil {
+		assert.NoError(t, err, err.Error())
+	} else {
+		assert.NoError(t, err)
+	}
+
+	// Checking the installation worked
+	projectInstalled, err := isYarnProjectInstalled(tempDirPath)
+	assert.NoError(t, err)
+	assert.True(t, projectInstalled)
+}

xray/commands/audit/scarunner.go

@@ -218,7 +218,7 @@ func GetTechDependencyTree(params xrayutils.AuditParams, tech coreutils.Technolo
 	case coreutils.Npm:
 		fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params)
 	case coreutils.Yarn:
-		fullDependencyTrees, uniqueDeps, err = yarn.BuildDependencyTree()
+		fullDependencyTrees, uniqueDeps, err = yarn.BuildDependencyTree(params)
 	case coreutils.Go:
 		fullDependencyTrees, uniqueDeps, err = _go.BuildDependencyTree(serverDetails, params.DepsRepo())
 	case coreutils.Pipenv, coreutils.Pip, coreutils.Poetry: