From 703f429ad000889c6e7e47fde45d8bf7a1de76c2 Mon Sep 17 00:00:00 2001 From: Or Geva Date: Mon, 13 Nov 2023 17:57:02 +0200 Subject: [PATCH 1/6] Ensure JAS scanners scan all available roots when no configuration file found --- .../com/jfrog/ide/idea/scan/ScanUtils.java | 5 +---- .../jfrog/ide/idea/scan/ScannerFactory.java | 4 +++- .../applications/JFrogApplicationsConfig.java | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java b/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java index fab31beb..05e2b703 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java @@ -12,8 +12,6 @@ import java.io.IOException; import java.nio.file.Path; -import java.util.Collection; -import java.util.Map; import java.util.Set; /** @@ -28,7 +26,7 @@ public class ScanUtils { * @param project - the project * @return local scan paths */ - static Set createScanPaths(Map scannersMap, Project project) { + public static Set createScanPaths(Project project) { final Set paths = Sets.newHashSet(); paths.add(com.jfrog.ide.idea.utils.Utils.getProjectBasePath(project)); for (Module module : ModuleManager.getInstance(project).getModules()) { @@ -37,7 +35,6 @@ static Set createScanPaths(Map scannersMap, Project paths.add(modulePath.toNioPath()); } } - scannersMap.values().stream().map(ScannerBase::getProjectPaths).flatMap(Collection::stream).forEach(paths::add); Logger.getInstance().debug("Scanning projects in the following paths: " + paths); return paths; } diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScannerFactory.java b/src/main/java/com/jfrog/ide/idea/scan/ScannerFactory.java index 63755203..3fa120a1 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScannerFactory.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScannerFactory.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.Collection; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; @@ -45,7 +46,8 @@ public Map refreshScanners(Map oldSc Map scanners = Maps.newHashMap(); refreshMavenScanner(scanners, oldScanners, executor, scanLogic); refreshPypiScanners(scanners, oldScanners, executor, scanLogic); - Set scanPaths = createScanPaths(oldScanners, project); + Set scanPaths = createScanPaths(project); + oldScanners.values().stream().map(ScannerBase::getProjectPaths).flatMap(Collection::stream).forEach(scanPaths::add); refreshGenericScanners(scanners, oldScanners, scanPaths, executor, scanLogic); return scanners; } diff --git a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java index 158257da..4bea741a 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java +++ b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java @@ -6,11 +6,12 @@ import lombok.Getter; import lombok.NoArgsConstructor; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Set; import static com.jfrog.ide.idea.scan.SourceCodeScannerManager.convertToSkippedFolders; -import static com.jfrog.ide.idea.utils.Utils.getProjectBasePath; @Getter @NoArgsConstructor @@ -22,15 +23,16 @@ public class JFrogApplicationsConfig { public static JFrogApplicationsConfig createApplicationConfigWithDefaultModule(Project project) { JFrogApplicationsConfig applicationsConfig = new JFrogApplicationsConfig(); - - ModuleConfig defualtModuleConfig = new ModuleConfig(); - defualtModuleConfig.setSourceRoot(getProjectBasePath(project).toAbsolutePath().toString()); - defualtModuleConfig.setExcludePatterns(convertToSkippedFolders(GlobalSettings.getInstance().getServerConfig().getExcludedPaths())); - + Set paths = com.jfrog.ide.idea.scan.ScanUtils.createScanPaths(project); applicationsConfig.modules = new ArrayList<>(); - applicationsConfig.modules.add(defualtModuleConfig); + + for (Path path : paths) { + ModuleConfig defualtModuleConfig = new ModuleConfig(); + defualtModuleConfig.setSourceRoot(path.toString()); + defualtModuleConfig.setExcludePatterns(convertToSkippedFolders(GlobalSettings.getInstance().getServerConfig().getExcludedPaths())); + applicationsConfig.modules.add(defualtModuleConfig); + } return applicationsConfig; } - } From 7b5f760d9570dbbe12e2e4c4eb893c167faaf5ae Mon Sep 17 00:00:00 2001 From: Or Geva Date: Mon, 13 Nov 2023 17:57:22 +0200 Subject: [PATCH 2/6] Remove JAS scanners timeout --- src/main/java/com/jfrog/ide/idea/scan/ScanManager.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java b/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java index d849148d..487ec425 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java @@ -31,14 +31,12 @@ import static com.jfrog.ide.common.utils.XrayConnectionUtils.createXrayClientBuilder; public class ScanManager { - private final int SCAN_TIMEOUT_MINUTES = 60; private final Project project; private final ScannerFactory factory; private final SourceCodeScannerManager sourceCodeScannerManager; private Map scanners = Maps.newHashMap(); private ExecutorService executor; - private ScanManager(@NotNull Project project) { this.project = project; factory = new ScannerFactory(project); @@ -97,8 +95,8 @@ public void startScan() { scanner.asyncScanAndUpdateResults(); } executor.shutdown(); - if (!executor.awaitTermination(SCAN_TIMEOUT_MINUTES, TimeUnit.MINUTES)) { - logError(Logger.getInstance(), "Scan timeout of " + SCAN_TIMEOUT_MINUTES + " minutes elapsed. The scan is being canceled.", true); + if (!executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES)) { + logError(Logger.getInstance(), "Scan timeout elapsed. The scan is being canceled.", true); } // Cache tree only if no errors occurred during scan. if (scanners.values().stream().anyMatch(ScannerBase::isScanErrorOccurred)) { From a1b51693a0f226d0ca91daa6b22577b61ccfd86a Mon Sep 17 00:00:00 2001 From: Or Geva Date: Tue, 14 Nov 2023 10:32:05 +0200 Subject: [PATCH 3/6] Revert "Remove JAS scanners timeout" This reverts commit 7b5f760d9570dbbe12e2e4c4eb893c167faaf5ae. --- src/main/java/com/jfrog/ide/idea/scan/ScanManager.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java b/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java index 487ec425..d849148d 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScanManager.java @@ -31,12 +31,14 @@ import static com.jfrog.ide.common.utils.XrayConnectionUtils.createXrayClientBuilder; public class ScanManager { + private final int SCAN_TIMEOUT_MINUTES = 60; private final Project project; private final ScannerFactory factory; private final SourceCodeScannerManager sourceCodeScannerManager; private Map scanners = Maps.newHashMap(); private ExecutorService executor; + private ScanManager(@NotNull Project project) { this.project = project; factory = new ScannerFactory(project); @@ -95,8 +97,8 @@ public void startScan() { scanner.asyncScanAndUpdateResults(); } executor.shutdown(); - if (!executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES)) { - logError(Logger.getInstance(), "Scan timeout elapsed. The scan is being canceled.", true); + if (!executor.awaitTermination(SCAN_TIMEOUT_MINUTES, TimeUnit.MINUTES)) { + logError(Logger.getInstance(), "Scan timeout of " + SCAN_TIMEOUT_MINUTES + " minutes elapsed. The scan is being canceled.", true); } // Cache tree only if no errors occurred during scan. if (scanners.values().stream().anyMatch(ScannerBase::isScanErrorOccurred)) { From 2a50f1cb98e4e0cfbdf803220d727cca268cb683 Mon Sep 17 00:00:00 2001 From: Or Geva Date: Tue, 14 Nov 2023 10:32:40 +0200 Subject: [PATCH 4/6] Apply CR changes --- src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java | 4 +--- .../data/applications/JFrogApplicationsConfig.java | 11 ++++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java b/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java index 05e2b703..cc9d7638 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java @@ -20,10 +20,8 @@ public class ScanUtils { /** * This method gets a set of modules from IDEA, and searches for projects to be scanned. - * It appends the root path of each module it finds into a set. * - * @param scannersMap - current scanners - * @param project - the project + * @param project - the project * @return local scan paths */ public static Set createScanPaths(Project project) { diff --git a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java index 4bea741a..fb7acb76 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java +++ b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.intellij.openapi.project.Project; +import com.jfrog.ide.common.utils.Utils; import com.jfrog.ide.idea.configuration.GlobalSettings; import lombok.Getter; import lombok.NoArgsConstructor; @@ -26,11 +27,11 @@ public static JFrogApplicationsConfig createApplicationConfigWithDefaultModule(P Set paths = com.jfrog.ide.idea.scan.ScanUtils.createScanPaths(project); applicationsConfig.modules = new ArrayList<>(); - for (Path path : paths) { - ModuleConfig defualtModuleConfig = new ModuleConfig(); - defualtModuleConfig.setSourceRoot(path.toString()); - defualtModuleConfig.setExcludePatterns(convertToSkippedFolders(GlobalSettings.getInstance().getServerConfig().getExcludedPaths())); - applicationsConfig.modules.add(defualtModuleConfig); + for (Path path : Utils.consolidatePaths(paths)) { + ModuleConfig defaultModuleConfig = new ModuleConfig(); + defaultModuleConfig.setSourceRoot(path.toString()); + defaultModuleConfig.setExcludePatterns(convertToSkippedFolders(GlobalSettings.getInstance().getServerConfig().getExcludedPaths())); + applicationsConfig.modules.add(defaultModuleConfig); } return applicationsConfig; From dca7125f512fffd182b8ecff4f29c21a247ae26a Mon Sep 17 00:00:00 2001 From: Or Geva Date: Tue, 14 Nov 2023 12:15:18 +0200 Subject: [PATCH 5/6] Apply CR changes --- src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java | 4 +++- .../idea/scan/data/applications/JFrogApplicationsConfig.java | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java b/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java index cc9d7638..5022cacb 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScanUtils.java @@ -6,6 +6,7 @@ import com.intellij.openapi.project.Project; import com.intellij.openapi.project.ProjectUtil; import com.intellij.openapi.vfs.VirtualFile; +import com.jfrog.ide.common.utils.Utils; import com.jfrog.ide.idea.log.Logger; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; @@ -25,7 +26,7 @@ public class ScanUtils { * @return local scan paths */ public static Set createScanPaths(Project project) { - final Set paths = Sets.newHashSet(); + Set paths = Sets.newHashSet(); paths.add(com.jfrog.ide.idea.utils.Utils.getProjectBasePath(project)); for (Module module : ModuleManager.getInstance(project).getModules()) { VirtualFile modulePath = ProjectUtil.guessModuleDir(module); @@ -33,6 +34,7 @@ public static Set createScanPaths(Project project) { paths.add(modulePath.toNioPath()); } } + paths = Utils.consolidatePaths(paths); Logger.getInstance().debug("Scanning projects in the following paths: " + paths); return paths; } diff --git a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java index fb7acb76..17dcb04b 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java +++ b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.intellij.openapi.project.Project; -import com.jfrog.ide.common.utils.Utils; import com.jfrog.ide.idea.configuration.GlobalSettings; import lombok.Getter; import lombok.NoArgsConstructor; @@ -27,7 +26,7 @@ public static JFrogApplicationsConfig createApplicationConfigWithDefaultModule(P Set paths = com.jfrog.ide.idea.scan.ScanUtils.createScanPaths(project); applicationsConfig.modules = new ArrayList<>(); - for (Path path : Utils.consolidatePaths(paths)) { + for (Path path : paths) { ModuleConfig defaultModuleConfig = new ModuleConfig(); defaultModuleConfig.setSourceRoot(path.toString()); defaultModuleConfig.setExcludePatterns(convertToSkippedFolders(GlobalSettings.getInstance().getServerConfig().getExcludedPaths())); From e2c14c380405cc061f0d02a30aff33964cc4bd54 Mon Sep 17 00:00:00 2001 From: Or Geva Date: Tue, 14 Nov 2023 13:12:15 +0200 Subject: [PATCH 6/6] Apply CR changes --- .../idea/scan/data/applications/JFrogApplicationsConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java index 17dcb04b..dde08b0a 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java +++ b/src/main/java/com/jfrog/ide/idea/scan/data/applications/JFrogApplicationsConfig.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.intellij.openapi.project.Project; import com.jfrog.ide.idea.configuration.GlobalSettings; +import com.jfrog.ide.idea.scan.ScanUtils; import lombok.Getter; import lombok.NoArgsConstructor; @@ -23,7 +24,7 @@ public class JFrogApplicationsConfig { public static JFrogApplicationsConfig createApplicationConfigWithDefaultModule(Project project) { JFrogApplicationsConfig applicationsConfig = new JFrogApplicationsConfig(); - Set paths = com.jfrog.ide.idea.scan.ScanUtils.createScanPaths(project); + Set paths = ScanUtils.createScanPaths(project); applicationsConfig.modules = new ArrayList<>(); for (Path path : paths) {