From 5408ca3733a7efef62f236f8f5cf8307c3e94b99 Mon Sep 17 00:00:00 2001 From: Vitor Cardoso Date: Wed, 10 Jan 2024 15:19:23 +0000 Subject: [PATCH] [DEVO-9536] - Add Jfrog Security Analysis to Pentaho Wingman --- resources/default-properties.yaml | 3 + .../minion-multibranch-pipeline-default.vm | 8 ++- .../hitachivantara/ci/build/Builder.groovy | 6 ++ .../ci/build/impl/AntBuilder.groovy | 6 ++ .../ci/build/impl/DSLScriptBuilder.groovy | 7 +++ .../ci/build/impl/GradleBuilder.groovy | 6 ++ .../ci/build/impl/JenkinsJobBuilder.groovy | 7 +++ .../ci/build/impl/MavenBuilder.groovy | 42 +++++++++++++ .../ci/config/LibraryProperties.groovy | 6 +- .../ci/github/GitHubManager.groovy | 2 +- .../ci/github/GitHubPullRequest.groovy | 2 + .../ci/github/IssueComment.groovy | 1 + .../hitachivantara/ci/TestLogReport.groovy | 6 +- .../ci/github/GitHubPullRequestTest.groovy | 4 +- vars/audit.groovy | 62 ++++++++++++++----- vars/stages.groovy | 20 ++++++ 16 files changed, 162 insertions(+), 26 deletions(-) diff --git a/resources/default-properties.yaml b/resources/default-properties.yaml index cd1eb34..54287b2 100644 --- a/resources/default-properties.yaml +++ b/resources/default-properties.yaml @@ -197,3 +197,6 @@ DOCKER_PRIVATE_PUSH_REPO: LOGS_TO_KEEP: ARTIFACTS_TO_KEEP: DISABLE_CONCURRENT_BUILDS: false + +# Artifacts +ARTIFACTORY_BASE_URL: https://one.hitachivantara.com diff --git a/resources/templates/minion-multibranch-pipeline-default.vm b/resources/templates/minion-multibranch-pipeline-default.vm index 1124220..52d48df 100644 --- a/resources/templates/minion-multibranch-pipeline-default.vm +++ b/resources/templates/minion-multibranch-pipeline-default.vm @@ -5,7 +5,8 @@ libraries.each { library -> runCheckout = properties.getBool('RUN_STAGE_CHECKOUT') runBuild = properties.getBool('RUN_STAGE_BUILD') -runSonar = properties.getBool('RUN_STAGE_SONAR') && item.auditable +runSonar = properties.getBool('RUN_SONAR') && item.auditable +runFrogbot = properties.getBool('RUN_FROGBOT') && item.auditable %> Map defaultParams = [ SLAVE_NODE_LABEL : '$properties.getString('SLAVE_NODE_LABEL')', @@ -15,7 +16,8 @@ Map defaultParams = [ RUN_STAGE_CHECKOUT : $runCheckout, RUN_STAGE_BUILD : $runBuild, - RUN_STAGE_SONAR : $runSonar, + RUN_SONARQUBE : $runSonar, + RUN_FROGBOT : $runFrogbot, OVERRIDE_PARAMS : [:], OVERRIDE_JOB_PARAMS : [:] @@ -29,7 +31,7 @@ node(params.SLAVE_NODE_LABEL ?: defaultParams.SLAVE_NODE_LABEL) { timeout(config.get().timeout) { stages.checkout() stages.buildAndTest() - stages.sonar() + stages.scans() } } diff --git a/src/org/hitachivantara/ci/build/Builder.groovy b/src/org/hitachivantara/ci/build/Builder.groovy index c6dbeb1..c422228 100644 --- a/src/org/hitachivantara/ci/build/Builder.groovy +++ b/src/org/hitachivantara/ci/build/Builder.groovy @@ -48,4 +48,10 @@ interface Builder { * @return a closure execution for sonar analysis */ Closure getSonarExecution() + + /** + * For frogbot execution + * @return a closure execution for frogbot pull request analysis + */ + Closure getFrogbotExecution() } \ No newline at end of file diff --git a/src/org/hitachivantara/ci/build/impl/AntBuilder.groovy b/src/org/hitachivantara/ci/build/impl/AntBuilder.groovy index ef04f4e..bdfa58c 100644 --- a/src/org/hitachivantara/ci/build/impl/AntBuilder.groovy +++ b/src/org/hitachivantara/ci/build/impl/AntBuilder.groovy @@ -131,6 +131,12 @@ class AntBuilder extends AbstractBuilder implements IBuilder, Serializable { return { -> } } + @Override + Closure getFrogbotExecution() { + // not implemented + return { -> } + } + private Closure getAntDsl(JobItem jobItem, String antCmd) { Map buildProperties = buildData.getBuildProperties() diff --git a/src/org/hitachivantara/ci/build/impl/DSLScriptBuilder.groovy b/src/org/hitachivantara/ci/build/impl/DSLScriptBuilder.groovy index e72a274..37673d6 100644 --- a/src/org/hitachivantara/ci/build/impl/DSLScriptBuilder.groovy +++ b/src/org/hitachivantara/ci/build/impl/DSLScriptBuilder.groovy @@ -101,4 +101,11 @@ class DSLScriptBuilder extends AbstractBuilder implements IBuilder, Serializable // not implemented return { -> } } + + @Override + Closure getFrogbotExecution() { + // not implemented + return { -> } + } + } diff --git a/src/org/hitachivantara/ci/build/impl/GradleBuilder.groovy b/src/org/hitachivantara/ci/build/impl/GradleBuilder.groovy index 2233400..4a2e2a4 100644 --- a/src/org/hitachivantara/ci/build/impl/GradleBuilder.groovy +++ b/src/org/hitachivantara/ci/build/impl/GradleBuilder.groovy @@ -79,6 +79,12 @@ class GradleBuilder extends AbstractBuilder implements Serializable { return getGradleDsl(gradleCommand) } + @Override + Closure getFrogbotExecution() { + // not implemented + return { -> } + } + Closure getGradleDsl(String cmd) { String localRepoPath = "${buildData.getString(LIB_CACHE_ROOT_PATH)}/gradle" diff --git a/src/org/hitachivantara/ci/build/impl/JenkinsJobBuilder.groovy b/src/org/hitachivantara/ci/build/impl/JenkinsJobBuilder.groovy index af2aa04..fda2723 100644 --- a/src/org/hitachivantara/ci/build/impl/JenkinsJobBuilder.groovy +++ b/src/org/hitachivantara/ci/build/impl/JenkinsJobBuilder.groovy @@ -101,4 +101,11 @@ class JenkinsJobBuilder extends AbstractBuilder implements IBuilder, Serializabl // not implemented return { -> } } + + @Override + Closure getFrogbotExecution() { + // not implemented + return { -> } + } + } diff --git a/src/org/hitachivantara/ci/build/impl/MavenBuilder.groovy b/src/org/hitachivantara/ci/build/impl/MavenBuilder.groovy index eb88278..f64f24d 100644 --- a/src/org/hitachivantara/ci/build/impl/MavenBuilder.groovy +++ b/src/org/hitachivantara/ci/build/impl/MavenBuilder.groovy @@ -27,6 +27,7 @@ import static org.hitachivantara.ci.GroovyUtils.intersect import static org.hitachivantara.ci.FileUtils.isChild import static org.hitachivantara.ci.build.helper.BuilderUtils.process import static org.hitachivantara.ci.FileUtils.resolve +import static org.hitachivantara.ci.config.LibraryProperties.ARTIFACTORY_BASE_URL import static org.hitachivantara.ci.config.LibraryProperties.ARTIFACT_DEPLOYER_CREDENTIALS_ID import static org.hitachivantara.ci.config.LibraryProperties.SCM_API_TOKEN_CREDENTIALS_ID import static org.hitachivantara.ci.config.LibraryProperties.BRANCH_NAME @@ -78,6 +79,47 @@ class MavenBuilder extends AbstractBuilder implements IBuilder, Serializable { return getMvnDsl(mvnCommand) } + @Override + Closure getFrogbotExecution() { + // Frogbot will only run on a PR + if (!buildData.isPullRequest()) { + steps.log.info "This is not a Pull Request build! Skipping..." + return { -> } + } + + String artifactoryURL = buildData.getString(ARTIFACTORY_BASE_URL) + String gitProvider = "github" + String gitRepo = item.scmInfo.repository + String gitOwner = item.scmInfo.organization + String gitPrNbr = buildData.get(CHANGE_ID) + String deployCredentials = buildData.getString(ARTIFACT_DEPLOYER_CREDENTIALS_ID) + String scmApiTokenCredential = buildData.getString(SCM_API_TOKEN_CREDENTIALS_ID) + + return { -> + steps.dir(item.buildWorkDir) { + steps.withEnv([ + "JF_URL=${artifactoryURL}", + "JF_GIT_PROVIDER=${gitProvider}", + "JF_GIT_REPO=${gitRepo}", + "JF_GIT_PULL_REQUEST_ID=${gitPrNbr}", + "JF_GIT_OWNER=${gitOwner}" + ]) { + steps.withCredentials([steps.usernamePassword(credentialsId: deployCredentials, + usernameVariable: 'JF_USER', passwordVariable: 'JF_PASSWORD'), + steps.string(credentialsId: scmApiTokenCredential, variable: 'JF_GIT_TOKEN')]) { + + //String localSettingsFile = item.settingsFile ?: settingsFile + + steps.log.info "Running /opt/frogbot scan-pull-request" + if (item.containerized) { + process("/opt/frogbot scan-pull-request", steps) + } + } + } + } + } + } + @Override void setBuilderData(Map builderData) { throw new BuilderException('Deprecated method that should not be used') diff --git a/src/org/hitachivantara/ci/config/LibraryProperties.groovy b/src/org/hitachivantara/ci/config/LibraryProperties.groovy index 88fc2eb..e24419c 100644 --- a/src/org/hitachivantara/ci/config/LibraryProperties.groovy +++ b/src/org/hitachivantara/ci/config/LibraryProperties.groovy @@ -84,6 +84,7 @@ class LibraryProperties implements Serializable { public static final String RUN_NEXUS_LIFECYCLE = 'RUN_NEXUS_LIFECYCLE' public static final String RUN_DEPENDENCY_CHECK = 'RUN_DEPENDENCY_CHECK' public static final String RUN_SONARQUBE = 'RUN_SONARQUBE' + public static final String RUN_FROGBOT = 'RUN_FROGBOT' public static final String PUSH_CHANGES = 'PUSH_CHANGES' public static final String PARALLEL_SIZE = 'PARALLEL_SIZE' @@ -154,7 +155,7 @@ class LibraryProperties implements Serializable { public static final String STAGE_LABEL_CHECKOUT = 'Checkout' public static final String STAGE_LABEL_VERSIONING = 'Version' public static final String STAGE_LABEL_BUILD = 'Build' - public static final String STAGE_LABEL_AUDIT = 'Audit' + public static final String STAGE_LABEL_AUDIT = 'Scans' public static final String STAGE_LABEL_UNIT_TEST = 'Test' public static final String STAGE_LABEL_PUSH = 'Push' public static final String STAGE_LABEL_TAG = 'Tag' @@ -235,4 +236,7 @@ class LibraryProperties implements Serializable { public static final String LOGS_TO_KEEP = 'LOGS_TO_KEEP' public static final String ARTIFACTS_TO_KEEP = 'ARTIFACTS_TO_KEEP' public static final String DISABLE_CONCURRENT_BUILDS = 'DISABLE_CONCURRENT_BUILDS' + + public static final String ARTIFACTORY_BASE_URL = 'ARTIFACTORY_BASE_URL' + } diff --git a/src/org/hitachivantara/ci/github/GitHubManager.groovy b/src/org/hitachivantara/ci/github/GitHubManager.groovy index fe1887c..2351698 100644 --- a/src/org/hitachivantara/ci/github/GitHubManager.groovy +++ b/src/org/hitachivantara/ci/github/GitHubManager.groovy @@ -211,7 +211,7 @@ class GitHubManager implements Serializable { GitHubPullRequest pullRequest = repository.getPullRequest(prNumber) // minimize previous comments - List comments = pullRequest.comments.findAll { issueComment -> issueComment.viewerDidAuthor & !issueComment.isMinimized } + List comments = pullRequest.comments.findAll { issueComment -> !issueComment.body.contains('frogbot') & issueComment.viewerDidAuthor & !issueComment.isMinimized } comments.each { IssueComment comment -> comment.hide(GitHubMinimizeContentReason.OUTDATED) } diff --git a/src/org/hitachivantara/ci/github/GitHubPullRequest.groovy b/src/org/hitachivantara/ci/github/GitHubPullRequest.groovy index 4f23bcb..d042b71 100644 --- a/src/org/hitachivantara/ci/github/GitHubPullRequest.groovy +++ b/src/org/hitachivantara/ci/github/GitHubPullRequest.groovy @@ -29,6 +29,7 @@ class GitHubPullRequest implements Serializable { isMinimized viewerDidAuthor minimizedReason + body } } } @@ -48,6 +49,7 @@ class GitHubPullRequest implements Serializable { it.id = issueComment.id it.isMinimized = issueComment.isMinimized it.viewerDidAuthor = issueComment.viewerDidAuthor + it.body = issueComment.body if (reason) { it.minimizedReason = GitHubMinimizeContentReason.valueOf(reason.toUpperCase()) } diff --git a/src/org/hitachivantara/ci/github/IssueComment.groovy b/src/org/hitachivantara/ci/github/IssueComment.groovy index b4a889e..a1f9ffb 100644 --- a/src/org/hitachivantara/ci/github/IssueComment.groovy +++ b/src/org/hitachivantara/ci/github/IssueComment.groovy @@ -11,6 +11,7 @@ class IssueComment implements Serializable { Boolean isMinimized Boolean viewerDidAuthor GitHubMinimizeContentReason minimizedReason + String body /** Minimizes a comment on an Issue, Commit, Pull Request, or Gist * @param reason The reason why to minimize diff --git a/test/src/org/hitachivantara/ci/TestLogReport.groovy b/test/src/org/hitachivantara/ci/TestLogReport.groovy index 01ca78e..3985931 100644 --- a/test/src/org/hitachivantara/ci/TestLogReport.groovy +++ b/test/src/org/hitachivantara/ci/TestLogReport.groovy @@ -130,8 +130,8 @@ class TestLogReport extends BasePipelineSpecification { setup: LogReport report = new LogReport(mockScript) - configRule.time('Audit', 100000) - configRule.time('Audit', 'my custom item ID', 100000) + configRule.time('Scans', 100000) + configRule.time('Scans', 'my custom item ID', 100000) when: report.build(configRule.buildData) @@ -143,7 +143,7 @@ class TestLogReport extends BasePipelineSpecification { 'Warnings:\n' + ' No warnings\n' + 'Timings:\n' + - ' [Audit] (1m 40s)\n' + + ' [Scans] (1m 40s)\n' + ' my custom item ID : 1m 40s\n' + 'Releases:\n' + ' No releases' diff --git a/test/src/org/hitachivantara/ci/github/GitHubPullRequestTest.groovy b/test/src/org/hitachivantara/ci/github/GitHubPullRequestTest.groovy index a2d105d..451e203 100644 --- a/test/src/org/hitachivantara/ci/github/GitHubPullRequestTest.groovy +++ b/test/src/org/hitachivantara/ci/github/GitHubPullRequestTest.groovy @@ -61,8 +61,8 @@ class GitHubPullRequestTest extends BasePipelineSpecification { def "test commentPullRequest"() { given: GitHubPullRequest pullRequest = new GitHubPullRequest(comments: [ - new IssueComment(id: '1', isMinimized: false, viewerDidAuthor: true), - new IssueComment(id: '2', isMinimized: false, viewerDidAuthor: false), + new IssueComment(id: '1', isMinimized: false, viewerDidAuthor: true, body: 'something'), + new IssueComment(id: '2', isMinimized: false, viewerDidAuthor: false, body: 'also something'), ]) replacements.addReplacement(GitHubPullRequest, ['static.get': { String owner, String name, Integer number -> pullRequest }]) replacements.addReplacement(GitHubManager, [ diff --git a/vars/audit.groovy b/vars/audit.groovy index b38dd6f..0d13dfb 100644 --- a/vars/audit.groovy +++ b/vars/audit.groovy @@ -24,30 +24,29 @@ import static org.hitachivantara.ci.config.LibraryProperties.RUN_SONARQUBE import static org.hitachivantara.ci.config.LibraryProperties.STAGE_LABEL_AUDIT import static org.hitachivantara.ci.config.LibraryProperties.RUN_BUILDS import static org.hitachivantara.ci.config.LibraryProperties.RUN_CHECKOUTS +import static org.hitachivantara.ci.config.LibraryProperties.RUN_FROGBOT // must be power of 2 @Field int DEPENDENCY_CHECK = 0x1 @Field int NEXUS_IQ_SCAN = 0x2 @Field int SONARQUBE = 0x4 -@Field int ALL = DEPENDENCY_CHECK | NEXUS_IQ_SCAN | SONARQUBE +@Field int FROGBOT = 0x8 +@Field int ALL = DEPENDENCY_CHECK | NEXUS_IQ_SCAN | SONARQUBE | FROGBOT def call() { BuildData buildData = BuildData.instance - if (buildData.runAudit) { - utils.timer( - { - runStage(buildData, getEnabledScanners(buildData)) - }, - { long duration -> - buildData.time(STAGE_LABEL_AUDIT, duration) - log.info "${STAGE_LABEL_AUDIT} completed in ${TimeCategory.minus(new Date(duration), new Date(0))}" - } - ) - } else { - utils.createStageSkipped(STAGE_LABEL_AUDIT) - buildData.time(STAGE_LABEL_AUDIT, 0) - } + + utils.timer( + { + runStage(buildData, getEnabledScanners(buildData)) + }, + { long duration -> + buildData.time(STAGE_LABEL_AUDIT, duration) + log.info "${STAGE_LABEL_AUDIT} completed in ${TimeCategory.minus(new Date(duration), new Date(0))}" + } + ) + } int getEnabledScanners(BuildData buildData) { @@ -56,6 +55,7 @@ int getEnabledScanners(BuildData buildData) { enabledScanners |= buildData.getBool(RUN_DEPENDENCY_CHECK) ? DEPENDENCY_CHECK : 0 enabledScanners |= buildData.getBool(RUN_NEXUS_LIFECYCLE) ? NEXUS_IQ_SCAN : 0 enabledScanners |= buildData.getBool(RUN_SONARQUBE) ? SONARQUBE : 0 + enabledScanners |= buildData.getBool(RUN_FROGBOT) && buildData.isPullRequest() ? FROGBOT : 0 return enabledScanners } @@ -135,6 +135,9 @@ Map getEntries(BuildData buildData, JobItem jobItem, int enable if (enabledScanners & SONARQUBE) { scanners << [(jobItem.jobID + ': SonarQube'): { -> sonar(buildData, jobItem) }] } + if (enabledScanners & FROGBOT) { + scanners << [(jobItem.jobID + ': Frogbot'): { -> frogbot(buildData, jobItem) }] + } return scanners } @@ -184,4 +187,31 @@ void nexusIQScan(/*BuildData buildData, JobItem jobItem*/) { void dependencyCheck(/*BuildData buildData, JobItem jobItem*/) { //TODO log.warn "Not implemented yet" -} \ No newline at end of file +} + +/** + * Perform frogobot's pull request analysis + * + * @param buildData + * @param jobItem + */ +void frogbot(BuildData buildData, JobItem jobItem) { + Builder builder = BuilderFactory.builderFor(jobItem) + Closure execution = builder.frogbotExecution + + // apply retries + if (buildData.getInt(BUILD_RETRIES)) { + Closure current = execution + execution = { -> retry(buildData.getInt(BUILD_RETRIES), current) } + } + + // apply container + if (jobItem.containerized) { + Closure current = execution + execution = { -> utils.withContainer(jobItem.dockerImage, current) } + } + + utils.timer(execution) { long duration -> + buildData.time(STAGE_LABEL_AUDIT, jobItem.jobID + ': Frogbot', duration) + } +} diff --git a/vars/stages.groovy b/vars/stages.groovy index 4e068ab..8bd8e2e 100644 --- a/vars/stages.groovy +++ b/vars/stages.groovy @@ -315,6 +315,26 @@ void sonar(String id = 'sonar', String label = '') { ).run() } +void frogbot(String id = 'frogbot', String label = '') { + new ParallelItemWorkStage(id: id, label: label ?: id.capitalize(), + ignoreGroups: true, + allowMinions: true, + itemFilter: { List items -> + BuilderUtils.applyChanges(id, items) + items.findAll { JobItem item -> !item.skip && item.auditable } + }, + itemExecution: { JobItem item -> + BuilderFactory.builderFor(id, item) + .getFrogbotExecution() + .call() + } + ).run() +} + +void scans() { + audit.call() +} + void report() { new SimpleStage(id: 'report', label: STAGE_LABEL_REPORT, body: {