Merge pull request #50 from pentaho/DEVO-9536

[DEVO-9536] - Add Jfrog Security Analysis to Pentaho Wingman

resources/default-properties.yaml

@@ -197,3 +197,6 @@ DOCKER_PRIVATE_PUSH_REPO:
 LOGS_TO_KEEP:
 ARTIFACTS_TO_KEEP:
 DISABLE_CONCURRENT_BUILDS: false
+
+# Artifacts
+ARTIFACTORY_BASE_URL: https://one.hitachivantara.com

resources/templates/minion-multibranch-pipeline-default.vm

@@ -5,7 +5,8 @@ libraries.each { library ->
 
 runCheckout = properties.getBool('RUN_STAGE_CHECKOUT')
 runBuild = properties.getBool('RUN_STAGE_BUILD')
-runSonar = properties.getBool('RUN_STAGE_SONAR') && item.auditable
+runSonar = properties.getBool('RUN_SONAR') && item.auditable
+runFrogbot = properties.getBool('RUN_FROGBOT') && item.auditable
 %>
 Map defaultParams = [
   SLAVE_NODE_LABEL    : '$properties.getString('SLAVE_NODE_LABEL')',
@@ -15,7 +16,8 @@ Map defaultParams = [
 
   RUN_STAGE_CHECKOUT  : $runCheckout,
   RUN_STAGE_BUILD     : $runBuild,
-  RUN_STAGE_SONAR     : $runSonar,
+  RUN_SONARQUBE       : $runSonar,
+  RUN_FROGBOT         : $runFrogbot,
 
   OVERRIDE_PARAMS     : [:],
   OVERRIDE_JOB_PARAMS : [:]
@@ -29,7 +31,7 @@ node(params.SLAVE_NODE_LABEL ?: defaultParams.SLAVE_NODE_LABEL) {
       timeout(config.get().timeout) {
         stages.checkout()
         stages.buildAndTest()
-        stages.sonar()
+        stages.scans()
       }
     }
 

src/org/hitachivantara/ci/build/Builder.groovy

@@ -48,4 +48,10 @@ interface Builder {
    * @return a closure execution for sonar analysis
    */
   Closure getSonarExecution()
+
+  /**
+   * For frogbot execution
+   * @return a closure execution for frogbot pull request analysis
+   */
+  Closure getFrogbotExecution()
 }

src/org/hitachivantara/ci/build/impl/AntBuilder.groovy

@@ -131,6 +131,12 @@ class AntBuilder extends AbstractBuilder implements IBuilder, Serializable {
     return { -> }
   }
 
+  @Override
+  Closure getFrogbotExecution() {
+    // not implemented
+    return { -> }
+  }
+
   private Closure getAntDsl(JobItem jobItem, String antCmd) {
     Map buildProperties = buildData.getBuildProperties()
 

src/org/hitachivantara/ci/build/impl/DSLScriptBuilder.groovy

@@ -101,4 +101,11 @@ class DSLScriptBuilder extends AbstractBuilder implements IBuilder, Serializable
     // not implemented
     return { -> }
   }
+
+  @Override
+  Closure getFrogbotExecution() {
+    // not implemented
+    return { -> }
+  }
+
 }

src/org/hitachivantara/ci/build/impl/GradleBuilder.groovy

@@ -79,6 +79,12 @@ class GradleBuilder extends AbstractBuilder implements Serializable {
     return getGradleDsl(gradleCommand)
   }
 
+  @Override
+  Closure getFrogbotExecution() {
+    // not implemented
+    return { -> }
+  }
+
   Closure getGradleDsl(String cmd) {
     String localRepoPath = "${buildData.getString(LIB_CACHE_ROOT_PATH)}/gradle"
 

src/org/hitachivantara/ci/build/impl/JenkinsJobBuilder.groovy

@@ -101,4 +101,11 @@ class JenkinsJobBuilder extends AbstractBuilder implements IBuilder, Serializabl
     // not implemented
     return { -> }
   }
+
+  @Override
+  Closure getFrogbotExecution() {
+    // not implemented
+    return { -> }
+  }
+
 }

src/org/hitachivantara/ci/build/impl/MavenBuilder.groovy

@@ -27,6 +27,7 @@ import static org.hitachivantara.ci.GroovyUtils.intersect
 import static org.hitachivantara.ci.FileUtils.isChild
 import static org.hitachivantara.ci.build.helper.BuilderUtils.process
 import static org.hitachivantara.ci.FileUtils.resolve
+import static org.hitachivantara.ci.config.LibraryProperties.ARTIFACTORY_BASE_URL
 import static org.hitachivantara.ci.config.LibraryProperties.ARTIFACT_DEPLOYER_CREDENTIALS_ID
 import static org.hitachivantara.ci.config.LibraryProperties.SCM_API_TOKEN_CREDENTIALS_ID
 import static org.hitachivantara.ci.config.LibraryProperties.BRANCH_NAME
@@ -78,6 +79,47 @@ class MavenBuilder extends AbstractBuilder implements IBuilder, Serializable {
     return getMvnDsl(mvnCommand)
   }
 
+  @Override
+  Closure getFrogbotExecution() {
+    // Frogbot will only run on a PR
+    if (!buildData.isPullRequest()) {
+      steps.log.info "This is not a Pull Request build! Skipping..."
+      return { -> }
+    }
+
+    String artifactoryURL = buildData.getString(ARTIFACTORY_BASE_URL)
+    String gitProvider = "github"
+    String gitRepo = item.scmInfo.repository
+    String gitOwner = item.scmInfo.organization
+    String gitPrNbr = buildData.get(CHANGE_ID)
+    String deployCredentials = buildData.getString(ARTIFACT_DEPLOYER_CREDENTIALS_ID)
+    String scmApiTokenCredential = buildData.getString(SCM_API_TOKEN_CREDENTIALS_ID)
+
+    return { ->
+      steps.dir(item.buildWorkDir) {
+        steps.withEnv([
+            "JF_URL=${artifactoryURL}",
+            "JF_GIT_PROVIDER=${gitProvider}",
+            "JF_GIT_REPO=${gitRepo}",
+            "JF_GIT_PULL_REQUEST_ID=${gitPrNbr}",
+            "JF_GIT_OWNER=${gitOwner}"
+        ]) {
+          steps.withCredentials([steps.usernamePassword(credentialsId: deployCredentials,
+              usernameVariable: 'JF_USER', passwordVariable: 'JF_PASSWORD'),
+                                 steps.string(credentialsId: scmApiTokenCredential, variable: 'JF_GIT_TOKEN')]) {
+
+            //String localSettingsFile = item.settingsFile ?: settingsFile
+
+            steps.log.info "Running /opt/frogbot scan-pull-request"
+            if (item.containerized) {
+              process("/opt/frogbot scan-pull-request", steps)
+            }
+          }
+        }
+      }
+    }
+  }
+
   @Override
   void setBuilderData(Map builderData) {
     throw new BuilderException('Deprecated method that should not be used')

src/org/hitachivantara/ci/config/LibraryProperties.groovy

@@ -84,6 +84,7 @@ class LibraryProperties implements Serializable {
   public static final String RUN_NEXUS_LIFECYCLE = 'RUN_NEXUS_LIFECYCLE'
   public static final String RUN_DEPENDENCY_CHECK = 'RUN_DEPENDENCY_CHECK'
   public static final String RUN_SONARQUBE = 'RUN_SONARQUBE'
+  public static final String RUN_FROGBOT = 'RUN_FROGBOT'
 
   public static final String PUSH_CHANGES = 'PUSH_CHANGES'
   public static final String PARALLEL_SIZE = 'PARALLEL_SIZE'
@@ -154,7 +155,7 @@ class LibraryProperties implements Serializable {
   public static final String STAGE_LABEL_CHECKOUT = 'Checkout'
   public static final String STAGE_LABEL_VERSIONING = 'Version'
   public static final String STAGE_LABEL_BUILD = 'Build'
-  public static final String STAGE_LABEL_AUDIT = 'Audit'
+  public static final String STAGE_LABEL_AUDIT = 'Scans'
   public static final String STAGE_LABEL_UNIT_TEST = 'Test'
   public static final String STAGE_LABEL_PUSH = 'Push'
   public static final String STAGE_LABEL_TAG = 'Tag'
@@ -235,4 +236,7 @@ class LibraryProperties implements Serializable {
   public static final String LOGS_TO_KEEP = 'LOGS_TO_KEEP'
   public static final String ARTIFACTS_TO_KEEP = 'ARTIFACTS_TO_KEEP'
   public static final String DISABLE_CONCURRENT_BUILDS = 'DISABLE_CONCURRENT_BUILDS'
+
+  public static final String ARTIFACTORY_BASE_URL = 'ARTIFACTORY_BASE_URL'
+
 }

src/org/hitachivantara/ci/github/GitHubManager.groovy

@@ -211,7 +211,7 @@ class GitHubManager implements Serializable {
       GitHubPullRequest pullRequest = repository.getPullRequest(prNumber)
 
       // minimize previous comments
-      List<IssueComment> comments = pullRequest.comments.findAll { issueComment -> issueComment.viewerDidAuthor & !issueComment.isMinimized }
+      List<IssueComment> comments = pullRequest.comments.findAll { issueComment -> !issueComment.body.contains('frogbot') & issueComment.viewerDidAuthor & !issueComment.isMinimized }
       comments.each { IssueComment comment ->
         comment.hide(GitHubMinimizeContentReason.OUTDATED)
       }

src/org/hitachivantara/ci/github/GitHubPullRequest.groovy

@@ -29,6 +29,7 @@ class GitHubPullRequest implements Serializable {
                     isMinimized
                     viewerDidAuthor
                     minimizedReason
+                    body
                   }
                 }
               }
@@ -48,6 +49,7 @@ class GitHubPullRequest implements Serializable {
           it.id = issueComment.id
           it.isMinimized = issueComment.isMinimized
           it.viewerDidAuthor = issueComment.viewerDidAuthor
+          it.body = issueComment.body
           if (reason) {
             it.minimizedReason = GitHubMinimizeContentReason.valueOf(reason.toUpperCase())
           }

src/org/hitachivantara/ci/github/IssueComment.groovy

@@ -11,6 +11,7 @@ class IssueComment implements Serializable {
   Boolean isMinimized
   Boolean viewerDidAuthor
   GitHubMinimizeContentReason minimizedReason
+  String body
 
   /** Minimizes a comment on an Issue, Commit, Pull Request, or Gist
    * @param reason The reason why to minimize

test/src/org/hitachivantara/ci/TestLogReport.groovy

@@ -130,8 +130,8 @@ class TestLogReport extends BasePipelineSpecification {
     setup:
       LogReport report = new LogReport(mockScript)
 
-      configRule.time('Audit', 100000)
-      configRule.time('Audit', 'my custom item ID', 100000)
+      configRule.time('Scans', 100000)
+      configRule.time('Scans', 'my custom item ID', 100000)
 
     when:
       report.build(configRule.buildData)
@@ -143,7 +143,7 @@ class TestLogReport extends BasePipelineSpecification {
         'Warnings:\n' +
         '  No warnings\n' +
         'Timings:\n' +
-        '  [Audit] (1m 40s)\n' +
+        '  [Scans] (1m 40s)\n' +
         '    my custom item ID : 1m 40s\n' +
         'Releases:\n' +
         '  No releases'

test/src/org/hitachivantara/ci/github/GitHubPullRequestTest.groovy

@@ -61,8 +61,8 @@ class GitHubPullRequestTest extends BasePipelineSpecification {
   def "test commentPullRequest"() {
     given:
       GitHubPullRequest pullRequest = new GitHubPullRequest(comments: [
-        new IssueComment(id: '1', isMinimized: false, viewerDidAuthor: true),
-        new IssueComment(id: '2', isMinimized: false, viewerDidAuthor: false),
+        new IssueComment(id: '1', isMinimized: false, viewerDidAuthor: true, body: 'something'),
+        new IssueComment(id: '2', isMinimized: false, viewerDidAuthor: false, body: 'also something'),
       ])
       replacements.addReplacement(GitHubPullRequest, ['static.get': { String owner, String name, Integer number -> pullRequest }])
       replacements.addReplacement(GitHubManager, [

vars/audit.groovy

@@ -24,30 +24,29 @@ import static org.hitachivantara.ci.config.LibraryProperties.RUN_SONARQUBE
 import static org.hitachivantara.ci.config.LibraryProperties.STAGE_LABEL_AUDIT
 import static org.hitachivantara.ci.config.LibraryProperties.RUN_BUILDS
 import static org.hitachivantara.ci.config.LibraryProperties.RUN_CHECKOUTS
+import static org.hitachivantara.ci.config.LibraryProperties.RUN_FROGBOT
 
 // must be power of 2
 @Field int DEPENDENCY_CHECK = 0x1
 @Field int NEXUS_IQ_SCAN = 0x2
 @Field int SONARQUBE = 0x4
-@Field int ALL = DEPENDENCY_CHECK | NEXUS_IQ_SCAN | SONARQUBE
+@Field int FROGBOT = 0x8
+@Field int ALL = DEPENDENCY_CHECK | NEXUS_IQ_SCAN | SONARQUBE | FROGBOT
 
 
 def call() {
   BuildData buildData = BuildData.instance
-  if (buildData.runAudit) {
-    utils.timer(
-      {
-        runStage(buildData, getEnabledScanners(buildData))
-      },
-      { long duration ->
-        buildData.time(STAGE_LABEL_AUDIT, duration)
-        log.info "${STAGE_LABEL_AUDIT} completed in ${TimeCategory.minus(new Date(duration), new Date(0))}"
-      }
-    )
-  } else {
-    utils.createStageSkipped(STAGE_LABEL_AUDIT)
-    buildData.time(STAGE_LABEL_AUDIT, 0)
-  }
+
+  utils.timer(
+    {
+      runStage(buildData, getEnabledScanners(buildData))
+    },
+    { long duration ->
+      buildData.time(STAGE_LABEL_AUDIT, duration)
+      log.info "${STAGE_LABEL_AUDIT} completed in ${TimeCategory.minus(new Date(duration), new Date(0))}"
+    }
+  )
+
 }
 
 int getEnabledScanners(BuildData buildData) {
@@ -56,6 +55,7 @@ int getEnabledScanners(BuildData buildData) {
   enabledScanners |= buildData.getBool(RUN_DEPENDENCY_CHECK) ? DEPENDENCY_CHECK : 0
   enabledScanners |= buildData.getBool(RUN_NEXUS_LIFECYCLE) ? NEXUS_IQ_SCAN : 0
   enabledScanners |= buildData.getBool(RUN_SONARQUBE) ? SONARQUBE : 0
+  enabledScanners |= buildData.getBool(RUN_FROGBOT) && buildData.isPullRequest() ? FROGBOT : 0
   return enabledScanners
 }
 
@@ -135,6 +135,9 @@ Map<String, Closure> getEntries(BuildData buildData, JobItem jobItem, int enable
   if (enabledScanners & SONARQUBE) {
     scanners << [(jobItem.jobID + ': SonarQube'): { -> sonar(buildData, jobItem) }]
   }
+  if (enabledScanners & FROGBOT) {
+    scanners << [(jobItem.jobID + ': Frogbot'): { -> frogbot(buildData, jobItem) }]
+  }
 
   return scanners
 }
@@ -184,4 +187,31 @@ void nexusIQScan(/*BuildData buildData, JobItem jobItem*/) {
 void dependencyCheck(/*BuildData buildData, JobItem jobItem*/) {
   //TODO
   log.warn "Not implemented yet"
-}
+}
+
+/**
+ * Perform frogobot's pull request analysis
+ *
+ * @param buildData
+ * @param jobItem
+ */
+void frogbot(BuildData buildData, JobItem jobItem) {
+  Builder builder = BuilderFactory.builderFor(jobItem)
+  Closure execution = builder.frogbotExecution
+
+  // apply retries
+  if (buildData.getInt(BUILD_RETRIES)) {
+    Closure current = execution
+    execution = { -> retry(buildData.getInt(BUILD_RETRIES), current) }
+  }
+
+  // apply container
+  if (jobItem.containerized) {
+    Closure current = execution
+    execution = { -> utils.withContainer(jobItem.dockerImage, current) }
+  }
+
+  utils.timer(execution) { long duration ->
+    buildData.time(STAGE_LABEL_AUDIT, jobItem.jobID + ': Frogbot', duration)
+  }
+}

vars/stages.groovy

@@ -315,6 +315,26 @@ void sonar(String id = 'sonar', String label = '') {
   ).run()
 }
 
+void frogbot(String id = 'frogbot', String label = '') {
+  new ParallelItemWorkStage(id: id, label: label ?: id.capitalize(),
+      ignoreGroups: true,
+      allowMinions: true,
+      itemFilter: { List<JobItem> items ->
+        BuilderUtils.applyChanges(id, items)
+        items.findAll { JobItem item -> !item.skip && item.auditable }
+      },
+      itemExecution: { JobItem item ->
+        BuilderFactory.builderFor(id, item)
+            .getFrogbotExecution()
+            .call()
+      }
+  ).run()
+}
+
+void scans() {
+  audit.call()
+}
+
 void report() {
   new SimpleStage(id: 'report', label: STAGE_LABEL_REPORT,
     body: {