Merge pull request #3102 from jtobard/dev/scm-cluster-config

Scm cluster config

core/src/main/java/com/dtolabs/rundeck/plugins/scm/ScmExportPlugin.java

@@ -129,4 +129,14 @@ ScmExportResult export(
      * @param originalPath original path
      */
     ScmDiffResult getFileDiff(JobExportReference job, String originalPath);
+
+
+    /**
+     * Function to fix status of the jobs on cluster environment.
+     * To automatically match the job status on every node.
+     *
+     * @param jobs rundeck jobs
+     * @return map with information on the process
+     */
+    Map clusterFixJobs(List<JobExportReference> jobs);
 }

core/src/main/java/com/dtolabs/rundeck/plugins/scm/ScmImportPlugin.java

@@ -144,4 +144,14 @@ ScmExportResult scmImport(
      * @param originalPath original path
      */
     ScmImportDiffResult getFileDiff(JobScmReference job, String originalPath);
+
+
+    /**
+     * Function to fix status of the jobs on cluster environment.
+     * To automatically match the job status on every node.
+     *
+     * @param jobs rundeck jobs
+     * @return map with information on the process
+     */
+    Map clusterFixJobs(List<JobScmReference> jobs);
 }

plugins/git-plugin/src/main/groovy/org/rundeck/plugin/scm/git/GitExportPlugin.groovy

@@ -24,6 +24,8 @@ import com.dtolabs.rundeck.core.plugins.views.BasicInputView
 import com.dtolabs.rundeck.plugins.scm.*
 import org.apache.log4j.Logger
 import org.eclipse.jgit.api.Status
+import org.eclipse.jgit.api.errors.GitAPIException
+import org.eclipse.jgit.api.errors.JGitInternalException
 import org.eclipse.jgit.lib.BranchTrackingStatus
 import org.eclipse.jgit.revwalk.RevCommit
 import org.rundeck.plugin.scm.git.config.Export
@@ -419,6 +421,7 @@ class GitExportPlugin extends BaseGitPlugin implements ScmExportPlugin {
         if (!status) {
             status = refreshJobStatus(job, originalPath)
         }
+
         return createJobStatus(status, jobActionsForStatus(status))
     }
 
@@ -463,4 +466,57 @@ class GitExportPlugin extends BaseGitPlugin implements ScmExportPlugin {
     }
 
 
+    Map clusterFixJobs(final List<JobExportReference> jobs){
+        def retSt = [:]
+        retSt.deleted = []
+        retSt.restored = []
+        def toPull = false
+        jobs.each { job ->
+            def storedCommitId = ((JobScmReference)job).scmImportMetadata?.commitId
+            def commitId = lastCommitForPath(getRelativePathForJob(job))
+            def path = getRelativePathForJob(job)
+            if(storedCommitId != null && commitId == null){
+                //file to delete-pull
+                git.rm().addFilepattern(path).call()
+                toPull = true
+                retSt.deleted.add(path)
+            }else if(storedCommitId != null && commitId?.name != storedCommitId){
+                git.checkout().addPath(path).call()
+                toPull = true
+                retSt.restored.add(job)
+            }
+        }
+        Status status = git.status().call()
+        if (status.isClean()) {
+            //behind branch on deleted job
+            def bstat = BranchTrackingStatus.of(repo, branch)
+            if (bstat && bstat.behindCount > 0) {
+                toPull = true
+                retSt.behind = true
+            }
+        }
+        if(toPull){
+            retSt.pull = true
+            try{
+                git.pull().call()
+            }catch (JGitInternalException e){
+                retSt.error=e
+            }catch(GitAPIException e){
+                retSt.error = e
+                log.info(e)
+            }
+        }
+
+        try{
+            jobs.each{job ->
+                refreshJobStatus(job,null)
+            }
+        }catch (ScmPluginException e){
+            retSt.error = e
+        }
+
+        retSt
+    }
+
+
 }

plugins/git-plugin/src/main/groovy/org/rundeck/plugin/scm/git/GitImportPlugin.groovy

@@ -21,6 +21,8 @@ import com.dtolabs.rundeck.core.plugins.views.Action
 import com.dtolabs.rundeck.core.plugins.views.BasicInputView
 import com.dtolabs.rundeck.plugins.scm.*
 import org.apache.log4j.Logger
+import org.eclipse.jgit.api.PullResult
+import org.eclipse.jgit.api.Status
 import org.eclipse.jgit.diff.DiffEntry
 import org.eclipse.jgit.lib.BranchTrackingStatus
 import org.eclipse.jgit.lib.ObjectId
@@ -140,7 +142,7 @@ class GitImportPlugin extends BaseGitPlugin implements ScmImportPlugin {
     }
 
 
-    GitImportSynchState getStatusInternal(ScmOperationContext context, boolean performFetch) {
+    GitImportSynchState   getStatusInternal(ScmOperationContext context, boolean performFetch) {
         //look for any unimported paths
         if (!trackedItemsSelected) {
             return null
@@ -574,4 +576,18 @@ class GitImportPlugin extends BaseGitPlugin implements ScmImportPlugin {
     boolean isTrackedPath(final String path) {
         return trackedItems?.contains(path) || isUseTrackingRegex() && trackingRegex && path.matches(trackingRegex)
     }
+
+
+    Map clusterFixJobs(List<JobScmReference> jobs){
+        Status st = git.status().call()
+        def bstat = BranchTrackingStatus.of(repo, branch)
+        if(st.clean && bstat && bstat.behindCount>0){
+            PullResult result = git.pull().call()
+            jobs.each{job ->
+                refreshJobStatus(job,null)
+            }
+            return [updated:true]
+        }
+        [:]
+    }
 }

plugins/git-plugin/src/test/groovy/org/rundeck/plugin/scm/git/GitExportPluginSpec.groovy

@@ -1424,4 +1424,102 @@ class GitExportPluginSpec extends Specification {
         theEventType                                    | _
         JobChangeEvent.JobChangeEventType.MODIFY_RENAME | _
     }
+
+
+    def "fix cluster status with new files behind"() {
+        given:
+
+        def gitdir = new File(tempdir, 'scm')
+        def origindir = new File(tempdir, 'origin')
+        Export config = createTestConfig(gitdir, origindir, [exportUuidBehavior: 'preserve'])
+
+        //create a git dir
+        def git = createGit(origindir)
+
+        git.close()
+        def plugin = new GitExportPlugin(config)
+        plugin.initialize(Mock(ScmOperationContext))
+
+        def serializer = Mock(JobSerializer)
+        def jobref = Stub(JobScmReference) {
+            getJobName() >> 'name'
+            getGroupPath() >> 'a/b'
+            getId() >> 'abc'
+            getSourceId() >> 'xyz'
+            getScmImportMetadata() >> [commitId:'a']
+        }
+
+        when:
+        def status = plugin.clusterFixJobs([jobref])
+
+        then:
+        status != null
+        status.deleted
+        status.deleted.size() == 1
+        status.deleted[0]=='a/b/name-abc.xml'
+    }
+
+    def "fix cluster status with clean status"() {
+        given:
+
+        def gitdir = new File(tempdir, 'scm')
+        def origindir = new File(tempdir, 'origin')
+        Export config = createTestConfig(gitdir, origindir, [exportUuidBehavior: 'preserve'])
+
+        //create a git dir
+        def git = createGit(origindir)
+
+        git.close()
+        def plugin = new GitExportPlugin(config)
+        plugin.initialize(Mock(ScmOperationContext))
+
+        def serializer = Mock(JobSerializer)
+        def jobref = Stub(JobScmReference) {
+            getJobName() >> 'name'
+            getGroupPath() >> 'a/b'
+            getId() >> 'abc'
+            getSourceId() >> 'xyz'
+            getScmImportMetadata() >> [:]
+        }
+
+        when:
+        def status = plugin.clusterFixJobs([jobref])
+
+        then:
+        status != null
+        status.deleted.size() == 0
+        status.restored.size() == 0
+
+    }
+    def "fix cluster status with modified status"() {
+        given:
+
+        def gitdir = new File(tempdir, 'scm')
+        def origindir = new File(tempdir, 'origin')
+        Export config = createTestConfig(gitdir, origindir, [exportUuidBehavior: 'preserve'])
+
+        //create a git dir
+        def git = createGit(origindir)
+        def commit = addCommitFile(origindir, git, 'a/b/name-abc.xml', 'blah')
+        git.close()
+        def plugin = new GitExportPlugin(config)
+        plugin.initialize(Mock(ScmOperationContext))
+
+        def jobref = Stub(JobScmReference) {
+            getJobName() >> 'name'
+            getGroupPath() >> 'a/b'
+            getId() >> 'abc'
+            getSourceId() >> 'xyz'
+            getScmImportMetadata() >> [commitId:'a']
+        }
+
+        when:
+        def status = plugin.clusterFixJobs([jobref])
+
+        then:
+        status != null
+        status.deleted.size() == 0
+        status.restored.size() == 1
+
+    }
 }

rundeckapp/grails-app/controllers/rundeck/controllers/MenuController.groovy

@@ -429,15 +429,27 @@ class MenuController extends ControllerBase implements ApplicationContextAware{
                                                                  ),
                                                                  [AuthConstants.ACTION_ADMIN, AuthConstants.ACTION_EXPORT]
             )) {
+                if(frameworkService.isClusterModeEnabled()){
+                    if (!scmService.projectHasConfiguredExportPlugin(params.project)) {
+                        //initialize if in another node
+                        scmService.initProject(params.project, 'export')
+                    }
+                    if(minScm){
+                        scmService.fixExportStatus(params.project, results.nextScheduled)
+                    }
+                }
                 def pluginData = [:]
                 try {
                     if (scmService.projectHasConfiguredExportPlugin(params.project)) {
-                        pluginData.scmExportEnabled = true
-                        pluginData.scmExportActions = scmService.exportPluginActions(authContext, params.project)
-                        if(!minScm){
-                            pluginData.scmStatus = scmService.exportStatusForJobs(results.nextScheduled)
-                            pluginData.scmExportStatus = scmService.exportPluginStatus(authContext, params.project)
-                            pluginData.scmExportRenamed = scmService.getRenamedJobPathsForProject(params.project)
+                        pluginData.scmExportEnabled = scmService.loadScmConfig(params.project, 'export').enabled
+                        if(pluginData.scmExportEnabled){
+
+                            if(!minScm){
+                                pluginData.scmStatus = scmService.exportStatusForJobs(results.nextScheduled)
+                                pluginData.scmExportStatus = scmService.exportPluginStatus(authContext, params.project)
+                                pluginData.scmExportRenamed = scmService.getRenamedJobPathsForProject(params.project)
+                            }
+                            pluginData.scmExportActions = scmService.exportPluginActions(authContext, params.project)
                         }
                         results.putAll(pluginData)
                     }
@@ -451,15 +463,27 @@ class MenuController extends ControllerBase implements ApplicationContextAware{
                                                                  ),
                                                                  [AuthConstants.ACTION_ADMIN, AuthConstants.ACTION_IMPORT]
             )) {
-
+                if(frameworkService.isClusterModeEnabled()){
+                    if (!scmService.projectHasConfiguredImportPlugin(params.project)) {
+                        //initialize if in another node
+                        scmService.initProject(params.project, 'import')
+                    }
+                    if(minScm){
+                        scmService.fixImportStatus(params.project, results.nextScheduled)
+                        scmService.importPluginStatus(authContext, params.project)
+                    }
+                }
                 def pluginData = [:]
                 try {
                     if (scmService.projectHasConfiguredImportPlugin(params.project)) {
-                        pluginData.scmImportEnabled = true
-                        pluginData.scmImportActions = scmService.importPluginActions(authContext, params.project)
-                        if(!minScm){
-                            pluginData.scmImportJobStatus = scmService.importStatusForJobs(results.nextScheduled)
-                            pluginData.scmImportStatus = scmService.importPluginStatus(authContext, params.project)
+                        pluginData.scmImportEnabled = scmService.loadScmConfig(params.project, 'import').enabled
+                        if(pluginData.scmImportEnabled){
+
+                            if(!minScm){
+                                pluginData.scmImportJobStatus = scmService.importStatusForJobs(results.nextScheduled)
+                                pluginData.scmImportStatus = scmService.importPluginStatus(authContext, params.project)
+                            }
+                            pluginData.scmImportActions = scmService.importPluginActions(authContext, params.project)
                         }
                         results.putAll(pluginData)
                     }
@@ -2990,13 +3014,19 @@ class MenuController extends ControllerBase implements ApplicationContextAware{
                 [AuthConstants.ACTION_ADMIN, AuthConstants.ACTION_EXPORT]
         )) {
             def pluginData = [:]
+            if(frameworkService.isClusterModeEnabled()){
+                //initialize if in another node
+                scmService.initProject(params.project,'export')
+            }
             try {
                 if (scmService.projectHasConfiguredExportPlugin(params.project)) {
-                    pluginData.scmExportEnabled = true
-                    pluginData.scmStatus = scmService.exportStatusForJobs(result.nextScheduled)
-                    pluginData.scmExportStatus = scmService.exportPluginStatus(authContext, params.project)
-                    pluginData.scmExportActions = scmService.exportPluginActions(authContext, params.project)
-                    pluginData.scmExportRenamed = scmService.getRenamedJobPathsForProject(params.project)
+                    pluginData.scmExportEnabled = scmService.loadScmConfig(params.project, 'export')?.enabled
+                    if(pluginData.scmExportEnabled){
+                        pluginData.scmStatus = scmService.exportStatusForJobs(result.nextScheduled)
+                        pluginData.scmExportStatus = scmService.exportPluginStatus(authContext, params.project)
+                        pluginData.scmExportActions = scmService.exportPluginActions(authContext, params.project)
+                        pluginData.scmExportRenamed = scmService.getRenamedJobPathsForProject(params.project)
+                    }
                     results.putAll(pluginData)
                 }
             } catch (ScmPluginException e) {
@@ -3009,14 +3039,19 @@ class MenuController extends ControllerBase implements ApplicationContextAware{
                 ),
                 [AuthConstants.ACTION_ADMIN, AuthConstants.ACTION_IMPORT]
         )) {
-
+            if(frameworkService.isClusterModeEnabled()){
+                //initialize if in another node
+                scmService.initProject(params.project,'import')
+            }
             def pluginData = [:]
             try {
                 if (scmService.projectHasConfiguredImportPlugin(params.project)) {
-                    pluginData.scmImportEnabled = true
-                    pluginData.scmImportJobStatus = scmService.importStatusForJobs(result.nextScheduled)
-                    pluginData.scmImportStatus = scmService.importPluginStatus(authContext, params.project)
-                    pluginData.scmImportActions = scmService.importPluginActions(authContext, params.project)
+                    pluginData.scmImportEnabled = scmService.loadScmConfig(params.project, 'import')?.enabled
+                    if(pluginData.scmImportEnabled){
+                        pluginData.scmImportJobStatus = scmService.importStatusForJobs(result.nextScheduled)
+                        pluginData.scmImportStatus = scmService.importPluginStatus(authContext, params.project)
+                        pluginData.scmImportActions = scmService.importPluginActions(authContext, params.project)
+                    }
                     results.putAll(pluginData)
                 }