Merge pull request jenkinsci#2 from LinuxSuRen/auto-load

Add support to backup and restore automatically

README.md

@@ -136,6 +136,7 @@ You can find more documentation about JCasC here:
 - [Exporting configurations](./docs/features/configExport.md)
 - [Validating configurations](./docs/features/jsonSchema.md)
 - [Triggering Configuration Reload](./docs/features/configurationReload.md)
+- [Auto backup](./docs/features/auto-backup.md)
 
 The configuration file format depends on the version of jenkins-core and installed plugins.
 Documentation is generated from a live instance, as well as a JSON schema you can use to validate configuration file

docs/features/auto-backup.md

@@ -0,0 +1,39 @@
+This feature provides a solution to allow users to upgrade their Jenkins Configuration-as-Code config file.
+
+## Use case
+
+For the users who wants to build a Jenkins distribution, configuration-as-code could be a good
+ option to provide a initial configuration which lets Jenkins has the feature of out-of-the-box.
+
+But there's one problem here, after the Jenkins distribution runs for a while. User must wants to
+ change the configuration base on his use case. So there're two YAML config files needed.
+ One is the initial one which we call it `system.yaml` here, another one belongs to user's data
+ which is `user.yaml`.
+
+The behaviour of generating the user's configuration automatically is still
+ [working in progress](https://github.com/jenkinsci/configuration-as-code-plugin/pull/1218).
+
+## How does it work?
+
+First, check if there's a new version of the initial config file which is
+ `${JENKINS_HOME}/war/jenkins.yaml`. If there isn't, skip all the following steps.
+
+Second, check if there's a user data file. If it exists, than calculate the diff between
+ the previous config file and the user file. Or just replace the old file simply and skip
+ all the following steps.
+
+Third, apply the patch into the new config file as the result of user file.
+
+Finally, replace the old config file with the new one and delete the new config file.
+
+We deal with three config files:
+
+|Config file path|Description|
+|---|---|
+|`${JENKINS_HOME}/war/jenkins.yaml`|Initial config file, put the new config files in here|
+|`${JENKINS_HOME}/war/WEB-INF/jenkins.yaml`|Should be the last version of config file|
+|`${JENKINS_HOME}/war/WEB-INF/jenkins.yaml.d/user.yaml`|All current config file, auto generate it when a user change the config|
+
+## TODO
+
+- let the name of config file can be configurable

plugin/pom.xml

@@ -69,6 +69,23 @@
       <version>0.10.2</version>
     </dependency>
 
+
+    <dependency>
+      <groupId>com.flipkart.zjsonpatch</groupId>
+      <artifactId>zjsonpatch</artifactId>
+      <version>0.4.9</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <version>2.10.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>2.10.1</version>
+    </dependency>
+
     <dependency>
       <groupId>com.github.stefanbirkner</groupId>
       <artifactId>system-rules</artifactId>

plugin/src/main/java/io/jenkins/plugins/casc/ConfigurationContext.java

@@ -17,6 +17,7 @@ public class ConfigurationContext implements ConfiguratorRegistry {
     private Deprecation deprecation = Deprecation.reject;
     private Restriction restriction = Restriction.reject;
     private Unknown unknown = Unknown.reject;
+    private boolean enableBackup = false;
 
     /**
      * the model-introspection model to be applied by configuration-as-code.
@@ -50,6 +51,10 @@ public void warning(@NonNull CNode node, @NonNull String message) {
 
     public Unknown getUnknown() { return unknown; }
 
+    public boolean isEnableBackup() {
+        return enableBackup;
+    }
+
     public void setDeprecated(Deprecation deprecation) {
         this.deprecation = deprecation;
     }
@@ -62,8 +67,11 @@ public void setUnknown(Unknown unknown) {
         this.unknown = unknown;
     }
 
+    public void setEnableBackup(boolean enableBackup) {
+        this.enableBackup = enableBackup;
+    }
 
-    // --- delegate methods for ConfigurationContext
+// --- delegate methods for ConfigurationContext
 
 
     @Override

plugin/src/main/java/io/jenkins/plugins/casc/auto/CasCBackup.java

@@ -0,0 +1,86 @@
+package io.jenkins.plugins.casc.auto;
+
+import hudson.Extension;
+import hudson.XmlFile;
+import hudson.model.Saveable;
+import hudson.model.listeners.SaveableListener;
+import io.jenkins.plugins.casc.ConfigurationAsCode;
+import io.jenkins.plugins.casc.ConfigurationContext;
+import io.jenkins.plugins.casc.impl.DefaultConfiguratorRegistry;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.inject.Inject;
+import javax.servlet.ServletContext;
+import jenkins.model.GlobalConfiguration;
+import jenkins.model.Jenkins;
+
+@Extension(ordinal = 100)
+public class CasCBackup extends SaveableListener {
+    private static final Logger LOGGER = Logger.getLogger(CasCBackup.class.getName());
+
+    private static final String DEFAULT_JENKINS_YAML_PATH = "jenkins.yaml";
+    private static final String cascDirectory = "/WEB-INF/" + DEFAULT_JENKINS_YAML_PATH + ".d/";
+
+    @Inject
+    private DefaultConfiguratorRegistry registry;
+
+    @Override
+    public void onChange(Saveable o, XmlFile file) {
+        ConfigurationContext context = new ConfigurationContext(registry);
+        if (!context.isEnableBackup()) {
+            return;
+        }
+
+        // only take care of the configuration which controlled by casc
+        if (!(o instanceof GlobalConfiguration)) {
+            return;
+        }
+
+        ByteArrayOutputStream buf = new ByteArrayOutputStream();
+        try {
+            ConfigurationAsCode.get().export(buf);
+        } catch (Exception e) {
+            LOGGER.log(Level.WARNING, "error happen when exporting the whole config into a YAML", e);
+            return;
+        }
+
+        final ServletContext servletContext = Jenkins.getInstance().servletContext;
+        try {
+            URL bundled = servletContext.getResource(cascDirectory);
+            if (bundled != null) {
+                File cascDir = new File(bundled.getFile());
+
+                boolean hasDir = false;
+                if(!cascDir.exists()) {
+                    hasDir = cascDir.mkdirs();
+                } else if (cascDir.isFile()) {
+                    LOGGER.severe(String.format("%s is a regular file", cascDir));
+                } else {
+                    hasDir = true;
+                }
+
+                if(hasDir) {
+                    File backupFile = new File(cascDir, "user.yaml");
+                    try (OutputStream writer = new FileOutputStream(backupFile)) {
+                        writer.write(buf.toByteArray());
+
+                        LOGGER.fine(String.format("backup file was saved, %s", backupFile.getAbsolutePath()));
+                    } catch (IOException e) {
+                        LOGGER.log(Level.WARNING, String.format("error happen when saving %s", backupFile.getAbsolutePath()), e);
+                    }
+                } else {
+                    LOGGER.severe(String.format("cannot create casc backup directory %s", cascDir));
+                }
+            }
+        } catch (MalformedURLException e) {
+            LOGGER.log(Level.WARNING, String.format("error happen when finding %s", cascDirectory), e);
+        }
+    }
+}

plugin/src/main/java/io/jenkins/plugins/casc/auto/PatchConfig.java

@@ -0,0 +1,135 @@
+package io.jenkins.plugins.casc.auto;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.flipkart.zjsonpatch.JsonDiff;
+import com.flipkart.zjsonpatch.JsonPatch;
+import hudson.init.InitMilestone;
+import hudson.init.Initializer;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import javax.servlet.ServletContext;
+import jenkins.model.Jenkins;
+import org.apache.commons.io.IOUtils;
+
+/**
+ * Apply the patch between two versions of the initial config files
+ */
+public class PatchConfig {
+    private static final Logger LOGGER = Logger.getLogger(CasCBackup.class.getName());
+
+    final static  String DEFAULT_JENKINS_YAML_PATH = "jenkins.yaml";
+    final static String cascFile = "/WEB-INF/" + DEFAULT_JENKINS_YAML_PATH;
+    final static String cascDirectory = "/WEB-INF/" + DEFAULT_JENKINS_YAML_PATH + ".d/";
+    final static  String cascUserConfigFile = "user.yaml";
+
+    @Initializer(after= InitMilestone.STARTED, fatal=false)
+    public static void patchConfig() {
+        LOGGER.fine("start to calculate the patch of casc");
+
+        URL newSystemConfig = findConfig("/" + DEFAULT_JENKINS_YAML_PATH);
+        URL systemConfig = findConfig(cascFile);
+        URL userConfig = findConfig(cascDirectory + cascUserConfigFile);
+        URL userConfigDir = findConfig(cascDirectory);
+
+        if (newSystemConfig == null || userConfigDir == null) {
+            LOGGER.warning("no need to upgrade the configuration of Jenkins");
+            return;
+        }
+
+        JsonNode patch = null;
+        if (systemConfig != null && userConfig != null) {
+            ObjectMapper objectMapper = new ObjectMapper();
+            try {
+                JsonNode source = objectMapper.readTree(yamlToJson(systemConfig.openStream()));
+                JsonNode target = objectMapper.readTree(yamlToJson(userConfig.openStream()));
+
+                patch = JsonDiff.asJson(source, target);
+            } catch (IOException e) {
+                LOGGER.log(Level.SEVERE, "error happen when calculate the patch", e);
+                return;
+            }
+
+            try {
+                // give systemConfig a real path
+                PatchConfig.copyAndDelSrc(newSystemConfig, systemConfig);
+            } catch (IOException e) {
+                LOGGER.log(Level.SEVERE, "error happen when copy the new system config", e);
+                return;
+            }
+        }
+
+        if (patch != null) {
+            File userYamlFile = new File(userConfigDir.getFile(), "user.yaml");
+            File userJSONFile = new File(userConfigDir.getFile(), "user.json");
+
+            try (InputStream newSystemInput = systemConfig.openStream();
+                 OutputStream userFileOutput = new FileOutputStream(userYamlFile);
+                 OutputStream patchFileOutput = new FileOutputStream(userJSONFile)){
+                ObjectMapper jsonReader = new ObjectMapper();
+                JsonNode target = JsonPatch.apply(patch, jsonReader.readTree(yamlToJson(newSystemInput)));
+
+                String userYaml = jsonToYaml(new ByteArrayInputStream(target.toString().getBytes()));
+
+                userFileOutput.write(userYaml.getBytes());
+                patchFileOutput.write(patch.toString().getBytes());
+            } catch (Exception e) {
+                LOGGER.log(Level.SEVERE, "error happen when copy the new system config", e);
+            }
+        } else {
+            LOGGER.warning("there's no patch of casc");
+        }
+    }
+
+    private static URL findConfig(String path) {
+        final ServletContext servletContext = Jenkins.getInstance().servletContext;
+        try {
+            return servletContext.getResource(path);
+        } catch (IOException e) {
+            LOGGER.log(Level.SEVERE, String.format("error happen when finding path %s", path), e);
+        }
+        return null;
+    }
+
+    private static void copyAndDelSrc(URL src, URL target) throws IOException {
+        try {
+            PatchConfig.copy(src, target);
+        } finally {
+            boolean result = new File(src.getFile()).delete();
+            LOGGER.fine("src file delete " + result);
+        }
+    }
+
+    private static void copy(URL src, URL target) throws IOException {
+        try (InputStream input = src.openStream();
+            OutputStream output = new FileOutputStream(target.getFile())) {
+            IOUtils.copy(input, output);
+        }
+    }
+
+    private static String jsonToYaml(InputStream input) throws IOException {
+        ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
+        ObjectMapper jsonReader = new ObjectMapper();
+
+        Object obj = jsonReader.readValue(input, Object.class);
+
+        return yamlReader.writeValueAsString(obj);
+    }
+
+    private  static String yamlToJson(InputStream input) throws IOException {
+        ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
+        ObjectMapper jsonReader = new ObjectMapper();
+
+        Object obj = yamlReader.readValue(input, Object.class);
+
+        return jsonReader.writeValueAsString(obj);
+    }
+}