[ISSUE#12994]Optimize config operation.

api/src/main/java/com/alibaba/nacos/api/model/v2/ErrorCode.java

@@ -110,6 +110,26 @@ public enum ErrorCode {
      */
     CONFIG_GRAY_NAME_UNRECOGNIZED_ERROR(20013, "config gray name not recognized"),
 
+    /**
+     * over cluster quota.
+     */
+    OVER_CLUSTER_QUOTA(20014, "Exceeded the maximum number of configurations in the cluster"),
+
+    /**
+     * over group quota.
+     */
+    OVER_GROUP_QUOTA(20015, "Exceeded the maximum number of configurations in this group"),
+
+    /**
+     * over tenant quota.
+     */
+    OVER_TENANT_QUOTA(20016, "Exceeded the maximum number of configurations for this namespaceId"),
+
+    /**
+     * over max content size.
+     */
+    OVER_MAX_SIZE(20017, "Exceeded the maximum size limit of the configuration content"),
+
     /**
      *  service name error.
      */
@@ -217,6 +237,15 @@ public String getMsg() {
         return msg;
     }
 
+    public static ErrorCode getErrorCode(String name) {
+        for (ErrorCode errorCode : ErrorCode.values()) {
+            if (errorCode.name().equals(name)) {
+                return errorCode;
+            }
+        }
+        return null;
+    }
+
     ErrorCode(Integer code, String msg) {
         this.code = code;
         this.msg = msg;

config/src/main/java/com/alibaba/nacos/config/server/constant/Constants.java

@@ -317,6 +317,18 @@ public class Constants {
 
     public static final String GRAY = "gray";
 
+    /**
+     * request source type.
+     */
+    public static final String HTTP = "http";
+
+    public static final String RPC = "rpc";
+
+    /**
+     * Separator.
+     */
+    public static final String COLON =  ":";
+
     static String getPersistEncode() {
         String persistEncode = System.getenv(NACOS_PERSIST_ENCODE_KEY);
         if (StringUtils.isBlank(persistEncode)) {

config/src/main/java/com/alibaba/nacos/config/server/controller/ConfigController.java

@@ -39,7 +39,6 @@
 import com.alibaba.nacos.config.server.model.GroupkeyListenserStatus;
 import com.alibaba.nacos.config.server.model.SameConfigPolicy;
 import com.alibaba.nacos.config.server.model.SampleResult;
-import com.alibaba.nacos.config.server.model.event.ConfigDataChangeEvent;
 import com.alibaba.nacos.config.server.model.form.ConfigForm;
 import com.alibaba.nacos.config.server.model.gray.BetaGrayRule;
 import com.alibaba.nacos.config.server.model.gray.GrayRuleManager;
@@ -48,17 +47,14 @@
 import com.alibaba.nacos.config.server.paramcheck.ConfigDefaultHttpParamExtractor;
 import com.alibaba.nacos.config.server.paramcheck.ConfigListenerHttpParamExtractor;
 import com.alibaba.nacos.config.server.result.code.ResultCodeEnum;
-import com.alibaba.nacos.config.server.service.ConfigChangePublisher;
 import com.alibaba.nacos.config.server.service.ConfigOperationService;
 import com.alibaba.nacos.config.server.service.ConfigSubService;
 import com.alibaba.nacos.config.server.service.repository.ConfigInfoGrayPersistService;
 import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService;
-import com.alibaba.nacos.config.server.service.trace.ConfigTraceService;
 import com.alibaba.nacos.config.server.utils.GroupKey;
 import com.alibaba.nacos.config.server.utils.MD5Util;
 import com.alibaba.nacos.config.server.utils.ParamUtils;
 import com.alibaba.nacos.config.server.utils.RequestUtil;
-import com.alibaba.nacos.config.server.utils.TimeUtils;
 import com.alibaba.nacos.config.server.utils.YamlParserUtil;
 import com.alibaba.nacos.config.server.utils.ZipUtils;
 import com.alibaba.nacos.core.control.TpsControl;
@@ -68,7 +64,6 @@
 import com.alibaba.nacos.plugin.auth.constant.ActionTypes;
 import com.alibaba.nacos.plugin.auth.constant.SignType;
 import com.alibaba.nacos.plugin.encryption.handler.EncryptionHandler;
-import com.alibaba.nacos.sys.utils.InetUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeanUtils;
@@ -90,7 +85,6 @@
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
 import java.net.URLDecoder;
-import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Date;
@@ -210,6 +204,7 @@ public Boolean publishConfig(HttpServletRequest request, HttpServletResponse res
 
         ConfigRequestInfo configRequestInfo = new ConfigRequestInfo();
         configRequestInfo.setSrcIp(RequestUtil.getRemoteIp(request));
+        configRequestInfo.setSrcType(Constants.HTTP);
         configRequestInfo.setRequestIpApp(RequestUtil.getAppName(request));
         configRequestInfo.setBetaIps(request.getHeader("betaIps"));
         configRequestInfo.setCasMd5(request.getHeader("casMd5"));
@@ -295,38 +290,32 @@ public Boolean deleteConfig(HttpServletRequest request, HttpServletResponse resp
         String clientIp = RequestUtil.getRemoteIp(request);
         String srcUser = RequestUtil.getSrcUserName(request);
 
-        return configOperationService.deleteConfig(dataId, group, tenant, tag, clientIp, srcUser);
+        return configOperationService.deleteConfig(dataId, group, tenant, tag, clientIp, srcUser, Constants.HTTP);
     }
 
     /**
-     * Execute delete config operation.
-     *
-     * @return java.lang.Boolean
-     * @author klw
-     * @Description: delete configuration based on multiple config ids
-     * @Date 2019/7/5 10:26
-     * @Param [request, response, dataId, group, tenant, tag]
+     * delete configs based on the IDs list.
      */
     @DeleteMapping(params = "delType=ids")
     @Secured(action = ActionTypes.WRITE, signType = SignType.CONFIG)
     public RestResult<Boolean> deleteConfigs(HttpServletRequest request, @RequestParam(value = "ids") List<Long> ids) {
         String clientIp = RequestUtil.getRemoteIp(request);
         String srcUser = RequestUtil.getSrcUserName(request);
-        final Timestamp time = TimeUtils.getCurrentTime();
-        List<ConfigAllInfo> configInfoList = configInfoPersistService.removeConfigInfoByIds(ids, clientIp, srcUser);
-        if (CollectionUtils.isEmpty(configInfoList)) {
+        try {
+            for (Long id : ids) {
+                ConfigInfo configInfo = configInfoPersistService.findConfigInfo(id);
+                if (configInfo == null) {
+                    LOGGER.warn("[deleteConfigs] configInfo is null, id: {}", id);
+                    continue;
+                }
+                configOperationService.deleteConfig(configInfo.getDataId(), configInfo.getGroup(),
+                        configInfo.getTenant(), null, clientIp, srcUser, Constants.HTTP);
+            }
             return RestResultUtils.success(true);
+        } catch (Exception e) {
+            LOGGER.error("delete configs based on the IDs list error, IDs: {}", ids, e);
+            return RestResultUtils.failed(e.getMessage());
         }
-        for (ConfigAllInfo configInfo : configInfoList) {
-            ConfigChangePublisher.notifyConfigChange(
-                    new ConfigDataChangeEvent(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant(),
-                            time.getTime()));
-
-            ConfigTraceService.logPersistenceEvent(configInfo.getDataId(), configInfo.getGroup(),
-                    configInfo.getTenant(), null, time.getTime(), clientIp, ConfigTraceService.PERSISTENCE_EVENT,
-                    ConfigTraceService.PERSISTENCE_TYPE_REMOVE, null);
-        }
-        return RestResultUtils.success(true);
     }
 
     @GetMapping("/catalog")
@@ -464,7 +453,7 @@ public RestResult<Boolean> stopBeta(HttpServletRequest httpServletRequest,
         String remoteIp = getRemoteIp(httpServletRequest);
         try {
 
-            configOperationService.deleteConfig(dataId, group, tenant, BetaGrayRule.TYPE_BETA, remoteIp, srcUser);
+            configOperationService.deleteConfig(dataId, group, tenant, BetaGrayRule.TYPE_BETA, remoteIp, srcUser, Constants.HTTP);
         } catch (Throwable e) {
             LOGGER.error("remove beta data error", e);
             return RestResultUtils.failed(500, false, "remove beta data error");
@@ -667,26 +656,15 @@ public RestResult<Map<String, Object>> importAndPublishConfig(HttpServletRequest
             failedData.put("succCount", 0);
             return RestResultUtils.buildResult(ResultCodeEnum.DATA_EMPTY, failedData);
         }
-        final String srcIp = RequestUtil.getRemoteIp(request);
-        String requestIpApp = RequestUtil.getAppName(request);
-        final Timestamp time = TimeUtils.getCurrentTime();
-        Map<String, Object> saveResult = configInfoPersistService.batchInsertOrUpdate(configInfoList, srcUser, srcIp,
-                null, policy);
-        for (ConfigInfo configInfo : configInfoList) {
-            ConfigChangePublisher.notifyConfigChange(
-                    new ConfigDataChangeEvent(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant(),
-                            time.getTime()));
-            ConfigTraceService.logPersistenceEvent(configInfo.getDataId(), configInfo.getGroup(),
-                    configInfo.getTenant(), requestIpApp, time.getTime(), InetUtils.getSelfIP(),
-                    ConfigTraceService.PERSISTENCE_EVENT, ConfigTraceService.PERSISTENCE_TYPE_PUB,
-                    configInfo.getContent());
-        }
+
+        Map<String, Object> saveResult = batchImportAndPublishConfigs(configInfoList, request, srcUser, namespace, policy);
+
         // unrecognizedCount
         if (!unrecognizedList.isEmpty()) {
             saveResult.put("unrecognizedCount", unrecognizedList.size());
             saveResult.put("unrecognizedData", unrecognizedList);
         }
-        return RestResultUtils.success("导入成功", saveResult);
+        return RestResultUtils.success("import success", saveResult);
     }
 
     /**
@@ -900,7 +878,6 @@ public RestResult<Map<String, Object>> cloneConfig(HttpServletRequest request,
         for (ConfigAllInfo ci : queryedDataList) {
             SameNamespaceCloneConfigBean paramBean = configBeansMap.get(ci.getId());
             ConfigAllInfo ci4save = new ConfigAllInfo();
-            ci4save.setTenant(namespace);
             ci4save.setType(ci.getType());
             ci4save.setGroup((paramBean != null && StringUtils.isNotBlank(paramBean.getGroup())) ? paramBean.getGroup()
                     : ci.getGroup());
@@ -914,26 +891,93 @@ public RestResult<Map<String, Object>> cloneConfig(HttpServletRequest request,
             ci4save.setDesc(ci.getDesc());
             ci4save.setEncryptedDataKey(
                     ci.getEncryptedDataKey() == null ? StringUtils.EMPTY : ci.getEncryptedDataKey());
+            ParamUtils.checkParam(ci4save.getDataId(), ci4save.getGroup(), "datumId", ci4save.getContent());
             configInfoList4Clone.add(ci4save);
         }
         if (StringUtils.isBlank(srcUser)) {
             srcUser = RequestUtil.getSrcUserName(request);
         }
-        final String srcIp = RequestUtil.getRemoteIp(request);
-        String requestIpApp = RequestUtil.getAppName(request);
-        final Timestamp time = TimeUtils.getCurrentTime();
-        Map<String, Object> saveResult = configInfoPersistService.batchInsertOrUpdate(configInfoList4Clone, srcUser,
-                srcIp, null, policy);
-        for (ConfigInfo configInfo : configInfoList4Clone) {
-            ConfigChangePublisher.notifyConfigChange(
-                    new ConfigDataChangeEvent(configInfo.getDataId(), configInfo.getGroup(), configInfo.getTenant(),
-                            time.getTime()));
-            ConfigTraceService.logPersistenceEvent(configInfo.getDataId(), configInfo.getGroup(),
-                    configInfo.getTenant(), requestIpApp, time.getTime(), InetUtils.getSelfIP(),
-                    ConfigTraceService.PERSISTENCE_EVENT, ConfigTraceService.PERSISTENCE_TYPE_PUB,
-                    configInfo.getContent());
-        }
+        Map<String, Object> saveResult = batchImportAndPublishConfigs(configInfoList4Clone, request, srcUser, namespace, policy);
+
         return RestResultUtils.success("Clone Completed Successfully", saveResult);
     }
 
+    private Map<String, Object> batchImportAndPublishConfigs(List<ConfigAllInfo> configAllInfoList, HttpServletRequest request,
+            String srcUser, String targetNamespaceId, SameConfigPolicy sameConfigPolicy) throws NacosException {
+        Map<String, Object> saveResult = new HashMap<>(16);
+        int succCount = 0;
+        int skipCount = 0;
+        List<Map<String, String>> failData = new ArrayList<>();
+        List<Map<String, String>> skipData = new ArrayList<>();
+        for (int i = 0; i < configAllInfoList.size(); i++) {
+            ConfigAllInfo configAllInfo = configAllInfoList.get(i);
+            ConfigForm configForm = transferToConfigForm(configAllInfo, srcUser, targetNamespaceId);
+            ConfigRequestInfo configRequestInfo = transferToConfigRequestInfo(request);
+            if (sameConfigPolicy != SameConfigPolicy.OVERWRITE) {
+                configRequestInfo.setUpdateForExist(false);
+            }
+            Boolean importRes = configOperationService.publishConfig(configForm, configRequestInfo, configAllInfo.getEncryptedDataKey());
+            if (importRes) {
+                succCount++;
+            } else if (SameConfigPolicy.SKIP == sameConfigPolicy) {
+                skipCount++;
+                Map<String, String> skipItem = new HashMap<>(2);
+                skipItem.put("dataId", configAllInfo.getDataId());
+                skipItem.put("group", configAllInfo.getGroup());
+                skipData.add(skipItem);
+            } else if (SameConfigPolicy.ABORT == sameConfigPolicy) {
+                Map<String, String> failedItem = new HashMap<>(2);
+                failedItem.put("dataId", configAllInfo.getDataId());
+                failedItem.put("group", configAllInfo.getGroup());
+                failData.add(failedItem);
+                // skip remaining configs
+                for (int j = i + 1; j < configAllInfoList.size(); j++) {
+                    ConfigAllInfo skipConfigInfo = configAllInfoList.get(j);
+                    Map<String, String> skipitem = new HashMap<>(2);
+                    skipitem.put("dataId", skipConfigInfo.getDataId());
+                    skipitem.put("group", skipConfigInfo.getGroup());
+                    skipData.add(skipitem);
+                    skipCount++;
+                }
+                break;
+            }
+        }
+
+        saveResult.put("succCount", succCount);
+        saveResult.put("skipCount", skipCount);
+        if (!failData.isEmpty()) {
+            saveResult.put("failData", failData);
+        }
+        if (!skipData.isEmpty()) {
+            saveResult.put("skipData", skipData);
+        }
+
+        return saveResult;
+    }
+
+    private ConfigForm transferToConfigForm(ConfigAllInfo configInfo, String srcUser, String targetNamespaceId) {
+        ConfigForm configForm = new ConfigForm();
+        configForm.setDataId(configInfo.getDataId());
+        configForm.setGroup(configInfo.getGroup());
+        configForm.setNamespaceId(targetNamespaceId);
+        configForm.setContent(configInfo.getContent());
+        configForm.setAppName(configInfo.getAppName());
+        configForm.setConfigTags(configInfo.getConfigTags());
+        configForm.setDesc(configInfo.getDesc());
+        configForm.setUse(configInfo.getUse());
+        configForm.setEffect(configInfo.getEffect());
+        configForm.setType(configInfo.getType());
+        configForm.setSchema(configInfo.getSchema());
+        configForm.setEncryptedDataKey(configInfo.getEncryptedDataKey());
+        configForm.setSrcUser(srcUser);
+        return configForm;
+    }
+
+    private ConfigRequestInfo transferToConfigRequestInfo(HttpServletRequest request) {
+        ConfigRequestInfo configRequestInfo = new ConfigRequestInfo();
+        configRequestInfo.setSrcIp(RequestUtil.getRemoteIp(request));
+        configRequestInfo.setSrcType(Constants.HTTP);
+        configRequestInfo.setRequestIpApp(RequestUtil.getAppName(request));
+        return configRequestInfo;
+    }
 }

config/src/main/java/com/alibaba/nacos/config/server/controller/v2/ConfigControllerV2.java

@@ -142,6 +142,7 @@ public Result<Boolean> publishConfig(ConfigForm configForm, HttpServletRequest r
 
         ConfigRequestInfo configRequestInfo = new ConfigRequestInfo();
         configRequestInfo.setSrcIp(RequestUtil.getRemoteIp(request));
+        configRequestInfo.setSrcType(Constants.HTTP);
         configRequestInfo.setRequestIpApp(RequestUtil.getAppName(request));
         configRequestInfo.setBetaIps(request.getHeader("betaIps"));
         configRequestInfo.setCasMd5(request.getHeader("casMd5"));
@@ -169,7 +170,7 @@ public Result<Boolean> deleteConfig(HttpServletRequest request, @RequestParam("d
 
         String clientIp = RequestUtil.getRemoteIp(request);
         String srcUser = RequestUtil.getSrcUserName(request);
-        return Result.success(configOperationService.deleteConfig(dataId, group, namespaceId, tag, clientIp, srcUser));
+        return Result.success(configOperationService.deleteConfig(dataId, group, namespaceId, tag, clientIp, srcUser, Constants.HTTP));
     }
 
     /**