reslve issue :datavane/tis#289 add reade header error message when en…

…count header with data formate
qlangtech · Feb 26, 2024 · 026932d · 026932d
1 parent 1807a07
commit 026932d
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 9 deletions.
diff --git a/...tis-datax-common-plugin/src/main/java/com/qlangtech/tis/plugin/datax/ParseColsResult.java b/...tis-datax-common-plugin/src/main/java/com/qlangtech/tis/plugin/datax/ParseColsResult.java
@@ -66,15 +66,15 @@ public static ParseColsResult parseColsResult(String tabName, FileFormat.FileHea
         List<String> headerCols = null;
         if (containHeader) {
             headerCols = fileHeader.getHeaderCols();
+            if (fileHeader.getTypes().size() != headerCols.size()) {
+                throw new IllegalStateException("fileHeader.getTypes().size():"
+                        + fileHeader.getTypes().size()
+                        + " is not equal with fileHeader.getHeaderCols().size():"
+                        + headerCols.size()
+                        + " colCount:" + fileHeader.colCount);
+            }
         }
         DataXColMeta colMeta = null;
-        if (fileHeader.getTypes().size() != fileHeader.getHeaderCols().size()) {
-            throw new IllegalStateException("fileHeader.getTypes().size():"
-                    + fileHeader.getTypes().size()
-                    + " is not equal with fileHeader.getHeaderCols().size():"
-                    + fileHeader.getHeaderCols().size()
-                    + " colCount:" + fileHeader.colCount);
-        }
         List<DataType> types = fileHeader.getTypes();
         for (int index = 0; index < fileHeader.colCount; index++) {
             colMeta = new DataXColMeta(types.get(index));

diff --git a/...tax/tis-datax-dfs-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXDFSReader.java b/...tax/tis-datax-dfs-plugin/src/main/java/com/qlangtech/tis/plugin/datax/DataXDFSReader.java
@@ -55,7 +55,7 @@ public static List<? extends Descriptor> dfsLinkerFilter(List<? extends Descript
             return Collections.emptyList();
         }
         return descs.stream().filter((desc) -> {
-           return !(desc instanceof IExclusiveTDFSType);
+            return !(desc instanceof IExclusiveTDFSType);
         }).collect(Collectors.toList());
     }
 
@@ -113,7 +113,18 @@ protected boolean verify(IControlMsgHandler msgHandler, Context context, PostFor
         }
 
         protected boolean verify(IControlMsgHandler msgHandler, Context context, PostFormVals postFormVals, boolean showRecognizedResMsg) {
-            AbstractDFSReader dataxReader = (AbstractDFSReader) postFormVals.newInstance();
+            AbstractDFSReader dataxReader = postFormVals.newInstance();
+
+            //try {
+                //
+                /**
+                 * 当reader header读取，但是在head 无法正确读取到colName 则在此处会抛异常
+                 * @see CSVFormat#setCsvReaderConfig 会抛出TisException
+                 */
+//                dataxReader.getSelectedTabs();
+//            } catch (Exception e) {
+//                throw new RuntimeException(e);
+//            }
 
             Set<ITDFSSession.Res> matchRes = dataxReader.dfsLinker.useTdfsSession((session) -> {
                 return dataxReader.resMatcher.findAllRes(session);

diff --git a/...x/tis-datax-dfs-plugin/src/main/java/com/qlangtech/tis/plugin/datax/format/CSVFormat.java b/...x/tis-datax-dfs-plugin/src/main/java/com/qlangtech/tis/plugin/datax/format/CSVFormat.java
@@ -25,7 +25,10 @@
 import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.TypeReference;
 import com.csvreader.CsvReader;
+import com.google.common.collect.Lists;
 import com.qlangtech.tis.extension.TISExtension;
+import com.qlangtech.tis.lang.TisException;
+import com.qlangtech.tis.plugin.ValidatorCommons;
 import com.qlangtech.tis.plugin.annotation.FormField;
 import com.qlangtech.tis.plugin.annotation.FormFieldType;
 import com.qlangtech.tis.plugin.datax.common.PluginFieldValidators;
@@ -46,6 +49,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.regex.Matcher;
 
 /**
  * @author: 百岁（[email protected]）
@@ -142,6 +146,14 @@ protected void setCsvReaderConfig(CsvReader csvReader) {
                     Objects.requireNonNull(csvReader, "csvFormat.reader can not be null");
                     if (csvReader.readHeaders()) {
                         this.fileHeader = csvReader.getHeaders();// new FileHeader(csvReader.getHeaderCount(), Lists.newArrayList());
+                        List<String> headers = Lists.newArrayList(this.fileHeader);
+                        Matcher matcher = null;
+                        for (String colName : headers) {
+                            matcher = ValidatorCommons.PATTERN_DB_COL_NAME.matcher(colName);
+                            if (!matcher.matches()) {
+                                throw TisException.create("invalid file header:" + String.join(",", headers));
+                            }
+                        }
                     } else {
                         throw new IllegalStateException("must contain head");
                     }