Feature: required columns, column alias binding (#157)

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [0.14.0] - 2020-05-25
+
+### Added
+- support for required and optional columns: `isRequiredColumn` in  `ImportAttributeMapper`
+
 ## [0.13.0] - 2020-05-15
 
 ### Bugfix

README.md

@@ -406,6 +406,9 @@ An attribute mapping contains the following information:
 * column name in the import file
 * column number (only relevant for CSV / Excel)
 * entity attribute
+* column required flag
+
+When column required flag is set to true, addon validates that the column name exists in the imported file. It doesn't check their content, so you still need to validate that additionally. 
 
 #### Auto Detection of Entity Attribute Mappings
 

gradle.properties

@@ -1 +1 @@
-version=0.13.1-SNAPSHOT
+version=0.14.0

modules/core/db/init/hsql/10.create-db.sql

@@ -42,6 +42,7 @@ create table DDCDI_IMPORT_ATTRIBUTE_MAPPER (
     ASSOCIATION_LOOKUP_ATTRIBUTE varchar(255),
     FILE_COLUMN_NUMBER integer not null,
     FILE_COLUMN_ALIAS varchar(255),
+    IS_REQUIRED_COLUMN boolean,
     CUSTOM_ATTRIBUTE_BIND_SCRIPT longvarchar,
     --
     primary key (ID)

modules/core/db/init/mysql/10.create-db.sql

@@ -42,6 +42,7 @@ create table DDCDI_IMPORT_ATTRIBUTE_MAPPER (
     ASSOCIATION_LOOKUP_ATTRIBUTE varchar(255),
     FILE_COLUMN_NUMBER integer not null,
     FILE_COLUMN_ALIAS varchar(255),
+    IS_REQUIRED_COLUMN boolean,
     CUSTOM_ATTRIBUTE_BIND_SCRIPT longtext,
     --
     primary key (ID)

modules/core/db/init/postgres/10.create-db.sql

@@ -42,6 +42,7 @@ create table DDCDI_IMPORT_ATTRIBUTE_MAPPER (
     ASSOCIATION_LOOKUP_ATTRIBUTE varchar(255),
     FILE_COLUMN_NUMBER integer not null,
     FILE_COLUMN_ALIAS varchar(255),
+    IS_REQUIRED_COLUMN boolean,
     CUSTOM_ATTRIBUTE_BIND_SCRIPT text,
     --
     primary key (ID)

modules/core/db/update/hsql/20/200522-2-updateImportAttributeMapper.sql

@@ -0,0 +1 @@
+alter table DDCDI_IMPORT_ATTRIBUTE_MAPPER add column IS_REQUIRED_COLUMN boolean ;

modules/core/db/update/mysql/20/200527-2-updateImportAttributeMapper.sql

@@ -0,0 +1 @@
+alter table DDCDI_IMPORT_ATTRIBUTE_MAPPER add column IS_REQUIRED_COLUMN boolean ;

modules/core/db/update/postgres/20/200527-2-updateImportAttributeMapper.sql

@@ -0,0 +1 @@
+alter table DDCDI_IMPORT_ATTRIBUTE_MAPPER add column IS_REQUIRED_COLUMN boolean ;

modules/core/src/de/diedavids/cuba/dataimport/binding/AttributeBindRequest.groovy

@@ -28,8 +28,11 @@ class AttributeBindRequest {
 
 
     String getRawValue() {
-        def columnValue = dataRow[importAttributeMapper.fileColumnAlias] ?: ''
-        ((String) columnValue).trim()
+        def alias = importAttributeMapper.fileColumnAlias
+        if (importAttributeMapper.isRequiredColumn && !dataRow.hasProperty(alias))
+            throw new MissingPropertyException(alias)
+
+        ((String) dataRow[alias]).trim()
     }
 
     String getImportEntityClassName() {

modules/global/src/de/diedavids/cuba/dataimport/converter/ExcelReader.groovy

@@ -46,7 +46,7 @@ class ExcelReader {
         def linesRead = 0
 
         if (params.labels) {
-            labels = rowIterator.next().collect { it.toString().toLowerCase() }
+            labels = rowIterator.next().collect { it.toString().toLowerCase().replace('.', '') }
         }
         offset.times { rowIterator.next() }
 

modules/global/src/de/diedavids/cuba/dataimport/converter/ImportAttributeMapperCreator.groovy

@@ -40,6 +40,7 @@ class ImportAttributeMapperCreator {
         result.attributeType = attrType
         result.fileColumnAlias = column
         result.fileColumnNumber = fileColumnNumber
+        result.isRequiredColumn = true
         result
     }
 

modules/global/src/de/diedavids/cuba/dataimport/dto/ColumnValidationResult.java

@@ -0,0 +1,33 @@
+package de.diedavids.cuba.dataimport.dto;
+
+import java.util.List;
+
+public class ColumnValidationResult {
+    public ColumnValidationResult(boolean valid, List<String> columns) {
+        this.valid = valid;
+        this.columns = columns;
+    }
+
+    public boolean getValid() {
+        return valid;
+    }
+
+    public boolean isValid() {
+        return valid;
+    }
+
+    public void setValid(boolean valid) {
+        this.valid = valid;
+    }
+
+    public List<String> getColumns() {
+        return columns;
+    }
+
+    public void setColumns(List<String> columns) {
+        this.columns = columns;
+    }
+
+    private boolean valid;
+    private List<String> columns;
+}

modules/global/src/de/diedavids/cuba/dataimport/dto/DataRowImpl.groovy

@@ -43,15 +43,19 @@ class DataRowImpl implements DataRow {
         if (index != null) {
             values[index]
         } else {
-            throw new MissingPropertyException(name)
+            ''
         }
     }
 
+    def hasProperty(String name) {
+        def index = columns[name]
+        index != null
+    }
+
     def getAt(Integer index) {
         values[index]
     }
 
-
     String toString() {
         columns.collect { key, index -> "$key: ${values[index]}" }.join(', ')
     }

modules/global/src/de/diedavids/cuba/dataimport/dto/ImportData.java

@@ -1,6 +1,5 @@
 package de.diedavids.cuba.dataimport.dto;
 
-import de.diedavids.cuba.dataimport.entity.ImportConfiguration;
 import de.diedavids.cuba.dataimport.entity.attributemapper.ImportAttributeMapper;
 
 import java.io.Serializable;
@@ -11,5 +10,5 @@ public interface ImportData extends Serializable {
     List<DataRow> getRows();
     List<String> getColumns();
 
-    boolean isCompatibleWith(List<ImportAttributeMapper> attributeMappers);
+    ColumnValidationResult isCompatibleWith(List<ImportAttributeMapper> attributeMappers);
 }

modules/global/src/de/diedavids/cuba/dataimport/dto/ImportDataImpl.groovy

@@ -7,9 +7,15 @@ class ImportDataImpl implements ImportData {
     List<String> columns = []
 
     @Override
-    boolean isCompatibleWith(List<ImportAttributeMapper> attributeMappers) {
-        attributeMappers.every { attributeMapper ->
-            columns[attributeMapper.fileColumnNumber] == attributeMapper.fileColumnAlias
-        }
+    ColumnValidationResult isCompatibleWith(List<ImportAttributeMapper> attributeMappers) {
+        List<String> invalidColumns = attributeMappers
+                .findAll { attributeMapper -> isInvalidColumn(attributeMapper)}
+                .collect { it.fileColumnAlias }
+
+        new ColumnValidationResult(invalidColumns.isEmpty(), invalidColumns)
+    }
+
+    private boolean isInvalidColumn(ImportAttributeMapper attributeMapper) {
+        attributeMapper.isRequiredColumn && !columns.contains(attributeMapper.fileColumnAlias)
     }
 }

modules/global/src/de/diedavids/cuba/dataimport/entity/attributemapper/ImportAttributeMapper.java

@@ -1,20 +1,17 @@
 package de.diedavids.cuba.dataimport.entity.attributemapper;
 
-import javax.annotation.PostConstruct;
-import javax.persistence.Entity;
-import javax.persistence.Table;
-import javax.persistence.Column;
-import javax.validation.constraints.NotNull;
-
-import com.haulmont.chile.core.datatypes.FormatStrings;
 import com.haulmont.cuba.core.entity.StandardEntity;
 import de.diedavids.cuba.dataimport.entity.ImportConfiguration;
-import de.diedavids.cuba.dataimport.entity.ImportTransactionStrategy;
 
+import javax.annotation.PostConstruct;
+import javax.persistence.Column;
+import javax.persistence.Entity;
 import javax.persistence.FetchType;
 import javax.persistence.JoinColumn;
-import javax.persistence.ManyToOne;
 import javax.persistence.Lob;
+import javax.persistence.ManyToOne;
+import javax.persistence.Table;
+import javax.validation.constraints.NotNull;
 
 @Table(name = "DDCDI_IMPORT_ATTRIBUTE_MAPPER")
 @Entity(name = "ddcdi$ImportAttributeMapper")
@@ -48,11 +45,21 @@ public class ImportAttributeMapper extends StandardEntity {
     protected String fileColumnAlias;
 
 
+    @Column(name = "IS_REQUIRED_COLUMN")
+    protected Boolean isRequiredColumn;
 
     @Lob
     @Column(name = "CUSTOM_ATTRIBUTE_BIND_SCRIPT")
     protected String customAttributeBindScript;
 
+    public Boolean getIsRequiredColumn() {
+        return isRequiredColumn;
+    }
+
+    public void setIsRequiredColumn(Boolean isRequiredColumn) {
+        this.isRequiredColumn = isRequiredColumn;
+    }
+
     public void setMapperMode(AttributeMapperMode mapperMode) {
         this.mapperMode = mapperMode == null ? null : mapperMode.getId();
     }

modules/global/src/de/diedavids/cuba/dataimport/entity/attributemapper/messages.properties

@@ -10,6 +10,7 @@ ImportAttributeMapper.attributeType = Attribute type
 ImportAttributeMapper.associationLookupAttribute = Association lookup attribute
 ImportAttributeMapper.mapperMode = Mode
 
+ImportAttributeMapper.isRequiredColumn=Column required
 AttributeType.DYNAMIC_ATTRIBUTE = Dynamic Attribute
 AttributeType.ASSOCIATION_ATTRIBUTE = Association Attribute
 AttributeType.DIRECT_ATTRIBUTE = Direct Attribute

modules/global/test/de/diedavids/cuba/dataimport/dto/ImportDataImplSpec.groovy

@@ -1,7 +1,6 @@
 package de.diedavids.cuba.dataimport.dto
 
 import de.diedavids.cuba.dataimport.converter.CsvImportDataConverter
-import de.diedavids.cuba.dataimport.entity.ImportConfiguration
 import de.diedavids.cuba.dataimport.entity.attributemapper.AttributeType
 import de.diedavids.cuba.dataimport.entity.attributemapper.ImportAttributeMapper
 import spock.lang.Specification
@@ -11,7 +10,7 @@ class ImportDataImplSpec extends Specification {
     ImportDataImpl sut
 
 
-    def "ImportData is compatible with attribute mappers, if all headers match an entry in the attribute mappers fileColumnAlias"() {
+    def "ImportData is compatible with attribute mappers, if all headers match an entry in the required attribute mappers fileColumnAlias"() {
 
         given:
 
@@ -20,15 +19,17 @@ Mark,Andersson
 Pete,Hansen''')
 
         def attributeMappers = [
-                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name', fileColumnAlias: 'name', fileColumnNumber: 0),
-                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname', fileColumnAlias: 'lastname', fileColumnNumber: 1),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name',
+                        fileColumnAlias: 'name', fileColumnNumber: 0, isRequiredColumn: true),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname',
+                        fileColumnAlias: 'lastname', fileColumnNumber: 1, isRequiredColumn: true),
         ]
 
         expect:
-        sut.isCompatibleWith(attributeMappers)
+        sut.isCompatibleWith(attributeMappers).isValid()
     }
 
-    def "ImportData is incompatible with attribute mappers, if one header does not match any entry in the attribute mappers fileColumnAlias"() {
+    def "ImportData is incompatible with attribute mappers, if one header does not match any entry in the required attribute mappers fileColumnAlias"() {
 
         given:
 
@@ -37,16 +38,18 @@ Mark,Andersson
 Pete,Hansen''')
 
         def attributeMappers = [
-                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name', fileColumnAlias: 'name', fileColumnNumber: 0),
-                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname', fileColumnAlias: 'firstName', fileColumnNumber: 1),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name',
+                        fileColumnAlias: 'name', fileColumnNumber: 0, isRequiredColumn: true),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname',
+                        fileColumnAlias: 'firstName', fileColumnNumber: 1, isRequiredColumn: true),
         ]
 
         expect:
-        !sut.isCompatibleWith(attributeMappers)
+        !sut.isCompatibleWith(attributeMappers).isValid()
     }
 
 
-    def "ImportData is incompatible with attribute mappers, if one header does not match any entry in the attribute mappers fileColumnNumber"() {
+    def "ImportData is compatible with attribute mappers, even if one header does not match any entry in the required attribute mappers fileColumnNumber"() {
 
         given:
 
@@ -55,11 +58,53 @@ Mark,Andersson
 Pete,Hansen''')
 
         def attributeMappers = [
-                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname', fileColumnAlias: 'lastname', fileColumnNumber: 0),
-                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name', fileColumnAlias: 'name', fileColumnNumber: 1),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname',
+                        fileColumnAlias: 'lastname', fileColumnNumber: 0, isRequiredColumn: true),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name',
+                        fileColumnAlias: 'name', fileColumnNumber: 1, isRequiredColumn: true),
         ]
 
         expect:
-        !sut.isCompatibleWith(attributeMappers)
+        sut.isCompatibleWith(attributeMappers).isValid()
+    }
+
+    def "ImportData is compatible with attribute mappers, if one header does not match any entry in the not required attribute mappers fileColumnAlias"() {
+
+        given:
+
+        ImportData sut = new CsvImportDataConverter().convert('''name,lastname
+Mark,Andersson
+Pete,Hansen''')
+
+        def attributeMappers = [
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name',
+                        fileColumnAlias: 'name', fileColumnNumber: 0, isRequiredColumn: true),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname',
+                        fileColumnAlias: 'firstname', fileColumnNumber: 1, isRequiredColumn: false),
+        ]
+
+        expect:
+        sut.isCompatibleWith(attributeMappers).isValid()
+    }
+
+    def "ImportData is compatible with attribute mappers, if header is missing for not required attribute mappers fileColumnAlias"() {
+
+        given:
+
+        ImportData sut = new CsvImportDataConverter().convert('''name,lastname
+Mark,Andersson
+Pete,Hansen''')
+
+        def attributeMappers = [
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'name',
+                        fileColumnAlias: 'name', fileColumnNumber: 0, isRequiredColumn: true),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'firstname',
+                        fileColumnAlias: 'firstname', fileColumnNumber: 1, isRequiredColumn: false),
+                new ImportAttributeMapper(attributeType: AttributeType.DIRECT_ATTRIBUTE, entityAttribute: 'lastname',
+                        fileColumnAlias: 'lastname', fileColumnNumber: 2, isRequiredColumn: true),
+        ]
+
+        expect:
+        sut.isCompatibleWith(attributeMappers).isValid()
     }
 }

modules/web/src/de/diedavids/cuba/dataimport/web/importattributemapper/import-attribute-mapper-automatic-frame.xml

@@ -32,6 +32,7 @@
                        property="associationLookupAttribute"/>
                 <field property="fileColumnNumber"/>
                 <field property="fileColumnAlias"/>
+                <field property="isRequiredColumn"/>
             </column>
         </fieldGroup>
     </layout>

modules/web/src/de/diedavids/cuba/dataimport/web/importattributemapper/import-attribute-mapper-custom-frame.xml

@@ -22,6 +22,7 @@
 
                 <field property="fileColumnNumber"/>
                 <field property="fileColumnAlias"/>
+                <field property="isColumnRequired"/>
             </column>
         </fieldGroup>
         <sourceCodeEditor id="customAttributeBindScriptEditor"

modules/web/src/de/diedavids/cuba/dataimport/web/importconfiguration/import-configuration-edit.xml

@@ -87,6 +87,7 @@
                         <column id="fileColumnAlias"/>
                         <column id="attributeType"/>
                         <column id="entityAttribute"/>
+                        <column id="isRequiredColumn"/>
                     </columns>
                     <rows datasource="importAttributeMappersDs"/>
                     <buttonsPanel>