Feature/fix 1644/asset ingestor filtering

CHANGELOG.md

@@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com)
 
 ### Changed
 - #1945 - Added support for jcr:content creation and update to the Data Importer
+- #1644 - Asset Ingestor | Add include section
 - #1989 - Updated maven dependency org.owasp:dependency-check-maven to 5.1.1
 
 ### Fixed

bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/asset/AssetIngestor.java

@@ -46,7 +46,6 @@
 import javax.jcr.Session;
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.List;
@@ -135,31 +134,34 @@ public AssetIngestor(MimeTypeService mimeTypeService) {
     )
     String jcrBasePath = "/content/dam";
     @FormField(
-            name = "Ignore folders",
-            description = "List of folder names to be ignored",
+            name = "Folders filter",
+            description = "Comma-delimited list of folders to filter, useful for bypassing thumnail folders and such. If you want to exclude folder name add '-' sign before name."
+                          + "If you want to include name, just write folder name or add '+' sign before name.",
             hint = "tmp,.DS_STORE",
-            options = {"default=tmp,ds_store,.ds_store,.thumbs,.appledouble"}
+            options = {"default=-tmp,-ds_store,-.ds_store,-.thumbs,-.appledouble"}
     )
-    String ignoreFolders = "tmp,ds_store,.ds_store,.thumbs,.appledouble";
-    List<String> ignoreFolderList;
+    String foldersFilterExpression = "-tmp,-ds_store,-.ds_store,-.thumbs,-.appledouble";
+    NamesFilter folderFilter;
 
     @FormField(
-            name = "Ignore files",
-            description = "List of file names to ignore",
+            name = "Files Filter",
+            description = "Comma-delimited list of files to filter, also useful for bypassing additional metadata files which might not be useful in a DAM setting. If you want to exclude file name add '-' sign before name."
+                          + "If you want to include name, just write file name or add '+' sign before name.",
             hint = "full file names, comma separated",
-            options = {"default=ds_store,.ds_store"}
+            options = {"default=-ds_store,-.ds_store"}
     )
-    String ignoreFiles = "ds_store,.ds_store";
-    List<String> ignoreFileList;
+    String filesFilterExpression = "-ds_store,-.ds_store";
+    NamesFilter fileFilter;
 
     @FormField(
-            name = "Ignore extensions",
-            description = "List of file extensions to ignore",
+            name = "Extensions filter",
+            description = "Comma-delimited list of file extensions to filter. If you want to exclude extension add '-' sign before name."
+                          + "If you want to include extension, just write extension or add '+' sign before name",
             hint = "mp4,txt, etc.",
-            options = {"default=txt,html,css,js,thm,exe,db"}
+            options = {"default=-txt,-html,-css,-js,-thm,-exe,-db"}
     )
-    String ignoreExtensions = "txt,html,css,js,thm,exe,db";
-    List<String> ignoreExtensionList;
+    String extensionsFilterExpression = "-txt,-html,-css,-js,-thm,-exe,-db";
+    NamesFilter extensionFilter;
 
     @FormField(
             name = "Existing action",
@@ -247,20 +249,20 @@ protected long getCount(EnumMap<ReportColumns, Object> row) {
 
     @Override
     public void init() throws RepositoryException {
-        if (ignoreFolders == null) {
-            ignoreFolders = "";
+        if (foldersFilterExpression == null) {
+            foldersFilterExpression = "";
         }
-        ignoreFolderList = Arrays.asList(ignoreFolders.trim().toLowerCase().split(","));
+        folderFilter = new NamesFilter(foldersFilterExpression);
 
-        if (ignoreFiles == null) {
-            ignoreFiles = "";
+        if (filesFilterExpression == null) {
+            filesFilterExpression = "";
         }
-        ignoreFileList = Arrays.asList(ignoreFiles.trim().toLowerCase().split(","));
+        fileFilter = new NamesFilter(filesFilterExpression);
 
-        if (ignoreExtensions == null) {
-            ignoreExtensions = "";
+        if (extensionsFilterExpression == null) {
+            extensionsFilterExpression = "";
         }
-        ignoreExtensionList = Arrays.asList(ignoreExtensions.trim().toLowerCase().split(","));
+        extensionFilter = new NamesFilter(extensionsFilterExpression);
 
         if (this.retries <= 0) {
             this.retries = DEFAULT_RETRIES;
@@ -456,13 +458,13 @@ protected boolean canImportFile(Source source) throws IOException {
         if (maximumSize > 0 && source.getLength() > maximumSize) {
             return false;
         }
-        if (name.startsWith(".") || ignoreFileList.contains(name)) {
+        if (name.startsWith(".") || fileFilter.isNotValidName(name)) {
             return false;
         }
         if (name.contains(".")) {
             int extPos = name.lastIndexOf('.');
             String ext = name.substring(extPos + 1);
-            if (ignoreExtensionList.contains(ext)) {
+            if (extensionFilter.isNotValidName(ext)) {
                 return false;
             }
         }
@@ -471,7 +473,7 @@ protected boolean canImportFile(Source source) throws IOException {
 
     protected boolean canImportFolder(HierarchicalElement element) {
         String name = element.getName();
-        if (ignoreFolderList.contains(name.toLowerCase())) {
+        if (folderFilter.isNotValidName(name.toLowerCase())) {
             return false;
         } else {
             HierarchicalElement parent = element.getParent();

bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/asset/NamesFilter.java

@@ -0,0 +1,98 @@
+/*
+ * #%L
+ * ACS AEM Commons Bundle
+ * %%
+ * Copyright (C) 2018 Adobe
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+package com.adobe.acs.commons.mcp.impl.processes.asset;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+class NamesFilter {
+
+    private List<String> include = new ArrayList<>();
+    private List<String> exclude = new ArrayList<>();
+
+    NamesFilter() {
+    }
+
+    NamesFilter(String filterString) {
+        List<String> filterExpressions = Optional.ofNullable(filterString)
+                                                 .filter(StringUtils::isNotEmpty)
+                                                 .map(str -> str.trim().toLowerCase().split(","))
+                                                 .map(Arrays::asList)
+                                                 .orElse(Collections.emptyList());
+
+        for (String filterExpression : filterExpressions) {
+            if (filterExpression.startsWith("+")) {
+                include.add(filterExpression.substring(1));
+            } else if (filterExpression.startsWith("-")) {
+                exclude.add(filterExpression.substring(1));
+            } else {
+                include.add(filterExpression);
+            }
+        }
+
+        include.removeAll(exclude);
+    }
+
+    boolean isNotValidName(String name) {
+        return !isValidName(name);
+    }
+
+    private boolean isValidName(String name) {
+        if (isEmptyFilter()) {
+            return true;
+        }
+
+        if (isOnlyIncludeFiler()) {
+            return isIncluded(name);
+        }
+
+        if (isOnlyExcludeFilter()) {
+            return isNotExcluded(name);
+        }
+
+        return isIncluded(name) && isNotExcluded(name);
+    }
+
+    private boolean isNotExcluded(final String name) {
+        return !exclude.contains(name);
+    }
+
+    private boolean isIncluded(final String name) {
+        return include.contains(name);
+    }
+
+    private boolean isOnlyExcludeFilter() {
+        return CollectionUtils.isNotEmpty(exclude) && CollectionUtils.isEmpty(include);
+    }
+
+    private boolean isOnlyIncludeFiler() {
+        return CollectionUtils.isEmpty(exclude) && CollectionUtils.isNotEmpty(include);
+    }
+
+    private boolean isEmptyFilter() {
+        return CollectionUtils.isEmpty(exclude) && CollectionUtils.isEmpty(include);
+    }
+}

bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/asset/FileAssetIngestorTest.java

@@ -28,7 +28,6 @@
 import com.jcraft.jsch.ChannelSftp;
 import com.jcraft.jsch.JSchException;
 import com.jcraft.jsch.Session;
-import com.jcraft.jsch.SftpATTRS;
 import com.jcraft.jsch.SftpException;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
@@ -111,9 +110,9 @@ public AssetManager apply(@Nullable ResourceResolver input) {
         ingestor = new FileAssetIngestor(context.getService(MimeTypeService.class));
         ingestor.timeout = 1;
         ingestor.jcrBasePath = "/content/dam";
-        ingestor.ignoreFileList = Collections.emptyList();
-        ingestor.ignoreExtensionList = Collections.emptyList();
-        ingestor.ignoreFolderList = Arrays.asList(".ds_store");
+        ingestor.fileFilter = new NamesFilter();
+        ingestor.extensionFilter = new NamesFilter();
+        ingestor.folderFilter = new NamesFilter("-.ds_store");
         ingestor.existingAssetAction = AssetIngestor.AssetAction.skip;
         ingestor.fileBasePath = tempDirectory.getAbsolutePath();
 

bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/asset/NamesFilterTest.java

@@ -0,0 +1,80 @@
+/*
+ * #%L
+ * ACS AEM Commons Bundle
+ * %%
+ * Copyright (C) 2018 Adobe
+ * %%
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * #L%
+ */
+package com.adobe.acs.commons.mcp.impl.processes.asset;
+
+import org.apache.commons.lang3.StringUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class NamesFilterTest {
+
+    @Test
+    public void testIsNotValidNameReturnsFalseWhenFilterIsEmpty() {
+        assertFalse(new NamesFilter(StringUtils.EMPTY).isNotValidName("test1"));
+        assertFalse(new NamesFilter(StringUtils.EMPTY).isNotValidName("test2"));
+        assertFalse(new NamesFilter(StringUtils.EMPTY).isNotValidName("test3"));
+        assertFalse(new NamesFilter(null).isNotValidName("test3"));
+    }
+
+    @Test
+    public void testIsNotValidNameReturnsTrueWhenOnlyIncludeWasProvided() {
+        assertTrue(new NamesFilter("+test1,test2,test3").isNotValidName("smthElse1"));
+        assertTrue(new NamesFilter("test1,test2,test3").isNotValidName("smthElse2"));
+        assertTrue(new NamesFilter("+test1,+test2,+test3").isNotValidName("smthElse3"));
+    }
+
+    @Test
+    public void testIsNotValidNameReturnsFalseWhenOnlyIncludesWasProvided() {
+        assertFalse(new NamesFilter("+test1,test2,test3").isNotValidName("test1"));
+        assertFalse(new NamesFilter("test1,test2,+test3").isNotValidName("test2"));
+        assertFalse(new NamesFilter("+test1,+test2,+test3").isNotValidName("test3"));
+    }
+
+    @Test
+    public void testIsNotValidNameReturnsTrueWhenOnlyExcludeWasProvided() {
+        assertTrue(new NamesFilter("-test1,-test2,-test3").isNotValidName("test1"));
+        assertTrue(new NamesFilter("-test1,-test2,-test3").isNotValidName("test2"));
+        assertTrue(new NamesFilter("-test1,-test2,-test3").isNotValidName("test3"));
+    }
+
+    @Test
+    public void testIsNotValidNameReturnsFalseWhenOnlyExcludeWasProvided() {
+        assertFalse(new NamesFilter("-test1,-test2,-test3").isNotValidName("smthElse1"));
+        assertFalse(new NamesFilter("-test1,-test2,-test3").isNotValidName("smthElse2"));
+        assertFalse(new NamesFilter("-test1,-test2,-test3").isNotValidName("smthElse3"));
+    }
+
+    @Test
+    public void testIsNotValidNameReturnsTrueWhenItemWasIncludedAndExcluded() {
+        assertTrue(new NamesFilter("-test1,test1").isNotValidName("test1"));
+    }
+
+    @Test
+    public void testIsNotValidNameWhenBothFiltersAreApplied() {
+        assertTrue(new NamesFilter("-test1,+test2,test3,-test4").isNotValidName("test1"));
+        assertTrue(new NamesFilter("-test1,test2,+test3,-test4").isNotValidName("test4"));
+        assertTrue(new NamesFilter("-test1,+test2,+test3,-test4").isNotValidName("test5"));
+
+        assertFalse(new NamesFilter("-test1,test2,test3,-test4").isNotValidName("test2"));
+        assertFalse(new NamesFilter("-test1,test2,test3,-test4").isNotValidName("test3"));
+    }
+}

bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/asset/S3AssetIngestorTest.java

@@ -103,9 +103,9 @@ public AssetManager apply(@Nullable ResourceResolver input) {
         context.resourceResolver().commit();
         ingestor = new S3AssetIngestor(context.getService(MimeTypeService.class));
         ingestor.jcrBasePath = "/content/dam";
-        ingestor.ignoreFileList = Collections.emptyList();
-        ingestor.ignoreExtensionList = Collections.emptyList();
-        ingestor.ignoreFolderList = Arrays.asList(".ds_store");
+        ingestor.fileFilter = new NamesFilter();
+        ingestor.extensionFilter = new NamesFilter();
+        ingestor.folderFilter = new NamesFilter("-.ds_store");
         ingestor.existingAssetAction = AssetIngestor.AssetAction.skip;
 
         int port = FreePortFinder.findFreeLocalPort();
@@ -147,7 +147,6 @@ public void testCreateFoldersWithEmptyBucket() throws Exception {
         verify(actionManager).setCurrentItem(currentItemCaptor.capture());
         assertEquals(1, currentItemCaptor.getAllValues().size());
         assertEquals(TEST_BUCKET, currentItemCaptor.getValue());
-
     }
 
     @Test