diff --git a/bundle/pom.xml b/bundle/pom.xml index 8fffe60f11..f5a6c8a3db 100644 --- a/bundle/pom.xml +++ b/bundle/pom.xml @@ -55,6 +55,7 @@ sun.misc.*;resolution:=optional, org.apache.sling.models.annotations.*;resolution:=optional, org.apache.sling.models.spi.injectorspecific.*;resolution:=optional, + com.amazonaws.*;resolution:=optional, com.day.cq.wcm.workflow.process;version="[6.0,8)", com.day.cq.mailer;version="[5.9,7)", com.adobe.cq.sightly;version="[2.5,4)", @@ -294,6 +295,12 @@ tika-parsers 1.14 + + com.amazonaws + aws-java-sdk-osgi + 1.10.76 + provided + org.mockito @@ -349,8 +356,8 @@ org.apache.jackrabbit - jackrabbit-core - 2.10.1 + jackrabbit-jcr-commons + 2.12.1 test @@ -359,6 +366,10 @@ 1.0-R1534292 test + + org.slf4j + jcl-over-slf4j + org.apache.sling org.apache.sling.testing.sling-mock @@ -366,6 +377,12 @@ org.apache.sling org.apache.sling.testing.sling-mock-jackrabbit + + + org.slf4j + slf4j-simple + + org.apache.sling @@ -378,6 +395,12 @@ io.wcm io.wcm.testing.aem-mock + + + org.slf4j + slf4j-simple + + org.skyscreamer @@ -387,7 +410,24 @@ org.hamcrest hamcrest-library - + + me.alexpanov + free-port-finder + 1.0 + test + + + io.findify + s3mock_2.11 + 0.2.3 + test + + + com.amazonaws + aws-java-sdk-s3 + + + com.adobe.aem diff --git a/bundle/src/main/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImpl.java b/bundle/src/main/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImpl.java index 595d370625..a194ef2d1e 100644 --- a/bundle/src/main/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImpl.java +++ b/bundle/src/main/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImpl.java @@ -52,6 +52,8 @@ public class SharpenImageTransformerImpl implements ImageTransformer { static final String TYPE = "sharpen"; private static final String KEY_UNSHARP_MASK = "op_usm"; + private static final int NUM_SHARPEN_PARAMS = 2; + @Override public final Layer transform(final Layer layer, final ValueMap properties) { @@ -70,12 +72,12 @@ public final Layer transform(final Layer layer, final ValueMap properties) { String[] param = unsharpenMask.split(","); // Support is provided for amount and radius only. - if(param.length <= 4 && param.length >= 2) { + if(param.length == NUM_SHARPEN_PARAMS) { float amount = Float.parseFloat(param[0]); float radius = Float.parseFloat(param[1]); layer.sharpen(amount, radius); } else { - log.warn("Transform [ {} ] requires 2 or 4 parameters.", TYPE); + log.warn("Transform [ {} ] requires 2 parameters.", TYPE); } } } catch (NumberFormatException exception) { diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/form/PasswordComponent.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/form/PasswordComponent.java new file mode 100644 index 0000000000..77cb37a23b --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/form/PasswordComponent.java @@ -0,0 +1,29 @@ +/* + * Copyright 2017 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.adobe.acs.commons.mcp.form; + +import aQute.bnd.annotation.ProviderType; + +/** + * Text field component + */ +@ProviderType +public class PasswordComponent extends FieldComponent { + @Override + public void init() { + setResourceType("granite/ui/components/coral/foundation/form/password"); + } +} diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/form/package-info.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/form/package-info.java new file mode 100644 index 0000000000..e093364a80 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/form/package-info.java @@ -0,0 +1,22 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +@aQute.bnd.annotation.Version("3.11.0") +package com.adobe.acs.commons.mcp.form; + diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestor.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestor.java index 7760dc9945..717dde1eec 100644 --- a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestor.java +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestor.java @@ -1,6 +1,9 @@ /* - * Copyright 2017 Adobe. - * + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -12,11 +15,11 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * #L% */ package com.adobe.acs.commons.mcp.impl.processes; import com.adobe.acs.commons.fam.ActionManager; -import com.adobe.acs.commons.fam.actions.Actions; import com.adobe.acs.commons.functions.CheckedConsumer; import com.adobe.acs.commons.mcp.ProcessDefinition; import com.adobe.acs.commons.mcp.ProcessInstance; @@ -26,49 +29,38 @@ import com.adobe.acs.commons.mcp.model.FieldFormat; import com.adobe.acs.commons.mcp.model.GenericReport; import com.adobe.acs.commons.mcp.model.ValueFormat; +import com.day.cq.commons.jcr.JcrConstants; import com.day.cq.commons.jcr.JcrUtil; import com.day.cq.dam.api.Asset; import com.day.cq.dam.api.AssetManager; -import java.io.File; -import java.io.FileInputStream; +import org.apache.commons.lang3.StringUtils; +import org.apache.sling.api.resource.PersistenceException; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.commons.mime.MimeTypeService; + +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.Session; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumMap; import java.util.List; -import javax.jcr.Node; -import javax.jcr.RepositoryException; -import javax.jcr.Session; -import org.apache.sling.api.resource.LoginException; -import org.apache.sling.api.resource.PersistenceException; -import org.apache.sling.api.resource.ResourceResolver; -import org.apache.sling.commons.mime.MimeTypeService; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; -/** - * Asset Ingestor reads a directory structure recursively and imports it as-is into AEM. - */ -public class AssetIngestor extends ProcessDefinition { +public abstract class AssetIngestor extends ProcessDefinition { private final MimeTypeService mimetypeService; + public enum AssetAction { + skip, version, replace + } + public AssetIngestor(MimeTypeService mimeTypeService) { this.mimetypeService = mimeTypeService; } - - public static enum AssetAction { - skip, version, replace - } - - @FormField( - name = "Source", - description = "Source folder for content ingestion", - hint = "/var/mycontent, /tmp, /mnt/all_the_things, ...", - required = true - ) - String fileBasePath; - File baseFolder; - + @FormField( name = "Target JCR Folder", description = "Base folder for ingestion", @@ -95,7 +87,7 @@ public static enum AssetAction { ) String ignoreFiles; List ignoreFileList; - + @FormField( name = "Ignore extensions", description = "List of file extensions to ignore", @@ -104,13 +96,13 @@ public static enum AssetAction { ) String ignoreExtensions; List ignoreExtensionList; - + @FormField( name = "Existing action", description = "What to do if an asset exists", component = RadioComponent.EnumerationSelector.class, options={"default=skip","vertical"} - ) + ) AssetAction existingAssetAction; @FormField( name = "Minimum size", @@ -127,21 +119,16 @@ public static enum AssetAction { ) long maximumSize; - private static final String DEFAULT_FOLDER_TYPE = "sling:Folder"; - private static final String JCR_TITLE = "jcr:title"; - private static final String CHANGED_BY_WORKFLOW = "changedByWorkflowProcess"; - - int folderCount = 0; - int assetCount = 0; - int filesSkipped = 0; - long totalImportedData = 0; - + protected static final String DEFAULT_FOLDER_TYPE = "sling:Folder"; + protected static final String CHANGED_BY_WORKFLOW = "changedByWorkflowProcess"; + + AtomicInteger folderCount = new AtomicInteger(); + AtomicInteger assetCount = new AtomicInteger(); + AtomicInteger filesSkipped = new AtomicInteger(); + AtomicLong totalImportedData = new AtomicLong(); + @Override public void init() throws RepositoryException { - baseFolder = new File(fileBasePath); - if (!baseFolder.exists()) { - throw new RepositoryException("Source folder does not exist!"); - } if (ignoreFolders == null) { ignoreFolders = ""; } @@ -156,88 +143,103 @@ public void init() throws RepositoryException { ignoreExtensionList = Arrays.asList(ignoreExtensions.trim().toLowerCase().split(",")); } - @Override - public void buildProcess(ProcessInstance instance, ResourceResolver rr) throws LoginException, RepositoryException { - instance.getInfo().setDescription(fileBasePath + "->" + jcrBasePath); - instance.defineCriticalAction("Create Folders", rr, this::createFolders); - instance.defineCriticalAction("Import Assets", rr, this::importAssets); - } - - private void createFolders(ActionManager manager) throws IOException { - manager.deferredWithResolver(r->{ - manager.setCurrentItem(fileBasePath); - Files.walk(baseFolder.toPath()).map(Path::toFile).filter(File::isDirectory).filter(this::canImportFolder).forEach(f->{ - manager.deferredWithResolver(Actions.retry(10, 100, rr-> { - manager.setCurrentItem(f.getPath()); - createFolderNode(folderToNodePath(f), f, rr); - })); - }); - }); + private void createAsset(Source source, String assetPath, ResourceResolver r, boolean versioning) throws Exception { + r.adaptTo(Session.class).getWorkspace().getObservationManager().setUserData(CHANGED_BY_WORKFLOW); + AssetManager assetManager = r.adaptTo(AssetManager.class); + String type = mimetypeService.getMimeType(source.getName()); + if (versioning) { + //is asset is null, no version gets created + Asset asset = r.getResource(assetPath).adaptTo(Asset.class); + //once you are past this first version, default behavior is to start numbering 1.0, 1.1 and so on + assetManager.createRevision(asset, "initial version of asset", asset.getName()); + r.commit(); + r.refresh(); + //once version is committed we are safe to create, which only replaces the original version + } + assetManager.createAsset(assetPath, source.getStream(), type, false); + r.commit(); + r.refresh(); + totalImportedData.accumulateAndGet(source.getLength(), (p,x) -> p+x); + assetCount.incrementAndGet(); } - private String folderToNodePath(File current) { - if (baseFolder.equals(current)) { - return jcrBasePath; - } else { - return folderToNodePath(current.getParentFile()) + "/" + JcrUtil.createValidName(current.getName()); + protected void handleExistingAsset(Source source, String assetPath, ResourceResolver r) throws Exception { + switch (existingAssetAction) { + case skip: + //if skip then we only create asset if it doesn't exist + if (r.getResource(assetPath) == null) { + createAsset(source, assetPath, r, false); + } else { + filesSkipped.incrementAndGet(); + } + break; + case replace: + //if replace we just create a new one and the old one goes away + createAsset(source, assetPath, r, false); + break; + case version: + //only option left is replace, we'll save current version as a version and then replace it + versionExistingAsset(source, assetPath, r); } } - - private boolean createFolderNode(String node, File folder, ResourceResolver r) throws RepositoryException, PersistenceException { + + protected boolean createFolderNode(HierarchialElement el, ResourceResolver r) throws RepositoryException, PersistenceException { + if (el == null || !el.isFolder()) { + return false; + } + String folderPath = el.getNodePath(); + String name = el.getName(); Session s = r.adaptTo(Session.class); - if (s.nodeExists(node)) { - Node folderNode = s.getNode(node); - if (folderNode.hasProperty(JCR_TITLE) && folderNode.getProperty(JCR_TITLE).getString().equals(folder.getName())) { + if (s.nodeExists(folderPath)) { + Node folderNode = s.getNode(folderPath); + if (folderPath.equals(jcrBasePath) || (folderNode.hasProperty(JcrConstants.JCR_TITLE) && folderNode.getProperty(JcrConstants.JCR_TITLE).getString().equals(name))) { return false; } else { - folderNode.setProperty(JCR_TITLE, folder.getName()); + folderNode.setProperty(JcrConstants.JCR_TITLE, name); r.commit(); r.refresh(); return true; } } - String parentNode = node.substring(0, node.lastIndexOf("/")); - String childNode = node.substring(node.lastIndexOf("/") + 1); - if (!folder.getParentFile().equals(baseFolder)) { - createFolderNode(parentNode, folder.getParentFile(), r); + HierarchialElement parent = el.getParent(); + String parentPath; + if (parent == null) { + parentPath = jcrBasePath; + } else { + parentPath = parent.getNodePath(); + } + if (!jcrBasePath.equals(parentPath)) { + createFolderNode(parent, r); + } + Node child = s.getNode(parentPath).addNode(el.getNodeName(), DEFAULT_FOLDER_TYPE); + folderCount.incrementAndGet(); + if (!folderPath.equals(jcrBasePath)) { + child.setProperty(JcrConstants.JCR_TITLE, name); } - Node child = s.getNode(parentNode).addNode(childNode, DEFAULT_FOLDER_TYPE); - folderCount++; - child.setProperty(JCR_TITLE, folder.getName()); r.commit(); r.refresh(); return true; } - private void importAssets(ActionManager manager) throws IOException { - manager.deferredWithResolver(rr->{ - Actions.setCurrentItem(fileBasePath); - Files.walk(baseFolder.toPath()).map(Path::toFile).filter(File::isFile).filter(f->canImportFolder(f.getParentFile())).forEach(f->{ - if (canImportFile(f)) { - manager.deferredWithResolver(Actions.retry(5, 25, importFile(f))); - } else { - filesSkipped++; - } - }); - }); + private void versionExistingAsset(Source source, String assetPath, ResourceResolver r) throws Exception { + createAsset(source, assetPath, r, r.getResource(assetPath) != null); } - private boolean canImportFolder(File f) { - if (f.equals(baseFolder)) { - return true; - } - if (ignoreFolderList.contains(f.getName().toLowerCase())) { - return false; - } - return canImportFolder(f.getParentFile()); + protected CheckedConsumer importAsset(final Source source, ActionManager actionManager) { + return (ResourceResolver r) -> { + String path = source.getElement().getNodePath(); + createFolderNode(source.getElement().getParent(), r); + actionManager.setCurrentItem(source.getElement().getItemName()); + handleExistingAsset(source, path, r); + }; } - - private boolean canImportFile(File f) { - String name = f.getName().toLowerCase(); - if (minimumSize > 0 && f.length() < minimumSize) { + + protected boolean canImportFile(Source source) { + String name = source.getName().toLowerCase(); + if (minimumSize > 0 && source.getLength() < minimumSize) { return false; } - if (maximumSize > 0 && f.length() > maximumSize) { + if (maximumSize > 0 && source.getLength() > maximumSize) { return false; } if (name.startsWith(".") || ignoreFileList.contains(name)) { @@ -252,60 +254,30 @@ private boolean canImportFile(File f) { } return true; } - - private CheckedConsumer importFile(final File sourceFile) { - return (ResourceResolver r) -> { - String basePath = folderToNodePath(sourceFile.getParentFile()); - createFolderNode(basePath, sourceFile.getParentFile(), r); - String destPath = basePath + "/" + sourceFile.getName(); - Actions.setCurrentItem(destPath); - handleExistingAsset(sourceFile, destPath, r); - }; - } - private void createAsset(File sourceFile, String assetPath, ResourceResolver r, boolean versioning) throws Exception { - r.adaptTo(Session.class).getWorkspace().getObservationManager().setUserData(CHANGED_BY_WORKFLOW); - AssetManager assetManager = r.adaptTo(AssetManager.class); - String type = mimetypeService.getMimeType(sourceFile.getName()); - if (versioning) { - //is asset is null, no version gets created - Asset asset = r.getResource(assetPath).adaptTo(Asset.class); - //once you are past this first version, default behavior is to start numbering 1.0, 1.1 and so on - assetManager.createRevision(asset, "initial version of asset", asset.getName()); - r.commit(); - r.refresh(); - //once version is committed we are safe to create, which only replaces the original version + protected boolean canImportFolder(HierarchialElement element) { + String name = element.getName(); + if (ignoreFolderList.contains(name.toLowerCase())) { + return false; + } else { + HierarchialElement parent = element.getParent(); + if (parent == null) { + return true; + } else { + return canImportFolder(parent); + } } - assetManager.createAsset(assetPath, new FileInputStream(sourceFile), type, false); - r.commit(); - r.refresh(); - totalImportedData += sourceFile.length(); - assetCount++; } - private void handleExistingAsset(File sourceFile, String assetPath, ResourceResolver r) throws Exception { - switch (existingAssetAction) { - case skip: - //if skip then we only create asset if it doesn't exist - if (r.getResource(assetPath) == null) { - createAsset(sourceFile, assetPath, r, false); - } else { - filesSkipped++; - } - break; - case replace: - //if replace we just create a new one and the old one goes away - createAsset(sourceFile, assetPath, r, false); - break; - case version: - //only option left is replace, we'll save current version as a version and then replace it - versionExistingAsset(sourceFile, assetPath, r); + protected boolean canImportContainingFolder(HierarchialElement element) { + HierarchialElement parent = element.getParent(); + if (parent == null) { + return true; + } else { + return canImportFolder(parent); } } - private void versionExistingAsset(File sourceFile, String assetPath, ResourceResolver r) throws Exception { - createAsset(sourceFile, assetPath, r, r.getResource(assetPath) != null); - } enum ReportColumns {folder_count, asset_count, files_skipped, @FieldFormat(ValueFormat.storageSize) data_imported}; GenericReport report = new GenericReport(); @@ -320,5 +292,39 @@ public void storeReport(ProcessInstance instance, ResourceResolver rr) throws Re values.put(ReportColumns.data_imported, totalImportedData); report.setRows(rows, ReportColumns.class); report.persist(rr, instance.getPath() + "/jcr:content/report"); - } -} \ No newline at end of file + } + + protected interface Source { + + String getName(); + InputStream getStream() throws IOException; + long getLength(); + HierarchialElement getElement(); + + } + + protected interface HierarchialElement { + boolean isFile(); + boolean isFolder(); + HierarchialElement getParent(); + String getName(); + String getItemName(); + Source getSource(); + String getJcrBasePath(); + + default String getNodePath() { + HierarchialElement parent = getParent(); + return (parent == null ? getJcrBasePath() : parent.getNodePath()) + "/" + getNodeName(); + } + default String getNodeName() { + String name = getName(); + if (isFile() && name.contains(".")) { + String baseName = StringUtils.substringBeforeLast(name, "."); + String extension = StringUtils.substringAfterLast(name, "."); + return JcrUtil.createValidName(baseName) + "." + JcrUtil.createValidName(extension); + } else { + return JcrUtil.createValidName(name); + } + } + } +} diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestor.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestor.java new file mode 100644 index 0000000000..74abaf65b3 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestor.java @@ -0,0 +1,180 @@ +/* + * Copyright 2017 Adobe. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.adobe.acs.commons.fam.ActionManager; +import com.adobe.acs.commons.fam.actions.Actions; +import com.adobe.acs.commons.mcp.ProcessInstance; +import com.adobe.acs.commons.mcp.form.FormField; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import javax.jcr.RepositoryException; +import javax.jcr.Session; + +import com.day.cq.commons.jcr.JcrUtil; +import org.apache.sling.api.resource.LoginException; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.commons.mime.MimeTypeService; + +/** + * Asset Ingestor reads a directory structure recursively and imports it as-is into AEM. + */ +public class FileAssetIngestor extends AssetIngestor { + + public FileAssetIngestor(MimeTypeService mimeTypeService) { + super(mimeTypeService); + } + @FormField( + name = "Source", + description = "Source folder for content ingestion", + hint = "/var/mycontent, /tmp, /mnt/all_the_things, ...", + required = true + ) + String fileBasePath; + File baseFolder; + + @Override + public void init() throws RepositoryException { + baseFolder = new File(fileBasePath); + if (!baseFolder.exists()) { + throw new RepositoryException("Source folder does not exist!"); + } + super.init(); + } + + @Override + public void buildProcess(ProcessInstance instance, ResourceResolver rr) throws LoginException, RepositoryException { + instance.getInfo().setDescription(fileBasePath + "->" + jcrBasePath); + instance.defineCriticalAction("Create Folders", rr, this::createFolders); + instance.defineCriticalAction("Import Assets", rr, this::importAssets); + } + + void createFolders(ActionManager manager) throws IOException { + manager.deferredWithResolver(r->{ + JcrUtil.createPath(jcrBasePath, DEFAULT_FOLDER_TYPE, DEFAULT_FOLDER_TYPE, r.adaptTo(Session.class), true); + manager.setCurrentItem(fileBasePath); + Files.walk(baseFolder.toPath()).map(Path::toFile).filter(f -> !f.equals(baseFolder)). + map(FileHierarchialElement::new).filter(FileHierarchialElement::isFolder).filter(this::canImportFolder).forEach(f->{ + manager.deferredWithResolver(Actions.retry(10, 100, rr-> { + manager.setCurrentItem(f.getItemName()); + createFolderNode(f, rr); + })); + }); + }); + } + + void importAssets(ActionManager manager) throws IOException { + manager.deferredWithResolver(rr->{ + JcrUtil.createPath(jcrBasePath, DEFAULT_FOLDER_TYPE, DEFAULT_FOLDER_TYPE, rr.adaptTo(Session.class), true); + manager.setCurrentItem(fileBasePath); + Files.walk(baseFolder.toPath()).map(FileHierarchialElement::new).filter(FileHierarchialElement::isFile). + filter(this::canImportContainingFolder).map(FileHierarchialElement::getSource).forEach(fs->{ + if (canImportFile(fs)) { + manager.deferredWithResolver(Actions.retry(5, 25, importAsset(fs, manager))); + } else { + filesSkipped.incrementAndGet(); + } + }); + }); + } + + private class FileSource implements Source { + private final File file; + private final HierarchialElement element; + + private FileSource(File f, FileHierarchialElement el) { + this.file = f; + this.element = el; + } + + @Override + public String getName() { + return file.getName(); + } + + @Override + public InputStream getStream() throws IOException { + return new FileInputStream(file); + } + + @Override + public long getLength() { + return file.length(); + } + + @Override + public HierarchialElement getElement() { + return element; + } + } + + class FileHierarchialElement implements HierarchialElement { + + private final File file; + + FileHierarchialElement(File f) { + this.file = f; + } + + private FileHierarchialElement(Path p) { + this(p.toFile()); + } + + @Override + public Source getSource() { + return new FileSource(file, this); + } + + @Override + public String getItemName() { + return file.getAbsolutePath(); + } + + @Override + public String getName() { + return file.getName(); + } + + @Override + public HierarchialElement getParent() { + File parent = file.getParentFile(); + if (parent.equals(baseFolder)) { + return null; + } + return new FileHierarchialElement(file.getParentFile()); + } + + @Override + public boolean isFolder() { + return file.isDirectory(); + } + + @Override + public boolean isFile() { + return file.isFile(); + } + + @Override + public String getJcrBasePath() { + return jcrBasePath; + } + } + +} \ No newline at end of file diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestorFactory.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorFactory.java similarity index 84% rename from bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestorFactory.java rename to bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorFactory.java index 0186e1c633..34d4d1258d 100644 --- a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/AssetIngestorFactory.java +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorFactory.java @@ -28,7 +28,7 @@ @Component @Service(ProcessDefinitionFactory.class) -public class AssetIngestorFactory extends AdministratorsOnlyProcessDefinitionFactory { +public class FileAssetIngestorFactory extends AdministratorsOnlyProcessDefinitionFactory { @Reference MimeTypeService mimetypeService; @@ -39,7 +39,7 @@ public String getName() { } @Override - public AssetIngestor createProcessDefinitionInstance() { - return new AssetIngestor(mimetypeService); + public FileAssetIngestor createProcessDefinitionInstance() { + return new FileAssetIngestor(mimetypeService); } } diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestor.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestor.java new file mode 100644 index 0000000000..0c0bcf17c8 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestor.java @@ -0,0 +1,268 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.adobe.acs.commons.fam.ActionManager; +import com.adobe.acs.commons.fam.actions.Actions; +import com.adobe.acs.commons.mcp.ProcessInstance; +import com.adobe.acs.commons.mcp.form.FormField; +import com.adobe.acs.commons.mcp.form.PasswordComponent; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.day.cq.commons.jcr.JcrUtil; +import org.apache.commons.lang3.StringUtils; +import org.apache.sling.api.resource.LoginException; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.commons.mime.MimeTypeService; + +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import java.io.IOException; +import java.io.InputStream; + +public class S3AssetIngestor extends AssetIngestor { + + public S3AssetIngestor(MimeTypeService mimeTypeService) { + super(mimeTypeService); + } + + @FormField( + name = "Bucket", + description = "S3 Bucket Name" + ) + String bucket; + + @FormField( + name = "Access Key", + description = "S3 Access Key" + ) + String accessKey; + + @FormField( + name = "Secret Key", + description = "S3 Secret Key", + component = PasswordComponent.class + ) + String secretKey; + + @FormField( + name = "S3 Base Path", + description = "S3 Base Path (Prefix)", + required = false + ) + String s3BasePath; + + @FormField( + name = "Endpoint URL", + description = "Endpoint URL, leave blank for default. Used primarily for S3-compatible object-storage solutions.", + required = false + ) + String endpointUrl; + + transient AmazonS3 s3Client; + + transient String baseItemName; + + @Override + public void init() throws RepositoryException { + super.init(); + if (StringUtils.isNotBlank(s3BasePath)) { + baseItemName = bucket + ":" + s3BasePath; + } else { + baseItemName = bucket; + } + if (StringUtils.isNotBlank(endpointUrl)) { + baseItemName = endpointUrl + "/" + baseItemName; + } + } + + @Override + public void buildProcess(ProcessInstance instance, ResourceResolver rr) throws LoginException, RepositoryException { + if (StringUtils.isNotBlank(s3BasePath) && !s3BasePath.endsWith("/")) { + s3BasePath = s3BasePath + "/"; + } + instance.getInfo().setDescription(baseItemName + "->" + jcrBasePath); + instance.defineCriticalAction("Create Folders", rr, this::createFolders); + instance.defineCriticalAction("Import Assets", rr, this::importAssets); + s3Client = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey)); + if (StringUtils.isNotBlank(endpointUrl)) { + s3Client.setEndpoint(endpointUrl); + } + } + + void createFolders(ActionManager manager) { + manager.deferredWithResolver(r->{ + JcrUtil.createPath(jcrBasePath, DEFAULT_FOLDER_TYPE, DEFAULT_FOLDER_TYPE, r.adaptTo(Session.class), true); + manager.setCurrentItem(baseItemName); + + ObjectListing listing = s3Client.listObjects(bucket, s3BasePath); + createFolders(manager, listing); + }); + } + private void createFolders(ActionManager manager, ObjectListing listing) { + listing.getObjectSummaries().stream().filter(sum -> !sum.getKey().equals(s3BasePath)).map(S3HierarchialElement::new). + filter(S3HierarchialElement::isFolder).filter(this::canImportFolder).forEach(el-> { + manager.deferredWithResolver(Actions.retry(10, 100, rr-> { + manager.setCurrentItem(el.getItemName()); + createFolderNode(el, rr); + })); + }); + if (listing.isTruncated()) { + createFolders(manager, s3Client.listNextBatchOfObjects(listing)); + } + } + + void importAssets(ActionManager manager) { + manager.deferredWithResolver(rr->{ + JcrUtil.createPath(jcrBasePath, DEFAULT_FOLDER_TYPE, DEFAULT_FOLDER_TYPE, rr.adaptTo(Session.class), true); + manager.setCurrentItem(baseItemName); + ObjectListing listing = s3Client.listObjects(bucket, s3BasePath); + importAssets(manager, listing); + }); + } + + private void importAssets(ActionManager manager, ObjectListing listing) { + listing.getObjectSummaries().stream().map(S3HierarchialElement::new). + filter(S3HierarchialElement::isFile).filter(this::canImportContainingFolder).map(S3HierarchialElement::getSource).forEach(ss-> { + if (canImportFile(ss)) { + manager.deferredWithResolver(Actions.retry(5, 25, importAsset(ss, manager))); + } else { + filesSkipped.incrementAndGet(); + } + }); + if (listing.isTruncated()) { + createFolders(manager, s3Client.listNextBatchOfObjects(listing)); + } + } + + private class S3Source implements Source { + + private final S3ObjectSummary s3ObjectSummary; + private final HierarchialElement element; + + private S3Source(S3ObjectSummary s3ObjectSummary, S3HierarchialElement element) { + this.s3ObjectSummary = s3ObjectSummary; + this.element = element; + } + + @Override + public long getLength() { + return s3ObjectSummary.getSize(); + } + + @Override + public InputStream getStream() throws IOException { + return s3Client.getObject(bucket, s3ObjectSummary.getKey()).getObjectContent(); + } + + @Override + public String getName() { + return element.getName(); + } + + @Override + public HierarchialElement getElement() { + return element; + } + } + + class S3HierarchialElement implements HierarchialElement { + private final S3ObjectSummary original; + private final String negativePath; + final String effectiveKey; + + S3HierarchialElement(S3ObjectSummary original) { + this(original, null); + } + + private S3HierarchialElement(S3ObjectSummary original, String negativePath) { + this.original = original; + this.negativePath = negativePath != null ? negativePath : ""; + this.effectiveKey = original.getKey().substring(0, original.getKey().length() - this.negativePath.length()); + } + + @Override + public boolean isFile() { + return !isFolder(); + } + + @Override + public boolean isFolder() { + return effectiveKey.endsWith("/"); + } + + @Override + public HierarchialElement getParent() { + if (isFolder()) { + String newNegativePath = getName() + "/" + this.negativePath; + String newEffectiveKey = original.getKey().substring(0, original.getKey().length() - newNegativePath.length()); + if (newNegativePath.equals(original.getKey()) || newEffectiveKey.equals(s3BasePath)) { + return null; + } + return new S3HierarchialElement(original, newNegativePath); + } else { + String newNegativePath = getName(); + String newEffectiveKey = original.getKey().substring(0, original.getKey().length() - newNegativePath.length()); + if (newNegativePath.equals(original.getKey()) || newEffectiveKey.equals(s3BasePath)) { + return null; + } + return new S3HierarchialElement(original, newNegativePath); + } + } + + @Override + public String getName() { + String keyWithoutTrailingSlash; + if (isFolder()) { + keyWithoutTrailingSlash = effectiveKey.substring(0, effectiveKey.length() - 1); + } else { + keyWithoutTrailingSlash = effectiveKey; + } + String name = StringUtils.substringAfterLast(keyWithoutTrailingSlash, "/"); + if (StringUtils.isEmpty(name)) { + return keyWithoutTrailingSlash; + } else { + return name; + } + } + + @Override + public String getItemName() { + return bucket + ":" + effectiveKey; + } + + @Override + public Source getSource() { + if (StringUtils.isNotBlank(negativePath)) { + return null; + } else { + return new S3Source(original, this); + } + } + + @Override + public String getJcrBasePath() { + return jcrBasePath; + } + } +} diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorFactory.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorFactory.java new file mode 100644 index 0000000000..3a0d47a563 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorFactory.java @@ -0,0 +1,62 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.adobe.acs.commons.mcp.AdministratorsOnlyProcessDefinitionFactory; +import com.adobe.acs.commons.mcp.ProcessDefinition; +import com.adobe.acs.commons.mcp.ProcessDefinitionFactory; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.Service; +import org.apache.jackrabbit.api.security.user.User; +import org.apache.sling.commons.mime.MimeTypeService; + +@Component +@Service(ProcessDefinitionFactory.class) +public class S3AssetIngestorFactory extends AdministratorsOnlyProcessDefinitionFactory { + + @Reference + MimeTypeService mimetypeService; + + @Override + public String getName() { + return "S3 Asset Ingestor"; + } + + @Override + public ProcessDefinition createProcessDefinitionInstance() { + return new S3AssetIngestor(mimetypeService); + } + + @Override + public boolean isAllowed(User user) { + if (super.isAllowed(user)) { + // check if S3 SDK is available + try { + AmazonS3 s3Client = new AmazonS3Client(); + return true; + } catch (NoClassDefFoundError e) { + } + } + return false; + } +} diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/mbean/package-info.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/mbean/package-info.java new file mode 100644 index 0000000000..e828ef4937 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/mbean/package-info.java @@ -0,0 +1,23 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +@Version("3.10.0") +package com.adobe.acs.commons.mcp.mbean; + +import aQute.bnd.annotation.Version; \ No newline at end of file diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/model/package-info.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/model/package-info.java new file mode 100644 index 0000000000..bf4d3de9b7 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/model/package-info.java @@ -0,0 +1,23 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +@Version("3.10.0") +package com.adobe.acs.commons.mcp.model; + +import aQute.bnd.annotation.Version; \ No newline at end of file diff --git a/bundle/src/main/java/com/adobe/acs/commons/mcp/util/package-info.java b/bundle/src/main/java/com/adobe/acs/commons/mcp/util/package-info.java new file mode 100644 index 0000000000..120a4aa7a6 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/mcp/util/package-info.java @@ -0,0 +1,23 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +@Version("3.10.0") +package com.adobe.acs.commons.mcp.util; + +import aQute.bnd.annotation.Version; \ No newline at end of file diff --git a/bundle/src/main/java/com/adobe/acs/commons/replication/packages/automatic/model/package-info.java b/bundle/src/main/java/com/adobe/acs/commons/replication/packages/automatic/model/package-info.java new file mode 100644 index 0000000000..72ebeb4eb0 --- /dev/null +++ b/bundle/src/main/java/com/adobe/acs/commons/replication/packages/automatic/model/package-info.java @@ -0,0 +1,23 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +@Version("3.10.0") +package com.adobe.acs.commons.replication.packages.automatic.model; + +import aQute.bnd.annotation.Version; \ No newline at end of file diff --git a/bundle/src/test/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImplTest.java b/bundle/src/test/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImplTest.java index f45551e637..0bd8b3a7d2 100644 --- a/bundle/src/test/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImplTest.java +++ b/bundle/src/test/java/com/adobe/acs/commons/images/transformers/impl/SharpenImageTransformerImplTest.java @@ -75,7 +75,7 @@ public void testTransform_withFourValues() throws Exception { transformer.transform(layer, properties); - verify(layer, times(1)).sharpen(2.0f,1.0f); + verify(layer, times(0)).sharpen(2.0f,1.0f); verifyNoMoreInteractions(layer); } } diff --git a/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/FactoryInjectionTest.java b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/FactoryInjectionTest.java index b724443328..32200e7cb2 100644 --- a/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/FactoryInjectionTest.java +++ b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/FactoryInjectionTest.java @@ -21,8 +21,8 @@ import com.adobe.acs.commons.fam.ActionManagerFactory; import com.adobe.acs.commons.mcp.ProcessDefinition; -import com.adobe.acs.commons.mcp.impl.processes.AssetIngestor; -import com.adobe.acs.commons.mcp.impl.processes.AssetIngestorFactory; +import com.adobe.acs.commons.mcp.impl.processes.FileAssetIngestor; +import com.adobe.acs.commons.mcp.impl.processes.FileAssetIngestorFactory; import com.adobe.acs.commons.mcp.impl.processes.AssetReport; import com.adobe.acs.commons.mcp.impl.processes.AssetReportFactory; import com.adobe.acs.commons.mcp.impl.processes.DeepPrune; @@ -32,6 +32,8 @@ import com.adobe.acs.commons.mcp.impl.processes.PageRelocatorFactory; import com.adobe.acs.commons.mcp.impl.processes.ProcessCleanup; import com.adobe.acs.commons.mcp.impl.processes.ProcessCleanupFactory; +import com.adobe.acs.commons.mcp.impl.processes.S3AssetIngestor; +import com.adobe.acs.commons.mcp.impl.processes.S3AssetIngestorFactory; import com.day.cq.replication.Replicator; import com.day.cq.wcm.api.PageManagerFactory; import org.apache.sling.event.jobs.JobManager; @@ -65,10 +67,17 @@ public void setup() { } @Test - public void testAssetIngestorFactory() throws Exception { + public void testFileIngestorFactory() throws Exception { ProcessDefinition def = cpm.findDefinitionByNameOrPath("Asset Ingestor"); assertNotNull(def); - assertTrue(def instanceof AssetIngestor); + assertTrue(def instanceof FileAssetIngestor); + } + + @Test + public void testS3IngestorFactory() throws Exception { + ProcessDefinition def = cpm.findDefinitionByNameOrPath("S3 Asset Ingestor"); + assertNotNull(def); + assertTrue(def instanceof S3AssetIngestor); } @Test @@ -100,12 +109,13 @@ public void testProcessCleanupFactory() throws Exception { } private void registerFactories() { - slingContext.registerInjectActivateService(new AssetIngestorFactory()); + slingContext.registerInjectActivateService(new FileAssetIngestorFactory()); slingContext.registerInjectActivateService(new AssetReportFactory()); slingContext.registerInjectActivateService(new DeepPruneFactory()); slingContext.registerInjectActivateService(new FolderRelocatorFactory()); slingContext.registerInjectActivateService(new PageRelocatorFactory()); slingContext.registerInjectActivateService(new ProcessCleanupFactory()); + slingContext.registerInjectActivateService(new S3AssetIngestorFactory()); } private void registerCommonServices() { diff --git a/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorTest.java b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorTest.java new file mode 100644 index 0000000000..d79949b9a8 --- /dev/null +++ b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorTest.java @@ -0,0 +1,273 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.adobe.acs.commons.fam.ActionManager; +import com.adobe.acs.commons.functions.CheckedConsumer; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.day.cq.dam.api.AssetManager; +import com.google.common.base.Function; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.JcrConstants; +import org.apache.sling.api.resource.PersistenceException; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.commons.mime.MimeTypeService; +import org.apache.sling.testing.mock.sling.ResourceResolverType; +import org.apache.sling.testing.mock.sling.junit.SlingContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import javax.annotation.Nullable; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.Assert.*; +import static org.mockito.Matchers.*; +import static org.mockito.Mockito.*; +import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; + +@RunWith(MockitoJUnitRunner.class) +public class FileAssetIngestorTest { + private static final int FILE_SIZE = 57797; + + @Rule + public final SlingContext context = new SlingContext(ResourceResolverType.JCR_OAK); + + @Mock + private ActionManager actionManager; + + @Mock + private AssetManager assetManager; + + @Captor + private ArgumentCaptor currentItemCaptor; + + @Captor + private ArgumentCaptor assetPathCaptor; + + private FileAssetIngestor ingestor; + + private File tempDirectory; + + @Before + public void setup() throws PersistenceException { + context.registerAdapter(ResourceResolver.class, AssetManager.class, new Function() { + @Nullable + @Override + public AssetManager apply(@Nullable ResourceResolver input) { + return assetManager; + } + }); + + context.create().resource("/content/dam", JcrConstants.JCR_PRIMARYTYPE, "sling:Folder"); + context.resourceResolver().commit(); + ingestor = new FileAssetIngestor(context.getService(MimeTypeService.class)); + ingestor.jcrBasePath = "/content/dam"; + ingestor.ignoreFileList = Collections.emptyList(); + ingestor.ignoreExtensionList = Collections.emptyList(); + ingestor.ignoreFolderList = Arrays.asList(".ds_store"); + ingestor.existingAssetAction = AssetIngestor.AssetAction.skip; + + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + CheckedConsumer method = (CheckedConsumer) invocation.getArguments()[0]; + method.accept(context.resourceResolver()); + return null; + } + }).when(actionManager).deferredWithResolver(any(CheckedConsumer.class)); + tempDirectory = Files.createTempDir(); + ingestor.fileBasePath = tempDirectory.getAbsolutePath(); + } + + @After + public void teardown() throws Exception { + FileUtils.deleteDirectory(tempDirectory); + } + + @Test + public void testCreateFoldersWithEmptyDirectory() throws Exception { + ingestor.init(); + ingestor.createFolders(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertFalse(context.resourceResolver().getResource("/content/dam").hasChildren()); + + verify(actionManager).setCurrentItem(currentItemCaptor.capture()); + assertEquals(1, currentItemCaptor.getAllValues().size()); + assertEquals(tempDirectory.getAbsolutePath(), currentItemCaptor.getValue()); + + } + + @Test + public void testCreateFolders() throws Exception { + ingestor.init(); + addFile(tempDirectory, "image.png", "/img/test.png"); + File folder1 = mkdir(tempDirectory, "folder1"); + addFile(folder1, "image.png", "/img/test.png"); + File folder2 = mkdir(tempDirectory, "folder2"); + File folder3 = mkdir(folder2, "folder3"); + addFile(folder3, "image.png", "/img/test.png"); + + java.nio.file.Files.walk(tempDirectory.toPath()).forEach(p -> System.out.println(p.toString())); + + ingestor.createFolders(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertEquals(3, ingestor.folderCount.get()); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder1")); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder2")); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder2/folder3")); + + verify(actionManager, times(4)).setCurrentItem(currentItemCaptor.capture()); + assertThat(currentItemCaptor.getAllValues(), + containsInAnyOrder(tempDirectory.getAbsolutePath(), folder1.getAbsolutePath(), folder2.getAbsolutePath(), folder3.getAbsolutePath())); + } + + + @Test + public void testImportAssetsWithEmptyDirectory() throws Exception { + ingestor.init(); + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertFalse(context.resourceResolver().getResource("/content/dam").hasChildren()); + + verify(actionManager).setCurrentItem(currentItemCaptor.capture()); + assertEquals(1, currentItemCaptor.getAllValues().size()); + assertEquals(tempDirectory.getAbsolutePath(), currentItemCaptor.getValue()); + + } + + @Test + public void testImportAssetsWithDirectoryContainingJustFolders() throws Exception { + ingestor.init(); + mkdir(tempDirectory, "folder1"); + mkdir(mkdir(tempDirectory, "folder2"), "folder3"); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertFalse(context.resourceResolver().getResource("/content/dam").hasChildren()); + + verify(actionManager).setCurrentItem(currentItemCaptor.capture()); + assertEquals(1, currentItemCaptor.getAllValues().size()); + assertEquals(tempDirectory.getAbsolutePath(), currentItemCaptor.getValue()); + + } + + @Test + public void testImportAssets() throws Exception { + ingestor.init(); + File rootImage = addFile(tempDirectory, "image.png", "/img/test.png"); + File folder1 = mkdir(tempDirectory, "folder1"); + File folder1Image = addFile(folder1, "image.png", "/img/test.png"); + File folder2 = mkdir(tempDirectory, "folder2"); + File folder3 = mkdir(folder2, "folder3"); + File folder3Image = addFile(folder3, "image.png", "/img/test.png"); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertEquals(3, ingestor.assetCount.get()); + assertEquals(3, ingestor.folderCount.get()); + assertEquals(FILE_SIZE * 3, ingestor.totalImportedData.get()); + verify(assetManager, times(3)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertThat(assetPathCaptor.getAllValues(), + containsInAnyOrder("/content/dam/folder1/image.png", "/content/dam/folder2/folder3/image.png", "/content/dam/image.png")); + + verify(actionManager, times(4)).setCurrentItem(currentItemCaptor.capture()); + assertThat(currentItemCaptor.getAllValues(), + containsInAnyOrder(tempDirectory.getAbsolutePath(), folder1Image.getAbsolutePath(), folder3Image.getAbsolutePath(), rootImage.getAbsolutePath())); + } + + + @Test + public void testImportAssetsToNewRootFolder() throws Exception { + ingestor.jcrBasePath = "/content/dam/test"; + ingestor.init(); + File rootImage = addFile(tempDirectory, "image.png", "/img/test.png"); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + + assertNull(context.resourceResolver().getResource("/content/dam/test").getValueMap().get("jcr:title")); + assertEquals(1, ingestor.assetCount.get()); + assertEquals(0, ingestor.folderCount.get()); + assertEquals(FILE_SIZE, ingestor.totalImportedData.get()); + verify(assetManager, times(1)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertEquals("/content/dam/test/image.png", assetPathCaptor.getValue()); + + verify(actionManager, times(2)).setCurrentItem(currentItemCaptor.capture()); + assertThat(currentItemCaptor.getAllValues(), + containsInAnyOrder(tempDirectory.getAbsolutePath(), rootImage.getAbsolutePath())); + } + + + @Test + public void testImportAssetsToExistingRootFolder() throws Exception { + ingestor.jcrBasePath = "/content/dam/test"; + ingestor.init(); + context.create().resource("/content/dam/test", "jcr:primaryType", "sling:Folder", "jcr:title", "testTitle"); + File rootImage = addFile(tempDirectory, "image.png", "/img/test.png"); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + + assertEquals("testTitle", context.resourceResolver().getResource("/content/dam/test").getValueMap().get("jcr:title")); + assertEquals(1, ingestor.assetCount.get()); + assertEquals(0, ingestor.folderCount.get()); + assertEquals(FILE_SIZE, ingestor.totalImportedData.get()); + verify(assetManager, times(1)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertEquals("/content/dam/test/image.png", assetPathCaptor.getValue()); + + verify(actionManager, times(2)).setCurrentItem(currentItemCaptor.capture()); + assertThat(currentItemCaptor.getAllValues(), + containsInAnyOrder(tempDirectory.getAbsolutePath(), rootImage.getAbsolutePath())); + } + + private File mkdir(File dir, String name) { + File newDir = new File(dir, name); + newDir.mkdir(); + return newDir; + } + + private File addFile(File dir, String name, String resourcePath) throws IOException { + File newFile = new File(dir, name); + FileUtils.copyInputStreamToFile(getClass().getResourceAsStream(resourcePath), newFile); + return newFile; + } + +} diff --git a/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorUtilitiesTest.java b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorUtilitiesTest.java new file mode 100644 index 0000000000..aa42c08ca5 --- /dev/null +++ b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/FileAssetIngestorUtilitiesTest.java @@ -0,0 +1,137 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.google.common.io.Files; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; + +import static org.junit.Assert.*; + +/** + * Test of utility methods in FileAssetIngestor. Separated out from FileAssetIngestorTest for minimal bootstrapping + */ +public class FileAssetIngestorUtilitiesTest { + + private FileAssetIngestor ingestor; + + private File tempDirectory; + + @Before + public void setup() throws Exception { + ingestor = new FileAssetIngestor(null); + ingestor.jcrBasePath = "/content/dam"; + tempDirectory = Files.createTempDir(); + ingestor.fileBasePath = tempDirectory.getAbsolutePath(); + ingestor.init(); + } + + @After + public void teardown() throws Exception { + FileUtils.deleteDirectory(tempDirectory); + } + + @Test + public void testHierarchialElementForFolder() { + File folder1 = new File(tempDirectory, "folder1"); + folder1.mkdir(); + File folder2 = new File(folder1, "folder2"); + folder2.mkdir(); + + AssetIngestor.HierarchialElement el = ingestor.new FileHierarchialElement(folder2); + assertEquals(folder2.getAbsolutePath(), el.getItemName()); + assertTrue(el.isFolder()); + assertFalse(el.isFile()); + assertEquals("/content/dam/folder1/folder2", el.getNodePath()); + assertEquals("folder2", el.getName()); + + AssetIngestor.HierarchialElement parent = el.getParent(); + assertEquals(folder1.getAbsolutePath(), parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder1", parent.getName()); + + assertNull(parent.getParent()); + } + + @Test + public void testHierarchialElementForFile() throws Exception { + File folder1 = new File(tempDirectory, "folder1"); + folder1.mkdir(); + File folder2 = new File(folder1, "folder2"); + folder2.mkdir(); + File image = new File(folder2, "image.png"); + FileUtils.writeByteArrayToFile(image, new byte[0]); + + AssetIngestor.HierarchialElement el = ingestor.new FileHierarchialElement(image); + assertEquals(image.getAbsolutePath(), el.getItemName()); + assertFalse(el.isFolder()); + assertTrue(el.isFile()); + assertEquals("image.png", el.getName()); + + AssetIngestor.HierarchialElement parent = el.getParent(); + assertEquals(folder2.getAbsolutePath(), parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder2", parent.getName()); + + parent = parent.getParent(); + assertEquals(folder1.getAbsolutePath(), parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder1", parent.getName()); + + assertNull(parent.getParent()); + } + @Test + public void testHierarchialElementForFileInRoot() throws Exception { + File image = new File(tempDirectory, "image.png"); + FileUtils.writeByteArrayToFile(image, new byte[0]); + AssetIngestor.HierarchialElement el = ingestor.new FileHierarchialElement(image); + assertEquals(image.getAbsolutePath(), el.getItemName()); + assertFalse(el.isFolder()); + assertTrue(el.isFile()); + assertEquals("image.png", el.getName()); + + assertNull(el.getParent()); + } + + @Test + public void testHierarchialElementForFolderInRoot() { + File folder1 = new File(tempDirectory, "folder1"); + folder1.mkdir(); + AssetIngestor.HierarchialElement el = ingestor.new FileHierarchialElement(folder1); + assertEquals(folder1.getAbsolutePath(), el.getItemName()); + assertTrue(el.isFolder()); + assertFalse(el.isFile()); + assertEquals("folder1", el.getName()); + + assertNull(el.getParent()); + } + +} diff --git a/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorTest.java b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorTest.java new file mode 100644 index 0000000000..7a17147db1 --- /dev/null +++ b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorTest.java @@ -0,0 +1,319 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.adobe.acs.commons.fam.ActionManager; +import com.adobe.acs.commons.functions.CheckedConsumer; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.S3ClientOptions; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.day.cq.dam.api.AssetManager; +import com.google.common.base.Function; +import io.findify.s3mock.S3Mock; +import me.alexpanov.net.FreePortFinder; +import org.apache.jackrabbit.JcrConstants; +import org.apache.sling.api.resource.PersistenceException; +import org.apache.sling.api.resource.ResourceResolver; +import org.apache.sling.commons.mime.MimeTypeService; +import org.apache.sling.testing.mock.sling.ResourceResolverType; +import org.apache.sling.testing.mock.sling.junit.SlingContext; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.runners.MockitoJUnitRunner; +import org.mockito.stubbing.Answer; + +import javax.annotation.Nullable; +import java.io.ByteArrayInputStream; +import java.util.Arrays; +import java.util.Collections; + +import static org.mockito.Mockito.*; +import static org.junit.Assert.*; + +@RunWith(MockitoJUnitRunner.class) +public class S3AssetIngestorTest { + + private static final String TEST_BUCKET = "testbucket"; + + private static final int FILE_SIZE = 57797; + + private S3Mock s3Mock; + + @Rule + public final SlingContext context = new SlingContext(ResourceResolverType.JCR_OAK); + + @Mock + private ActionManager actionManager; + + @Mock + private AssetManager assetManager; + + @Captor + private ArgumentCaptor currentItemCaptor; + + @Captor + private ArgumentCaptor assetPathCaptor; + + private S3AssetIngestor ingestor; + + private AmazonS3 s3Client; + + @Before + public void setup() throws PersistenceException { + context.registerAdapter(ResourceResolver.class, AssetManager.class, new Function() { + @Nullable + @Override + public AssetManager apply(@Nullable ResourceResolver input) { + return assetManager; + } + }); + + context.create().resource("/content/dam", JcrConstants.JCR_PRIMARYTYPE, "sling:Folder"); + context.resourceResolver().commit(); + ingestor = new S3AssetIngestor(context.getService(MimeTypeService.class)); + ingestor.jcrBasePath = "/content/dam"; + ingestor.ignoreFileList = Collections.emptyList(); + ingestor.ignoreExtensionList = Collections.emptyList(); + ingestor.ignoreFolderList = Arrays.asList(".ds_store"); + ingestor.existingAssetAction = AssetIngestor.AssetAction.skip; + + int port = FreePortFinder.findFreeLocalPort(); + s3Mock = new S3Mock.Builder().withPort(port).withInMemoryBackend().build(); + s3Mock.start(); + + S3ClientOptions options = S3ClientOptions.builder().setPathStyleAccess(true).build(); + s3Client = new AmazonS3Client(new AnonymousAWSCredentials()); + s3Client.setS3ClientOptions(options); + s3Client.setEndpoint("http://localhost:" + port); + ingestor.s3Client = s3Client; + ingestor.bucket = TEST_BUCKET; + + s3Client.createBucket(TEST_BUCKET); + + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) throws Throwable { + CheckedConsumer method = (CheckedConsumer) invocation.getArguments()[0]; + method.accept(context.resourceResolver()); + return null; + } + }).when(actionManager).deferredWithResolver(any(CheckedConsumer.class)); + } + + @After + public void teardown() { + s3Mock.stop(); + } + + @Test + public void testCreateFoldersWithEmptyBucket() throws Exception { + ingestor.init(); + ingestor.createFolders(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertFalse(context.resourceResolver().getResource("/content/dam").hasChildren()); + + verify(actionManager).setCurrentItem(currentItemCaptor.capture()); + assertEquals(1, currentItemCaptor.getAllValues().size()); + assertEquals(TEST_BUCKET, currentItemCaptor.getValue()); + + } + + @Test + public void testImportAssetsWithEmptyBucket() throws Exception { + ingestor.init(); + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertFalse(context.resourceResolver().getResource("/content/dam").hasChildren()); + + verify(actionManager).setCurrentItem(currentItemCaptor.capture()); + assertEquals(1, currentItemCaptor.getAllValues().size()); + assertEquals(TEST_BUCKET, currentItemCaptor.getValue()); + + } + + @Test + public void testImportAssetsWithBucketContainingJustFolders() throws Exception { + ingestor.init(); + s3Client.putObject(TEST_BUCKET, "folder1/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertFalse(context.resourceResolver().getResource("/content/dam").hasChildren()); + + verify(actionManager).setCurrentItem(currentItemCaptor.capture()); + assertEquals(1, currentItemCaptor.getAllValues().size()); + assertEquals(TEST_BUCKET, currentItemCaptor.getValue()); + + } + + @Test + public void testImportAssets() throws Exception { + ingestor.init(); + s3Client.putObject(TEST_BUCKET, "image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder1/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder1/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertEquals(3, ingestor.assetCount.get()); + assertEquals(3, ingestor.folderCount.get()); + assertEquals(FILE_SIZE * 3, ingestor.totalImportedData.get()); + verify(assetManager, times(3)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertEquals(Arrays.asList("/content/dam/folder1/image.png", "/content/dam/folder2/folder3/image.png", "/content/dam/image.png"), assetPathCaptor.getAllValues()); + + verify(actionManager, times(4)).setCurrentItem(currentItemCaptor.capture()); + assertEquals(Arrays.asList("testbucket", "testbucket:folder1/image.png", "testbucket:folder2/folder3/image.png", "testbucket:image.png"), currentItemCaptor.getAllValues()); + } + + @Test + public void testImportAssetsToNewRootFolder() throws Exception { + ingestor.jcrBasePath = "/content/dam/test"; + ingestor.init(); + s3Client.putObject(TEST_BUCKET, "image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + + assertNull(context.resourceResolver().getResource("/content/dam/test").getValueMap().get("jcr:title")); + assertEquals(1, ingestor.assetCount.get()); + assertEquals(0, ingestor.folderCount.get()); + assertEquals(FILE_SIZE, ingestor.totalImportedData.get()); + verify(assetManager, times(1)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertEquals("/content/dam/test/image.png", assetPathCaptor.getValue()); + + verify(actionManager, times(2)).setCurrentItem(currentItemCaptor.capture()); + assertEquals(Arrays.asList("testbucket", "testbucket:image.png"), currentItemCaptor.getAllValues()); + } + + + @Test + public void testImportAssetsToExistingRootFolder() throws Exception { + ingestor.jcrBasePath = "/content/dam/test"; + ingestor.init(); + context.create().resource("/content/dam/test", "jcr:primaryType", "sling:Folder", "jcr:title", "testTitle"); + s3Client.putObject(TEST_BUCKET, "image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + + assertEquals("testTitle", context.resourceResolver().getResource("/content/dam/test").getValueMap().get("jcr:title")); + assertEquals(1, ingestor.assetCount.get()); + assertEquals(0, ingestor.folderCount.get()); + assertEquals(FILE_SIZE, ingestor.totalImportedData.get()); + verify(assetManager, times(1)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertEquals("/content/dam/test/image.png", assetPathCaptor.getValue()); + + verify(actionManager, times(2)).setCurrentItem(currentItemCaptor.capture()); + assertEquals(Arrays.asList("testbucket", "testbucket:image.png"), currentItemCaptor.getAllValues()); + } + + @Test + public void testImportAssetsWithBasePath() throws Exception { + ingestor.s3BasePath = "folder2/"; + ingestor.init(); + + s3Client.putObject(TEST_BUCKET, "image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder1/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder1/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + + ingestor.importAssets(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertEquals(1, ingestor.assetCount.get()); + assertEquals(1, ingestor.folderCount.get()); + assertEquals(FILE_SIZE, ingestor.totalImportedData.get()); + verify(assetManager, times(1)).createAsset(assetPathCaptor.capture(), any(), any(), eq(false)); + assertEquals("/content/dam/folder3/image.png", assetPathCaptor.getValue()); + + verify(actionManager, times(2)).setCurrentItem(currentItemCaptor.capture()); + assertEquals(Arrays.asList("testbucket:folder2/", "testbucket:folder2/folder3/image.png"), currentItemCaptor.getAllValues()); + } + + @Test + public void testCreateFolders() throws Exception { + ingestor.init(); + s3Client.putObject(TEST_BUCKET, "image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder1/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder1/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "folder2/folder3/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + + ingestor.createFolders(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertEquals(3, ingestor.folderCount.get()); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder1")); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder2")); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder2/folder3")); + + verify(actionManager, times(4)).setCurrentItem(currentItemCaptor.capture()); + assertEquals(Arrays.asList("testbucket", "testbucket:folder1/", "testbucket:folder2/", "testbucket:folder2/folder3/"), currentItemCaptor.getAllValues()); + } + + @Test + public void testCreateFoldersWithBasePath() throws Exception { + ingestor.s3BasePath = "a/"; + ingestor.init(); + s3Client.putObject(TEST_BUCKET, "image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "a/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "a/folder1/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "a/folder1/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "a/folder2/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "a/folder2/folder3/", new ByteArrayInputStream(new byte[0]), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "a/folder2/folder3/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + s3Client.putObject(TEST_BUCKET, "b/image.png", getClass().getResourceAsStream("/img/test.png"), new ObjectMetadata()); + + ingestor.createFolders(actionManager); + + assertFalse(context.resourceResolver().hasChanges()); + assertEquals(3, ingestor.folderCount.get()); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder1")); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder2")); + assertNotNull(context.resourceResolver().getResource("/content/dam/folder2/folder3")); + + verify(actionManager, times(4)).setCurrentItem(currentItemCaptor.capture()); + assertEquals(Arrays.asList("testbucket:a/", "testbucket:a/folder1/", "testbucket:a/folder2/", "testbucket:a/folder2/folder3/"), currentItemCaptor.getAllValues()); + } + +} diff --git a/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorUtilitiesTest.java b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorUtilitiesTest.java new file mode 100644 index 0000000000..3c7839a966 --- /dev/null +++ b/bundle/src/test/java/com/adobe/acs/commons/mcp/impl/processes/S3AssetIngestorUtilitiesTest.java @@ -0,0 +1,188 @@ +/* + * #%L + * ACS AEM Commons Bundle + * %% + * Copyright (C) 2017 Adobe + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package com.adobe.acs.commons.mcp.impl.processes; + +import com.amazonaws.services.s3.model.S3ObjectSummary; +import org.junit.Before; +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Test of utility methods in S3AssetIngestor. Separated out from S3AssetIngestorTest for minimal bootstrapping + */ +public class S3AssetIngestorUtilitiesTest { + + private S3AssetIngestor ingestor; + + @Before + public void setup() { + ingestor = new S3AssetIngestor(null); + ingestor.bucket = "testbucket"; + ingestor.jcrBasePath = "/content/dam"; + } + + @Test + public void testHierarchialElementForFolderNoBasePath() { + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/folder2/")); + assertEquals("testbucket:folder1/folder2/", el.getItemName()); + assertTrue(el.isFolder()); + assertFalse(el.isFile()); + assertEquals("/content/dam/folder1/folder2", el.getNodePath()); + assertEquals("folder2", el.getName()); + + AssetIngestor.HierarchialElement parent = el.getParent(); + assertEquals("testbucket:folder1/", parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder1", parent.getName()); + + assertNull(parent.getParent()); + } + + @Test + public void testHierarchialElementForFolderWithBasePath() { + ingestor.s3BasePath = "folder1/"; + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/folder2/folder3/")); + assertEquals("testbucket:folder1/folder2/folder3/", el.getItemName()); + assertTrue(el.isFolder()); + assertFalse(el.isFile()); + assertEquals("/content/dam/folder2/folder3", el.getNodePath()); + assertEquals("folder3", el.getName()); + + AssetIngestor.HierarchialElement parent = el.getParent(); + assertEquals("testbucket:folder1/folder2/", parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("/content/dam/folder2", parent.getNodePath()); + assertEquals("folder2", parent.getName()); + + assertNull(parent.getParent()); + } + + @Test + public void testHierarchialElementForFileNoBasePath() { + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/folder2/image.png")); + assertEquals("testbucket:folder1/folder2/image.png", el.getItemName()); + assertFalse(el.isFolder()); + assertTrue(el.isFile()); + assertEquals("image.png", el.getName()); + + AssetIngestor.HierarchialElement parent = el.getParent(); + assertEquals("testbucket:folder1/folder2/", parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder2", parent.getName()); + + parent = parent.getParent(); + assertEquals("testbucket:folder1/", parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder1", parent.getName()); + + assertNull(parent.getParent()); + } + + @Test + public void testHierarchialElementForFileWithBasePath() { + ingestor.s3BasePath = "folder1/"; + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/folder2/folder3/image.png")); + assertEquals("testbucket:folder1/folder2/folder3/image.png", el.getItemName()); + assertFalse(el.isFolder()); + assertTrue(el.isFile()); + assertEquals("image.png", el.getName()); + assertEquals("/content/dam/folder2/folder3/image.png", el.getNodePath()); + + AssetIngestor.HierarchialElement parent = el.getParent(); + assertEquals("testbucket:folder1/folder2/folder3/", parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder3", parent.getName()); + assertEquals("/content/dam/folder2/folder3", parent.getNodePath()); + + parent = parent.getParent(); + assertEquals("testbucket:folder1/folder2/", parent.getItemName()); + assertNotNull(parent); + assertTrue(parent.isFolder()); + assertFalse(parent.isFile()); + assertEquals("folder2", parent.getName()); + assertEquals("/content/dam/folder2", parent.getNodePath()); + + assertNull(parent.getParent()); + } + + @Test + public void testHierarchialElementForFileInRootNoBasePath() { + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("image.png")); + assertEquals("testbucket:image.png", el.getItemName()); + assertFalse(el.isFolder()); + assertTrue(el.isFile()); + assertEquals("image.png", el.getName()); + + assertNull(el.getParent()); + } + + @Test + public void testHierarchialElementForFolderInRootNoBasePath() { + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/")); + assertEquals("testbucket:folder1/", el.getItemName()); + assertTrue(el.isFolder()); + assertFalse(el.isFile()); + assertEquals("folder1", el.getName()); + + assertNull(el.getParent()); + } + + @Test + public void testHierarchialElementForFileInRootWithBasePath() { + ingestor.s3BasePath = "folder1/"; + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/image.png")); + assertEquals("testbucket:folder1/image.png", el.getItemName()); + assertFalse(el.isFolder()); + assertTrue(el.isFile()); + assertEquals("image.png", el.getName()); + + assertNull(el.getParent()); + } + + @Test + public void testHierarchialElementForFolderInRootWithBasePath() { + ingestor.s3BasePath = "folder1/"; + AssetIngestor.HierarchialElement el = ingestor.new S3HierarchialElement(forKey("folder1/folder2/")); + assertEquals("testbucket:folder1/folder2/", el.getItemName()); + assertTrue(el.isFolder()); + assertFalse(el.isFile()); + assertEquals("folder2", el.getName()); + + assertNull(el.getParent()); + } + + + private S3ObjectSummary forKey(String key) { + S3ObjectSummary s = new S3ObjectSummary(); + s.setKey(key); + return s; + } +} diff --git a/bundle/src/test/java/com/amazonaws/util/StringUtils.java b/bundle/src/test/java/com/amazonaws/util/StringUtils.java new file mode 100644 index 0000000000..25e1d4dcf7 --- /dev/null +++ b/bundle/src/test/java/com/amazonaws/util/StringUtils.java @@ -0,0 +1,306 @@ +/* + * Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ +package com.amazonaws.util; + +import static com.amazonaws.util.BinaryUtils.copyBytesFrom; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; +import java.text.Collator; +import java.util.Date; +import java.util.Locale; + +/** + * Utilities for converting objects to strings. + * + * This is the latest version of this class (from 1.11.76) as otherwise S3 Mock won't work, + * but we need to depend upon the old version of the S3 SDK for compatibility with AEM + */ +public class StringUtils { + + private static final String DEFAULT_ENCODING = "UTF-8"; + + public static final String COMMA_SEPARATOR = ","; + + public static final Charset UTF8 = Charset.forName(DEFAULT_ENCODING); + + private static final Locale LOCALE_ENGLISH = Locale.ENGLISH; + + // white space character that match Pattern.compile("\\s") + private static final char CHAR_SPACE = ' '; + private static final char CHAR_TAB = '\t'; + private static final char CHAR_NEW_LINE = '\n'; + private static final char CHAR_VERTICAL_TAB = '\u000b'; + private static final char CHAR_CARRIAGE_RETURN = '\r'; + private static final char CHAR_FORM_FEED = '\f'; + + public static Integer toInteger(StringBuilder value) { + return Integer.parseInt(value.toString()); + } + + public static String toString(StringBuilder value) { + return value.toString(); + } + + public static Boolean toBoolean(StringBuilder value) { + return Boolean.getBoolean(value.toString()); + } + + public static String fromInteger(Integer value) { + return Integer.toString(value); + } + + public static String fromLong(Long value) { + return Long.toString(value); + } + + public static String fromString(String value) { + return value; + } + + public static String fromBoolean(Boolean value) { + return Boolean.toString(value); + } + + public static String fromBigInteger(BigInteger value) { + return value.toString(); + } + + public static String fromBigDecimal(BigDecimal value) { + return value.toString(); + } + + + public static BigInteger toBigInteger(String s) { + return new BigInteger(s); + } + + public static BigDecimal toBigDecimal(String s) { + return new BigDecimal(s); + } + + public static String fromFloat(Float value) { + return Float.toString(value); + } + + /** + * Converts the specified date to an ISO 8601 timestamp string and returns + * it. + * + * @param value + * The date to format as an ISO 8601 timestamp string. + * + * @return An ISO 8601 timestamp string created from the specified date. + */ + public static String fromDate(Date value) { + return DateUtils.formatISO8601Date(value); + } + + /** + * Returns the string representation of the specified double. + * + * @param d + * The double to represent as a string. + * + * @return The string representation of the specified double. + */ + public static String fromDouble(Double d) { + return Double.toString(d); + } + + /** + * Returns the string representation of the specified Byte. + * + * @param b + * The Byte to represent as a string. + * + * @return The string representation of the specified Byte. + */ + public static String fromByte(Byte b) { + return Byte.toString(b); + } + + /** + * Base64 encodes the data in the specified byte buffer (from the current + * position to the buffer's limit) and returns it as a base64 encoded + * string. + * + * @param byteBuffer + * The data to base64 encode and return as a string; must not be + * null. + * + * @return The base64 encoded contents of the specified byte buffer. + */ + public static String fromByteBuffer(ByteBuffer byteBuffer) { + return Base64.encodeAsString(copyBytesFrom(byteBuffer)); + } + + public static String replace( String originalString, String partToMatch, String replacement ) { + StringBuilder buffer = new StringBuilder( originalString.length() ); + buffer.append( originalString ); + + int indexOf = buffer.indexOf( partToMatch ); + while (indexOf != -1) { + buffer = buffer.replace(indexOf, indexOf + partToMatch.length(), replacement); + indexOf = buffer.indexOf(partToMatch, indexOf + replacement.length()); + } + + return buffer.toString(); + } + + /** + * Joins the strings in parts with joiner between each string + * @param joiner the string to insert between the strings in parts + * @param parts the parts to join + */ + public static String join(String joiner, String... parts) { + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < parts.length; i++) { + builder.append(parts[i]); + if (i < parts.length - 1) { + builder.append(joiner); + } + } + return builder.toString(); + } + + /** + * A null-safe trim method. If the input string is null, returns null; + * otherwise returns a trimmed version of the input. + */ + public static String trim(String value) { + if (value == null) { + return null; + } + return value.trim(); + } + + /** + * @return true if the given value is either null or the empty string + */ + public static boolean isNullOrEmpty(String value) { + return value == null || value.isEmpty(); + } + + /** + * @return true if the given value is non-null and non-empty + */ + public static boolean hasValue(String str) { + return !isNullOrEmpty(str); + } + + /** + * Converts a given String to lower case with Locale.ENGLISH + * + * @param str the string to be converted to lower case + * @return the lower case of string, or itself if string is null/empty + */ + public static String lowerCase(String str) { + if(isNullOrEmpty(str)) { + return str; + } + return str.toLowerCase(LOCALE_ENGLISH); + } + + /** + * Converts a given String to upper case with Locale.ENGLISH + * + * @param str the string to be converted to upper case + * @return the upper case of string, or itself if string is null/empty + */ + public static String upperCase(String str) { + if(isNullOrEmpty(str)) { + return str; + } + return str.toUpperCase(LOCALE_ENGLISH); + } + + /** + * Compare two strings with Locale.ENGLISH + * This method is preferred over String.compareTo() method. + * @param str1 String 1 + * @param str2 String 2 + * @return negative integer if str1 lexicographically precedes str2 + * positive integer if str1 lexicographically follows str2 + * 0 if both strings are equal + * @throws IllegalArgumentException throws exception if both or either of the strings is null + */ + public static int compare(String str1, String str2) { + if( str1 == null || str2 == null) { + throw new IllegalArgumentException("Arguments cannot be null"); + } + + Collator collator = Collator.getInstance(LOCALE_ENGLISH); + return collator.compare(str1, str2); + } + + /** + * Tests a char to see if is it whitespace. + * This method considers the same characters to be white + * space as the Pattern class does when matching \s + * + * @param ch the character to be tested + * @return true if the character is white space, false otherwise. + */ + private static boolean isWhiteSpace(final char ch) { + if (ch == CHAR_SPACE) return true; + if (ch == CHAR_TAB) return true; + if (ch == CHAR_NEW_LINE) return true; + if (ch == CHAR_VERTICAL_TAB) return true; + if (ch == CHAR_CARRIAGE_RETURN) return true; + if (ch == CHAR_FORM_FEED) return true; + return false; + } + + /** + * This method appends a string to a string builder and collapses contiguous + * white space is a single space. + * + * This is equivalent to: + * destination.append(source.replaceAll("\\s+", " ")) + * but does not create a Pattern object that needs to compile the match + * string; it also prevents us from having to make a Matcher object as well. + * + */ + public static void appendCompactedString(final StringBuilder destination, final String source) { + boolean previousIsWhiteSpace = false; + int length = source.length(); + + for (int i = 0; i < length; i++) { + char ch = source.charAt(i); + if (isWhiteSpace(ch)) { + if (previousIsWhiteSpace) { + continue; + } + destination.append(CHAR_SPACE); + previousIsWhiteSpace = true; + } else { + destination.append(ch); + previousIsWhiteSpace = false; + } + } + } + + /** + * Performs a case insensitive comparison and returns true if the data + * begins with the given sequence. + */ + public static boolean beginsWithIgnoreCase(final String data, final String seq) { + return data.regionMatches(true, 0, seq, 0, seq.length()); + } + +} diff --git a/bundle/src/test/resources/logback-test.xml b/bundle/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..afaebf8e17 --- /dev/null +++ b/bundle/src/test/resources/logback-test.xml @@ -0,0 +1,14 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + \ No newline at end of file diff --git a/content/src/main/content/jcr_root/apps/cq/core/content/nav/tools/acs-commons/users-to-csv/.content.xml b/content/src/main/content/jcr_root/apps/cq/core/content/nav/tools/acs-commons/users-to-csv/.content.xml deleted file mode 100644 index c06b791da2..0000000000 --- a/content/src/main/content/jcr_root/apps/cq/core/content/nav/tools/acs-commons/users-to-csv/.content.xml +++ /dev/null @@ -1,8 +0,0 @@ - - diff --git a/pom.xml b/pom.xml index 17d8f951fa..67a23ff07b 100644 --- a/pom.xml +++ b/pom.xml @@ -279,6 +279,12 @@ 1.6.4 test + + org.slf4j + jcl-over-slf4j + 1.6.4 + test + javax.servlet javax.servlet-api