spring-projectsGH-3482: (S)FTP: Fix Recursive LS (ARFOG)

Resolves spring-projects#3482 `.` and `..` should be ignored when recursing. **cherry-pick to 5.4.x, 5.3.x**
garyrussell · Feb 1, 2021 · 9037232 · 9037232
1 parent 98fbf62
commit 9037232
Show file tree

Hide file tree

Showing 3 changed files with 112 additions and 6 deletions.
diff --git a/...rg/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java b/...rg/springframework/integration/file/remote/gateway/AbstractRemoteFileOutboundGateway.java
@@ -974,18 +974,23 @@ protected final List<F> filterFiles(F[] files) {
 	private void processFile(Session<F> session, String directory, String subDirectory, List<F> lsFiles,
 			boolean recursion, F file) throws IOException {
 
+		String fileName = getFilename(file);
+		String fileSep = this.remoteFileTemplate.getRemoteFileSeparator();
+		boolean isDots = ".".equals(fileName)
+				|| "..".equals(fileName)
+				|| fileName.endsWith(fileSep + ".")
+				|| fileName.endsWith(fileSep + "..");
 		if (this.options.contains(Option.SUBDIRS) || !isDirectory(file)) {
-			if (recursion && StringUtils.hasText(subDirectory)) {
+			if (recursion && StringUtils.hasText(subDirectory) && (!isDots || this.options.contains(Option.ALL))) {
 				lsFiles.add(enhanceNameWithSubDirectory(file, subDirectory));
 			}
-			else {
+			else if (this.options.contains(Option.ALL) || !isDots){
 				lsFiles.add(file);
 			}
 		}
-		String fileName = getFilename(file);
-		if (recursion && isDirectory(file) && !(".".equals(fileName)) && !("..".equals(fileName))) {
+		if (recursion && isDirectory(file) && !isDots) {
 			lsFiles.addAll(listFilesInRemoteDir(session, directory,
-					subDirectory + fileName + this.remoteFileTemplate.getRemoteFileSeparator()));
+					subDirectory + fileName + fileSep));
 		}
 	}
 

diff --git a/...st/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml b/...st/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests-context.xml
@@ -59,6 +59,39 @@
 							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
 							  reply-channel="output"/>
 
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
+							  request-channel="inboundLSRecursive"
+							  command="ls"
+							  expression="payload"
+							  command-options="-R -dirs"
+							  mode="REPLACE_IF_MODIFIED"
+							  filter="dotStarDotTxtFilter"
+							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
+							  reply-channel="output"/>
+
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
+							  request-channel="inboundLSRecursiveALL"
+							  command="ls"
+							  expression="payload"
+							  command-options="-a -R -dirs"
+							  mode="REPLACE_IF_MODIFIED"
+							  filter="dotStarDotTxtFilter"
+							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
+							  reply-channel="output"/>
+
+	<int-sftp:outbound-gateway session-factory="sftpSessionFactory"
+							  request-channel="inboundLSRecursiveNoDirs"
+							  command="ls"
+							  expression="payload"
+							  command-options="-R"
+							  mode="REPLACE_IF_MODIFIED"
+							  filter="dotStarDotTxtFilter"
+							  local-directory-expression="@extraConfig.targetLocalDirectoryName + #remoteDirectory"
+							  local-filename-generator-expression="#remoteFileName.replaceFirst('sftpSource', 'localTarget')"
+							  reply-channel="output"/>
+
 	<bean id="dotStarDotTxtFilter"
 				class="org.springframework.integration.sftp.filters.SftpRegexPatternFileListFilter">
 		<constructor-arg value="^.*\.txt$" />

diff --git a/.../src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java b/.../src/test/java/org/springframework/integration/sftp/outbound/SftpServerOutboundTests.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2020 the original author or authors.
+ * Copyright 2013-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -35,6 +35,7 @@
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.BeforeEach;
@@ -60,6 +61,7 @@
 import org.springframework.integration.sftp.server.SessionClosedEvent;
 import org.springframework.integration.sftp.server.SessionOpenedEvent;
 import org.springframework.integration.sftp.session.DefaultSftpSessionFactory;
+import org.springframework.integration.sftp.session.SftpFileInfo;
 import org.springframework.integration.sftp.session.SftpRemoteFileTemplate;
 import org.springframework.integration.support.MessageBuilder;
 import org.springframework.integration.test.util.TestUtils;
@@ -100,6 +102,15 @@ public class SftpServerOutboundTests extends SftpTestSupport {
 	@Autowired
 	private DirectChannel inboundMGetRecursive;
 
+	@Autowired
+	private DirectChannel inboundLSRecursive;
+
+	@Autowired
+	private DirectChannel inboundLSRecursiveALL;
+
+	@Autowired
+	private DirectChannel inboundLSRecursiveNoDirs;
+
 	@Autowired
 	private DirectChannel inboundMGetRecursiveFiltered;
 
@@ -254,6 +265,63 @@ public void testInt3172LocalDirectoryExpressionMGETRecursive() throws IOExceptio
 		FileUtils.copyInputStreamToFile(new ByteArrayInputStream(localAsString.getBytes()), secondRemote);
 	}
 
+	@Test
+	@SuppressWarnings("unchecked")
+	void testLSRecursive() throws IOException {
+		String dir = "sftpSource/";
+		this.inboundLSRecursive.send(new GenericMessage<Object>(dir));
+		Message<?> result = this.output.receive(1000);
+		assertThat(result).isNotNull();
+		List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
+		assertThat(files).hasSize(4);
+		assertThat(files.stream()
+				.map(fi -> fi.getFilename())
+				.collect(Collectors.toList())).contains(
+						" sftpSource1.txt",
+						"sftpSource2.txt",
+						"subSftpSource",
+						"subSftpSource/subSftpSource1.txt");
+	}
+
+	@Test
+	@SuppressWarnings("unchecked")
+	void testLSRecursiveALL() throws IOException {
+		String dir = "sftpSource/";
+		this.inboundLSRecursiveALL.send(new GenericMessage<Object>(dir));
+		Message<?> result = this.output.receive(1000);
+		assertThat(result).isNotNull();
+		List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
+		assertThat(files).hasSize(8);
+		assertThat(files.stream()
+				.map(fi -> fi.getFilename())
+				.collect(Collectors.toList())).contains(
+						" sftpSource1.txt",
+						"sftpSource2.txt",
+						"subSftpSource",
+						"subSftpSource/subSftpSource1.txt",
+						".",
+						"..",
+						"subSftpSource/.",
+						"subSftpSource/..");
+	}
+
+	@Test
+	@SuppressWarnings("unchecked")
+	void testLSRecursiveNoDirs() throws IOException {
+		String dir = "sftpSource/";
+		this.inboundLSRecursiveNoDirs.send(new GenericMessage<Object>(dir));
+		Message<?> result = this.output.receive(1000);
+		assertThat(result).isNotNull();
+		List<SftpFileInfo> files = (List<SftpFileInfo>) result.getPayload();
+		assertThat(files).hasSize(3);
+		assertThat(files.stream()
+				.map(fi -> fi.getFilename())
+				.collect(Collectors.toList())).contains(
+						" sftpSource1.txt",
+						"sftpSource2.txt",
+						"subSftpSource/subSftpSource1.txt");
+	}
+
 	private long setModifiedOnSource1() {
 		File firstRemote = new File(getSourceRemoteDirectory(), " sftpSource1.txt");
 		firstRemote.setLastModified(System.currentTimeMillis() - 1_000_000);