List and copy log files from running container

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/EnvironmentUp.java

@@ -40,6 +40,8 @@
 import static io.prestosql.tests.product.launcher.cli.Commands.runCommand;
 import static io.prestosql.tests.product.launcher.env.EnvironmentContainers.COORDINATOR;
 import static io.prestosql.tests.product.launcher.env.EnvironmentContainers.TESTS;
+import static io.prestosql.tests.product.launcher.env.EnvironmentListener.compose;
+import static io.prestosql.tests.product.launcher.env.EnvironmentListener.logCopyingListener;
 import static io.prestosql.tests.product.launcher.env.EnvironmentListener.loggingListener;
 import static java.util.Objects.requireNonNull;
 import static picocli.CommandLine.Mixin;
@@ -142,9 +144,7 @@ public void run()
             }
 
             Optional<Path> environmentLogPath = logsDirBase.map(dir -> dir.resolve(environment));
-            environmentLogPath.ifPresent(builder::exposeLogsInHostPath);
-
-            Environment environment = builder.build(loggingListener());
+            Environment environment = builder.build(compose(loggingListener(), logCopyingListener(environmentLogPath)));
             environment.start();
 
             if (background) {

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/cli/TestRun.java

@@ -45,6 +45,8 @@
 import static io.prestosql.tests.product.launcher.docker.ContainerUtil.exposePort;
 import static io.prestosql.tests.product.launcher.env.DockerContainer.cleanOrCreateHostPath;
 import static io.prestosql.tests.product.launcher.env.EnvironmentContainers.TESTS;
+import static io.prestosql.tests.product.launcher.env.EnvironmentListener.compose;
+import static io.prestosql.tests.product.launcher.env.EnvironmentListener.logCopyingListener;
 import static io.prestosql.tests.product.launcher.env.EnvironmentListener.loggingListener;
 import static io.prestosql.tests.product.launcher.env.common.Standard.CONTAINER_TEMPTO_PROFILE_CONFIG;
 import static java.util.Objects.requireNonNull;
@@ -240,8 +242,7 @@ private Environment getEnvironment()
 
             environmentConfig.extendEnvironment(this.environment).ifPresent(extender -> extender.extendEnvironment(environment));
 
-            logsDirBase.ifPresent(environment::exposeLogsInHostPath);
-            return environment.build(loggingListener());
+            return environment.build(compose(loggingListener(), logCopyingListener(logsDirBase)));
         }
 
         private static Iterable<? extends String> reportsDirOptions(Path path)

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/DockerContainer.java

@@ -14,6 +14,7 @@
 package io.prestosql.tests.product.launcher.env;
 
 import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.google.common.base.Splitter;
 import com.google.common.base.Stopwatch;
 import com.google.common.io.RecursiveDeleteOption;
 import io.airlift.log.Logger;
@@ -33,11 +34,12 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Stream;
 
 import static com.google.common.io.MoreFiles.deleteRecursively;
+import static java.lang.String.format;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.testcontainers.containers.BindMode.READ_WRITE;
 
 public class DockerContainer
         extends FixedHostPortGenericContainer<DockerContainer>
@@ -116,17 +118,6 @@ public DockerContainer withExposedLogPaths(String... logPaths)
         return this;
     }
 
-    public void exposeLogsInHostPath(Path hostBasePath)
-    {
-        for (String containerLogPath : logPaths) {
-            Path hostLogPath = Paths.get(hostBasePath.toString(), containerLogPath);
-            cleanOrCreateHostPath(hostLogPath);
-            withFileSystemBind(hostLogPath.toString(), containerLogPath, READ_WRITE);
-        }
-
-        logPaths = null;
-    }
-
     @Override
     protected void containerIsStarting(InspectContainerResponse containerInfo)
     {
@@ -167,6 +158,58 @@ public void clearDependencies()
         dependencies.clear();
     }
 
+    public void copyLogsToHostPath(Path hostPath)
+    {
+        if (!isRunning()) {
+            log.warn("Could not copy files from stopped container %s", logicalName);
+            return;
+        }
+
+        log.info("Copying container %s logs to '%s'", logicalName, hostPath);
+
+        Path hostLogPath = Paths.get(hostPath.toString(), logicalName);
+        cleanOrCreateHostPath(hostLogPath);
+
+        for (String containerLogPath : logPaths) {
+            try {
+                listFilesInContainer(containerLogPath).forEach(filename ->
+                        copyFileFromContainer(filename, hostLogPath));
+            }
+            catch (Exception e) {
+                log.warn("Could not copy logs from %s to '%s': %s", logicalName, hostPath, e);
+            }
+        }
+    }
+
+    private void copyFileFromContainer(String filename, Path rootHostPath)
+    {
+        Path targetPath = rootHostPath.resolve(filename.replaceFirst("^\\/", ""));
+
+        log.info("Copying file %s to %s", filename, targetPath);
+        ensurePathExists(targetPath.getParent());
+        copyFileFromContainer(filename, targetPath.toString());
+    }
+
+    private Stream<String> listFilesInContainer(String path)
+    {
+        try {
+            ExecResult result = execInContainer("/usr/bin/find", path, "-type", "f", "-print");
+
+            if (result.getExitCode() == 0L) {
+                return Splitter.on("\n")
+                        .omitEmptyStrings()
+                        .splitToStream(result.getStdout());
+            }
+
+            throw new RuntimeException(format("Could not list files in %s: %s", path, result.getStderr()));
+        }
+        catch (Exception e) {
+            log.warn("Could not list files in container '%s': %s", logicalName, e);
+        }
+
+        return Stream.empty();
+    }
+
     @Override
     public String toString()
     {

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/Environment.java

@@ -33,7 +33,6 @@
 import java.io.PrintStream;
 import java.io.UnsupportedEncodingException;
 import java.nio.charset.Charset;
-import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
@@ -307,16 +306,5 @@ private Environment build(Optional<EnvironmentListener> listener)
 
             return new Environment(name, containers, listener);
         }
-
-        public Builder exposeLogsInHostPath(Path basePath)
-        {
-            log.info("Exposing environment '%s' logs in: '%s'", name, basePath);
-
-            return configureContainers(dockerContainer -> {
-                Path containerLogPath = basePath.resolve(dockerContainer.getLogicalName());
-                log.info("Exposing container '%s' logs in host directory '%s'", dockerContainer.getLogicalName(), containerLogPath);
-                dockerContainer.exposeLogsInHostPath(containerLogPath);
-            });
-        }
     }
 }

presto-product-tests-launcher/src/main/java/io/prestosql/tests/product/launcher/env/EnvironmentListener.java

@@ -16,7 +16,9 @@
 import com.github.dockerjava.api.command.InspectContainerResponse;
 import io.airlift.log.Logger;
 
+import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.Optional;
 
 public interface EnvironmentListener
 {
@@ -161,4 +163,24 @@ public void containerStopped(DockerContainer container, InspectContainerResponse
             }
         };
     }
+
+    static EnvironmentListener logCopyingListener(Optional<Path> logBaseDir)
+    {
+        if (logBaseDir.isEmpty()) {
+            return noop();
+        }
+
+        return new EnvironmentListener() {
+            @Override
+            public void containerStopping(DockerContainer container, InspectContainerResponse response)
+            {
+                container.copyLogsToHostPath(logBaseDir.get());
+            }
+        };
+    }
+
+    static EnvironmentListener noop()
+    {
+        return new EnvironmentListener() {};
+    }
 }