From 8398b21787d209685d143a599d109397c78ae87e Mon Sep 17 00:00:00 2001 From: Bulgakov Alexander Date: Wed, 17 Apr 2024 21:49:03 +0300 Subject: [PATCH] refactor: extracts Kubernetes related classes --- .../m4gshm/testcontainers/AbstractPod.java | 131 ++---------------- .../testcontainers/KubernetesUtils.java | 102 ++++++++++++++ .../m4gshm/testcontainers/MongoDBPod.java | 2 - .../testcontainers/PodContainerDelegate.java | 30 +++- .../testcontainers/PodContainerUtils.java | 11 +- 5 files changed, 149 insertions(+), 127 deletions(-) diff --git a/src/main/java/io/github/m4gshm/testcontainers/AbstractPod.java b/src/main/java/io/github/m4gshm/testcontainers/AbstractPod.java index 91a475f..943e4d4 100644 --- a/src/main/java/io/github/m4gshm/testcontainers/AbstractPod.java +++ b/src/main/java/io/github/m4gshm/testcontainers/AbstractPod.java @@ -9,17 +9,13 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientBuilder; import io.fabric8.kubernetes.client.LocalPortForward; -import io.fabric8.kubernetes.client.RequestConfig; import io.fabric8.kubernetes.client.dsl.PodResource; -import io.fabric8.kubernetes.client.dsl.internal.OperationSupport; import lombok.Getter; import lombok.NonNull; import lombok.Setter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.jetbrains.annotations.NotNull; -import org.testcontainers.containers.Container; import org.testcontainers.containers.output.OutputFrame; import org.testcontainers.images.builder.Transferable; import org.testcontainers.shaded.com.google.common.hash.Hashing; @@ -30,9 +26,7 @@ import java.io.InputStream; import java.net.Inet6Address; import java.net.InetAddress; -import java.nio.charset.Charset; import java.time.Duration; -import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -46,10 +40,10 @@ import static io.github.m4gshm.testcontainers.KubernetesUtils.UNKNOWN; import static io.github.m4gshm.testcontainers.KubernetesUtils.createPod; import static io.github.m4gshm.testcontainers.KubernetesUtils.escapeQuotes; -import static io.github.m4gshm.testcontainers.KubernetesUtils.getError; import static io.github.m4gshm.testcontainers.KubernetesUtils.getFirstNotReadyContainer; import static io.github.m4gshm.testcontainers.KubernetesUtils.resource; import static io.github.m4gshm.testcontainers.KubernetesUtils.shellQuote; +import static io.github.m4gshm.testcontainers.KubernetesUtils.uploadStdIn; import static io.github.m4gshm.testcontainers.KubernetesUtils.waitEmptyQueue; import static io.github.m4gshm.testcontainers.PodContainerUtils.config; import static java.lang.Boolean.getBoolean; @@ -223,7 +217,7 @@ public String getDockerImageName() { protected abstract void containerIsStarted(InspectContainerResponse containerInfo, boolean reused); - protected abstract @NotNull List getEnvVars(); + protected abstract List getEnvVars(); protected abstract String[] getCommandParts(); @@ -266,16 +260,19 @@ public void copyFileToContainer(Transferable transferable, String containerPath) @SneakyThrows public void uploadTmpTar(byte[] payload) { + var podName = this.podName; + var podResource = pod; + var tmpDir = "/tmp"; - var tarName = tmpDir + "/" + this.podName + ".tar"; + var tarName = tmpDir + "/" + podName + ".tar"; log.debug("tar uploading {}", tarName); var escapedTarPath = escapeQuotes(tarName); - try (var exec = pod.terminateOnError().exec("touch", escapedTarPath)) { + try (var exec = podResource.terminateOnError().exec("touch", escapedTarPath)) { waitEmptyQueue(exec); } - uploadStdIn(payload, escapedTarPath); + uploadStdIn(pod, getRequestTimeout(), escapedTarPath, payload); // uploadBase64(payload, escapedTarPath); var unpackDir = "/"; @@ -284,7 +281,7 @@ public void uploadTmpTar(byte[] payload) { var out = new ByteArrayOutputStream(); var err = new ByteArrayOutputStream(); - try (var exec = pod.redirectingInput().writingOutput(out).writingError(err).exec("sh", "-c", extractTarCmd)) { + try (var exec = podResource.redirectingInput().writingOutput(out).writingError(err).exec("sh", "-c", extractTarCmd)) { waitEmptyQueue(exec); var exitedCode = exec.exitCode(); var exitCode = exitedCode.get(getRequestTimeout(), MILLISECONDS); @@ -301,40 +298,6 @@ public void uploadTmpTar(byte[] payload) { } } - @SneakyThrows - private void uploadStdIn(byte[] payload, String escapedTarPath) { - try (var exec = pod.redirectingInput().terminateOnError().exec("cp", "/dev/stdin", escapedTarPath)) { - var input = exec.getInput(); - input.write(payload); - input.flush(); - waitEmptyQueue(exec); - checkSize(escapedTarPath, payload.length); - } - } - - @SneakyThrows - protected void checkSize(String filePath, long expected) { - var size = getSize(filePath); - if (size != expected) { - Thread.sleep(100); - size = getSize(filePath); - } - if (size != expected) { - throw new UploadFileException("Unexpected file size " + size + ", expected " + expected + ", file '" + filePath + "'"); - } - } - - @SneakyThrows - private long getSize(String filePath) { - var byteCount = new ByteArrayOutputStream(); - try (var exec = pod.writingOutput(byteCount).terminateOnError().exec("sh", "-c", "wc -c < " + filePath)) { - var exitCode = exec.exitCode().get(getRequestTimeout(), MILLISECONDS); - var remoteSizeRaw = byteCount.toString(UTF_8).trim(); - waitEmptyQueue(exec); - return Integer.parseInt(remoteSizeRaw); - } - } - protected void waitUntilPodStarted() { var startTime = System.currentTimeMillis(); var pod = getPodResource().get(); @@ -382,7 +345,6 @@ protected void waitUntilPodStarted() { throw new StartPodException("unexpected pod status", pod.getMetadata().getName(), status.getPhase()); } - var firstNotReadyContainer = getFirstNotReadyContainer(status); if (firstNotReadyContainer != null) { @@ -441,7 +403,6 @@ protected int getRequestTimeout() { return kubernetesClient().getConfiguration().getRequestTimeout(); } - public void addHostPort(Integer port, Integer hostPort) { podBuilderFactory.addHostPort(port, hostPort); } @@ -501,80 +462,6 @@ public String getPodIP() { return getPod().map(pod -> pod.getStatus().getHostIP()).orElse(null); } - - @SneakyThrows - public T copyFileFromContainer(String containerPath, ThrowingFunction function) { - assertPodRunning("copyFileFromContainer"); - try (var inputStream = pod.file(containerPath).read()) { - return function.apply(inputStream); - } - } - - @SneakyThrows - private void uploadBase64(byte[] payload, String escapedTarPath) { - var encoded = Base64.getEncoder().encodeToString(payload); - try (var uploadWatch = pod.terminateOnError().exec("sh", "-c", "echo " + encoded + "| base64 -d >" + "'" + escapedTarPath + "'")) { - var code = uploadWatch.exitCode().get(getRequestConfig().getRequestTimeout(), MILLISECONDS); - if (code != 0) { - throw new UploadFileException("Unexpected exit code " + code + ", file '" + escapedTarPath + "'"); - } - checkSize(escapedTarPath, payload.length); - } - } - - private RequestConfig getRequestConfig() { - return ((OperationSupport) pod).getRequestConfig(); - } - - @SneakyThrows - public boolean removeFile(String tarName) { - try (var exec = waitEmptyQueue(pod.redirectingError().exec("rm", tarName))) { - var exitCode = exec.exitCode().get(getRequestTimeout(), MILLISECONDS); - var deleted = exitCode != 0; - if (deleted) { - log.warn("deleting of temporary file {} finished with unexpected code {}, errOut: {}", - tarName, exitCode, getError(exec)); - } - return deleted; - } - } - - @SneakyThrows - public Container.ExecResult execInContainer(Charset outputCharset, String... command) { - var hasShellCall = command.length > 1 && command[0].equals("sh") && command[1].equals("-c"); - if (!hasShellCall) { - var newCmd = new String[command.length + 2]; - newCmd[0] = "sh"; - newCmd[1] = "-c"; - System.arraycopy(command, 0, newCmd, 2, command.length); - command = newCmd; - } - - try (var execWatch = pod.redirectingOutput().redirectingError().exec(command)) { - var exited = execWatch.exitCode(); - var exitCode = exited.get(getRequestTimeout(), MILLISECONDS); - ; - String errOut; - try { - errOut = new String(execWatch.getError().readAllBytes(), outputCharset); - } catch (IOException e) { - errOut = ""; - log.info("err output read error", e); - } - String output; - try { - output = new String(execWatch.getOutput().readAllBytes(), outputCharset); - } catch (IOException e) { - output = ""; - log.info("output read error", e); - } - var constructor = Container.ExecResult.class.getDeclaredConstructor(int.class, String.class, String.class); - constructor.setAccessible(true); - - return constructor.newInstance(exitCode, output, errOut); - } - } - @Override public String toString() { return getClass().getSimpleName() + "{" + toStringFields() + '}'; diff --git a/src/main/java/io/github/m4gshm/testcontainers/KubernetesUtils.java b/src/main/java/io/github/m4gshm/testcontainers/KubernetesUtils.java index 0bb8046..0b50725 100644 --- a/src/main/java/io/github/m4gshm/testcontainers/KubernetesUtils.java +++ b/src/main/java/io/github/m4gshm/testcontainers/KubernetesUtils.java @@ -15,14 +15,18 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.InetAddress; +import java.nio.charset.Charset; +import java.util.Base64; import java.util.Collection; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static java.lang.Boolean.TRUE; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.stream.Collectors.toMap; @Slf4j @@ -124,4 +128,102 @@ public static Map startPortForward( })); } + + @SneakyThrows + public boolean removeFile(PodResource podResource, int requestTimeout, String tarName) { + try (var exec = waitEmptyQueue(podResource.redirectingError().exec("rm", tarName))) { + var exitCode = exec.exitCode().get(requestTimeout, MILLISECONDS); + var deleted = exitCode != 0; + if (deleted) { + log.warn("deleting of temporary file {} finished with unexpected code {}, errOut: {}", + tarName, exitCode, getError(exec)); + } + return deleted; + } + } + + @SneakyThrows + public static ExecResult exec(PodResource podResource, int requestTimeout, Charset outputCharset, String... command) { + var hasShellCall = command.length > 1 && command[0].equals("sh") && command[1].equals("-c"); + if (!hasShellCall) { + var newCmd = new String[command.length + 2]; + newCmd[0] = "sh"; + newCmd[1] = "-c"; + System.arraycopy(command, 0, newCmd, 2, command.length); + command = newCmd; + } + + try (var execWatch = podResource.redirectingOutput().redirectingError().exec(command)) { + var exited = execWatch.exitCode(); + var exitCode = exited.get(requestTimeout, MILLISECONDS); + ; + String errOut; + try { + errOut = new String(execWatch.getError().readAllBytes(), outputCharset); + } catch (IOException e) { + errOut = ""; + log.info("err output read error", e); + } + String output; + try { + output = new String(execWatch.getOutput().readAllBytes(), outputCharset); + } catch (IOException e) { + output = ""; + log.info("output read error", e); + } + + return new ExecResult(exitCode, output, errOut); + } + } + + + @SneakyThrows + public static void uploadBase64(PodResource podResource, int requestTimeout, String escapedTarPath, byte[] payload) { + var encoded = Base64.getEncoder().encodeToString(payload); + try (var uploadWatch = podResource.terminateOnError().exec("sh", "-c", "echo " + encoded + "| base64 -d >" + "'" + escapedTarPath + "'")) { + var code = uploadWatch.exitCode().get(requestTimeout, MILLISECONDS); + if (code != 0) { + throw new UploadFileException("Unexpected exit code " + code + ", file '" + escapedTarPath + "'"); + } + checkSize(podResource, requestTimeout, escapedTarPath, payload.length); + } + } + + @SneakyThrows + public static void uploadStdIn(PodResource podResource, int requestTimeout, String escapedTarPath, byte[] payload) { + try (var exec = podResource.redirectingInput().terminateOnError().exec("cp", "/dev/stdin", escapedTarPath)) { + var input = exec.getInput(); + input.write(payload); + input.flush(); + waitEmptyQueue(exec); + checkSize(podResource, requestTimeout, escapedTarPath, payload.length); + } + } + + @SneakyThrows + public static long getFileSize(PodResource podResource, int requestTimeout, String filePath) { + var byteCount = new ByteArrayOutputStream(); + try (var exec = podResource.writingOutput(byteCount).terminateOnError().exec("sh", "-c", "wc -c < " + filePath)) { + var exitCode = exec.exitCode().get(requestTimeout, MILLISECONDS); + var remoteSizeRaw = byteCount.toString(UTF_8).trim(); + waitEmptyQueue(exec); + return Integer.parseInt(remoteSizeRaw); + } + } + + @SneakyThrows + public static void checkSize(PodResource podResource, int requestTimeout, String filePath, long expected) { + var size = getFileSize(podResource, requestTimeout, filePath); + if (size != expected) { + Thread.sleep(100); + size = getFileSize(podResource, requestTimeout, filePath); + } + if (size != expected) { + throw new UploadFileException("Unexpected file size " + size + ", expected " + expected + ", file '" + filePath + "'"); + } + } + + public record ExecResult(int exitCode, String output, String error) { + } + } diff --git a/src/main/java/io/github/m4gshm/testcontainers/MongoDBPod.java b/src/main/java/io/github/m4gshm/testcontainers/MongoDBPod.java index cb76c24..46eac13 100644 --- a/src/main/java/io/github/m4gshm/testcontainers/MongoDBPod.java +++ b/src/main/java/io/github/m4gshm/testcontainers/MongoDBPod.java @@ -5,8 +5,6 @@ import lombok.extern.slf4j.Slf4j; import org.testcontainers.containers.MongoDBContainer; -import static java.util.Objects.requireNonNull; - /** * Kubernetes based extension of the {@link org.testcontainers.containers.MongoDBContainer}. */ diff --git a/src/main/java/io/github/m4gshm/testcontainers/PodContainerDelegate.java b/src/main/java/io/github/m4gshm/testcontainers/PodContainerDelegate.java index 37559c5..200d4a5 100644 --- a/src/main/java/io/github/m4gshm/testcontainers/PodContainerDelegate.java +++ b/src/main/java/io/github/m4gshm/testcontainers/PodContainerDelegate.java @@ -17,6 +17,7 @@ import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.testcontainers.containers.Container; +import org.testcontainers.containers.Container.ExecResult; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; @@ -25,7 +26,9 @@ import org.testcontainers.images.PullPolicy; import org.testcontainers.images.builder.Transferable; import org.testcontainers.utility.MountableFile; +import org.testcontainers.utility.ThrowingFunction; +import java.io.InputStream; import java.net.InetAddress; import java.nio.charset.Charset; import java.util.List; @@ -90,6 +93,29 @@ private static WaitStrategy replacePodWaiters(@NotNull WaitStrategy waitStrategy : waitStrategy; } + @SneakyThrows + public ExecResult execInContainer(Charset outputCharset, String... command) { + var execResult = KubernetesUtils.exec(pod, getRequestTimeout(), outputCharset, command); + + var constructor = ExecResult.class.getDeclaredConstructor(int.class, String.class, String.class); + constructor.setAccessible(true); + + return constructor.newInstance(execResult.exitCode(), execResult.output(), execResult.error()); + } + + @SneakyThrows + public T copyFileFromContainer(String containerPath, ThrowingFunction function) { + assertPodRunning("copyFileFromContainer"); + try (var inputStream = pod.file(containerPath).read()) { + return function.apply(inputStream); + } + } + + @SneakyThrows + public boolean removeFile(String tarName) { + return KubernetesUtils.removeFile(pod, getRequestTimeout(), tarName); + } + @SneakyThrows protected void containerIsStarted(InspectContainerResponse containerInfo, boolean reused) { var containerIsStarted = GenericContainer.class.getDeclaredMethod( @@ -282,11 +308,11 @@ protected String[] getCommandParts() { return container.getCommandParts(); } - public Container.ExecResult execInContainerWithUser(String user, String... command) { + public ExecResult execInContainerWithUser(String user, String... command) { return execInContainerWithUser(UTF_8, user, command); } - public Container.ExecResult execInContainerWithUser(Charset outputCharset, String user, String... command) { + public ExecResult execInContainerWithUser(Charset outputCharset, String user, String... command) { throw new UnsupportedOperationException("execInContainerWithUser"); } diff --git a/src/main/java/io/github/m4gshm/testcontainers/PodContainerUtils.java b/src/main/java/io/github/m4gshm/testcontainers/PodContainerUtils.java index 9929ebe..d2b9976 100644 --- a/src/main/java/io/github/m4gshm/testcontainers/PodContainerUtils.java +++ b/src/main/java/io/github/m4gshm/testcontainers/PodContainerUtils.java @@ -2,14 +2,19 @@ import com.fasterxml.jackson.databind.json.JsonMapper; import com.github.dockerjava.api.command.CreateContainerCmd; +import io.fabric8.kubernetes.client.dsl.PodResource; +import lombok.SneakyThrows; import lombok.experimental.UtilityClass; import org.jetbrains.annotations.NotNull; +import java.io.IOException; +import java.nio.charset.Charset; import java.util.List; import static com.fasterxml.jackson.databind.MapperFeature.SORT_PROPERTIES_ALPHABETICALLY; import static com.fasterxml.jackson.databind.SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS; import static java.lang.reflect.Proxy.newProxyInstance; +import static java.util.concurrent.TimeUnit.MILLISECONDS; @UtilityClass public class PodContainerUtils { @@ -19,7 +24,10 @@ public static JsonMapper config(JsonMapper jsonMapper) { .enable(ORDER_MAP_ENTRIES_BY_KEYS) .build(); } - public static @NotNull CreateContainerCmd newCreateContainerCmd(ClassLoader classLoader, PodBuilderFactory podBuilderFactory) { + + public static @NotNull CreateContainerCmd newCreateContainerCmd( + ClassLoader classLoader, PodBuilderFactory podBuilderFactory + ) { return (CreateContainerCmd) newProxyInstance(classLoader, new Class[]{CreateContainerCmd.class}, ( proxy, method, args ) -> { @@ -38,4 +46,5 @@ public static JsonMapper config(JsonMapper jsonMapper) { }; }); } + }