diff --git a/build-info-client/src/main/java/org/jfrog/build/client/DownloadResponse.java b/build-info-client/src/main/java/org/jfrog/build/client/DownloadResponse.java new file mode 100644 index 000000000..368568d02 --- /dev/null +++ b/build-info-client/src/main/java/org/jfrog/build/client/DownloadResponse.java @@ -0,0 +1,37 @@ +package org.jfrog.build.client; + +import org.apache.http.Header; + +/** + * @author yahavi + **/ +@SuppressWarnings("unused") +public class DownloadResponse { + public static final String SHA256_HEADER_NAME = "X-Checksum-Sha256"; + Header[] headers; + String content; + + public DownloadResponse() { + } + + public DownloadResponse(String content, Header[] headers) { + this.headers = headers; + this.content = content; + } + + public void setHeaders(Header[] headers) { + this.headers = headers; + } + + public void setContent(String content) { + this.content = content; + } + + public Header[] getHeaders() { + return headers; + } + + public String getContent() { + return content; + } +} diff --git a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/BuildDockerCreate.java b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/BuildDockerCreate.java deleted file mode 100644 index 59baa2dcd..000000000 --- a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/BuildDockerCreate.java +++ /dev/null @@ -1,167 +0,0 @@ -package org.jfrog.build.extractor.docker.extractor; - -import com.google.common.collect.ArrayListMultimap; -import org.apache.commons.lang3.exception.ExceptionUtils; -import org.jfrog.build.api.Build; -import org.jfrog.build.api.Module; -import org.jfrog.build.api.util.Log; -import org.jfrog.build.extractor.clientConfiguration.ArtifactoryClientConfiguration; -import org.jfrog.build.extractor.clientConfiguration.ArtifactoryManagerBuilder; -import org.jfrog.build.extractor.clientConfiguration.client.artifactory.ArtifactoryManager; -import org.jfrog.build.extractor.docker.DockerUtils; -import org.jfrog.build.extractor.docker.types.DockerImage; -import org.jfrog.build.extractor.docker.types.DockerLayer; -import org.jfrog.build.extractor.docker.types.DockerLayers; -import org.jfrog.build.extractor.packageManager.PackageManagerExtractor; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -import static org.jfrog.build.extractor.packageManager.PackageManagerUtils.createArtifactoryClientConfiguration; - -public class BuildDockerCreate extends PackageManagerExtractor { - private final ArtifactoryManagerBuilder artifactoryManagerBuilder; - private final Log logger; - private final String kanikoImageFile; - private final String targetRepository; - private final List modulesList = new ArrayList<>(); - private final ArrayListMultimap artifactProperties; - - - /** - * @param artifactoryManagerBuilder - Artifactory manager builder. - * @param targetRepository - The repository it'll deploy to. - * @param kanikoImageFile - Image file to add. - * @param logger - The logger. - * @param artifactProperties - Properties to be attached to the docker layers deployed to Artifactory. - */ - public BuildDockerCreate(ArtifactoryManagerBuilder artifactoryManagerBuilder, - String kanikoImageFile, ArrayListMultimap artifactProperties, String targetRepository, Log logger) { - this.artifactoryManagerBuilder = artifactoryManagerBuilder; - this.targetRepository = targetRepository; - this.logger = logger; - this.kanikoImageFile = kanikoImageFile; - this.artifactProperties = artifactProperties; - } - - /** - * Allow running docker push using a new Java process. - * - * @param ignored ignores input incoming params. - */ - public static void main(String[] ignored) { - try { - ArtifactoryClientConfiguration clientConfiguration = createArtifactoryClientConfiguration(); - // Client builders. - ArtifactoryManagerBuilder artifactoryManagerBuilder = new ArtifactoryManagerBuilder().setClientConfiguration(clientConfiguration, clientConfiguration.publisher); - // Load artifact and BuildInfo properties from publisher section in the BuildInfo.properties file. - ArtifactoryClientConfiguration.KanikoHandler kanikoHandler = clientConfiguration.kanikoHandler; - // Init BuildDockerCreate. - BuildDockerCreate dockerBuildCreate = new BuildDockerCreate(artifactoryManagerBuilder, - kanikoHandler.getImageFile(), - ArrayListMultimap.create(clientConfiguration.publisher.getMatrixParams().asMultimap()), - clientConfiguration.publisher.getRepoKey(), - clientConfiguration.getLog()); - - // Exe docker push & collect build info. - dockerBuildCreate.executeAndSaveBuildInfo(clientConfiguration); - } catch (RuntimeException e) { - ExceptionUtils.printRootCauseStackTrace(e, System.out); - System.exit(1); - } - } - - @Override - public Build execute() { - logger.info("Getting build info for: " + kanikoImageFile); - try { - Build build = new Build(); - for (ImageFileWithDigest imageFileWithDigest : getImageFileWithDigests(kanikoImageFile)) { - DockerImage image = new DockerImage(imageFileWithDigest.manifestSha256, imageFileWithDigest.imageName, targetRepository, artifactoryManagerBuilder, "", ""); - Module module = image.generateBuildInfoModule(logger, DockerUtils.CommandType.Push); - if (module.getArtifacts() == null || module.getArtifacts().size() == 0) { - logger.warn("Could not find docker image: " + imageFileWithDigest.imageName + " in Artifactory."); - } else { - setImageLayersProps(image.getLayers(), artifactProperties, artifactoryManagerBuilder); - } - modulesList.add(module); - logger.info("Successfully created build info for image: " + imageFileWithDigest.imageName); - } - build.setModules(modulesList); - return build; - } catch (IOException | InterruptedException e) { - logger.error(e.getMessage(), e); - throw new RuntimeException(e); - } catch (RuntimeException e) { - logger.error(e.getMessage(), e); - throw e; - } - } - - /** - * Update each layer's properties with artifactProperties. - */ - private void setImageLayersProps(DockerLayers layers, ArrayListMultimap artifactProperties, ArtifactoryManagerBuilder artifactoryManagerBuilder) throws IOException { - if (layers == null){ - return; - } - try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { - for (DockerLayer layer : layers.getLayers()) { - artifactoryManager.setProperties(layer.getFullPath(), artifactProperties, false); - } - } - } - - /** - * Read the file which contains the following format on each line: 'IMAGE-TAG-IN-ARTIFACTORY'@sha256'SHA256-OF-THE-IMAGE-MANIFEST'. - * - * @param filePath the image file path. - * @return the image file with names and digests extracted. - * @throws IOException if an issue occurs reading the file. - */ - private List getImageFileWithDigests(String filePath) throws IOException { - List dataLines = Files.readAllLines(Paths.get(filePath)); - if (dataLines.isEmpty()) { - throw new IOException("empty image file \"" + filePath + "\"."); - } - final List imageFileWithDigests = - dataLines.stream().map(this::getImageFileWithDigest).filter(Objects::nonNull).collect(Collectors.toList()); - if (imageFileWithDigests.size() != dataLines.size()) { - throw new RuntimeException("missing image-tag/sha256 in file: \"" + filePath + "\""); - } - return imageFileWithDigests; - } - - /** - * Read the file which contains the following format: 'IMAGE-TAG-IN-ARTIFACTORY'@sha256'SHA256-OF-THE-IMAGE-MANIFEST'. - * - * @param data the image file data. - * @return the image file with name and digest extracted. - */ - private ImageFileWithDigest getImageFileWithDigest(String data) { - String[] splittedData = data.split("@"); - if (splittedData.length != 2) { - throw new RuntimeException("unexpected file format \"" + data + "\". The file should include one line in the following format: image-tag@sha256"); - } - ImageFileWithDigest imageFileWithDigest = new ImageFileWithDigest(splittedData[0], splittedData[1]); - if (imageFileWithDigest.imageName.isEmpty() || imageFileWithDigest.manifestSha256.isEmpty()) { - return null; - } - return imageFileWithDigest; - } - - private static class ImageFileWithDigest { - final String imageName; - final String manifestSha256; - - ImageFileWithDigest(String imageName, String manifestSha256) { - this.imageName = imageName.trim(); - this.manifestSha256 = manifestSha256.trim(); - } - } -} diff --git a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/BuildDockerCreator.java b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/BuildDockerCreator.java new file mode 100644 index 000000000..bdcd7a897 --- /dev/null +++ b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/BuildDockerCreator.java @@ -0,0 +1,242 @@ +package org.jfrog.build.extractor.docker.extractor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ArrayListMultimap; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.jfrog.build.api.Build; +import org.jfrog.build.api.Module; +import org.jfrog.build.api.util.Log; +import org.jfrog.build.extractor.clientConfiguration.ArtifactoryClientConfiguration; +import org.jfrog.build.extractor.clientConfiguration.ArtifactoryManagerBuilder; +import org.jfrog.build.extractor.clientConfiguration.client.artifactory.ArtifactoryManager; +import org.jfrog.build.extractor.clientConfiguration.util.PathsUtils; +import org.jfrog.build.extractor.clientConfiguration.util.spec.UploadSpecHelper; +import org.jfrog.build.extractor.docker.DockerUtils; +import org.jfrog.build.extractor.docker.types.DockerImage; +import org.jfrog.build.extractor.docker.types.DockerLayer; +import org.jfrog.build.extractor.docker.types.DockerLayers; +import org.jfrog.build.extractor.packageManager.PackageManagerExtractor; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.jfrog.build.extractor.docker.DockerUtils.createMapper; +import static org.jfrog.build.extractor.packageManager.PackageManagerUtils.createArtifactoryClientConfiguration; + +public class BuildDockerCreator extends PackageManagerExtractor { + private final ArrayListMultimap artifactProperties; + private final ArtifactoryManagerBuilder artifactoryManagerBuilder; + private final ImageFileType imageFileType; + private final String sourceRepo; + private final String imageFile; + private final Log logger; + + enum ImageFileType { + KANIKO, + JIB + } + + /** + * @param artifactoryManagerBuilder - Artifactory manager builder. + * @param sourceRepo - The repository it'll resolve from. + * @param imageFileType - The input imageFile format - JIB or Kaniko + * @param imageFile - Image file to add. + * @param logger - The logger. + * @param artifactProperties - Properties to be attached to the docker layers deployed to Artifactory. + */ + public BuildDockerCreator(ArtifactoryManagerBuilder artifactoryManagerBuilder, String imageFile, ImageFileType imageFileType, + ArrayListMultimap artifactProperties, String sourceRepo, Log logger) { + this.artifactoryManagerBuilder = artifactoryManagerBuilder; + this.artifactProperties = artifactProperties; + this.sourceRepo = sourceRepo; + this.imageFileType = imageFileType; + this.imageFile = imageFile; + this.logger = logger; + } + + /** + * Allow creating build-info for a published docker image in a new Java process. + * + * @param ignored ignores input incoming params. + */ + public static void main(String[] ignored) { + try { + ArtifactoryClientConfiguration clientConfiguration = createArtifactoryClientConfiguration(); + // Client builders. + ArtifactoryManagerBuilder artifactoryManagerBuilder = new ArtifactoryManagerBuilder().setClientConfiguration(clientConfiguration, clientConfiguration.publisher); + // Load artifact and BuildInfo properties from publisher section in the BuildInfo.properties file. + ArtifactoryClientConfiguration.DockerHandler dockerHandler = clientConfiguration.dockerHandler; + String imageFile; + ImageFileType imageFileType; + if (StringUtils.isNotBlank(dockerHandler.getKanikoImageFile())) { + imageFile = dockerHandler.getKanikoImageFile(); + imageFileType = ImageFileType.KANIKO; + } else if (StringUtils.isNotBlank(dockerHandler.getJibImageFile())) { + imageFile = dockerHandler.getJibImageFile(); + imageFileType = ImageFileType.JIB; + } else { + throw new RuntimeException("kaniko.image.file or jib.image.file property is expected"); + } + // Init BuildDockerCreate. + BuildDockerCreator dockerBuildCreate = new BuildDockerCreator(artifactoryManagerBuilder, + imageFile, + imageFileType, + ArrayListMultimap.create(clientConfiguration.publisher.getMatrixParams().asMultimap()), + clientConfiguration.publisher.getRepoKey(), + clientConfiguration.getLog()); + + // Exe build-docker-create & collect build info. + dockerBuildCreate.executeAndSaveBuildInfo(clientConfiguration); + } catch (RuntimeException e) { + ExceptionUtils.printRootCauseStackTrace(e, System.out); + System.exit(1); + } + } + + @Override + public Build execute() { + logger.info("Generating build info for: " + imageFile); + try { + List modules = new ArrayList<>(); + List imageFilesWithDigest = imageFileType == ImageFileType.KANIKO ? + getKanikoImageFileWithDigests(imageFile) : getJibImageFilesWithDigests(imageFile); + if (imageFilesWithDigest.isEmpty()) { + throw new RuntimeException("No image files found at path '" + imageFile + "'"); + } + for (ImageFileWithDigest imageFileWithDigest : imageFilesWithDigest) { + DockerImage image = new DockerImage("", imageFileWithDigest.imageName, imageFileWithDigest.manifestSha256, sourceRepo, artifactoryManagerBuilder, "", ""); + Module module = image.generateBuildInfoModule(logger, DockerUtils.CommandType.Push); + if (module.getArtifacts() == null || module.getArtifacts().size() == 0) { + logger.warn("Could not find docker image: " + imageFileWithDigest.imageName + " in Artifactory."); + } else { + setImageLayersProps(image.getLayers(), artifactProperties, artifactoryManagerBuilder); + } + modules.add(module); + logger.info("Successfully created build info for image: " + imageFileWithDigest.imageName); + } + Build build = new Build(); + build.setModules(modules); + return build; + } catch (Exception e) { + logger.error(ExceptionUtils.getRootCauseMessage(e), e); + throw new RuntimeException(e); + } + } + + /** + * Update each layer's properties with artifactProperties. + */ + private void setImageLayersProps(DockerLayers layers, ArrayListMultimap artifactProperties, ArtifactoryManagerBuilder artifactoryManagerBuilder) throws IOException { + if (layers == null) { + return; + } + try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { + for (DockerLayer layer : layers.getLayers()) { + artifactoryManager.setProperties(layer.getFullPath(), artifactProperties, true); + } + } + } + + /** + * Read the file which contains the following format on each line: 'IMAGE-TAG-IN-ARTIFACTORY'@sha256'SHA256-OF-THE-IMAGE-MANIFEST'. + * + * @param kanikoImageFile - Kaniko image-file path. + * @return the image file with names and digests extracted. + * @throws IOException if an issue occurs reading the file. + */ + private List getKanikoImageFileWithDigests(String kanikoImageFile) throws IOException { + List dataLines = Files.readAllLines(Paths.get(kanikoImageFile)); + if (dataLines.isEmpty()) { + throw new IOException("Couldn't read image file \"" + kanikoImageFile + "\"."); + } + final List imageFileWithDigests = + dataLines.stream().map(this::getImageFileWithDigest).filter(Objects::nonNull).collect(Collectors.toList()); + if (imageFileWithDigests.size() != dataLines.size()) { + throw new RuntimeException("missing image-tag/sha256 in file: \"" + kanikoImageFile + "\""); + } + return imageFileWithDigests; + } + + /** + * Read the file which contains the following format: + * { + * "image":"IMAGE-NAME-IN-ARTIFACTORY", + * "tags":["IMAGE-TAGS-IN-ARTIFACTORY"] + * "imageId":"sha256:SHA256-OF-THE-IMAGE-MANIFEST", + * "imageDigest":"sha256:SHA256-OF-THE-IMAGE", + * } + * + * @param jibImageFiles - Pattern matches JIB's image JSON files. For example: "*target/jib-image.json". + * @return the image file with names and digests extracted. + */ + private List getJibImageFilesWithDigests(String jibImageFiles) { + ObjectMapper mapper = createMapper(); + String baseDir = UploadSpecHelper.getWildcardBaseDir(new File(""), jibImageFiles); + String newPattern = UploadSpecHelper.prepareWildcardPattern(new File(""), jibImageFiles, baseDir); + String regexPath = PathsUtils.pathToRegExp(newPattern); + List imageFilesWithDigests = new ArrayList<>(); + Path baseDirPath = Paths.get(baseDir); + try (Stream files = Files.find(baseDirPath, + Integer.MAX_VALUE, + (path, basicFileAttributes) -> baseDirPath.relativize(path).toString().matches(regexPath))) { + files.forEach(jibImageFile -> { + JsonNode jsonNode; + try { + jsonNode = mapper.readTree(jibImageFile.toFile()); + } catch (IOException e) { + throw new RuntimeException("Couldn't read image file \"" + jibImageFiles + "\"."); + } + JsonNode imageName = jsonNode.get("image"); + if (imageName == null || imageName.isNull()) { + throw new RuntimeException("Missing \"image\" in file: \"" + jibImageFiles + "\""); + } + JsonNode imageDigest = jsonNode.get("imageDigest"); + if (imageDigest == null || imageDigest.isNull()) { + throw new RuntimeException("Missing \"imageDigest\" in file: \"" + jibImageFiles + "\""); + } + imageFilesWithDigests.add(new ImageFileWithDigest(imageName.asText(), imageDigest.asText())); + }); + } catch (IOException e) { + throw new RuntimeException("Couldn't find JIB image files using the following pattern: '" + jibImageFiles + "'", e); + } + return imageFilesWithDigests; + } + + /** + * Read the file which contains the following format: 'IMAGE-TAG-IN-ARTIFACTORY'@sha256'SHA256-OF-THE-IMAGE-MANIFEST'. + * + * @param data the image file data. + * @return the image file with name and digest extracted. + */ + private ImageFileWithDigest getImageFileWithDigest(String data) { + String[] splitData = data.split("@"); + if (splitData.length != 2) { + throw new RuntimeException("unexpected file format \"" + data + "\". The file should include one line in the following format: image-tag@sha256"); + } + ImageFileWithDigest imageFileWithDigest = new ImageFileWithDigest(splitData[0], splitData[1]); + if (imageFileWithDigest.imageName.isEmpty() || imageFileWithDigest.manifestSha256.isEmpty()) { + return null; + } + return imageFileWithDigest; + } + + private static class ImageFileWithDigest { + final String imageName; + final String manifestSha256; + + ImageFileWithDigest(String imageName, String manifestSha256) { + this.imageName = imageName.trim(); + this.manifestSha256 = manifestSha256.trim(); + } + } +} diff --git a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPull.java b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPull.java index 5215bf256..f8032aedb 100644 --- a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPull.java +++ b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPull.java @@ -74,7 +74,7 @@ public Build execute() { DockerJavaWrapper.pullImage(imageTag, username, password, host, env, logger); String imageId = DockerJavaWrapper.getImageIdFromTag(imageTag, host, env, logger); Pair archDetails = DockerJavaWrapper.getImageArch(imageTag, host, env, logger); - DockerImage image = new DockerImage(imageId, imageTag, targetRepository, artifactoryManagerBuilder, archDetails.getLeft(), archDetails.getRight()); + DockerImage image = new DockerImage(imageId, imageTag, "", targetRepository, artifactoryManagerBuilder, archDetails.getLeft(), archDetails.getRight()); Module module = image.generateBuildInfoModule(logger, DockerUtils.CommandType.Pull); if (module.getDependencies() == null || module.getDependencies().size() == 0) { logger.warn("Could not find docker image: " + imageTag + " in Artifactory."); diff --git a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPush.java b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPush.java index 93f52f31d..78252f62b 100644 --- a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPush.java +++ b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/extractor/DockerPush.java @@ -83,7 +83,7 @@ public Build execute() { try { DockerJavaWrapper.pushImage(imageTag, username, password, host, env, logger); String imageId = DockerJavaWrapper.getImageIdFromTag(imageTag, host, env, logger); - DockerImage image = new DockerImage(imageId, imageTag, targetRepository, artifactoryManagerBuilder, "", ""); + DockerImage image = new DockerImage(imageId, imageTag, "", targetRepository, artifactoryManagerBuilder, "", ""); Module module = image.generateBuildInfoModule(logger, DockerUtils.CommandType.Push); if (module.getArtifacts() == null || module.getArtifacts().size() == 0) { logger.warn("Could not find docker image: " + imageTag + " in Artifactory."); diff --git a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/types/DockerImage.java b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/types/DockerImage.java index a1fa45e24..addc7148d 100644 --- a/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/types/DockerImage.java +++ b/build-info-extractor-docker/src/main/java/org/jfrog/build/extractor/docker/types/DockerImage.java @@ -1,7 +1,8 @@ package org.jfrog.build.extractor.docker.types; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.apache.http.Header; import org.jfrog.build.api.Artifact; import org.jfrog.build.api.Dependency; import org.jfrog.build.api.Module; @@ -12,6 +13,7 @@ import org.jfrog.build.api.search.AqlSearchResult; import org.jfrog.build.api.util.Log; import org.jfrog.build.client.ArtifactoryVersion; +import org.jfrog.build.client.DownloadResponse; import org.jfrog.build.extractor.clientConfiguration.ArtifactoryManagerBuilder; import org.jfrog.build.extractor.clientConfiguration.client.artifactory.ArtifactoryManager; import org.jfrog.build.extractor.docker.DockerUtils; @@ -21,8 +23,10 @@ import java.util.*; import java.util.stream.Collectors; +import static org.jfrog.build.client.DownloadResponse.SHA256_HEADER_NAME; + public class DockerImage implements Serializable { - private final String imageId; + private String imageId; private final String imageTag; private final String targetRepo; // Properties to be attached to the docker layers deployed to Artifactory. @@ -35,14 +39,16 @@ public class DockerImage implements Serializable { private String manifest; private String imagePath; private DockerLayers layers; + private final String manifestSha256; - public DockerImage(String imageId, String imageTag, String targetRepo, ArtifactoryManagerBuilder artifactoryManagerBuilder, String arch, String os) { + public DockerImage(String imageId, String imageTag, String manifestSha256, String targetRepo, ArtifactoryManagerBuilder artifactoryManagerBuilder, String arch, String os) { this.imageId = imageId; this.imageTag = imageTag; this.targetRepo = targetRepo; this.artifactoryManagerBuilder = artifactoryManagerBuilder; this.architecture = arch; this.os = os; + this.manifestSha256 = manifestSha256; } public DockerLayers getLayers() { @@ -50,19 +56,47 @@ public DockerLayers getLayers() { } /** - * Check if the provided manifestPath is correct. + * Check if the provided manifestPath is correct by comparing the SHA256 of the image or of the manifest. * Set the manifest and imagePath in case of the correct manifest. + * Also, if manifestSha256 is provided, set the actual imageId in case of the correct manifest. */ private void checkAndSetManifestAndImagePathCandidates(String candidateManifestPath, ArtifactoryManager artifactoryManager, Log logger) throws IOException { - Pair candidateDetails = getManifestFromArtifactory(artifactoryManager, candidateManifestPath, logger); - String manifestContent = candidateDetails.getLeft(); + Pair candidateDetails = getManifestFromArtifactory(artifactoryManager, candidateManifestPath, logger); + DownloadResponse downloadResponse = candidateDetails.getLeft(); + String manifestContent = downloadResponse.getContent(); String manifestPath = candidateDetails.getRight(); - String imageDigest = DockerUtils.getConfigDigest(manifestContent); - if (imageDigest.equals(imageId)) { - manifest = manifestContent; - imagePath = manifestPath; - loadLayers(manifestPath); + if (StringUtils.isNotBlank(manifestSha256)) { + // If manifestSha256 is set, we should check the manifest's SHA256 instead of the image's SHA256. + // This scenario is used for Kaniko and JIB. + if (!checkDownloadedManifest(downloadResponse, manifestPath)) { + // The downloaded manifest is not the expected one. + return; + } + // Extract the image ID from the downloaded manifest. + imageId = DockerUtils.getConfigDigest(manifestContent); + } else if (!DockerUtils.getConfigDigest(manifestContent).equals(imageId)) { + // The downloaded manifest does not contain the expected image SHA256 + return; } + manifest = manifestContent; + imagePath = manifestPath; + loadLayers(manifestPath); + } + + /** + * Return true if the SHA256 of the downloaded manifest.json file is matched the expected one. + * + * @param downloadResponse - The download response from the 'GET ' REST API + * @param manifestPath - The actual path to the manifest.json file in Artifactory + * @return if the SHA256 of the downloaded manifest.json file is matched the expected one. + * @throws IOException in case the 'X-Checksum-Sha256' is missing in the response. + */ + private boolean checkDownloadedManifest(DownloadResponse downloadResponse, String manifestPath) throws IOException { + Header manifestSha256 = Arrays.stream(downloadResponse.getHeaders()) + .filter(header -> SHA256_HEADER_NAME.equals(header.getName())) + .findFirst() + .orElseThrow(() -> new IOException(String.format("'%s' header is missing in GET '%s'.", SHA256_HEADER_NAME, manifestPath))); + return StringUtils.endsWith(this.manifestSha256, manifestSha256.getValue()); } /** @@ -76,7 +110,8 @@ private void checkAndSetManifestAndImagePathCandidates(String candidateManifestP * @return A pair of (manifest content, path to manifest). * @throws IOException fail to search for manifest json in manifestPath. */ - private Pair getManifestFromArtifactory(ArtifactoryManager artifactoryManager, String manifestPath, Log logger) throws IOException { + private Pair getManifestFromArtifactory(ArtifactoryManager artifactoryManager, String + manifestPath, Log logger) throws IOException { String pathWithoutRepo = StringUtils.substringAfter(manifestPath, "/"); String downloadUrl = manifestPath + "/manifest.json"; logger.info("Trying to download manifest from " + downloadUrl); @@ -88,7 +123,7 @@ private Pair getManifestFromArtifactory(ArtifactoryManager artif } downloadUrl = manifestPath + "/list.manifest.json"; logger.info("Fallback for remote/virtual repository. Trying to download fat-manifest from " + downloadUrl); - String digestsFromFatManifest = DockerUtils.getImageDigestFromFatManifest(artifactoryManager.download(downloadUrl), os, architecture); + String digestsFromFatManifest = DockerUtils.getImageDigestFromFatManifest(artifactoryManager.download(downloadUrl).getContent(), os, architecture); if (digestsFromFatManifest.isEmpty()) { logger.info("Failed to get image digest from fat manifest"); throw e; @@ -125,37 +160,38 @@ private DockerLayers createLayers(ArtifactoryManager artifactoryManager, String /** * Search the docker image in Artifactory and add all artifacts & dependencies into Module. */ - private void setDependenciesAndArtifacts(ModuleBuilder moduleBuilder, ArtifactoryManager artifactoryManager) throws IOException { + private void setDependenciesAndArtifacts(ModuleBuilder moduleBuilder, ArtifactoryManager artifactoryManager) throws + IOException { DockerLayer historyLayer = layers.getByDigest(imageId); if (historyLayer == null) { throw new IllegalStateException("Could not find the history docker layer: " + imageId + " for image: " + imageTag + " in Artifactory."); } - int dependencyLayerNum = DockerUtils.getNumberOfDependentLayers(artifactoryManager.download(historyLayer.getFullPath())); + int dependencyLayerNum = DockerUtils.getNumberOfDependentLayers(artifactoryManager.download(historyLayer.getFullPath()).getContent()); - LinkedHashSet dependencies = new LinkedHashSet<>(); - LinkedHashSet artifacts = new LinkedHashSet<>(); - // Filter out duplicate layers from manifest by using HashSet. - // Docker manifest may hold 'empty layers', as a result, docker promote will fail to promote the same layer more than once. - Iterator it = DockerUtils.getLayersDigests(manifest).iterator(); - for (int i = 0; i < dependencyLayerNum; i++) { - String digest = it.next(); - DockerLayer layer = layers.getByDigest(digest); - Dependency dependency = new DependencyBuilder().id(layer.getFileName()).sha1(layer.getSha1()).build(); - dependencies.add(dependency); - Artifact artifact = new ArtifactBuilder(layer.getFileName()).sha1(layer.getSha1()).remotePath(layer.getPath()).build(); - artifacts.add(artifact); - } - moduleBuilder.dependencies(new ArrayList<>(dependencies)); - while (it.hasNext()) { - String digest = it.next(); - DockerLayer layer = layers.getByDigest(digest); - if (layer == null) { - continue; - } - Artifact artifact = new ArtifactBuilder(layer.getFileName()).sha1(layer.getSha1()).remotePath(layer.getPath()).build(); - artifacts.add(artifact); + LinkedHashSet dependencies = new LinkedHashSet<>(); + LinkedHashSet artifacts = new LinkedHashSet<>(); + // Filter out duplicate layers from manifest by using HashSet. + // Docker manifest may hold 'empty layers', as a result, docker promote will fail to promote the same layer more than once. + Iterator it = DockerUtils.getLayersDigests(manifest).iterator(); + for (int i = 0; i < dependencyLayerNum; i++) { + String digest = it.next(); + DockerLayer layer = layers.getByDigest(digest); + Dependency dependency = new DependencyBuilder().id(layer.getFileName()).sha1(layer.getSha1()).build(); + dependencies.add(dependency); + Artifact artifact = new ArtifactBuilder(layer.getFileName()).sha1(layer.getSha1()).remotePath(layer.getPath()).build(); + artifacts.add(artifact); + } + moduleBuilder.dependencies(new ArrayList<>(dependencies)); + while (it.hasNext()) { + String digest = it.next(); + DockerLayer layer = layers.getByDigest(digest); + if (layer == null) { + continue; } - moduleBuilder.artifacts(new ArrayList<>(artifacts)); + Artifact artifact = new ArtifactBuilder(layer.getFileName()).sha1(layer.getSha1()).remotePath(layer.getPath()).build(); + artifacts.add(artifact); + } + moduleBuilder.artifacts(new ArrayList<>(artifacts)); } private void setDependencies(ModuleBuilder moduleBuilder) throws IOException { @@ -186,7 +222,8 @@ private String getAqlQuery(boolean includeVirtualRepos, String Repo, String mani return aqlRequestForDockerSha.toString(); } - public Module generateBuildInfoModule(Log logger, DockerUtils.CommandType cmdType) throws IOException, InterruptedException { + public Module generateBuildInfoModule(Log logger, DockerUtils.CommandType cmdType) throws + IOException, InterruptedException { try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) { ModuleBuilder moduleBuilder = new ModuleBuilder() .type(ModuleType.DOCKER) @@ -245,7 +282,8 @@ private DockerLayers getLayers(ArtifactoryManager artifactoryManager, String man * Find and validate manifest.json file in Artifactory for the current image. * Since provided imageTag differs between reverse-proxy and proxy-less configuration, try to build the correct manifest path. */ - private void findAndSetManifestFromArtifactory(ArtifactoryManager artifactoryManager, Log logger, DockerUtils.CommandType cmdType) throws IOException { + private void findAndSetManifestFromArtifactory(ArtifactoryManager artifactoryManager, Log + logger, DockerUtils.CommandType cmdType) throws IOException { // Try to get manifest, assuming reverse proxy String ImagePath = DockerUtils.getImagePath(imageTag); ArrayList manifestPathCandidate = new ArrayList<>(DockerUtils.getArtManifestPath(ImagePath, targetRepo, cmdType)); diff --git a/build-info-extractor-docker/src/test/java/org/jfrog/build/extractor/docker/extractor/DockerExtractorTest.java b/build-info-extractor-docker/src/test/java/org/jfrog/build/extractor/docker/extractor/DockerExtractorTest.java index 649a5614e..8fe2a1bb5 100644 --- a/build-info-extractor-docker/src/test/java/org/jfrog/build/extractor/docker/extractor/DockerExtractorTest.java +++ b/build-info-extractor-docker/src/test/java/org/jfrog/build/extractor/docker/extractor/DockerExtractorTest.java @@ -1,27 +1,32 @@ package org.jfrog.build.extractor.docker.extractor; - +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.lang3.SystemUtils; import org.jfrog.build.IntegrationTestsBase; -import org.jfrog.build.api.Artifact; -import org.jfrog.build.api.Build; -import org.jfrog.build.api.Dependency; -import org.jfrog.build.api.Module; -import org.jfrog.build.extractor.clientConfiguration.ArtifactoryManagerBuilder; +import org.jfrog.build.api.*; import org.jfrog.build.extractor.docker.DockerJavaWrapper; +import org.jfrog.build.extractor.executor.CommandExecutor; +import org.jfrog.build.extractor.executor.CommandResults; import org.testng.SkipException; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import java.io.File; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; import java.util.Collections; import java.util.List; @@ -29,12 +34,12 @@ @Test public class DockerExtractorTest extends IntegrationTestsBase { - private static final Path PROJECTS_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "org", "jfrog", "build", "extractor")); + private static final Path PROJECTS_ROOT = Paths.get(".").toAbsolutePath().normalize().resolve(Paths.get("src", "test", "resources", "org", "jfrog", "build", "extractor", "docker")); + private static final String PROJECT_PATH = PROJECTS_ROOT.toAbsolutePath().toString(); private static final String SHORT_IMAGE_NAME = "jfrog_artifactory_buildinfo_tests"; private static final String SHORT_IMAGE_TAG_LOCAL = "2"; private static final String SHORT_IMAGE_TAG_VIRTUAL = "3"; - private static final String EXPECTED_REMOTE_PATH_LOCAL = SHORT_IMAGE_NAME + "/" + SHORT_IMAGE_TAG_LOCAL; - private static final String EXPECTED_REMOTE_PATH_VIRTUAL = SHORT_IMAGE_NAME + "/" + SHORT_IMAGE_TAG_VIRTUAL; + private static final String EXPECTED_REMOTE_PATH_KANIKO = "hello-world/latest"; private static final String LOCAL_DOMAIN = "BITESTS_ARTIFACTORY_DOCKER_LOCAL_DOMAIN"; private static final String REMOTE_DOMAIN = "BITESTS_ARTIFACTORY_DOCKER_REMOTE_DOMAIN"; @@ -44,9 +49,6 @@ public class DockerExtractorTest extends IntegrationTestsBase { private static final String DOCKER_VIRTUAL_REPO = "BITESTS_ARTIFACTORY_DOCKER_VIRTUAL_REPO"; private static final String DOCKER_HOST = "BITESTS_ARTIFACTORY_DOCKER_HOST"; private final ArrayListMultimap artifactProperties = ArrayListMultimap.create(); - private ArtifactoryManagerBuilder artifactoryManagerBuilder; - private String localDomainName; - private String remoteDomainName; private String pullImageFromVirtual; private String virtualDomainName; private String dockerLocalRepo; @@ -68,17 +70,15 @@ public DockerExtractorTest() { .build()); } - private static boolean isWindows() { - return System.getProperty("os.name").toLowerCase().contains("win"); - } - @BeforeClass private void setUp() { - artifactoryManagerBuilder = new ArtifactoryManagerBuilder().setArtifactoryUrl(getUrl()).setUsername(getUsername()).setPassword(getPassword()).setLog(getLog()); - // Get image name - localDomainName = validateDomainSuffix(System.getenv(LOCAL_DOMAIN)); - remoteDomainName = validateDomainSuffix(System.getenv(REMOTE_DOMAIN)); - virtualDomainName = validateDomainSuffix(System.getenv(VIRTUAL_DOMAIN)); + if (SystemUtils.IS_OS_WINDOWS) { + throw new SkipException("Skipping Docker tests on Windows OS"); + } + assertEnvironment(); + String localDomainName = StringUtils.appendIfMissing(System.getenv(LOCAL_DOMAIN), "/"); + String remoteDomainName = StringUtils.appendIfMissing(System.getenv(REMOTE_DOMAIN), "/"); + virtualDomainName = System.getenv(VIRTUAL_DOMAIN); dockerLocalRepo = System.getenv(DOCKER_LOCAL_REPO); dockerRemoteRepo = System.getenv(DOCKER_REMOTE_REPO); dockerVirtualRepo = System.getenv(DOCKER_VIRTUAL_REPO); @@ -86,195 +86,137 @@ private void setUp() { imageTagLocal = localDomainName + SHORT_IMAGE_NAME + ":" + SHORT_IMAGE_TAG_LOCAL; imageTagVirtual = localDomainName + SHORT_IMAGE_NAME + ":" + SHORT_IMAGE_TAG_VIRTUAL; pullImageFromRemote = remoteDomainName + "hello-world:latest"; - pullImageFromVirtual = virtualDomainName + "hello-world:latest"; + pullImageFromVirtual = StringUtils.appendIfMissing(virtualDomainName, "/") + "hello-world:latest"; } - @Test - public void dockerPushFromLocalTest() { - if (isWindows()) { - throw new SkipException("Skipping Docker tests on Windows OS"); - } - try { - if (StringUtils.isBlank(localDomainName)) { - throw new IOException("The " + LOCAL_DOMAIN + " environment variable is not set, failing docker tests."); - } - if (StringUtils.isBlank(dockerLocalRepo)) { - throw new IOException("The " + DOCKER_LOCAL_REPO + " environment variable is not set, failing docker tests."); - } - String projectPath = PROJECTS_ROOT.resolve("docker-push").toAbsolutePath().toString(); - DockerJavaWrapper.buildImage(imageTagLocal, host, Collections.emptyMap(), projectPath); - - DockerPush dockerPush = new DockerPush(artifactoryManagerBuilder, imageTagLocal, host, artifactProperties, dockerLocalRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); - Build build = dockerPush.execute(); - assertEquals(build.getModules().size(), 1); - Module module = build.getModules().get(0); + private void assertEnvironment() { + Lists.newArrayList(LOCAL_DOMAIN, DOCKER_LOCAL_REPO, + REMOTE_DOMAIN, DOCKER_REMOTE_REPO, + VIRTUAL_DOMAIN, DOCKER_VIRTUAL_REPO) + .forEach(envKey -> assertNotNull(System.getenv(envKey), "The '" + envKey + "' environment variable is not set, failing docker tests.")); + } - assertEquals(module.getType(), "docker"); - assertEquals(module.getRepository(), dockerLocalRepo); - List artifacts = module.getArtifacts(); - validateImageArtifacts(artifacts, imageTagLocal); - assertEquals(7, artifacts.size()); - module.getArtifacts().forEach(artifact -> assertEquals(artifact.getRemotePath(), EXPECTED_REMOTE_PATH_LOCAL)); - assertEquals(5, module.getDependencies().size()); - } catch (Exception e) { - fail(ExceptionUtils.getStackTrace(e)); - } + @AfterClass + private void tearDown() throws IOException { + deleteContentFromRepo(dockerLocalRepo); } @Test - public void dockerPushFromVirtualTest() { - if (isWindows()) { - throw new SkipException("Skipping Docker tests on Windows OS"); - } - try { - if (StringUtils.isBlank(virtualDomainName)) { - throw new IOException("The " + LOCAL_DOMAIN + " environment variable is not set, failing docker tests."); - } - if (StringUtils.isBlank(dockerVirtualRepo)) { - throw new IOException("The " + DOCKER_LOCAL_REPO + " environment variable is not set, failing docker tests."); - } - String projectPath = PROJECTS_ROOT.resolve("docker-push").toAbsolutePath().toString(); - DockerJavaWrapper.buildImage(imageTagVirtual, host, Collections.emptyMap(), projectPath); - - DockerPush dockerPush = new DockerPush(artifactoryManagerBuilder, imageTagVirtual, host, artifactProperties, dockerVirtualRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); - Build build = dockerPush.execute(); - assertEquals(build.getModules().size(), 1); - Module module = build.getModules().get(0); + public void dockerPushToLocalTest() { + DockerJavaWrapper.buildImage(imageTagLocal, host, Collections.emptyMap(), PROJECT_PATH); + DockerPush dockerPush = new DockerPush(artifactoryManagerBuilder, imageTagLocal, host, artifactProperties, dockerLocalRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); + pushAndValidateImage(dockerPush, dockerLocalRepo, imageTagLocal, SHORT_IMAGE_TAG_LOCAL); + } - assertEquals(module.getType(), "docker"); - assertEquals(module.getRepository(), dockerVirtualRepo); - List artifacts = module.getArtifacts(); - validateImageArtifacts(artifacts, imageTagVirtual); - assertEquals(7, artifacts.size()); - module.getArtifacts().forEach(artifact -> assertEquals(artifact.getRemotePath(), EXPECTED_REMOTE_PATH_VIRTUAL)); - assertEquals(5, module.getDependencies().size()); - } catch (Exception e) { - fail(ExceptionUtils.getStackTrace(e)); - } + @Test + public void dockerPushToVirtualTest() { + DockerJavaWrapper.buildImage(imageTagVirtual, host, Collections.emptyMap(), PROJECT_PATH); + DockerPush dockerPush = new DockerPush(artifactoryManagerBuilder, imageTagVirtual, host, artifactProperties, dockerVirtualRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); + pushAndValidateImage(dockerPush, dockerVirtualRepo, imageTagVirtual, SHORT_IMAGE_TAG_VIRTUAL); } @Test public void dockerPullFromRemoteTest() { - if (isWindows()) { - getLog().info("Skipping Docker tests on Windows OS"); - return; - } - try { - if (StringUtils.isBlank(remoteDomainName)) { - throw new IOException("The " + REMOTE_DOMAIN + " environment variable is not set, failing docker tests."); - } - if (StringUtils.isBlank(dockerRemoteRepo)) { - throw new IOException("The " + DOCKER_REMOTE_REPO + " environment variable is not set, failing docker tests."); - } - DockerPull dockerPull = new DockerPull(artifactoryManagerBuilder, pullImageFromRemote, host, dockerRemoteRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); - validatePulledDockerImage(dockerPull.execute(), pullImageFromRemote); - } catch (Exception e) { - fail(ExceptionUtils.getStackTrace(e)); - } + DockerPull dockerPull = new DockerPull(artifactoryManagerBuilder, pullImageFromRemote, host, dockerRemoteRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); + validatePulledDockerImage(dockerPull.execute(), pullImageFromRemote); } @Test public void dockerPullFromVirtualTest() { - if (isWindows()) { - getLog().info("Skipping Docker tests on Windows OS"); - return; - } - try { - if (StringUtils.isBlank(virtualDomainName)) { - throw new IOException("The " + VIRTUAL_DOMAIN + " environment variable is not set, failing docker tests."); - } - if (StringUtils.isBlank(dockerVirtualRepo)) { - throw new IOException("The " + DOCKER_VIRTUAL_REPO + " environment variable is not set, failing docker tests."); - } - DockerPull dockerPull = new DockerPull(artifactoryManagerBuilder, pullImageFromVirtual, host, dockerVirtualRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); - validatePulledDockerImage(dockerPull.execute(), pullImageFromVirtual); - } catch (Exception e) { - fail(ExceptionUtils.getStackTrace(e)); - } + DockerPull dockerPull = new DockerPull(artifactoryManagerBuilder, pullImageFromVirtual, host, dockerVirtualRepo, getUsername(), getPassword(), getLog(), Collections.emptyMap()); + validatePulledDockerImage(dockerPull.execute(), pullImageFromVirtual); } @Test - public void buildDockerCreateFromRemoteTest() { - if (isWindows()) { - throw new SkipException("Skipping Docker tests on Windows OS"); - } - Path kanikoFile = null; + public void buildDockerCreateKanikoTest() throws IOException, InterruptedException { + Path workingDirectory = Files.createTempDirectory("build-docker-create-kaniko-test", + PosixFilePermissions.asFileAttribute(Sets.newHashSet(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE))); try { - if (StringUtils.isBlank(remoteDomainName)) { - throw new IOException("The " + REMOTE_DOMAIN + " environment variable is not set, failing docker tests."); - } - if (StringUtils.isBlank(dockerRemoteRepo)) { - throw new IOException("The " + DOCKER_REMOTE_REPO + " environment variable is not set, failing docker tests."); - } - // Get the image id for the image already in artifactory - DockerJavaWrapper.pullImage(pullImageFromRemote, getUsername(), getPassword(), host, Collections.emptyMap(), getLog()); - String imageId = DockerJavaWrapper.getImageIdFromTag(pullImageFromRemote, host, Collections.emptyMap(), getLog()); - // Create the image file from the already created image - String kanikoImageData = pullImageFromRemote + '@' + imageId; - kanikoFile = Files.createTempFile("hello-world", ".image").toAbsolutePath(); - Files.write(kanikoFile, Collections.singleton(kanikoImageData), StandardOpenOption.TRUNCATE_EXISTING); + FileUtils.copyDirectory(new File(PROJECT_PATH), workingDirectory.toFile()); + Path kanikoConfig = createKanikoConfig(workingDirectory, virtualDomainName); + String kanikoFile = execKaniko(workingDirectory, virtualDomainName, kanikoConfig); - BuildDockerCreate buildDockerCreate = new BuildDockerCreate(artifactoryManagerBuilder, kanikoFile.toString(), artifactProperties, dockerRemoteRepo, getLog()); - Build build = buildDockerCreate.execute(); + BuildDockerCreator buildDockerCreator = new BuildDockerCreator(artifactoryManagerBuilder, kanikoFile, BuildDockerCreator.ImageFileType.KANIKO, artifactProperties, dockerVirtualRepo, getLog()); + Build build = buildDockerCreator.execute(); assertEquals(build.getModules().size(), 1); - Module module = build.getModules().get(0); - - assertEquals(module.getType(), "docker"); - assertEquals(module.getRepository(), dockerRemoteRepo); - List artifacts = module.getArtifacts(); - validateImageArtifacts(artifacts, pullImageFromRemote); - } catch (Exception e) { - fail(ExceptionUtils.getStackTrace(e)); + Module module = getAndValidateModule(build, "hello-world:latest", dockerVirtualRepo); + module.getArtifacts().stream().map(BaseBuildFileBean::getRemotePath).forEach(remotePath -> assertEquals(remotePath, EXPECTED_REMOTE_PATH_KANIKO)); } finally { - if (kanikoFile != null) { - try { - Files.deleteIfExists(kanikoFile); - } catch (IOException ex) {} - } + FileUtils.deleteDirectory(workingDirectory.toFile()); } } @Test - public void buildDockerCreateFromVirtualTest() { - if (isWindows()) { - throw new SkipException("Skipping Docker tests on Windows OS"); - } - Path kanikoFile = null; + public void buildDockerCreateJibTest() throws IOException, InterruptedException { + Path wd = Files.createTempDirectory("build-docker-create-jib-test", + PosixFilePermissions.asFileAttribute(Sets.newHashSet(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE))); try { - if (StringUtils.isBlank(virtualDomainName)) { - throw new IOException("The " + VIRTUAL_DOMAIN + " environment variable is not set, failing docker tests."); - } - if (StringUtils.isBlank(dockerVirtualRepo)) { - throw new IOException("The " + DOCKER_VIRTUAL_REPO + " environment variable is not set, failing docker tests."); - } - // Get the image id for the image already in artifactory - DockerJavaWrapper.pullImage(pullImageFromVirtual, getUsername(), getPassword(), host, Collections.emptyMap(), getLog()); - String imageId = DockerJavaWrapper.getImageIdFromTag(pullImageFromVirtual, host, Collections.emptyMap(), getLog()); - // Create the image file from the already created image - String kanikoImageData = pullImageFromVirtual + '@' + imageId; - kanikoFile = Files.createTempFile("hello-world", ".image").toAbsolutePath(); - Files.write(kanikoFile, Collections.singleton(kanikoImageData), StandardOpenOption.TRUNCATE_EXISTING); - - BuildDockerCreate buildDockerCreate = new BuildDockerCreate(artifactoryManagerBuilder, kanikoFile.toString(), artifactProperties, dockerVirtualRepo, getLog()); - Build build = buildDockerCreate.execute(); - assertEquals(build.getModules().size(), 1); - Module module = build.getModules().get(0); - - assertEquals(module.getType(), "docker"); - assertEquals(module.getRepository(), dockerVirtualRepo); - List artifacts = module.getArtifacts(); - validateImageArtifacts(artifacts, pullImageFromVirtual); - } catch (Exception e) { - fail(ExceptionUtils.getStackTrace(e)); + FileUtils.copyDirectory(PROJECTS_ROOT.resolve("maven-jib-example").toFile(), wd.toFile()); + execJib(wd); + + // Run build-docker-create + BuildDockerCreator.ImageFileType imageFileType = BuildDockerCreator.ImageFileType.JIB; + Build build = new BuildDockerCreator(artifactoryManagerBuilder, getJibImageJsonPath(wd), + imageFileType, artifactProperties, dockerVirtualRepo, getLog()).execute(); + + // Check modules + assertEquals(build.getModules().size(), 3); + Module module = getAndValidateModule(build, "multi1", dockerVirtualRepo); + assertEquals(module.getArtifacts().size(), 10); + module = getAndValidateModule(build, "multi2", dockerVirtualRepo); + assertEquals(module.getArtifacts().size(), 9); + module = getAndValidateModule(build, "multi3", dockerVirtualRepo); + assertEquals(module.getArtifacts().size(), 11); } finally { - if (kanikoFile != null) { - try { - Files.deleteIfExists(kanikoFile); - } catch (IOException ex) {} - } + FileUtils.deleteDirectory(wd.toFile()); } } + private Path createKanikoConfig(Path workingDirectory, String registry) throws IOException { + Path kanikoConfigPath = workingDirectory.toAbsolutePath().resolve("kaniko-config.json"); + ObjectMapper mapper = new ObjectMapper(); + ObjectNode credentials = mapper.createObjectNode() + .put("username", getUsername()) + .put("password", getPassword()); + ObjectNode registryNode = mapper.createObjectNode().set(registry, credentials); + String kanikoConfig = mapper.createObjectNode().set("auths", registryNode).toPrettyString(); + Files.write(kanikoConfigPath, kanikoConfig.getBytes(StandardCharsets.UTF_8)); + return kanikoConfigPath; + } + + private String execKaniko(Path workingDirectory, String registry, Path kanikoConfigPath) throws IOException, InterruptedException { + CommandExecutor commandExecutor = new CommandExecutor("docker", null); + List args = Lists.newArrayList("run", "--rm", "-v", + workingDirectory.toAbsolutePath() + ":/workspace", "-v", + kanikoConfigPath + ":/kaniko/.docker/config.json:ro", "gcr.io/kaniko-project/executor:latest", + "--dockerfile=Dockerfile", "--destination=" + registry + "/hello-world", + "--image-name-tag-with-digest-file=image-file"); + CommandResults results = commandExecutor.exeCommand(workingDirectory.toFile(), args, null, getLog()); + assertTrue(results.isOk(), results.getErr()); + return workingDirectory.resolve("image-file").toAbsolutePath().toString(); + } + + private void execJib(Path workingDirectory) throws IOException, InterruptedException { + CommandExecutor commandExecutor = new CommandExecutor("mvn", null); + List args = Lists.newArrayList("compile", "jib:build"); + CommandResults results = commandExecutor.exeCommand(workingDirectory.toFile(), args, null, getLog()); + assertTrue(results.isOk(), results.getErr()); + } + + private String getJibImageJsonPath(Path workingDirectory) { + return workingDirectory.resolve("*").resolve("target").resolve("jib-image.json").toAbsolutePath().toString(); + } + + private void pushAndValidateImage(DockerPush dockerPush, String repo, String imageTag, String shortImageTag) { + Build build = dockerPush.execute(); + Module module = getAndValidateModule(build, SHORT_IMAGE_NAME + ":" + shortImageTag, repo); + List artifacts = module.getArtifacts(); + validateImageArtifacts(artifacts, imageTag); + assertEquals(7, artifacts.size()); + module.getArtifacts().forEach(artifact -> assertEquals(artifact.getRemotePath(), SHORT_IMAGE_NAME + "/" + shortImageTag)); + assertEquals(5, module.getDependencies().size()); + } + private void validatePulledDockerImage(Build build, String image) { assertEquals(build.getModules().size(), 1); Module module = build.getModules().get(0); @@ -290,6 +232,16 @@ private void validateImageDependencies(List deps, String image) { assertTrue(deps.stream().anyMatch(dep -> dep.getId().equals(imageDigest))); } + private Module getAndValidateModule(Build build, String id, String repo) { + Module module = build.getModules().stream() + .filter(moduleCandidate -> StringUtils.equals(moduleCandidate.getId(), id)) + .findFirst().orElse(null); + assertNotNull(module); + assertEquals(module.getType(), "docker"); + assertEquals(module.getRepository(), repo); + return module; + } + private void validateImageArtifacts(List arts, String image) { String imageDigest = getImageId(image); assertTrue(arts.stream().anyMatch(art -> art.getName().equals(imageDigest))); @@ -300,13 +252,4 @@ private String getImageId(String image) { assertNotNull(id); return id.replace(":", "__"); } - - private String validateDomainSuffix(String domain) { - return StringUtils.appendIfMissing(domain, "/"); - } - - @AfterClass - private void tearDown() throws IOException { - deleteContentFromRepo(dockerLocalRepo); - } -} \ No newline at end of file +} diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker-push/Dockerfile b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/Dockerfile similarity index 100% rename from build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker-push/Dockerfile rename to build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/Dockerfile diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi1/pom.xml b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi1/pom.xml new file mode 100644 index 000000000..e64e9c224 --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi1/pom.xml @@ -0,0 +1,89 @@ + + + 4.0.0 + + org.jfrog.test + multi + 3.7-SNAPSHOT + + + multi1 + jar + Multi 1 + + + + apache + none + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + + org.apache.commons + commons-email + 1.1 + compile + + + org.codehaus.plexus + plexus-utils + 1.5.1 + + + javax.servlet.jsp + jsp-api + 2.1 + compile + + + commons-io + commons-io + 1.4 + + + org.springframework + spring-aop + 2.5.6 + + + + + org.testng + testng + jdk15 + 5.9 + test + + + + diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi1/src/main/java/artifactory/test/Multi1.java b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi1/src/main/java/artifactory/test/Multi1.java new file mode 100644 index 000000000..8e766e7a1 --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi1/src/main/java/artifactory/test/Multi1.java @@ -0,0 +1,10 @@ +package artifactory.test; + +/** + * Hello world! + */ +public class Multi1 { + public static void main(String[] args) { + System.out.println("Hello World!"); + } +} diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi2/pom.xml b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi2/pom.xml new file mode 100644 index 000000000..e29f1edaa --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi2/pom.xml @@ -0,0 +1,19 @@ + + + 4.0.0 + + org.jfrog.test + multi + 3.7-SNAPSHOT + + + + ${project.parent.version} + + + multi2 + 3.7-SNAPSHOT + jar + Multi 2 + + diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi2/src/main/java/artifactory/test/App.java b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi2/src/main/java/artifactory/test/App.java new file mode 100644 index 000000000..56915fde1 --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi2/src/main/java/artifactory/test/App.java @@ -0,0 +1,13 @@ +package artifactory.test; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/pom.xml b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/pom.xml new file mode 100644 index 000000000..324a1312a --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/pom.xml @@ -0,0 +1,75 @@ + + + 4.0.0 + + org.jfrog.test + multi + 3.7-SNAPSHOT + + + multi3 + Multi 3 + + + + + ${project.groupId} + multi1 + ${project.parent.version} + + + + + hsqldb + hsqldb + 1.8.0.10 + runtime + + + + + javax.servlet + servlet-api + 2.5 + provided + + + + + + assembly + + + + maven-assembly-plugin + 2.2-beta-5 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + org.apache.maven.plugins + maven-war-plugin + + + false + + + + + + + + + diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/src/main/java/artifactory/test/Multi3.java b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/src/main/java/artifactory/test/Multi3.java new file mode 100644 index 000000000..76af503e0 --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/src/main/java/artifactory/test/Multi3.java @@ -0,0 +1,11 @@ +package artifactory.test; + +/** + * Hello world! + */ +public class Multi3 { + public static void main(String[] args) { + new Multi1(); + System.out.println("Hello World!"); + } +} diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/src/main/webapp/WEB-INF/web.xml b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 000000000..f97cbaace --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/multi3/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,9 @@ + + + test-webapp + + \ No newline at end of file diff --git a/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/pom.xml b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/pom.xml new file mode 100644 index 000000000..ed355e56e --- /dev/null +++ b/build-info-extractor-docker/src/test/resources/org/jfrog/build/extractor/docker/maven-jib-example/pom.xml @@ -0,0 +1,95 @@ + + + 4.0.0 + org.jfrog.test + multi + 3.7-SNAPSHOT + pom + Simple Multi Modules Build + + + multi1 + multi2 + multi3 + + + + UTF-8 + 1.8 + 1.8 + + + + + junit + junit + 3.8.1 + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 2.4 + + + org.apache.maven.plugins + maven-war-plugin + 2.4 + + + org.apache.maven.plugins + maven-source-plugin + 2.1.2 + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + false + + + + + org.apache.maven.plugins + maven-war-plugin + + + false + + + + + com.google.cloud.tools + jib-maven-plugin + 3.0.0 + + + ${env.BITESTS_ARTIFACTORY_DOCKER_VIRTUAL_DOMAIN}/adoptopenjdk:8-jdk-hotspot + + ${env.BITESTS_ARTIFACTORY_USERNAME} + ${env.BITESTS_ARTIFACTORY_PASSWORD} + + + + ${env.BITESTS_ARTIFACTORY_DOCKER_VIRTUAL_DOMAIN}/${project.artifactId} + + ${env.BITESTS_ARTIFACTORY_USERNAME} + ${env.BITESTS_ARTIFACTORY_PASSWORD} + + + + + + + diff --git a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ArtifactoryClientConfiguration.java b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ArtifactoryClientConfiguration.java index 4d042c7bf..5ec113e06 100644 --- a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ArtifactoryClientConfiguration.java +++ b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ArtifactoryClientConfiguration.java @@ -63,7 +63,6 @@ public class ArtifactoryClientConfiguration { public final PipHandler pipHandler; public final DotnetHandler dotnetHandler; public final DockerHandler dockerHandler; - public final KanikoHandler kanikoHandler; public final PrefixPropertyHandler root; /** * To configure the props builder itself, so all method of this classes delegated from here @@ -82,7 +81,6 @@ public ArtifactoryClientConfiguration(Log log) { this.pipHandler = new PipHandler(); this.dotnetHandler = new DotnetHandler(); this.dockerHandler = new DockerHandler(); - this.kanikoHandler = new KanikoHandler(); } public void fillFromProperties(Map props, IncludeExcludePatterns patterns) { @@ -588,19 +586,21 @@ public String getHost() { public void setHost(String host) { rootConfig.setStringValue(DOCKER_HOST, host); } - } - public class KanikoHandler extends PrefixPropertyHandler { - public KanikoHandler() { - super(root, PROP_KANIKO_PREFIX); + public String getKanikoImageFile() { + return rootConfig.getStringValue(KANIKO_IMAGE_FILE); } - public String getImageFile() { - return rootConfig.getStringValue(KANIKO_IMAGE_FILE); + public void setKanikoImageFile(String kanikoImageFile) { + rootConfig.setStringValue(KANIKO_IMAGE_FILE, kanikoImageFile); + } + + public String getJibImageFile() { + return rootConfig.getStringValue(JIB_IMAGE_FILE); } - public void setImageFile(String imageManifestPath) { - rootConfig.setStringValue(KANIKO_IMAGE_FILE, imageManifestPath); + public void setJibImageFile(String jibImageFile) { + rootConfig.setStringValue(JIB_IMAGE_FILE, jibImageFile); } } diff --git a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ClientConfigurationFields.java b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ClientConfigurationFields.java index ab6063f9a..8a4b8edf9 100644 --- a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ClientConfigurationFields.java +++ b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/ClientConfigurationFields.java @@ -39,6 +39,7 @@ public interface ClientConfigurationFields { String DOTNET_NUGET_PROTOCOL = "dotnet.nuget.protocol"; String DOCKER_IMAGE_TAG = "docker.image.tag"; String KANIKO_IMAGE_FILE = "kaniko.image.file"; + String JIB_IMAGE_FILE = "jib.image.file"; String DOCKER_HOST = "docker.host"; String URL = "url"; String REPO_KEY = "repoKey"; diff --git a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/ArtifactoryManager.java b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/ArtifactoryManager.java index 419a3103e..4abd0c7ca 100644 --- a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/ArtifactoryManager.java +++ b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/ArtifactoryManager.java @@ -16,6 +16,7 @@ import org.jfrog.build.api.util.Log; import org.jfrog.build.client.ArtifactoryUploadResponse; import org.jfrog.build.client.ArtifactoryVersion; +import org.jfrog.build.client.DownloadResponse; import org.jfrog.build.client.ItemLastModified; import org.jfrog.build.client.artifactoryXrayResponse.ArtifactoryXrayResponse; import org.jfrog.build.extractor.clientConfiguration.client.ManagerBase; @@ -82,11 +83,11 @@ public void distributeBuild(String buildName, String buildNumber, Distribution p distributeBuildService.execute(jfrogHttpClient); } - public String download(String downloadFrom) throws IOException { + public DownloadResponse download(String downloadFrom) throws IOException { return download(downloadFrom, null); } - public String download(String downloadFrom, Map headers) throws IOException { + public DownloadResponse download(String downloadFrom, Map headers) throws IOException { Download downloadService = new Download(downloadFrom, headers, log); return downloadService.execute(jfrogHttpClient); } diff --git a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/services/Download.java b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/services/Download.java index dcb60fea2..68f8e4e00 100644 --- a/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/services/Download.java +++ b/build-info-extractor/src/main/java/org/jfrog/build/extractor/clientConfiguration/client/artifactory/services/Download.java @@ -2,19 +2,20 @@ import org.apache.commons.io.IOUtils; import org.jfrog.build.api.util.Log; +import org.jfrog.build.client.DownloadResponse; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Map; -public class Download extends DownloadBase { +public class Download extends DownloadBase { public Download(String downloadFrom, Map headers, Log log) { super(downloadFrom, false, headers, log); } @Override protected void setResponse(InputStream stream) throws IOException { - result = IOUtils.toString(stream, StandardCharsets.UTF_8.name()); + result = new DownloadResponse(IOUtils.toString(stream, StandardCharsets.UTF_8.name()), getHeaders()); } }