From 56ef1b0cc3334f9f773324a1051ad11fcc6f9607 Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger <43503240+paullatzelsperger@users.noreply.github.com> Date: Fri, 11 Oct 2024 07:43:31 +0200 Subject: [PATCH] feat: add manifest resolve task (#276) * feat: add manifest resolve task * checkstyle --- .../edc/plugins/autodoc/AutodocPlugin.java | 7 +- .../tasks/AbstractManifestResolveTask.java | 104 +++++++++++++++++ .../autodoc/tasks/DependencySource.java | 43 +++++++ ...oadTask.java => DownloadManifestTask.java} | 108 +++++------------- .../autodoc/tasks/MergeManifestsTask.java | 47 +++++--- .../autodoc/tasks/ResolveManifestTask.java | 66 +++++++++++ 6 files changed, 280 insertions(+), 95 deletions(-) create mode 100644 plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/AbstractManifestResolveTask.java create mode 100644 plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DependencySource.java rename plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/{ManifestDownloadTask.java => DownloadManifestTask.java} (52%) create mode 100644 plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ResolveManifestTask.java diff --git a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/AutodocPlugin.java b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/AutodocPlugin.java index b1f75a90..99c3380f 100644 --- a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/AutodocPlugin.java +++ b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/AutodocPlugin.java @@ -14,10 +14,11 @@ package org.eclipse.edc.plugins.autodoc; -import org.eclipse.edc.plugins.autodoc.tasks.ManifestDownloadTask; +import org.eclipse.edc.plugins.autodoc.tasks.DownloadManifestTask; import org.eclipse.edc.plugins.autodoc.tasks.MarkdownRendererTask.ToHtml; import org.eclipse.edc.plugins.autodoc.tasks.MarkdownRendererTask.ToMarkdown; import org.eclipse.edc.plugins.autodoc.tasks.MergeManifestsTask; +import org.eclipse.edc.plugins.autodoc.tasks.ResolveManifestTask; import org.gradle.api.Plugin; import org.gradle.api.Project; @@ -44,6 +45,8 @@ public void apply(Project project) { project.getTasks().register(MergeManifestsTask.NAME, MergeManifestsTask.class, t -> t.dependsOn(AUTODOC_TASK_NAME).setGroup(GROUP_NAME)); project.getTasks().register(ToMarkdown.NAME, ToMarkdown.class, t -> t.setGroup(GROUP_NAME)); project.getTasks().register(ToHtml.NAME, ToHtml.class, t -> t.setGroup(GROUP_NAME)); - project.getTasks().register(ManifestDownloadTask.NAME, ManifestDownloadTask.class, t -> t.setGroup(GROUP_NAME)); + project.getTasks().register(DownloadManifestTask.NAME, DownloadManifestTask.class, t -> t.setGroup(GROUP_NAME)); + // resolving manifests requires the Autodoc manifests of all dependencies to exist already + project.getTasks().register(ResolveManifestTask.NAME, ResolveManifestTask.class, t -> t.dependsOn(AUTODOC_TASK_NAME).setGroup(GROUP_NAME)); } } diff --git a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/AbstractManifestResolveTask.java b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/AbstractManifestResolveTask.java new file mode 100644 index 00000000..1f304107 --- /dev/null +++ b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/AbstractManifestResolveTask.java @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.plugins.autodoc.tasks; + +import org.eclipse.edc.plugins.autodoc.AutodocExtension; +import org.gradle.api.DefaultTask; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +/** + * Abstract gradle task, that "resolves" an already-existing autodoc manifest from a URI and transfers (=copies, downloads,...) + * the file to a directory on the local file system. + *

+ * Implementations must provide a reference to that autodoc manifest file in the form of a {@link DependencySource}. + */ +public abstract class AbstractManifestResolveTask extends DefaultTask { + public static final String MANIFEST_CLASSIFIER = "manifest"; + public static final String MANIFEST_TYPE = "json"; + protected Path downloadDirectory; + private File outputDirectoryOverride; + + public AbstractManifestResolveTask() { + downloadDirectory = getProject().getLayout().getBuildDirectory().getAsFile().get().toPath().resolve("autodoc"); + } + + @TaskAction + public void resolveAutodocManifest() { + var autodocExt = getProject().getExtensions().findByType(AutodocExtension.class); + requireNonNull(autodocExt, "AutodocExtension cannot be null"); + + if (autodocExt.getDownloadDirectory().isPresent()) { + downloadDirectory = autodocExt.getDownloadDirectory().get().toPath(); + } + + if (outputDirectoryOverride != null) { + downloadDirectory = outputDirectoryOverride.toPath(); + } + + getProject().getConfigurations() + .stream().flatMap(config -> config.getDependencies().stream()) + .filter(dep -> dep instanceof DefaultProjectDependency) + .filter(dep -> !getExclusions().contains(dep.getName())) + .map(this::createSource) + .filter(Optional::isPresent) + .forEach(dt -> transferDependencyFile(dt.get(), downloadDirectory)); + } + + @Option(option = "output", description = "CLI option to override the output directory") + public void setOutput(String output) { + this.outputDirectoryOverride = new File(output); + } + + /** + * Returns an {@link InputStream} that points to the physical location of the autodoc manifest file. + */ + @Internal //otherwise it would get interpreted as task input :/ + protected abstract InputStream resolveManifest(DependencySource autodocManifest); + + @Internal //otherwise it would get interpreted as task input :/ + protected Set getExclusions() { + return Set.of(); + } + + @Internal //otherwise it would get interpreted as task input :/ + protected abstract Optional createSource(Dependency dependency); + + private void transferDependencyFile(DependencySource dependencySource, Path downloadDirectory) { + var targetFilePath = downloadDirectory.resolve(dependencySource.filename()); + try (var inputStream = resolveManifest(dependencySource)) { + downloadDirectory.toFile().mkdirs(); + getLogger().debug("Downloading {} into {}", dependencySource, downloadDirectory); + try (var fos = new FileOutputStream(targetFilePath.toFile())) { + inputStream.transferTo(fos); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DependencySource.java b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DependencySource.java new file mode 100644 index 00000000..76b871ec --- /dev/null +++ b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DependencySource.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.plugins.autodoc.tasks; + +import org.gradle.api.artifacts.Dependency; + +import java.net.URI; + +import static java.lang.String.format; + +/** + * Represents the combination of a dependency and a pointer (URL) to its physical location. + * + * @param dependency the dependency in question + * @param uri the location where the physical file exists + * @param classifier what type of dependency we have, e.g. sources, sources, manifest etc + * @param type file extension + */ +public record DependencySource(Dependency dependency, URI uri, String classifier, String type) { + @Override + public String toString() { + return "{" + + "dependency=" + dependency + + ", uri=" + uri + + '}'; + } + + String filename() { + return format("%s-%s-%s.%s", dependency.getName(), dependency.getVersion(), classifier, type); + } +} diff --git a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ManifestDownloadTask.java b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DownloadManifestTask.java similarity index 52% rename from plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ManifestDownloadTask.java rename to plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DownloadManifestTask.java index 50d0610f..a2fb12fb 100644 --- a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ManifestDownloadTask.java +++ b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/DownloadManifestTask.java @@ -14,103 +14,66 @@ package org.eclipse.edc.plugins.autodoc.tasks; -import org.eclipse.edc.plugins.autodoc.AutodocExtension; -import org.gradle.api.DefaultTask; import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.repositories.MavenArtifactRepository; -import org.gradle.api.tasks.TaskAction; -import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.file.Files; -import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.Objects; import java.util.Optional; -import java.util.Set; import static java.lang.String.format; -import static java.util.Objects.requireNonNull; -public class ManifestDownloadTask extends DefaultTask { +public class DownloadManifestTask extends AbstractManifestResolveTask { public static final String NAME = "downloadManifests"; - private static final String EDC_GROUP = "org.eclipse.edc"; private static final Duration MAX_MANIFEST_AGE = Duration.ofHours(24); - private static final String MANIFEST_CLASSIFIER = "manifest"; - private static final String MANIFEST_TYPE = "json"; + private final HttpClient httpClient; - private Path downloadDirectory; - public ManifestDownloadTask() { + public DownloadManifestTask() { httpClient = HttpClient.newHttpClient(); - downloadDirectory = getProject().getRootProject().getBuildDir().toPath().resolve("manifests"); - } - - @TaskAction - public void downloadManifests() { - var autodocExt = getProject().getExtensions().findByType(AutodocExtension.class); - requireNonNull(autodocExt, "AutodocExtension cannot be null"); - - if (autodocExt.getDownloadDirectory().isPresent()) { - downloadDirectory = autodocExt.getDownloadDirectory().get().toPath(); - } - - getProject().getConfigurations() - .stream().flatMap(config -> config.getDependencies().stream()) - .filter(dep -> EDC_GROUP.equals(dep.getGroup())) - .filter(dep -> !getExclusions().contains(dep.getName())) - .map(this::createDownloadRequest) - .filter(Optional::isPresent) - .forEach(dt -> downloadDependency(dt.get(), downloadDirectory)); } - private String createArtifactUrl(Dependency dep, MavenArtifactRepository repo) { - return format("%s%s/%s/%s/%s-%s-%s.%s", repo.getUrl(), dep.getGroup().replace(".", "/"), dep.getName(), dep.getVersion(), - dep.getName(), dep.getVersion(), MANIFEST_CLASSIFIER, MANIFEST_TYPE); - } - - private void downloadDependency(DependencyDownload dt, Path outputDirectory) { - - var p = outputDirectory.resolve(dt.filename()); - var request = HttpRequest.newBuilder().uri(dt.uri()).GET().build(); + @Override + protected InputStream resolveManifest(DependencySource autodocManifest) { + var request = HttpRequest.newBuilder().uri(autodocManifest.uri()).GET().build(); + HttpResponse response; try { - var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); - if (response.statusCode() != 200) { - getLogger().warn("Could not download {}, HTTP response: {}", dt.dependency, response); - return; - } - outputDirectory.toFile().mkdirs(); - getLogger().debug("Downloading {} into {}", dt, outputDirectory); - try (var is = response.body(); var fos = new FileOutputStream(p.toFile())) { - is.transferTo(fos); - } + response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); } catch (IOException | InterruptedException e) { throw new RuntimeException(e); } + if (response.statusCode() != 200) { + getLogger().warn("Could not download {}, HTTP response: {}", autodocManifest.dependency(), response); + return InputStream.nullInputStream(); + } + return response.body(); } /** * Creates a download request for a given dependency, classifier, and type. A download request is successfully created if: *

* - * @param dep the dependency to download - * @return an optional DownloadRequest if the artifact should be downloaded, otherwise an empty optional + * @param dependency the dependency to download + * @return an optional DownloadRequest if the artifact can be downloaded, otherwise an empty optional */ - private Optional createDownloadRequest(Dependency dep) { - if (isLocalFileValid(dep)) { - getLogger().debug("Local file {} was deemed to be viable, will not download", new DependencyDownload(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename()); + @Override + protected Optional createSource(Dependency dependency) { + if (isLocalFileValid(dependency)) { + getLogger().debug("Local file {} was deemed to be viable, will not download", new DependencySource(dependency, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename()); return Optional.empty(); } var repos = getProject().getRepositories().stream().toList(); @@ -118,7 +81,7 @@ private Optional createDownloadRequest(Dependency dep) { .filter(repo -> repo instanceof MavenArtifactRepository) .map(repo -> (MavenArtifactRepository) repo) .map(repo -> { - var repoUrl = createArtifactUrl(dep, repo); + var repoUrl = createArtifactUrl(dependency, repo); try { // we use a HEAD request, because we only want to see whether that module has a `-manifest.json` var uri = URI.create(repoUrl); @@ -128,7 +91,7 @@ private Optional createDownloadRequest(Dependency dep) { .build(); var response = httpClient.send(headRequest, HttpResponse.BodyHandlers.discarding()); if (response.statusCode() == 200) { - return new DependencyDownload(dep, uri, MANIFEST_CLASSIFIER, MANIFEST_TYPE); + return new DependencySource(dependency, uri, MANIFEST_CLASSIFIER, MANIFEST_TYPE); } return null; } catch (IOException | InterruptedException | IllegalArgumentException e) { @@ -139,6 +102,11 @@ private Optional createDownloadRequest(Dependency dep) { .findFirst(); } + private String createArtifactUrl(Dependency dep, MavenArtifactRepository repo) { + return format("%s%s/%s/%s/%s-%s-%s.%s", repo.getUrl(), dep.getGroup().replace(".", "/"), dep.getName(), dep.getVersion(), + dep.getName(), dep.getVersion(), MANIFEST_CLASSIFIER, MANIFEST_TYPE); + } + /** * Checks if the manifest for a dependency exists locally. A local file is considered valid if: *
    @@ -152,7 +120,7 @@ private Optional createDownloadRequest(Dependency dep) { */ private boolean isLocalFileValid(Dependency dep) { if (!downloadDirectory.toFile().exists()) return false; - var filePath = downloadDirectory.resolve(new DependencyDownload(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename()); + var filePath = downloadDirectory.resolve(new DependencySource(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename()); var file = filePath.toFile(); if (!file.exists() || !file.canRead()) return false; @@ -164,21 +132,5 @@ private boolean isLocalFileValid(Dependency dep) { } } - private Set getExclusions() { - return Set.of(); - } - - private record DependencyDownload(Dependency dependency, URI uri, String classifier, String type) { - @Override - public String toString() { - return "{" + - "dependency=" + dependency + - ", uri=" + uri + - '}'; - } - String filename() { - return format("%s-%s-%s.%s", dependency.getName(), dependency.getVersion(), classifier, type); - } - } } diff --git a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/MergeManifestsTask.java b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/MergeManifestsTask.java index 2cbf8e1d..1edf6e90 100644 --- a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/MergeManifestsTask.java +++ b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/MergeManifestsTask.java @@ -19,6 +19,7 @@ import org.gradle.api.GradleException; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; import org.gradle.util.internal.GFileUtils; import java.io.File; @@ -28,19 +29,33 @@ /** * Task that takes an input file (JSON) and appends its contents to a destination file. This task is intended to be called per-project. */ -public class MergeManifestsTask extends DefaultTask { +public abstract class MergeManifestsTask extends DefaultTask { public static final String NAME = "mergeManifests"; private static final String MERGED_MANIFEST_FILENAME = "manifest.json"; private final JsonFileAppender appender; + private final File projectBuildDirectory; private File destinationFile; + private File inputDirectory; public MergeManifestsTask() { appender = new JsonFileAppender(getProject().getLogger()); - var rootProjectPath = getProject().getRootProject().getLayout().getBuildDirectory().getAsFile().get().getAbsolutePath(); - destinationFile = Path.of(rootProjectPath, MERGED_MANIFEST_FILENAME).toFile(); + projectBuildDirectory = getProject().getLayout().getBuildDirectory().getAsFile().get(); + destinationFile = projectBuildDirectory.toPath().resolve(MERGED_MANIFEST_FILENAME).toFile(); + inputDirectory = projectBuildDirectory.toPath().resolve("autodoc").toFile(); } + /** + * The destination file. By default, it is set to {@code /build/manifest.json} + */ + @OutputFile + public File getDestinationFile() { + return destinationFile; + } + + public void setDestinationFile(File destinationFile) { + this.destinationFile = destinationFile; + } @TaskAction public void mergeManifests() { @@ -54,14 +69,24 @@ public void mergeManifests() { throw new GradleException("destinationFile must be configured but was null!"); } - var projectBuildDirectory = getProject().getLayout().getBuildDirectory().getAsFile(); var sourceFile = Path.of(autodocExt.getOutputDirectory().convention(projectBuildDirectory).get().getAbsolutePath(), "edc.json").toFile(); + if (sourceFile.exists()) { appender.append(destination, sourceFile); } else { getProject().getLogger().lifecycle("Skip project [{}] - no manifest file found", sourceFile); } + + // if an additional input directory was specified via CLI, lets include the files in it. + if (inputDirectory != null && + inputDirectory.exists() && + autodocExt.isIncludeTransitive()) { + var files = GFileUtils.listFiles(inputDirectory, new String[]{ "json" }, false); + getLogger().lifecycle("Appending [{}] additional JSON files from the inputDirectory to the merged manifest", files.size()); + files.forEach(f -> appender.append(destination, f)); + } + // if an additional input directory was specified, lets include the files in it. if (autodocExt.getAdditionalInputDirectory().isPresent() && autodocExt.getAdditionalInputDirectory().get().exists() && @@ -75,16 +100,8 @@ public void mergeManifests() { } - /** - * The destination file. By default, it is set to {@code /build/manifest.json} - */ - @OutputFile - public File getDestinationFile() { - return destinationFile; - } - - public void setDestinationFile(File destinationFile) { - this.destinationFile = destinationFile; + @Option(option = "input", description = "Directory where previously downloaded or resolved manifest files reside") + public void setInputDirectory(String inputDirectory) { + this.inputDirectory = new File(inputDirectory); } - } diff --git a/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ResolveManifestTask.java b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ResolveManifestTask.java new file mode 100644 index 00000000..0bfe57ed --- /dev/null +++ b/plugins/autodoc/autodoc-plugin/src/main/java/org/eclipse/edc/plugins/autodoc/tasks/ResolveManifestTask.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG) + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation + * + */ + +package org.eclipse.edc.plugins.autodoc.tasks; + +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Optional; + +import static java.io.InputStream.nullInputStream; + +public class ResolveManifestTask extends AbstractManifestResolveTask { + + public static final String NAME = "resolveManifests"; + + + @Override + protected InputStream resolveManifest(DependencySource autodocManifest) { + var uri = autodocManifest.uri(); + try { + var file = new File(uri); + if (file.exists()) { + return new FileInputStream(file); + } + + getLogger().info("File {} does not exist", file); + return nullInputStream(); + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } + } + + @Override + protected Optional createSource(Dependency dependency) { + if (dependency instanceof DefaultProjectDependency localDepdendency) { + var manifestFile = localDepdendency.getDependencyProject().getLayout().getBuildDirectory().file("edc.json"); + if (manifestFile.isPresent()) { + return Optional.of(new DependencySource(localDepdendency, manifestFile.get().getAsFile().toURI(), MANIFEST_CLASSIFIER, MANIFEST_TYPE)); + } else { + getLogger().debug("No manifest file found for dependency {}", dependency); + } + + } else { + getLogger().debug("Dependency {} is not a DefaultProjectDependency", dependency); + } + + return Optional.empty(); + } + +}