Skip to content

Commit

Permalink
feat: add manifest resolve task (#276)
Browse files Browse the repository at this point in the history
* feat: add manifest resolve task

* checkstyle
  • Loading branch information
paullatzelsperger committed Oct 11, 2024
1 parent b56fd39 commit 56ef1b0
Show file tree
Hide file tree
Showing 6 changed files with 280 additions and 95 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,11 @@

package org.eclipse.edc.plugins.autodoc;

import org.eclipse.edc.plugins.autodoc.tasks.ManifestDownloadTask;
import org.eclipse.edc.plugins.autodoc.tasks.DownloadManifestTask;
import org.eclipse.edc.plugins.autodoc.tasks.MarkdownRendererTask.ToHtml;
import org.eclipse.edc.plugins.autodoc.tasks.MarkdownRendererTask.ToMarkdown;
import org.eclipse.edc.plugins.autodoc.tasks.MergeManifestsTask;
import org.eclipse.edc.plugins.autodoc.tasks.ResolveManifestTask;
import org.gradle.api.Plugin;
import org.gradle.api.Project;

Expand All @@ -44,6 +45,8 @@ public void apply(Project project) {
project.getTasks().register(MergeManifestsTask.NAME, MergeManifestsTask.class, t -> t.dependsOn(AUTODOC_TASK_NAME).setGroup(GROUP_NAME));
project.getTasks().register(ToMarkdown.NAME, ToMarkdown.class, t -> t.setGroup(GROUP_NAME));
project.getTasks().register(ToHtml.NAME, ToHtml.class, t -> t.setGroup(GROUP_NAME));
project.getTasks().register(ManifestDownloadTask.NAME, ManifestDownloadTask.class, t -> t.setGroup(GROUP_NAME));
project.getTasks().register(DownloadManifestTask.NAME, DownloadManifestTask.class, t -> t.setGroup(GROUP_NAME));
// resolving manifests requires the Autodoc manifests of all dependencies to exist already
project.getTasks().register(ResolveManifestTask.NAME, ResolveManifestTask.class, t -> t.dependsOn(AUTODOC_TASK_NAME).setGroup(GROUP_NAME));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

package org.eclipse.edc.plugins.autodoc.tasks;

import org.eclipse.edc.plugins.autodoc.AutodocExtension;
import org.gradle.api.DefaultTask;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency;
import org.gradle.api.tasks.Internal;
import org.gradle.api.tasks.TaskAction;
import org.gradle.api.tasks.options.Option;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.Optional;
import java.util.Set;

import static java.util.Objects.requireNonNull;

/**
* Abstract gradle task, that "resolves" an already-existing autodoc manifest from a URI and transfers (=copies, downloads,...)
* the file to a directory on the local file system.
* <p>
* Implementations must provide a reference to that autodoc manifest file in the form of a {@link DependencySource}.
*/
public abstract class AbstractManifestResolveTask extends DefaultTask {
public static final String MANIFEST_CLASSIFIER = "manifest";
public static final String MANIFEST_TYPE = "json";
protected Path downloadDirectory;
private File outputDirectoryOverride;

public AbstractManifestResolveTask() {
downloadDirectory = getProject().getLayout().getBuildDirectory().getAsFile().get().toPath().resolve("autodoc");
}

@TaskAction
public void resolveAutodocManifest() {
var autodocExt = getProject().getExtensions().findByType(AutodocExtension.class);
requireNonNull(autodocExt, "AutodocExtension cannot be null");

if (autodocExt.getDownloadDirectory().isPresent()) {
downloadDirectory = autodocExt.getDownloadDirectory().get().toPath();
}

if (outputDirectoryOverride != null) {
downloadDirectory = outputDirectoryOverride.toPath();
}

getProject().getConfigurations()
.stream().flatMap(config -> config.getDependencies().stream())
.filter(dep -> dep instanceof DefaultProjectDependency)
.filter(dep -> !getExclusions().contains(dep.getName()))
.map(this::createSource)
.filter(Optional::isPresent)
.forEach(dt -> transferDependencyFile(dt.get(), downloadDirectory));
}

@Option(option = "output", description = "CLI option to override the output directory")
public void setOutput(String output) {
this.outputDirectoryOverride = new File(output);
}

/**
* Returns an {@link InputStream} that points to the physical location of the autodoc manifest file.
*/
@Internal //otherwise it would get interpreted as task input :/
protected abstract InputStream resolveManifest(DependencySource autodocManifest);

@Internal //otherwise it would get interpreted as task input :/
protected Set<String> getExclusions() {
return Set.of();
}

@Internal //otherwise it would get interpreted as task input :/
protected abstract Optional<DependencySource> createSource(Dependency dependency);

private void transferDependencyFile(DependencySource dependencySource, Path downloadDirectory) {
var targetFilePath = downloadDirectory.resolve(dependencySource.filename());
try (var inputStream = resolveManifest(dependencySource)) {
downloadDirectory.toFile().mkdirs();
getLogger().debug("Downloading {} into {}", dependencySource, downloadDirectory);
try (var fos = new FileOutputStream(targetFilePath.toFile())) {
inputStream.transferTo(fos);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright (c) 2024 Bayerische Motoren Werke Aktiengesellschaft (BMW AG)
*
* This program and the accompanying materials are made available under the
* terms of the Apache License, Version 2.0 which is available at
* https://www.apache.org/licenses/LICENSE-2.0
*
* SPDX-License-Identifier: Apache-2.0
*
* Contributors:
* Bayerische Motoren Werke Aktiengesellschaft (BMW AG) - initial API and implementation
*
*/

package org.eclipse.edc.plugins.autodoc.tasks;

import org.gradle.api.artifacts.Dependency;

import java.net.URI;

import static java.lang.String.format;

/**
* Represents the combination of a dependency and a pointer (URL) to its physical location.
*
* @param dependency the dependency in question
* @param uri the location where the physical file exists
* @param classifier what type of dependency we have, e.g. sources, sources, manifest etc
* @param type file extension
*/
public record DependencySource(Dependency dependency, URI uri, String classifier, String type) {
@Override
public String toString() {
return "{" +
"dependency=" + dependency +
", uri=" + uri +
'}';
}

String filename() {
return format("%s-%s-%s.%s", dependency.getName(), dependency.getVersion(), classifier, type);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,111 +14,74 @@

package org.eclipse.edc.plugins.autodoc.tasks;

import org.eclipse.edc.plugins.autodoc.AutodocExtension;
import org.gradle.api.DefaultTask;
import org.gradle.api.artifacts.Dependency;
import org.gradle.api.artifacts.repositories.MavenArtifactRepository;
import org.gradle.api.tasks.TaskAction;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;

import static java.lang.String.format;
import static java.util.Objects.requireNonNull;

public class ManifestDownloadTask extends DefaultTask {
public class DownloadManifestTask extends AbstractManifestResolveTask {

public static final String NAME = "downloadManifests";
private static final String EDC_GROUP = "org.eclipse.edc";
private static final Duration MAX_MANIFEST_AGE = Duration.ofHours(24);
private static final String MANIFEST_CLASSIFIER = "manifest";
private static final String MANIFEST_TYPE = "json";

private final HttpClient httpClient;
private Path downloadDirectory;

public ManifestDownloadTask() {
public DownloadManifestTask() {
httpClient = HttpClient.newHttpClient();
downloadDirectory = getProject().getRootProject().getBuildDir().toPath().resolve("manifests");
}

@TaskAction
public void downloadManifests() {
var autodocExt = getProject().getExtensions().findByType(AutodocExtension.class);
requireNonNull(autodocExt, "AutodocExtension cannot be null");

if (autodocExt.getDownloadDirectory().isPresent()) {
downloadDirectory = autodocExt.getDownloadDirectory().get().toPath();
}

getProject().getConfigurations()
.stream().flatMap(config -> config.getDependencies().stream())
.filter(dep -> EDC_GROUP.equals(dep.getGroup()))
.filter(dep -> !getExclusions().contains(dep.getName()))
.map(this::createDownloadRequest)
.filter(Optional::isPresent)
.forEach(dt -> downloadDependency(dt.get(), downloadDirectory));
}

private String createArtifactUrl(Dependency dep, MavenArtifactRepository repo) {
return format("%s%s/%s/%s/%s-%s-%s.%s", repo.getUrl(), dep.getGroup().replace(".", "/"), dep.getName(), dep.getVersion(),
dep.getName(), dep.getVersion(), MANIFEST_CLASSIFIER, MANIFEST_TYPE);
}

private void downloadDependency(DependencyDownload dt, Path outputDirectory) {

var p = outputDirectory.resolve(dt.filename());
var request = HttpRequest.newBuilder().uri(dt.uri()).GET().build();
@Override
protected InputStream resolveManifest(DependencySource autodocManifest) {
var request = HttpRequest.newBuilder().uri(autodocManifest.uri()).GET().build();
HttpResponse<InputStream> response;
try {
var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
if (response.statusCode() != 200) {
getLogger().warn("Could not download {}, HTTP response: {}", dt.dependency, response);
return;
}
outputDirectory.toFile().mkdirs();
getLogger().debug("Downloading {} into {}", dt, outputDirectory);
try (var is = response.body(); var fos = new FileOutputStream(p.toFile())) {
is.transferTo(fos);
}
response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream());
} catch (IOException | InterruptedException e) {
throw new RuntimeException(e);
}
if (response.statusCode() != 200) {
getLogger().warn("Could not download {}, HTTP response: {}", autodocManifest.dependency(), response);
return InputStream.nullInputStream();
}
return response.body();
}

/**
* Creates a download request for a given dependency, classifier, and type. A download request is successfully created if:
* <ul>
* <li>the output directory does not exists</li>
* <li>the file does not exist locally</li>
* <li>the file exists locally, but is too old (<24hrs) </li>
* <li>the file does not exist locally, e.g. from a previous run</li>
* <li>the file exists locally, but is too old (older than 24hrs) </li>
* <li>the file exists locally, but is not readable</li>
* <li>the file is found in at least one Maven repository. MavenLocal is ignored.</li>
* </ul>
*
* @param dep the dependency to download
* @return an optional DownloadRequest if the artifact should be downloaded, otherwise an empty optional
* @param dependency the dependency to download
* @return an optional DownloadRequest if the artifact can be downloaded, otherwise an empty optional
*/
private Optional<DependencyDownload> createDownloadRequest(Dependency dep) {
if (isLocalFileValid(dep)) {
getLogger().debug("Local file {} was deemed to be viable, will not download", new DependencyDownload(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename());
@Override
protected Optional<DependencySource> createSource(Dependency dependency) {
if (isLocalFileValid(dependency)) {
getLogger().debug("Local file {} was deemed to be viable, will not download", new DependencySource(dependency, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename());
return Optional.empty();
}
var repos = getProject().getRepositories().stream().toList();
return repos.stream()
.filter(repo -> repo instanceof MavenArtifactRepository)
.map(repo -> (MavenArtifactRepository) repo)
.map(repo -> {
var repoUrl = createArtifactUrl(dep, repo);
var repoUrl = createArtifactUrl(dependency, repo);
try {
// we use a HEAD request, because we only want to see whether that module has a `-manifest.json`
var uri = URI.create(repoUrl);
Expand All @@ -128,7 +91,7 @@ private Optional<DependencyDownload> createDownloadRequest(Dependency dep) {
.build();
var response = httpClient.send(headRequest, HttpResponse.BodyHandlers.discarding());
if (response.statusCode() == 200) {
return new DependencyDownload(dep, uri, MANIFEST_CLASSIFIER, MANIFEST_TYPE);
return new DependencySource(dependency, uri, MANIFEST_CLASSIFIER, MANIFEST_TYPE);
}
return null;
} catch (IOException | InterruptedException | IllegalArgumentException e) {
Expand All @@ -139,6 +102,11 @@ private Optional<DependencyDownload> createDownloadRequest(Dependency dep) {
.findFirst();
}

private String createArtifactUrl(Dependency dep, MavenArtifactRepository repo) {
return format("%s%s/%s/%s/%s-%s-%s.%s", repo.getUrl(), dep.getGroup().replace(".", "/"), dep.getName(), dep.getVersion(),
dep.getName(), dep.getVersion(), MANIFEST_CLASSIFIER, MANIFEST_TYPE);
}

/**
* Checks if the manifest for a dependency exists locally. A local file is considered valid if:
* <ul>
Expand All @@ -152,7 +120,7 @@ private Optional<DependencyDownload> createDownloadRequest(Dependency dep) {
*/
private boolean isLocalFileValid(Dependency dep) {
if (!downloadDirectory.toFile().exists()) return false;
var filePath = downloadDirectory.resolve(new DependencyDownload(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename());
var filePath = downloadDirectory.resolve(new DependencySource(dep, null, MANIFEST_CLASSIFIER, MANIFEST_TYPE).filename());
var file = filePath.toFile();
if (!file.exists() || !file.canRead()) return false;

Expand All @@ -164,21 +132,5 @@ private boolean isLocalFileValid(Dependency dep) {
}
}

private Set<String> getExclusions() {
return Set.of();
}

private record DependencyDownload(Dependency dependency, URI uri, String classifier, String type) {
@Override
public String toString() {
return "{" +
"dependency=" + dependency +
", uri=" + uri +
'}';
}

String filename() {
return format("%s-%s-%s.%s", dependency.getName(), dependency.getVersion(), classifier, type);
}
}
}
Loading

0 comments on commit 56ef1b0

Please sign in to comment.