Skip to content

Commit

Permalink
Add Support for Composer 2 Metadata API
Browse files Browse the repository at this point in the history 
Signed-off-by: ch8matt <[email protected]>
  • Loading branch information
@ch8matt
ch8matt committed Dec 4, 2024
1 parent 241ad03 commit 7f9f245
Show file tree
Hide file tree
Showing 5 changed files with 128 additions and 70 deletions.
194 changes: 126 additions & 68 deletions src/main/java/org/dependencytrack/tasks/repositories/ComposerMetaAnalyzer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import alpine.common.logging.Logger;
import com.github.packageurl.PackageURL;
import org.dependencytrack.exception.MetaAnalyzerException;
import org.json.JSONException;
import org.json.JSONObject;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.CloseableHttpResponse;
Expand All @@ -37,112 +38,169 @@
/**
* An IMetaAnalyzer implementation that supports Composer.
*
* @author Szabolcs (Szasza) Palmer
* @since 4.1.0
* @authors:
* - Szabolcs (Szasza) Palmer
* - ch8matt
*/
public class ComposerMetaAnalyzer extends AbstractMetaAnalyzer {

private static final Logger LOGGER = Logger.getLogger(ComposerMetaAnalyzer.class);
private static final String DEFAULT_BASE_URL = "https://repo.packagist.org";

/**
* @see <a href="https://packagist.org/apidoc#get-package-metadata-v1">Packagist's API doc for "Getting package data - Using the Composer v1 metadata (DEPRECATED)"</a>
*/
private static final String API_URL = "/p/%s/%s.json";
private static final String API_URL_V1 = "/p/%s/%s.json";
private static final String API_URL_V2 = "/p2/%s/%s.json";

ComposerMetaAnalyzer() {
this.baseUrl = DEFAULT_BASE_URL;
}

/**
* {@inheritDoc}
*/
public boolean isApplicable(final Component component) {
return component.getPurl() != null && PackageURL.StandardTypes.COMPOSER.equals(component.getPurl().getType());
}

/**
* {@inheritDoc}
*/
public RepositoryType supportedRepositoryType() {
return RepositoryType.COMPOSER;
}

/**
* {@inheritDoc}
*/
public MetaModel analyze(final Component component) {
final MetaModel meta = new MetaModel(component);
if (component.getPurl() == null) {
return meta;
}

final String url = String.format(baseUrl + API_URL, urlEncode(component.getPurl().getNamespace()), urlEncode(component.getPurl().getName()));
try (final CloseableHttpResponse response = processHttpRequest(url)) {
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
handleUnexpectedHttpResponse(LOGGER, url, response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase(), component);
return meta;
}
if (response.getEntity().getContent() == null) {
return meta;
}
String jsonString = EntityUtils.toString(response.getEntity());
if (jsonString.equalsIgnoreCase("")) {
return meta;
}
if (jsonString.equalsIgnoreCase("{}")) {
final String urlV2 = String.format(baseUrl + API_URL_V2, urlEncode(component.getPurl().getNamespace()), urlEncode(component.getPurl().getName()));
final String urlV1 = String.format(baseUrl + API_URL_V1, urlEncode(component.getPurl().getNamespace()), urlEncode(component.getPurl().getName()));

try {
if (processRepository(urlV2, meta)) {
return meta;
}
JSONObject jsonObject = new JSONObject(jsonString);
final String expectedResponsePackage = component.getPurl().getNamespace() + "/" + component.getPurl().getName();
final JSONObject responsePackages = jsonObject
.getJSONObject("packages");
if (!responsePackages.has(expectedResponsePackage)) {
// the package no longer exists - like this one: https://repo.packagist.org/p/magento/adobe-ims.json
if (processRepository(urlV1, meta)) {
return meta;
}
final JSONObject composerPackage = responsePackages.getJSONObject(expectedResponsePackage);

final ComparableVersion latestVersion = new ComparableVersion(stripLeadingV(component.getPurl().getVersion()));
final DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
LOGGER.warn("Failed to retrieve package metadata from both Composer V1 and V2 endpoints.");
} catch (IOException ex) {
handleRequestException(LOGGER, ex);
} catch (Exception ex) {
LOGGER.error("Unexpected error during analysis", ex);
throw new MetaAnalyzerException(ex);
}
return meta;
}

composerPackage.names().forEach(key_ -> {
String key = (String) key_;
if (key.startsWith("dev-") || key.endsWith("-dev")) {
// dev versions are excluded, since they are not pinned but a VCS-branch.
return;
private boolean processRepository(String url, MetaModel meta) throws IOException {
try (final CloseableHttpResponse response = processHttpRequest(url)) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK && response.getEntity() != null) {
String jsonString = EntityUtils.toString(response.getEntity());
if (jsonString.isEmpty() || jsonString.equalsIgnoreCase("{}")) {
LOGGER.info("Empty or default response received from: " + url);
return false;
}

final String version_normalized = composerPackage.getJSONObject(key).getString("version_normalized");
ComparableVersion currentComparableVersion = new ComparableVersion(version_normalized);
if (currentComparableVersion.compareTo(latestVersion) < 0) {
// smaller version can be skipped
return;
JSONObject jsonObject = new JSONObject(jsonString);
if (jsonObject.has("packages")) {
parseComposerData(jsonObject.getJSONObject("packages"), meta);
return true;
} else {
LOGGER.warn("Unexpected JSON structure from: " + url);
}
} else {
LOGGER.warn("HTTP response status not OK for URL: " + url + " Status: " + response.getStatusLine());
}
} catch (JSONException e) {
LOGGER.error("Invalid JSON response from: " + url, e);
}
return false;
}

final String version = composerPackage.getJSONObject(key).getString("version");
latestVersion.parseVersion(stripLeadingV(version_normalized));
meta.setLatestVersion(version);

final String published = composerPackage.getJSONObject(key).getString("time");
try {
meta.setPublishedTimestamp(dateFormat.parse(published));
} catch (ParseException e) {
LOGGER.warn("An error occurred while parsing upload time", e);
private void parseComposerData(JSONObject packages, MetaModel meta) {
try {
for (String packageName : packages.keySet()) {
Object packageData = packages.get(packageName);

if (packageData instanceof JSONObject) {
JSONObject packageObject = (JSONObject) packageData;

if (packageObject.has("versions")) {
parseVersions(packageObject.getJSONObject("versions"), meta);
} else {
parseVersions(packageObject, meta);
}
} else {
LOGGER.warn("Skipping malformed package data for package: " + packageName);
}
});
} catch (IOException ex) {
handleRequestException(LOGGER, ex);
} catch (Exception ex) {
throw new MetaAnalyzerException(ex);
}
} catch (JSONException e) {
LOGGER.error("Failed to parse JSON data", e);
}
}

private void parseVersions(JSONObject packageVersions, MetaModel meta) {
if (packageVersions == null || packageVersions.isEmpty()) {
LOGGER.warn("Package versions JSON is null or empty. Skipping processing.");
return;
}

for (String versionKey : packageVersions.keySet()) {
try {
JSONObject versionData = packageVersions.optJSONObject(versionKey);
if (versionData == null) {
LOGGER.warn("Version entry is null or malformed for key: " + versionKey);
continue;
}
parseVersionData(versionData, meta);
} catch (JSONException e) {
LOGGER.error("Skipping malformed version entry for key: " + versionKey + ". Error: " + e.getMessage(), e);
}
}
}

return meta;
private void parseVersionData(JSONObject versionData, MetaModel meta) {
if (versionData == null || versionData.isEmpty()) {
LOGGER.warn("Version data JSON is null or empty. Skipping.");
return;
}

try {
if (!versionData.has("version") || !versionData.has("version_normalized")) {
LOGGER.warn("Skipping version data. Missing 'version' or 'version_normalized' field.");
return;
}

String versionNormalized = versionData.optString("version_normalized", "");
if (versionNormalized.isEmpty() || !versionNormalized.matches("^[0-9]+(\\.[0-9]+)*$")) {
LOGGER.warn("Skipping version data. Invalid version format: " + versionNormalized);
return;
}

ComparableVersion currentComparableVersion = new ComparableVersion(stripLeadingV(versionNormalized));
if (meta.getLatestVersion() == null || currentComparableVersion.compareTo(new ComparableVersion(stripLeadingV(meta.getLatestVersion()))) > 0) {
meta.setLatestVersion(versionData.optString("version", "unknown"));
LOGGER.debug("Setting latest version to: " + versionData.optString("version", "unknown"));

if (versionData.has("time")) {
String timeString = versionData.optString("time", "");
if (!timeString.isEmpty()) {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX");
try {
meta.setPublishedTimestamp(dateFormat.parse(timeString));
}
catch (ParseException e) {
LOGGER.warn("Failed to parse time for version: " + versionData.optString("version", "unknown") + ". Error: " + e.getMessage(), e);
}
}
else {
LOGGER.warn("Version data contains an empty 'time' field.");
}
}
}
}
catch (JSONException e) {
LOGGER.error("Skipping invalid version data due to JSON error: " + e.getMessage(), e);
}
}

private static String stripLeadingV(String s) {
return s.startsWith("v")
? s.substring(1)
: s;
if (s == null || s.isEmpty()) return s;
return s.startsWith("v") ? s.substring(1) : s;
}
}
1 change: 0 additions & 1 deletion ...est/resources/unit/tasks/repositories/https---repo.packagist.org-p-magento-adobe-ims.json
View file

This file was deleted.

Loading

0 comments on commit 7f9f245

Please sign in to comment.