Skip to content

Commit

Permalink
Fix unique constraint violation during NVD mirroring
Browse files Browse the repository at this point in the history 
The fix is achieved by using the same logic for persisting `Vulnerability` and `VulnerableSoftware` records that `NistApiMirrorTask` was already using. It handles duplicate records.

This should also yield a performance boost (did not benchmark because that wasn't the focus of this change), since the transaction commit frequency is reduced compared to the previous logic.

Fixes DependencyTrack#3663

Signed-off-by: nscuro <[email protected]>
  • Loading branch information
@nscuro @MM-msr
nscuro authored and MM-msr committed Jun 18, 2024
1 parent bb7ab8c commit c13dbdd
Show file tree
Hide file tree
Showing 6 changed files with 428 additions and 360 deletions.
2 changes: 2 additions & 0 deletions docs/_posts/2024-xx-xx-v4.11.0.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ It is also available through [Artifact Hub](https://artifacthub.io/packages/helm
* Fix severity not being set for vulnerabilities from VulnDB - [apiserver/#3595]
* Fix `JDOFatalUserException` for long reference URLs from OSS Index - [apiserver/#3650]
* Fix unhandled `ClientErrorException`s causing a `HTTP 500` response - [apiserver/#3659]
* Fix unique constraint violation during NVD mirroring via feed files - [apiserver/#3664]
* Fix `VUE_APP_SERVER_URL` being ignored - [frontend/#682]
* Fix visibility of "Vulnerabilities" and "Policy Violations" columns not being toggle-able individually - [frontend/#686]
* Fix finding search routes - [frontend/#689]
Expand Down Expand Up @@ -252,6 +253,7 @@ Special thanks to everyone who contributed code to implement enhancements and fi
[apiserver/#3657]: https://github.com/DependencyTrack/dependency-track/pull/3657
[apiserver/#3659]: https://github.com/DependencyTrack/dependency-track/pull/3659
[apiserver/#3661]: https://github.com/DependencyTrack/dependency-track/pull/3661
[apiserver/#3664]: https://github.com/DependencyTrack/dependency-track/pull/3664

[frontend/#682]: https://github.com/DependencyTrack/frontend/pull/682
[frontend/#683]: https://github.com/DependencyTrack/frontend/pull/683
Expand Down
225 changes: 107 additions & 118 deletions src/main/java/org/dependencytrack/parser/nvd/NvdParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,11 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.commons.lang3.StringUtils;
import org.datanucleus.PropertyNames;
import org.dependencytrack.event.IndexEvent;
import org.dependencytrack.model.Cwe;
import org.dependencytrack.model.Vulnerability;
import org.dependencytrack.model.VulnerableSoftware;
import org.dependencytrack.parser.common.resolver.CweResolver;
import org.dependencytrack.persistence.QueryManager;
import org.dependencytrack.util.VulnerabilityUtil;
import us.springett.cvss.Cvss;
import us.springett.parsers.cpe.exceptions.CpeEncodingException;
Expand All @@ -51,6 +49,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.function.BiConsumer;

/**
* Parser and processor of NVD data feeds.
Expand All @@ -71,6 +70,11 @@ private enum Operator {
// https://github.com/DependencyTrack/dependency-track/pull/2520
// is merged.
private final ObjectMapper objectMapper = new ObjectMapper();
private final BiConsumer<Vulnerability, List<VulnerableSoftware>> vulnerabilityConsumer;

public NvdParser(final BiConsumer<Vulnerability, List<VulnerableSoftware>> vulnerabilityConsumer) {
this.vulnerabilityConsumer = vulnerabilityConsumer;
}

public void parse(final File file) {
if (!file.getName().endsWith(".json")) {
Expand Down Expand Up @@ -111,127 +115,115 @@ public void parse(final File file) {
}

private void parseCveItem(final ObjectNode cveItem) {
try (QueryManager qm = new QueryManager().withL2CacheDisabled()) {
qm.getPersistenceManager().setProperty(PropertyNames.PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false");

final Vulnerability vulnerability = new Vulnerability();
vulnerability.setSource(Vulnerability.Source.NVD);
final Vulnerability vulnerability = new Vulnerability();
vulnerability.setSource(Vulnerability.Source.NVD);

// CVE ID
final var cve = (ObjectNode) cveItem.get("cve");
final var meta0 = (ObjectNode) cve.get("CVE_data_meta");
vulnerability.setVulnId(meta0.get("ID").asText());
// CVE ID
final var cve = (ObjectNode) cveItem.get("cve");
final var meta0 = (ObjectNode) cve.get("CVE_data_meta");
vulnerability.setVulnId(meta0.get("ID").asText());

// CVE Published and Modified dates
final String publishedDateString = cveItem.get("publishedDate").asText();
final String lastModifiedDateString = cveItem.get("lastModifiedDate").asText();
try {
if (StringUtils.isNotBlank(publishedDateString)) {
vulnerability.setPublished(Date.from(OffsetDateTime.parse(publishedDateString).toInstant()));
}
if (StringUtils.isNotBlank(lastModifiedDateString)) {
vulnerability.setUpdated(Date.from(OffsetDateTime.parse(lastModifiedDateString).toInstant()));
}
} catch (DateTimeParseException | NullPointerException | IllegalArgumentException e) {
LOGGER.error("Unable to parse dates from NVD data feed", e);
// CVE Published and Modified dates
final String publishedDateString = cveItem.get("publishedDate").asText();
final String lastModifiedDateString = cveItem.get("lastModifiedDate").asText();
try {
if (StringUtils.isNotBlank(publishedDateString)) {
vulnerability.setPublished(Date.from(OffsetDateTime.parse(publishedDateString).toInstant()));
}
if (StringUtils.isNotBlank(lastModifiedDateString)) {
vulnerability.setUpdated(Date.from(OffsetDateTime.parse(lastModifiedDateString).toInstant()));
}
} catch (DateTimeParseException | NullPointerException | IllegalArgumentException e) {
LOGGER.error("Unable to parse dates from NVD data feed", e);
}

// CVE Description
final var descO = (ObjectNode) cve.get("description");
final var desc1 = (ArrayNode) descO.get("description_data");
final StringBuilder descriptionBuilder = new StringBuilder();
for (int j = 0; j < desc1.size(); j++) {
final var desc2 = (ObjectNode) desc1.get(j);
if ("en".equals(desc2.get("lang").asText())) {
descriptionBuilder.append(desc2.get("value").asText());
if (j < desc1.size() - 1) {
descriptionBuilder.append("\n\n");
}
// CVE Description
final var descO = (ObjectNode) cve.get("description");
final var desc1 = (ArrayNode) descO.get("description_data");
final StringBuilder descriptionBuilder = new StringBuilder();
for (int j = 0; j < desc1.size(); j++) {
final var desc2 = (ObjectNode) desc1.get(j);
if ("en".equals(desc2.get("lang").asText())) {
descriptionBuilder.append(desc2.get("value").asText());
if (j < desc1.size() - 1) {
descriptionBuilder.append("\n\n");
}
}
vulnerability.setDescription(descriptionBuilder.toString());
}
vulnerability.setDescription(descriptionBuilder.toString());

// CVE Impact
parseCveImpact(cveItem, vulnerability);
// CVE Impact
parseCveImpact(cveItem, vulnerability);

// CWE
final var prob0 = (ObjectNode) cve.get("problemtype");
final var prob1 = (ArrayNode) prob0.get("problemtype_data");
for (int j = 0; j < prob1.size(); j++) {
final var prob2 = (ObjectNode) prob1.get(j);
final var prob3 = (ArrayNode) prob2.get("description");
for (int k = 0; k < prob3.size(); k++) {
final var prob4 = (ObjectNode) prob3.get(k);
if ("en".equals(prob4.get("lang").asText())) {
final String cweString = prob4.get("value").asText();
if (cweString != null && cweString.startsWith("CWE-")) {
final Cwe cwe = CweResolver.getInstance().lookup(cweString);
if (cwe != null) {
vulnerability.addCwe(cwe);
} else {
LOGGER.warn("CWE " + cweString + " not found in Dependency-Track database. This could signify an issue with the NVD or with Dependency-Track not having advanced knowledge of this specific CWE identifier.");
}
// CWE
final var prob0 = (ObjectNode) cve.get("problemtype");
final var prob1 = (ArrayNode) prob0.get("problemtype_data");
for (int j = 0; j < prob1.size(); j++) {
final var prob2 = (ObjectNode) prob1.get(j);
final var prob3 = (ArrayNode) prob2.get("description");
for (int k = 0; k < prob3.size(); k++) {
final var prob4 = (ObjectNode) prob3.get(k);
if ("en".equals(prob4.get("lang").asText())) {
final String cweString = prob4.get("value").asText();
if (cweString != null && cweString.startsWith("CWE-")) {
final Cwe cwe = CweResolver.getInstance().lookup(cweString);
if (cwe != null) {
vulnerability.addCwe(cwe);
} else {
LOGGER.warn("CWE " + cweString + " not found in Dependency-Track database. This could signify an issue with the NVD or with Dependency-Track not having advanced knowledge of this specific CWE identifier.");
}
}
}
}
}

// References
final var ref0 = (ObjectNode) cve.get("references");
final var ref1 = (ArrayNode) ref0.get("reference_data");
final StringBuilder sb = new StringBuilder();
for (int l = 0; l < ref1.size(); l++) {
final var ref2 = (ObjectNode) ref1.get(l);
final Iterator<String> fieldNameIter = ref2.fieldNames();
while (fieldNameIter.hasNext()) {
final String s = fieldNameIter.next();
if ("url".equals(s)) {
// Convert reference to Markdown format
final String url = ref2.get("url").asText();
sb.append("* [").append(url).append("](").append(url).append(")\n");
}
// References
final var ref0 = (ObjectNode) cve.get("references");
final var ref1 = (ArrayNode) ref0.get("reference_data");
final StringBuilder sb = new StringBuilder();
for (int l = 0; l < ref1.size(); l++) {
final var ref2 = (ObjectNode) ref1.get(l);
final Iterator<String> fieldNameIter = ref2.fieldNames();
while (fieldNameIter.hasNext()) {
final String s = fieldNameIter.next();
if ("url".equals(s)) {
// Convert reference to Markdown format
final String url = ref2.get("url").asText();
sb.append("* [").append(url).append("](").append(url).append(")\n");
}
}
final String references = sb.toString();
if (references.length() > 0) {
vulnerability.setReferences(references.substring(0, references.lastIndexOf("\n")));
}

// Update the vulnerability
LOGGER.debug("Synchronizing: " + vulnerability.getVulnId());
final Vulnerability synchronizeVulnerability = qm.synchronizeVulnerability(vulnerability, false);
final List<VulnerableSoftware> vsListOld = qm.detach(qm.getVulnerableSoftwareByVulnId(synchronizeVulnerability.getSource(), synchronizeVulnerability.getVulnId()));
}
final String references = sb.toString();
if (!references.isEmpty()) {
vulnerability.setReferences(references.substring(0, references.lastIndexOf("\n")));
}

// CPE
List<VulnerableSoftware> vsList = new ArrayList<>();
final var configurations = (ObjectNode) cveItem.get("configurations");
final var nodes = (ArrayNode) configurations.get("nodes");
for (int j = 0; j < nodes.size(); j++) {
final var node = (ObjectNode) nodes.get(j);
final List<VulnerableSoftware> vulnerableSoftwareInNode = new ArrayList<>();
final Operator nodeOperator = Operator.valueOf(node.get("operator").asText(Operator.NONE.name()));
if (node.has("children")) {
// https://github.com/DependencyTrack/dependency-track/issues/1033
final var children = (ArrayNode) node.get("children");
if (children.size() > 0) {
for (int l = 0; l < children.size(); l++) {
final var child = (ObjectNode) children.get(l);
vulnerableSoftwareInNode.addAll(parseCpes(qm, child));
}
} else {
vulnerableSoftwareInNode.addAll(parseCpes(qm, node));
// CPE
List<VulnerableSoftware> vsList = new ArrayList<>();
final var configurations = (ObjectNode) cveItem.get("configurations");
final var nodes = (ArrayNode) configurations.get("nodes");
for (int j = 0; j < nodes.size(); j++) {
final var node = (ObjectNode) nodes.get(j);
final List<VulnerableSoftware> vulnerableSoftwareInNode = new ArrayList<>();
final Operator nodeOperator = Operator.valueOf(node.get("operator").asText(Operator.NONE.name()));
if (node.has("children")) {
// https://github.com/DependencyTrack/dependency-track/issues/1033
final var children = (ArrayNode) node.get("children");
if (!children.isEmpty()) {
for (int l = 0; l < children.size(); l++) {
final var child = (ObjectNode) children.get(l);
vulnerableSoftwareInNode.addAll(parseCpes(child));
}
} else {
vulnerableSoftwareInNode.addAll(parseCpes(qm, node));
vulnerableSoftwareInNode.addAll(parseCpes(node));
}
vsList.addAll(reconcile(vulnerableSoftwareInNode, nodeOperator));
} else {
vulnerableSoftwareInNode.addAll(parseCpes(node));
}
qm.persist(vsList);
qm.updateAffectedVersionAttributions(synchronizeVulnerability, vsList, Vulnerability.Source.NVD);
vsList = qm.reconcileVulnerableSoftware(synchronizeVulnerability, vsListOld, vsList, Vulnerability.Source.NVD);
synchronizeVulnerability.setVulnerableSoftware(vsList);
qm.persist(synchronizeVulnerability);
vsList.addAll(reconcile(vulnerableSoftwareInNode, nodeOperator));
}

vulnerabilityConsumer.accept(vulnerability, vsList);
}

/**
Expand Down Expand Up @@ -302,14 +294,14 @@ private void parseCveImpact(final ObjectNode cveItem, final Vulnerability vuln)
));
}

private List<VulnerableSoftware> parseCpes(final QueryManager qm, final ObjectNode node) {
private List<VulnerableSoftware> parseCpes(final ObjectNode node) {
final List<VulnerableSoftware> vsList = new ArrayList<>();
if (node.has("cpe_match")) {
final var cpeMatches = (ArrayNode) node.get("cpe_match");
for (int k = 0; k < cpeMatches.size(); k++) {
final var cpeMatch = (ObjectNode) cpeMatches.get(k);
if (cpeMatch.get("vulnerable").asBoolean(true)) { // only parse the CPEs marked as vulnerable
final VulnerableSoftware vs = generateVulnerableSoftware(qm, cpeMatch);
final VulnerableSoftware vs = generateVulnerableSoftware(cpeMatch);
if (vs != null) {
vsList.add(vs);
}
Expand All @@ -319,29 +311,26 @@ private List<VulnerableSoftware> parseCpes(final QueryManager qm, final ObjectNo
return vsList;
}

private VulnerableSoftware generateVulnerableSoftware(final QueryManager qm, final ObjectNode cpeMatch) {
private VulnerableSoftware generateVulnerableSoftware(final ObjectNode cpeMatch) {
final String cpe23Uri = cpeMatch.get("cpe23Uri").asText();
final String versionEndExcluding = Optional.ofNullable(cpeMatch.get("versionEndExcluding")).map(JsonNode::asText).orElse(null);
final String versionEndIncluding = Optional.ofNullable(cpeMatch.get("versionEndIncluding")).map(JsonNode::asText).orElse(null);
final String versionStartExcluding = Optional.ofNullable(cpeMatch.get("versionStartExcluding")).map(JsonNode::asText).orElse(null);
final String versionStartIncluding = Optional.ofNullable(cpeMatch.get("versionStartIncluding")).map(JsonNode::asText).orElse(null);
VulnerableSoftware vs = qm.getVulnerableSoftwareByCpe23(cpe23Uri, versionEndExcluding,
versionEndIncluding, versionStartExcluding, versionStartIncluding);
if (vs != null) {
return vs;
}

final VulnerableSoftware vs;
try {
vs = ModelConverter.convertCpe23UriToVulnerableSoftware(cpe23Uri);
vs.setVulnerable(cpeMatch.get("vulnerable").asBoolean(true));
vs.setVersionEndExcluding(versionEndExcluding);
vs.setVersionEndIncluding(versionEndIncluding);
vs.setVersionStartExcluding(versionStartExcluding);
vs.setVersionStartIncluding(versionStartIncluding);
//Event.dispatch(new IndexEvent(IndexEvent.Action.CREATE, qm.detach(VulnerableSoftware.class, vs.getId())));
return vs;
} catch (CpeParsingException | CpeEncodingException e) {
LOGGER.warn("An error occurred while parsing: " + cpe23Uri + " - The CPE is invalid and will be discarded.");
return null;
}
return null;

vs.setVulnerable(cpeMatch.get("vulnerable").asBoolean(true));
vs.setVersionEndExcluding(versionEndExcluding);
vs.setVersionEndIncluding(versionEndIncluding);
vs.setVersionStartExcluding(versionStartExcluding);
vs.setVersionStartIncluding(versionStartIncluding);
return vs;
}
}
Loading

0 comments on commit c13dbdd

Please sign in to comment.