Refactor BOM upload processing for better efficiency, correctness, and consistency

docs/_posts/2024-xx-xx-v4.11.0.md

@@ -0,0 +1,82 @@
+---
+title: v4.11.0
+type: major
+---
+
+**Highlights:**
+
+* **Optimized BOM Ingestion**. The logic how uploaded BOMs are processed and ingested into Dependency-Track has been
+overhauled to be more reliable and efficient. Further, BOM processing is now an atomic operation, such that errors
+occurring midway do not cause a partial state to be left behind. De-duplication of components and services is more
+predictable, and log messages emitted during processing contain additional context, making them easier to correlate.
+Because the new implementation can have a big impact on how Dependency-Track behaves regarding BOM uploads,
+it is disabled by default for this release. It may be enabled by setting the environment variable `BOM_PROCESSING_TASK_V2_ENABLED`
+to `true`. Users are highly encouraged to do so.
+
+**Features:**
+
+* Make processing of uploaded BOMs atomic - [apiserver/#3357]
+* Improve performance of BOM processing - [apiserver/#3357]
+* Add more context to logs emitted during BOM processing - [apiserver/#3357]
+  * BOM format, spec version, serial number, and version
+  * Project UUID, name, and version
+
+**Fixes:**
+
+* Fix `StackOverflowError` when processing BOMs with deeply nested component structures - [apiserver/#3357]
+* Fix inconsistent component de-duplication during BOM processing, causing varying components counts in successive uploads - [apiserver/#3357]
+* Fix components erroneously being de-duplicated when only a single attribute of their [component identity] is identical - [apiserver/#3357]
+* Fix components defined in the BOM node `metadata.component.components` not being imported - [apiserver/#3357]
+
+**Upgrade Notes:**
+
+* The default logging configuration ([logback.xml]) was updated to include the [Mapped Diagnostic Context] (MDC)
+  * Users who [customized their logging configuration] are recommended to follow this change
+
+For a complete list of changes, refer to the respective GitHub milestones:
+
+* [API server milestone 4.11.0](https://github.com/DependencyTrack/dependency-track/milestone/25?closed=1)
+* [Frontend milestone 4.11.0](https://github.com/DependencyTrack/frontend/milestone/16?closed=1)
+
+We thank all organizations and individuals who contributed to this release, from logging issues to taking part in discussions on GitHub & Slack to testing of fixes.
+
+Special thanks to everyone who contributed code to implement enhancements and fix defects:  
+[@malice00], [@mehab], [@sahibamittal], [@VithikaS]
+
+###### dependency-track-apiserver.jar
+
+| Algorithm | Checksum |
+|:----------|:---------|
+| SHA-1     |          |
+| SHA-256   |          |
+
+###### dependency-track-bundled.jar
+
+| Algorithm | Checksum |
+|:----------|:---------|
+| SHA-1     |          |
+| SHA-256   |          |
+
+###### frontend-dist.zip
+
+| Algorithm | Checksum |
+|:----------|:---------|
+| SHA-1     |          |
+| SHA-256   |          |
+
+###### Software Bill of Materials (SBOM)
+
+* API Server: [bom.json](https://github.com/DependencyTrack/dependency-track/releases/download/4.11.0/bom.json)
+* Frontend: [bom.json](https://github.com/DependencyTrack/frontend/releases/download/4.11.0/bom.json)
+
+[apiserver/#3357]: https://github.com/DependencyTrack/dependency-track/pull/3357
+
+[@malice00]: https://github.com/malice00
+[@mehab]: https://github.com/mehab
+[@sahibamittal]: https://github.com/sahibamittal
+[@VithikaS]: https://github.com/VithikaS
+
+[Mapped Diagnostic Context]: https://logback.qos.ch/manual/mdc.html
+[component identity]: https://docs.dependencytrack.org/analysis-types/component-identity/
+[customized their logging configuration]: https://docs.dependencytrack.org/getting-started/monitoring/#custom-logging-configuration
+[logback.xml]: https://github.com/DependencyTrack/dependency-track/blob/master/src/main/docker/logback.xml

src/main/docker/logback.xml

@@ -12,13 +12,13 @@
             <maxFileSize>10MB</maxFileSize>
         </triggeringPolicy>
         <encoder>
-            <pattern>%date [%marker] %level [%logger] %msg%n</pattern>
+            <pattern>%date [%marker] %level [%logger] %msg%replace( [%mdc{}]){' \[\]', ''}%n</pattern>
         </encoder>
     </appender>
 
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
         <encoder>
-            <pattern>%date %level [%logger{0}] %msg%n</pattern>
+            <pattern>%date %level [%logger{0}] %msg%replace( [%mdc{}]){' \[\]', ''}%n</pattern>
         </encoder>
     </appender>
 
@@ -34,7 +34,7 @@
         </triggeringPolicy>
         <filter class="org.owasp.security.logging.filter.SecurityMarkerFilter"/>
         <encoder>
-            <pattern>%date [%marker] %level - %msg%n</pattern>
+            <pattern>%date [%marker] %level - %msg%replace( [%mdc{}]){' \[\]', ''}%n</pattern>
         </encoder>
     </appender>
 
@@ -49,7 +49,7 @@
             <maxFileSize>10MB</maxFileSize>
         </triggeringPolicy>
         <encoder>
-            <pattern>%date [%marker] %level [%logger] %msg%n</pattern>
+            <pattern>%date [%marker] %level [%logger] %msg%replace( [%mdc{}]){' \[\]', ''}%n</pattern>
         </encoder>
     </appender>
 

src/main/java/org/dependencytrack/common/ConfigKey.java

@@ -36,7 +36,8 @@ public enum ConfigKey implements Config.Key {
     REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_ENABLED("repo.meta.analyzer.cacheStampedeBlocker.enabled", true),
     REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_LOCK_BUCKETS("repo.meta.analyzer.cacheStampedeBlocker.lock.buckets", 1000),
     REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_MAX_ATTEMPTS("repo.meta.analyzer.cacheStampedeBlocker.max.attempts", 10),
-    SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true);
+    SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true),
+    BOM_PROCESSING_TASK_V2_ENABLED("bom.processing.task.v2.enabled", false);
 
     private final String propertyName;
     private final Object defaultValue;

src/main/java/org/dependencytrack/common/MdcKeys.java

@@ -0,0 +1,40 @@
+/*
+ * This file is part of Dependency-Track.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) Steve Springett. All Rights Reserved.
+ */
+package org.dependencytrack.common;
+
+/**
+ * Common fields for use with SLF4J's {@link org.slf4j.MDC}.
+ *
+ * @since 4.11.0
+ */
+public final class MdcKeys {
+
+    public static final String MDC_BOM_FORMAT = "bomFormat";
+    public static final String MDC_BOM_SERIAL_NUMBER = "bomSerialNumber";
+    public static final String MDC_BOM_SPEC_VERSION = "bomSpecVersion";
+    public static final String MDC_BOM_UPLOAD_TOKEN = "bomUploadToken";
+    public static final String MDC_BOM_VERSION = "bomVersion";
+    public static final String MDC_PROJECT_NAME = "projectName";
+    public static final String MDC_PROJECT_UUID = "projectUuid";
+    public static final String MDC_PROJECT_VERSION = "projectVersion";
+
+    private MdcKeys() {
+    }
+
+}

src/main/java/org/dependencytrack/event/BomUploadEvent.java

@@ -19,9 +19,9 @@
 package org.dependencytrack.event;
 
 import alpine.event.framework.AbstractChainableEvent;
+import org.dependencytrack.model.Project;
 
-import java.io.File;
-import java.util.UUID;
+import static org.dependencytrack.util.PersistenceUtil.assertNonPersistent;
 
 /**
  * Defines an event triggered when a bill-of-material (bom) document is submitted.
@@ -31,31 +31,21 @@
  */
 public class BomUploadEvent extends AbstractChainableEvent {
 
-    private final UUID projectUuid;
-    private File file;
-    private byte[] bom;
+    private final Project project;
+    private final byte[] bom;
 
-    public BomUploadEvent(final UUID projectUuid, final byte[] bom) {
-        this.projectUuid = projectUuid;
-        if (bom != null) {
-            this.bom = bom.clone();
-        }
+    public BomUploadEvent(final Project project, final byte[] bom) {
+        assertNonPersistent(project, "project must not be persistent");
+        this.project = project;
+        this.bom = bom.clone();
     }
 
-    public BomUploadEvent(final UUID projectUuid, final File file) {
-        this.projectUuid = projectUuid;
-        this.file = file;
-    }
-
-    public UUID getProjectUuid() {
-        return projectUuid;
+    public Project getProject() {
+        return project;
     }
 
     public byte[] getBom() {
         return bom == null ? null : bom.clone();
     }
 
-    public File getFile() {
-        return file;
-    }
 }

src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java

@@ -18,13 +18,16 @@
  */
 package org.dependencytrack.event;
 
+import alpine.Config;
 import alpine.common.logging.Logger;
 import alpine.event.LdapSyncEvent;
 import alpine.event.framework.EventService;
 import alpine.event.framework.SingleThreadedEventService;
 import alpine.server.tasks.LdapSyncTask;
 import org.dependencytrack.RequirementsVerifier;
+import org.dependencytrack.common.ConfigKey;
 import org.dependencytrack.tasks.BomUploadProcessingTask;
+import org.dependencytrack.tasks.BomUploadProcessingTaskV2;
 import org.dependencytrack.tasks.CallbackTask;
 import org.dependencytrack.tasks.ClearComponentAnalysisCacheTask;
 import org.dependencytrack.tasks.CloneProjectTask;
@@ -83,7 +86,11 @@ public void contextInitialized(final ServletContextEvent event) {
         if (RequirementsVerifier.failedValidation()) {
             return;
         }
-        EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTask.class);
+        if (Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED)) {
+            EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTaskV2.class);
+        } else {
+            EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTask.class);
+        }
         EVENT_SERVICE.subscribe(VexUploadEvent.class, VexUploadProcessingTask.class);
         EVENT_SERVICE.subscribe(LdapSyncEvent.class, LdapSyncTask.class);
         EVENT_SERVICE.subscribe(InternalAnalysisEvent.class, InternalAnalysisTask.class);
@@ -126,7 +133,11 @@ public void contextDestroyed(final ServletContextEvent event) {
         LOGGER.info("Shutting down asynchronous event subsystem");
         TaskScheduler.getInstance().shutdown();
 
-        EVENT_SERVICE.unsubscribe(BomUploadProcessingTask.class);
+        if (Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED)) {
+            EVENT_SERVICE.unsubscribe(BomUploadProcessingTaskV2.class);
+        } else {
+            EVENT_SERVICE.unsubscribe(BomUploadProcessingTask.class);
+        }
         EVENT_SERVICE.unsubscribe(VexUploadProcessingTask.class);
         EVENT_SERVICE.unsubscribe(LdapSyncTask.class);
         EVENT_SERVICE.unsubscribe(InternalAnalysisTask.class);

src/main/java/org/dependencytrack/model/Component.java

@@ -361,8 +361,10 @@ public enum FetchGroup {
     private UUID uuid;
 
     private transient String bomRef;
+    private transient List<org.cyclonedx.model.License> licenseCandidates;
     private transient DependencyMetrics metrics;
     private transient RepositoryMetaComponent repositoryMeta;
+    private transient boolean isNew;
     private transient int usedBy;
     private transient JsonObject cacheResult;
     private transient Set<String> dependencyGraph;
@@ -765,6 +767,14 @@ public void setRepositoryMeta(RepositoryMetaComponent repositoryMeta) {
         this.repositoryMeta = repositoryMeta;
     }
 
+    public boolean isNew() {
+        return isNew;
+    }
+
+    public void setNew(final boolean aNew) {
+        isNew = aNew;
+    }
+
     public Double getLastInheritedRiskScore() {
         return lastInheritedRiskScore;
     }
@@ -781,6 +791,14 @@ public void setBomRef(String bomRef) {
         this.bomRef = bomRef;
     }
 
+    public List<org.cyclonedx.model.License> getLicenseCandidates() {
+        return licenseCandidates;
+    }
+
+    public void setLicenseCandidates(final List<org.cyclonedx.model.License> licenseCandidates) {
+        this.licenseCandidates = licenseCandidates;
+    }
+
     public int getUsedBy() {
         return usedBy;
     }

src/main/java/org/dependencytrack/model/ComponentIdentity.java

@@ -23,6 +23,7 @@
 import org.dependencytrack.util.PurlUtil;
 import org.json.JSONObject;
 
+import java.util.Objects;
 import java.util.UUID;
 
 /**
@@ -137,6 +138,27 @@ public UUID getUuid() {
         return uuid;
     }
 
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final ComponentIdentity that = (ComponentIdentity) o;
+        return objectType == that.objectType
+                && Objects.equals(purl, that.purl)
+                && Objects.equals(purlCoordinates, that.purlCoordinates)
+                && Objects.equals(cpe, that.cpe)
+                && Objects.equals(swidTagId, that.swidTagId)
+                && Objects.equals(group, that.group)
+                && Objects.equals(name, that.name)
+                && Objects.equals(version, that.version)
+                && Objects.equals(uuid, that.uuid);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(objectType, purl, purlCoordinates, cpe, swidTagId, group, name, version, uuid);
+    }
+
     public JSONObject toJSON() {
         final JSONObject jsonObject = new JSONObject();
         jsonObject.put("uuid", this.getUuid());

src/main/java/org/dependencytrack/model/License.java

@@ -92,6 +92,7 @@ public enum FetchGroup {
     }
 
     private static final long serialVersionUID = -1707920279688859358L;
+    public static final License UNRESOLVED = new License();
 
     @PrimaryKey
     @Persistent(valueStrategy = IdGeneratorStrategy.NATIVE)

src/main/java/org/dependencytrack/model/Project.java

@@ -281,6 +281,7 @@ public enum FetchGroup {
     @ApiModelProperty(accessMode = ApiModelProperty.AccessMode.READ_ONLY)
     private ProjectMetadata metadata;
 
+    private transient String bomRef;
     private transient ProjectMetrics metrics;
     private transient List<ProjectVersion> versions;
     private transient List<Component> dependencyGraph;
@@ -490,6 +491,14 @@ public void setActive(Boolean active) {
         this.active = active;
     }
 
+    public String getBomRef() {
+        return bomRef;
+    }
+
+    public void setBomRef(String bomRef) {
+        this.bomRef = bomRef;
+    }
+
     public ProjectMetrics getMetrics() {
         return metrics;
     }