From 23cb76f837df17db7731a91b7d37c91ca4e1cf3c Mon Sep 17 00:00:00 2001 From: nscuro Date: Sat, 13 Jan 2024 00:14:34 +0100 Subject: [PATCH] Handle L2 cache eviction Signed-off-by: nscuro --- .../tasks/BomUploadProcessingTaskV2.java | 17 +++++- .../dependencytrack/util/PersistenceUtil.java | 59 +++++++++++++++++++ .../util/PersistenceUtilTest.java | 27 +++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTaskV2.java b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTaskV2.java index f65a761bef..70fb6a49b9 100644 --- a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTaskV2.java +++ b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTaskV2.java @@ -101,6 +101,7 @@ import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.flatten; import static org.dependencytrack.util.PersistenceUtil.applyIfChanged; import static org.dependencytrack.util.PersistenceUtil.assertPersistent; +import static org.dependencytrack.util.PersistenceUtil.evictFromL2Cache; /** * @since 4.11.0 @@ -226,7 +227,9 @@ private void processBom(final Context ctx, final org.cyclonedx.model.Bom cdxBom) dispatchBomConsumedNotification(ctx); - final var processedComponents = new ArrayList(); + final var processedProject = new Project[1]; + final var processedComponents = new ArrayList(components.size()); + final var processedServices = new ArrayList(services.size()); try (final var qm = new QueryManager().withL2CacheDisabled()) { // Disable reachability checks on commit. // See https://www.datanucleus.org/products/accessplatform_4_1/jdo/performance_tuning.html @@ -288,14 +291,24 @@ private void processBom(final Context ctx, final org.cyclonedx.model.Bom cdxBom) processComponents(qm, persistentProject, finalComponents, identitiesByBomRef, bomRefsByIdentity); LOGGER.info("Processing %d services".formatted(finalServices.size())); - processServices(qm, persistentProject, finalServices, identitiesByBomRef, bomRefsByIdentity); + final Map persistentServicesByIdentity = + processServices(qm, persistentProject, finalServices, identitiesByBomRef, bomRefsByIdentity); LOGGER.info("Processing %d dependency graph entries".formatted(numDependencyGraphEntries)); processDependencyGraph(qm, persistentProject, dependencyGraph, persistentComponentsByIdentity, identitiesByBomRef); recordBomImport(ctx, qm, persistentProject); + + processedProject[0] = persistentProject; processedComponents.addAll(persistentComponentsByIdentity.values()); + processedServices.addAll(persistentServicesByIdentity.values()); }); + + // Ensure other areas of the application that still use L2 caching do + // not operate on stale data. + evictFromL2Cache(qm, processedProject[0]); + evictFromL2Cache(qm, processedComponents); + evictFromL2Cache(qm, processedServices); } eventsToDispatch.add(createVulnAnalysisEvent(ctx, processedComponents)); diff --git a/src/main/java/org/dependencytrack/util/PersistenceUtil.java b/src/main/java/org/dependencytrack/util/PersistenceUtil.java index e039882a28..c924c58533 100644 --- a/src/main/java/org/dependencytrack/util/PersistenceUtil.java +++ b/src/main/java/org/dependencytrack/util/PersistenceUtil.java @@ -19,9 +19,13 @@ package org.dependencytrack.util; import org.apache.commons.collections4.CollectionUtils; +import org.datanucleus.enhancement.Persistable; +import org.dependencytrack.persistence.QueryManager; import javax.jdo.JDOHelper; import javax.jdo.ObjectState; +import javax.jdo.PersistenceManager; +import javax.jdo.PersistenceManagerFactory; import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -201,4 +205,59 @@ private static boolean isPersistent(final Object object) { || objectState == HOLLOW_PERSISTENT_NONTRANSACTIONAL; } + /** + * Evict a given object from the JDO L2 cache. + * + * @param qm The {@link QueryManager} to use + * @param object The object to evict from the cache + * @since 4.11.0 + */ + public static void evictFromL2Cache(final QueryManager qm, final Object object) { + final PersistenceManagerFactory pmf = qm.getPersistenceManager().getPersistenceManagerFactory(); + pmf.getDataStoreCache().evict(getDataNucleusJdoObjectId(object)); + } + + /** + * Evict a given {@link Collection} of objects from the JDO L2 cache. + * + * @param qm The {@link QueryManager} to use + * @param objects The objects to evict from the cache + * @since 4.11.0 + */ + public static void evictFromL2Cache(final QueryManager qm, final Collection objects) { + final PersistenceManagerFactory pmf = qm.getPersistenceManager().getPersistenceManagerFactory(); + pmf.getDataStoreCache().evictAll(getDataNucleusJdoObjectIds(objects)); + } + + private static Collection getDataNucleusJdoObjectIds(final Collection objects) { + return objects.stream().map(PersistenceUtil::getDataNucleusJdoObjectId).toList(); + } + + /** + * {@link JDOHelper#getObjectId(Object)} and {@link PersistenceManager#getObjectId(Object)} + * return instances of {@link javax.jdo.identity.LongIdentity}, but the DataNucleus L2 cache is maintained + * with DataNucleus-specific {@link org.datanucleus.identity.LongId}s instead. + *

+ * Calling {@link javax.jdo.datastore.DataStoreCache#evict(Object)} with {@link javax.jdo.identity.LongIdentity} + * is pretty much a no-op. The mismatch is undetectable because {@code evict} doesn't throw when a wrong identity + * type is passed either. + *

+ * (╯°□°)╯︵ ┻━┻ + * + * @param object The object to get the JDO object ID for + * @return A JDO object ID + */ + private static Object getDataNucleusJdoObjectId(final Object object) { + if (!(object instanceof final Persistable persistable)) { + throw new IllegalArgumentException("Can't get JDO object ID from non-Persistable objects"); + } + + final Object objectId = persistable.dnGetObjectId(); + if (objectId == null) { + throw new IllegalStateException("Object does not have a JDO object ID"); + } + + return objectId; + } + } diff --git a/src/test/java/org/dependencytrack/util/PersistenceUtilTest.java b/src/test/java/org/dependencytrack/util/PersistenceUtilTest.java index bd57843e65..09d1cb7567 100644 --- a/src/test/java/org/dependencytrack/util/PersistenceUtilTest.java +++ b/src/test/java/org/dependencytrack/util/PersistenceUtilTest.java @@ -18,6 +18,8 @@ */ package org.dependencytrack.util; +import org.datanucleus.api.jdo.JDOPersistenceManagerFactory; +import org.datanucleus.cache.Level2Cache; import org.dependencytrack.PersistenceCapableTest; import org.dependencytrack.model.Project; import org.dependencytrack.util.PersistenceUtil.Diff; @@ -25,6 +27,7 @@ import org.junit.Before; import org.junit.Test; +import javax.jdo.JDOHelper; import javax.jdo.PersistenceManager; import javax.jdo.Transaction; import java.util.Map; @@ -34,6 +37,7 @@ import static org.assertj.core.api.Assertions.assertThatNoException; import static org.dependencytrack.util.PersistenceUtil.assertNonPersistent; import static org.dependencytrack.util.PersistenceUtil.assertPersistent; +import static org.dependencytrack.util.PersistenceUtil.evictFromL2Cache; public class PersistenceUtilTest extends PersistenceCapableTest { @@ -176,4 +180,27 @@ public void testDifferWithoutChanges() { assertThat(differ.getDiffs()).isEmpty(); } + @Test + public void testEvictFromL2Cache() { + final var project = new Project(); + project.setName("acme-app"); + qm.persist(project); + + final PersistenceManager pm = qm.getPersistenceManager(); + final var pmf = (JDOPersistenceManagerFactory) pm.getPersistenceManagerFactory(); + final Level2Cache l2Cache = pmf.getNucleusContext().getLevel2Cache(); + assertThat(l2Cache.getSize()).isEqualTo(1); + + // Try to evict using ID obtained from JDOHelper... + pmf.getDataStoreCache().evict(JDOHelper.getObjectId(project)); + assertThat(l2Cache.getSize()).isEqualTo(1); + + // Try to evict using ID obtained from PersistenceManager... + pmf.getDataStoreCache().evict(qm.getPersistenceManager().getObjectId(project)); + assertThat(l2Cache.getSize()).isEqualTo(1); + + evictFromL2Cache(qm, project); + assertThat(l2Cache.getSize()).isEqualTo(0); + } + } \ No newline at end of file