From ffa9c168896d1e8323057b646744eaa63f708ebf Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Fri, 8 Nov 2024 15:22:49 -0500 Subject: [PATCH 01/19] Add endpoint and logic to clear cdm_cache and achilles_cache --- pom.xml | 1 + src/main/java/org/ohdsi/webapi/Constants.java | 1 + .../repository/AchillesCacheRepository.java | 4 +- .../service/AchillesCacheService.java | 5 + .../cdmresults/AchillesClearCacheTasklet.java | 24 +++++ .../CDMResultsClearCacheTasklet.java | 24 +++++ .../repository/CDMCacheRepository.java | 3 + .../cdmresults/service/CDMCacheService.java | 4 + .../webapi/service/CDMResultsService.java | 93 ++++++++++++++++++- 9 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java create mode 100644 src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java diff --git a/pom.xml b/pom.xml index f82a63ccb3..e16116c53f 100644 --- a/pom.xml +++ b/pom.xml @@ -272,6 +272,7 @@ 3 true + true 47 diff --git a/src/main/java/org/ohdsi/webapi/Constants.java b/src/main/java/org/ohdsi/webapi/Constants.java index 2069ed108d..d3d7c38e83 100644 --- a/src/main/java/org/ohdsi/webapi/Constants.java +++ b/src/main/java/org/ohdsi/webapi/Constants.java @@ -12,6 +12,7 @@ public interface Constants { String GENERATE_PREDICTION_ANALYSIS = "generatePredictionAnalysis"; String GENERATE_ESTIMATION_ANALYSIS = "generateEstimationAnalysis"; String WARM_CACHE = "warmCache"; + String CLEAR_CACHE = "clearCache"; String USERS_IMPORT = "usersImport"; String JOB_IS_ALREADY_SCHEDULED = "Job for provider %s is already scheduled"; diff --git a/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java b/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java index 2479cc7da7..9571ddf0b5 100644 --- a/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java +++ b/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java @@ -2,7 +2,6 @@ import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity; import org.ohdsi.webapi.source.Source; -import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; @@ -16,4 +15,7 @@ public interface AchillesCacheRepository extends CrudRepository findBySourceAndNames(@Param("source") Source source, @Param("names") List names); + + @Query("delete from AchillesCacheEntity where source = :source") + void deleteBySource(@Param("source") Source source); } diff --git a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java index c55180929b..a65d033e7a 100644 --- a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java +++ b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java @@ -85,6 +85,11 @@ public void saveDrilldownCacheMap(Source source, String domain, Map nodes) { List cacheEntities = getEntities(source, nodes); cacheRepository.save(cacheEntities); diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java b/src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java new file mode 100644 index 0000000000..364f97242f --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java @@ -0,0 +1,24 @@ +package org.ohdsi.webapi.cdmresults; + +import org.ohdsi.webapi.achilles.service.AchillesCacheService; +import org.ohdsi.webapi.source.Source; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; + +public class AchillesClearCacheTasklet implements Tasklet { + private final Source source; + private final AchillesCacheService cacheService; + + public AchillesClearCacheTasklet(Source source, AchillesCacheService cacheService) { + this.source = source; + this.cacheService = cacheService; + } + + @Override + public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) { + cacheService.clearCache(this.source); + return RepeatStatus.FINISHED; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java b/src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java new file mode 100644 index 0000000000..2d2d9b2fa8 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java @@ -0,0 +1,24 @@ +package org.ohdsi.webapi.cdmresults; + +import org.ohdsi.webapi.cdmresults.service.CDMCacheService; +import org.ohdsi.webapi.source.Source; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; + +public class CDMResultsClearCacheTasklet implements Tasklet { + private final Source source; + private final CDMCacheService cdmCacheService; + + public CDMResultsClearCacheTasklet(Source source, CDMCacheService cdmCacheService) { + this.source = source; + this.cdmCacheService = cdmCacheService; + } + + @Override + public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) { + cdmCacheService.clearCache(this.source); + return RepeatStatus.FINISHED; + } +} \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java b/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java index f329ddfcd2..e900aa218c 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java @@ -12,4 +12,7 @@ public interface CDMCacheRepository extends CrudRepository { @Query("select c from CDMCacheEntity c where c.sourceId = :sourceId and c.conceptId in :conceptIds") List findBySourceAndConceptIds(@Param("sourceId") int sourceId, @Param("conceptIds") List conceptIds); + + @Query("delete from CDMCacheEntity c where c.sourceId = :sourceId") + void deleteBySource(@Param("sourceId") int sourceId); } diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java index e3870609d4..106c4775de 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java @@ -73,6 +73,10 @@ public void warm(Source source) { } } + public void clearCache(Source source) { + cdmCacheRepository.deleteBySource(source.getSourceId()); + } + public List findAndCache(Source source, List conceptIds) { if (CollectionUtils.isEmpty(conceptIds)) { return Collections.emptyList(); diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index 7b83d9d3bd..024adc4c22 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -8,7 +8,9 @@ import org.ohdsi.webapi.achilles.aspect.AchillesCache; import org.ohdsi.webapi.achilles.service.AchillesCacheService; import org.ohdsi.webapi.cdmresults.AchillesCacheTasklet; +import org.ohdsi.webapi.cdmresults.AchillesClearCacheTasklet; import org.ohdsi.webapi.cdmresults.CDMResultsCacheTasklet; +import org.ohdsi.webapi.cdmresults.CDMResultsClearCacheTasklet; import org.ohdsi.webapi.cdmresults.DescendantRecordAndPersonCount; import org.ohdsi.webapi.cdmresults.DescendantRecordCount; import org.ohdsi.webapi.cdmresults.domain.CDMCacheEntity; @@ -116,6 +118,9 @@ public class CDMResultsService extends AbstractDaoService implements Initializin @Value("${cdm.cache.achilles.warming.enable}") private boolean cdmAchillesCacheWarmingEnable; + @Value("${cdm.result.clear.cache.enable}") + private boolean cdmResultClearCacheEnable; + @Value("${cache.achilles.usePersonCount:false}") private boolean usePersonCount; @@ -283,6 +288,47 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so return new JobExecutionResource(); } + /** + * Refresh the results cache for a selected source + * + * @summary Refresh results cache + * @param sourceKey The source key + * @return The job execution resource + */ + @GET + @Path("clearCaches") + @Produces(MediaType.APPLICATION_JSON) + public List clearCaches() { + List jobs = new ArrayList<>(); + + if (!cdmResultClearCacheEnable) { + logger.info("Clearing cache is disabled for CDM results"); + return jobs; + } + + if (!isSecured() || !isAdmin()) { + return jobs; + } + + List sources = getSourceRepository().findAll(); + for (Source source : sources) { + if (!sourceAccessor.hasAccess(source)) { + continue; + } + JobExecutionResource jobExecutionResource = jobService.findJobByName(Constants.CLEAR_CACHE, + getClearCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey())); + if (jobExecutionResource == null) { + if (source.getDaimons().stream() + .anyMatch(sd -> Objects.equals(sd.getDaimonType(), SourceDaimon.DaimonType.Results))) { + jobs.add(clearCache(source)); + } + } else { + jobs.add(jobExecutionResource); + } + } + return jobs; + } + /** * Queries for data density report for the given sourceKey * @@ -503,6 +549,18 @@ private void warmCaches(Collection sources) { } } + /* + * Warm cache for a single source + */ + private JobExecutionResource clearCache(Source source) { + String jobName = getClearCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey()); + List jobSteps = createClearCacheJobSteps(source, jobName); + SimpleJobBuilder builder = createJob(String.valueOf(source.getSourceId()), + source.getSourceKey(), + jobSteps); + return runJob(source.getSourceKey(), source.getSourceId(), jobName, builder); + } + private SimpleJobBuilder createJob(String sourceIds, String sourceKeys, List steps) { final SimpleJobBuilder[] stepBuilder = {null}; String jobName = getWarmCacheJobName(sourceIds, sourceKeys); @@ -566,6 +624,29 @@ private Step getCountStep(Source source, String jobStepName) { .build(); } + private List createClearCacheJobSteps(Source source, String jobName) { + SimpleJob job = new SimpleJob(jobName); + job.setJobRepository(jobRepository); + List steps = new ArrayList<>(); + steps.add(getAchillesClearCacheStep(source, jobName)); + steps.add(getCountClearCacheStep(source, jobName)); + return steps; + } + + private Step getAchillesClearCacheStep(Source source, String jobStepName) { + AchillesClearCacheTasklet achillesTasklet = new AchillesClearCacheTasklet(source, cacheService); + return stepBuilderFactory.get(jobStepName + " achilles") + .tasklet(achillesTasklet) + .build(); + } + + private Step getCountClearCacheStep(Source source, String jobStepName) { + CDMResultsClearCacheTasklet countTasklet = new CDMResultsClearCacheTasklet(source, cdmCacheService); + return stepBuilderFactory.get(jobStepName + " counts") + .tasklet(countTasklet) + .build(); + } + private int getResultsDaimonPriority(Source source) { Optional resultsPriority = source.getDaimons().stream() .filter(d -> d.getDaimonType().equals(SourceDaimon.DaimonType.Results)) @@ -576,11 +657,19 @@ private int getResultsDaimonPriority(Source source) { } private String getWarmCacheJobName(String sourceIds, String sourceKeys) { + return getJobName("warming cache", sourceIds, sourceKeys); + } + + private String getClearCacheJobName(String sourceIds, String sourceKeys) { + return getJobName("clearing cache", sourceIds, sourceKeys); + } + + private String getJobName(String jobType, String sourceIds, String sourceKeys) { // for multiple sources: try to compose a job name from source keys, and if it is too long - use source ids - String jobName = String.format("warming cache: %s", sourceKeys); + String jobName = String.format("%s: %s", jobType, sourceKeys); if (jobName.length() >= 100) { // job name in batch_job_instance is varchar(100) - jobName = String.format("warming cache: %s", sourceIds); + jobName = String.format("%s: %s", jobType, sourceIds); if (jobName.length() >= 100) { // if we still have more than 100 symbols jobName = jobName.substring(0, 88); From c69c87e55b8f79c084397e7d2a910e7be941de43 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Mon, 11 Nov 2024 09:44:43 -0500 Subject: [PATCH 02/19] Write a test --- pom.xml | 2 +- .../webapi/service/CDMResultsService.java | 11 ++++---- .../webapi/test/CDMResultsServiceIT.java | 25 ++++++++++++++++++- .../resources/application-test.properties | 6 ++++- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index e16116c53f..17c7673356 100644 --- a/pom.xml +++ b/pom.xml @@ -252,6 +252,7 @@ true false + true false @@ -272,7 +273,6 @@ 3 true - true 47 diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index 024adc4c22..0b843977b6 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -289,16 +289,15 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so } /** - * Refresh the results cache for a selected source + * Clear the cdm_cache and achilles_cache for all sources * - * @summary Refresh results cache - * @param sourceKey The source key - * @return The job execution resource + * @summary Clear the cdm_cache and achilles_cache for all sources + * @return List of JobExecutionResources */ @GET - @Path("clearCaches") + @Path("clearCache") @Produces(MediaType.APPLICATION_JSON) - public List clearCaches() { + public List clearCache() { List jobs = new ArrayList<>(); if (!cdmResultClearCacheEnable) { diff --git a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java index 72635bad7a..f747ebffea 100644 --- a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java +++ b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java @@ -17,6 +17,7 @@ import org.ohdsi.circe.helper.ResourceHelper; import org.ohdsi.sql.SqlRender; import org.ohdsi.sql.SqlTranslate; +import org.ohdsi.webapi.job.JobExecutionResource; import org.ohdsi.webapi.source.SourceRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -28,6 +29,9 @@ public class CDMResultsServiceIT extends WebApiIT { @Value("${cdmResultsService.endpoint.conceptRecordCount}") private String conceptRecordCountEndpoint; + @Value("${cdmResultsService.endpoint.clearCache}") + private String clearCacheEndpoint; + @Autowired private SourceRepository sourceRepository; @@ -65,7 +69,7 @@ public void requestConceptRecordCounts_firstTime_returnsResults() { final ResponseEntity>>> entity = getRestTemplate().postForEntity(this.conceptRecordCountEndpoint, conceptIds, returnClass, queryParameters ); - // Assertion + // Assert assertOK(entity); List>> results = entity.getBody(); assertEquals(1, results.size()); @@ -78,4 +82,23 @@ public void requestConceptRecordCounts_firstTime_returnsResults() { assertEquals(102, counts.get(2).intValue()); assertEquals(103, counts.get(3).intValue()); } + + @Test + public void clearCache_nothingInCache_returns() { + + // Arrange + List list = new ArrayList<>(); + @SuppressWarnings("unchecked") + Class> returnClass = (Class>) list + .getClass(); + + // Act + final ResponseEntity> entity = getRestTemplate().getForEntity(this.clearCacheEndpoint, returnClass); + + // Assert + assertOK(entity); + // Right now we don't have security enabled in the test environment, so the cache is not cleared (there's a check that we're using security) + List results = entity.getBody(); + assertEquals(0, results.size()); + } } diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index 3d9a789b5b..d1fcefacfe 100644 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -11,8 +11,10 @@ vocabularyservice.endpoint.concept=${vocabularyservice.endpoint}/concept/1 #GET cohortdefinitions cohortdefinitionservice.endpoint.cohortdefinitions=${baseUri}/cohortdefinition -#POST cdmResults +#POST conceptRecordCount cdmResultsService.endpoint.conceptRecordCount=${cdmResultsService.endpoint}/{sourceName}/conceptRecordCount +#GET clearCache +cdmResultsService.endpoint.clearCache=${cdmResultsService.endpoint}/clearCache #Example application service exampleservice.endpoint=${baseUri}/example @@ -47,3 +49,5 @@ security.provider=DisabledSecurity i18n.enabled=true i18n.defaultLocale=en + +cdm.result.clear.cache.enable=true From 7095df85cebd3a113e763f1500a74ae90a10342e Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Mon, 11 Nov 2024 10:27:11 -0500 Subject: [PATCH 03/19] Add property to application.properties --- src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index cd1afb2013..64352d8884 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -249,6 +249,7 @@ analysis.result.zipVolumeSizeMb=${analysis.result.zipVolumeSizeMb} #Cache Config cdm.result.cache.warming.enable=${cdm.result.cache.warming.enable} cdm.cache.achilles.warming.enable=${cdm.cache.achilles.warming.enable} +cdm.result.clear.cache.enable=${cdm.result.clear.cache.enable} cdm.cache.cron.warming.enable=${cdm.cache.cron.warming.enable} cdm.cache.cron.expression=${cdm.cache.cron.expression} From c730a64de250034f76e87ab46130a9ea35476e22 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Mon, 11 Nov 2024 11:13:28 -0500 Subject: [PATCH 04/19] Add the @modifying attribute to the methods that delete from the database --- .../webapi/achilles/repository/AchillesCacheRepository.java | 2 ++ .../ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java b/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java index 9571ddf0b5..9e5e8c3805 100644 --- a/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java +++ b/src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java @@ -2,6 +2,7 @@ import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity; import org.ohdsi.webapi.source.Source; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; @@ -16,6 +17,7 @@ public interface AchillesCacheRepository extends CrudRepository findBySourceAndNames(@Param("source") Source source, @Param("names") List names); + @Modifying @Query("delete from AchillesCacheEntity where source = :source") void deleteBySource(@Param("source") Source source); } diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java b/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java index e900aa218c..140bc48bfa 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java @@ -1,6 +1,7 @@ package org.ohdsi.webapi.cdmresults.repository; import org.ohdsi.webapi.cdmresults.domain.CDMCacheEntity; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.data.repository.query.Param; @@ -13,6 +14,7 @@ public interface CDMCacheRepository extends CrudRepository @Query("select c from CDMCacheEntity c where c.sourceId = :sourceId and c.conceptId in :conceptIds") List findBySourceAndConceptIds(@Param("sourceId") int sourceId, @Param("conceptIds") List conceptIds); + @Modifying @Query("delete from CDMCacheEntity c where c.sourceId = :sourceId") void deleteBySource(@Param("sourceId") int sourceId); } From 6f6b17a7c73b2b39d6c2bfcd1603bd54dea1c615 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Mon, 11 Nov 2024 11:14:11 -0500 Subject: [PATCH 05/19] Fix the job name for the clear cache job --- .../webapi/service/CDMResultsService.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index 0b843977b6..fb195c93cb 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -486,9 +486,7 @@ private JobExecutionResource warmCaches(Source source) { String jobName = getWarmCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey()); List jobSteps = createCacheWarmingJobSteps(source, jobName); - SimpleJobBuilder builder = createJob(String.valueOf(source.getSourceId()), - source.getSourceKey(), - jobSteps); + SimpleJobBuilder builder = createJob(jobName, jobSteps); return runJob(source.getSourceKey(), source.getSourceId(), jobName, builder); } @@ -533,9 +531,9 @@ private void warmCaches(Collection sources) { if (counter++ >= bucketSizes[bucketIndex] - 1) { if (!allJobSteps.isEmpty()) { - SimpleJobBuilder builder = createJob(sourceIds.stream().map(String::valueOf).collect(Collectors.joining(",")), - String.join(",", sourceKeys), - allJobSteps); + String compositeJobName = getWarmCacheJobName(sourceIds.stream().map(String::valueOf) + .collect(Collectors.joining(",")), String.join(",", sourceKeys)); + SimpleJobBuilder builder = createJob(compositeJobName, allJobSteps); runJob(source.getSourceKey(), source.getSourceId(), jobName, builder); } @@ -549,20 +547,18 @@ private void warmCaches(Collection sources) { } /* - * Warm cache for a single source + * Clear cache for a single source */ private JobExecutionResource clearCache(Source source) { String jobName = getClearCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey()); List jobSteps = createClearCacheJobSteps(source, jobName); - SimpleJobBuilder builder = createJob(String.valueOf(source.getSourceId()), - source.getSourceKey(), + SimpleJobBuilder builder = createJob(jobName, jobSteps); return runJob(source.getSourceKey(), source.getSourceId(), jobName, builder); } - private SimpleJobBuilder createJob(String sourceIds, String sourceKeys, List steps) { + private SimpleJobBuilder createJob(String jobName, List steps) { final SimpleJobBuilder[] stepBuilder = {null}; - String jobName = getWarmCacheJobName(sourceIds, sourceKeys); if (jobService.findJobByName(jobName, jobName) == null && !steps.isEmpty()) { JobBuilder jobBuilder = jobBuilders.get(jobName); From 4f0219e9119d98256e9a2b7136bd3904fb476d21 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Tue, 12 Nov 2024 16:21:21 -0500 Subject: [PATCH 06/19] Remove feature flag for clear cache --- pom.xml | 66 ++++++++++--------- .../webapi/service/CDMResultsService.java | 8 --- src/main/resources/application.properties | 1 - .../resources/application-test.properties | 2 - 4 files changed, 34 insertions(+), 43 deletions(-) diff --git a/pom.xml b/pom.xml index 17c7673356..add25b9214 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,5 @@ - + 4.0.0 @@ -122,7 +122,7 @@ (&(objectClass=person)(cn=%s)) true - 30000 + 30000 public (&(objectClass=person)(userPrincipalName=%s)) displayname @@ -191,8 +191,8 @@ - true - + true + 8080 @@ -216,7 +216,7 @@ false PBEWithMD5AndDES - + OHDSI @@ -252,7 +252,6 @@ true false - true false @@ -299,6 +298,7 @@ /tmp/atlas/audit/audit.log /tmp/atlas/audit/audit-extra.log + WebAPI ${basedir}/src/main/java @@ -369,8 +369,8 @@ - git.branch - git.commit.id.abbrev + git.branch + git.commit.id.abbrev @@ -576,6 +576,7 @@ + com.fasterxml.jackson.core @@ -809,12 +810,12 @@ jackson-databind - org.slf4j - slf4j-log4j12 + org.slf4j + slf4j-log4j12 - log4j - log4j + log4j + log4j @@ -1160,7 +1161,7 @@ ${pac4j.version} - com.fasterxml.jackson.core + com.fasterxml.jackson.core jackson-databind @@ -1206,12 +1207,12 @@ spring-boot-starter-test ${spring.boot.version} test - - - com.vaadin.external.google - android-json - - + + + com.vaadin.external.google + android-json + + org.dbunit @@ -1232,12 +1233,13 @@ test - com.github.mjeanroy - dbunit-plus - 2.0.1 - test + com.github.mjeanroy + dbunit-plus + 2.0.1 + test + webapi-oracle @@ -1330,17 +1332,17 @@ lower(email) = lower(?) - + ohdsi.snapshots repo.ohdsi.org-snapshots https://repo.ohdsi.org/nexus/content/repositories/snapshots false - + true - - + + @@ -1413,7 +1415,7 @@ true 2.6.15 - ...path/to/impala/jdbc/drivers... + ...path/to/impala/jdbc/drivers... @@ -1518,9 +1520,9 @@ v2-rev20220326-1.32.1 - com.google.cloud - google-cloud-bigquery - 1.2.15 + com.google.cloud + google-cloud-bigquery + 1.2.15 com.google.http-client diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index fb195c93cb..65e65b96b9 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -118,9 +118,6 @@ public class CDMResultsService extends AbstractDaoService implements Initializin @Value("${cdm.cache.achilles.warming.enable}") private boolean cdmAchillesCacheWarmingEnable; - @Value("${cdm.result.clear.cache.enable}") - private boolean cdmResultClearCacheEnable; - @Value("${cache.achilles.usePersonCount:false}") private boolean usePersonCount; @@ -300,11 +297,6 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so public List clearCache() { List jobs = new ArrayList<>(); - if (!cdmResultClearCacheEnable) { - logger.info("Clearing cache is disabled for CDM results"); - return jobs; - } - if (!isSecured() || !isAdmin()) { return jobs; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 64352d8884..cd1afb2013 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -249,7 +249,6 @@ analysis.result.zipVolumeSizeMb=${analysis.result.zipVolumeSizeMb} #Cache Config cdm.result.cache.warming.enable=${cdm.result.cache.warming.enable} cdm.cache.achilles.warming.enable=${cdm.cache.achilles.warming.enable} -cdm.result.clear.cache.enable=${cdm.result.clear.cache.enable} cdm.cache.cron.warming.enable=${cdm.cache.cron.warming.enable} cdm.cache.cron.expression=${cdm.cache.cron.expression} diff --git a/src/test/resources/application-test.properties b/src/test/resources/application-test.properties index d1fcefacfe..6b79fe5e0e 100644 --- a/src/test/resources/application-test.properties +++ b/src/test/resources/application-test.properties @@ -49,5 +49,3 @@ security.provider=DisabledSecurity i18n.enabled=true i18n.defaultLocale=en - -cdm.result.clear.cache.enable=true From e3bcb2dc022c7202efe68e652f6fd038643f0eb2 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Tue, 12 Nov 2024 16:22:55 -0500 Subject: [PATCH 07/19] Test fix --- src/test/java/org/ohdsi/webapi/test/ITStarter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/ohdsi/webapi/test/ITStarter.java b/src/test/java/org/ohdsi/webapi/test/ITStarter.java index 1c12bdd850..cf7593e17f 100644 --- a/src/test/java/org/ohdsi/webapi/test/ITStarter.java +++ b/src/test/java/org/ohdsi/webapi/test/ITStarter.java @@ -21,7 +21,8 @@ SecurityIT.class, JobServiceIT.class, CohortAnalysisServiceIT.class, - VocabularyServiceIT.class + VocabularyServiceIT.class, + CDMResultsServiceIT.class }) @TestPropertySource(locations = "/application-test.properties") public class ITStarter extends AbstractShiro { From a37083e5af668a1e80dec5e7772193c96e6ebdb4 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Wed, 13 Nov 2024 09:21:45 -0500 Subject: [PATCH 08/19] Add an editorconfig file to use correct formatting in this project --- .editorconfig | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..b9c127d689 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = crlf +charset = utf-8 +trim_trailing_whitespace = false +insert_final_newline = false \ No newline at end of file From af59f70d9681325548f4f78ddb77d5b17d27e17b Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Wed, 13 Nov 2024 09:22:18 -0500 Subject: [PATCH 09/19] Don't do jobs. --- .../cdmresults/AchillesClearCacheTasklet.java | 24 ------- .../CDMResultsClearCacheTasklet.java | 24 ------- .../webapi/service/CDMResultsService.java | 70 ++++--------------- 3 files changed, 12 insertions(+), 106 deletions(-) delete mode 100644 src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java delete mode 100644 src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java b/src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java deleted file mode 100644 index 364f97242f..0000000000 --- a/src/main/java/org/ohdsi/webapi/cdmresults/AchillesClearCacheTasklet.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.ohdsi.webapi.cdmresults; - -import org.ohdsi.webapi.achilles.service.AchillesCacheService; -import org.ohdsi.webapi.source.Source; -import org.springframework.batch.core.StepContribution; -import org.springframework.batch.core.scope.context.ChunkContext; -import org.springframework.batch.core.step.tasklet.Tasklet; -import org.springframework.batch.repeat.RepeatStatus; - -public class AchillesClearCacheTasklet implements Tasklet { - private final Source source; - private final AchillesCacheService cacheService; - - public AchillesClearCacheTasklet(Source source, AchillesCacheService cacheService) { - this.source = source; - this.cacheService = cacheService; - } - - @Override - public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) { - cacheService.clearCache(this.source); - return RepeatStatus.FINISHED; - } -} diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java b/src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java deleted file mode 100644 index 2d2d9b2fa8..0000000000 --- a/src/main/java/org/ohdsi/webapi/cdmresults/CDMResultsClearCacheTasklet.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.ohdsi.webapi.cdmresults; - -import org.ohdsi.webapi.cdmresults.service.CDMCacheService; -import org.ohdsi.webapi.source.Source; -import org.springframework.batch.core.StepContribution; -import org.springframework.batch.core.scope.context.ChunkContext; -import org.springframework.batch.core.step.tasklet.Tasklet; -import org.springframework.batch.repeat.RepeatStatus; - -public class CDMResultsClearCacheTasklet implements Tasklet { - private final Source source; - private final CDMCacheService cdmCacheService; - - public CDMResultsClearCacheTasklet(Source source, CDMCacheService cdmCacheService) { - this.source = source; - this.cdmCacheService = cdmCacheService; - } - - @Override - public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) { - cdmCacheService.clearCache(this.source); - return RepeatStatus.FINISHED; - } -} \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index 65e65b96b9..483299df53 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -8,9 +8,7 @@ import org.ohdsi.webapi.achilles.aspect.AchillesCache; import org.ohdsi.webapi.achilles.service.AchillesCacheService; import org.ohdsi.webapi.cdmresults.AchillesCacheTasklet; -import org.ohdsi.webapi.cdmresults.AchillesClearCacheTasklet; import org.ohdsi.webapi.cdmresults.CDMResultsCacheTasklet; -import org.ohdsi.webapi.cdmresults.CDMResultsClearCacheTasklet; import org.ohdsi.webapi.cdmresults.DescendantRecordAndPersonCount; import org.ohdsi.webapi.cdmresults.DescendantRecordCount; import org.ohdsi.webapi.cdmresults.domain.CDMCacheEntity; @@ -48,6 +46,7 @@ import org.springframework.stereotype.Component; import javax.ws.rs.Consumes; +import javax.ws.rs.ForbiddenException; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; @@ -289,35 +288,17 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so * Clear the cdm_cache and achilles_cache for all sources * * @summary Clear the cdm_cache and achilles_cache for all sources - * @return List of JobExecutionResources + * @return void + * @throws ForbiddenException if the user is not an admin */ @GET @Path("clearCache") - @Produces(MediaType.APPLICATION_JSON) - public List clearCache() { - List jobs = new ArrayList<>(); - + public void clearCache() { if (!isSecured() || !isAdmin()) { - return jobs; + throw new ForbiddenException(); } - List sources = getSourceRepository().findAll(); - for (Source source : sources) { - if (!sourceAccessor.hasAccess(source)) { - continue; - } - JobExecutionResource jobExecutionResource = jobService.findJobByName(Constants.CLEAR_CACHE, - getClearCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey())); - if (jobExecutionResource == null) { - if (source.getDaimons().stream() - .anyMatch(sd -> Objects.equals(sd.getDaimonType(), SourceDaimon.DaimonType.Results))) { - jobs.add(clearCache(source)); - } - } else { - jobs.add(jobExecutionResource); - } - } - return jobs; + sources.parallelStream().forEach(this::clearCache); } /** @@ -541,12 +522,12 @@ private void warmCaches(Collection sources) { /* * Clear cache for a single source */ - private JobExecutionResource clearCache(Source source) { - String jobName = getClearCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey()); - List jobSteps = createClearCacheJobSteps(source, jobName); - SimpleJobBuilder builder = createJob(jobName, - jobSteps); - return runJob(source.getSourceKey(), source.getSourceId(), jobName, builder); + private void clearCache(Source source) { + if (!sourceAccessor.hasAccess(source)) { + return; + } + cacheService.clearCache(source); + cdmCacheService.clearCache(source); } private SimpleJobBuilder createJob(String jobName, List steps) { @@ -611,29 +592,6 @@ private Step getCountStep(Source source, String jobStepName) { .build(); } - private List createClearCacheJobSteps(Source source, String jobName) { - SimpleJob job = new SimpleJob(jobName); - job.setJobRepository(jobRepository); - List steps = new ArrayList<>(); - steps.add(getAchillesClearCacheStep(source, jobName)); - steps.add(getCountClearCacheStep(source, jobName)); - return steps; - } - - private Step getAchillesClearCacheStep(Source source, String jobStepName) { - AchillesClearCacheTasklet achillesTasklet = new AchillesClearCacheTasklet(source, cacheService); - return stepBuilderFactory.get(jobStepName + " achilles") - .tasklet(achillesTasklet) - .build(); - } - - private Step getCountClearCacheStep(Source source, String jobStepName) { - CDMResultsClearCacheTasklet countTasklet = new CDMResultsClearCacheTasklet(source, cdmCacheService); - return stepBuilderFactory.get(jobStepName + " counts") - .tasklet(countTasklet) - .build(); - } - private int getResultsDaimonPriority(Source source) { Optional resultsPriority = source.getDaimons().stream() .filter(d -> d.getDaimonType().equals(SourceDaimon.DaimonType.Results)) @@ -647,10 +605,6 @@ private String getWarmCacheJobName(String sourceIds, String sourceKeys) { return getJobName("warming cache", sourceIds, sourceKeys); } - private String getClearCacheJobName(String sourceIds, String sourceKeys) { - return getJobName("clearing cache", sourceIds, sourceKeys); - } - private String getJobName(String jobType, String sourceIds, String sourceKeys) { // for multiple sources: try to compose a job name from source keys, and if it is too long - use source ids String jobName = String.format("%s: %s", jobType, sourceKeys); From 3408a2c1786ac91db20dd64abd9de22df61742ee Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Wed, 13 Nov 2024 09:24:38 -0500 Subject: [PATCH 10/19] Method should be a post --- src/main/java/org/ohdsi/webapi/service/CDMResultsService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index 483299df53..edf04493b2 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -291,7 +291,7 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so * @return void * @throws ForbiddenException if the user is not an admin */ - @GET + @POST @Path("clearCache") public void clearCache() { if (!isSecured() || !isAdmin()) { From 7df35d07a473927855bef6f7162972be808c3191 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Wed, 13 Nov 2024 12:43:17 -0500 Subject: [PATCH 11/19] No longer need this constant --- src/main/java/org/ohdsi/webapi/Constants.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/Constants.java b/src/main/java/org/ohdsi/webapi/Constants.java index d3d7c38e83..2069ed108d 100644 --- a/src/main/java/org/ohdsi/webapi/Constants.java +++ b/src/main/java/org/ohdsi/webapi/Constants.java @@ -12,7 +12,6 @@ public interface Constants { String GENERATE_PREDICTION_ANALYSIS = "generatePredictionAnalysis"; String GENERATE_ESTIMATION_ANALYSIS = "generateEstimationAnalysis"; String WARM_CACHE = "warmCache"; - String CLEAR_CACHE = "clearCache"; String USERS_IMPORT = "usersImport"; String JOB_IS_ALREADY_SCHEDULED = "Job for provider %s is already scheduled"; From a857290d296f0259987823b7c3d2642a999d3362 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Wed, 13 Nov 2024 14:32:25 -0500 Subject: [PATCH 12/19] Fix the test and ignore it --- .../org/ohdsi/webapi/test/CDMResultsServiceIT.java | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java index f747ebffea..1820ee00ca 100644 --- a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java +++ b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java @@ -13,11 +13,11 @@ import java.util.Map; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.ohdsi.circe.helper.ResourceHelper; import org.ohdsi.sql.SqlRender; import org.ohdsi.sql.SqlTranslate; -import org.ohdsi.webapi.job.JobExecutionResource; import org.ohdsi.webapi.source.SourceRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -83,22 +83,17 @@ public void requestConceptRecordCounts_firstTime_returnsResults() { assertEquals(103, counts.get(3).intValue()); } + // This is ignored right now because the clearCache method requires that security be set up and I'm not sure how to do that in this context + @Ignore @Test public void clearCache_nothingInCache_returns() { // Arrange - List list = new ArrayList<>(); - @SuppressWarnings("unchecked") - Class> returnClass = (Class>) list - .getClass(); // Act - final ResponseEntity> entity = getRestTemplate().getForEntity(this.clearCacheEndpoint, returnClass); + final ResponseEntity entity = getRestTemplate().getForEntity(this.clearCacheEndpoint, String.class); // Assert assertOK(entity); - // Right now we don't have security enabled in the test environment, so the cache is not cleared (there's a check that we're using security) - List results = entity.getBody(); - assertEquals(0, results.size()); } } From aa73c6f3340ab953572726c6977f011922272688 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Wed, 13 Nov 2024 14:32:36 -0500 Subject: [PATCH 13/19] Change the indent size to 2 --- .editorconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.editorconfig b/.editorconfig index b9c127d689..88bf97c508 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,7 +5,7 @@ root = true [*] indent_style = space -indent_size = 4 +indent_size = 2 end_of_line = crlf charset = utf-8 trim_trailing_whitespace = false From 1e4dd19fa4f48faa486d59b0e1db78c0481c2e80 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Thu, 14 Nov 2024 09:42:45 -0500 Subject: [PATCH 14/19] Fix test method to send post instead of get --- src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java index 1820ee00ca..f23a104f00 100644 --- a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java +++ b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java @@ -91,7 +91,7 @@ public void clearCache_nothingInCache_returns() { // Arrange // Act - final ResponseEntity entity = getRestTemplate().getForEntity(this.clearCacheEndpoint, String.class); + final ResponseEntity entity = getRestTemplate().postForEntity(this.clearCacheEndpoint, null, String.class); // Assert assertOK(entity); From e89e4ab45a8ff04634b93272915df28d131df4c0 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Thu, 14 Nov 2024 12:22:41 -0500 Subject: [PATCH 15/19] Add a migration to add the required permission for clearing the cache --- ...0.20241113115700__add_clearcache_permission.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql diff --git a/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql b/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql new file mode 100644 index 0000000000..b2ba6fc697 --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql @@ -0,0 +1,14 @@ +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'cdmresults:clearcache:post', + 'Clear the achilles and cdm results caches'; + +INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) +SELECT sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission sp, + ${ohdsiSchema}.sec_role sr +WHERE sp."value" in + ( + 'cdmresults:clearcache:post' + ) + AND sr.name IN ('admin'); From 3d8f33d7d2036e4a6acb43ac75e912ce852c2867 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Thu, 14 Nov 2024 12:22:48 -0500 Subject: [PATCH 16/19] Add missing transaction --- .../org/ohdsi/webapi/cdmresults/service/CDMCacheService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java index 106c4775de..dd732220ba 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java @@ -19,6 +19,7 @@ import org.springframework.core.convert.ConversionService; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionTemplate; import java.util.ArrayList; @@ -73,6 +74,7 @@ public void warm(Source source) { } } + @Transactional() public void clearCache(Source source) { cdmCacheRepository.deleteBySource(source.getSourceId()); } From d3b3e25109ee490d9aba18efa0f69cce909e18aa Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Thu, 14 Nov 2024 15:07:47 -0500 Subject: [PATCH 17/19] Refactor and write tests against the internal services --- .../service/AchillesCacheService.java | 204 +++++++++--------- .../cdmresults/service/CDMCacheService.java | 15 +- .../webapi/service/CDMResultsService.java | 15 +- .../webapi/test/CDMResultsServiceIT.java | 71 +++++- 4 files changed, 184 insertions(+), 121 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java index a65d033e7a..d80ea18c41 100644 --- a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java +++ b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java @@ -6,8 +6,10 @@ import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity; import org.ohdsi.webapi.achilles.repository.AchillesCacheRepository; import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Propagation; @@ -22,108 +24,118 @@ @Service public class AchillesCacheService { - private static final Logger LOG = LoggerFactory.getLogger(AchillesCacheService.class); - - private final AchillesCacheRepository cacheRepository; - - private final ObjectMapper objectMapper; - - @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}") - private int batchSize; - - public AchillesCacheService(AchillesCacheRepository cacheRepository, - ObjectMapper objectMapper) { - this.cacheRepository = cacheRepository; - this.objectMapper = objectMapper; - } - - @Transactional(readOnly = true) - public AchillesCacheEntity getCache(Source source, String cacheName) { - return cacheRepository.findBySourceAndCacheName(source, cacheName); + private static final Logger LOG = LoggerFactory.getLogger(AchillesCacheService.class); + + private final AchillesCacheRepository cacheRepository; + + private final ObjectMapper objectMapper; + + @Value("${spring.jpa.properties.hibernate.jdbc.batch_size}") + private int batchSize; + + @Autowired + private SourceRepository sourceRepository; + + public AchillesCacheService(AchillesCacheRepository cacheRepository, + ObjectMapper objectMapper) { + this.cacheRepository = cacheRepository; + this.objectMapper = objectMapper; + } + + @Transactional(readOnly = true) + public AchillesCacheEntity getCache(Source source, String cacheName) { + return cacheRepository.findBySourceAndCacheName(source, cacheName); + } + + @Transactional(readOnly = true) + public List findBySourceAndNames(Source source, List names) { + return cacheRepository.findBySourceAndNames(source, names); + } + + @Transactional(propagation = Propagation.REQUIRES_NEW) + public AchillesCacheEntity createCache(Source source, String cacheName, Object result) + throws JsonProcessingException { + AchillesCacheEntity cacheEntity = getCache(source, cacheName); + String cache = objectMapper.writeValueAsString(result); + if (Objects.nonNull(cacheEntity)) { + cacheEntity.setCache(cache); + } else { + cacheEntity = new AchillesCacheEntity(); + cacheEntity.setSource(source); + cacheEntity.setCacheName(cacheName); + cacheEntity.setCache(cache); } - @Transactional(readOnly = true) - public List findBySourceAndNames(Source source, List names) { - return cacheRepository.findBySourceAndNames(source, names); - } - - @Transactional(propagation = Propagation.REQUIRES_NEW) - public AchillesCacheEntity createCache(Source source, String cacheName, Object result) throws JsonProcessingException { - AchillesCacheEntity cacheEntity = getCache(source, cacheName); - String cache = objectMapper.writeValueAsString(result); - if (Objects.nonNull(cacheEntity)) { - cacheEntity.setCache(cache); - } else { - cacheEntity = new AchillesCacheEntity(); - cacheEntity.setSource(source); - cacheEntity.setCacheName(cacheName); - cacheEntity.setCache(cache); - } + return cacheRepository.save(cacheEntity); + } - return cacheRepository.save(cacheEntity); + @Transactional(propagation = Propagation.REQUIRES_NEW) + public void saveDrilldownCacheMap(Source source, String domain, Map conceptNodes) { + if (conceptNodes.isEmpty()) { // nothing to cache + LOG.warn( + "Cannot cache drilldown reports for {}, domain {}. Check if result schema contains achilles results tables.", + source.getSourceKey(), domain); + return; } - @Transactional(propagation = Propagation.REQUIRES_NEW) - public void saveDrilldownCacheMap(Source source, String domain, Map conceptNodes) { - if (conceptNodes.isEmpty()) { // nothing to cache - LOG.warn("Cannot cache drilldown reports for {}, domain {}. Check if result schema contains achilles results tables.", - source.getSourceKey(), domain); - return; - } - - Map nodes = new HashMap<>(batchSize); - for (Map.Entry entry : conceptNodes.entrySet()) { - if (nodes.size() >= batchSize) { - createCacheEntities(source, nodes); - nodes.clear(); - } - Integer key = entry.getKey(); - String cacheName = getCacheName(domain, key); - nodes.put(cacheName, entry.getValue()); - } + Map nodes = new HashMap<>(batchSize); + for (Map.Entry entry : conceptNodes.entrySet()) { + if (nodes.size() >= batchSize) { createCacheEntities(source, nodes); nodes.clear(); + } + Integer key = entry.getKey(); + String cacheName = getCacheName(domain, key); + nodes.put(cacheName, entry.getValue()); } - - @Transactional() - public void clearCache(Source source) { - cacheRepository.deleteBySource(source); - } - - private void createCacheEntities(Source source, Map nodes) { - List cacheEntities = getEntities(source, nodes); - cacheRepository.save(cacheEntities); - } - - private List getEntities(Source source, Map nodes) { - List cacheNames = new ArrayList<>(nodes.keySet()); - List cacheEntities = findBySourceAndNames(source, cacheNames); - nodes.forEach((key, value) -> { - // check if the entity with given cache name already exists - Optional cacheEntity = cacheEntities.stream() - .filter(entity -> entity.getCacheName().equals(key)) - .findAny(); - try { - String newValue = objectMapper.writeValueAsString(value); - if (cacheEntity.isPresent()) { - // if cache entity already exists update its value - cacheEntity.get().setCache(newValue); - } else { - // if cache entity does not exist - create new one - AchillesCacheEntity newEntity = new AchillesCacheEntity(); - newEntity.setCacheName(key); - newEntity.setSource(source); - newEntity.setCache(newValue); - cacheEntities.add(newEntity); - } - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - }); - return cacheEntities; - } - - private String getCacheName(String domain, int conceptId) { - return String.format("drilldown_%s_%d", domain, conceptId); - } + createCacheEntities(source, nodes); + nodes.clear(); + } + + @Transactional() + public void clearCache() { + List sources = sourceRepository.findAll(); + sources.parallelStream().forEach(this::clearCache); + } + + private void clearCache(Source source) { + cacheRepository.deleteBySource(source); + } + + private void createCacheEntities(Source source, Map nodes) { + List cacheEntities = getEntities(source, nodes); + cacheRepository.save(cacheEntities); + } + + private List getEntities(Source source, Map nodes) { + List cacheNames = new ArrayList<>(nodes.keySet()); + List cacheEntities = findBySourceAndNames(source, cacheNames); + nodes.forEach((key, value) -> { + // check if the entity with given cache name already exists + Optional cacheEntity = cacheEntities.stream() + .filter(entity -> entity.getCacheName().equals(key)) + .findAny(); + try { + String newValue = objectMapper.writeValueAsString(value); + if (cacheEntity.isPresent()) { + // if cache entity already exists update its value + cacheEntity.get().setCache(newValue); + } else { + // if cache entity does not exist - create new one + AchillesCacheEntity newEntity = new AchillesCacheEntity(); + newEntity.setCacheName(key); + newEntity.setSource(source); + newEntity.setCache(newValue); + cacheEntities.add(newEntity); + } + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + return cacheEntities; + } + + private String getCacheName(String domain, int conceptId) { + return String.format("drilldown_%s_%d", domain, conceptId); + } } diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java index dd732220ba..2e053b3c26 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java @@ -74,11 +74,6 @@ public void warm(Source source) { } } - @Transactional() - public void clearCache(Source source) { - cdmCacheRepository.deleteBySource(source.getSourceId()); - } - public List findAndCache(Source source, List conceptIds) { if (CollectionUtils.isEmpty(conceptIds)) { return Collections.emptyList(); @@ -101,6 +96,16 @@ public List findAndCache(Source source, List conceptIds return cacheEntities; } + @Transactional() + public void clearCache() { + List sources = getSourceRepository().findAll(); + sources.parallelStream().forEach(this::clearCache); + } + + private void clearCache(Source source) { + cdmCacheRepository.deleteBySource(source.getSourceId()); + } + private List find(Source source, List conceptIds) { if (CollectionUtils.isEmpty(conceptIds)) { return Collections.emptyList(); diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index edf04493b2..938577fd85 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -297,8 +297,8 @@ public void clearCache() { if (!isSecured() || !isAdmin()) { throw new ForbiddenException(); } - List sources = getSourceRepository().findAll(); - sources.parallelStream().forEach(this::clearCache); + cacheService.clearCache(); + cdmCacheService.clearCache(); } /** @@ -519,17 +519,6 @@ private void warmCaches(Collection sources) { } } - /* - * Clear cache for a single source - */ - private void clearCache(Source source) { - if (!sourceAccessor.hasAccess(source)) { - return; - } - cacheService.clearCache(source); - cdmCacheService.clearCache(source); - } - private SimpleJobBuilder createJob(String jobName, List steps) { final SimpleJobBuilder[] stepBuilder = {null}; if (jobService.findJobByName(jobName, jobName) == null && !steps.isEmpty()) { diff --git a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java index f23a104f00..51e0a356c5 100644 --- a/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java +++ b/src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java @@ -13,11 +13,12 @@ import java.util.Map; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.ohdsi.circe.helper.ResourceHelper; import org.ohdsi.sql.SqlRender; import org.ohdsi.sql.SqlTranslate; +import org.ohdsi.webapi.achilles.service.AchillesCacheService; +import org.ohdsi.webapi.cdmresults.service.CDMCacheService; import org.ohdsi.webapi.source.SourceRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -35,6 +36,12 @@ public class CDMResultsServiceIT extends WebApiIT { @Autowired private SourceRepository sourceRepository; + @Autowired + private AchillesCacheService achillesService; + + @Autowired + private CDMCacheService cdmCacheService; + @Before public void init() throws Exception { truncateTable(String.format("%s.%s", "public", "source")); @@ -83,17 +90,67 @@ public void requestConceptRecordCounts_firstTime_returnsResults() { assertEquals(103, counts.get(3).intValue()); } - // This is ignored right now because the clearCache method requires that security be set up and I'm not sure how to do that in this context - @Ignore @Test - public void clearCache_nothingInCache_returns() { + public void achillesService_clearCache_nothingInCache_doesNothing() { // Arrange // Act - final ResponseEntity entity = getRestTemplate().postForEntity(this.clearCacheEndpoint, null, String.class); + achillesService.clearCache(); // Assert - assertOK(entity); + String sql = "SELECT COUNT(*) FROM achilles_cache"; + Integer count = jdbcTemplate.queryForObject(sql, Integer.class); + assertEquals(0, count.intValue()); + } + + @Test + public void achillesService_clearCache_somethingInCache_clearsAllRowsForSource() { + + // Arrange + String insertSqlRow1 = "INSERT INTO achilles_cache (id, source_id, cache_name, cache) VALUES (1, 1, 'cache1', 'cache1')"; + jdbcTemplate.execute(insertSqlRow1); + String insertSqlRow2 = "INSERT INTO achilles_cache (id, source_id, cache_name, cache) VALUES (2, 1, 'cache2', 'cache2')"; + jdbcTemplate.execute(insertSqlRow2); + + // Act + achillesService.clearCache(); + + // Assert + String sql = "SELECT COUNT(*) FROM achilles_cache"; + Integer count = jdbcTemplate.queryForObject(sql, Integer.class); + assertEquals(0, count.intValue()); + } + + @Test + public void cdmCacheService_clearCache_nothingInCache_doesNothing() { + + // Arrange + + // Act + cdmCacheService.clearCache(); + + // Assert + String sql = "SELECT COUNT(*) FROM cdm_cache"; + Integer count = jdbcTemplate.queryForObject(sql, Integer.class); + assertEquals(0, count.intValue()); + } + + @Test + public void cdmCacheService_clearCache_somethingInCache_clearsAllRowsForSource() { + + // Arrange + String insertSqlRow1 = "INSERT INTO cdm_cache (id, concept_id, source_id, record_count, descendant_record_count, person_count, descendant_person_count) VALUES (1, 1, 1, 100, 101, 102, 103)"; + jdbcTemplate.execute(insertSqlRow1); + String insertSqlRow2 = "INSERT INTO cdm_cache (id, concept_id, source_id, record_count, descendant_record_count, person_count, descendant_person_count) VALUES (2, 2, 1, 200, 201, 202, 203)"; + jdbcTemplate.execute(insertSqlRow2); + + // Act + cdmCacheService.clearCache(); + + // Assert + String sql = "SELECT COUNT(*) FROM cdm_cache"; + Integer count = jdbcTemplate.queryForObject(sql, Integer.class); + assertEquals(0, count.intValue()); } -} + } From a1a29cd5727e817b9e84bdebd1c3da034ea05e88 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Fri, 15 Nov 2024 10:36:47 -0500 Subject: [PATCH 18/19] Check for source access. Get the transactions correct. --- .../webapi/achilles/service/AchillesCacheService.java | 10 ++++++++-- .../webapi/cdmresults/service/CDMCacheService.java | 11 +++++++++-- .../org/ohdsi/webapi/service/CDMResultsService.java | 2 ++ 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java index d80ea18c41..4502dd07fb 100644 --- a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java +++ b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity; import org.ohdsi.webapi.achilles.repository.AchillesCacheRepository; +import org.ohdsi.webapi.shiro.management.datasource.SourceAccessor; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceRepository; import org.slf4j.Logger; @@ -36,6 +37,9 @@ public class AchillesCacheService { @Autowired private SourceRepository sourceRepository; + @Autowired + private SourceAccessor sourceAccessor; + public AchillesCacheService(AchillesCacheRepository cacheRepository, ObjectMapper objectMapper) { this.cacheRepository = cacheRepository; @@ -95,11 +99,13 @@ public void saveDrilldownCacheMap(Source source, String domain, Map sources = sourceRepository.findAll(); - sources.parallelStream().forEach(this::clearCache); + sources.stream().forEach(this::clearCache); } private void clearCache(Source source) { - cacheRepository.deleteBySource(source); + if (sourceAccessor.hasAccess(source)) { + cacheRepository.deleteBySource(source); + } } private void createCacheEntities(Source source, Map nodes) { diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java index 2e053b3c26..14855bc5f4 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java @@ -10,11 +10,13 @@ import org.ohdsi.webapi.cdmresults.mapper.DescendantRecordCountMapper; import org.ohdsi.webapi.cdmresults.repository.CDMCacheRepository; import org.ohdsi.webapi.service.AbstractDaoService; +import org.ohdsi.webapi.shiro.management.datasource.SourceAccessor; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceDaimon; import org.ohdsi.webapi.util.PreparedSqlRender; import org.ohdsi.webapi.util.PreparedStatementRenderer; import org.ohdsi.webapi.util.SessionUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.convert.ConversionService; import org.springframework.jdbc.core.JdbcTemplate; @@ -52,6 +54,9 @@ public class CDMCacheService extends AbstractDaoService { private final ConversionService conversionService; + @Autowired + private SourceAccessor sourceAccessor; + public CDMCacheService(CDMCacheBatchService cdmCacheBatchService, ConversionService conversionService, CDMCacheRepository cdmCacheRepository) { @@ -99,11 +104,13 @@ public List findAndCache(Source source, List conceptIds @Transactional() public void clearCache() { List sources = getSourceRepository().findAll(); - sources.parallelStream().forEach(this::clearCache); + sources.stream().forEach(this::clearCache); } private void clearCache(Source source) { - cdmCacheRepository.deleteBySource(source.getSourceId()); + if (sourceAccessor.hasAccess(source)) { + cdmCacheRepository.deleteBySource(source.getSourceId()); + } } private List find(Source source, List conceptIds) { diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index 938577fd85..f44353a017 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -44,6 +44,7 @@ import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import javax.ws.rs.Consumes; import javax.ws.rs.ForbiddenException; @@ -293,6 +294,7 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so */ @POST @Path("clearCache") + @Transactional() public void clearCache() { if (!isSecured() || !isAdmin()) { throw new ForbiddenException(); From 3eadef7ea5574d983c32b069bb26e89a14f750e8 Mon Sep 17 00:00:00 2001 From: Anne Marsan Date: Mon, 25 Nov 2024 13:59:56 -0500 Subject: [PATCH 19/19] Add endpoint for clearing individual source caches --- .../service/AchillesCacheService.java | 3 ++- .../cdmresults/service/CDMCacheService.java | 3 ++- .../model/SourcePermissionSchema.java | 1 + .../webapi/service/CDMResultsService.java | 19 +++++++++++++++++++ ...41113115700__add_clearcache_permission.sql | 17 ++++++++++++++++- 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java index 4502dd07fb..023f6d7d69 100644 --- a/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java +++ b/src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java @@ -102,7 +102,8 @@ public void clearCache() { sources.stream().forEach(this::clearCache); } - private void clearCache(Source source) { + @Transactional() + public void clearCache(Source source) { if (sourceAccessor.hasAccess(source)) { cacheRepository.deleteBySource(source); } diff --git a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java index 14855bc5f4..8b7a6e520b 100644 --- a/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java +++ b/src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java @@ -107,7 +107,8 @@ public void clearCache() { sources.stream().forEach(this::clearCache); } - private void clearCache(Source source) { + @Transactional() + public void clearCache(Source source) { if (sourceAccessor.hasAccess(source)) { cdmCacheRepository.deleteBySource(source.getSourceId()); } diff --git a/src/main/java/org/ohdsi/webapi/security/model/SourcePermissionSchema.java b/src/main/java/org/ohdsi/webapi/security/model/SourcePermissionSchema.java index c510776be8..ae457d252f 100644 --- a/src/main/java/org/ohdsi/webapi/security/model/SourcePermissionSchema.java +++ b/src/main/java/org/ohdsi/webapi/security/model/SourcePermissionSchema.java @@ -33,6 +33,7 @@ public class SourcePermissionSchema extends EntityPermissionSchema { put("cdmresults:%s:*:get", "Get Achilles reports on Source with SourceKey = %s"); put("cdmresults:%s:conceptRecordCount:post", "Get Achilles concept counts on Source with SourceKey = %s"); put("cdmresults:%s:*:*:get", "Get Achilles reports details on Source with SourceKey = %s"); + put("cdmresults:%s:clearcache:post", "Clear the Achilles and CDM results caches on Source with SourceKey = %s"); put("cohortresults:%s:*:*:get", "Get cohort results on Source with SourceKey = %s"); put("cohortresults:%s:*:*:*:get", "Get cohort results details on Source with SourceKey = %s"); put("cohortresults:%s:*:healthcareutilization:*:*:get", "Get cohort results baseline on period for Source with SourceKey = %s"); diff --git a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java index f44353a017..c887962d1c 100644 --- a/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java +++ b/src/main/java/org/ohdsi/webapi/service/CDMResultsService.java @@ -284,6 +284,25 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so } return new JobExecutionResource(); } + + /** + * Clear the cdm_cache and achilles_cache for all sources + * + * @summary Clear the cdm_cache and achilles_cache for all sources + * @return void + * @throws ForbiddenException if the user is not an admin + */ + @POST + @Path("{sourceKey}/clearCache") + @Transactional() + public void clearCacheForSource(@PathParam("sourceKey") final String sourceKey) { + if (!isSecured() || !isAdmin()) { + throw new ForbiddenException(); + } + Source source = getSourceRepository().findBySourceKey(sourceKey); + cacheService.clearCache(source); + cdmCacheService.clearCache(source); + } /** * Clear the cdm_cache and achilles_cache for all sources diff --git a/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql b/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql index b2ba6fc697..080e455933 100644 --- a/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql +++ b/src/main/resources/db/migration/postgresql/V2.15.0.20241113115700__add_clearcache_permission.sql @@ -1,7 +1,7 @@ INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cdmresults:clearcache:post', - 'Clear the achilles and cdm results caches'; + 'Clear the Achilles and CDM results caches'; INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) SELECT sr.id, sp.id @@ -12,3 +12,18 @@ WHERE sp."value" in 'cdmresults:clearcache:post' ) AND sr.name IN ('admin'); + +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'cdmresults:*:clearcache:post', + 'Clear the Achilles and CDM results caches'; + +INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) +SELECT sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission sp, + ${ohdsiSchema}.sec_role sr +WHERE sp."value" in + ( + 'cdmresults:*:clearcache:post' + ) + AND sr.name IN ('admin');