Add endpoint to clear the cdm_cache and achilles_cache

.editorconfig

@@ -0,0 +1,12 @@
+# EditorConfig is awesome: https://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = crlf
+charset = utf-8
+trim_trailing_whitespace = false
+insert_final_newline = false

pom.xml

@@ -1,5 +1,5 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
 
   <modelVersion>4.0.0</modelVersion>
 
@@ -122,7 +122,7 @@
     <security.ad.system.password></security.ad.system.password>
     <security.ad.searchFilter>(&amp;(objectClass=person)(cn=%s))</security.ad.searchFilter>
     <security.ad.ignore.partial.result.exception>true</security.ad.ignore.partial.result.exception>
-    <security.ad.result.count.limit>30000</security.ad.result.count.limit> <!-- 0 means no limit -->
+    <security.ad.result.count.limit>30000</security.ad.result.count.limit>    <!-- 0 means no limit -->
     <security.ad.default.import.group>public</security.ad.default.import.group>
     <security.ad.searchString>(&amp;(objectClass=person)(userPrincipalName=%s))</security.ad.searchString>
     <security.ad.userMapping.displaynameAttr>displayname</security.ad.userMapping.displaynameAttr>
@@ -191,8 +191,8 @@
     <!-- If defaultGlobalReadPermissions is set to true (default), then all users can see every artifact.  -->
     <!-- If it is set to false, WebAPI will filter out the artifacts that a user does not explicitly have -->
     <!-- read permissions to -->
-    <security.defaultGlobalReadPermissions>true</security.defaultGlobalReadPermissions>	
-    
+    <security.defaultGlobalReadPermissions>true</security.defaultGlobalReadPermissions>
+
     <!-- EMBEDDED SERVER CONFIGURATION (ServerProperties) -->
     <server.port>8080</server.port>
     <server.ssl.key-store></server.ssl.key-store>
@@ -216,7 +216,7 @@
     <jasypt.encryptor.enabled>false</jasypt.encryptor.enabled>
     <jasypt.encryptor.password></jasypt.encryptor.password>
     <jasypt.encryptor.algorithm>PBEWithMD5AndDES</jasypt.encryptor.algorithm>
-                
+
     <!-- ORGANIZATION SETTINGS -->
     <organization.name>OHDSI</organization.name>
 
@@ -298,6 +298,7 @@
     <audit.trail.log.file>/tmp/atlas/audit/audit.log</audit.trail.log.file>
     <audit.trail.log.extraFile>/tmp/atlas/audit/audit-extra.log</audit.trail.log.extraFile>
   </properties>
+
   <build>
     <finalName>WebAPI</finalName>
     <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
@@ -368,8 +369,8 @@
           <!-- only include properties to speed up plugin -->
           <!-- ref: https://github.com/git-commit-id/git-commit-id-maven-plugin/issues/462 -->
           <includeOnlyProperties>
-              <includeOnlyProperty>git.branch</includeOnlyProperty>
-              <includeOnlyProperty>git.commit.id.abbrev</includeOnlyProperty>
+            <includeOnlyProperty>git.branch</includeOnlyProperty>
+            <includeOnlyProperty>git.commit.id.abbrev</includeOnlyProperty>
           </includeOnlyProperties>
         </configuration>
       </plugin>
@@ -575,6 +576,7 @@
       </dependency>
     </dependencies>
   </dependencyManagement>
+
   <dependencies>
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
@@ -808,12 +810,12 @@
           <artifactId>jackson-databind</artifactId>
         </exclusion>
         <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
         </exclusion>
         <exclusion>
-            <groupId>log4j</groupId>
-            <artifactId>log4j</artifactId>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
         </exclusion>
       </exclusions>
     </dependency>
@@ -1159,7 +1161,7 @@
       <version>${pac4j.version}</version>
       <exclusions>
         <exclusion>
-      <groupId>com.fasterxml.jackson.core</groupId>
+          <groupId>com.fasterxml.jackson.core</groupId>
           <artifactId>jackson-databind</artifactId>
         </exclusion>
       </exclusions>
@@ -1205,12 +1207,12 @@
       <artifactId>spring-boot-starter-test</artifactId>
       <version>${spring.boot.version}</version>
       <scope>test</scope>
-        <exclusions>
-            <exclusion>
-                <groupId>com.vaadin.external.google</groupId>
-                <artifactId>android-json</artifactId>
-            </exclusion>
-        </exclusions>
+      <exclusions>
+        <exclusion>
+          <groupId>com.vaadin.external.google</groupId>
+          <artifactId>android-json</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
       <groupId>org.dbunit</groupId>
@@ -1231,12 +1233,13 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-        <groupId>com.github.mjeanroy</groupId>
-        <artifactId>dbunit-plus</artifactId>
-        <version>2.0.1</version>
-        <scope>test</scope>
+      <groupId>com.github.mjeanroy</groupId>
+      <artifactId>dbunit-plus</artifactId>
+      <version>2.0.1</version>
+      <scope>test</scope>
     </dependency>
   </dependencies>
+
   <profiles>
     <profile>
       <id>webapi-oracle</id>
@@ -1329,17 +1332,17 @@
           lower(email) = lower(?)</security.db.datasource.authenticationQuery>
       </properties>
       <repositories>
-	<repository>
+        <repository>
           <id>ohdsi.snapshots</id>
           <name>repo.ohdsi.org-snapshots</name>
           <url>https://repo.ohdsi.org/nexus/content/repositories/snapshots</url>
           <releases>
             <enabled>false</enabled>
           </releases>
-      	  <snapshots>
+          <snapshots>
             <enabled>true</enabled>
-      	  </snapshots>
-    	</repository>
+          </snapshots>
+        </repository>
       </repositories>
     </profile>
     <profile>
@@ -1412,7 +1415,7 @@
         <impala.enabled>true</impala.enabled>
         <impala.driver.version>2.6.15</impala.driver.version>
         <!-- Impala JDBC driver path -->
-		<impala.classpath>...path/to/impala/jdbc/drivers...</impala.classpath>
+        <impala.classpath>...path/to/impala/jdbc/drivers...</impala.classpath>
       </properties>
       <dependencies>
         <dependency>
@@ -1517,9 +1520,9 @@
           <version>v2-rev20220326-1.32.1</version>
         </dependency>
         <dependency>
-            <groupId>com.google.cloud</groupId>
-            <artifactId>google-cloud-bigquery</artifactId>
-            <version>1.2.15</version>
+          <groupId>com.google.cloud</groupId>
+          <artifactId>google-cloud-bigquery</artifactId>
+          <version>1.2.15</version>
         </dependency>
         <dependency>
           <groupId>com.google.http-client</groupId>

src/main/java/org/ohdsi/webapi/Constants.java

@@ -12,6 +12,7 @@ public interface Constants {
   String GENERATE_PREDICTION_ANALYSIS = "generatePredictionAnalysis";
   String GENERATE_ESTIMATION_ANALYSIS = "generateEstimationAnalysis";
   String WARM_CACHE = "warmCache";
+  String CLEAR_CACHE = "clearCache";
   String USERS_IMPORT = "usersImport";
   String JOB_IS_ALREADY_SCHEDULED = "Job for provider %s is already scheduled";
 

src/main/java/org/ohdsi/webapi/achilles/repository/AchillesCacheRepository.java

@@ -2,7 +2,7 @@
 
 import org.ohdsi.webapi.achilles.domain.AchillesCacheEntity;
 import org.ohdsi.webapi.source.Source;
-import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.CrudRepository;
 import org.springframework.data.repository.query.Param;
@@ -16,4 +16,8 @@ public interface AchillesCacheRepository extends CrudRepository<AchillesCacheEnt
 
     @Query("select ac from AchillesCacheEntity ac where source = :source and cacheName in :names")
     List<AchillesCacheEntity> findBySourceAndNames(@Param("source") Source source, @Param("names") List<String> names);
+
+    @Modifying
+    @Query("delete from AchillesCacheEntity where source = :source")
+    void deleteBySource(@Param("source") Source source);
 }

src/main/java/org/ohdsi/webapi/achilles/service/AchillesCacheService.java

@@ -85,6 +85,11 @@ public void saveDrilldownCacheMap(Source source, String domain, Map<Integer, Obj
         nodes.clear();
     }
 
+    @Transactional()
+    public void clearCache(Source source) {
+        cacheRepository.deleteBySource(source);
+    }
+
     private void createCacheEntities(Source source, Map<String, ObjectNode> nodes) {
         List<AchillesCacheEntity> cacheEntities = getEntities(source, nodes);
         cacheRepository.save(cacheEntities);

src/main/java/org/ohdsi/webapi/cdmresults/repository/CDMCacheRepository.java

@@ -1,6 +1,7 @@
 package org.ohdsi.webapi.cdmresults.repository;
 
 import org.ohdsi.webapi.cdmresults.domain.CDMCacheEntity;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.CrudRepository;
 import org.springframework.data.repository.query.Param;
@@ -12,4 +13,8 @@
 public interface CDMCacheRepository extends CrudRepository<CDMCacheEntity, Long> {
     @Query("select c from CDMCacheEntity c where c.sourceId = :sourceId and c.conceptId in :conceptIds")
     List<CDMCacheEntity> findBySourceAndConceptIds(@Param("sourceId") int sourceId, @Param("conceptIds") List<Integer> conceptIds);
+
+    @Modifying
+    @Query("delete from CDMCacheEntity c where c.sourceId = :sourceId")
+    void deleteBySource(@Param("sourceId") int sourceId);
 }

src/main/java/org/ohdsi/webapi/cdmresults/service/CDMCacheService.java

@@ -73,6 +73,10 @@ public void warm(Source source) {
     }
   }
 
+  public void clearCache(Source source) {
+    cdmCacheRepository.deleteBySource(source.getSourceId());
+  }
+
   public List<CDMCacheEntity> findAndCache(Source source, List<Integer> conceptIds) {
     if (CollectionUtils.isEmpty(conceptIds)) {
       return Collections.emptyList();

src/main/java/org/ohdsi/webapi/service/CDMResultsService.java

@@ -46,6 +46,7 @@
 import org.springframework.stereotype.Component;
 
 import javax.ws.rs.Consumes;
+import javax.ws.rs.ForbiddenException;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
@@ -283,6 +284,23 @@ public JobExecutionResource refreshCache(@PathParam("sourceKey") final String so
         return new JobExecutionResource();
     }
 
+    /**
+     * Clear the cdm_cache and achilles_cache for all sources
+     * 
+     * @summary Clear the cdm_cache and achilles_cache for all sources
+     * @return void
+     * @throws ForbiddenException if the user is not an admin
+     */
+    @POST
+    @Path("clearCache")
+    public void clearCache() {
+        if (!isSecured() || !isAdmin()) {
+            throw new ForbiddenException();
+        }
+        List<Source> sources = getSourceRepository().findAll();
+        sources.parallelStream().forEach(this::clearCache);
+    }
+
     /**
      * Queries for data density report for the given sourceKey
      * 
@@ -441,9 +459,7 @@ private JobExecutionResource warmCaches(Source source) {
 
         String jobName = getWarmCacheJobName(String.valueOf(source.getSourceId()), source.getSourceKey());
         List<Step> jobSteps = createCacheWarmingJobSteps(source, jobName);
-        SimpleJobBuilder builder = createJob(String.valueOf(source.getSourceId()),
-                source.getSourceKey(),
-                jobSteps);
+        SimpleJobBuilder builder = createJob(jobName, jobSteps);
         return runJob(source.getSourceKey(), source.getSourceId(), jobName, builder);
     }
 
@@ -488,9 +504,9 @@ private void warmCaches(Collection<Source> sources) {
 
             if (counter++ >= bucketSizes[bucketIndex] - 1) {
                 if (!allJobSteps.isEmpty()) {
-                    SimpleJobBuilder builder = createJob(sourceIds.stream().map(String::valueOf).collect(Collectors.joining(",")),
-                            String.join(",", sourceKeys),
-                            allJobSteps);
+                    String compositeJobName = getWarmCacheJobName(sourceIds.stream().map(String::valueOf)
+                            .collect(Collectors.joining(",")), String.join(",", sourceKeys));
+                    SimpleJobBuilder builder = createJob(compositeJobName, allJobSteps);
                     runJob(source.getSourceKey(), source.getSourceId(), jobName, builder);
                 }
 
@@ -503,9 +519,19 @@ private void warmCaches(Collection<Source> sources) {
         }
     }
 
-    private SimpleJobBuilder createJob(String sourceIds, String sourceKeys, List<Step> steps) {
+    /*
+     * Clear cache for a single source
+     */
+    private void clearCache(Source source) {
+        if (!sourceAccessor.hasAccess(source)) {
+            return;
+        }
+        cacheService.clearCache(source);
+        cdmCacheService.clearCache(source);
+    }
+
+    private SimpleJobBuilder createJob(String jobName, List<Step> steps) {
         final SimpleJobBuilder[] stepBuilder = {null};
-        String jobName = getWarmCacheJobName(sourceIds, sourceKeys);
         if (jobService.findJobByName(jobName, jobName) == null && !steps.isEmpty()) {
             JobBuilder jobBuilder = jobBuilders.get(jobName);
 
@@ -576,11 +602,15 @@ private int getResultsDaimonPriority(Source source) {
     }
 
     private String getWarmCacheJobName(String sourceIds, String sourceKeys) {
+        return getJobName("warming cache", sourceIds, sourceKeys);
+    }
+
+    private String getJobName(String jobType, String sourceIds, String sourceKeys) {
         // for multiple sources: try to compose a job name from source keys, and if it is too long - use source ids
-        String jobName = String.format("warming cache: %s", sourceKeys);
+        String jobName = String.format("%s: %s", jobType, sourceKeys);
 
         if (jobName.length() >= 100) { // job name in batch_job_instance is varchar(100)
-            jobName = String.format("warming cache: %s", sourceIds);
+            jobName = String.format("%s: %s", jobType, sourceIds);
 
             if (jobName.length() >= 100) { // if we still have more than 100 symbols
                 jobName = jobName.substring(0, 88);

src/test/java/org/ohdsi/webapi/test/CDMResultsServiceIT.java

@@ -17,6 +17,7 @@
 import org.ohdsi.circe.helper.ResourceHelper;
 import org.ohdsi.sql.SqlRender;
 import org.ohdsi.sql.SqlTranslate;
+import org.ohdsi.webapi.job.JobExecutionResource;
 import org.ohdsi.webapi.source.SourceRepository;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -28,6 +29,9 @@ public class CDMResultsServiceIT extends WebApiIT {
     @Value("${cdmResultsService.endpoint.conceptRecordCount}")
     private String conceptRecordCountEndpoint;
 
+    @Value("${cdmResultsService.endpoint.clearCache}")
+    private String clearCacheEndpoint;
+
     @Autowired
     private SourceRepository sourceRepository;
 
@@ -65,7 +69,7 @@ public void requestConceptRecordCounts_firstTime_returnsResults() {
         final ResponseEntity<List<LinkedHashMap<String, List<Integer>>>> entity = getRestTemplate().postForEntity(this.conceptRecordCountEndpoint, conceptIds, 
                 returnClass, queryParameters );
 
-        // Assertion
+        // Assert
         assertOK(entity);
         List<LinkedHashMap<String, List<Integer>>> results = entity.getBody();
         assertEquals(1, results.size());
@@ -78,4 +82,23 @@ public void requestConceptRecordCounts_firstTime_returnsResults() {
         assertEquals(102, counts.get(2).intValue());
         assertEquals(103, counts.get(3).intValue());
     }
+
+    @Test
+    public void clearCache_nothingInCache_returns() {
+
+        // Arrange
+        List<JobExecutionResource> list = new ArrayList<>();
+        @SuppressWarnings("unchecked")
+        Class<List<JobExecutionResource>> returnClass = (Class<List<JobExecutionResource>>) list
+                .getClass();
+
+        // Act
+        final ResponseEntity<List<JobExecutionResource>> entity = getRestTemplate().getForEntity(this.clearCacheEndpoint, returnClass);
+
+        // Assert
+        assertOK(entity);
+        // Right now we don't have security enabled in the test environment, so the cache is not cleared (there's a check that we're using security)
+        List<JobExecutionResource> results = entity.getBody();
+        assertEquals(0, results.size());
+    }
 }