WIP [HTM-1076][HTM-1077] TMFeatureType event handler to propagate cha…

…nges to solr indexes

src/main/java/nl/b3p/tailormap/api/controller/SearchController.java

@@ -12,7 +12,6 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.lang.invoke.MethodHandles;
-import java.util.concurrent.TimeUnit;
 import nl.b3p.tailormap.api.annotation.AppRestController;
 import nl.b3p.tailormap.api.persistence.Application;
 import nl.b3p.tailormap.api.persistence.GeoService;
@@ -21,10 +20,10 @@
 import nl.b3p.tailormap.api.persistence.json.GeoServiceLayer;
 import nl.b3p.tailormap.api.repository.SearchIndexRepository;
 import nl.b3p.tailormap.api.solr.SolrHelper;
+import nl.b3p.tailormap.api.solr.SolrService;
 import nl.b3p.tailormap.api.viewer.model.SearchResponse;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.common.SolrException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -50,17 +49,14 @@ public class SearchController {
 
   private final SearchIndexRepository searchIndexRepository;
 
-  @Value("${tailormap-api.solr-url}")
-  private String solrUrl;
-
-  @Value("${tailormap-api.solr-core-name:tailormap}")
-  private String solrCoreName;
-
   @Value("${tailormap-api.pageSize:100}")
   private int numResultsToReturn;
 
-  public SearchController(SearchIndexRepository searchIndexRepository) {
+  private final SolrService solrService;
+
+  public SearchController(SearchIndexRepository searchIndexRepository, SolrService solrService) {
     this.searchIndexRepository = searchIndexRepository;
+    this.solrService = solrService;
   }
 
   @Transactional(readOnly = true)
@@ -90,7 +86,7 @@ public ResponseEntity<Serializable> search(
                         "Layer '%s' does not have a search index"
                             .formatted(appTreeLayerNode.getLayerName())));
 
-    try (SolrClient solrClient = getSolrClient();
+    try (SolrClient solrClient = solrService.getSolrClientForSearching();
         SolrHelper solrHelper = new SolrHelper(solrClient)) {
       final SearchResponse searchResponse =
           solrHelper.findInIndex(searchIndex, solrQuery, start, numResultsToReturn);
@@ -107,11 +103,4 @@ public ResponseEntity<Serializable> search(
           HttpStatus.BAD_REQUEST, "Error while searching with given query", e);
     }
   }
-
-  private SolrClient getSolrClient() {
-    return new Http2SolrClient.Builder(solrUrl + solrCoreName)
-        .withConnectionTimeout(10, TimeUnit.SECONDS)
-        .withFollowRedirects(true)
-        .build();
-  }
 }

src/main/java/nl/b3p/tailormap/api/controller/admin/SolrAdminController.java

@@ -14,7 +14,6 @@
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
 import java.util.NoSuchElementException;
-import java.util.concurrent.TimeUnit;
 import nl.b3p.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper;
 import nl.b3p.tailormap.api.persistence.SearchIndex;
 import nl.b3p.tailormap.api.persistence.TMFeatureType;
@@ -23,11 +22,10 @@
 import nl.b3p.tailormap.api.repository.GeoServiceRepository;
 import nl.b3p.tailormap.api.repository.SearchIndexRepository;
 import nl.b3p.tailormap.api.solr.SolrHelper;
+import nl.b3p.tailormap.api.solr.SolrService;
 import nl.b3p.tailormap.api.viewer.model.ErrorResponse;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.client.solrj.response.SolrPingResponse;
 import org.apache.solr.common.SolrException;
 import org.slf4j.Logger;
@@ -61,18 +59,21 @@ public class SolrAdminController {
   private final FeatureTypeRepository featureTypeRepository;
   private final GeoServiceRepository geoServiceRepository;
   private final SearchIndexRepository searchIndexRepository;
+  private final SolrService solrService;
 
   public SolrAdminController(
       FeatureSourceFactoryHelper featureSourceFactoryHelper,
       FeatureSourceRepository featureSourceRepository,
       FeatureTypeRepository featureTypeRepository,
       GeoServiceRepository geoServiceRepository,
-      SearchIndexRepository searchIndexRepository) {
+      SearchIndexRepository searchIndexRepository,
+      SolrService solrService) {
     this.featureSourceFactoryHelper = featureSourceFactoryHelper;
     this.featureSourceRepository = featureSourceRepository;
     this.featureTypeRepository = featureTypeRepository;
     this.geoServiceRepository = geoServiceRepository;
     this.searchIndexRepository = searchIndexRepository;
+    this.solrService = solrService;
   }
 
   /**
@@ -99,7 +100,7 @@ public SolrAdminController(
       path = "${tailormap-api.admin.base-path}/index/ping",
       produces = MediaType.APPLICATION_JSON_VALUE)
   public ResponseEntity<?> pingSolr() {
-    try (SolrClient solrClient = getSolrClient()) {
+    try (SolrClient solrClient = solrService.getSolrClientForIndexing()) {
       final SolrPingResponse ping = solrClient.ping();
       logger.info("Solr ping status {}", ping.getResponse().get("status"));
       return ResponseEntity.ok(
@@ -118,24 +119,6 @@ public ResponseEntity<?> pingSolr() {
     }
   }
 
-  /**
-   * Get a concurrent update Solr client for bulk operations.
-   *
-   * @return the Solr client
-   */
-  private SolrClient getSolrClient() {
-    return new ConcurrentUpdateHttp2SolrClient.Builder(
-            solrUrl + solrCoreName,
-            new Http2SolrClient.Builder()
-                .withFollowRedirects(true)
-                .withConnectionTimeout(10000, TimeUnit.MILLISECONDS)
-                .withRequestTimeout(60000, TimeUnit.MILLISECONDS)
-                .build())
-        .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2)
-        .withThreadCount(10)
-        .build();
-  }
-
   /**
    * (re-) Index a layer.
    *
@@ -185,7 +168,7 @@ public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
     boolean createNewIndex =
         (null == searchIndex.getLastIndexed()
             || searchIndex.getStatus() == SearchIndex.Status.INITIAL);
-    try (SolrClient solrClient = getSolrClient();
+    try (SolrClient solrClient = solrService.getSolrClientForIndexing();
         SolrHelper solrHelper = new SolrHelper(solrClient)) {
       solrHelper.addFeatureTypeIndex(searchIndex, indexingFT, featureSourceFactoryHelper);
       searchIndexRepository.save(searchIndex);
@@ -234,7 +217,7 @@ public ResponseEntity<?> index(@PathVariable Long searchIndexId) {
       produces = MediaType.APPLICATION_JSON_VALUE)
   @Transactional
   public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
-    try (SolrClient solrClient = getSolrClient();
+    try (SolrClient solrClient = solrService.getSolrClientForIndexing();
         SolrHelper solrHelper = new SolrHelper(solrClient)) {
       solrHelper.clearIndexForLayer(searchIndexId);
       // do not delete the SearchIndex metadata object

src/main/java/nl/b3p/tailormap/api/repository/GeoServiceRepository.java

@@ -52,4 +52,17 @@ public interface GeoServiceRepository extends JpaRepository<GeoService, String>
   @PreAuthorize("permitAll()")
   @Query("from GeoService s where id not in :ids")
   List<GeoService> getAllExcludingIds(@Param("ids") List<String> ids);
+
+  /**
+   * Find all geo-services that have a layer that is linked to a specific (Solr) index.
+   *
+   * @param indexId The index id to search for
+   */
+  @NonNull
+  @PreAuthorize("permitAll()")
+  @Query(
+      value =
+          "select * from geo_service gs, lateral jsonb_path_query(gs.settings, ('$.layerSettings.**{1}.searchIndex.searchIndexId ? (@ == '||:indexId||')')::jsonpath)",
+      nativeQuery = true)
+  List<GeoService> findByIndexId(@Param("indexId") @NonNull Long indexId);
 }

src/main/java/nl/b3p/tailormap/api/repository/events/SolrTMFeatureTypeEventHandler.java

@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2024 B3Partners B.V.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+package nl.b3p.tailormap.api.repository.events;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import nl.b3p.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper;
+import nl.b3p.tailormap.api.persistence.SearchIndex;
+import nl.b3p.tailormap.api.persistence.TMFeatureType;
+import nl.b3p.tailormap.api.repository.GeoServiceRepository;
+import nl.b3p.tailormap.api.repository.SearchIndexRepository;
+import nl.b3p.tailormap.api.solr.SolrHelper;
+import nl.b3p.tailormap.api.solr.SolrService;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.common.SolrException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.data.rest.core.annotation.HandleAfterDelete;
+import org.springframework.data.rest.core.annotation.HandleBeforeSave;
+import org.springframework.data.rest.core.annotation.RepositoryEventHandler;
+
+/** Event handler for Solr indexes when a {@code TMFeatureType} is updated or deleted. */
+@RepositoryEventHandler
+public class SolrTMFeatureTypeEventHandler {
+
+  private static final Logger logger =
+      LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private final SearchIndexRepository searchIndexRepository;
+  private final SolrService solrService;
+  private final FeatureSourceFactoryHelper featureSourceFactoryHelper;
+  private final GeoServiceRepository geoServiceRepository;
+
+  public SolrTMFeatureTypeEventHandler(
+      SearchIndexRepository searchIndexRepository,
+      SolrService solrService,
+      FeatureSourceFactoryHelper featureSourceFactoryHelper,
+      GeoServiceRepository geoServiceRepository) {
+    this.searchIndexRepository = searchIndexRepository;
+    this.solrService = solrService;
+    this.featureSourceFactoryHelper = featureSourceFactoryHelper;
+    this.geoServiceRepository = geoServiceRepository;
+  }
+
+  /**
+   * Handle the update of a TMFeatureType.
+   *
+   * @param tmFeatureType the TMFeatureType to handle
+   */
+  @HandleBeforeSave
+  public void handleTMFeatureTypeUpdate(TMFeatureType tmFeatureType) {
+    logger.debug("Handling TMFeatureType save event for: {}", tmFeatureType);
+    // determine if it is a new FT or an update
+    if (null == tmFeatureType.getId()) {
+      // do nothing as there is no index defined for a new feature type
+      logger.debug("New TMFeatureType: {}", tmFeatureType);
+    } else {
+      logger.debug("Updated TMFeatureType: {}", tmFeatureType);
+      searchIndexRepository.findByFeatureTypeId(tmFeatureType.getId()).stream()
+          .findAny()
+          .ifPresent(
+              searchIndex -> {
+                logger.debug(
+                    "Updating search index {} for feature type: {}",
+                    searchIndex.getName(),
+                    searchIndex);
+
+                try (SolrHelper solrHelper =
+                    new SolrHelper(solrService.getSolrClientForIndexing())) {
+                  solrHelper.addFeatureTypeIndex(
+                      searchIndex, tmFeatureType, featureSourceFactoryHelper);
+                } catch (UnsupportedOperationException
+                    | IOException
+                    | SolrServerException
+                    | SolrException e) {
+                  logger.error("Error re-indexing", e);
+                  searchIndex.setStatus(SearchIndex.Status.ERROR);
+                  searchIndexRepository.save(searchIndex);
+                }
+              });
+    }
+  }
+
+  /**
+   * Handle the deletion of a TMFeatureType.
+   *
+   * @param tmFeatureType the TMFeatureType to handle
+   */
+  @HandleAfterDelete
+  public void handleTMFeatureTypeDeleteForSolr(TMFeatureType tmFeatureType) {
+    logger.debug("Handling TMFeatureType delete event for: {}", tmFeatureType);
+    searchIndexRepository.findByFeatureTypeId(tmFeatureType.getId()).stream()
+        .findAny()
+        .ifPresent(
+            searchIndex -> {
+              logger.info(
+                  "Deleting search index {} for feature type: {}",
+                  searchIndex.getName(),
+                  searchIndex);
+
+              try (SolrHelper solrHelper = new SolrHelper(solrService.getSolrClientForIndexing())) {
+                solrHelper.clearIndexForLayer(searchIndex.getId());
+                searchIndexRepository.delete(searchIndex);
+                // find layers that use this index in the layer settings and clear them
+                geoServiceRepository
+                    .findByIndexId(searchIndex.getId())
+                    .forEach(
+                        geoService ->
+                            geoService
+                                .getLayers()
+                                .forEach(
+                                    layer -> {
+                                      logger.debug(
+                                          "Checking layer {} for search index {}",
+                                          layer.getName(),
+                                          searchIndex.getName());
+                                      geoService
+                                          .findSearchIndexForLayer(layer, searchIndexRepository)
+                                          .ifPresent(
+                                              searchIndex1 -> {
+                                                if (searchIndex1
+                                                    .getId()
+                                                    .equals(searchIndex.getId())) {
+                                                  logger.debug(
+                                                      "Clearing search index for layer {}",
+                                                      layer.getName());
+                                                  geoService
+                                                      .getLayerSettings(layer.getName())
+                                                      .setSearchIndex(null);
+                                                  geoServiceRepository.save(geoService);
+                                                }
+                                              });
+                                    }));
+              } catch (UnsupportedOperationException
+                  | IOException
+                  | SolrServerException
+                  | SolrException e) {
+                logger.error("Error deleting index for {}", searchIndex, e);
+              }
+            });
+  }
+}

src/main/java/nl/b3p/tailormap/api/solr/SolrHelper.java

@@ -60,7 +60,7 @@ public class SolrHelper implements AutoCloseable, Constants {
   /**
    * Constructor
    *
-   * @param solrClient the Solr client, this will be closed when this class is closed
+   * @param solrClient the Solr client, this client will be closed when this class is closed.
    */
   public SolrHelper(@NotNull SolrClient solrClient) {
     this.solrClient = solrClient;

src/main/java/nl/b3p/tailormap/api/solr/SolrService.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2024 B3Partners B.V.
+ *
+ * SPDX-License-Identifier: MIT
+ */
+package nl.b3p.tailormap.api.solr;
+
+import java.util.concurrent.TimeUnit;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+@Service
+public class SolrService {
+  @Value("${tailormap-api.solr-url}")
+  private String solrUrl;
+
+  @Value("${tailormap-api.solr-core-name:tailormap}")
+  private String solrCoreName;
+
+  /**
+   * Get a concurrent update Solr client for bulk operations.
+   *
+   * @return the Solr client
+   */
+  public SolrClient getSolrClientForIndexing() {
+    return new ConcurrentUpdateHttp2SolrClient.Builder(
+            solrUrl + solrCoreName,
+            new Http2SolrClient.Builder()
+                .withFollowRedirects(true)
+                .withConnectionTimeout(10000, TimeUnit.MILLISECONDS)
+                .withRequestTimeout(60000, TimeUnit.MILLISECONDS)
+                .build())
+        .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2)
+        .withThreadCount(10)
+        .build();
+  }
+
+  /**
+   * Get a Solr client for searching.
+   *
+   * @return the Solr client
+   */
+  public SolrClient getSolrClientForSearching() {
+    return new Http2SolrClient.Builder(solrUrl + solrCoreName)
+        .withConnectionTimeout(10, TimeUnit.SECONDS)
+        .withFollowRedirects(true)
+        .build();
+  }
+}