From 8e3d842e8734e4ad3c48f025d1b1ce396dabd4b1 Mon Sep 17 00:00:00 2001 From: Mark Prins <1165786+mprins@users.noreply.github.com> Date: Tue, 11 Jun 2024 17:08:49 +0200 Subject: [PATCH] WIP [HTM-1076][HTM-1077] TMFeatureType event handler to propagate changes to solr indexes fix PMD style fixup after API changes in https://github.com/B3Partners/tailormap-api/pull/876 move testcases and remove now useless test, use solr service inject fix rebase issues --- .../configuration/dev/PopulateTestData.java | 33 ++--- .../api/controller/SearchController.java | 23 +-- .../controller/admin/SolrAdminController.java | 43 ++---- .../api/repository/ApplicationRepository.java | 16 +++ .../events/SolrTMFeatureTypeEventHandler.java | 136 ++++++++++++++++++ .../org/tailormap/api/solr/SolrHelper.java | 2 +- .../org/tailormap/api/solr/SolrService.java | 56 ++++++++ .../ApplicationRepositoryIntegrationTest.java | 15 ++ 8 files changed, 247 insertions(+), 77 deletions(-) create mode 100644 src/main/java/org/tailormap/api/repository/events/SolrTMFeatureTypeEventHandler.java create mode 100644 src/main/java/org/tailormap/api/solr/SolrService.java diff --git a/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java b/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java index ce2460e769..fd3037fda8 100644 --- a/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java +++ b/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java @@ -19,9 +19,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -77,6 +74,7 @@ import org.tailormap.api.repository.UserRepository; import org.tailormap.api.security.InternalAdminAuthentication; import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; import org.tailormap.api.viewer.model.AppStyling; import org.tailormap.api.viewer.model.Component; import org.tailormap.api.viewer.model.ComponentConfig; @@ -107,16 +105,13 @@ public class PopulateTestData { @Value("${MAP5_URL:#{null}}") private String map5url; - @Value("${tailormap-api.solr-core-name:tailormap}") - private String solrCoreName; - private final ApplicationContext appContext; private final UserRepository userRepository; private final GroupRepository groupRepository; private final CatalogRepository catalogRepository; private final GeoServiceRepository geoServiceRepository; private final GeoServiceHelper geoServiceHelper; - + private final SolrService solrService; private final FeatureSourceRepository featureSourceRepository; private final ApplicationRepository applicationRepository; private final ConfigurationRepository configurationRepository; @@ -136,6 +131,7 @@ public PopulateTestData( ConfigurationRepository configurationRepository, FeatureSourceFactoryHelper featureSourceFactoryHelper, SearchIndexRepository searchIndexRepository, + SolrService solrService, UploadRepository uploadRepository) { this.appContext = appContext; this.userRepository = userRepository; @@ -148,6 +144,7 @@ public PopulateTestData( this.configurationRepository = configurationRepository; this.featureSourceFactoryHelper = featureSourceFactoryHelper; this.searchIndexRepository = searchIndexRepository; + this.solrService = solrService; this.uploadRepository = uploadRepository; } @@ -169,7 +166,7 @@ public void populate() throws Exception { InternalAdminAuthentication.clearSecurityContextAuthentication(); } if (exit) { - // Exit after transaction is completed - for 'mvn verify' to populate testdata before + // Exit after transaction is completed — for 'mvn verify' to populate testdata before // integration tests new Thread( () -> { @@ -1344,23 +1341,9 @@ public void createSolrIndex() throws Exception { logger.info("Creating Solr index"); @SuppressWarnings("PMD.AvoidUsingHardCodedIP") final String solrUrl = - "http://" - + (connectToSpatialDbsAtLocalhost ? "127.0.0.1" : "solr") - + ":8983/solr/" - + solrCoreName; - SolrHelper solrHelper = - new SolrHelper( - new ConcurrentUpdateHttp2SolrClient.Builder( - solrUrl, - new Http2SolrClient.Builder() - .useHttp1_1(true) - .withFollowRedirects(true) - .withConnectionTimeout(10000, TimeUnit.MILLISECONDS) - .withRequestTimeout(60000, TimeUnit.MILLISECONDS) - .build()) - .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2) - .withThreadCount(10) - .build()); + "http://" + (connectToSpatialDbsAtLocalhost ? "127.0.0.1" : "solr") + ":8983/solr/"; + this.solrService.setSolrUrl(solrUrl); + SolrHelper solrHelper = new SolrHelper(this.solrService.getSolrClientForIndexing()); GeoService geoService = geoServiceRepository.findById("snapshot-geoserver").orElseThrow(); Application defaultApp = applicationRepository.findByName("default"); diff --git a/src/main/java/org/tailormap/api/controller/SearchController.java b/src/main/java/org/tailormap/api/controller/SearchController.java index b3c1c5fd83..3465c97d77 100644 --- a/src/main/java/org/tailormap/api/controller/SearchController.java +++ b/src/main/java/org/tailormap/api/controller/SearchController.java @@ -12,10 +12,8 @@ import java.io.IOException; import java.io.Serializable; import java.lang.invoke.MethodHandles; -import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +35,7 @@ import org.tailormap.api.persistence.json.AppTreeLayerNode; import org.tailormap.api.repository.SearchIndexRepository; import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; import org.tailormap.api.viewer.model.SearchResponse; @AppRestController @@ -50,17 +49,14 @@ public class SearchController { private final SearchIndexRepository searchIndexRepository; - @Value("${tailormap-api.solr-url}") - private String solrUrl; - - @Value("${tailormap-api.solr-core-name:tailormap}") - private String solrCoreName; - @Value("${tailormap-api.pageSize:100}") private int numResultsToReturn; - public SearchController(SearchIndexRepository searchIndexRepository) { + private final SolrService solrService; + + public SearchController(SearchIndexRepository searchIndexRepository, SolrService solrService) { this.searchIndexRepository = searchIndexRepository; + this.solrService = solrService; } @Transactional(readOnly = true) @@ -92,7 +88,7 @@ public ResponseEntity search( "Layer '%s' does not have a search index" .formatted(appTreeLayerNode.getLayerName()))); - try (SolrClient solrClient = getSolrClient(); + try (SolrClient solrClient = solrService.getSolrClientForSearching(); SolrHelper solrHelper = new SolrHelper(solrClient)) { final SearchResponse searchResponse = solrHelper.findInIndex(searchIndex, solrQuery, start, numResultsToReturn); @@ -109,11 +105,4 @@ public ResponseEntity search( HttpStatus.BAD_REQUEST, "Error while searching with given query", e); } } - - private SolrClient getSolrClient() { - return new Http2SolrClient.Builder(solrUrl + solrCoreName) - .withConnectionTimeout(10, TimeUnit.SECONDS) - .withFollowRedirects(true) - .build(); - } } diff --git a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java index 8bb4515a7d..978a29cecb 100644 --- a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java +++ b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java @@ -14,16 +14,12 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.NoSuchElementException; -import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -40,37 +36,34 @@ import org.tailormap.api.repository.FeatureTypeRepository; import org.tailormap.api.repository.SearchIndexRepository; import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; import org.tailormap.api.viewer.model.ErrorResponse; /** Admin controller for Solr. */ @RestController public class SolrAdminController { - @Value("${tailormap-api.solr-url}") - private String solrUrl; - - @Value("${tailormap-api.solr-core-name:tailormap}") - private String solrCoreName; - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final FeatureSourceFactoryHelper featureSourceFactoryHelper; - private final FeatureTypeRepository featureTypeRepository; private final SearchIndexRepository searchIndexRepository; + private final SolrService solrService; public SolrAdminController( FeatureSourceFactoryHelper featureSourceFactoryHelper, FeatureTypeRepository featureTypeRepository, - SearchIndexRepository searchIndexRepository) { + SearchIndexRepository searchIndexRepository, + SolrService solrService) { this.featureSourceFactoryHelper = featureSourceFactoryHelper; this.featureTypeRepository = featureTypeRepository; this.searchIndexRepository = searchIndexRepository; + this.solrService = solrService; } /** * Ping solr. * - * @return the response entity (ok or an error response) + * @return the response entity (OK or an error response) */ @Operation(summary = "Ping Solr", description = "Ping Solr to check if it is available") @ApiResponse( @@ -91,7 +84,7 @@ public SolrAdminController( path = "${tailormap-api.admin.base-path}/index/ping", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity pingSolr() { - try (SolrClient solrClient = getSolrClient()) { + try (SolrClient solrClient = solrService.getSolrClientForIndexing()) { final SolrPingResponse ping = solrClient.ping(); logger.info("Solr ping status {}", ping.getResponse().get("status")); return ResponseEntity.ok( @@ -110,24 +103,6 @@ public ResponseEntity pingSolr() { } } - /** - * Get a concurrent update Solr client for bulk operations. - * - * @return the Solr client - */ - private SolrClient getSolrClient() { - return new ConcurrentUpdateHttp2SolrClient.Builder( - solrUrl + solrCoreName, - new Http2SolrClient.Builder() - .withFollowRedirects(true) - .withConnectionTimeout(10000, TimeUnit.MILLISECONDS) - .withRequestTimeout(60000, TimeUnit.MILLISECONDS) - .build()) - .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2) - .withThreadCount(10) - .build(); - } - /** * (re-) Index a layer. * @@ -177,7 +152,7 @@ public ResponseEntity index(@PathVariable Long searchIndexId) { boolean createNewIndex = (null == searchIndex.getLastIndexed() || searchIndex.getStatus() == SearchIndex.Status.INITIAL); - try (SolrClient solrClient = getSolrClient(); + try (SolrClient solrClient = solrService.getSolrClientForIndexing(); SolrHelper solrHelper = new SolrHelper(solrClient)) { solrHelper.addFeatureTypeIndex(searchIndex, indexingFT, featureSourceFactoryHelper); searchIndexRepository.save(searchIndex); @@ -226,7 +201,7 @@ public ResponseEntity index(@PathVariable Long searchIndexId) { produces = MediaType.APPLICATION_JSON_VALUE) @Transactional public ResponseEntity clearIndex(@PathVariable Long searchIndexId) { - try (SolrClient solrClient = getSolrClient(); + try (SolrClient solrClient = solrService.getSolrClientForIndexing(); SolrHelper solrHelper = new SolrHelper(solrClient)) { solrHelper.clearIndexForLayer(searchIndexId); // do not delete the SearchIndex metadata object diff --git a/src/main/java/org/tailormap/api/repository/ApplicationRepository.java b/src/main/java/org/tailormap/api/repository/ApplicationRepository.java index 7fe9fc88eb..e50bb30334 100644 --- a/src/main/java/org/tailormap/api/repository/ApplicationRepository.java +++ b/src/main/java/org/tailormap/api/repository/ApplicationRepository.java @@ -5,8 +5,11 @@ */ package org.tailormap.api.repository; +import java.util.List; import java.util.Optional; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.lang.NonNull; import org.springframework.security.access.prepost.PreAuthorize; import org.tailormap.api.persistence.Application; @@ -21,4 +24,17 @@ public interface ApplicationRepository extends JpaRepository @Override @NonNull Optional findById(@NonNull Long aLong); + + /** + * Find all applications that have a layer that is linked to a specific (Solr) index. + * + * @param indexId The index id to search for + */ + @NonNull + @PreAuthorize("permitAll()") + @Query( + value = + "select * from application app, lateral jsonb_path_query(app.settings, ('$.layerSettings.**{1}.searchIndexId ? (@ == '||:indexId||')')::jsonpath)", + nativeQuery = true) + List findByIndexId(@Param("indexId") @NonNull Long indexId); } diff --git a/src/main/java/org/tailormap/api/repository/events/SolrTMFeatureTypeEventHandler.java b/src/main/java/org/tailormap/api/repository/events/SolrTMFeatureTypeEventHandler.java new file mode 100644 index 0000000000..43cc9a291b --- /dev/null +++ b/src/main/java/org/tailormap/api/repository/events/SolrTMFeatureTypeEventHandler.java @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2024 B3Partners B.V. + * + * SPDX-License-Identifier: MIT + */ +package org.tailormap.api.repository.events; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.rest.core.annotation.HandleAfterDelete; +import org.springframework.data.rest.core.annotation.HandleBeforeSave; +import org.springframework.data.rest.core.annotation.RepositoryEventHandler; +import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper; +import org.tailormap.api.persistence.SearchIndex; +import org.tailormap.api.persistence.TMFeatureType; +import org.tailormap.api.persistence.json.AppLayerSettings; +import org.tailormap.api.repository.ApplicationRepository; +import org.tailormap.api.repository.SearchIndexRepository; +import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; + +/** Event handler for Solr indexes when a {@code TMFeatureType} is updated or deleted. */ +@RepositoryEventHandler +public class SolrTMFeatureTypeEventHandler { + + private static final Logger logger = + LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final SearchIndexRepository searchIndexRepository; + private final SolrService solrService; + private final FeatureSourceFactoryHelper featureSourceFactoryHelper; + private final ApplicationRepository applicationRepository; + + public SolrTMFeatureTypeEventHandler( + SearchIndexRepository searchIndexRepository, + SolrService solrService, + FeatureSourceFactoryHelper featureSourceFactoryHelper, + ApplicationRepository applicationRepository) { + this.searchIndexRepository = searchIndexRepository; + this.solrService = solrService; + this.featureSourceFactoryHelper = featureSourceFactoryHelper; + this.applicationRepository = applicationRepository; + } + + /** + * Handle the update of a TMFeatureType. + * + * @param tmFeatureType the TMFeatureType to handle + */ + @HandleBeforeSave + public void handleTMFeatureTypeUpdate(TMFeatureType tmFeatureType) { + logger.debug("Handling TMFeatureType save event for: {}", tmFeatureType); + // determine if it is a new FT or an update + if (null == tmFeatureType.getId()) { + // do nothing as there is no index defined for a new feature type + logger.debug("New TMFeatureType: {}", tmFeatureType); + } else { + logger.debug("Updated TMFeatureType: {}", tmFeatureType); + searchIndexRepository.findByFeatureTypeId(tmFeatureType.getId()).stream() + .findAny() + .ifPresent( + searchIndex -> { + logger.debug( + "Updating search index {} for feature type: {}", + searchIndex.getName(), + searchIndex); + + try (SolrHelper solrHelper = + new SolrHelper(solrService.getSolrClientForIndexing())) { + solrHelper.addFeatureTypeIndex( + searchIndex, tmFeatureType, featureSourceFactoryHelper); + } catch (UnsupportedOperationException + | IOException + | SolrServerException + | SolrException e) { + logger.error("Error re-indexing", e); + searchIndex.setStatus(SearchIndex.Status.ERROR); + searchIndexRepository.save(searchIndex); + } + }); + } + } + + /** + * Handle the deletion of a TMFeatureType. + * + * @param tmFeatureType the TMFeatureType to handle + */ + @HandleAfterDelete + public void handleTMFeatureTypeDeleteForSolr(TMFeatureType tmFeatureType) { + logger.debug("Handling TMFeatureType delete event for: {}", tmFeatureType); + searchIndexRepository.findByFeatureTypeId(tmFeatureType.getId()).stream() + .findAny() + .ifPresent( + searchIndex -> { + logger.info( + "Deleting search index {} for feature type: {}", + searchIndex.getName(), + searchIndex); + + try (SolrHelper solrHelper = new SolrHelper(solrService.getSolrClientForIndexing())) { + solrHelper.clearIndexForLayer(searchIndex.getId()); + searchIndexRepository.delete(searchIndex); + // find any application layers that use this index clear the index from them + applicationRepository + .findByIndexId(searchIndex.getId()) + .forEach( + application -> { + application + .getAllAppTreeLayerNode() + .forEach( + appTreeLayerNode -> { + AppLayerSettings appLayerSettings = + application.getAppLayerSettings(appTreeLayerNode); + if (null != appLayerSettings.getSearchIndexId() + && appLayerSettings + .getSearchIndexId() + .equals(searchIndex.getId())) { + appLayerSettings.setSearchIndexId(null); + } + }); + applicationRepository.save(application); + }); + } catch (UnsupportedOperationException + | IOException + | SolrServerException + | SolrException e) { + logger.error("Error deleting index for {}", searchIndex, e); + } + }); + } +} diff --git a/src/main/java/org/tailormap/api/solr/SolrHelper.java b/src/main/java/org/tailormap/api/solr/SolrHelper.java index 98c09611b4..2cb6b979a0 100644 --- a/src/main/java/org/tailormap/api/solr/SolrHelper.java +++ b/src/main/java/org/tailormap/api/solr/SolrHelper.java @@ -60,7 +60,7 @@ public class SolrHelper implements AutoCloseable, Constants { /** * Constructor * - * @param solrClient the Solr client, this will be closed when this class is closed + * @param solrClient the Solr client, this client will be closed when this class is closed. */ public SolrHelper(@NotNull SolrClient solrClient) { this.solrClient = solrClient; diff --git a/src/main/java/org/tailormap/api/solr/SolrService.java b/src/main/java/org/tailormap/api/solr/SolrService.java new file mode 100644 index 0000000000..ee8f94b7a9 --- /dev/null +++ b/src/main/java/org/tailormap/api/solr/SolrService.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 B3Partners B.V. + * + * SPDX-License-Identifier: MIT + */ +package org.tailormap.api.solr; + +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Service +public class SolrService { + @Value("${tailormap-api.solr-url}") + private String solrUrl; + + @Value("${tailormap-api.solr-core-name:tailormap}") + private String solrCoreName; + + /** + * Get a concurrent update Solr client for bulk operations. + * + * @return the Solr client + */ + public SolrClient getSolrClientForIndexing() { + return new ConcurrentUpdateHttp2SolrClient.Builder( + this.solrUrl + this.solrCoreName, + new Http2SolrClient.Builder() + .withFollowRedirects(true) + .withConnectionTimeout(10000, TimeUnit.MILLISECONDS) + .withRequestTimeout(60000, TimeUnit.MILLISECONDS) + .build()) + .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2) + .withThreadCount(10) + .build(); + } + + /** + * Get a Solr client for searching. + * + * @return the Solr client + */ + public SolrClient getSolrClientForSearching() { + return new Http2SolrClient.Builder(this.solrUrl + this.solrCoreName) + .withConnectionTimeout(10, TimeUnit.SECONDS) + .withFollowRedirects(true) + .build(); + } + + public void setSolrUrl(String solrUrl) { + this.solrUrl = solrUrl; + } +} diff --git a/src/test/java/org/tailormap/api/repository/ApplicationRepositoryIntegrationTest.java b/src/test/java/org/tailormap/api/repository/ApplicationRepositoryIntegrationTest.java index 6fb458e51f..306a36f833 100644 --- a/src/test/java/org/tailormap/api/repository/ApplicationRepositoryIntegrationTest.java +++ b/src/test/java/org/tailormap/api/repository/ApplicationRepositoryIntegrationTest.java @@ -8,7 +8,9 @@ import static org.hibernate.validator.internal.util.Contracts.assertNotNull; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.List; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.tailormap.api.annotation.PostgresIntegrationTest; @@ -30,4 +32,17 @@ void should_not_find_application_by_nonexistent_name() { Application a = applicationRepository.findByName("does-not-exist"); assertNull(a); } + + @Test + void it_should_find_application_using_findByIndexId_with_valid_ID() { + final Application application = applicationRepository.findByIndexId(2L).get(0); + assertNotNull(application, "application should not be null"); + assertEquals("default", application.getName(), "application name is incorrect"); + } + + @Test + void it_should_not_find_applications_findByIndexId_with_invalid_ID() { + final List applications = applicationRepository.findByIndexId(-2L); + assertTrue(applications.isEmpty(), "applications should be empty"); + } }