From 6af4b90d24a6fe45599464e10d30f3e30362a45f Mon Sep 17 00:00:00 2001 From: mark <1165786+mprins@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:06:32 +0200 Subject: [PATCH] Introduce the SolrService from PR #844 --- .../configuration/dev/PopulateTestData.java | 31 +++------- .../api/controller/SearchController.java | 22 ++------ .../controller/admin/SolrAdminController.java | 40 +++---------- .../org/tailormap/api/solr/SolrHelper.java | 4 +- .../org/tailormap/api/solr/SolrService.java | 56 +++++++++++++++++++ 5 files changed, 78 insertions(+), 75 deletions(-) create mode 100644 src/main/java/org/tailormap/api/solr/SolrService.java diff --git a/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java b/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java index ce2460e769..434f7583e1 100644 --- a/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java +++ b/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java @@ -19,9 +19,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -77,6 +74,7 @@ import org.tailormap.api.repository.UserRepository; import org.tailormap.api.security.InternalAdminAuthentication; import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; import org.tailormap.api.viewer.model.AppStyling; import org.tailormap.api.viewer.model.Component; import org.tailormap.api.viewer.model.ComponentConfig; @@ -107,15 +105,13 @@ public class PopulateTestData { @Value("${MAP5_URL:#{null}}") private String map5url; - @Value("${tailormap-api.solr-core-name:tailormap}") - private String solrCoreName; - private final ApplicationContext appContext; private final UserRepository userRepository; private final GroupRepository groupRepository; private final CatalogRepository catalogRepository; private final GeoServiceRepository geoServiceRepository; private final GeoServiceHelper geoServiceHelper; + private final SolrService solrService; private final FeatureSourceRepository featureSourceRepository; private final ApplicationRepository applicationRepository; @@ -131,6 +127,7 @@ public PopulateTestData( CatalogRepository catalogRepository, GeoServiceRepository geoServiceRepository, GeoServiceHelper geoServiceHelper, + SolrService solrService, FeatureSourceRepository featureSourceRepository, ApplicationRepository applicationRepository, ConfigurationRepository configurationRepository, @@ -143,6 +140,7 @@ public PopulateTestData( this.catalogRepository = catalogRepository; this.geoServiceRepository = geoServiceRepository; this.geoServiceHelper = geoServiceHelper; + this.solrService = solrService; this.featureSourceRepository = featureSourceRepository; this.applicationRepository = applicationRepository; this.configurationRepository = configurationRepository; @@ -1344,24 +1342,9 @@ public void createSolrIndex() throws Exception { logger.info("Creating Solr index"); @SuppressWarnings("PMD.AvoidUsingHardCodedIP") final String solrUrl = - "http://" - + (connectToSpatialDbsAtLocalhost ? "127.0.0.1" : "solr") - + ":8983/solr/" - + solrCoreName; - SolrHelper solrHelper = - new SolrHelper( - new ConcurrentUpdateHttp2SolrClient.Builder( - solrUrl, - new Http2SolrClient.Builder() - .useHttp1_1(true) - .withFollowRedirects(true) - .withConnectionTimeout(10000, TimeUnit.MILLISECONDS) - .withRequestTimeout(60000, TimeUnit.MILLISECONDS) - .build()) - .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2) - .withThreadCount(10) - .build()); - + "http://" + (connectToSpatialDbsAtLocalhost ? "127.0.0.1" : "solr") + ":8983/solr/"; + this.solrService.setSolrUrl(solrUrl); + SolrHelper solrHelper = new SolrHelper(this.solrService.getSolrClientForIndexing()); GeoService geoService = geoServiceRepository.findById("snapshot-geoserver").orElseThrow(); Application defaultApp = applicationRepository.findByName("default"); diff --git a/src/main/java/org/tailormap/api/controller/SearchController.java b/src/main/java/org/tailormap/api/controller/SearchController.java index b3c1c5fd83..cdb5181d77 100644 --- a/src/main/java/org/tailormap/api/controller/SearchController.java +++ b/src/main/java/org/tailormap/api/controller/SearchController.java @@ -12,10 +12,8 @@ import java.io.IOException; import java.io.Serializable; import java.lang.invoke.MethodHandles; -import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,6 +35,7 @@ import org.tailormap.api.persistence.json.AppTreeLayerNode; import org.tailormap.api.repository.SearchIndexRepository; import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; import org.tailormap.api.viewer.model.SearchResponse; @AppRestController @@ -49,18 +48,14 @@ public class SearchController { LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final SearchIndexRepository searchIndexRepository; - - @Value("${tailormap-api.solr-url}") - private String solrUrl; - - @Value("${tailormap-api.solr-core-name:tailormap}") - private String solrCoreName; + private final SolrService solrService; @Value("${tailormap-api.pageSize:100}") private int numResultsToReturn; - public SearchController(SearchIndexRepository searchIndexRepository) { + public SearchController(SearchIndexRepository searchIndexRepository, SolrService solrService) { this.searchIndexRepository = searchIndexRepository; + this.solrService = solrService; } @Transactional(readOnly = true) @@ -92,7 +87,7 @@ public ResponseEntity search( "Layer '%s' does not have a search index" .formatted(appTreeLayerNode.getLayerName()))); - try (SolrClient solrClient = getSolrClient(); + try (SolrClient solrClient = solrService.getSolrClientForSearching(); SolrHelper solrHelper = new SolrHelper(solrClient)) { final SearchResponse searchResponse = solrHelper.findInIndex(searchIndex, solrQuery, start, numResultsToReturn); @@ -109,11 +104,4 @@ public ResponseEntity search( HttpStatus.BAD_REQUEST, "Error while searching with given query", e); } } - - private SolrClient getSolrClient() { - return new Http2SolrClient.Builder(solrUrl + solrCoreName) - .withConnectionTimeout(10, TimeUnit.SECONDS) - .withFollowRedirects(true) - .build(); - } } diff --git a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java index 8614516d41..66441eef57 100644 --- a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java +++ b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java @@ -14,16 +14,12 @@ import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.NoSuchElementException; -import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; -import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.common.SolrException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -41,31 +37,29 @@ import org.tailormap.api.repository.FeatureTypeRepository; import org.tailormap.api.repository.SearchIndexRepository; import org.tailormap.api.solr.SolrHelper; +import org.tailormap.api.solr.SolrService; import org.tailormap.api.viewer.model.ErrorResponse; /** Admin controller for Solr. */ @RestController public class SolrAdminController { - @Value("${tailormap-api.solr-url}") - private String solrUrl; - - @Value("${tailormap-api.solr-core-name:tailormap}") - private String solrCoreName; - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private final FeatureSourceFactoryHelper featureSourceFactoryHelper; private final FeatureTypeRepository featureTypeRepository; private final SearchIndexRepository searchIndexRepository; + private final SolrService solrService; public SolrAdminController( FeatureSourceFactoryHelper featureSourceFactoryHelper, FeatureTypeRepository featureTypeRepository, - SearchIndexRepository searchIndexRepository) { + SearchIndexRepository searchIndexRepository, + SolrService solrService) { this.featureSourceFactoryHelper = featureSourceFactoryHelper; this.featureTypeRepository = featureTypeRepository; this.searchIndexRepository = searchIndexRepository; + this.solrService = solrService; } /** @@ -92,7 +86,7 @@ public SolrAdminController( path = "${tailormap-api.admin.base-path}/index/ping", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity pingSolr() { - try (SolrClient solrClient = getSolrClient()) { + try (SolrClient solrClient = solrService.getSolrClientForSearching()) { final SolrPingResponse ping = solrClient.ping(); logger.info("Solr ping status {}", ping.getResponse().get("status")); return ResponseEntity.ok( @@ -111,24 +105,6 @@ public ResponseEntity pingSolr() { } } - /** - * Get a concurrent update Solr client for bulk operations. - * - * @return the Solr client - */ - private SolrClient getSolrClient() { - return new ConcurrentUpdateHttp2SolrClient.Builder( - solrUrl + solrCoreName, - new Http2SolrClient.Builder() - .withFollowRedirects(true) - .withConnectionTimeout(10000, TimeUnit.MILLISECONDS) - .withRequestTimeout(60000, TimeUnit.MILLISECONDS) - .build()) - .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2) - .withThreadCount(10) - .build(); - } - /** * (re-) Index a layer. * @@ -195,7 +171,7 @@ public ResponseEntity index(@PathVariable Long searchIndexId) { boolean createNewIndex = (null == searchIndex.getLastIndexed() || searchIndex.getStatus() == SearchIndex.Status.INITIAL); - try (SolrClient solrClient = getSolrClient(); + try (SolrClient solrClient = solrService.getSolrClientForIndexing(); SolrHelper solrHelper = new SolrHelper(solrClient)) { solrHelper.addFeatureTypeIndex(searchIndex, indexingFT, featureSourceFactoryHelper); searchIndexRepository.save(searchIndex); @@ -244,7 +220,7 @@ public ResponseEntity index(@PathVariable Long searchIndexId) { produces = MediaType.APPLICATION_JSON_VALUE) @Transactional public ResponseEntity clearIndex(@PathVariable Long searchIndexId) { - try (SolrClient solrClient = getSolrClient(); + try (SolrClient solrClient = solrService.getSolrClientForSearching(); SolrHelper solrHelper = new SolrHelper(solrClient)) { solrHelper.clearIndexForLayer(searchIndexId); // do not delete the SearchIndex metadata object diff --git a/src/main/java/org/tailormap/api/solr/SolrHelper.java b/src/main/java/org/tailormap/api/solr/SolrHelper.java index 98c09611b4..d330a17b1a 100644 --- a/src/main/java/org/tailormap/api/solr/SolrHelper.java +++ b/src/main/java/org/tailormap/api/solr/SolrHelper.java @@ -258,8 +258,8 @@ public SearchResponse findInIndex( if (null == solrQuery || solrQuery.isBlank()) { solrQuery = "*"; } - // TODO We could escape special/syntax characters, but that also prevents using keys like ~ and - // * + // TODO We could escape special/syntax characters, but that also prevents using + // keys like ~ and * // solrQuery = ClientUtils.escapeQueryChars(solrQuery); final SolrQuery query = diff --git a/src/main/java/org/tailormap/api/solr/SolrService.java b/src/main/java/org/tailormap/api/solr/SolrService.java new file mode 100644 index 0000000000..ee8f94b7a9 --- /dev/null +++ b/src/main/java/org/tailormap/api/solr/SolrService.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2024 B3Partners B.V. + * + * SPDX-License-Identifier: MIT + */ +package org.tailormap.api.solr; + +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateHttp2SolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +@Service +public class SolrService { + @Value("${tailormap-api.solr-url}") + private String solrUrl; + + @Value("${tailormap-api.solr-core-name:tailormap}") + private String solrCoreName; + + /** + * Get a concurrent update Solr client for bulk operations. + * + * @return the Solr client + */ + public SolrClient getSolrClientForIndexing() { + return new ConcurrentUpdateHttp2SolrClient.Builder( + this.solrUrl + this.solrCoreName, + new Http2SolrClient.Builder() + .withFollowRedirects(true) + .withConnectionTimeout(10000, TimeUnit.MILLISECONDS) + .withRequestTimeout(60000, TimeUnit.MILLISECONDS) + .build()) + .withQueueSize(SolrHelper.SOLR_BATCH_SIZE * 2) + .withThreadCount(10) + .build(); + } + + /** + * Get a Solr client for searching. + * + * @return the Solr client + */ + public SolrClient getSolrClientForSearching() { + return new Http2SolrClient.Builder(this.solrUrl + this.solrCoreName) + .withConnectionTimeout(10, TimeUnit.SECONDS) + .withFollowRedirects(true) + .build(); + } + + public void setSolrUrl(String solrUrl) { + this.solrUrl = solrUrl; + } +}