diff --git a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java index a39c76bd3..40c81c6c4 100644 --- a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java +++ b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java @@ -38,6 +38,7 @@ import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.persistence.SearchIndex; import org.tailormap.api.persistence.TMFeatureSource; import org.tailormap.api.persistence.TMFeatureType; @@ -291,7 +292,9 @@ private SearchIndex validateInputAndFindIndex(Long searchIndexId) { if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) { // the search index should not exist for WFS feature types, but test just in case - searchIndex.setStatus(SearchIndex.Status.ERROR).setComment("WFS indexing not supported"); + searchIndex + .setStatus(SearchIndex.Status.ERROR) + .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported")); throw new ResponseStatusException( HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing"); } @@ -336,7 +339,7 @@ public ResponseEntity clearIndex(@PathVariable Long searchIndexId) { searchIndex .setLastIndexed(null) .setStatus(SearchIndex.Status.INITIAL) - .setComment("Index cleared"); + .setSummary(new SearchIndexSummary().total(0)); searchIndexRepository.save(searchIndex); } catch (IOException | SolrServerException | NoSuchElementException e) { logger.warn("Error clearing index", e); diff --git a/src/main/java/org/tailormap/api/persistence/SearchIndex.java b/src/main/java/org/tailormap/api/persistence/SearchIndex.java index eba5af963..3edcbf6b7 100644 --- a/src/main/java/org/tailormap/api/persistence/SearchIndex.java +++ b/src/main/java/org/tailormap/api/persistence/SearchIndex.java @@ -23,6 +23,7 @@ import java.util.List; import org.hibernate.annotations.Type; import org.springframework.format.annotation.DateTimeFormat; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.admin.model.TaskSchedule; import org.tailormap.api.persistence.listener.EntityEventPublisher; @@ -53,8 +54,11 @@ public class SearchIndex implements Serializable { @Valid private List searchDisplayFieldsUsed = new ArrayList<>(); - @Column(columnDefinition = "text") - private String comment; + @JsonProperty("summary") + @Type(value = io.hypersistence.utils.hibernate.type.json.JsonBinaryType.class) + @Column(columnDefinition = "jsonb") + @Valid + private SearchIndexSummary summary; /** Date and time of last index creation. */ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) @@ -118,15 +122,6 @@ public SearchIndex setSearchDisplayFieldsUsed(List searchDisplayFieldsUs return this; } - public String getComment() { - return comment; - } - - public SearchIndex setComment(String comment) { - this.comment = comment; - return this; - } - public OffsetDateTime getLastIndexed() { return lastIndexed; } @@ -154,6 +149,15 @@ public SearchIndex setSchedule(@Valid TaskSchedule schedule) { return this; } + public SearchIndexSummary getSummary() { + return summary; + } + + public SearchIndex setSummary(SearchIndexSummary summary) { + this.summary = summary; + return this; + } + public enum Status { INITIAL("initial"), INDEXING("indexing"), diff --git a/src/main/java/org/tailormap/api/scheduling/IndexTask.java b/src/main/java/org/tailormap/api/scheduling/IndexTask.java index 7b11b967b..a51df3fb0 100644 --- a/src/main/java/org/tailormap/api/scheduling/IndexTask.java +++ b/src/main/java/org/tailormap/api/scheduling/IndexTask.java @@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.lang.NonNull; import org.springframework.scheduling.quartz.QuartzJobBean; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.admin.model.ServerSentEvent; import org.tailormap.api.admin.model.TaskProgressEvent; import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper; @@ -108,8 +109,7 @@ protected void executeInternal(@NonNull JobExecutionContext context) .withBatchSize(solrBatchSize) .withGeometryValidationRule(solrGeometryValidationRule)) { - searchIndex.setStatus(SearchIndex.Status.INDEXING); - searchIndex = searchIndexRepository.save(searchIndex); + searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXING)); searchIndex = solrHelper.addFeatureTypeIndex( @@ -119,8 +119,7 @@ protected void executeInternal(@NonNull JobExecutionContext context) searchIndexRepository, this::taskProgress, UUID.fromString(context.getTrigger().getJobKey().getName())); - searchIndex = searchIndex.setStatus(SearchIndex.Status.INDEXED); - searchIndexRepository.save(searchIndex); + searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXED)); persistedJobData.put( "executions", (1 + (int) context.getMergedJobDataMap().getOrDefault("executions", 0))); persistedJobData.put("lastExecutionFinished", Instant.now()); @@ -128,12 +127,14 @@ protected void executeInternal(@NonNull JobExecutionContext context) context.setResult("Index task executed successfully"); } catch (UnsupportedOperationException | IOException | SolrServerException | SolrException e) { logger.error("Error indexing", e); - searchIndex.setStatus(SearchIndex.Status.ERROR).setComment(e.getMessage()); persistedJobData.put("lastExecutionFinished", null); persistedJobData.put( Task.LAST_RESULT_KEY, "Index task failed with " + e.getMessage() + ". Check logs for details"); - searchIndexRepository.save(searchIndex); + searchIndexRepository.save( + searchIndex + .setStatus(SearchIndex.Status.ERROR) + .setSummary(new SearchIndexSummary().errorMessage(e.getMessage()))); context.setResult("Error indexing. Check logs for details."); throw new JobExecutionException("Error indexing", e); } diff --git a/src/main/java/org/tailormap/api/solr/SolrHelper.java b/src/main/java/org/tailormap/api/solr/SolrHelper.java index b8e11a4fd..6d7d3860f 100644 --- a/src/main/java/org/tailormap/api/solr/SolrHelper.java +++ b/src/main/java/org/tailormap/api/solr/SolrHelper.java @@ -9,6 +9,7 @@ import jakarta.validation.constraints.Positive; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.math.BigDecimal; import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; @@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory; import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.admin.model.TaskProgressEvent; import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper; import org.tailormap.api.geotools.processing.GeometryProcessor; @@ -183,9 +185,7 @@ public SearchIndex addFeatureTypeIndex( throws IOException, SolrServerException { // use a dummy/logging listener when not given Consumer progressListener = - (event) -> { - logger.debug("Progress event: {}", event); - }; + (event) -> logger.debug("Progress event: {}", event); return this.addFeatureTypeIndex( searchIndex, @@ -234,13 +234,16 @@ public SearchIndex addFeatureTypeIndex( taskUuid = searchIndex.getSchedule().getUuid(); } + SearchIndexSummary summary = + new SearchIndexSummary().startedAt(startedAtOffset).total(0).duration(0.0); + if (null == searchIndex.getSearchFieldsUsed()) { logger.warn( "No search fields configured for search index: {}, bailing out.", searchIndex.getName()); return searchIndexRepository.save( searchIndex .setStatus(SearchIndex.Status.ERROR) - .setComment("No search fields configured")); + .setSummary(summary.errorMessage("No search fields configured"))); } progressListener.accept( @@ -267,7 +270,7 @@ public SearchIndex addFeatureTypeIndex( return searchIndexRepository.save( searchIndex .setStatus(SearchIndex.Status.ERROR) - .setComment("No search fields configured")); + .setSummary(summary.errorMessage("No search fields configured"))); } // add search and display properties to query @@ -397,28 +400,21 @@ public SearchIndex addFeatureTypeIndex( logger.warn( "{} features were skipped because no search or display values were found.", indexSkippedCounter); - searchIndex = - searchIndex.setComment( - "Indexed %s features in %s.%s seconds, started at %s. %s features were skipped because no search or display values were found." - .formatted( - total, - processTime.getSeconds(), - processTime.getNano(), - startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt)), - indexSkippedCounter)); - } else { - searchIndex = - searchIndex.setComment( - "Indexed %s features in %s.%s seconds, started at %s." - .formatted( - total, - processTime.getSeconds(), - processTime.getNano(), - startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt)))); } return searchIndexRepository.save( - searchIndex.setLastIndexed(finishedAtOffset).setStatus(SearchIndex.Status.INDEXED)); + searchIndex + .setLastIndexed(finishedAtOffset) + .setStatus(SearchIndex.Status.INDEXED) + .setSummary( + summary + .total(total) + .skippedCounter(indexSkippedCounter) + .duration( + BigDecimal.valueOf(processTime.getSeconds()) + .add(BigDecimal.valueOf(processTime.getNano(), 9)) + .doubleValue()) + .errorMessage(null))); } /** diff --git a/src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql b/src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql new file mode 100644 index 000000000..b8bd532ba --- /dev/null +++ b/src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql @@ -0,0 +1,4 @@ +alter table if exists search_index + drop column comment; +alter table if exists search_index + add column summary jsonb default null; \ No newline at end of file diff --git a/src/main/resources/openapi/admin-schemas.yaml b/src/main/resources/openapi/admin-schemas.yaml index 451fe656e..59ee33e2a 100644 --- a/src/main/resources/openapi/admin-schemas.yaml +++ b/src/main/resources/openapi/admin-schemas.yaml @@ -100,3 +100,33 @@ components: description: 'Priority of the task' type: integer nullable: true + + SearchIndexSummary: + description: 'Summary of a search index run. This is created/updated when the index is finished.' + type: object + properties: + total: + description: 'Total number of features counted for indexing. When 0 or null, the index was cleared.' + type: integer + format: int32 + nullable: true + skippedCounter: + description: 'Number of features skipped during indexing.' + type: integer + format: int32 + nullable: true + startedAt: + description: 'Zoned date-time when the task started.' + type: string + format: date-time + nullable: true + example: '2024-12-13T11:30:40.863829185+01:00' + duration: + description: 'Time taken to index the source in seconds.' + type: number + format: double + nullable: true + errorMessage: + description: 'Error message if the task failed. Check the status field of the search index.' + type: string + nullable: true