HTM-1366 | HTM-1367: Indexing process summary information should be s…

…tored as a json blob (#1082)

src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java

@@ -38,6 +38,7 @@
 import org.springframework.web.bind.annotation.PutMapping;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.server.ResponseStatusException;
+import org.tailormap.api.admin.model.SearchIndexSummary;
 import org.tailormap.api.persistence.SearchIndex;
 import org.tailormap.api.persistence.TMFeatureSource;
 import org.tailormap.api.persistence.TMFeatureType;
@@ -291,7 +292,9 @@ private SearchIndex validateInputAndFindIndex(Long searchIndexId) {
 
     if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) {
       // the search index should not exist for WFS feature types, but test just in case
-      searchIndex.setStatus(SearchIndex.Status.ERROR).setComment("WFS indexing not supported");
+      searchIndex
+          .setStatus(SearchIndex.Status.ERROR)
+          .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported"));
       throw new ResponseStatusException(
           HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing");
     }
@@ -336,7 +339,7 @@ public ResponseEntity<?> clearIndex(@PathVariable Long searchIndexId) {
       searchIndex
           .setLastIndexed(null)
           .setStatus(SearchIndex.Status.INITIAL)
-          .setComment("Index cleared");
+          .setSummary(new SearchIndexSummary().total(0));
       searchIndexRepository.save(searchIndex);
     } catch (IOException | SolrServerException | NoSuchElementException e) {
       logger.warn("Error clearing index", e);

src/main/java/org/tailormap/api/persistence/SearchIndex.java

@@ -23,6 +23,7 @@
 import java.util.List;
 import org.hibernate.annotations.Type;
 import org.springframework.format.annotation.DateTimeFormat;
+import org.tailormap.api.admin.model.SearchIndexSummary;
 import org.tailormap.api.admin.model.TaskSchedule;
 import org.tailormap.api.persistence.listener.EntityEventPublisher;
 
@@ -53,8 +54,11 @@ public class SearchIndex implements Serializable {
   @Valid
   private List<String> searchDisplayFieldsUsed = new ArrayList<>();
 
-  @Column(columnDefinition = "text")
-  private String comment;
+  @JsonProperty("summary")
+  @Type(value = io.hypersistence.utils.hibernate.type.json.JsonBinaryType.class)
+  @Column(columnDefinition = "jsonb")
+  @Valid
+  private SearchIndexSummary summary;
 
   /** Date and time of last index creation. */
   @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME)
@@ -118,15 +122,6 @@ public SearchIndex setSearchDisplayFieldsUsed(List<String> searchDisplayFieldsUs
     return this;
   }
 
-  public String getComment() {
-    return comment;
-  }
-
-  public SearchIndex setComment(String comment) {
-    this.comment = comment;
-    return this;
-  }
-
   public OffsetDateTime getLastIndexed() {
     return lastIndexed;
   }
@@ -154,6 +149,15 @@ public SearchIndex setSchedule(@Valid TaskSchedule schedule) {
     return this;
   }
 
+  public SearchIndexSummary getSummary() {
+    return summary;
+  }
+
+  public SearchIndex setSummary(SearchIndexSummary summary) {
+    this.summary = summary;
+    return this;
+  }
+
   public enum Status {
     INITIAL("initial"),
     INDEXING("indexing"),

src/main/java/org/tailormap/api/scheduling/IndexTask.java

@@ -31,6 +31,7 @@
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.lang.NonNull;
 import org.springframework.scheduling.quartz.QuartzJobBean;
+import org.tailormap.api.admin.model.SearchIndexSummary;
 import org.tailormap.api.admin.model.ServerSentEvent;
 import org.tailormap.api.admin.model.TaskProgressEvent;
 import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper;
@@ -108,8 +109,7 @@ protected void executeInternal(@NonNull JobExecutionContext context)
                 .withBatchSize(solrBatchSize)
                 .withGeometryValidationRule(solrGeometryValidationRule)) {
 
-      searchIndex.setStatus(SearchIndex.Status.INDEXING);
-      searchIndex = searchIndexRepository.save(searchIndex);
+      searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXING));
 
       searchIndex =
           solrHelper.addFeatureTypeIndex(
@@ -119,21 +119,22 @@ protected void executeInternal(@NonNull JobExecutionContext context)
               searchIndexRepository,
               this::taskProgress,
               UUID.fromString(context.getTrigger().getJobKey().getName()));
-      searchIndex = searchIndex.setStatus(SearchIndex.Status.INDEXED);
-      searchIndexRepository.save(searchIndex);
+      searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXED));
       persistedJobData.put(
           "executions", (1 + (int) context.getMergedJobDataMap().getOrDefault("executions", 0)));
       persistedJobData.put("lastExecutionFinished", Instant.now());
       persistedJobData.put(Task.LAST_RESULT_KEY, "Index task executed successfully");
       context.setResult("Index task executed successfully");
     } catch (UnsupportedOperationException | IOException | SolrServerException | SolrException e) {
       logger.error("Error indexing", e);
-      searchIndex.setStatus(SearchIndex.Status.ERROR).setComment(e.getMessage());
       persistedJobData.put("lastExecutionFinished", null);
       persistedJobData.put(
           Task.LAST_RESULT_KEY,
           "Index task failed with " + e.getMessage() + ". Check logs for details");
-      searchIndexRepository.save(searchIndex);
+      searchIndexRepository.save(
+          searchIndex
+              .setStatus(SearchIndex.Status.ERROR)
+              .setSummary(new SearchIndexSummary().errorMessage(e.getMessage())));
       context.setResult("Error indexing. Check logs for details.");
       throw new JobExecutionException("Error indexing", e);
     }

src/main/java/org/tailormap/api/solr/SolrHelper.java

@@ -9,6 +9,7 @@
 import jakarta.validation.constraints.Positive;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
+import java.math.BigDecimal;
 import java.time.Duration;
 import java.time.Instant;
 import java.time.OffsetDateTime;
@@ -43,6 +44,7 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
+import org.tailormap.api.admin.model.SearchIndexSummary;
 import org.tailormap.api.admin.model.TaskProgressEvent;
 import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper;
 import org.tailormap.api.geotools.processing.GeometryProcessor;
@@ -183,9 +185,7 @@ public SearchIndex addFeatureTypeIndex(
       throws IOException, SolrServerException {
     // use a dummy/logging listener when not given
     Consumer<TaskProgressEvent> progressListener =
-        (event) -> {
-          logger.debug("Progress event: {}", event);
-        };
+        (event) -> logger.debug("Progress event: {}", event);
 
     return this.addFeatureTypeIndex(
         searchIndex,
@@ -234,13 +234,16 @@ public SearchIndex addFeatureTypeIndex(
       taskUuid = searchIndex.getSchedule().getUuid();
     }
 
+    SearchIndexSummary summary =
+        new SearchIndexSummary().startedAt(startedAtOffset).total(0).duration(0.0);
+
     if (null == searchIndex.getSearchFieldsUsed()) {
       logger.warn(
           "No search fields configured for search index: {}, bailing out.", searchIndex.getName());
       return searchIndexRepository.save(
           searchIndex
               .setStatus(SearchIndex.Status.ERROR)
-              .setComment("No search fields configured"));
+              .setSummary(summary.errorMessage("No search fields configured")));
     }
 
     progressListener.accept(
@@ -267,7 +270,7 @@ public SearchIndex addFeatureTypeIndex(
       return searchIndexRepository.save(
           searchIndex
               .setStatus(SearchIndex.Status.ERROR)
-              .setComment("No search fields configured"));
+              .setSummary(summary.errorMessage("No search fields configured")));
     }
 
     // add search and display properties to query
@@ -397,28 +400,21 @@ public SearchIndex addFeatureTypeIndex(
       logger.warn(
           "{} features were skipped because no search or display values were found.",
           indexSkippedCounter);
-      searchIndex =
-          searchIndex.setComment(
-              "Indexed %s features in %s.%s seconds, started at %s. %s features were skipped because no search or display values were found."
-                  .formatted(
-                      total,
-                      processTime.getSeconds(),
-                      processTime.getNano(),
-                      startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt)),
-                      indexSkippedCounter));
-    } else {
-      searchIndex =
-          searchIndex.setComment(
-              "Indexed %s features in %s.%s seconds, started at %s."
-                  .formatted(
-                      total,
-                      processTime.getSeconds(),
-                      processTime.getNano(),
-                      startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt))));
     }
 
     return searchIndexRepository.save(
-        searchIndex.setLastIndexed(finishedAtOffset).setStatus(SearchIndex.Status.INDEXED));
+        searchIndex
+            .setLastIndexed(finishedAtOffset)
+            .setStatus(SearchIndex.Status.INDEXED)
+            .setSummary(
+                summary
+                    .total(total)
+                    .skippedCounter(indexSkippedCounter)
+                    .duration(
+                        BigDecimal.valueOf(processTime.getSeconds())
+                            .add(BigDecimal.valueOf(processTime.getNano(), 9))
+                            .doubleValue())
+                    .errorMessage(null)));
   }
 
   /**

src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql

@@ -0,0 +1,4 @@
+alter table if exists search_index
+    drop column comment;
+alter table if exists search_index
+    add column summary jsonb default null;

src/main/resources/openapi/admin-schemas.yaml

@@ -100,3 +100,33 @@ components:
           description: 'Priority of the task'
           type: integer
           nullable: true
+
+    SearchIndexSummary:
+      description: 'Summary of a search index run. This is created/updated when the index is finished.'
+      type: object
+      properties:
+        total:
+          description: 'Total number of features counted for indexing. When 0 or null, the index was cleared.'
+          type: integer
+          format: int32
+          nullable: true
+        skippedCounter:
+          description: 'Number of features skipped during indexing.'
+          type: integer
+          format: int32
+          nullable: true
+        startedAt:
+          description: 'Zoned date-time when the task started.'
+          type: string
+          format: date-time
+          nullable: true
+          example: '2024-12-13T11:30:40.863829185+01:00'
+        duration:
+          description: 'Time taken to index the source in seconds.'
+          type: number
+          format: double
+          nullable: true
+        errorMessage:
+          description: 'Error message if the task failed. Check the status field of the search index.'
+          type: string
+          nullable: true