From 265c72088cf298c7c8a14cfb777e791b9cb61594 Mon Sep 17 00:00:00 2001 From: Matthijs Laan Date: Fri, 13 Dec 2024 13:39:07 +0100 Subject: [PATCH 1/8] [maven-release-plugin] prepare release tailormap-api-11.4.4 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 202d09d4c..035e2955d 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ SPDX-License-Identifier: MIT org.tailormap tailormap-api - 11.4.4-SNAPSHOT + 11.4.4 jar Tailormap API Tailormap API provides the backend for Tailormap @@ -63,7 +63,7 @@ SPDX-License-Identifier: MIT scm:git:git@github.com:Tailormap/tailormap-api.git scm:git:git@github.com:Tailormap/tailormap-api.git - tailormap-api-11.4.0 + tailormap-api-11.4.4 https://github.com/Tailormap/tailormap-api/ @@ -99,7 +99,7 @@ SPDX-License-Identifier: MIT ${java.version} ${java.version} 3.12.1 - 2024-11-29T15:05:44Z + 2024-12-13T12:34:49Z 32.1 1.20.0 5.0.0-alpha.14 From 72d66f9230248ce495452810ba33a59abd8fcaf8 Mon Sep 17 00:00:00 2001 From: Matthijs Laan Date: Fri, 13 Dec 2024 13:39:10 +0100 Subject: [PATCH 2/8] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 035e2955d..dfd1a43fb 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,7 @@ SPDX-License-Identifier: MIT org.tailormap tailormap-api - 11.4.4 + 11.4.5-SNAPSHOT jar Tailormap API Tailormap API provides the backend for Tailormap @@ -99,7 +99,7 @@ SPDX-License-Identifier: MIT ${java.version} ${java.version} 3.12.1 - 2024-12-13T12:34:49Z + 2024-12-13T12:39:10Z 32.1 1.20.0 5.0.0-alpha.14 From 81986a9b75523f2d90d4eaa560835e024494c3c3 Mon Sep 17 00:00:00 2001 From: Mark Prins <1165786+mprins@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:41:01 +0100 Subject: [PATCH 3/8] HTM-1372: add operationIds for the page endpoints (#1086) HTM-1372: add oprationIds for the page endpoints --- src/main/resources/openapi/viewer-api.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/resources/openapi/viewer-api.yaml b/src/main/resources/openapi/viewer-api.yaml index ef9d99ae8..0f622d5eb 100644 --- a/src/main/resources/openapi/viewer-api.yaml +++ b/src/main/resources/openapi/viewer-api.yaml @@ -1366,6 +1366,7 @@ paths: /page: summary: 'Use this endpoint to get the home page' get: + operationId: 'getHomePage' responses: '200': description: 'OK' @@ -1389,6 +1390,7 @@ paths: schema: type: string get: + operationId: 'getPage' responses: '200': description: 'OK' From 84d52be99e5285511098368c7c64e338cf9eb339 Mon Sep 17 00:00:00 2001 From: Mark Prins <1165786+mprins@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:42:55 +0100 Subject: [PATCH 4/8] HTM-1375: fix OpenAPI errors and warnings (#1087) --- src/main/resources/openapi/viewer-api.yaml | 12 ++++++------ src/main/resources/openapi/viewer-schemas.yaml | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/resources/openapi/viewer-api.yaml b/src/main/resources/openapi/viewer-api.yaml index 0f622d5eb..8b3db703e 100644 --- a/src/main/resources/openapi/viewer-api.yaml +++ b/src/main/resources/openapi/viewer-api.yaml @@ -156,7 +156,7 @@ components: Layers: description: 'An array of app layers.' type: array - minLength: 0 + minItems: 0 items: $ref: './viewer-schemas.yaml#/components/schemas/AppLayer' @@ -202,7 +202,7 @@ components: components: description: 'Configured components (controls) for the viewer' type: array - minLength: 0 + minItems: 0 items: $ref: './viewer-schemas.yaml#/components/schemas/Component' nullable: false @@ -235,13 +235,13 @@ components: features: description: 'list of features. When the pagenumber*pagesize exceeds the number of features this list will be empty' type: array - minLength: 0 + minItems: 0 items: $ref: './viewer-schemas.yaml#/components/schemas/Feature' columnMetadata: description: 'presentation metadata' type: array - minLength: 0 + minItems: 0 items: $ref: './viewer-schemas.yaml#/components/schemas/ColumnMetadata' template: @@ -267,7 +267,7 @@ components: values: description: 'list of unique values' type: array - minLength: 0 + minItems: 0 uniqueItems: true # any one type of string|number|boolean is allowed, but that generates unusable java code # so allow any non-null value, we don't need to worry about mixed types as the data comes from a @@ -300,7 +300,7 @@ components: documents: description: 'list of search documents. When the pagenumber*pagesize exceeds the number of documents this list will be empty' type: array - minLength: 0 + minItems: 0 items: $ref: '#/components/schemas/SearchDocument' example: { "total": 2, diff --git a/src/main/resources/openapi/viewer-schemas.yaml b/src/main/resources/openapi/viewer-schemas.yaml index 8a7eb9628..cfc224257 100644 --- a/src/main/resources/openapi/viewer-schemas.yaml +++ b/src/main/resources/openapi/viewer-schemas.yaml @@ -293,7 +293,7 @@ components: attributes: description: 'feature attributes in key/value pairs' type: object - minItems: 1 + minProperties: 1 additionalProperties: true example: __fid: 1 @@ -439,13 +439,13 @@ components: tiles: description: list of tiles, tiles are used to link to applications or other pages type: array - minLength: 0 + minItems: 0 items: $ref: '#/components/schemas/ViewerPageTile' menu: description: list of menu items for this page type: array - minLength: 0 + minItems: 0 items: $ref: '#/components/schemas/ViewerMenuItem' From a55998a5b096cf7c30d4bf63a3f4a281129497d4 Mon Sep 17 00:00:00 2001 From: Mark Prins <1165786+mprins@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:01:45 +0100 Subject: [PATCH 5/8] HTM-1366: replace SearchIndex#comment with SearchIndex#summary in datamodel and persistence --- .../api/persistence/SearchIndex.java | 26 +++++++++------- ...r_index_summary_delete_comment_columns.sql | 4 +++ src/main/resources/openapi/admin-schemas.yaml | 30 +++++++++++++++++++ 3 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql diff --git a/src/main/java/org/tailormap/api/persistence/SearchIndex.java b/src/main/java/org/tailormap/api/persistence/SearchIndex.java index eba5af963..3edcbf6b7 100644 --- a/src/main/java/org/tailormap/api/persistence/SearchIndex.java +++ b/src/main/java/org/tailormap/api/persistence/SearchIndex.java @@ -23,6 +23,7 @@ import java.util.List; import org.hibernate.annotations.Type; import org.springframework.format.annotation.DateTimeFormat; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.admin.model.TaskSchedule; import org.tailormap.api.persistence.listener.EntityEventPublisher; @@ -53,8 +54,11 @@ public class SearchIndex implements Serializable { @Valid private List searchDisplayFieldsUsed = new ArrayList<>(); - @Column(columnDefinition = "text") - private String comment; + @JsonProperty("summary") + @Type(value = io.hypersistence.utils.hibernate.type.json.JsonBinaryType.class) + @Column(columnDefinition = "jsonb") + @Valid + private SearchIndexSummary summary; /** Date and time of last index creation. */ @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) @@ -118,15 +122,6 @@ public SearchIndex setSearchDisplayFieldsUsed(List searchDisplayFieldsUs return this; } - public String getComment() { - return comment; - } - - public SearchIndex setComment(String comment) { - this.comment = comment; - return this; - } - public OffsetDateTime getLastIndexed() { return lastIndexed; } @@ -154,6 +149,15 @@ public SearchIndex setSchedule(@Valid TaskSchedule schedule) { return this; } + public SearchIndexSummary getSummary() { + return summary; + } + + public SearchIndex setSummary(SearchIndexSummary summary) { + this.summary = summary; + return this; + } + public enum Status { INITIAL("initial"), INDEXING("indexing"), diff --git a/src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql b/src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql new file mode 100644 index 000000000..b8bd532ba --- /dev/null +++ b/src/main/resources/db/migration/V13__add_solr_index_summary_delete_comment_columns.sql @@ -0,0 +1,4 @@ +alter table if exists search_index + drop column comment; +alter table if exists search_index + add column summary jsonb default null; \ No newline at end of file diff --git a/src/main/resources/openapi/admin-schemas.yaml b/src/main/resources/openapi/admin-schemas.yaml index 451fe656e..59ee33e2a 100644 --- a/src/main/resources/openapi/admin-schemas.yaml +++ b/src/main/resources/openapi/admin-schemas.yaml @@ -100,3 +100,33 @@ components: description: 'Priority of the task' type: integer nullable: true + + SearchIndexSummary: + description: 'Summary of a search index run. This is created/updated when the index is finished.' + type: object + properties: + total: + description: 'Total number of features counted for indexing. When 0 or null, the index was cleared.' + type: integer + format: int32 + nullable: true + skippedCounter: + description: 'Number of features skipped during indexing.' + type: integer + format: int32 + nullable: true + startedAt: + description: 'Zoned date-time when the task started.' + type: string + format: date-time + nullable: true + example: '2024-12-13T11:30:40.863829185+01:00' + duration: + description: 'Time taken to index the source in seconds.' + type: number + format: double + nullable: true + errorMessage: + description: 'Error message if the task failed. Check the status field of the search index.' + type: string + nullable: true From af61d3aebb1f19f967013fb61424bbaa09638834 Mon Sep 17 00:00:00 2001 From: mark <1165786+mprins@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:32:55 +0100 Subject: [PATCH 6/8] HTM-1367: Refactor usage of `setComment(...)` to `setSummary(...)` in the API --- .../controller/admin/SolrAdminController.java | 7 ++- .../tailormap/api/scheduling/IndexTask.java | 13 +++--- .../org/tailormap/api/solr/SolrHelper.java | 44 +++++++++---------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java index a39c76bd3..40c81c6c4 100644 --- a/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java +++ b/src/main/java/org/tailormap/api/controller/admin/SolrAdminController.java @@ -38,6 +38,7 @@ import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.server.ResponseStatusException; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.persistence.SearchIndex; import org.tailormap.api.persistence.TMFeatureSource; import org.tailormap.api.persistence.TMFeatureType; @@ -291,7 +292,9 @@ private SearchIndex validateInputAndFindIndex(Long searchIndexId) { if (TMFeatureSource.Protocol.WFS.equals(indexingFT.getFeatureSource().getProtocol())) { // the search index should not exist for WFS feature types, but test just in case - searchIndex.setStatus(SearchIndex.Status.ERROR).setComment("WFS indexing not supported"); + searchIndex + .setStatus(SearchIndex.Status.ERROR) + .setSummary(new SearchIndexSummary().errorMessage("WFS indexing not supported")); throw new ResponseStatusException( HttpStatus.BAD_REQUEST, "Layer does not have valid feature type for indexing"); } @@ -336,7 +339,7 @@ public ResponseEntity clearIndex(@PathVariable Long searchIndexId) { searchIndex .setLastIndexed(null) .setStatus(SearchIndex.Status.INITIAL) - .setComment("Index cleared"); + .setSummary(new SearchIndexSummary().total(0)); searchIndexRepository.save(searchIndex); } catch (IOException | SolrServerException | NoSuchElementException e) { logger.warn("Error clearing index", e); diff --git a/src/main/java/org/tailormap/api/scheduling/IndexTask.java b/src/main/java/org/tailormap/api/scheduling/IndexTask.java index 7b11b967b..a51df3fb0 100644 --- a/src/main/java/org/tailormap/api/scheduling/IndexTask.java +++ b/src/main/java/org/tailormap/api/scheduling/IndexTask.java @@ -31,6 +31,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.lang.NonNull; import org.springframework.scheduling.quartz.QuartzJobBean; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.admin.model.ServerSentEvent; import org.tailormap.api.admin.model.TaskProgressEvent; import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper; @@ -108,8 +109,7 @@ protected void executeInternal(@NonNull JobExecutionContext context) .withBatchSize(solrBatchSize) .withGeometryValidationRule(solrGeometryValidationRule)) { - searchIndex.setStatus(SearchIndex.Status.INDEXING); - searchIndex = searchIndexRepository.save(searchIndex); + searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXING)); searchIndex = solrHelper.addFeatureTypeIndex( @@ -119,8 +119,7 @@ protected void executeInternal(@NonNull JobExecutionContext context) searchIndexRepository, this::taskProgress, UUID.fromString(context.getTrigger().getJobKey().getName())); - searchIndex = searchIndex.setStatus(SearchIndex.Status.INDEXED); - searchIndexRepository.save(searchIndex); + searchIndex = searchIndexRepository.save(searchIndex.setStatus(SearchIndex.Status.INDEXED)); persistedJobData.put( "executions", (1 + (int) context.getMergedJobDataMap().getOrDefault("executions", 0))); persistedJobData.put("lastExecutionFinished", Instant.now()); @@ -128,12 +127,14 @@ protected void executeInternal(@NonNull JobExecutionContext context) context.setResult("Index task executed successfully"); } catch (UnsupportedOperationException | IOException | SolrServerException | SolrException e) { logger.error("Error indexing", e); - searchIndex.setStatus(SearchIndex.Status.ERROR).setComment(e.getMessage()); persistedJobData.put("lastExecutionFinished", null); persistedJobData.put( Task.LAST_RESULT_KEY, "Index task failed with " + e.getMessage() + ". Check logs for details"); - searchIndexRepository.save(searchIndex); + searchIndexRepository.save( + searchIndex + .setStatus(SearchIndex.Status.ERROR) + .setSummary(new SearchIndexSummary().errorMessage(e.getMessage()))); context.setResult("Error indexing. Check logs for details."); throw new JobExecutionException("Error indexing", e); } diff --git a/src/main/java/org/tailormap/api/solr/SolrHelper.java b/src/main/java/org/tailormap/api/solr/SolrHelper.java index b8e11a4fd..6d7d3860f 100644 --- a/src/main/java/org/tailormap/api/solr/SolrHelper.java +++ b/src/main/java/org/tailormap/api/solr/SolrHelper.java @@ -9,6 +9,7 @@ import jakarta.validation.constraints.Positive; import java.io.IOException; import java.lang.invoke.MethodHandles; +import java.math.BigDecimal; import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; @@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory; import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; +import org.tailormap.api.admin.model.SearchIndexSummary; import org.tailormap.api.admin.model.TaskProgressEvent; import org.tailormap.api.geotools.featuresources.FeatureSourceFactoryHelper; import org.tailormap.api.geotools.processing.GeometryProcessor; @@ -183,9 +185,7 @@ public SearchIndex addFeatureTypeIndex( throws IOException, SolrServerException { // use a dummy/logging listener when not given Consumer progressListener = - (event) -> { - logger.debug("Progress event: {}", event); - }; + (event) -> logger.debug("Progress event: {}", event); return this.addFeatureTypeIndex( searchIndex, @@ -234,13 +234,16 @@ public SearchIndex addFeatureTypeIndex( taskUuid = searchIndex.getSchedule().getUuid(); } + SearchIndexSummary summary = + new SearchIndexSummary().startedAt(startedAtOffset).total(0).duration(0.0); + if (null == searchIndex.getSearchFieldsUsed()) { logger.warn( "No search fields configured for search index: {}, bailing out.", searchIndex.getName()); return searchIndexRepository.save( searchIndex .setStatus(SearchIndex.Status.ERROR) - .setComment("No search fields configured")); + .setSummary(summary.errorMessage("No search fields configured"))); } progressListener.accept( @@ -267,7 +270,7 @@ public SearchIndex addFeatureTypeIndex( return searchIndexRepository.save( searchIndex .setStatus(SearchIndex.Status.ERROR) - .setComment("No search fields configured")); + .setSummary(summary.errorMessage("No search fields configured"))); } // add search and display properties to query @@ -397,28 +400,21 @@ public SearchIndex addFeatureTypeIndex( logger.warn( "{} features were skipped because no search or display values were found.", indexSkippedCounter); - searchIndex = - searchIndex.setComment( - "Indexed %s features in %s.%s seconds, started at %s. %s features were skipped because no search or display values were found." - .formatted( - total, - processTime.getSeconds(), - processTime.getNano(), - startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt)), - indexSkippedCounter)); - } else { - searchIndex = - searchIndex.setComment( - "Indexed %s features in %s.%s seconds, started at %s." - .formatted( - total, - processTime.getSeconds(), - processTime.getNano(), - startedAt.atOffset(ZoneId.systemDefault().getRules().getOffset(startedAt)))); } return searchIndexRepository.save( - searchIndex.setLastIndexed(finishedAtOffset).setStatus(SearchIndex.Status.INDEXED)); + searchIndex + .setLastIndexed(finishedAtOffset) + .setStatus(SearchIndex.Status.INDEXED) + .setSummary( + summary + .total(total) + .skippedCounter(indexSkippedCounter) + .duration( + BigDecimal.valueOf(processTime.getSeconds()) + .add(BigDecimal.valueOf(processTime.getNano(), 9)) + .doubleValue()) + .errorMessage(null))); } /** From 81ff8a82a12f9478d0420f51b13915f8cfa935ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 10:21:45 +0100 Subject: [PATCH 7/8] Bump sentry.version from 7.18.1 to 7.19.0 (#1088) Bumps `sentry.version` from 7.18.1 to 7.19.0. Updates `io.sentry:sentry` from 7.18.1 to 7.19.0 - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/7.18.1...7.19.0) Updates `io.sentry:sentry-jdbc` from 7.18.1 to 7.19.0 - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/7.18.1...7.19.0) Updates `io.sentry:sentry-quartz` from 7.18.1 to 7.19.0 - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/7.18.1...7.19.0) Updates `io.sentry:sentry-spring-boot-starter-jakarta` from 7.18.1 to 7.19.0 - [Release notes](https://github.com/getsentry/sentry-java/releases) - [Changelog](https://github.com/getsentry/sentry-java/blob/main/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-java/compare/7.18.1...7.19.0) --- updated-dependencies: - dependency-name: io.sentry:sentry dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: io.sentry:sentry-jdbc dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: io.sentry:sentry-quartz dependency-type: direct:production update-type: version-update:semver-minor - dependency-name: io.sentry:sentry-spring-boot-starter-jakarta dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dfd1a43fb..516de875c 100644 --- a/pom.xml +++ b/pom.xml @@ -155,7 +155,7 @@ SPDX-License-Identifier: MIT 2.9.0 2.0.1 5.18.2 - 7.18.1 + 7.19.0 2.36.0 -XepDisableWarningsInGeneratedCode ${project.build.directory}/generated-sources/.* From 4b807d5651a3b5c3751978181eb2595c17d29c2d Mon Sep 17 00:00:00 2001 From: Mark Prins <1165786+mprins@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:17:41 +0100 Subject: [PATCH 8/8] CI: update Austria basemap capabilities url see https://basemap.at/highdpi/ and https://github.com/openlayers/openlayers/pull/16463#issue-2739879777 --- .../org/tailormap/api/configuration/dev/PopulateTestData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java b/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java index 81749927a..0a19a1bb0 100644 --- a/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java +++ b/src/main/java/org/tailormap/api/configuration/dev/PopulateTestData.java @@ -493,7 +493,7 @@ private void createCatalogTestData() throws Exception { .setId("at-basemap") .setProtocol(WMTS) .setTitle("basemap.at") - .setUrl("https://basemap.at/wmts/1.0.0/WMTSCapabilities.xml") + .setUrl("https://mapsneu.wien.gv.at/basemapneu/1.0.0/WMTSCapabilities.xml") .setAuthorizationRules(ruleAnonymousRead) .setPublished(true) .setSettings(