From 00097bfc4a06a47b4738c354989cac907d0b44dc Mon Sep 17 00:00:00 2001 From: Othello Maurer Date: Wed, 21 Sep 2022 14:56:07 +0200 Subject: [PATCH 1/4] Invalidate registration cache only for affected sidecars --- .../sidecar/rest/resources/ConfigurationResource.java | 8 ++++++-- .../graylog/plugins/sidecar/services/SidecarService.java | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java index 808bfe08e0f0..838458dc520a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java @@ -19,6 +19,7 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -114,7 +115,7 @@ public ConfigurationResource(ConfigurationService configurationService, this.sidecarService = sidecarService; this.etagService = etagService; this.importService = importService; - this.searchQueryParser = new SearchQueryParser(Configuration.FIELD_NAME, SEARCH_FIELD_MAPPING);; + this.searchQueryParser = new SearchQueryParser(Configuration.FIELD_NAME, SEARCH_FIELD_MAPPING); } @GET @@ -333,8 +334,11 @@ public Response updateConfiguration(@ApiParam(name = "id", required = true) return Response.status(Response.Status.BAD_REQUEST).entity(validationResult).build(); } etagService.invalidateAllConfigurations(); + if (! previousConfiguration.tags().equals(updatedConfiguration.tags())) { - etagService.invalidateAllRegistrations(); + sidecarService.findByTags(Sets.union(previousConfiguration.tags(), updatedConfiguration.tags())) + .map(Sidecar::id) + .forEach(etagService::invalidateRegistration); } return Response.ok().entity(configurationService.save(updatedConfiguration)).build(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java index 8675cea6994a..e6c56ea2302a 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java @@ -271,4 +271,8 @@ public List toSummaryList(List sidecars, Predicate collector.toSummary(isActiveFunction)) .collect(Collectors.toList()); } + + public Stream findByTags(Collection tags) { + return streamQuery(DBQuery.in("node_details.tags", tags)); + } } From 3f2f883ff67c8e3cfac4d6b4445e0e075dc78f12 Mon Sep 17 00:00:00 2001 From: Othello Maurer Date: Wed, 21 Sep 2022 18:19:54 +0200 Subject: [PATCH 2/4] Use node-id for cache invalidation The cache key has been changed since this PR has been started. --- .../plugins/sidecar/rest/resources/ConfigurationResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java index 838458dc520a..838cca3fe424 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java @@ -337,7 +337,7 @@ public Response updateConfiguration(@ApiParam(name = "id", required = true) if (! previousConfiguration.tags().equals(updatedConfiguration.tags())) { sidecarService.findByTags(Sets.union(previousConfiguration.tags(), updatedConfiguration.tags())) - .map(Sidecar::id) + .map(Sidecar::nodeId) .forEach(etagService::invalidateRegistration); } From 56898fab71c5d10cc71e327d422e66ab304dd7c6 Mon Sep 17 00:00:00 2001 From: Othello Maurer Date: Thu, 22 Sep 2022 09:24:06 +0200 Subject: [PATCH 3/4] Consider OS for cache invalidation Also use symmetric difference of tags to further reduce unnecessary invalidations --- .../rest/resources/ConfigurationResource.java | 13 +++++++++++-- .../plugins/sidecar/services/SidecarService.java | 8 ++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java index 838cca3fe424..79849c4cdfe2 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java @@ -27,6 +27,7 @@ import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.sidecar.audit.SidecarAuditEventTypes; import org.graylog.plugins.sidecar.permissions.SidecarRestPermissions; +import org.graylog.plugins.sidecar.rest.models.Collector; import org.graylog.plugins.sidecar.rest.models.CollectorUpload; import org.graylog.plugins.sidecar.rest.models.Configuration; import org.graylog.plugins.sidecar.rest.models.ConfigurationSummary; @@ -37,6 +38,7 @@ import org.graylog.plugins.sidecar.rest.responses.ConfigurationListResponse; import org.graylog.plugins.sidecar.rest.responses.ConfigurationPreviewRenderResponse; import org.graylog.plugins.sidecar.rest.responses.ConfigurationSidecarsResponse; +import org.graylog.plugins.sidecar.services.CollectorService; import org.graylog.plugins.sidecar.services.ConfigurationService; import org.graylog.plugins.sidecar.services.EtagService; import org.graylog.plugins.sidecar.services.ImportService; @@ -78,6 +80,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -99,6 +102,7 @@ public class ConfigurationResource extends RestResource implements PluginRestRes private final SidecarService sidecarService; private final EtagService etagService; private final ImportService importService; + private final CollectorService collectorService; private final SearchQueryParser searchQueryParser; private static final ImmutableMap SEARCH_FIELD_MAPPING = ImmutableMap.builder() .put("id", SearchQueryField.create(Configuration.FIELD_ID)) @@ -110,11 +114,13 @@ public class ConfigurationResource extends RestResource implements PluginRestRes public ConfigurationResource(ConfigurationService configurationService, SidecarService sidecarService, EtagService etagService, - ImportService importService) { + ImportService importService, + CollectorService collectorService) { this.configurationService = configurationService; this.sidecarService = sidecarService; this.etagService = etagService; this.importService = importService; + this.collectorService = collectorService; this.searchQueryParser = new SearchQueryParser(Configuration.FIELD_NAME, SEARCH_FIELD_MAPPING); } @@ -336,7 +342,10 @@ public Response updateConfiguration(@ApiParam(name = "id", required = true) etagService.invalidateAllConfigurations(); if (! previousConfiguration.tags().equals(updatedConfiguration.tags())) { - sidecarService.findByTags(Sets.union(previousConfiguration.tags(), updatedConfiguration.tags())) + final Set tags = Sets.symmetricDifference(previousConfiguration.tags(), updatedConfiguration.tags()); + final String os = Optional.ofNullable(collectorService.find(request.collectorId())) + .map(Collector::nodeOperatingSystem).orElse(""); + sidecarService.findByTagsAndOS(tags, os) .map(Sidecar::nodeId) .forEach(etagService::invalidateRegistration); } diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java index bbfcf5162c82..6871c2aa2cea 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Set; import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -275,7 +276,10 @@ public List toSummaryList(List sidecars, Predicate findByTags(Collection tags) { - return streamQuery(DBQuery.in("node_details.tags", tags)); + public Stream findByTagsAndOS(Collection tags, String os) { + return streamQuery(DBQuery.and( + DBQuery.in("node_details.tags", tags), + DBQuery.regex("node_details.operating_system", Pattern.compile("^" + Pattern.quote(os) + "$", Pattern.CASE_INSENSITIVE)) + )); } } From 4b161ea77deb21d81b4ce8ad61c4ff727d43f1dc Mon Sep 17 00:00:00 2001 From: Othello Maurer Date: Thu, 22 Sep 2022 11:16:52 +0200 Subject: [PATCH 4/4] Invalidate less when creating new configs --- .../sidecar/rest/resources/ConfigurationResource.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java index 79849c4cdfe2..312937e1d101 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java @@ -289,7 +289,11 @@ public Response createConfiguration(@ApiParam(name = "JSON body", required = tru final Configuration config = configurationService.save(configuration); if (!config.tags().isEmpty()) { - etagService.invalidateAllRegistrations(); + final String os = Optional.ofNullable(collectorService.find(request.collectorId())) + .map(Collector::nodeOperatingSystem).orElse(""); + sidecarService.findByTagsAndOS(config.tags(), os) + .map(Sidecar::nodeId) + .forEach(etagService::invalidateRegistration); } return Response.ok().entity(config).build();