From ac799b11957222caee959ba7e930b3d9afe200f7 Mon Sep 17 00:00:00 2001 From: Othello Maurer Date: Thu, 22 Sep 2022 15:37:01 +0200 Subject: [PATCH] Invalidate registration cache only for affected sidecars (#13514) * Invalidate registration cache only for affected sidecars * Use node-id for cache invalidation The cache key has been changed since this PR has been started. * Consider OS for cache invalidation Also use symmetric difference of tags to further reduce unnecessary invalidations * Invalidate less when creating new configs --- .../rest/resources/ConfigurationResource.java | 25 ++++++++++++++++--- .../sidecar/services/SidecarService.java | 8 ++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java index 808bfe08e0f0..312937e1d101 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/rest/resources/ConfigurationResource.java @@ -19,6 +19,7 @@ import com.codahale.metrics.annotation.Timed; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Sets; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -26,6 +27,7 @@ import org.apache.shiro.authz.annotation.RequiresPermissions; import org.graylog.plugins.sidecar.audit.SidecarAuditEventTypes; import org.graylog.plugins.sidecar.permissions.SidecarRestPermissions; +import org.graylog.plugins.sidecar.rest.models.Collector; import org.graylog.plugins.sidecar.rest.models.CollectorUpload; import org.graylog.plugins.sidecar.rest.models.Configuration; import org.graylog.plugins.sidecar.rest.models.ConfigurationSummary; @@ -36,6 +38,7 @@ import org.graylog.plugins.sidecar.rest.responses.ConfigurationListResponse; import org.graylog.plugins.sidecar.rest.responses.ConfigurationPreviewRenderResponse; import org.graylog.plugins.sidecar.rest.responses.ConfigurationSidecarsResponse; +import org.graylog.plugins.sidecar.services.CollectorService; import org.graylog.plugins.sidecar.services.ConfigurationService; import org.graylog.plugins.sidecar.services.EtagService; import org.graylog.plugins.sidecar.services.ImportService; @@ -77,6 +80,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -98,6 +102,7 @@ public class ConfigurationResource extends RestResource implements PluginRestRes private final SidecarService sidecarService; private final EtagService etagService; private final ImportService importService; + private final CollectorService collectorService; private final SearchQueryParser searchQueryParser; private static final ImmutableMap SEARCH_FIELD_MAPPING = ImmutableMap.builder() .put("id", SearchQueryField.create(Configuration.FIELD_ID)) @@ -109,12 +114,14 @@ public class ConfigurationResource extends RestResource implements PluginRestRes public ConfigurationResource(ConfigurationService configurationService, SidecarService sidecarService, EtagService etagService, - ImportService importService) { + ImportService importService, + CollectorService collectorService) { this.configurationService = configurationService; this.sidecarService = sidecarService; this.etagService = etagService; this.importService = importService; - this.searchQueryParser = new SearchQueryParser(Configuration.FIELD_NAME, SEARCH_FIELD_MAPPING);; + this.collectorService = collectorService; + this.searchQueryParser = new SearchQueryParser(Configuration.FIELD_NAME, SEARCH_FIELD_MAPPING); } @GET @@ -282,7 +289,11 @@ public Response createConfiguration(@ApiParam(name = "JSON body", required = tru final Configuration config = configurationService.save(configuration); if (!config.tags().isEmpty()) { - etagService.invalidateAllRegistrations(); + final String os = Optional.ofNullable(collectorService.find(request.collectorId())) + .map(Collector::nodeOperatingSystem).orElse(""); + sidecarService.findByTagsAndOS(config.tags(), os) + .map(Sidecar::nodeId) + .forEach(etagService::invalidateRegistration); } return Response.ok().entity(config).build(); @@ -333,8 +344,14 @@ public Response updateConfiguration(@ApiParam(name = "id", required = true) return Response.status(Response.Status.BAD_REQUEST).entity(validationResult).build(); } etagService.invalidateAllConfigurations(); + if (! previousConfiguration.tags().equals(updatedConfiguration.tags())) { - etagService.invalidateAllRegistrations(); + final Set tags = Sets.symmetricDifference(previousConfiguration.tags(), updatedConfiguration.tags()); + final String os = Optional.ofNullable(collectorService.find(request.collectorId())) + .map(Collector::nodeOperatingSystem).orElse(""); + sidecarService.findByTagsAndOS(tags, os) + .map(Sidecar::nodeId) + .forEach(etagService::invalidateRegistration); } return Response.ok().entity(configurationService.save(updatedConfiguration)).build(); diff --git a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java index 4c886ea10007..6871c2aa2cea 100644 --- a/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java +++ b/graylog2-server/src/main/java/org/graylog/plugins/sidecar/services/SidecarService.java @@ -49,6 +49,7 @@ import java.util.List; import java.util.Set; import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -274,4 +275,11 @@ public List toSummaryList(List sidecars, Predicate collector.toSummary(isActiveFunction)) .collect(Collectors.toList()); } + + public Stream findByTagsAndOS(Collection tags, String os) { + return streamQuery(DBQuery.and( + DBQuery.in("node_details.tags", tags), + DBQuery.regex("node_details.operating_system", Pattern.compile("^" + Pattern.quote(os) + "$", Pattern.CASE_INSENSITIVE)) + )); + } }