Skip to content

Commit

Permalink
Adding additional filters to alerts & events list. (#21056)
Browse files Browse the repository at this point in the history 
* Allow adding event definition filter.

* Allowing to filter by `priority`.

* Use better way to show priority labels.

* Allow filtering by aggregation timerange.

* Adding generic input filter for `key` property.

* Supporting multiple filters.

* Supporting multiple `key` filters.

* Adding license headers.

* Adding changelog snippet.

* Fixing linter hint.

* Adding tests for generic filter input.

* Handle non-absolute time ranges in aggregation time range filter as well.

* Inverting condition.

* Improve filtering logic for aggregation time range.

* Fixing timestamp filter parsing for all time/now.
  • Loading branch information
@dennisoelkers
dennisoelkers authored Dec 9, 2024
1 parent bea028b commit 9fca2ee
Show file tree
Hide file tree
Showing 14 changed files with 406 additions and 93 deletions.
5 changes: 5 additions & 0 deletions changelog/unreleased/pr-21056.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
type = "a"
message = "Allowing to filter events based on event definition, priority, key & aggregation time range."

issues = ["21055"]
pulls = ["21056"]
3 changes: 3 additions & 0 deletions graylog2-server/src/main/java/org/graylog/events/processor/EventDefinition.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,14 @@
import org.graylog.events.notifications.EventNotificationHandler;
import org.graylog.events.notifications.EventNotificationSettings;
import org.graylog.events.processor.storage.EventStorageHandler;
import org.graylog2.database.DbEntity;
import org.graylog2.shared.security.RestPermissions;
import org.joda.time.DateTime;

import javax.annotation.Nullable;
import java.util.Set;

@DbEntity(readPermission = RestPermissions.EVENT_DEFINITIONS_READ, collection = DBEventDefinitionService.COLLECTION_NAME)
public interface EventDefinition {
enum State {
ENABLED,
Expand Down
18 changes: 8 additions & 10 deletions graylog2-server/src/main/java/org/graylog/events/rest/EventsResource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,14 @@
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import jakarta.inject.Inject;
import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;
import org.apache.shiro.authz.annotation.RequiresAuthentication;
import org.graylog.events.event.EventDto;
import org.graylog.events.search.EventsSearchFilter;
Expand All @@ -31,16 +39,6 @@
import org.graylog2.plugin.rest.PluginRestResource;
import org.graylog2.shared.rest.resources.RestResource;

import jakarta.inject.Inject;

import jakarta.ws.rs.Consumes;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.POST;
import jakarta.ws.rs.Path;
import jakarta.ws.rs.PathParam;
import jakarta.ws.rs.Produces;
import jakarta.ws.rs.core.MediaType;

import java.util.Optional;

import static com.google.common.base.MoreObjects.firstNonNull;
Expand Down
29 changes: 27 additions & 2 deletions graylog2-server/src/main/java/org/graylog/events/search/EventsSearchFilter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,20 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.google.auto.value.AutoValue;
import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;

import java.util.Collections;
import java.util.Optional;
import java.util.Set;

@AutoValue
@JsonDeserialize(builder = EventsSearchFilter.Builder.class)
public abstract class EventsSearchFilter {
private static final String FIELD_ALERTS = "alerts";
private static final String FIELD_EVENT_DEFINITIONS = "event_definitions";
private static final String FIELD_PRIORITY = "priority";
private static final String FIELD_AGGREGATION_TIMERANGE = "aggregation_timerange";
private static final String FIELD_KEY = "key";

public enum Alerts {
@JsonProperty("include")
Expand All @@ -45,6 +50,15 @@ public enum Alerts {
@JsonProperty(FIELD_EVENT_DEFINITIONS)
public abstract Set<String> eventDefinitions();

@JsonProperty(FIELD_PRIORITY)
public abstract Set<String> priority();

@JsonProperty(FIELD_AGGREGATION_TIMERANGE)
public abstract Optional<TimeRange> aggregationTimerange();

@JsonProperty(FIELD_KEY)
public abstract Set<String> key();

public static EventsSearchFilter empty() {
return builder().build();
}
Expand All @@ -61,7 +75,9 @@ public static abstract class Builder {
public static Builder create() {
return new AutoValue_EventsSearchFilter.Builder()
.alerts(Alerts.INCLUDE)
.eventDefinitions(Collections.emptySet());
.eventDefinitions(Collections.emptySet())
.priority(Collections.emptySet())
.key(Collections.emptySet());
}

@JsonProperty(FIELD_ALERTS)
Expand All @@ -70,6 +86,15 @@ public static Builder create() {
@JsonProperty(FIELD_EVENT_DEFINITIONS)
public abstract Builder eventDefinitions(Set<String> eventDefinitions);

@JsonProperty(FIELD_PRIORITY)
public abstract Builder priority(Set<String> priority);

@JsonProperty(FIELD_AGGREGATION_TIMERANGE)
public abstract Builder aggregationTimerange(TimeRange aggregationTimerange);

@JsonProperty(FIELD_KEY)
public abstract Builder key(Set<String> key);

public abstract EventsSearchFilter build();
}
}
}
110 changes: 101 additions & 9 deletions graylog2-server/src/main/java/org/graylog/events/search/EventsSearchService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,31 +18,38 @@

import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSet;
import jakarta.inject.Inject;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.shiro.subject.Subject;
import org.graylog.events.event.EventDto;
import org.graylog.events.processor.DBEventDefinitionService;
import org.graylog.events.processor.EventDefinitionDto;
import org.graylog2.database.NotFoundException;
import org.graylog2.indexer.IndexMapping;
import org.graylog2.plugin.database.Persisted;
import org.graylog2.plugin.indexer.searches.timeranges.TimeRange;
import org.graylog2.shared.security.RestPermissions;
import org.graylog2.streams.StreamService;

import jakarta.inject.Inject;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collector;
import java.util.stream.Collectors;

import static org.graylog2.plugin.Tools.ES_DATE_FORMAT_FORMATTER;
import static org.graylog2.plugin.streams.Stream.DEFAULT_EVENTS_STREAM_ID;
import static org.graylog2.plugin.streams.Stream.DEFAULT_SYSTEM_EVENTS_STREAM_ID;

public class EventsSearchService {
private static final Collector<CharSequence, ?, String> joiningQueriesWithAND = Collectors.joining(" AND ");
private static final Collector<CharSequence, ?, String> joiningQueriesWithOR = Collectors.joining(" OR ");
private final MoreSearch moreSearch;
private final StreamService streamService;
private final DBEventDefinitionService eventDefinitionService;
Expand All @@ -64,17 +71,34 @@ private String eventDefinitionFilter(String id) {
}

public EventsSearchResult search(EventsSearchParameters parameters, Subject subject) {
final ImmutableSet.Builder<String> filterBuilder = ImmutableSet.<String>builder()
// Make sure we only filter for actual events and ignore anything else that might be in the event
// indices. (fixes an issue when users store non-event messages in event indices)
.add("_exists_:" + EventDto.FIELD_EVENT_DEFINITION_ID);
final var filterBuilder = new ArrayList<String>();
// Make sure we only filter for actual events and ignore anything else that might be in the event
// indices. (fixes an issue when users store non-event messages in event indices)
filterBuilder.add("_exists_:" + EventDto.FIELD_EVENT_DEFINITION_ID);

if (!parameters.filter().eventDefinitions().isEmpty()) {
final String eventDefinitionFilter = parameters.filter().eventDefinitions().stream()
.map(this::eventDefinitionFilter)
.collect(Collectors.joining(" OR "));
.collect(joiningQueriesWithOR);

filterBuilder.addAll(Collections.singleton("(" + eventDefinitionFilter + ")"));
filterBuilder.add(eventDefinitionFilter);
}

if (!parameters.filter().priority().isEmpty()) {
filterBuilder.add(parameters.filter().priority().stream()
.map(this::mapPriority)
.map(priority -> EventDto.FIELD_PRIORITY + ":" + priority)
.collect(joiningQueriesWithOR));
}

parameters.filter().aggregationTimerange()
.filter(range -> !range.isAllMessages())
.ifPresent(aggregationTimerange -> filterBuilder.add(createTimeRangeFilter(aggregationTimerange)));

if (!parameters.filter().key().isEmpty()) {
filterBuilder.add(parameters.filter().key().stream()
.map(keyFilter -> EventDto.FIELD_KEY_TUPLE + ":" + quote(keyFilter))
.collect(joiningQueriesWithOR));
}

switch (parameters.filter().alerts()) {
Expand All @@ -89,7 +113,9 @@ public EventsSearchResult search(EventsSearchParameters parameters, Subject subj
break;
}

final String filter = String.join(" AND ", filterBuilder.build());
final String filter = filterBuilder.stream()
.map(query -> "(" + query + ")")
.collect(joiningQueriesWithAND);
final ImmutableSet<String> eventStreams = ImmutableSet.of(DEFAULT_EVENTS_STREAM_ID, DEFAULT_SYSTEM_EVENTS_STREAM_ID);
final MoreSearch.Result result = moreSearch.eventSearch(parameters, filter, eventStreams, forbiddenSourceStreams(subject));

Expand Down Expand Up @@ -121,6 +147,72 @@ public EventsSearchResult search(EventsSearchParameters parameters, Subject subj
.build();
}

private String createTimeRangeFilter(TimeRange aggregationTimerange) {
final var formattedFrom = aggregationTimerange.getFrom().toString(ES_DATE_FORMAT_FORMATTER);
final var formattedTo = aggregationTimerange.getTo().toString(ES_DATE_FORMAT_FORMATTER);
return or(
group(
or(
TermRangeQuery.newStringRange(
EventDto.FIELD_TIMERANGE_START,
quote(formattedFrom),
quote(formattedTo),
true,
true).toString()
,
TermRangeQuery.newStringRange(
EventDto.FIELD_TIMERANGE_END,
quote(formattedFrom),
quote(formattedTo),
true,
true).toString()
)
),
group(
and(
TermRangeQuery.newStringRange(
EventDto.FIELD_TIMERANGE_START,
quote("1970-01-01 00:00:00.000"),
quote(formattedFrom),
true,
true).toString()
,
TermRangeQuery.newStringRange(
EventDto.FIELD_TIMERANGE_END,
quote(formattedTo),
quote("2038-01-01 00:00:00.000"),
true,
true).toString()
)
)
);
}

private String quote(String s) {
return "\"" + s + "\"";
}

private String group(String s) {
return "(" + s + ")";
}

private String or(String... queries) {
return Arrays.stream(queries).collect(joiningQueriesWithOR);
}

private String and(String... queries) {
return Arrays.stream(queries).collect(joiningQueriesWithAND);
}

private long mapPriority(String priorityFilter) {
return switch (priorityFilter) {
case "high", "3" -> 3;
case "normal", "2" -> 2;
case "low", "1" -> 1;
default -> throw new IllegalStateException("Invalid priority: " + priorityFilter);
};
}

// TODO: Loading all streams for a user is not very efficient. Not sure if we can find an alternative that is
// more efficient. Doing a separate ES query to get all source streams that would be in the result is
// most probably not more efficient.
Expand Down
5 changes: 5 additions & 0 deletions graylog2-server/src/main/java/org/graylog2/plugin/indexer/searches/timeranges/TimeRange.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,9 @@ public abstract class TimeRange {
public abstract DateTime getTo();

public abstract TimeRange withReferenceDate(DateTime now);

@JsonIgnore
public boolean isAllMessages() {
return false;
}
}
Loading

0 comments on commit 9fca2ee

Please sign in to comment.