Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Logtypes pr v2 #475

Merged
merged 18 commits into from
Jul 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ jobs:
os: [ ubuntu-latest, windows-latest, macos-latest ]
include:
- os: windows-latest
os_build_args: -x jacocoTestReport
os_build_args: -x integTest -x jacocoTestReport
working_directory: X:\
os_java_options: -Xmx4096M
- os: macos-latest
os_build_args: -x jacocoTestReport
os_build_args: -x integTest -x jacocoTestReport

name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,16 @@
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.ActionListener;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionResponse;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.component.LifecycleComponent;
Expand All @@ -31,7 +37,9 @@
import org.opensearch.index.codec.CodecServiceFactory;
import org.opensearch.index.engine.EngineFactory;
import org.opensearch.index.mapper.Mapper;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.plugins.ActionPlugin;
import org.opensearch.plugins.ClusterPlugin;
import org.opensearch.plugins.EnginePlugin;
import org.opensearch.plugins.MapperPlugin;
import org.opensearch.plugins.Plugin;
Expand All @@ -40,11 +48,13 @@
import org.opensearch.rest.RestController;
import org.opensearch.rest.RestHandler;
import org.opensearch.script.ScriptService;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.action.*;
import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder;
import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader;
import org.opensearch.securityanalytics.logtype.LogTypeService;
import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
import org.opensearch.securityanalytics.mapper.MapperService;
Expand All @@ -62,7 +72,9 @@
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin {
public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin {

private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class);

public static final String PLUGINS_BASE_URI = "/_plugins/_security_analytics";
public static final String MAPPER_BASE_URI = PLUGINS_BASE_URI + "/mappings";
Expand Down Expand Up @@ -91,8 +103,12 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map

private IndexTemplateManager indexTemplateManager;

private BuiltinLogTypeLoader builtinLogTypeLoader;

private LogTypeService logTypeService;

private Client client;

@Override
public Collection<Object> createComponents(Client client,
ClusterService clusterService,
Expand All @@ -105,21 +121,26 @@ public Collection<Object> createComponents(Client client,
NamedWriteableRegistry namedWriteableRegistry,
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<RepositoriesService> repositoriesServiceSupplier) {
logTypeService = new LogTypeService();
builtinLogTypeLoader = new BuiltinLogTypeLoader();
logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader);
detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
ruleTopicIndices = new RuleTopicIndices(client, clusterService);
ruleTopicIndices = new RuleTopicIndices(client, clusterService, logTypeService);
correlationIndices = new CorrelationIndices(client, clusterService);
indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry);
mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager);
mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService);
ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool);
correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
this.client = client;

return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices,
mapperService, indexTemplateManager, builtinLogTypeLoader
);
}

@Override
public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
return Collections.singletonList(DetectorIndexManagementService.class);
return List.of(DetectorIndexManagementService.class, BuiltinLogTypeLoader.class);
}

@Override
Expand Down Expand Up @@ -180,7 +201,7 @@ public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
@Override
public Optional<CodecServiceFactory> getCustomCodecServiceFactory(IndexSettings indexSettings) {
if (indexSettings.getValue(SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING)) {
return Optional.of(CorrelationCodecService::new);
return Optional.of(config -> new CorrelationCodecService(config, indexSettings));
}
return Optional.empty();
}
Expand Down Expand Up @@ -208,7 +229,8 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW,
SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA
);
}

Expand Down Expand Up @@ -239,4 +261,38 @@ public List<Setting<?>> getSettings() {
new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class)
);
}

@Override
public void onNodeStarted(DiscoveryNode localNode) {
// Trigger initialization of log types
logTypeService.ensureConfigIndexIsInitialized(new ActionListener<>() {
@Override
public void onResponse(Void unused) {
log.info("LogType config index successfully created and builtin log types loaded");
}

@Override
public void onFailure(Exception e) {
log.warn("Failed to initialize LogType config index and builtin log types");
}
});
// Trigger initialization of prepackaged rules by calling SearchRule API
SearchRequest searchRequest = new SearchRequest(Rule.PRE_PACKAGED_RULES_INDEX);
searchRequest.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(0));
client.execute(
SearchRuleAction.INSTANCE,
new SearchRuleRequest(true, searchRequest),
new ActionListener<>() {
@Override
public void onResponse(SearchResponse searchResponse) {
log.info("Successfully initialized prepackaged rules");
}

@Override
public void onFailure(Exception e) {
log.warn("Failed initializing prepackaged rules", e);
}
}
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
public class GetAlertsRequest extends ActionRequest {

private String detectorId;
private Detector.DetectorType detectorType;
private String logType;
private Table table;
private String severityLevel;
private String alertState;
Expand All @@ -28,22 +28,22 @@ public class GetAlertsRequest extends ActionRequest {

public GetAlertsRequest(
String detectorId,
Detector.DetectorType detectorType,
String logType,
Table table,
String severityLevel,
String alertState
) {
super();
this.detectorId = detectorId;
this.detectorType = detectorType;
this.logType = logType;
this.table = table;
this.severityLevel = severityLevel;
this.alertState = alertState;
}
public GetAlertsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalString(),
sin.readBoolean() ? sin.readEnum(Detector.DetectorType.class) : null,
sin.readOptionalString(),
Table.readFrom(sin),
sin.readString(),
sin.readString()
Expand All @@ -53,7 +53,7 @@ public GetAlertsRequest(StreamInput sin) throws IOException {
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if ((detectorId == null || detectorId.length() == 0) && detectorType == null) {
if ((detectorId == null || detectorId.length() == 0) && logType == null) {
validationException = addValidationError(String.format(Locale.getDefault(),
"At least one of detector type or detector id needs to be passed", DETECTOR_ID),
validationException);
Expand All @@ -64,10 +64,7 @@ public ActionRequestValidationException validate() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(detectorId);
if (detectorType != null) {
out.writeBoolean(true);
out.writeEnum(detectorType);
} else out.writeBoolean(false);
out.writeOptionalString(logType);
table.writeTo(out);
out.writeString(severityLevel);
out.writeString(alertState);
Expand All @@ -89,7 +86,7 @@ public String getAlertState() {
return alertState;
}

public Detector.DetectorType getDetectorType() {
return detectorType;
public String getLogType() {
return logType;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

public class GetFindingsRequest extends ActionRequest {

private Detector.DetectorType detectorType;
private String logType;
private String detectorId;
private Table table;

Expand All @@ -31,21 +31,21 @@ public GetFindingsRequest(String detectorId) {
public GetFindingsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalString(),
sin.readBoolean() ? sin.readEnum(Detector.DetectorType.class) : null,
sin.readOptionalString(),
Table.readFrom(sin)
);
}

public GetFindingsRequest(String detectorId, Detector.DetectorType detectorType, Table table) {
public GetFindingsRequest(String detectorId, String logType, Table table) {
this.detectorId = detectorId;
this.detectorType = detectorType;
this.logType = logType;
this.table = table;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if ((detectorId == null || detectorId.length() == 0) && detectorType == null) {
if ((detectorId == null || detectorId.length() == 0) && logType == null) {
validationException = addValidationError(String.format(Locale.getDefault(),
"At least one of detector type or detector id needs to be passed", DETECTOR_ID),
validationException);
Expand All @@ -56,21 +56,16 @@ public ActionRequestValidationException validate() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(detectorId);
if (detectorType != null) {
out.writeBoolean(true);
out.writeEnum(detectorType);
} else {
out.writeBoolean(false);
}
out.writeOptionalString(logType);
table.writeTo(out);
}

public String getDetectorId() {
return detectorId;
}

public Detector.DetectorType getDetectorType() {
return detectorType;
public String getLogType() {
return logType;
}

public Table getTable() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,6 @@ public ActionRequestValidationException validate() {

if (logType == null || logType.length() == 0) {
validationException = addValidationError("rule categoty is missing", validationException);
} else {
Optional<Detector.DetectorType> found =
Arrays.stream(Detector.DetectorType.values())
.filter(e -> e.getDetectorType().equals(logType))
.findFirst();
if (found.isPresent() == false) {
validationException = addValidationError("Invalid rule category", validationException);
}
}
return validationException;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ public void getAlertsByMonitorIds(
null,
alertIndex,
monitorIds,
null,
null
);

Expand Down Expand Up @@ -173,7 +174,7 @@ void setIndicesAdminClient(Client client) {

public void getAlerts(
List<Detector> detectors,
Detector.DetectorType detectorType,
String logType,
Table table,
String severityLevel,
String alertState,
Expand All @@ -199,7 +200,7 @@ public void getAlerts(
AlertsService.this.getAlertsByMonitorIds(
monitorToDetectorMapping,
allMonitorIds,
DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType.getDetectorType()),
DetectorMonitorConfig.getAllAlertsIndicesPattern(logType),
table,
severityLevel,
alertState,
Expand Down Expand Up @@ -253,6 +254,7 @@ public void getAlerts(List<String> alertIds,
null,
DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()),
null,
null,
alertIds);
AlertingPluginInterface.INSTANCE.getAlerts(
(NodeClient) client,
Expand Down
Loading