Skip to content

Commit

Permalink
[backport 2.x] Logtypes pr v2
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@petardz @sbcd90
petardz authored and sbcd90 committed Jul 11, 2023
1 parent 2e869e5 commit 85eceb6
Show file tree
Hide file tree
Showing 100 changed files with 5,043 additions and 2,182 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ jobs:
os: [ ubuntu-latest, windows-latest, macos-latest ]
include:
- os: windows-latest
os_build_args: -x jacocoTestReport
os_build_args: -x integTest -x jacocoTestReport
working_directory: X:\
os_java_options: -Xmx4096M
- os: macos-latest
os_build_args: -x jacocoTestReport
os_build_args: -x integTest -x jacocoTestReport

name: Build and Test security-analytics with JDK ${{ matrix.java }} on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
Expand Down
72 changes: 64 additions & 8 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,16 @@
import java.util.Map;
import java.util.Optional;
import java.util.function.Supplier;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.opensearch.action.ActionListener;
import org.opensearch.action.ActionRequest;
import org.opensearch.action.ActionResponse;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.Client;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
import org.opensearch.cluster.node.DiscoveryNode;
import org.opensearch.cluster.node.DiscoveryNodes;
import org.opensearch.cluster.service.ClusterService;
import org.opensearch.common.component.LifecycleComponent;
Expand All @@ -31,7 +37,9 @@
import org.opensearch.index.codec.CodecServiceFactory;
import org.opensearch.index.engine.EngineFactory;
import org.opensearch.index.mapper.Mapper;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.plugins.ActionPlugin;
import org.opensearch.plugins.ClusterPlugin;
import org.opensearch.plugins.EnginePlugin;
import org.opensearch.plugins.MapperPlugin;
import org.opensearch.plugins.Plugin;
Expand All @@ -40,11 +48,13 @@
import org.opensearch.rest.RestController;
import org.opensearch.rest.RestHandler;
import org.opensearch.script.ScriptService;
import org.opensearch.search.builder.SearchSourceBuilder;
import org.opensearch.securityanalytics.action.*;
import org.opensearch.securityanalytics.correlation.index.codec.CorrelationCodecService;
import org.opensearch.securityanalytics.correlation.index.mapper.CorrelationVectorFieldMapper;
import org.opensearch.securityanalytics.correlation.index.query.CorrelationQueryBuilder;
import org.opensearch.securityanalytics.indexmanagment.DetectorIndexManagementService;
import org.opensearch.securityanalytics.logtype.BuiltinLogTypeLoader;
import org.opensearch.securityanalytics.logtype.LogTypeService;
import org.opensearch.securityanalytics.mapper.IndexTemplateManager;
import org.opensearch.securityanalytics.mapper.MapperService;
Expand All @@ -62,7 +72,9 @@
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.watcher.ResourceWatcherService;

public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin {
public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin {

private static final Logger log = LogManager.getLogger(SecurityAnalyticsPlugin.class);

public static final String PLUGINS_BASE_URI = "/_plugins/_security_analytics";
public static final String MAPPER_BASE_URI = PLUGINS_BASE_URI + "/mappings";
Expand Down Expand Up @@ -91,8 +103,12 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map

private IndexTemplateManager indexTemplateManager;

private BuiltinLogTypeLoader builtinLogTypeLoader;

private LogTypeService logTypeService;

private Client client;

@Override
public Collection<Object> createComponents(Client client,
ClusterService clusterService,
Expand All @@ -105,21 +121,26 @@ public Collection<Object> createComponents(Client client,
NamedWriteableRegistry namedWriteableRegistry,
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<RepositoriesService> repositoriesServiceSupplier) {
logTypeService = new LogTypeService();
builtinLogTypeLoader = new BuiltinLogTypeLoader();
logTypeService = new LogTypeService(client, clusterService, xContentRegistry, builtinLogTypeLoader);
detectorIndices = new DetectorIndices(client.admin(), clusterService, threadPool);
ruleTopicIndices = new RuleTopicIndices(client, clusterService);
ruleTopicIndices = new RuleTopicIndices(client, clusterService, logTypeService);
correlationIndices = new CorrelationIndices(client, clusterService);
indexTemplateManager = new IndexTemplateManager(client, clusterService, indexNameExpressionResolver, xContentRegistry);
mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager);
mapperService = new MapperService(client, clusterService, indexNameExpressionResolver, indexTemplateManager, logTypeService);
ruleIndices = new RuleIndices(logTypeService, client, clusterService, threadPool);
correlationRuleIndices = new CorrelationRuleIndices(client, clusterService);
this.client = client;

return List.of(detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices, mapperService, indexTemplateManager);
return List.of(
detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, ruleIndices,
mapperService, indexTemplateManager, builtinLogTypeLoader
);
}

@Override
public Collection<Class<? extends LifecycleComponent>> getGuiceServiceClasses() {
return Collections.singletonList(DetectorIndexManagementService.class);
return List.of(DetectorIndexManagementService.class, BuiltinLogTypeLoader.class);
}

@Override
Expand Down Expand Up @@ -180,7 +201,7 @@ public Optional<EngineFactory> getEngineFactory(IndexSettings indexSettings) {
@Override
public Optional<CodecServiceFactory> getCustomCodecServiceFactory(IndexSettings indexSettings) {
if (indexSettings.getValue(SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING)) {
return Optional.of(CorrelationCodecService::new);
return Optional.of(config -> new CorrelationCodecService(config, indexSettings));
}
return Optional.empty();
}
Expand Down Expand Up @@ -208,7 +229,8 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW,
SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA
);
}

Expand Down Expand Up @@ -239,4 +261,38 @@ public List<Setting<?>> getSettings() {
new ActionPlugin.ActionHandler<>(SearchCorrelationRuleAction.INSTANCE, TransportSearchCorrelationRuleAction.class)
);
}

@Override
public void onNodeStarted(DiscoveryNode localNode) {
// Trigger initialization of log types
logTypeService.ensureConfigIndexIsInitialized(new ActionListener<>() {
@Override
public void onResponse(Void unused) {
log.info("LogType config index successfully created and builtin log types loaded");
}

@Override
public void onFailure(Exception e) {
log.warn("Failed to initialize LogType config index and builtin log types");
}
});
// Trigger initialization of prepackaged rules by calling SearchRule API
SearchRequest searchRequest = new SearchRequest(Rule.PRE_PACKAGED_RULES_INDEX);
searchRequest.source(new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(0));
client.execute(
SearchRuleAction.INSTANCE,
new SearchRuleRequest(true, searchRequest),
new ActionListener<>() {
@Override
public void onResponse(SearchResponse searchResponse) {
log.info("Successfully initialized prepackaged rules");
}

@Override
public void onFailure(Exception e) {
log.warn("Failed initializing prepackaged rules", e);
}
}
);
}
}
19 changes: 8 additions & 11 deletions src/main/java/org/opensearch/securityanalytics/action/GetAlertsRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
public class GetAlertsRequest extends ActionRequest {

private String detectorId;
private Detector.DetectorType detectorType;
private String logType;
private Table table;
private String severityLevel;
private String alertState;
Expand All @@ -28,22 +28,22 @@ public class GetAlertsRequest extends ActionRequest {

public GetAlertsRequest(
String detectorId,
Detector.DetectorType detectorType,
String logType,
Table table,
String severityLevel,
String alertState
) {
super();
this.detectorId = detectorId;
this.detectorType = detectorType;
this.logType = logType;
this.table = table;
this.severityLevel = severityLevel;
this.alertState = alertState;
}
public GetAlertsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalString(),
sin.readBoolean() ? sin.readEnum(Detector.DetectorType.class) : null,
sin.readOptionalString(),
Table.readFrom(sin),
sin.readString(),
sin.readString()
Expand All @@ -53,7 +53,7 @@ public GetAlertsRequest(StreamInput sin) throws IOException {
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if ((detectorId == null || detectorId.length() == 0) && detectorType == null) {
if ((detectorId == null || detectorId.length() == 0) && logType == null) {
validationException = addValidationError(String.format(Locale.getDefault(),
"At least one of detector type or detector id needs to be passed", DETECTOR_ID),
validationException);
Expand All @@ -64,10 +64,7 @@ public ActionRequestValidationException validate() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(detectorId);
if (detectorType != null) {
out.writeBoolean(true);
out.writeEnum(detectorType);
} else out.writeBoolean(false);
out.writeOptionalString(logType);
table.writeTo(out);
out.writeString(severityLevel);
out.writeString(alertState);
Expand All @@ -89,7 +86,7 @@ public String getAlertState() {
return alertState;
}

public Detector.DetectorType getDetectorType() {
return detectorType;
public String getLogType() {
return logType;
}
}
21 changes: 8 additions & 13 deletions src/main/java/org/opensearch/securityanalytics/action/GetFindingsRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

public class GetFindingsRequest extends ActionRequest {

private Detector.DetectorType detectorType;
private String logType;
private String detectorId;
private Table table;

Expand All @@ -31,21 +31,21 @@ public GetFindingsRequest(String detectorId) {
public GetFindingsRequest(StreamInput sin) throws IOException {
this(
sin.readOptionalString(),
sin.readBoolean() ? sin.readEnum(Detector.DetectorType.class) : null,
sin.readOptionalString(),
Table.readFrom(sin)
);
}

public GetFindingsRequest(String detectorId, Detector.DetectorType detectorType, Table table) {
public GetFindingsRequest(String detectorId, String logType, Table table) {
this.detectorId = detectorId;
this.detectorType = detectorType;
this.logType = logType;
this.table = table;
}

@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException validationException = null;
if ((detectorId == null || detectorId.length() == 0) && detectorType == null) {
if ((detectorId == null || detectorId.length() == 0) && logType == null) {
validationException = addValidationError(String.format(Locale.getDefault(),
"At least one of detector type or detector id needs to be passed", DETECTOR_ID),
validationException);
Expand All @@ -56,21 +56,16 @@ public ActionRequestValidationException validate() {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeOptionalString(detectorId);
if (detectorType != null) {
out.writeBoolean(true);
out.writeEnum(detectorType);
} else {
out.writeBoolean(false);
}
out.writeOptionalString(logType);
table.writeTo(out);
}

public String getDetectorId() {
return detectorId;
}

public Detector.DetectorType getDetectorType() {
return detectorType;
public String getLogType() {
return logType;
}

public Table getTable() {
Expand Down
8 changes: 0 additions & 8 deletions src/main/java/org/opensearch/securityanalytics/action/IndexRuleRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,14 +85,6 @@ public ActionRequestValidationException validate() {

if (logType == null || logType.length() == 0) {
validationException = addValidationError("rule categoty is missing", validationException);
} else {
Optional<Detector.DetectorType> found =
Arrays.stream(Detector.DetectorType.values())
.filter(e -> e.getDetectorType().equals(logType))
.findFirst();
if (found.isPresent() == false) {
validationException = addValidationError("Invalid rule category", validationException);
}
}
return validationException;
}
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/opensearch/securityanalytics/alerts/AlertsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ void setIndicesAdminClient(Client client) {

public void getAlerts(
List<Detector> detectors,
Detector.DetectorType detectorType,
String logType,
Table table,
String severityLevel,
String alertState,
Expand All @@ -200,7 +200,7 @@ public void getAlerts(
AlertsService.this.getAlertsByMonitorIds(
monitorToDetectorMapping,
allMonitorIds,
DetectorMonitorConfig.getAllAlertsIndicesPattern(detectorType.getDetectorType()),
DetectorMonitorConfig.getAllAlertsIndicesPattern(logType),
table,
severityLevel,
alertState,
Expand Down Expand Up @@ -254,8 +254,8 @@ public void getAlerts(List<String> alertIds,
null,
DetectorMonitorConfig.getAllAlertsIndicesPattern(detector.getDetectorType()),
null,
alertIds,
null);
null,
alertIds);
AlertingPluginInterface.INSTANCE.getAlerts(
(NodeClient) client,
request, actionListener);
Expand Down
Loading

0 comments on commit 85eceb6

Please sign in to comment.