Skip to content

Commit

Permalink
support object fields in aggregation based sigma rules (opensearch-pr…
Browse files Browse the repository at this point in the history 
…oject#789)

Signed-off-by: Subhobrata Dey <[email protected]>
  • Loading branch information
@sbcd90 @jowg-amazon
sbcd90 authored and jowg-amazon committed Mar 14, 2024
1 parent aa4706d commit f11acc5
Show file tree
Hide file tree
Showing 7 changed files with 1,033 additions and 102 deletions.
88 changes: 44 additions & 44 deletions ...erated/org/opensearch/securityanalytics/rules/condition/aggregation/AggregationLexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,50 +129,50 @@ public AggregationLexer(CharStream input) {
"\u0001\u000f\u0001\u000f\u0000\u0000\u0010\u0001\u0001\u0003\u0002\u0005"+
"\u0003\u0007\u0004\t\u0005\u000b\u0006\r\u0007\u000f\b\u0011\t\u0013\n"+
"\u0015\u000b\u0017\f\u0019\r\u001b\u000e\u001d\u000f\u001f\u0010\u0001"+
"\u0000\u0004\u0001\u000009\u0004\u0000**AZ__az\u0004\u000009AZ__az\u0003"+
"\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003\u0001"+
"\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007\u0001"+
"\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001\u0000"+
"\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000\u0000"+
"\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000\u0000"+
"\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000\u0000"+
"\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000\u0000"+
"\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000\u0000"+
"\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000\u0005"+
"&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001\u0000"+
"\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000\u0000"+
"\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013@"+
"\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001\u0000"+
"\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000\u0000"+
"\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!\"\u0005"+
">\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000$%\u0005"+
"=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000\'\u0006"+
"\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000*\b"+
"\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000-\n"+
"\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u000001\u0005"+
"u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001\u0000"+
"\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m\u0000"+
"\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005i\u0000"+
"\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005m\u0000"+
"\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000\u0000"+
"\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000\u0000"+
"C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000\u0000"+
"F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001\u0000"+
"\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000KM\u0005"+
"-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000MO\u0001"+
"\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000\u0000"+
"PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000\u0000"+
"\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000\u0000"+
"\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001\u0000"+
"\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000YS\u0001"+
"\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000\u0000"+
"\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001\u0000"+
"\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000_`\u0001"+
"\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000\u0000"+
"\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001\u0000"+
"\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000fg\u0001"+
"\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000\u0000"+
"\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
"\u0000\u0004\u0001\u000009\u0005\u0000**..AZ__az\u0005\u0000..09AZ__a"+
"z\u0003\u0000\t\n\f\r n\u0000\u0001\u0001\u0000\u0000\u0000\u0000\u0003"+
"\u0001\u0000\u0000\u0000\u0000\u0005\u0001\u0000\u0000\u0000\u0000\u0007"+
"\u0001\u0000\u0000\u0000\u0000\t\u0001\u0000\u0000\u0000\u0000\u000b\u0001"+
"\u0000\u0000\u0000\u0000\r\u0001\u0000\u0000\u0000\u0000\u000f\u0001\u0000"+
"\u0000\u0000\u0000\u0011\u0001\u0000\u0000\u0000\u0000\u0013\u0001\u0000"+
"\u0000\u0000\u0000\u0015\u0001\u0000\u0000\u0000\u0000\u0017\u0001\u0000"+
"\u0000\u0000\u0000\u0019\u0001\u0000\u0000\u0000\u0000\u001b\u0001\u0000"+
"\u0000\u0000\u0000\u001d\u0001\u0000\u0000\u0000\u0000\u001f\u0001\u0000"+
"\u0000\u0000\u0001!\u0001\u0000\u0000\u0000\u0003#\u0001\u0000\u0000\u0000"+
"\u0005&\u0001\u0000\u0000\u0000\u0007(\u0001\u0000\u0000\u0000\t+\u0001"+
"\u0000\u0000\u0000\u000b.\u0001\u0000\u0000\u0000\r4\u0001\u0000\u0000"+
"\u0000\u000f8\u0001\u0000\u0000\u0000\u0011<\u0001\u0000\u0000\u0000\u0013"+
"@\u0001\u0000\u0000\u0000\u0015D\u0001\u0000\u0000\u0000\u0017G\u0001"+
"\u0000\u0000\u0000\u0019I\u0001\u0000\u0000\u0000\u001bL\u0001\u0000\u0000"+
"\u0000\u001d[\u0001\u0000\u0000\u0000\u001fc\u0001\u0000\u0000\u0000!"+
"\"\u0005>\u0000\u0000\"\u0002\u0001\u0000\u0000\u0000#$\u0005>\u0000\u0000"+
"$%\u0005=\u0000\u0000%\u0004\u0001\u0000\u0000\u0000&\'\u0005<\u0000\u0000"+
"\'\u0006\u0001\u0000\u0000\u0000()\u0005<\u0000\u0000)*\u0005=\u0000\u0000"+
"*\b\u0001\u0000\u0000\u0000+,\u0005=\u0000\u0000,-\u0005=\u0000\u0000"+
"-\n\u0001\u0000\u0000\u0000./\u0005c\u0000\u0000/0\u0005o\u0000\u0000"+
"01\u0005u\u0000\u000012\u0005n\u0000\u000023\u0005t\u0000\u00003\f\u0001"+
"\u0000\u0000\u000045\u0005s\u0000\u000056\u0005u\u0000\u000067\u0005m"+
"\u0000\u00007\u000e\u0001\u0000\u0000\u000089\u0005m\u0000\u00009:\u0005"+
"i\u0000\u0000:;\u0005n\u0000\u0000;\u0010\u0001\u0000\u0000\u0000<=\u0005"+
"m\u0000\u0000=>\u0005a\u0000\u0000>?\u0005x\u0000\u0000?\u0012\u0001\u0000"+
"\u0000\u0000@A\u0005a\u0000\u0000AB\u0005v\u0000\u0000BC\u0005g\u0000"+
"\u0000C\u0014\u0001\u0000\u0000\u0000DE\u0005b\u0000\u0000EF\u0005y\u0000"+
"\u0000F\u0016\u0001\u0000\u0000\u0000GH\u0005(\u0000\u0000H\u0018\u0001"+
"\u0000\u0000\u0000IJ\u0005)\u0000\u0000J\u001a\u0001\u0000\u0000\u0000"+
"KM\u0005-\u0000\u0000LK\u0001\u0000\u0000\u0000LM\u0001\u0000\u0000\u0000"+
"MO\u0001\u0000\u0000\u0000NP\u0007\u0000\u0000\u0000ON\u0001\u0000\u0000"+
"\u0000PQ\u0001\u0000\u0000\u0000QO\u0001\u0000\u0000\u0000QR\u0001\u0000"+
"\u0000\u0000RY\u0001\u0000\u0000\u0000SU\u0005.\u0000\u0000TV\u0007\u0000"+
"\u0000\u0000UT\u0001\u0000\u0000\u0000VW\u0001\u0000\u0000\u0000WU\u0001"+
"\u0000\u0000\u0000WX\u0001\u0000\u0000\u0000XZ\u0001\u0000\u0000\u0000"+
"YS\u0001\u0000\u0000\u0000YZ\u0001\u0000\u0000\u0000Z\u001c\u0001\u0000"+
"\u0000\u0000[_\u0007\u0001\u0000\u0000\\^\u0007\u0002\u0000\u0000]\\\u0001"+
"\u0000\u0000\u0000^a\u0001\u0000\u0000\u0000_]\u0001\u0000\u0000\u0000"+
"_`\u0001\u0000\u0000\u0000`\u001e\u0001\u0000\u0000\u0000a_\u0001\u0000"+
"\u0000\u0000bd\u0007\u0003\u0000\u0000cb\u0001\u0000\u0000\u0000de\u0001"+
"\u0000\u0000\u0000ec\u0001\u0000\u0000\u0000ef\u0001\u0000\u0000\u0000"+
"fg\u0001\u0000\u0000\u0000gh\u0006\u000f\u0000\u0000h \u0001\u0000\u0000"+
"\u0000\u0007\u0000LQWY_e\u0001\u0006\u0000\u0000";
public static final ATN _ATN =
new ATNDeserializer().deserialize(_serializedATN.toCharArray());
static {
Expand Down
2 changes: 1 addition & 1 deletion src/main/grammars/Aggregation.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ RPAREN : ')' ;

DECIMAL : '-'?[0-9]+('.'[0-9]+)? ;

IDENTIFIER : [a-zA-Z*_][a-zA-Z_0-9]* ;
IDENTIFIER : [a-zA-Z*_.][a-zA-Z_0-9.]* ;
WS : [ \r\t\u000C\n]+ -> skip ;

comparison_expr : comparison_operand comp_operator comparison_operand # ComparisonExpressionWithOperator
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/opensearch/securityanalytics/rules/backend/AggregationBuilders.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public static AggregationBuilder getAggregationBuilderByFunction(String aggregat
aggregationBuilder = new TermsAggregationBuilder(name).field(name);
break;
case "count":
aggregationBuilder = new ValueCountAggregationBuilder(name).field(name);
aggregationBuilder = new ValueCountAggregationBuilder(name.replace(".", "_")).field(name);
break;
default:
throw new NotImplementedException(String.format(Locale.getDefault(), "Aggregation %s not supported by the backend", aggregationFunction));
Expand Down
27 changes: 17 additions & 10 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,32 +361,39 @@ public AggregationQueries convertAggregation(AggregationItem aggregation) {
BucketSelectorExtAggregationBuilder condition;
String bucketTriggerSelectorId = UUIDs.base64UUID();

if (aggregation.getAggFunction().equals("count")) {
if (aggregation.getAggFunction().equals("count") && aggregation.getAggField().equals("*")) {
String fieldName;
if (aggregation.getAggField().equals("*") && aggregation.getGroupByField() == null) {
if (aggregation.getGroupByField() == null) {
fieldName = "_index";
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", "_index");
} else {
fieldName = aggregation.getGroupByField();
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", aggregation.getGroupByField());
String mappedGroupByField = getMappedField(aggregation.getGroupByField());
fieldName = mappedGroupByField;
fmtAggQuery = String.format(Locale.getDefault(), aggCountQuery, "result_agg", mappedGroupByField);
}
aggBuilder.field(fieldName);
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, "_cnt", "_count", "result_agg", "_cnt", aggregation.getCompOperator(), aggregation.getThreshold());

Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, "_cnt", aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap("_cnt", "_count"), script, "result_agg", null);
} else {
fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", aggregation.getGroupByField(), aggregation.getAggField(), aggregation.getAggFunction(), aggregation.getAggField());
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, aggregation.getAggField(), aggregation.getAggField(), "result_agg", aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold());
/**
* removing dots to eliminate dots in aggregation names
*/
String mappedAggField = getFinalField(aggregation.getAggField());
String mappedAggFieldUpdated = mappedAggField.replace(".", "_");
String mappedGroupByField = getMappedField(aggregation.getGroupByField());
fmtAggQuery = String.format(Locale.getDefault(), aggQuery, "result_agg", mappedGroupByField, mappedAggFieldUpdated, aggregation.getAggFunction().equals("count")? "value_count": aggregation.getAggFunction(), mappedAggField);
fmtBucketTriggerQuery = String.format(Locale.getDefault(), bucketTriggerQuery, mappedAggFieldUpdated, mappedAggField, "result_agg", mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold());

// Add subaggregation
AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), aggregation.getAggField());
AggregationBuilder subAgg = AggregationBuilders.getAggregationBuilderByFunction(aggregation.getAggFunction(), mappedAggField);
if (subAgg != null) {
aggBuilder.field(aggregation.getGroupByField()).subAggregation(subAgg);
aggBuilder.field(mappedGroupByField).subAggregation(subAgg);
}

Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, aggregation.getAggField(), aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(aggregation.getAggField(), aggregation.getAggField()), script, "result_agg", null);
Script script = new Script(String.format(Locale.getDefault(), bucketTriggerScript, mappedAggFieldUpdated, aggregation.getCompOperator(), aggregation.getThreshold()));
condition = new BucketSelectorExtAggregationBuilder(bucketTriggerSelectorId, Collections.singletonMap(mappedAggFieldUpdated, mappedAggFieldUpdated), script, "result_agg", null);
}

AggregationQueries aggregationQueries = new AggregationQueries();
Expand Down
Loading

0 comments on commit f11acc5

Please sign in to comment.