Skip to content

Commit

Permalink
Add an "exists" check for "not" condition in sigma rules (opensearch-…
Browse files Browse the repository at this point in the history 
…project#852) (opensearch-project#897)

* test design

Signed-off-by: Joanne Wang <[email protected]>

* working version

Signed-off-by: Joanne Wang <[email protected]>

* cleaning up

Signed-off-by: Joanne Wang <[email protected]>

* testing

Signed-off-by: Joanne Wang <[email protected]>

* working version

Signed-off-by: Joanne Wang <[email protected]>

* working version

Signed-off-by: Joanne Wang <[email protected]>

* refactored querybackend

Signed-off-by: Joanne Wang <[email protected]>

* working on tests

Signed-off-by: Joanne Wang <[email protected]>

* fixed alerting and finding tests

Signed-off-by: Joanne Wang <[email protected]>

* fix correlation tests

Signed-off-by: Joanne Wang <[email protected]>

* working all tests

Signed-off-by: Joanne Wang <[email protected]>

* moved test and changed alias for adldap

Signed-off-by: Joanne Wang <[email protected]>

* added more tests

Signed-off-by: Joanne Wang <[email protected]>

* cleanup code

Signed-off-by: Joanne Wang <[email protected]>

* remove exists flag

Signed-off-by: Joanne Wang <[email protected]>

---------

Signed-off-by: Joanne Wang <[email protected]>
(cherry picked from commit 656a5fe)

Co-authored-by: Joanne Wang <[email protected]>
  • Loading branch information
@opensearch-trigger-bot @jowg-amazon
opensearch-trigger-bot[bot] and jowg-amazon authored Mar 8, 2024
1 parent ad3f5a5 commit 6d49626
Show file tree
Hide file tree
Showing 7 changed files with 779 additions and 89 deletions.
142 changes: 105 additions & 37 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file

Large diffs are not rendered by default.

102 changes: 56 additions & 46 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
*/
package org.opensearch.securityanalytics.rules.backend;

import org.opensearch.commons.alerting.aggregation.bucketselectorext.BucketSelectorExtAggregationBuilder;
import org.opensearch.search.aggregations.AggregationBuilder;
import org.opensearch.securityanalytics.rules.aggregation.AggregationItem;
import org.opensearch.securityanalytics.rules.backend.OSQueryBackend.AggregationQueries;
import org.opensearch.securityanalytics.rules.condition.ConditionAND;
Expand Down Expand Up @@ -47,7 +45,6 @@
import java.util.Set;

public abstract class QueryBackend {

private boolean convertOrAsIn;
private boolean convertAndAsIn;
private boolean collectErrors;
Expand Down Expand Up @@ -85,15 +82,15 @@ public List<Object> convertRule(SigmaRule rule) throws SigmaError {

Object query;
if (conditionItem instanceof ConditionAND) {
query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) conditionItem))));
query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.leftVal((ConditionAND) conditionItem))), false, false);
} else if (conditionItem instanceof ConditionOR) {
query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) conditionItem))));
query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.middleVal((ConditionOR) conditionItem))), false, false);
} else if (conditionItem instanceof ConditionNOT) {
query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) conditionItem))));
query = this.convertCondition(new ConditionType(Either.left(AnyOneOf.rightVal((ConditionNOT) conditionItem))), true, false);
} else if (conditionItem instanceof ConditionFieldEqualsValueExpression) {
query = this.convertCondition(new ConditionType(Either.right(Either.left((ConditionFieldEqualsValueExpression) conditionItem))));
query = this.convertCondition(new ConditionType(Either.right(Either.left((ConditionFieldEqualsValueExpression) conditionItem))), false, false);
} else {
query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))));
query = this.convertCondition(new ConditionType(Either.right(Either.right((ConditionValueExpression) conditionItem))), false, false);
}
queries.add(query);
if (aggItem != null) {
Expand All @@ -113,30 +110,41 @@ public List<Object> convertRule(SigmaRule rule) throws SigmaError {
return queries;
}

public Object convertCondition(ConditionType conditionType) throws SigmaValueError {
public Object convertCondition(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError {
if (conditionType.isConditionOR()) {
if (this.decideConvertConditionAsInExpression(Either.right(conditionType.getConditionOR()))) {
return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR()));
return this.convertConditionAsInExpression(Either.right(conditionType.getConditionOR()), isConditionNot, applyDeMorgans );
} else {
return this.convertConditionOr(conditionType.getConditionOR());
return this.convertConditionOr(conditionType.getConditionOR(), isConditionNot, applyDeMorgans);
}
} else if (conditionType.isConditionAND()) {
if (this.decideConvertConditionAsInExpression(Either.left(conditionType.getConditionAND()))) {
return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND()));
return this.convertConditionAsInExpression(Either.left(conditionType.getConditionAND()), isConditionNot, applyDeMorgans);
} else {
return this.convertConditionAnd(conditionType.getConditionAND());
return this.convertConditionAnd(conditionType.getConditionAND(), isConditionNot, applyDeMorgans);
}
} else if (conditionType.isConditionNOT()) {
return this.convertConditionNot(conditionType.getConditionNOT());
return this.convertConditionNot(conditionType.getConditionNOT(), isConditionNot, applyDeMorgans);
} else if (conditionType.isEqualsValueExpression()) {
return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression());
// check to see if conditionNot is an ancestor of the parse tree, otherwise return as normal
if (isConditionNot) {
return this.convertConditionFieldEqValNot(conditionType, isConditionNot, applyDeMorgans);
} else {
return this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans);
}
} else if (conditionType.isValueExpression()) {
return this.convertConditionVal(conditionType.getValueExpression());
return this.convertConditionVal(conditionType.getValueExpression(), applyDeMorgans);
} else {
throw new IllegalArgumentException("Unexpected data type in condition parse tree");
}
}

public String convertConditionFieldEqValNot(ConditionType conditionType, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError {
String baseString = this.convertConditionFieldEqVal(conditionType.getEqualsValueExpression(), isConditionNot, applyDeMorgans).toString();
String addExists = this.convertExistsField(conditionType.getEqualsValueExpression()).toString();
return String.format(Locale.getDefault(), ("%s" + "%s"), baseString, addExists);
}

public boolean decideConvertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition) {
if ((!this.convertOrAsIn && condition.isRight()) || (!this.convertAndAsIn && condition.isLeft())) {
return false;
Expand Down Expand Up @@ -181,74 +189,76 @@ public void resetQueryFields() {
}
}

public abstract Object convertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition);
public abstract Object convertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition, boolean isConditionNot, boolean applyDeMorgans);

public abstract Object convertConditionAnd(ConditionAND condition);
public abstract Object convertConditionAnd(ConditionAND condition, boolean isConditionNot, boolean applyDeMorgans);

public abstract Object convertConditionOr(ConditionOR condition);
public abstract Object convertConditionOr(ConditionOR condition, boolean isConditionNot, boolean applyDeMorgans);

public abstract Object convertConditionNot(ConditionNOT condition);
public abstract Object convertConditionNot(ConditionNOT condition, boolean isConditionNot, boolean applyDeMorgans);

public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition) throws SigmaValueError {
public Object convertConditionFieldEqVal(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) throws SigmaValueError {
if (condition.getValue() instanceof SigmaString) {
return this.convertConditionFieldEqValStr(condition);
return this.convertConditionFieldEqValStr(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaNumber) {
return this.convertConditionFieldEqValNum(condition);
return this.convertConditionFieldEqValNum(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaBool) {
return this.convertConditionFieldEqValBool(condition);
return this.convertConditionFieldEqValBool(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaRegularExpression) {
return this.convertConditionFieldEqValRe(condition);
return this.convertConditionFieldEqValRe(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaCIDRExpression) {
return this.convertConditionFieldEqValCidr(condition);
return this.convertConditionFieldEqValCidr(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaCompareExpression) {
return this.convertConditionFieldEqValOpVal(condition);
return this.convertConditionFieldEqValOpVal(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaNull) {
return this.convertConditionFieldEqValNull(condition);
return this.convertConditionFieldEqValNull(condition, applyDeMorgans);
}/* TODO: below methods will be supported when Sigma Expand Modifier is supported.
else if (condition.getValue() instanceof SigmaQueryExpression) {
return this.convertConditionFieldEqValQueryExpr(condition);
}*/ else if (condition.getValue() instanceof SigmaExpansion) {
return this.convertConditionFieldEqValQueryExpansion(condition);
return this.convertConditionFieldEqValQueryExpansion(condition, isConditionNot, applyDeMorgans);
} else {
throw new IllegalArgumentException("Unexpected value type class in condition parse tree: " + condition.getValue().getClass().getName());
}
}

public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition) throws SigmaValueError;
public abstract Object convertConditionFieldEqValStr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans) throws SigmaValueError;

public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionFieldEqValNum(ConditionFieldEqualsValueExpression condition);
public abstract Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionFieldEqValBool(ConditionFieldEqualsValueExpression condition);
public abstract Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionFieldEqValRe(ConditionFieldEqualsValueExpression condition);
public abstract Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionFieldEqValCidr(ConditionFieldEqualsValueExpression condition);
public abstract Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionFieldEqValOpVal(ConditionFieldEqualsValueExpression condition);
public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionFieldEqValNull(ConditionFieldEqualsValueExpression condition);
public abstract Object convertExistsField(ConditionFieldEqualsValueExpression condition);

/* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/
/* public abstract Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpression condition);*/

public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition) {
public Object convertConditionFieldEqValQueryExpansion(ConditionFieldEqualsValueExpression condition, boolean isConditionNot, boolean applyDeMorgans) {
List<Either<AnyOneOf<ConditionItem, ConditionFieldEqualsValueExpression, ConditionValueExpression>, String>> args = new ArrayList<>();
for (SigmaType sigmaType: ((SigmaExpansion) condition.getValue()).getValues()) {
args.add(Either.left(AnyOneOf.middleVal(new ConditionFieldEqualsValueExpression(condition.getField(), sigmaType))));
}

ConditionOR conditionOR = new ConditionOR(false, args);
return this.convertConditionOr(conditionOR);
return this.convertConditionOr(conditionOR, isConditionNot, applyDeMorgans);
}

public Object convertConditionVal(ConditionValueExpression condition) throws SigmaValueError {
public Object convertConditionVal(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError {
if (condition.getValue() instanceof SigmaString) {
return this.convertConditionValStr(condition);
return this.convertConditionValStr(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaNumber) {
return this.convertConditionValNum(condition);
return this.convertConditionValNum(condition, applyDeMorgans);
} else if (condition.getValue() instanceof SigmaBool) {
throw new SigmaValueError("Boolean values can't appear as standalone value without a field name.");
} else if (condition.getValue() instanceof SigmaRegularExpression) {
return this.convertConditionValRe(condition);
return this.convertConditionValRe(condition, applyDeMorgans);
}/* else if (condition.getValue() instanceof SigmaCIDRExpression) {
throw new SigmaValueError("CIDR values can't appear as standalone value without a field name.");
} else if (condition.getValue() instanceof SigmaQueryExpression) {
Expand All @@ -258,11 +268,11 @@ public Object convertConditionVal(ConditionValueExpression condition) throws Sig
}
}

public abstract Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError;
public abstract Object convertConditionValStr(ConditionValueExpression condition, boolean applyDeMorgans) throws SigmaValueError;

public abstract Object convertConditionValNum(ConditionValueExpression condition);
public abstract Object convertConditionValNum(ConditionValueExpression condition, boolean applyDeMorgans);

public abstract Object convertConditionValRe(ConditionValueExpression condition);
public abstract Object convertConditionValRe(ConditionValueExpression condition, boolean applyDeMorgans);

/* public abstract Object convertConditionValQueryExpr(ConditionValueExpression condition);*/

Expand Down
Loading

0 comments on commit 6d49626

Please sign in to comment.