Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Security Solution][Detection engine] skips geo_point non-ecs validation #163487

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
245ba57
add test element
vitaliidm May 5, 2023
c3d1806
Merge branch 'elastic:main' into main
vitaliidm May 5, 2023
cea3e3b
Revert "add test element"
vitaliidm May 5, 2023
8a240c5
Merge remote-tracking branch 'upstream/main'
vitaliidm May 10, 2023
d1b32ab
Merge branch 'elastic:main' into main
vitaliidm May 16, 2023
1b8bd2b
Merge branch 'elastic:main' into main
vitaliidm May 18, 2023
711fe3e
Merge remote-tracking branch 'upstream/main'
vitaliidm Jun 7, 2023
5272556
Merge branch 'main' of ssh://github.com/vitaliidm/kibana
vitaliidm Jun 7, 2023
9589ab4
Merge remote-tracking branch 'upstream/main'
vitaliidm Jun 9, 2023
fcd9618
Merge remote-tracking branch 'upstream/main'
vitaliidm Jun 30, 2023
5b12314
Merge remote-tracking branch 'upstream/main'
vitaliidm Jul 10, 2023
0b09ae1
Merge remote-tracking branch 'upstream/main'
vitaliidm Jul 17, 2023
82db206
Merge remote-tracking branch 'upstream/main'
vitaliidm Jul 25, 2023
d19ce2c
Merge remote-tracking branch 'upstream/main'
vitaliidm Jul 28, 2023
0870df1
Merge branch 'main' of https://github.com/vitaliidm/kibana
vitaliidm Jul 28, 2023
f298651
Merge remote-tracking branch 'upstream/main'
vitaliidm Jul 28, 2023
659f27b
Merge branch 'main' of ssh://github.com/vitaliidm/kibana
vitaliidm Jul 28, 2023
2fbc9a9
Merge remote-tracking branch 'upstream/main'
vitaliidm Aug 2, 2023
a6ab99c
Merge branch 'main' of https://github.com/vitaliidm/kibana
vitaliidm Aug 2, 2023
ef68911
Merge remote-tracking branch 'upstream/main'
vitaliidm Aug 3, 2023
5f9aedf
Merge branch 'elastic:main' into main
vitaliidm Aug 4, 2023
9e9300d
Merge remote-tracking branch 'upstream/main'
vitaliidm Aug 9, 2023
b76a432
Merge branch 'main' of https://github.com/vitaliidm/kibana
vitaliidm Aug 9, 2023
4500006
fix tests
vitaliidm Aug 9, 2023
46fd3d3
Merge branch 'main' into detection-engine/non-ecs-geo-point-fixes
vitaliidm Aug 9, 2023
8fa47b6
Merge branch 'main' into detection-engine/non-ecs-geo-point-fixes
vitaliidm Aug 9, 2023
b40eb89
change test title
vitaliidm Aug 9, 2023
26bda20
Update strip_non_ecs_fields.test.ts
vitaliidm Aug 9, 2023
4e38ed1
Merge branch 'main' into detection-engine/non-ecs-geo-point-fixes
vitaliidm Aug 10, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -551,4 +551,72 @@ describe('stripNonEcsFields', () => {
expect(removed).toEqual([]);
});
});

// geo_point is too complex so we going to skip its validation
describe('geo_point field', () => {
it('should not strip invalid geo_point field', () => {
const { result, removed } = stripNonEcsFields({
'client.location.geo': 'invalid geo_point',
});

expect(result).toEqual({
'client.location.geo': 'invalid geo_point',
});
expect(removed).toEqual([]);
});

it('should not strip valid geo_point fields', () => {
expect(
stripNonEcsFields({
'client.geo.location': [0, 90],
}).result
).toEqual({
'client.geo.location': [0, 90],
});

expect(
stripNonEcsFields({
'client.geo.location': {
type: 'Point',
coordinates: [-88.34, 20.12],
},
}).result
).toEqual({
'client.geo.location': {
type: 'Point',
coordinates: [-88.34, 20.12],
},
});

expect(
stripNonEcsFields({
'client.geo.location': 'POINT (-71.34 41.12)',
}).result
).toEqual({
'client.geo.location': 'POINT (-71.34 41.12)',
});

expect(
stripNonEcsFields({
client: {
geo: {
location: {
lat: 41.12,
lon: -71.34,
},
},
},
}).result
).toEqual({
client: {
geo: {
location: {
lat: 41.12,
lon: -71.34,
},
},
},
});
});
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,11 @@ const ecsObjectFields = getEcsObjectFields();

/**
* checks if path is a valid Ecs object type (object or flattened)
* geo_point also can be object
*/
const getIsEcsFieldObject = (path: string) => {
const ecsField = ecsFieldMap[path as keyof typeof ecsFieldMap];
return ['object', 'flattened'].includes(ecsField?.type) || ecsObjectFields[path];
return ['object', 'flattened', 'geo_point'].includes(ecsField?.type) || ecsObjectFields[path];
};

/**
Expand Down Expand Up @@ -117,6 +118,11 @@ const computeIsEcsCompliant = (value: SourceField, path: string) => {
const ecsField = ecsFieldMap[path as keyof typeof ecsFieldMap];
const isEcsFieldObject = getIsEcsFieldObject(path);

// do not validate geo_point, since it's very complex type that can be string/array/object
if (ecsField?.type === 'geo_point') {
return true;
}

// validate if value is a long type
if (ecsField?.type === 'long') {
return isValidLongType(value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,8 +258,8 @@ export default ({ getService }: FtrProviderContext) => {

// we don't validate it because geo_point is very complex type with many various representations: array, different object, string with few valid patterns
// more on geo_point type https://www.elastic.co/guide/en/elasticsearch/reference/current/geo-point.html
// since .alerts-* indices allow _ignore_malformed option, alert will be indexed for this document
it('should fail creating alert when ECS field mapping is geo_point', async () => {
// since .alerts-* indices allow _ignore_malformed option, alert will be created for this document
it('should not fail creating alert when ECS field mapping is geo_point', async () => {
const document = {
client: {
geo: {
Expand Down