Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Events Index & Alarms Dashboard #56

Merged
merged 4 commits into from
Aug 11, 2016
Merged

Events Index & Alarms Dashboard #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c45a39a
added events index
Aug 5, 2016
49ab3de
fixed error
Aug 8, 2016
d740700
craig comments, util.pyc deleted
Aug 10, 2016
586eefb
updated dashboard
Aug 10, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions resources/dashboards/Alarms-Dashboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"hits": 0,
"timeRestore": false,
"description": "",
"title": "Alarms Dashboard",
"panelsJSON": "[{\"col\":1,\"id\":\"Total-Alarms-Fired-(Count)\",\"row\":5,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"RuleName\",\"RuleSeverity\",\"Application\",\"Session\",\"TotalBytes\",\"Captured\"],\"id\":\"Alarms-Table\",\"row\":8,\"size_x\":12,\"size_y\":5,\"sort\":[\"TimeUpdated\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"Count-of-Total-Alarms-Fired-by-Name-(Bar-Chart)\",\"row\":1,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"id\":\"Summary-of-Total-Alarms-Fired-(Pie-Chart)\",\"type\":\"visualization\",\"size_x\":6,\"size_y\":4,\"col\":7,\"row\":1}]",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}"
}
}
20 changes: 20 additions & 0 deletions resources/searches/Alarms-Table.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"sort": [
"TimeUpdated",
"desc"
],
"hits": 0,
"description": "",
"title": "Alarms Table",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"[events_]YYYY_MM_DD\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"fragment_size\":2147483647},\"filter\":[]}"
},
"columns": [
"RuleName",
"RuleSeverity",
"Application",
"Session",
"TotalBytes"
]
}
9 changes: 9 additions & 0 deletions resources/visualizations/Count-of-Total-Alarms-Fired-by-Name-(Bar-Chart).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"visState": "{\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"TimeUpdated\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"RuleName.raw\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"[events_]YYYY_MM_DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
},
"version": 1,
"description": "",
"title": "Count of Total Alarms Fired by Name (Bar Chart)"
}
9 changes: 9 additions & 0 deletions resources/visualizations/Summary-of-Total-Alarms-Fired-(Pie-Chart).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"visState": "{\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"RuleSeverity\",\"size\":3,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"RuleName.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"Application\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"[events_]YYYY_MM_DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
},
"version": 1,
"description": "",
"title": "Summary of Total Alarms Fired (Pie Chart)"
}
9 changes: 9 additions & 0 deletions resources/visualizations/Total-Alarms-Fired-(Count).json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"visState": "{\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"3\",\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"Application\"}},{\"id\":\"6\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"RuleSeverity\",\"size\":3,\"order\":\"desc\",\"orderBy\":\"3\",\"json\":\"\\\"high\\\"\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"RuleName.raw\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"SrcIP\"}},{\"id\":\"5\",\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"DestIP\"}},{\"id\":\"1\",\"type\":\"count\",\"schema\":\"metric\",\"params\":{}}],\"listeners\":{}}",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"[events_]YYYY_MM_DD\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
},
"version": 1,
"description": "",
"title": "Total Alarms Fired (Count)"
}
71 changes: 52 additions & 19 deletions scripts/setDefaultIndex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,24 @@
UTIL = Utility()

NM_INDEX_PATTERN='[network_]YYYY_MM_DD'
EVENTS_INDEX_PATTERN='[events_]YYYY_MM_DD'
DEFAULT_INDEX='"defaultIndex": \"%s\"' % NM_INDEX_PATTERN
VERIFIED = 1

FIELD_FORMAT_MAPPINGS_FILE = "/usr/local/kibana-" + esUtil.KIBANA_VERSION + "-linux-x64/resources/mappings.json"

index_pattern_content = {
network_index_pattern_content = {
"title": "[network_]YYYY_MM_DD",
"intervalName": "days",
"timeFieldName": "TimeUpdated"
}

events_index_pattern_content = {
"title": "[events_]YYYY_MM_DD",
"intervalName": "days",
"timeFieldName": "TimeUpdated"
}

version_config_content = {
"defaultIndex": "[network_]YYYY_MM_DD"
}
Expand Down Expand Up @@ -99,47 +106,73 @@ def create_document_if_it_doesnt_exist(es_index, es_type, es_id, es_body):
return document_created

def get_field_mappings(filename):
global index_pattern_content
global network_index_pattern_content
corrected_mappings = {}
mappings_json = UTIL.read_json_from_file(filename)
value = UTIL.safe_list_read(mappings_json, 'fieldFormatMap')
# Quotations in the fieldFormatMap must be escaped
# for proper Elasticsearch insertion
escaped_mappings = replace_all_char(str=json.dumps(value), to_replace='"', new_char='\"')
corrected_mappings['fieldFormatMap'] = escaped_mappings
index_pattern_content.update(corrected_mappings)
network_index_pattern_content.update(corrected_mappings)

# ----------------- MAIN -----------------
def main():

global index_pattern_content
global network_index_pattern_content

# Add fieldFormatMap to index-pattern content
get_field_mappings(filename=FIELD_FORMAT_MAPPINGS_FILE)

logging.info("================================== INDEX PATTERN ==================================")
index_pattern_doc_created = create_document_if_it_doesnt_exist(esUtil.KIBANA_INDEX,
logging.info("================================== METADATA INDEX PATTERN ==================================")
network_index_pattern_doc_created = create_document_if_it_doesnt_exist(esUtil.KIBANA_INDEX,
esUtil.INDEX_PATTERN_TYPE,
NM_INDEX_PATTERN,
index_pattern_content)
index_pattern_missing_fields = verify_document_for_content(esUtil.KIBANA_INDEX,
network_index_pattern_content)
network_index_pattern_missing_fields = verify_document_for_content(esUtil.KIBANA_INDEX,
esUtil.INDEX_PATTERN_TYPE,
index_pattern_content)
if len(index_pattern_missing_fields) > 0:
logging.info("Updating Network Monitor index-pattern with missing fields: ")
for key in index_pattern_missing_fields:
logging.info(" " + key + ": " + index_pattern_missing_fields[key])
updated, update_ret = esUtil.function_with_timeout(esUtil.ES_QUERY_TIMEOUT,
network_index_pattern_content)
if len(network_index_pattern_missing_fields) > 0:
logging.info("Updating Network Monitor network index-pattern with missing fields: ")
for key in network_index_pattern_missing_fields:
logging.info(" " + key + ": " + network_index_pattern_missing_fields[key])
network_updated, network_update_ret = esUtil.function_with_timeout(esUtil.ES_QUERY_TIMEOUT,
esUtil.update_document,
esUtil.KIBANA_INDEX,
esUtil.INDEX_PATTERN_TYPE,
NM_INDEX_PATTERN,
esUtil.format_for_update(index_pattern_missing_fields))
if not updated:
logging.error("Unable to add missing index-pattern fields:")
logging.error(update_ret)
esUtil.format_for_update(network_index_pattern_missing_fields))
if not network_updated:
logging.error("Unable to add missing network index-pattern fields:")
logging.error(network_update_ret)
else:
logging.info("No missing index-pattern fields.")
logging.info("No missing network index-pattern fields.")

logging.info("================================== EVENTS INDEX PATTERN ==================================")

events_index_pattern_doc_created = create_document_if_it_doesnt_exist(esUtil.KIBANA_INDEX,
esUtil.INDEX_PATTERN_TYPE,
EVENTS_INDEX_PATTERN,
events_index_pattern_content)
events_index_pattern_missing_fields = verify_document_for_content(esUtil.KIBANA_INDEX,
esUtil.INDEX_PATTERN_TYPE,
events_index_pattern_content)
if len(events_index_pattern_missing_fields) > 0:
logging.info("Updating Network Monitor events index-pattern with missing fields: ")
for key in events_index_pattern_missing_fields:
logging.info(" " + key + ": " + events_index_pattern_missing_fields[key])
events_updated, events_update_ret = esUtil.function_with_timeout(esUtil.ES_QUERY_TIMEOUT,
esUtil.update_document,
esUtil.KIBANA_INDEX,
esUtil.INDEX_PATTERN_TYPE,
EVENTS_INDEX_PATTERN,
esUtil.format_for_update(events_index_pattern_missing_fields))
if not events_updated:
logging.error("Unable to add missing events index-pattern fields:")
logging.error(events_update_ret)
else:
logging.info("No missing events index-pattern fields.")


logging.info("================================== " + esUtil.KIBANA_VERSION + " CONFIG ==================================")
config_doc_created = create_document_if_it_doesnt_exist(esUtil.KIBANA_INDEX,
Expand Down